diff --git a/core/src/main/java/org/apache/calcite/jdbc/JavaRecordType.java b/core/src/main/java/org/apache/calcite/jdbc/JavaRecordType.java index 72242d3934cf..21b4c4415e5c 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/JavaRecordType.java +++ b/core/src/main/java/org/apache/calcite/jdbc/JavaRecordType.java @@ -41,14 +41,15 @@ public JavaRecordType(List fields, Class clazz) { this.clazz = requireNonNull(clazz, "clazz"); } - @Override public boolean equals(@Nullable Object obj) { + @Override public boolean deepEquals(@Nullable Object obj) { return this == obj || obj instanceof JavaRecordType && Objects.equals(fieldList, ((JavaRecordType) obj).fieldList) - && clazz == ((JavaRecordType) obj).clazz; + && clazz == ((JavaRecordType) obj).clazz + && this.isNullable() == ((JavaRecordType) obj).isNullable(); } - @Override public int hashCode() { - return Objects.hash(fieldList, clazz); + @Override public int deepHashCode() { + return Objects.hash(fieldList, this.isNullable(), clazz); } } diff --git a/core/src/main/java/org/apache/calcite/rel/HasDigestString.java b/core/src/main/java/org/apache/calcite/rel/HasDigestString.java new file mode 100644 index 000000000000..5058c3876959 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/HasDigestString.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel; + +/** + * Interface for objects that have a digest string. + */ +public interface HasDigestString { + String getDigestString(); +} diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataType.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataType.java index 4188d04698f4..071a1152a108 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelDataType.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataType.java @@ -322,4 +322,26 @@ default boolean equalsSansFieldNamesAndNullability(@Nullable RelDataType that) { default boolean isMeasure() { return getSqlTypeName() == SqlTypeName.MEASURE; } + + /** + * Returns the digest of this type. + * + * @return digest of this type + */ + RelDataTypeDigest getDigest(); + + /** + * Deep equality check for RelDataType digest. + * + * @return Whether the 2 RelDataTypes are equivalent or have the same digest. + * @see #deepHashCode() + */ + boolean deepEquals(@Nullable Object obj); + + /** + * Compute deep hash code for RelDataType digest. + * + * @see #deepEquals(Object) + */ + int deepHashCode(); } diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeDigest.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeDigest.java new file mode 100644 index 000000000000..ac805aae22ac --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeDigest.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.type; + +import org.apache.calcite.rel.HasDigestString; + +/** + * Digest of a RelDataType. + */ +public interface RelDataTypeDigest extends HasDigestString { + RelDataType getType(); +} diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeImpl.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeImpl.java index 67ab34101695..c0541d8c3914 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeImpl.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeImpl.java @@ -46,8 +46,8 @@ * RelDataTypeImpl is an abstract base for implementations of * {@link RelDataType}. * - *

Identity is based upon the {@link #digest} field, which each derived class - * should set during construction. + *

Identity is based upon the {@link #digest} or {@link #innerDigest} field, + * which each derived class should set {@link #digest} or {@link #innerDigest} during construction. */ public abstract class RelDataTypeImpl implements RelDataType, RelDataTypeFamily { @@ -60,7 +60,8 @@ public abstract class RelDataTypeImpl //~ Instance fields -------------------------------------------------------- protected final @Nullable List fieldList; - protected @Nullable String digest; + protected @Deprecated @Nullable String digest; + protected @Nullable RelDataTypeDigest innerDigest; //~ Constructors ----------------------------------------------------------- @@ -232,18 +233,50 @@ private static void getFieldRecurse(List slots, RelDataType type, return fieldList != null; } - @Override public boolean equals(@Nullable Object obj) { - return this == obj - || obj instanceof RelDataTypeImpl - && Objects.equals(this.digest, ((RelDataTypeImpl) obj).digest); + /** + * Gets the {@link RelDataTypeDigest} of this type. + * If a user has set the legacy string {@code digest} and {@code innerDigest} has not + * been initialized yet, this method computes and initializes it. + */ + @Override public RelDataTypeDigest getDigest() { + if (digest != null && innerDigest == null) { + computeDigest(); + } + return requireNonNull(innerDigest, "innerDigest"); + } + + @Override public final boolean equals(@Nullable Object obj) { + if (obj == this) { + return true; + } + if (obj instanceof RelDataTypeImpl) { + final RelDataTypeImpl that = (RelDataTypeImpl) obj; + return this.getDigest().equals(that.getDigest()); + } + return false; } - @Override public int hashCode() { - return Objects.hashCode(digest); + @Override public final int hashCode() { + return getDigest().hashCode(); + } + + @Override public boolean deepEquals(@Nullable Object obj) { + if (this == obj) { + return true; + } + if (obj == null || this.getClass() != obj.getClass()) { + return false; + } + return Objects.equals(this.getDigest().getDigestString(), + ((RelDataTypeImpl) obj).getDigest().getDigestString()); + } + + @Override public int deepHashCode() { + return Objects.hashCode(this.getDigest().getDigestString()); } @Override public String getFullTypeString() { - return requireNonNull(digest, "digest"); + return requireNonNull(this.getDigest().getDigestString(), "digest"); } @Override public boolean isNullable() { @@ -309,23 +342,81 @@ protected abstract void generateTypeString( boolean withDetail); /** - * Computes the digest field. This should be called in every non-abstract - * subclass constructor once the type is fully defined. + * Init the lazy digest computing field {@link #innerDigest}. + * This should be called in every non-abstract subclass + * constructor once the type is fully defined. */ @SuppressWarnings("method.invocation.invalid") protected void computeDigest(@UnknownInitialization RelDataTypeImpl this) { - StringBuilder sb = new StringBuilder(); - generateTypeString(sb, true); - if (!isNullable()) { - sb.append(NON_NULLABLE_SUFFIX); - } - digest = sb.toString(); + innerDigest = new InnerRelDataTypeDigest(); } @Override public String toString() { - StringBuilder sb = new StringBuilder(); - generateTypeString(sb, false); - return sb.toString(); + return getDigest().toString(); + } + + /** Implementation of {@link RelDataTypeDigest}. */ + private class InnerRelDataTypeDigest implements RelDataTypeDigest { + /** Cached hash code. */ + private int hash = 0; + /** Cached type string. */ + private @Nullable String digestWithDetail = null; // NOTE: shorter detail will be better + private @Nullable String digestWithoutDetail = null; + + @Override public RelDataType getType() { + return RelDataTypeImpl.this; + } + + @Override public boolean equals(@Nullable Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final RelDataTypeImpl.InnerRelDataTypeDigest otherDigest = + (RelDataTypeImpl.InnerRelDataTypeDigest) o; + if (digest != null) { + return digest.equals(otherDigest.getDigestString()); + } + return deepEquals(otherDigest.getType()); + } + + @Override public int hashCode() { + if (digest != null) { + return Objects.hashCode(digest); + } + if (hash == 0) { + hash = deepHashCode(); + } + return hash; + } + + @Override public String getDigestString() { + // return user defined digest by set legacy digest string field. + if (digest != null) { + return digest; + } + + if (digestWithDetail == null) { + StringBuilder sb = new StringBuilder(); + generateTypeString(sb, true); + if (!isNullable()) { + sb.append(NON_NULLABLE_SUFFIX); + } + digestWithDetail = sb.toString(); + } + return digestWithDetail; + } + + @Override public String toString() { + if (digestWithoutDetail == null) { + StringBuilder sb = new StringBuilder(); + RelDataTypeImpl.this.generateTypeString(sb, false); + digestWithoutDetail = sb.toString(); + } + return digestWithoutDetail; + } } @Override public RelDataTypePrecedenceList getPrecedenceList() { diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelRecordType.java b/core/src/main/java/org/apache/calcite/rel/type/RelRecordType.java index 4f253bf225d8..a63b107ca0cb 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelRecordType.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelRecordType.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import static java.util.Objects.requireNonNull; @@ -146,6 +147,39 @@ public RelRecordType(List fields) { sb.append(")"); } + @Override public boolean deepEquals(@Nullable Object obj) { + if (this == obj) { + return true; + } + if (obj == null || this.getClass() != obj.getClass()) { + return false; + } + + RelRecordType that = (RelRecordType) obj; + if (kind != that.kind || nullable != that.nullable) { + return false; + } + + if (fieldList == null || that.fieldList == null) { + return fieldList == null && that.fieldList == null; + } + + if (fieldList.size() != that.fieldList.size()) { + return false; + } + + for (int i = 0; i < fieldList.size(); i++) { + if (!fieldList.get(i).equals(that.fieldList.get(i))) { + return false; + } + } + return true; + } + + @Override public int deepHashCode() { + return Objects.hash(kind.ordinal(), nullable, fieldList); + } + /** * Per {@link Serializable} API, provides a replacement object to be written * during serialization. diff --git a/core/src/main/java/org/apache/calcite/rel/type/SingleColumnAliasRelDataType.java b/core/src/main/java/org/apache/calcite/rel/type/SingleColumnAliasRelDataType.java index b09196b57789..28e0406f6717 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/SingleColumnAliasRelDataType.java +++ b/core/src/main/java/org/apache/calcite/rel/type/SingleColumnAliasRelDataType.java @@ -136,4 +136,16 @@ public SingleColumnAliasRelDataType(RelDataType original, RelDataType alias) { @Override public boolean isDynamicStruct() { return original.isDynamicStruct(); } + + @Override public RelDataTypeDigest getDigest() { + return original.getDigest(); + } + + @Override public boolean deepEquals(@Nullable Object obj) { + return original.deepEquals(obj); + } + + @Override public int deepHashCode() { + return original.deepHashCode(); + } } diff --git a/core/src/main/java/org/apache/calcite/sql/type/ArraySqlType.java b/core/src/main/java/org/apache/calcite/sql/type/ArraySqlType.java index 1124939c7bec..7d1d1afc470f 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/ArraySqlType.java +++ b/core/src/main/java/org/apache/calcite/sql/type/ArraySqlType.java @@ -20,6 +20,10 @@ import org.apache.calcite.rel.type.RelDataTypeFamily; import org.apache.calcite.rel.type.RelDataTypePrecedenceList; +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.Objects; + import static org.apache.calcite.sql.type.NonNullableAccessors.getComponentTypeOrThrow; import static java.util.Objects.requireNonNull; @@ -56,6 +60,21 @@ public ArraySqlType(RelDataType elementType, boolean isNullable) { sb.append(" ARRAY"); } + @Override public boolean deepEquals(@Nullable Object obj) { + if (this == obj) { + return true; + } + if (obj == null || this.getClass() != obj.getClass()) { + return false; + } + ArraySqlType that = (ArraySqlType) obj; + return this.isNullable() == that.isNullable() && elementType.equals(that.elementType); + } + + @Override public int deepHashCode() { + return Objects.hash(SqlTypeName.ARRAY.ordinal(), isNullable, elementType.hashCode()); + } + // implement RelDataType @Override public RelDataType getComponentType() { return elementType; diff --git a/core/src/main/java/org/apache/calcite/sql/type/MapSqlType.java b/core/src/main/java/org/apache/calcite/sql/type/MapSqlType.java index 960fec7ac4ef..9ad216057577 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/MapSqlType.java +++ b/core/src/main/java/org/apache/calcite/sql/type/MapSqlType.java @@ -19,6 +19,10 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFamily; +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.Objects; + import static java.util.Objects.requireNonNull; /** @@ -69,6 +73,23 @@ public MapSqlType( .append(") MAP"); } + @Override public boolean deepEquals(@Nullable Object obj) { + if (this == obj) { + return true; + } + if (obj == null || this.getClass() != obj.getClass()) { + return false; + } + MapSqlType that = (MapSqlType) obj; + return this.isNullable() == that.isNullable() && keyType.equals(that.keyType) + && valueType.equals(that.valueType); + } + + @Override public int deepHashCode() { + return Objects.hash(SqlTypeName.MAP.ordinal(), this.isNullable, keyType.hashCode(), + valueType.hashCode()); + } + // implement RelDataType @Override public RelDataTypeFamily getFamily() { return this; diff --git a/core/src/main/java/org/apache/calcite/sql/type/MultisetSqlType.java b/core/src/main/java/org/apache/calcite/sql/type/MultisetSqlType.java index b12c836d89a0..cbea4062c2ea 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/MultisetSqlType.java +++ b/core/src/main/java/org/apache/calcite/sql/type/MultisetSqlType.java @@ -20,6 +20,10 @@ import org.apache.calcite.rel.type.RelDataTypeFamily; import org.apache.calcite.rel.type.RelDataTypePrecedenceList; +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.Objects; + import static org.apache.calcite.sql.type.NonNullableAccessors.getComponentTypeOrThrow; import static java.util.Objects.requireNonNull; @@ -56,6 +60,21 @@ public MultisetSqlType(RelDataType elementType, boolean isNullable) { sb.append(" MULTISET"); } + @Override public boolean deepEquals(@Nullable Object obj) { + if (this == obj) { + return true; + } + if (obj == null || this.getClass() != obj.getClass()) { + return false; + } + MultisetSqlType that = (MultisetSqlType) obj; + return this.isNullable() == that.isNullable() && elementType.equals(that.elementType); + } + + @Override public int deepHashCode() { + return Objects.hash(SqlTypeName.MULTISET.ordinal(), this.isNullable, elementType.hashCode()); + } + // implement RelDataType @Override public RelDataType getComponentType() { return elementType; diff --git a/core/src/test/java/org/apache/calcite/rex/RexBuilderTest.java b/core/src/test/java/org/apache/calcite/rex/RexBuilderTest.java index 7653d5370c76..e23679eaf5f5 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexBuilderTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexBuilderTest.java @@ -1477,6 +1477,7 @@ private void checkBigDecimalLiteral(RexBuilder builder, String val) { /** Emulate a user defined type. */ private static class UDT extends RelDataTypeImpl { + @SuppressWarnings("deprecation") UDT() { this.digest = "(udt)NOT NULL"; } diff --git a/core/src/test/java/org/apache/calcite/test/HepPlannerTest.java b/core/src/test/java/org/apache/calcite/test/HepPlannerTest.java index 262bba031392..3f80ab42ed9a 100644 --- a/core/src/test/java/org/apache/calcite/test/HepPlannerTest.java +++ b/core/src/test/java/org/apache/calcite/test/HepPlannerTest.java @@ -29,7 +29,11 @@ import org.apache.calcite.rel.logical.LogicalValues; import org.apache.calcite.rel.rules.CoerceInputsRule; import org.apache.calcite.rel.rules.CoreRules; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.SqlCollation; import org.apache.calcite.sql.SqlExplainLevel; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.tools.RelBuilder; import com.google.common.collect.ImmutableList; @@ -40,6 +44,7 @@ import java.io.PrintWriter; import java.io.StringWriter; +import java.nio.charset.StandardCharsets; import static org.apache.calcite.test.Matchers.isLinux; @@ -450,4 +455,59 @@ long getApplyTimes() { final RelNode result = planner.findBestExp(); assertThat(result, is(instanceOf(LogicalValues.class))); } + + @Test void testLargeTypeDigest() { + // if we don't support RelTypeDigest, this case will be OOM or very slow + int[] topFieldCounts = {1, 50, 500, 5000, 50000}; + for (int topN : topFieldCounts) { + final RelBuilder builder = RelBuilderTest.createBuilder(c -> c); + final RelDataTypeFactory typeFactory = builder.getTypeFactory(); + + RelDataType varchar = + typeFactory.createTypeWithCharsetAndCollation(typeFactory + .createSqlType(SqlTypeName.VARCHAR, 100), + StandardCharsets.UTF_8, SqlCollation.IMPLICIT); + + RelDataType leafObj = typeFactory.builder().add("k", varchar).add("v", varchar) + .add("attrs", typeFactory.createMapType(varchar, varchar)) + .add("tags", typeFactory.createArrayType(varchar, -1)).build(); + + final RelDataTypeFactory.Builder root = typeFactory.builder(); + for (int i = 0; i < topN; i++) { + int depth = 1 + (i % 8); + RelDataType t = leafObj; + + for (int d = 0; d < depth; d++) { + RelDataType arrObj = typeFactory.createArrayType(t, -1); + RelDataType mapObj = typeFactory.createMapType(varchar, t); + + t = + typeFactory.builder().add("lvl" + d, t).add("arr" + d, arrObj).add("map" + d, mapObj) + .add("s" + d, varchar).build(); + + } + + if ((i % 11) == 0) { + root.add("f" + i, typeFactory.createArrayType(t, -1)); + } else if ((i % 11) == 1) { + root.add("f" + i, typeFactory.createMapType(varchar, t)); + } else { + root.add("f" + i, t); + } + } + + // cache will reuse same digest object if we don't use string as composite type digest + final RelDataType type = root.build(); + final RelDataType type2 = root.build(); + + long start = System.currentTimeMillis(); + boolean equals = false; + for (int i = 0; i < 1000; i++) { + equals = type.equals(type2) && type.hashCode() == type2.hashCode(); + } + long end = System.currentTimeMillis(); + System.out.println("struct topFields=" + topN + ", Time=" + (end - start) + "ms"); + assertThat(equals, is(true)); + } + } }