HIVE-28773: CBO fails when a materialized view is dropped but its pre-compiled plan remains in the registry.

itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestHiveMaterializedViewRegistry.java

@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.txn.compactor;
+
+import org.apache.hadoop.hive.common.MaterializationSnapshot;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.common.TableName;
+import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.ddl.view.create.CreateMaterializedViewDesc;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveMaterializedViewsRegistry;
+import org.apache.hadoop.hive.ql.metadata.HiveRelOptMaterialization;
+import org.apache.hadoop.hive.ql.metadata.MaterializedViewMetadata;
+import org.apache.hadoop.hive.ql.metadata.RewriteAlgorithm;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.HiveMaterializedViewUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
+import static java.util.Arrays.asList;
+import static org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.executeStatementOnDriver;
+import static org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.executeStatementOnDriverSilently;
+
+public class TestHiveMaterializedViewRegistry extends CompactorOnTezTest {
+
+  private static final String DB = "default";
+  private static final String TABLE1 = "t1";
+  private static final String MV1 = "mat1";
+  private static final TableName MV1_NAME = new TableName("hive", DB, MV1);
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+
+    executeStatementOnDriverSilently("drop materialized view if exists " + MV1, driver);
+    executeStatementOnDriverSilently("drop table if exists " + TABLE1 , driver);
+
+    executeStatementOnDriver("create table " + TABLE1 + "(a int, b string, c float) stored as orc TBLPROPERTIES ('transactional'='true')", driver);
+    executeStatementOnDriver("insert into " + TABLE1 + "(a,b, c) values (1, 'one', 1.1), (2, 'two', 2.2), (NULL, NULL, NULL)", driver);
+  }
+
+  @Override
+  public void tearDown() {
+    executeStatementOnDriverSilently("drop materialized view " + MV1, driver);
+    executeStatementOnDriverSilently("drop table " + TABLE1 , driver);
+
+    super.tearDown();
+  }
+
+  @Test
+  public void testRefreshAddsNewMV() throws Exception {
+    CreateMaterializedViewDesc createMaterializedViewDesc = createMaterializedViewDesc();
+    Table mvTable = createMaterializedViewDesc.toTable(conf);
+    Hive.get().createTable(mvTable);
+
+    HiveMaterializedViewsRegistry.get().refresh(Hive.get());
+
+    HiveRelOptMaterialization materialization =
+            HiveMaterializedViewsRegistry.get().getRewritingMaterializedView(MV1_NAME, RewriteAlgorithm.ALL);
+
+    Assert.assertEquals(DB, materialization.qualifiedTableName.get(0));
+    Assert.assertEquals(MV1, materialization.qualifiedTableName.get(1));
+  }
+
+  @Test
+  public void testRefreshDoesNotAddMVDisabledForRewrite() throws Exception {
+    CreateMaterializedViewDesc createMaterializedViewDesc = createMaterializedViewDesc();
+    createMaterializedViewDesc.setRewriteEnabled(false);
+    Table mvTable = createMaterializedViewDesc.toTable(conf);
+    Hive.get().createTable(mvTable);
+
+    HiveMaterializedViewsRegistry.get().refresh(Hive.get());
+
+    HiveRelOptMaterialization materialization =
+            HiveMaterializedViewsRegistry.get().getRewritingMaterializedView(MV1_NAME, RewriteAlgorithm.ALL);
+    Assert.assertNull(materialization);
+  }
+
+  @Test
+  public void testRefreshUpdatesExistingMV() throws Exception {
+    // init the registry
+    HiveMaterializedViewsRegistry.get().refresh(Hive.get());
+
+    executeStatementOnDriver("create materialized view " + MV1 + " as " +
+            "select a,b,c from " + TABLE1 + " where a > 0 or a is null", driver);
+
+    // replace the MV
+    Hive.get().dropTable(DB, MV1);
+
+    CreateMaterializedViewDesc createMaterializedViewDesc = createMaterializedViewDesc();
+    Table mvTable = createMaterializedViewDesc.toTable(conf);
+    mvTable.setMaterializedViewMetadata(new MaterializedViewMetadata(
+            "hive", DB, MV1, new HashSet<>(), new MaterializationSnapshot("anything")));
+    Hive.get().createTable(mvTable);
+
+    // test refreshing the registry
+    HiveMaterializedViewsRegistry.get().refresh(Hive.get());
+
+    HiveRelOptMaterialization materialization =
+            HiveMaterializedViewsRegistry.get().getRewritingMaterializedView(MV1_NAME, RewriteAlgorithm.ALL);
+
+    Assert.assertEquals(DB, materialization.qualifiedTableName.get(0));
+    Assert.assertEquals(MV1, materialization.qualifiedTableName.get(1));
+    Table existingMVTable = HiveMaterializedViewUtils.extractTable(materialization);
+    Assert.assertEquals(mvTable.getTTable().getCreateTime(), existingMVTable.getCreateTime());
+  }
+
+  @Test
+  public void testRefreshRemovesMVDisabledForRewrite() throws Exception {
+    // init the registry
+    HiveMaterializedViewsRegistry.get().refresh(Hive.get());
+
+    executeStatementOnDriver("create materialized view " + MV1 + " as " +
+            "select a,b,c from " + TABLE1 + " where a > 0 or a is null", driver);
+
+    Table mvTable = Hive.get().getTable(DB, MV1);
+    mvTable.setRewriteEnabled(false);
+
+    EnvironmentContext environmentContext = new EnvironmentContext();
+    environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE);
+    Hive.get().alterTable(mvTable, false, environmentContext, true);
+
+    HiveMaterializedViewsRegistry.get().refresh(Hive.get());
+
+    HiveRelOptMaterialization materialization =
+            HiveMaterializedViewsRegistry.get().getRewritingMaterializedView(MV1_NAME, RewriteAlgorithm.ALL);
+    Assert.assertNull(materialization);
+  }
+
+  @Test
+  public void testRefreshAddsMVEnabledForRewrite() throws Exception {
+    // init the registry
+    HiveMaterializedViewsRegistry.get().refresh(Hive.get());
+
+    executeStatementOnDriver("create materialized view " + MV1 + " disabled rewrite as " +
+            "select a,b,c from " + TABLE1 + " where a > 0 or a is null", driver);
+
+    Table mvTable = Hive.get().getTable(DB, MV1);
+    mvTable.setRewriteEnabled(true);
+
+    EnvironmentContext environmentContext = new EnvironmentContext();
+    environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE);
+    Hive.get().alterTable(mvTable, false, environmentContext, true);
+
+    HiveMaterializedViewsRegistry.get().refresh(Hive.get());
+
+    HiveRelOptMaterialization materialization =
+            HiveMaterializedViewsRegistry.get().getRewritingMaterializedView(MV1_NAME, RewriteAlgorithm.ALL);
+
+    Assert.assertEquals(DB, materialization.qualifiedTableName.get(0));
+    Assert.assertEquals(MV1, materialization.qualifiedTableName.get(1));
+  }
+
+  @Test
+  public void testRefreshRemovesMVDoesNotExists() throws Exception {
+    // init the registry
+    HiveMaterializedViewsRegistry.get().refresh(Hive.get());
+
+    executeStatementOnDriver("create materialized view " + MV1 + " as " +
+            "select a,b,c from " + TABLE1 + " where a > 0 or a is null", driver);
+
+    Hive.get().dropTable(DB, MV1);
+
+    HiveMaterializedViewsRegistry.get().refresh(Hive.get());
+
+    HiveRelOptMaterialization materialization =
+            HiveMaterializedViewsRegistry.get().getRewritingMaterializedView(MV1_NAME, RewriteAlgorithm.ALL);
+    Assert.assertNull(materialization);
+  }
+
+  private static CreateMaterializedViewDesc createMaterializedViewDesc() {
+    Map<String, String> tableProperties = new HashMap<>();
+    tableProperties.put("created_with_ctas", "true");
+
+    CreateMaterializedViewDesc createMaterializedViewDesc = new CreateMaterializedViewDesc(
+            MV1,
+            null,
+            null,
+            tableProperties,
+            null,
+            null,
+            null,
+            false,
+            true,
+            "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
+            "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
+            null,
+            "org.apache.hadoop.hive.ql.io.orc.OrcSerde",
+            null,
+            new HashMap<>());
+
+    createMaterializedViewDesc.setViewOriginalText("select a,b,c from " + TABLE1 + " where a > 0 or a is null");
+    createMaterializedViewDesc.setViewExpandedText("select `t1`.`a`,`t1`.`b`,`t1`.`c` from `" + DB + "`.`" + TABLE1 + "` where `t1`.`a` > 0 or `t1`.`a` is null");
+    createMaterializedViewDesc.setCols(
+            asList(new FieldSchema("a", "int", null),
+                    new FieldSchema("b", "string", null),
+                    new FieldSchema("c", "float", null)));
+
+    return createMaterializedViewDesc;
+  }
+}

itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestQueryRewrite.java

@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.txn.compactor;
+
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.execSelectAndDumpData;
+import static org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.executeStatementOnDriver;
+import static org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.executeStatementOnDriverSilently;
+
+public class TestQueryRewrite extends CompactorOnTezTest {
+
+  private static final String DB = "default";
+  private static final String TABLE1 = "t1";
+  private static final String MV1 = "mat1";
+
+  private static final List<String> ORIGINAL_QUERY_PLAN = Arrays.asList(
+          "CBO PLAN:",
+          "HiveProject(a=[$0], b=[$1])",
+          "  HiveFilter(condition=[>($0, 0)])",
+          "    HiveTableScan(table=[[" + DB + ", " + TABLE1 + "]], table:alias=[" + TABLE1 + "])",
+          ""
+  );
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+
+    executeStatementOnDriverSilently("drop materialized view if exists " + MV1, driver);
+    executeStatementOnDriverSilently("drop table if exists " + TABLE1, driver);
+
+    executeStatementOnDriver("create table " + TABLE1 + "(a int, b string, c float) stored as orc TBLPROPERTIES ('transactional'='true')", driver);
+    executeStatementOnDriver("insert into " + TABLE1 + "(a,b, c) values (1, 'one', 1.1), (2, 'two', 2.2), (NULL, NULL, NULL)", driver);
+    executeStatementOnDriver("create materialized view " + MV1 + " stored by iceberg tblproperties('format-version'='2') as " +
+            "select a,b,c from " + TABLE1 + " where a > 0 or a is null", driver);
+  }
+
+  @Override
+  public void tearDown() {
+    executeStatementOnDriverSilently("drop materialized view " + MV1, driver);
+    executeStatementOnDriverSilently("drop table " + TABLE1 , driver);
+
+    super.tearDown();
+  }
+
+  @Test
+  public void testQueryIsNotRewrittenWhenMVIsDroppedButTheRegistryIsNotRefreshed() throws Exception {
+
+    // Simulate a multi HS2 cluster.
+    // Drop the MV using a direct API call to HMS. This is similar to what happens when the drop MV is executed by
+    // another HS2.
+    // In this case the MV is not removed from HiveMaterializedViewsRegistry of HS2 which runs the explain query.
+    msClient.dropTable(DB, MV1);
+
+    List<String> result = execSelectAndDumpData("explain cbo select a, b from " + TABLE1 + " where a > 0", driver, "");
+    Assert.assertEquals(ORIGINAL_QUERY_PLAN, result);
+  }
+
+  @Test
+  public void testQueryIsNotRewrittenWhenMVIsDisabledForRewriteButTheRegistryIsNotRefreshed() throws Exception {
+    Table mvTable = Hive.get().getTable(DB, MV1);
+    mvTable.setRewriteEnabled(false);
+
+    EnvironmentContext environmentContext = new EnvironmentContext();
+    environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE);
+    Hive.get().alterTable(mvTable, false, environmentContext, true);
+
+    List<String> result = execSelectAndDumpData("explain cbo select a, b from " + TABLE1 + " where a > 0", driver, "");
+    Assert.assertEquals(ORIGINAL_QUERY_PLAN, result);
+  }
+}

itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java

@@ -382,7 +382,7 @@ public void clearTablesCreatedDuringTests() throws Exception {
           continue;
         }
         db.dropTable(dbName, tblName, true, true, fsType == FsType.ENCRYPTED_HDFS);
-        HiveMaterializedViewsRegistry.get().dropMaterializedView(tblObj.getDbName(), tblObj.getTableName());
+        HiveMaterializedViewsRegistry.get().dropMaterializedView(tblObj.getFullTableName());
       }
     }
 

ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/drop/DropMaterializedViewOperation.java

@@ -54,7 +54,7 @@ public int execute() throws HiveException {
 
     // TODO: API w/catalog name
     context.getDb().dropTable(table, false);
-    HiveMaterializedViewsRegistry.get().dropMaterializedView(table.getDbName(), table.getTableName());
+    HiveMaterializedViewsRegistry.get().dropMaterializedView(table.getFullTableName());
     DDLUtils.addIfAbsentByName(new WriteEntity(table, WriteEntity.WriteType.DDL_NO_LOCK), context);
 
     return 0;

ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/show/ShowMaterializedViewsFormatter.java

@@ -137,7 +137,7 @@ void showMaterializedViews(DataOutputStream out, List<Table> materializedViews)
   @NotNull
   private static String formatIncrementalRebuildMode(Table materializedView) {
     HiveRelOptMaterialization relOptMaterialization = HiveMaterializedViewsRegistry.get().
-        getRewritingMaterializedView(materializedView.getDbName(), materializedView.getTableName(), ALL);
+        getRewritingMaterializedView(materializedView.getFullTableName(), ALL);
 
     if (relOptMaterialization == null) {
       return NOT_AVAILABLE.getMessage();

ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/update/MaterializedViewUpdateOperation.java

@@ -48,8 +48,8 @@ public int execute() throws HiveException {
         HiveMaterializedViewsRegistry.get().createMaterializedView(context.getDb().getConf(), mvTable);
       } else if (desc.isDisableRewrite()) {
         // Disabling rewriting, removing from cache
-        String[] names = desc.getName().split("\\.");
-        HiveMaterializedViewsRegistry.get().dropMaterializedView(names[0], names[1]);
+        Table mvTable = context.getDb().getTable(desc.getName());
+        HiveMaterializedViewsRegistry.get().dropMaterializedView(mvTable.getFullTableName());
       } else if (desc.isUpdateCreationMetadata()) {
         // We need to update the status of the creation signature
         Table mvTable = context.getDb().getTable(desc.getName());

ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java

@@ -212,6 +212,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.BitSet;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -2380,7 +2381,7 @@ private List<HiveRelOptMaterialization> getValidMaterializedViews(
         // It passed the test, load
         HiveRelOptMaterialization relOptMaterialization =
             HiveMaterializedViewsRegistry.get().getRewritingMaterializedView(
-                materializedViewTable.getDbName(), materializedViewTable.getTableName(), scope);
+                materializedViewTable.getFullTableName(), scope);
         if (relOptMaterialization != null) {
           Table cachedMaterializedViewTable = extractTable(relOptMaterialization);
           if (cachedMaterializedViewTable.equals(materializedViewTable)) {
@@ -2466,7 +2467,7 @@ public List<Table> getAllMaterializedViewObjectsForRewriting() throws HiveExcept
   public List<HiveRelOptMaterialization> getMaterializedViewsByAST(
           ASTNode astNode, Set<TableName> tablesUsed,  Supplier<String> validTxnsList, HiveTxnManager txnMgr) throws HiveException {
 
-    List<HiveRelOptMaterialization> materializedViews =
+    Collection<HiveRelOptMaterialization> materializedViews =
             HiveMaterializedViewsRegistry.get().getRewritingMaterializedViews(astNode);
     if (materializedViews.isEmpty()) {
       return Collections.emptyList();