[KYUUBI #7310] JSON metrics report path supports variables substitution

pan3793 · pan3793 · commit dfd9b4935378 · 2026-01-21T21:46:03.000+08:00
### Why are the changes needed? This PR enhances `kyuubi.metrics.json.location` to support variable substitution: `{{KYUUBI_HOME}}`, `{{KYUUBI_WORK_DIR_ROOT}}`, and changes the default value to `{{KYUUBI_HOME}}/metrics` - I suppose it is a more reasonable default value than `{{KYUUBI_WORK_DIR_ROOT}}/metrics`, given others dirs also under `KYUUBI_HOME` by default ``` export KYUUBI_LOG_DIR="${KYUUBI_LOG_DIR:-"${KYUUBI_HOME}/logs"}" export KYUUBI_PID_DIR="${KYUUBI_PID_DIR:-"${KYUUBI_HOME}/pid"}" export KYUUBI_WORK_DIR_ROOT="${KYUUBI_WORK_DIR_ROOT:-"${KYUUBI_HOME}/work"}" ``` To make things more consistent, variable substitution behavior is also updated for `kyuubi.metadata.store.jdbc.url` ### How was this patch tested? Manually tested by ``` bin/kyuubi run \ --conf kyuubi.metrics.reporters=JSON \ --conf kyuubi.metrics.json.location={{KYUUBI_HOME}}/metrics ``` ``` bin/kyuubi run \ --conf kyuubi.metrics.reporters=JSON \ --conf kyuubi.metrics.json.location={{KYUUBI_WORK_DIR_ROOT}}/metrics ``` and checks the metrics JSON file. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #7310 from pan3793/json-metrics. Closes #7310 71e4b28 [Cheng Pan] Update kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala 1100283 [Cheng Pan] nit 2b27c38 [Cheng Pan] rename d6db4e2 [Cheng Pan] fix 0fb1165 [Cheng Pan] JSON metrics report path supports variables substitution Authored-by: Cheng Pan <chengpan@apache.org> Signed-off-by: Cheng Pan <chengpan@apache.org>
diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md
diff --git a/docs/deployment/migration-guide.md b/docs/deployment/migration-guide.md
@@ -17,6 +17,11 @@
 
 # Kyuubi Migration Guide
 
+## Upgrading from Kyuubi 1.11 to 1.12
+
+* Since Kyuubi 1.12, the support of variable `<KYUUBI_HOME>` substitution in config `kyuubi.metadata.store.jdbc.url` is deprecated, use `{{KYUUBI_HOME}}` instead.
+* Since Kyuubi 1.12, default value of `kyuubi.metrics.json.location` is changed to `{{KYUUBI_HOME}}/metrics`, to restore previous behavior, change it to `{{KYUUBI_WORK_DIR_ROOT}}/metrics`.
+
 ## Upgrading from Kyuubi 1.10 to 1.11
 
 * Since Kyuubi 1.11, the support of Spark engine for Spark 3.2 is removed.
diff --git a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineLocal.scala b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineLocal.scala
@@ -209,7 +209,7 @@ trait WithFlinkSQLEngineLocal extends KyuubiFunSuite with WithFlinkTestResources
         }
     }.orElse {
       // 2. get the main resource jar from system build default
-      env.get(KYUUBI_HOME).toSeq
+      env.get(KYUUBI_HOME_ENV_VAR_NAME).toSeq
         .flatMap { p =>
           Seq(
             Paths.get(p, "externals", "engines", shortName, jarName),
diff --git a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineOnYarn.scala b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineOnYarn.scala
@@ -32,7 +32,7 @@ import org.apache.hadoop.yarn.server.MiniYARNCluster
 
 import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiFunSuite, SCALA_COMPILE_VERSION, Utils}
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.{ENGINE_FLINK_APPLICATION_JARS, KYUUBI_HOME}
+import org.apache.kyuubi.config.KyuubiConf.{ENGINE_FLINK_APPLICATION_JARS, KYUUBI_HOME_ENV_VAR_NAME}
 import org.apache.kyuubi.ha.HighAvailabilityConf.HA_ADDRESSES
 import org.apache.kyuubi.util.JavaUtils
 import org.apache.kyuubi.util.command.CommandLineUtils._
@@ -238,7 +238,7 @@ trait WithFlinkSQLEngineOnYarn extends KyuubiFunSuite with WithFlinkTestResource
       }
     }.orElse {
       // 2. get the main resource jar from system build default
-      env.get(KYUUBI_HOME).toSeq
+      env.get(KYUUBI_HOME_ENV_VAR_NAME).toSeq
         .flatMap { p =>
           Seq(
             Paths.get(p, "externals", "engines", shortName, jarName),
diff --git a/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/operation/FlinkOperationSuite.scala b/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/operation/FlinkOperationSuite.scala
@@ -28,7 +28,7 @@ class FlinkOperationSuite extends WithKyuubiServerAndFlinkMiniCluster
   with HiveJDBCTestHelper {
 
   override val conf: KyuubiConf = KyuubiConf()
-    .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome)
+    .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME_ENV_VAR_NAME", kyuubiHome)
     .set(ENGINE_TYPE, "FLINK_SQL")
     .set("flink.parallelism.default", "2")
 
diff --git a/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/operation/FlinkOperationSuiteOnYarn.scala b/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/operation/FlinkOperationSuiteOnYarn.scala
@@ -36,7 +36,7 @@ class FlinkOperationSuiteOnYarn extends WithKyuubiServerAndYarnMiniCluster
 
   override def beforeAll(): Unit = {
     conf
-      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome)
+      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME_ENV_VAR_NAME", kyuubiHome)
       .set(ENGINE_TYPE, "FLINK_SQL")
       .set("flink.execution.target", "yarn-application")
       .set("flink.parallelism.default", "2")
diff --git a/integration-tests/kyuubi-hive-it/src/test/scala/org/apache/kyuubi/it/hive/operation/KyuubiOperationHiveEnginePerConnectionSuite.scala b/integration-tests/kyuubi-hive-it/src/test/scala/org/apache/kyuubi/it/hive/operation/KyuubiOperationHiveEnginePerConnectionSuite.scala
@@ -34,7 +34,7 @@ class KyuubiOperationHiveEnginePerConnectionSuite extends WithKyuubiServer with
     metastore.toFile.delete()
     val currentUser = Utils.currentUser
     KyuubiConf()
-      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome)
+      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME_ENV_VAR_NAME", kyuubiHome)
       .set(ENGINE_TYPE, "HIVE_SQL")
       .set(ENGINE_SHARE_LEVEL, "connection")
       // increase this to 30s as hive session state and metastore client is slow initializing
diff --git a/integration-tests/kyuubi-hive-it/src/test/scala/org/apache/kyuubi/it/hive/operation/KyuubiOperationHiveEnginePerUserSuite.scala b/integration-tests/kyuubi-hive-it/src/test/scala/org/apache/kyuubi/it/hive/operation/KyuubiOperationHiveEnginePerUserSuite.scala
@@ -31,7 +31,7 @@ class KyuubiOperationHiveEnginePerUserSuite extends WithKyuubiServer with HiveEn
     val metastore = Utils.createTempDir(prefix = getClass.getSimpleName)
     metastore.toFile.delete()
     KyuubiConf()
-      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome)
+      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME_ENV_VAR_NAME", kyuubiHome)
       .set(ENGINE_TYPE, "HIVE_SQL")
       // increase this to 30s as hive session state and metastore client is slow initializing
       .setIfMissing(ENGINE_IDLE_TIMEOUT, 30000L)
diff --git a/integration-tests/kyuubi-hive-it/src/test/scala/org/apache/kyuubi/it/hive/operation/KyuubiOperationHiveEngineYarnModeSuite.scala b/integration-tests/kyuubi-hive-it/src/test/scala/org/apache/kyuubi/it/hive/operation/KyuubiOperationHiveEngineYarnModeSuite.scala
@@ -18,7 +18,7 @@ package org.apache.kyuubi.it.hive.operation
 
 import org.apache.kyuubi.{HiveEngineTests, Utils, WithKyuubiServerAndHadoopMiniCluster}
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_TYPE, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME}
+import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_TYPE, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME_ENV_VAR_NAME}
 import org.apache.kyuubi.engine.deploy.DeployMode
 
 class KyuubiOperationHiveEngineYarnModeSuite extends HiveEngineTests
@@ -28,7 +28,7 @@ class KyuubiOperationHiveEngineYarnModeSuite extends HiveEngineTests
     val metastore = Utils.createTempDir(prefix = getClass.getSimpleName)
     metastore.toFile.delete()
     KyuubiConf()
-      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome)
+      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME_ENV_VAR_NAME", kyuubiHome)
       .set(ENGINE_TYPE, "HIVE_SQL")
       .set(KyuubiConf.ENGINE_HIVE_DEPLOY_MODE, DeployMode.YARN.toString)
       // increase this to 30s as hive session state and metastore client is slow initializing
diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/WithKyuubiServerAndClickHouseContainer.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/clickhouse/WithKyuubiServerAndClickHouseContainer.scala
@@ -21,7 +21,7 @@ import java.time.Duration
 
 import org.apache.kyuubi.WithKyuubiServer
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_JDBC_EXTRA_CLASSPATH, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME}
+import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_JDBC_EXTRA_CLASSPATH, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME_ENV_VAR_NAME}
 import org.apache.kyuubi.engine.jdbc.clickhouse.WithClickHouseEngine
 import org.apache.kyuubi.util.JavaUtils
 
@@ -48,7 +48,7 @@ trait WithKyuubiServerAndClickHouseContainer extends WithKyuubiServer with WithC
 
   override protected val conf: KyuubiConf = {
     KyuubiConf()
-      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome)
+      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME_ENV_VAR_NAME", kyuubiHome)
       .set(ENGINE_JDBC_EXTRA_CLASSPATH, clickHouseJdbcConnectorPath)
       .set(ENGINE_IDLE_TIMEOUT, Duration.ofMinutes(1).toMillis)
   }
diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/WithKyuubiServerAndDorisContainer.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/WithKyuubiServerAndDorisContainer.scala
@@ -21,7 +21,7 @@ import java.nio.file.{Files, Path, Paths}
 
 import org.apache.kyuubi.WithKyuubiServer
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_EXTRA_CLASSPATH, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME}
+import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_EXTRA_CLASSPATH, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME_ENV_VAR_NAME}
 import org.apache.kyuubi.engine.jdbc.doris.WithDorisEngine
 import org.apache.kyuubi.util.JavaUtils
 
@@ -48,7 +48,7 @@ trait WithKyuubiServerAndDorisContainer extends WithKyuubiServer with WithDorisE
 
   override protected val conf: KyuubiConf = {
     KyuubiConf()
-      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome)
+      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME_ENV_VAR_NAME", kyuubiHome)
       .set(ENGINE_JDBC_EXTRA_CLASSPATH, mysqlJdbcConnectorPath)
   }
 
diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/WithKyuubiServerAndImpalaContainer.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/WithKyuubiServerAndImpalaContainer.scala
@@ -21,7 +21,7 @@ import java.nio.file.{Files, Path, Paths}
 
 import org.apache.kyuubi.WithKyuubiServer
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_EXTRA_CLASSPATH, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME}
+import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_EXTRA_CLASSPATH, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME_ENV_VAR_NAME}
 import org.apache.kyuubi.engine.jdbc.impala.WithImpalaEngine
 import org.apache.kyuubi.util.JavaUtils
 
@@ -48,7 +48,7 @@ trait WithKyuubiServerAndImpalaContainer extends WithKyuubiServer with WithImpal
 
   override protected val conf: KyuubiConf = {
     KyuubiConf()
-      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome)
+      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME_ENV_VAR_NAME", kyuubiHome)
       .set(ENGINE_JDBC_EXTRA_CLASSPATH, hiveJdbcConnectorPath)
   }
 
diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/WithKyuubiServerAndMySQLContainer.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/WithKyuubiServerAndMySQLContainer.scala
@@ -21,7 +21,7 @@ import java.nio.file.{Files, Path, Paths}
 
 import org.apache.kyuubi.WithKyuubiServer
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_EXTRA_CLASSPATH, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME}
+import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_EXTRA_CLASSPATH, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME_ENV_VAR_NAME}
 import org.apache.kyuubi.engine.jdbc.mysql.WithMySQLEngine
 import org.apache.kyuubi.util.JavaUtils
 
@@ -48,7 +48,7 @@ trait WithKyuubiServerAndMySQLContainer extends WithKyuubiServer with WithMySQLE
 
   override protected val conf: KyuubiConf = {
     KyuubiConf()
-      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome)
+      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME_ENV_VAR_NAME", kyuubiHome)
       .set(ENGINE_JDBC_EXTRA_CLASSPATH, mysqlJdbcConnectorPath)
   }
 
diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/WithKyuubiServerAndMySQLContainerYarnMode.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/mysql/WithKyuubiServerAndMySQLContainerYarnMode.scala
@@ -20,7 +20,7 @@ import java.nio.file.{Files, Path, Paths}
 
 import org.apache.kyuubi.WithKyuubiServerAndHadoopMiniCluster
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_TYPE, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME}
+import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_TYPE, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME_ENV_VAR_NAME}
 import org.apache.kyuubi.engine.deploy.DeployMode
 import org.apache.kyuubi.engine.jdbc.mysql.MySQLOperationSuite
 
@@ -45,7 +45,7 @@ class WithKyuubiServerAndMySQLContainerYarnMode extends MySQLOperationSuite
 
   override protected val conf: KyuubiConf = {
     KyuubiConf()
-      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome)
+      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME_ENV_VAR_NAME", kyuubiHome)
       .set(ENGINE_JDBC_EXTRA_CLASSPATH, mysqlJdbcConnectorPath)
       .set(ENGINE_TYPE, "JDBC")
       .set(KyuubiConf.ENGINE_JDBC_DEPLOY_MODE, DeployMode.YARN.toString)
diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/WithKyuubiServerAndPhoenixContainer.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/phoenix/WithKyuubiServerAndPhoenixContainer.scala
@@ -21,7 +21,7 @@ import java.time.Duration
 
 import org.apache.kyuubi.WithKyuubiServer
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_JDBC_EXTRA_CLASSPATH, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME}
+import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_JDBC_EXTRA_CLASSPATH, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME_ENV_VAR_NAME}
 import org.apache.kyuubi.engine.jdbc.phoenix.WithPhoenixEngine
 import org.apache.kyuubi.util.JavaUtils
 
@@ -48,7 +48,7 @@ trait WithKyuubiServerAndPhoenixContainer extends WithKyuubiServer with WithPhoe
 
   override protected val conf: KyuubiConf = {
     KyuubiConf()
-      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome)
+      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME_ENV_VAR_NAME", kyuubiHome)
       .set(ENGINE_JDBC_EXTRA_CLASSPATH, phoenixJdbcConnectorPath)
       .set(ENGINE_IDLE_TIMEOUT, Duration.ofMinutes(1).toMillis)
   }
diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/WithKyuubiServerAndPostgreSQLContainer.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/postgresql/WithKyuubiServerAndPostgreSQLContainer.scala
@@ -21,7 +21,7 @@ import java.time.Duration
 
 import org.apache.kyuubi.WithKyuubiServer
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_JDBC_EXTRA_CLASSPATH, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME}
+import org.apache.kyuubi.config.KyuubiConf.{ENGINE_IDLE_TIMEOUT, ENGINE_JDBC_EXTRA_CLASSPATH, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME_ENV_VAR_NAME}
 import org.apache.kyuubi.engine.jdbc.postgresql.WithPostgreSQLEngine
 import org.apache.kyuubi.util.JavaUtils
 
@@ -48,7 +48,7 @@ trait WithKyuubiServerAndPostgreSQLContainer extends WithKyuubiServer with WithP
 
   override protected val conf: KyuubiConf = {
     KyuubiConf()
-      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome)
+      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME_ENV_VAR_NAME", kyuubiHome)
       .set(ENGINE_JDBC_EXTRA_CLASSPATH, postgresqlJdbcConnectorPath)
       .set(ENGINE_IDLE_TIMEOUT, Duration.ofMinutes(1).toMillis)
   }
diff --git a/integration-tests/kyuubi-trino-it/src/test/scala/org/apache/kyuubi/it/trino/WithKyuubiServerAndTrinoContainer.scala b/integration-tests/kyuubi-trino-it/src/test/scala/org/apache/kyuubi/it/trino/WithKyuubiServerAndTrinoContainer.scala
@@ -36,7 +36,7 @@ trait WithKyuubiServerAndTrinoContainer extends WithKyuubiServer with TestContai
 
   override protected val conf: KyuubiConf = {
     KyuubiConf()
-      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome)
+      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME_ENV_VAR_NAME", kyuubiHome)
       .set(ENGINE_TYPE, "TRINO")
       .set(ENGINE_TRINO_CONNECTION_CATALOG, "memory")
   }
diff --git a/integration-tests/kyuubi-trino-it/src/test/scala/org/apache/kyuubi/it/trino/server/TrinoFrontendSuite.scala b/integration-tests/kyuubi-trino-it/src/test/scala/org/apache/kyuubi/it/trino/server/TrinoFrontendSuite.scala
@@ -21,7 +21,7 @@ import scala.util.control.NonFatal
 
 import org.apache.kyuubi.WithKyuubiServer
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.{KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME}
+import org.apache.kyuubi.config.KyuubiConf.{KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME_ENV_VAR_NAME}
 import org.apache.kyuubi.operation.SparkMetadataTests
 import org.apache.kyuubi.util.JavaUtils
 
@@ -60,7 +60,7 @@ class TrinoFrontendSuite extends WithKyuubiServer with SparkMetadataTests {
   override protected val conf: KyuubiConf = {
     KyuubiConf()
       .set(KyuubiConf.FRONTEND_PROTOCOLS, Seq("TRINO"))
-      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome)
+      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME_ENV_VAR_NAME", kyuubiHome)
   }
 
   override protected def jdbcUrl: String = {
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala
@@ -38,7 +38,7 @@ import org.apache.hadoop.util.ShutdownHookManager
 
 import org.apache.kyuubi.config.KyuubiConf
 import org.apache.kyuubi.config.internal.Tests.IS_TESTING
-import org.apache.kyuubi.util.TempFileCleanupUtils
+import org.apache.kyuubi.util.{JavaUtils, TempFileCleanupUtils}
 import org.apache.kyuubi.util.command.CommandLineUtils._
 
 object Utils extends Logging {
@@ -66,7 +66,7 @@ object Utils extends Logging {
 
   def getPropertiesFile(fileName: String, env: Map[String, String] = sys.env): Option[File] = {
     env.get(KYUUBI_CONF_DIR)
-      .orElse(env.get(KYUUBI_HOME).map(_ + File.separator + "conf"))
+      .orElse(env.get(KYUUBI_HOME_ENV_VAR_NAME).map(_ + File.separator + "conf"))
       .map(d => new File(d + File.separator + fileName))
       .filter(_.exists())
       .orElse {
@@ -135,6 +135,24 @@ object Utils extends Logging {
     }
   }
 
+  def substituteKyuubiEnvVars(original: String, env: Map[String, String] = sys.env): String = {
+    lazy val KYUUBI_HOME = env.getOrElse(
+      KYUUBI_HOME_ENV_VAR_NAME,
+      JavaUtils.getCodeSourceLocation(this.getClass).split("kyuubi-common").head)
+    lazy val KYUUBI_WORK_DIR_ROOT = env.getOrElse(
+      "KYUUBI_WORK_DIR_ROOT",
+      Paths.get(KYUUBI_HOME, "work").toAbsolutePath.toString)
+    // save cost of evaluating replacement when pattern not found
+    def substitute(input: String, pattern: String, replacement: => String): String = {
+      if (input.contains(pattern)) input.replace(pattern, replacement) else input
+    }
+    var substituted = original
+    substituted = substitute(original, "<KYUUBI_HOME>", KYUUBI_HOME) // deprecated since 1.12.0
+    substituted = substitute(substituted, "{{KYUUBI_HOME}}", KYUUBI_HOME)
+    substituted = substitute(substituted, "{{KYUUBI_WORK_DIR_ROOT}}", KYUUBI_WORK_DIR_ROOT)
+    substituted
+  }
+
   /**
    * Delete a directory recursively.
    */
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -246,7 +246,7 @@ object KyuubiConf {
 
   /** the default file that contains kyuubi properties */
   final val KYUUBI_CONF_FILE_NAME = "kyuubi-defaults.conf"
-  final val KYUUBI_HOME = "KYUUBI_HOME"
+  final val KYUUBI_HOME_ENV_VAR_NAME = "KYUUBI_HOME"
   final val KYUUBI_ENGINE_ENV_PREFIX = "kyuubi.engineEnv"
   final val KYUUBI_ENGINE_YARN_MODE_ENV_PREFIX = "kyuubi.engine.yarn.AMEnv"
   final val KYUUBI_BATCH_CONF_PREFIX = "kyuubi.batchConf"
diff --git a/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/JsonReporterService.scala b/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/JsonReporterService.scala
@@ -39,12 +39,12 @@ class JsonReporterService(registry: MetricRegistry)
   extends AbstractService("JsonReporterService") {
   private val jsonMapper = new ObjectMapper().registerModule(
     new MetricsModule(TimeUnit.SECONDS, TimeUnit.MILLISECONDS, false))
-  private val timer = new Timer(true)
+  private val timer = new Timer("JSON-metrics-timer", true)
   private var reportDir: Path = _
   private var reportPath: Path = _
 
   override def initialize(conf: KyuubiConf): Unit = synchronized {
-    reportDir = Utils.getAbsolutePathFromWork(conf.get(METRICS_JSON_LOCATION))
+    reportDir = Paths.get(Utils.substituteKyuubiEnvVars(conf.get(METRICS_JSON_LOCATION)))
     Files.createDirectories(reportDir)
     reportPath = Paths.get(reportDir.toString, "report.json").toAbsolutePath
     super.initialize(conf)
diff --git a/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConf.scala b/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConf.scala
@@ -59,10 +59,12 @@ object MetricsConf {
 
   val METRICS_JSON_LOCATION: ConfigEntry[String] = buildConf("kyuubi.metrics.json.location")
     .serverOnly
-    .doc("Where the JSON metrics file located")
+    .doc("Where the JSON metrics file located. " +
+      "Note: this value support the variables substitution: `{{KYUUBI_HOME}}`, " +
+      "`{{KYUUBI_WORK_DIR_ROOT}}`.")
     .version("1.2.0")
     .stringConf
-    .createWithDefault("metrics")
+    .createWithDefault("{{KYUUBI_HOME}}/metrics")
 
   val METRICS_JSON_INTERVAL: ConfigEntry[Long] = buildConf("kyuubi.metrics.json.interval")
     .serverOnly
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala
@@ -30,7 +30,7 @@ import org.apache.commons.lang3.StringUtils.containsIgnoreCase
 
 import org.apache.kyuubi._
 import org.apache.kyuubi.config.{KyuubiConf, KyuubiReservedKeys}
-import org.apache.kyuubi.config.KyuubiConf.KYUUBI_HOME
+import org.apache.kyuubi.config.KyuubiConf.KYUUBI_HOME_ENV_VAR_NAME
 import org.apache.kyuubi.operation.log.OperationLog
 import org.apache.kyuubi.util.{JavaUtils, NamedThreadFactory}
 
@@ -74,7 +74,7 @@ trait ProcBuilder {
       }
     }.orElse {
       // 2. get the main resource jar from system build default
-      env.get(KYUUBI_HOME).toSeq
+      env.get(KYUUBI_HOME_ENV_VAR_NAME).toSeq
         .flatMap { p =>
           Seq(
             Paths.get(p, "externals", "engines", shortName, jarName),
@@ -328,7 +328,7 @@ trait ProcBuilder {
     env.get(homeKey).filter(StringUtils.isNotBlank)
       .orElse {
         // 2. get from $KYUUBI_HOME/externals/kyuubi-download/target
-        env.get(KYUUBI_HOME).flatMap { p =>
+        env.get(KYUUBI_HOME_ENV_VAR_NAME).flatMap { p =>
           val candidates = Paths.get(p, "externals", "kyuubi-download", "target")
             .toFile.listFiles(engineHomeDirFilter)
           if (candidates == null) None else candidates.map(_.toPath).headOption
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/jdbc/JDBCMetadataStoreConf.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/jdbc/JDBCMetadataStoreConf.scala
@@ -19,24 +19,15 @@ package org.apache.kyuubi.server.metadata.jdbc
 
 import java.util.Properties
 
+import org.apache.kyuubi.Utils
 import org.apache.kyuubi.config.{ConfigEntry, KyuubiConf, OptionalConfigEntry}
 import org.apache.kyuubi.config.KyuubiConf.buildConf
-import org.apache.kyuubi.util.JavaUtils
 
 object JDBCMetadataStoreConf {
   final val METADATA_STORE_JDBC_DATASOURCE_PREFIX = "kyuubi.metadata.store.jdbc.datasource"
 
   def getMetadataStoreJdbcUrl(conf: KyuubiConf): String = {
-    val rawJdbcUrl = conf.get(METADATA_STORE_JDBC_URL)
-    if (rawJdbcUrl.contains("<KYUUBI_HOME>")) {
-      rawJdbcUrl.replace(
-        "<KYUUBI_HOME>",
-        sys.env.getOrElse(
-          "KYUUBI_HOME",
-          JavaUtils.getCodeSourceLocation(getClass).split("kyuubi-server").head))
-    } else {
-      rawJdbcUrl
-    }
+    Utils.substituteKyuubiEnvVars(conf.get(METADATA_STORE_JDBC_URL))
   }
 
   /** Get metadata store jdbc datasource properties. */
@@ -86,11 +77,12 @@ object JDBCMetadataStoreConf {
       .doc("The JDBC url for server JDBC metadata store. By default, it is a SQLite database " +
         "url, and the state information is not shared across Kyuubi instances. To enable high " +
         "availability for multiple kyuubi instances, please specify a production JDBC url. " +
-        "Note: this value support the variables substitution: `<KYUUBI_HOME>`.")
+        "Note: this value support the variables substitution: `{{KYUUBI_HOME}}`, " +
+        "`{{KYUUBI_WORK_DIR_ROOT}}`.")
       .version("1.6.0")
       .serverOnly
       .stringConf
-      .createWithDefault("jdbc:sqlite:<KYUUBI_HOME>/kyuubi_state_store.db")
+      .createWithDefault("jdbc:sqlite:{{KYUUBI_HOME}}/kyuubi_state_store.db")
 
   val METADATA_STORE_JDBC_USER: ConfigEntry[String] =
     buildConf("kyuubi.metadata.store.jdbc.user")
diff --git a/kyuubi-zookeeper/src/main/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeper.scala b/kyuubi-zookeeper/src/main/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeper.scala

Original file line number	Diff line number	Diff line change
`@@ -209,7 +209,7 @@ trait WithFlinkSQLEngineLocal extends KyuubiFunSuite with WithFlinkTestResources`
`209`	`209`	`}`
`210`	`210`	`}.orElse {`
`211`	`211`	`// 2. get the main resource jar from system build default`
`212`		`- env.get(KYUUBI_HOME).toSeq`
	`212`	`+ env.get(KYUUBI_HOME_ENV_VAR_NAME).toSeq`
`213`	`213`	`.flatMap { p =>`
`214`	`214`	`Seq(`
`215`	`215`	`Paths.get(p, "externals", "engines", shortName, jarName),`