[compiler] migrate from testng to munit

build.yaml

@@ -868,15 +868,8 @@ steps:
       set -ex
       cd /io/repo/hail
 
-      export MILLOPTS='--no-daemon' HAIL_BUILD_MODE=CI
-      time retry sh mill --no-daemon --version
-
-      # See `build_hail_jar_and_wheel`
-      time retry make shadowTestJar
-      if [ ! -f out/hail/2.12/test/assembly.dest/out.jar ]; then
-          echo 'no out.jar found after mill assembly returned. going to sleep'
-          sleep 5
-      fi
+      export HAIL_BUILD_MODE=CI
+      time sh mill --no-daemon hail[2.12].test.{compile,testForkGrouping}
     inputs:
       - from: /repo
         to: /io/repo
@@ -900,8 +893,6 @@ steps:
       time tar czf test.tar.gz -C python test pytest.ini
       time tar czf resources.tar.gz -C hail/test resources
       time tar czf data.tar.gz -C python/hail/docs data
-      time TESTNG_SPLITS=5 python3 generate_splits.py
-      time tar czf splits.tar.gz testng-splits-*.xml
     inputs:
       - from: /repo
         to: /io/repo
@@ -910,8 +901,6 @@ steps:
         to: /test.tar.gz
       - from: /io/repo/hail/resources.tar.gz
         to: /resources.tar.gz
-      - from: /io/repo/hail/splits.tar.gz
-        to: /splits.tar.gz
       - from: /io/repo/hail/data.tar.gz
         to: /data.tar.gz
     dependsOn:
@@ -1024,19 +1013,15 @@ steps:
       cpu: '2'
     script: |
       set -ex
-      cd /io
-      mkdir -p hail/test
-      tar xzf resources.tar.gz -C hail/test
-      tar xzf splits.tar.gz
-      export HAIL_TEST_SKIP_R=1
-      java -cp hail-test.jar:$SPARK_HOME/jars/* org.testng.TestNG -listener is.hail.LogTestListener testng-splits-$HAIL_RUN_IMAGE_SPLIT_INDEX.xml
+      cd /io/repo/hail
+      export HAIL_BUILD_MODE='CI'
+      export NO_COLOR=1
+      sh mill --no-daemon -j2 hail[2.12].test.testCISplit $HAIL_RUN_IMAGE_SPLIT_INDEX
     inputs:
-      - from: /resources.tar.gz
-        to: /io/resources.tar.gz
-      - from: /derived/debug/hail/out/hail/2.12/test/assembly.dest/out.jar
-        to: /io/hail-test.jar
-      - from: /splits.tar.gz
-        to: /io/splits.tar.gz
+      - from: /repo
+        to: /io/repo
+      - from: /derived/debug/hail/out
+        to: /io/repo/hail/out
     outputs:
       - from: /io/test-output
         to: /test-output
@@ -3589,23 +3574,18 @@ steps:
       memory: standard
       cpu: '2'
     script: |
-      set -ex
-      cd /io
-      mkdir -p src/test
-      tar xzf resources.tar.gz -C src/test
+      set -x
 
       export HAIL_CLOUD={{ global.cloud }}
       export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
       export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
       export HAIL_FS_TEST_CLOUD_RESOURCES_URI={{ global.test_storage_uri }}/{{ upload_test_resources_to_blob_storage.token }}/test/resources/fs
       export HAIL_TEST_STORAGE_URI={{ global.test_storage_uri }}
 
-      set +e
-      java -Xms7500M -Xmx7500M \
-           -cp hail-test.jar:$SPARK_HOME/jars/* \
-           org.testng.TestNG \
-           -listener is.hail.LogTestListener \
-           testng-fs.xml
+      cd /io/repo/hail
+      export HAIL_BUILD_MODE='CI'
+      export NO_COLOR=1
+      sh mill --no-daemon -j2 hail[2.12].test.test-fs
       exit_code=$?
       set -e
       if [[ $exit_code -eq 2 ]]
@@ -3616,12 +3596,10 @@ steps:
           exit $exit_code
       fi
     inputs:
-      - from: /resources.tar.gz
-        to: /io/resources.tar.gz
-      - from: /derived/debug/hail/out/hail/2.12/test/assembly.dest/out.jar
-        to: /io/hail-test.jar
-      - from: /repo/hail/testng-fs.xml
-        to: /io/testng-fs.xml
+      - from: /repo
+        to: /io/repo
+      - from: /derived/debug/hail/out
+        to: /io/repo/hail/out
     secrets:
       - name: test-gsa-key
         namespace:
@@ -3649,21 +3627,15 @@ steps:
       export HAIL_CLOUD={{ global.cloud }}
       export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
 
-      cd /io
-      mkdir -p src/test
-      tar xzf resources.tar.gz -C src/test
-      java -Xms7500M -Xmx7500M \
-           -cp hail-test.jar:$SPARK_HOME/jars/* \
-           org.testng.TestNG \
-           -listener is.hail.LogTestListener \
-           testng-services.xml
+      cd /io/repo/hail
+      export HAIL_BUILD_MODE='CI'
+      export NO_COLOR=1
+      sh mill --no-daemon -j2 hail[2.12].test.test-fs
     inputs:
-      - from: /resources.tar.gz
-        to: /io/resources.tar.gz
-      - from: /derived/debug/hail/out/hail/2.12/test/assembly.dest/out.jar
-        to: /io/hail-test.jar
-      - from: /repo/hail/testng-services.xml
-        to: /io/testng-services.xml
+      - from: /repo
+        to: /io/repo
+      - from: /derived/debug/hail/out
+        to: /io/repo/hail/out
     secrets:
       - name: test-gsa-key
         namespace:

hail/hail/package.mill

@@ -2,19 +2,21 @@ package build.hail
 
 import mill.*
 import mill.api.{BuildCtx, Result}
+import mill.javalib.testrunner.TestResult
 import mill.scalalib.*
 import mill.scalalib.Assembly.*
-import mill.scalalib.TestModule.TestNg
+import mill.scalalib.TestModule.Munit
 import mill.util.{Jvm, VcsVersion}
-
 import build.Env
 import build.Settings
 import build.HailScalaModule
-
+import mill.api.daemon.internal.TestReporter
 import millbuild.BuildConfig.*
 import millbuild.BuildMode.*
 import millbuild.MvnCoordinate.*
 
+import scala.collection.immutable.ArraySeq
+
 object `package` extends Cross[RootHailModule](enabledScalaVersions)
 
 trait RootHailModule extends CrossScalaModule, HailScalaModule:
@@ -172,7 +174,7 @@ trait RootHailModule extends CrossScalaModule, HailScalaModule:
       PathRef(Task.dest)
     }
 
-  object test extends HailScalaTests, TestNg, CrossValue:
+  object test extends HailScalaTests, Munit, CrossValue:
     override def forkArgs: T[Seq[String]] =
       Seq("-Xss4m", "-Xmx4096M")
 
@@ -181,17 +183,57 @@ trait RootHailModule extends CrossScalaModule, HailScalaModule:
 
     override def mvnDeps: T[Seq[Dep]] =
       outer.mvnDeps() ++ Seq(
-        `guice` :: "5.1.0",
-        `mockito-scala` :: "1.17.31",
+        `munit` :: "1.0.3",
         `scalacheck` :: "1.18.1",
-        `scalacheck-1-18` :: "3.2.19.0",
-        `scalatest` :: "3.2.19",
-        `scalatest-shouldmatchers` :: "3.2.19",
-        `testng-7-10` :: "3.2.19.0",
+        "org.scalameta::munit-scalacheck" :: "1.2.0",
+        `mockito-scala` :: "1.17.31",
       )
 
     override def runMvnDeps: T[Seq[Dep]] =
-      outer.runMvnDeps()
+      outer.runMvnDeps() ++ outer.compileMvnDeps()
 
     override def assemblyRules: Seq[Rule] =
       outer.assemblyRules
+
+    val numCISplits: Int = 5
+
+    // FIXME: we used to run this with `-Xms7500M -Xmx7500M`, will need to split into separate test module
+    // to give it different `forkArgs`
+    def `test-fs`(args: String*): Task.Command[(msg: String, results: Seq[TestResult])] = Task.Command {
+      testTask(
+        Task.Anon { args },
+        Task.Anon { Seq("is.hail.io.fs.*") },
+      )()
+    }
+
+    def `test-services`(args: String*): Task.Command[(msg: String, results: Seq[TestResult])] = Task.Command {
+      testTask(
+        Task.Anon { args },
+        Task.Anon { Seq("is.hail.services.*") },
+      )()
+    }
+
+    def testCISplit(args: String*): Task.Command[(msg: String, results: Seq[TestResult])] = {
+      require(args.length == 1)
+      Task.Command {
+        testTask(
+          Task.Anon {
+            Seq.empty
+          }, ciTestGrouping.map(_.apply(args.head.toInt))
+        )()
+      }
+    }
+
+    def ciTestGrouping: T[Seq[Seq[String]]] = Task {
+      val k = numCISplits
+      val classes = discoveredTestClasses().filter { cls =>
+        !cls.startsWith("is.hail.io.fs") && !cls.startsWith("is.hail.services")
+      }
+      val n = classes.length
+      val shuffled = scala.util.Random(12345).shuffle(classes)
+      val offsets = Range(0, k).map(i => (n - i + k - 1) / k).scan(0)(_ + _)
+      assert(offsets.last == n)
+      Range(0, k).map { i =>
+        shuffled.slice(offsets(i), offsets(i + 1))
+      }
+    }

hail/hail/src/is/hail/annotations/UnsafeUtils.scala

@@ -30,6 +30,12 @@ object UnsafeUtils {
     offset & -alignment
   }
 
+  def roundUpAlignment(offset: Int, alignment: Int): Int = {
+    assert(alignment > 0)
+    assert((alignment & (alignment - 1)) == 0) // power of 2
+    (offset + (alignment - 1)) & ~(alignment - 1)
+  }
+
   def roundDownAlignment(offset: Int, alignment: Int): Int = {
     assert(alignment > 0)
     assert((alignment & (alignment - 1)) == 0) // power of 2

hail/hail/test/src/is/hail/ExecStrategy.scala

@@ -0,0 +1,21 @@
+package is.hail
+
+object ExecStrategy extends Enumeration {
+  type ExecStrategy = Value
+  val Interpret, InterpretUnoptimized, JvmCompile, LoweredJVMCompile, JvmCompileUnoptimized = Value
+
+  val unoptimizedCompileOnly: Set[ExecStrategy] = Set(JvmCompileUnoptimized)
+  val compileOnly: Set[ExecStrategy] = Set(JvmCompile, JvmCompileUnoptimized)
+
+  val javaOnly: Set[ExecStrategy] =
+    Set(Interpret, InterpretUnoptimized, JvmCompile, JvmCompileUnoptimized)
+
+  val interpretOnly: Set[ExecStrategy] = Set(Interpret, InterpretUnoptimized)
+
+  val nonLowering: Set[ExecStrategy] =
+    Set(Interpret, InterpretUnoptimized, JvmCompile, JvmCompileUnoptimized)
+
+  val lowering: Set[ExecStrategy] = Set(LoweredJVMCompile)
+  val backendOnly: Set[ExecStrategy] = Set(LoweredJVMCompile)
+  val allRelational: Set[ExecStrategy] = interpretOnly.union(lowering)
+}