Add --calculate-optimal-total-split option

Donnerbart · Donnerbart · commit 6b6bc537fbea · 2025-03-07T14:45:10.000Z
diff --git a/README.md b/README.md
@@ -27,7 +27,7 @@ java -jar split-tests-java.jar --split-index 0 --split-total 10 --glob '**/*Test
 
 ### Using a JUnit report
 
-For example, check out the project into `project` and the JUnit reports into `reports`. 
+For example, check out the project into `project` and the JUnit reports into `reports`.
 
 ```
 java -jar split-tests-java.jar --split-index 0 --split-total 10 --glob 'project/**/*Test.java' --junit 'reports/**/*.xml'
@@ -38,6 +38,10 @@ java -jar split-tests-java.jar --split-index 0 --split-total 10 --glob 'project/
 ```plain
 Usage: <main class> [options]
   Options:
+    --calculate-optimal-total-split, -o
+      Calculates the optimal test split. Logs a warning if --split-total does 
+      not match.
+      Default: false
     --debug, -d
       Enables debug logging.
       Default: false
@@ -56,6 +60,9 @@ Usage: <main class> [options]
     --junit-glob, -j
       Glob pattern to find JUnit reports. Make sure to single-quote the 
       pattern to avoid shell expansion.
+    --max-optimal-total-split-calculations, -m
+      The maximum number of --calculate-optimal-total-split calculations.
+      Default: 50
     --new-test-time, -n
       Configures the calculation of the test time for tests without JUnit 
       reports. 
@@ -83,6 +90,7 @@ This tool is written in Java and uses Gradle as build tool.
 
 split-tests-java is inspired by [`split-test`](https://github.com/mtsmfm/split-test) for Ruby.
 
-In comparison to split-test, split-tests-java works with default JUnit reports.
+In comparison to `split-test`, `split-tests-java` works with standard JUnit reports without a `file` or `filepath`
+attribute.
 The output are also fully qualified class names instead of file names.
 This makes it compatible with default Java tooling.
diff --git a/src/main/java/de/donnerbart/split/Arguments.java b/src/main/java/de/donnerbart/split/Arguments.java
@@ -51,6 +51,14 @@ class Arguments {
                description = "The working directory. Defaults to the current directory.")
     @Nullable Path workingDirectory;
 
+    @Parameter(names = {"--calculate-optimal-total-split", "-o"},
+               description = "Calculates the optimal test split. Logs a warning if --split-total does not match.")
+    boolean calculateOptimalTotalSplit = false;
+
+    @Parameter(names = {"--max-optimal-total-split-calculations", "-m"},
+               description = "The maximum number of --calculate-optimal-total-split calculations.")
+    int maxOptimalTotalSplitCalculations = 50;
+
     @Parameter(names = {"--debug", "-d"}, description = "Enables debug logging.")
     boolean debug = false;
 
diff --git a/src/main/java/de/donnerbart/split/TestSplit.java b/src/main/java/de/donnerbart/split/TestSplit.java
@@ -12,6 +12,7 @@
 import org.jetbrains.annotations.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.slf4j.helpers.NOPLogger;
 
 import java.io.IOException;
 import java.nio.file.FileSystems;
@@ -33,15 +34,14 @@ public class TestSplit {
             Set.of("org.junit.jupiter.api.Disabled", "org.junit.Ignore");
     private static final @NotNull Set<String> SKIP_TEST_ANNOTATIONS = Set.of("Disabled", "Ignore");
 
-    private static final @NotNull Logger LOG = LoggerFactory.getLogger(TestSplit.class);
-
     private final int splitTotal;
     private final @NotNull String glob;
     private final @Nullable String excludeGlob;
     private final @Nullable String junitGlob;
     private final @NotNull FormatOption formatOption;
     private final @NotNull NewTestTimeOption newTestTimeOption;
     private final @NotNull Path workingDirectory;
+    private final @NotNull Logger log;
     private final boolean debug;
     private final @NotNull Consumer<Integer> exitCodeConsumer;
 
@@ -53,6 +53,7 @@ public TestSplit(
             final @NotNull FormatOption formatOption,
             final @NotNull NewTestTimeOption newTestTimeOption,
             final @NotNull Path workingDirectory,
+            final boolean log,
             final boolean debug,
             final @NotNull Consumer<Integer> exitCodeConsumer) {
         this.splitTotal = splitTotal;
@@ -62,6 +63,7 @@ public TestSplit(
         this.formatOption = formatOption;
         this.newTestTimeOption = newTestTimeOption;
         this.workingDirectory = workingDirectory;
+        this.log = log ? LoggerFactory.getLogger(TestSplit.class) : NOPLogger.NOP_LOGGER;
         this.debug = debug;
         this.exitCodeConsumer = exitCodeConsumer;
     }
@@ -70,17 +72,17 @@ public TestSplit(
         final var testPaths = getPaths(workingDirectory, glob, excludeGlob);
         final var classNames = fileToClassName(testPaths, exitCodeConsumer);
         if (classNames.isEmpty()) {
-            LOG.error("Found no test classes");
+            log.error("Found no test classes");
             exitCodeConsumer.accept(1);
         } else {
-            LOG.info("Found {} test classes", classNames.size());
+            log.info("Found {} test classes", classNames.size());
         }
 
         final var testCases = new HashSet<TestCase>();
         if (junitGlob != null) {
             // analyze JUnit reports
             final var junitPaths = getPaths(workingDirectory, junitGlob, null);
-            LOG.info("Found {} JUnit report files", junitPaths.size());
+            log.info("Found {} JUnit report files", junitPaths.size());
             if (!junitPaths.isEmpty()) {
                 var fastestTest = new TestCase("", Double.MAX_VALUE);
                 var slowestTest = new TestCase("", Double.MIN_VALUE);
@@ -90,7 +92,7 @@ public TestSplit(
                     final var testCase = new TestCase(testSuite.getName(), testSuite.getTime());
                     if (classNames.contains(testCase.name())) {
                         if (testCases.add(testCase)) {
-                            LOG.debug("Adding test {} [{}]", testCase.name(), formatTime(testCase.time()));
+                            log.debug("Adding test {} [{}]", testCase.name(), formatTime(testCase.time()));
                             if (testCase.time() < fastestTest.time()) {
                                 fastestTest = testCase;
                             }
@@ -99,53 +101,53 @@ public TestSplit(
                             }
                         }
                     } else {
-                        LOG.info("Skipping test {} from JUnit report", testCase.name());
+                        log.info("Skipping test {} from JUnit report", testCase.name());
                     }
                 }
-                LOG.debug("Found {} recorded test classes with time information", testCases.size());
-                LOG.debug("Fastest test class: {} ({})", fastestTest.name(), formatTime(fastestTest.time()));
-                LOG.debug("Slowest test class: {} ({})", slowestTest.name(), formatTime(slowestTest.time()));
+                log.debug("Found {} recorded test classes with time information", testCases.size());
+                log.debug("Fastest test class: {} ({})", fastestTest.name(), formatTime(fastestTest.time()));
+                log.debug("Slowest test class: {} ({})", slowestTest.name(), formatTime(slowestTest.time()));
             }
         }
         // add tests without timing records
         final var newTestTime = getNewTestTime(newTestTimeOption, testCases);
         classNames.forEach(className -> {
             final var testCase = new TestCase(className, newTestTime);
             if (testCases.add(testCase)) {
-                LOG.debug("Adding test {} [estimated {}]", testCase.name(), formatTime(testCase.time()));
+                log.debug("Adding test {} [estimated {}]", testCase.name(), formatTime(testCase.time()));
             }
         });
 
         // split tests
-        LOG.debug("Splitting {} tests", testCases.size());
+        log.debug("Splitting {} tests", testCases.size());
         final var splits = new Splits(splitTotal, formatOption);
         testCases.stream().sorted(Comparator.reverseOrder()).forEach(testCase -> {
             final var split = splits.add(testCase);
-            LOG.debug("Adding test {} to split #{}", testCase.name(), split.index());
+            log.debug("Adding test {} to split #{}", testCase.name(), split.index());
         });
 
         if (debug) {
             if (splitTotal > 1) {
                 final var fastestSplit = splits.getFastest();
-                LOG.debug("Fastest test plan is #{} with {} tests ({})",
+                log.debug("Fastest test plan is #{} with {} tests ({})",
                         fastestSplit.formatIndex(),
                         fastestSplit.tests().size(),
                         formatTime(fastestSplit.totalRecordedTime()));
                 final var slowestSplit = splits.getSlowest();
-                LOG.debug("Slowest test plan is #{} with {} tests ({})",
+                log.debug("Slowest test plan is #{} with {} tests ({})",
                         slowestSplit.formatIndex(),
                         slowestSplit.tests().size(),
                         formatTime(slowestSplit.totalRecordedTime()));
-                LOG.debug("Difference between the fastest and slowest test plan: {}",
+                log.debug("Difference between the fastest and slowest test plan: {}",
                         formatTime(slowestSplit.totalRecordedTime() - fastestSplit.totalRecordedTime()));
             }
-            LOG.debug("Test splits:");
-            splits.forEach(split -> LOG.debug(split.toString()));
+            log.debug("Test splits:");
+            splits.forEach(split -> log.debug(split.toString()));
         }
         return splits;
     }
 
-    private static @NotNull Set<Path> getPaths(
+    private @NotNull Set<Path> getPaths(
             final @NotNull Path rootPath,
             final @NotNull String glob,
             final @Nullable String excludeGlob) throws Exception {
@@ -161,7 +163,7 @@ public TestSplit(
                     final var candidate = path.normalize();
                     if (includeMatcher.matches(candidate)) {
                         if (excludeMatcher.matches(candidate)) {
-                            LOG.debug("Excluding test file {}", candidate);
+                            log.debug("Excluding test file {}", candidate);
                         } else {
                             files.add(candidate);
                         }
@@ -178,7 +180,7 @@ public TestSplit(
         return files;
     }
 
-    private static @NotNull Set<String> fileToClassName(
+    private @NotNull Set<String> fileToClassName(
             final @NotNull Set<Path> testPaths,
             final @NotNull Consumer<Integer> exitCodeConsumer) {
         final var javaParser = new JavaParser();
@@ -189,10 +191,10 @@ public TestSplit(
                 final var declaration = compilationUnit.findFirst(ClassOrInterfaceDeclaration.class).orElseThrow();
                 final var className = declaration.getFullyQualifiedName().orElseThrow();
                 if (declaration.isInterface()) {
-                    LOG.info("Skipping interface {}", className);
+                    log.info("Skipping interface {}", className);
                     continue;
                 } else if (declaration.isAbstract()) {
-                    LOG.info("Skipping abstract class {}", className);
+                    log.info("Skipping abstract class {}", className);
                     continue;
                 }
                 final var hasSkipTestImport = compilationUnit.getImports()
@@ -204,19 +206,19 @@ public TestSplit(
                         .map(AnnotationExpr::getNameAsString)
                         .anyMatch(SKIP_TEST_ANNOTATIONS::contains);
                 if (hasSkipTestImport && hasSkipTestAnnotation) {
-                    LOG.info("Skipping disabled test class {}", className);
+                    log.info("Skipping disabled test class {}", className);
                     continue;
                 }
                 classNames.add(className);
             } catch (final Exception e) {
-                LOG.error("Failed to parse test class {}", testPath, e);
+                log.error("Failed to parse test class {}", testPath, e);
                 exitCodeConsumer.accept(1);
             }
         }
         return classNames;
     }
 
-    private static double getNewTestTime(
+    private double getNewTestTime(
             final @NotNull NewTestTimeOption useAverageTimeForNewTests,
             final @NotNull Set<TestCase> testCases) {
         if (testCases.isEmpty()) {
@@ -227,17 +229,17 @@ private static double getNewTestTime(
             case AVERAGE -> {
                 final var averageTime =
                         testCases.stream().mapToDouble(TestCase::time).sum() / (double) testCases.size();
-                LOG.info("Average test time is {}", formatTime(averageTime));
+                log.info("Average test time is {}", formatTime(averageTime));
                 yield averageTime;
             }
             case MIN -> {
                 final var minTime = testCases.stream().mapToDouble(TestCase::time).min().orElseThrow();
-                LOG.info("Minimum test time is {}", formatTime(minTime));
+                log.info("Minimum test time is {}", formatTime(minTime));
                 yield minTime;
             }
             case MAX -> {
                 final var maxTime = testCases.stream().mapToDouble(TestCase::time).max().orElseThrow();
-                LOG.info("Maximum test time is {}", formatTime(maxTime));
+                log.info("Maximum test time is {}", formatTime(maxTime));
                 yield maxTime;
             }
         };
diff --git a/src/main/java/de/donnerbart/split/TestSplitMain.java b/src/main/java/de/donnerbart/split/TestSplitMain.java
@@ -56,13 +56,17 @@ public static void main(final @Nullable String @NotNull [] args) throws Exceptio
             LOG.info("JUnit glob: {}", arguments.junitGlob);
         }
         LOG.info("Output format: {}", arguments.formatOption);
+        if (arguments.calculateOptimalTotalSplit) {
+            calculateOptimalTotalSplit(arguments, workingDirectory);
+        }
         final var testSplit = new TestSplit(arguments.splitTotal,
                 arguments.glob,
                 arguments.excludeGlob,
                 arguments.junitGlob,
                 arguments.formatOption,
                 arguments.newTestTimeOption,
                 workingDirectory,
+                true,
                 arguments.debug,
                 System::exit);
         final var splits = testSplit.run();
@@ -88,4 +92,48 @@ static boolean validateArguments(final @NotNull Arguments arguments, final @NotN
         }
         return true;
     }
+
+    @VisibleForTesting
+    static int calculateOptimalTotalSplit(final @NotNull Arguments arguments, final @NotNull Path workingDirectory)
+            throws Exception {
+        if (arguments.junitGlob == null) {
+            LOG.warn("The option --calculate-optimal-total-split requires --junit-glob");
+            return 0;
+        }
+        LOG.info("Calculating optimal test split");
+        var optimalSplit = 1;
+        var lastSlowestSplit = Double.MAX_VALUE;
+        while (true) {
+            final var testSplit = new TestSplit(optimalSplit,
+                    arguments.glob,
+                    arguments.excludeGlob,
+                    arguments.junitGlob,
+                    arguments.formatOption,
+                    arguments.newTestTimeOption,
+                    workingDirectory,
+                    false,
+                    false,
+                    System::exit);
+            final var splits = testSplit.run();
+            final var slowestSplit = splits.getSlowest().totalRecordedTime();
+            if (Double.compare(slowestSplit, lastSlowestSplit) == 0) {
+                optimalSplit--;
+                LOG.info("The optimal --total-split value for this test suite is {}", optimalSplit);
+                if (optimalSplit != arguments.splitTotal) {
+                    LOG.warn("The --split-total value of {} does not match the optimal split of {}",
+                            arguments.splitTotal,
+                            optimalSplit);
+                }
+                return optimalSplit;
+            }
+            LOG.debug("The slowest split with {} splits takes {}", optimalSplit, formatTime(slowestSplit));
+            if (optimalSplit++ >= arguments.maxOptimalTotalSplitCalculations) {
+                LOG.warn(
+                        "The option --max-optimal-total-split-calculations of {} is too low to calculate the optimal test split",
+                        arguments.maxOptimalTotalSplitCalculations);
+                return 0;
+            }
+            lastSlowestSplit = slowestSplit;
+        }
+    }
 }
diff --git a/src/test/java/de/donnerbart/split/TestSplitMainTest.java b/src/test/java/de/donnerbart/split/TestSplitMainTest.java
diff --git a/src/test/java/de/donnerbart/split/TestSplitTest.java b/src/test/java/de/donnerbart/split/TestSplitTest.java
