chore: Add github step summaries for e2e test suites

.github/workflows/e2e.yaml

@@ -386,6 +386,13 @@ jobs:
         run: |
           ./hack/gh-workflow-ci.sh run_e2e_tests
 
+      - name: Generate GitHub Step Summary
+        if: ${{ always() }}
+        env:
+          TEST_PROVIDER: ${{ matrix.provider }}
+        run: |
+          ./hack/gh-workflow-ci.sh generate_github_summary
+
       - name: Collect logs
         if: ${{ always() }}
         env:

hack/generate_github_summary.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""Parse gotestsum JSON output and generate a GitHub Step Summary in markdown."""
+
+import collections
+import json
+import sys
+
+
+def main():
+    if len(sys.argv) != 3:
+        print(f"Usage: {sys.argv[0]} <json-file> <provider-name>", file=sys.stderr)
+        sys.exit(1)
+
+    json_file = sys.argv[1]
+    target = sys.argv[2]
+
+    results = {}  # test_name -> {action, elapsed}
+    test_output = collections.defaultdict(list)
+
+    with open(json_file) as fh:
+        for line in fh:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                entry = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            test = entry.get("Test")
+            if not test:
+                continue
+            action = entry.get("Action", "")
+            if action == "output":
+                test_output[test].append(entry.get("Output", ""))
+            elif action in ("pass", "fail", "skip"):
+                results[test] = {
+                    "action": action,
+                    "elapsed": entry.get("Elapsed", 0),
+                }
+
+    passed = sorted([t for t, r in results.items() if r["action"] == "pass"])
+    failed = sorted([t for t, r in results.items() if r["action"] == "fail"])
+    skipped = sorted([t for t, r in results.items() if r["action"] == "skip"])
+
+    # Filter to top-level tests only (no subtests) for counts
+    top_passed = [t for t in passed if "/" not in t]
+    top_failed = [t for t in failed if "/" not in t]
+    top_skipped = [t for t in skipped if "/" not in t]
+
+    status = ":x: FAILED" if top_failed else ":white_check_mark: PASSED"
+
+    lines = []
+    lines.append(f"## E2E Tests: {target} {status}")
+    lines.append("")
+    lines.append(
+        f":white_check_mark: **{len(top_passed)}** passed"
+        f" | :x: **{len(top_failed)}** failed"
+        f" | :fast_forward: **{len(top_skipped)}** skipped"
+    )
+    lines.append("")
+
+    if top_failed:
+        lines.append("### Failed Tests")
+        lines.append("")
+        for t in top_failed:
+            elapsed = results[t]["elapsed"]
+            lines.append(f"<details><summary>:x: {t} ({elapsed:.1f}s)</summary>")
+            lines.append("")
+            lines.append("```")
+            output = "".join(test_output.get(t, []))
+            # Include subtest output too
+            for sub, r in sorted(results.items()):
+                if sub.startswith(t + "/") and r["action"] == "fail":
+                    output += "".join(test_output.get(sub, []))
+            lines.append(output.rstrip())
+            lines.append("```")
+            lines.append("")
+            lines.append("</details>")
+            lines.append("")
+
+    if top_passed:
+        lines.append("<details><summary>Passed Tests</summary>")
+        lines.append("")
+        for t in top_passed:
+            elapsed = results[t]["elapsed"]
+            lines.append(f"- :white_check_mark: {t} ({elapsed:.1f}s)")
+        lines.append("")
+        lines.append("</details>")
+        lines.append("")
+
+    if top_skipped:
+        lines.append("<details><summary>Skipped Tests</summary>")
+        lines.append("")
+        for t in top_skipped:
+            lines.append(f"- :fast_forward: {t}")
+        lines.append("")
+        lines.append("</details>")
+
+    print("\n".join(lines))
+
+
+if __name__ == "__main__":
+    main()

hack/gh-workflow-ci.sh

@@ -115,8 +115,8 @@ get_tests() {
   # which runs the workflow YAML from main (old target names).
   local github_chunk_size github_remainder
   if [[ ${#github_tests[@]} -gt 0 ]]; then
-    github_chunk_size=$(( ${#github_tests[@]} / 2 ))
-    github_remainder=$(( ${#github_tests[@]} % 2 ))
+    github_chunk_size=$((${#github_tests[@]} / 2))
+    github_remainder=$((${#github_tests[@]} % 2))
   fi
 
   case "${target}" in
@@ -287,6 +287,23 @@ collect_logs() {
   detect_panic
 }
 
+generate_github_summary() {
+  local raw_output="/tmp/logs/e2e-test-output.json"
+  local target="${TEST_PROVIDER:-unknown}"
+
+  if [[ -z "${GITHUB_STEP_SUMMARY:-}" ]]; then
+    echo "GITHUB_STEP_SUMMARY not set, skipping summary generation"
+    return 0
+  fi
+
+  if [[ ! -f "${raw_output}" ]]; then
+    echo "No test output found at ${raw_output}, skipping summary generation"
+    return 0
+  fi
+
+  python3 ./hack/generate_github_summary.py "${raw_output}" "${target}" >>"${GITHUB_STEP_SUMMARY}"
+}
+
 detect_panic() {
   # shellcheck disable=SC2016
   (find /tmp/logs/ -type f -regex '.*/pipelines-as-code.*/[0-9]\.log$' | xargs -r sed -n '/stderr F panic:.*/,$p' | head -n 80) >/tmp/panic.log
@@ -328,6 +345,10 @@ help() {
     Will output logs using snazzy formatting when available or otherwise through a simple
     python formatter. This makes debugging easier from the GitHub Actions interface.
 
+  generate_github_summary
+    Parse gotestsum JSON output and write a markdown summary to GITHUB_STEP_SUMMARY.
+    Required env vars: TEST_PROVIDER, GITHUB_STEP_SUMMARY
+
   print_tests
     Print the list of tests that would be run for each provider target.
 
@@ -350,6 +371,9 @@ collect_logs)
 output_logs)
   output_logs
   ;;
+generate_github_summary)
+  generate_github_summary
+  ;;
 print_tests)
   set +x
   for target in github_public github_ghe_1 github_ghe_2 github_ghe_3 gitlab_bitbucket gitea_1 gitea_2 gitea_3 concurrency flaky; do