benchmarks-extended.yml

name: Extended Performance Benchmarks

on:
  # Run nightly at 2 AM UTC
  schedule:
    - cron: '0 2 * * *'

  # Run when transport code changes
  push:
    branches: [main]
    paths:
      - 'crates/core/src/transport/**'
      - 'crates/core/benches/**'
      - '.github/workflows/benchmarks-extended.yml'

  pull_request:
    paths:
      - 'crates/core/src/transport/**'
      - 'crates/core/benches/**'
      - '.github/workflows/benchmarks-extended.yml'

  # Allow manual trigger
  workflow_dispatch:

# Cancel in-progress runs when a new commit is pushed.
# On main, never cancel — each merge must complete its run (#3311).
concurrency:
  group: benchmarks-extended-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

jobs:
  extended-benchmark:
    name: Extended Benchmarks
    runs-on: ubuntu-latest
    timeout-minutes: 60  # Extended benchmarks take longer
    # Job will fail on severe regressions (>25%) for nightly/main builds
    # PRs only get informational comments, not failures

    env:
      CARGO_TARGET_DIR: ${{ github.workspace }}/target
      RUST_LOG: error

    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0  # Need history for merge-base

      - name: Install system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y liblzma-dev

      - uses: dtolnay/rust-toolchain@stable
        with:
          toolchain: 1.93.0

      - uses: Swatinem/rust-cache@v2
        with:
          prefix-key: bench-extended
          save-if: ${{ github.ref == 'refs/heads/main' }}

      # Determine baseline commit (merge-base for PRs, current for main)
      - name: Determine Baseline Commit
        id: baseline-commit
        run: |
          if [ "${{ github.event_name }}" == "pull_request" ]; then
            BASE_SHA=$(git merge-base origin/${{ github.base_ref }} HEAD)
            echo "Using merge-base for PR: $BASE_SHA"
          else
            BASE_SHA=${{ github.sha }}
            echo "Using current commit for main: $BASE_SHA"
          fi
          echo "sha=$BASE_SHA" >> $GITHUB_OUTPUT

      # Download baseline from merge-base commit
      - name: Download Baseline
        id: baseline-cache
        uses: actions/cache/restore@v5
        with:
          path: target/criterion
          key: criterion-extended-main-${{ runner.os }}-${{ steps.baseline-commit.outputs.sha }}
          restore-keys: |
            criterion-extended-main-${{ runner.os }}-

      - name: Report Baseline Status
        run: |
          echo "## Extended Benchmark Baseline" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

          if [ "${{ steps.baseline-cache.outputs.cache-hit }}" == "true" ]; then
            echo "✅ **Exact baseline match**: Comparing against merge-base commit" >> $GITHUB_STEP_SUMMARY
            echo "- Commit: ${{ steps.baseline-commit.outputs.sha }}" >> $GITHUB_STEP_SUMMARY
          elif [ -d "target/criterion" ]; then
            echo "⚠️ **Restored from fallback**: Using recent main branch baseline" >> $GITHUB_STEP_SUMMARY
            echo "- Target commit: ${{ steps.baseline-commit.outputs.sha }}" >> $GITHUB_STEP_SUMMARY
            echo "- Actual baseline may be from an older commit" >> $GITHUB_STEP_SUMMARY

            if [ -f "target/criterion/.baseline_info" ]; then
              echo "" >> $GITHUB_STEP_SUMMARY
              echo "**Baseline info:**" >> $GITHUB_STEP_SUMMARY
              echo '```' >> $GITHUB_STEP_SUMMARY
              cat target/criterion/.baseline_info >> $GITHUB_STEP_SUMMARY
              echo '```' >> $GITHUB_STEP_SUMMARY
            fi
          else
            echo "📊 **No baseline**: This run will establish the baseline" >> $GITHUB_STEP_SUMMARY
          fi
          echo "" >> $GITHUB_STEP_SUMMARY

      # Compile benchmarks first (fast fail on compilation errors)
      - name: Compile Extended Benchmarks
        run: |
          echo "## Compiling Extended Benchmarks" >> $GITHUB_STEP_SUMMARY
          cargo bench --bench transport_extended --features bench --no-run --color=never 2>&1 | tee compile_output.txt
          echo "✅ Extended benchmark compilation successful" >> $GITHUB_STEP_SUMMARY

          # Find the benchmark binary path for direct execution
          BENCH_BIN=$(find target/release/deps -name 'transport_extended-*' -type f -executable | head -1)
          echo "BENCH_BIN=$BENCH_BIN" >> $GITHUB_ENV
          echo "Benchmark binary: $BENCH_BIN"

      # Run Extended Benchmarks - execute pre-compiled binary directly
      - name: Run Extended Benchmark Suite
        id: bench_extended
        run: |
          echo "## Extended Transport Performance" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "Comprehensive resilience and performance testing:" >> $GITHUB_STEP_SUMMARY
          echo "- **High-latency paths**: 100ms, 200ms, 500ms RTT (detects ssthresh death spiral)" >> $GITHUB_STEP_SUMMARY
          echo "- **Packet loss**: 1%, 5% loss rates (tests reliability layer)" >> $GITHUB_STEP_SUMMARY
          echo "- **Large transfers**: 10MB, 50MB (sustained throughput)" >> $GITHUB_STEP_SUMMARY
          echo "- **Micro-benchmarks**: Component-level validation" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

          # Run pre-compiled benchmark binary directly (avoids cargo recompilation check)
          "$BENCH_BIN" --bench --color never 2>&1 | tee bench_output.txt || true

      # Parse results with structured script
      - name: Parse Benchmark Results
        id: parse_results
        run: |
          python3 --version

          # Run parser - captures exit code
          # Exit codes: 0=ok, 1=regressions, 2=severe regressions
          set +e
          python3 scripts/parse_bench_output.py bench_output.txt > parsed_output.txt 2>&1
          PARSE_EXIT=$?
          set -e

          if [ $PARSE_EXIT -eq 0 ]; then
            echo "regression_detected=false" >> $GITHUB_OUTPUT
            echo "severe_regression=false" >> $GITHUB_OUTPUT
          elif [ $PARSE_EXIT -eq 1 ]; then
            echo "regression_detected=true" >> $GITHUB_OUTPUT
            echo "severe_regression=false" >> $GITHUB_OUTPUT
          elif [ $PARSE_EXIT -eq 2 ]; then
            echo "regression_detected=true" >> $GITHUB_OUTPUT
            echo "severe_regression=true" >> $GITHUB_OUTPUT
          else
            echo "regression_detected=false" >> $GITHUB_OUTPUT
            echo "severe_regression=false" >> $GITHUB_OUTPUT
          fi

          # Append parsed results
          echo "" >> $GITHUB_STEP_SUMMARY
          cat bench_summary.md >> $GITHUB_STEP_SUMMARY 2>/dev/null || {
            echo "⚠️ Failed to parse results" >> $GITHUB_STEP_SUMMARY
            echo '```' >> $GITHUB_STEP_SUMMARY
            # Strip ANSI escape codes for clean markdown output
            tail -100 bench_output.txt | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY
            echo '```' >> $GITHUB_STEP_SUMMARY
          }

          exit 0

      # Save baseline metadata
      - name: Save Baseline Metadata
        if: github.ref == 'refs/heads/main'
        run: |
          mkdir -p target/criterion
          cat > target/criterion/.baseline_info <<EOF
          Benchmark Suite: Extended (transport_extended)
          Commit: ${{ github.sha }}
          Branch: ${{ github.ref }}
          Timestamp: $(date -u +"%Y-%m-%d %H:%M:%S UTC")
          Workflow Run: ${{ github.run_id }}
          Run Type: ${{ github.event_name }}
          EOF

      # Save baseline for future comparisons (only on main)
      - name: Save Baseline
        if: github.ref == 'refs/heads/main'
        uses: actions/cache/save@v5
        with:
          path: target/criterion
          key: criterion-extended-main-${{ runner.os }}-${{ github.sha }}

      # Post PR comment if regressions detected
      - name: Comment on PR
        if: github.event_name == 'pull_request' && steps.parse_results.outputs.regression_detected == 'true'
        uses: actions/github-script@v8
        with:
          script: |
            const fs = require('fs');

            let body;
            try {
              body = fs.readFileSync('bench_pr_comment.md', 'utf8');
            } catch (error) {
              body = `## ⚠️ Extended Benchmark Regressions Detected

            Some extended benchmarks show performance regressions.

            **Note**: Extended benchmarks include high-latency and packet-loss scenarios
            which have higher variance than standard CI benchmarks.

            [View full results](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID})`;
            }

            body += `\n\n[View full benchmark summary](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID})`;

            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: body
            });

      # Upload results as artifacts
      - name: Upload Benchmark Results
        uses: actions/upload-artifact@v7
        with:
          name: extended-benchmark-results
          path: |
            bench_output.txt
            bench_results.json
            bench_summary.md
            bench_pr_comment.md
            target/criterion/**/report/index.html
          retention-days: 90  # Keep extended results longer

      # Fail nightly/main builds on severe throughput regressions (>25%)
      # PRs only get informational comments, not failures
      - name: Fail on Severe Regressions
        if: steps.parse_results.outputs.severe_regression == 'true' && github.event_name != 'pull_request'
        run: |
          echo "🚨 SEVERE THROUGHPUT REGRESSION DETECTED" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "One or more throughput benchmarks regressed by more than 25%." >> $GITHUB_STEP_SUMMARY
          echo "This indicates a significant performance problem that should be investigated." >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "See the benchmark results above for details." >> $GITHUB_STEP_SUMMARY
          exit 1

  # Summary job - reports status but doesn't block PRs
  benchmark-summary:
    name: Extended Benchmark Summary
    runs-on: ubuntu-latest
    timeout-minutes: 5
    needs: extended-benchmark
    if: always()

    steps:
      - name: Check Status
        run: |
          if [ "${{ needs.extended-benchmark.result }}" == "failure" ]; then
            if [ "${{ github.event_name }}" == "pull_request" ]; then
              echo "⚠️ Extended benchmarks detected regressions (informational, not blocking PR)"
            else
              echo "🚨 Extended benchmarks failed - severe throughput regression detected (>25%)"
              echo "This indicates a significant performance problem that should be investigated."
            fi
          elif [ "${{ needs.extended-benchmark.result }}" == "cancelled" ]; then
            echo "⚠️ Extended benchmarks were cancelled"
          else
            echo "✅ Extended benchmarks completed successfully"
          fi