Merge pull request #28 from gianlucamazza/chore/ci-deps-dependabot-batch #115
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Performance Tests | |
| on: | |
| push: | |
| branches: [main, develop] | |
| pull_request: | |
| branches: [main, develop] | |
| schedule: | |
| # Run weekly on Monday at 2 AM UTC | |
| - cron: '0 2 * * 1' | |
| workflow_dispatch: # Allow manual triggering | |
| # Least privilege by default; jobs widen this to what they actually need. | |
| permissions: | |
| contents: read | |
| jobs: | |
| performance-tests: | |
| name: Run Performance Benchmarks | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| permissions: | |
| contents: read | |
| pull-requests: write # posts the benchmark summary comment | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 # Full history for baseline comparison | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.12' | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 | |
| with: | |
| enable-cache: true | |
| - name: Install dependencies | |
| run: | | |
| uv sync --frozen --all-extras | |
| - name: Run performance tests | |
| run: | | |
| uv run pytest -v -m performance \ | |
| --tb=short \ | |
| --no-cov \ | |
| --benchmark-only \ | |
| --benchmark-autosave \ | |
| --benchmark-storage=file://${{ github.workspace }}/.benchmarks \ | |
| --benchmark-name=short \ | |
| --benchmark-json=benchmark-results.json | |
| continue-on-error: true | |
| - name: Store benchmark results | |
| uses: actions/upload-artifact@v7 | |
| if: always() | |
| with: | |
| name: benchmark-results-${{ github.run_number }} | |
| path: | | |
| benchmark-results.json | |
| .benchmarks/ | |
| retention-days: 30 | |
| - name: Check for valid benchmark baseline | |
| if: github.event_name == 'pull_request' | |
| id: check_baseline | |
| run: | | |
| set -euo pipefail | |
| if [ -f ".benchmarks/baseline.json" ] && [ -s ".benchmarks/baseline.json" ]; then | |
| # Validate JSON using python to ensure we don't attempt to parse invalid/empty JSON later | |
| python3 - <<'PY' | |
| import json, sys | |
| try: | |
| with open('.benchmarks/baseline.json', 'r') as f: | |
| json.load(f) | |
| except Exception as e: | |
| print('Baseline exists but contains invalid JSON:', e, file=sys.stderr) | |
| sys.exit(2) | |
| print('baseline_valid') | |
| PY | |
| echo "has_baseline=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "has_baseline=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Compare with baseline | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| # Download baseline from main branch | |
| git fetch origin main:main | |
| # Compare with baseline (if valid baseline exists) | |
| if [ "${{ steps.check_baseline.outputs.has_baseline }}" = "true" ]; then | |
| uv run pytest -v -m performance \ | |
| --benchmark-only \ | |
| --benchmark-compare=baseline \ | |
| --benchmark-compare-fail=mean:20% || echo "Performance regression detected" | |
| else | |
| echo "No valid baseline found, skipping comparison" | |
| fi | |
| - name: Comment PR with results | |
| if: github.event_name == 'pull_request' && always() | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| let comment = '## Performance Test Results\n\n'; | |
| // pytest-benchmark leaves the file absent or empty when the run | |
| // collected no benchmarks, so guard the parse instead of assuming | |
| // that an existing file holds valid JSON. | |
| let results = null; | |
| if (fs.existsSync('benchmark-results.json')) { | |
| const raw = fs.readFileSync('benchmark-results.json', 'utf8').trim(); | |
| if (raw) { | |
| try { | |
| results = JSON.parse(raw); | |
| } catch (error) { | |
| comment += `Benchmark results file is not valid JSON: ${error.message}\n`; | |
| } | |
| } else { | |
| comment += 'Benchmark results file is empty (no benchmarks collected)\n'; | |
| } | |
| } else { | |
| comment += 'No benchmark results file found\n'; | |
| } | |
| if (results) { | |
| comment += '### Summary\n\n'; | |
| comment += '| Metric | Value |\n'; | |
| comment += '|--------|-------|\n'; | |
| comment += `| Total Tests | ${results.benchmarks?.length || 0} |\n`; | |
| if (results.benchmarks && results.benchmarks.length > 0) { | |
| comment += '\n### Top 5 Slowest Tests\n\n'; | |
| comment += '| Test | Mean | p95 |\n'; | |
| comment += '|------|------|-----|\n'; | |
| const sorted = results.benchmarks | |
| .sort((a, b) => (b.stats?.mean || 0) - (a.stats?.mean || 0)) | |
| .slice(0, 5); | |
| sorted.forEach(bench => { | |
| const mean = (bench.stats?.mean * 1000 || 0).toFixed(2); | |
| const p95 = (bench.stats?.['95th_percentile'] * 1000 || 0).toFixed(2); | |
| comment += `| ${bench.name} | ${mean}ms | ${p95}ms |\n`; | |
| }); | |
| } | |
| } | |
| comment += '\n---\n'; | |
| comment += 'View detailed results in the workflow artifacts\n'; | |
| github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: comment | |
| }); | |
| performance-report: | |
| name: Generate Performance Report | |
| runs-on: ubuntu-latest | |
| needs: performance-tests | |
| if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop' | |
| permissions: | |
| contents: write # publishes the HTML report to gh-pages | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.12' | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 | |
| - name: Install dependencies | |
| run: uv sync --frozen --all-extras | |
| - name: Download benchmark results | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: benchmark-results-${{ github.run_number }} | |
| path: ./benchmark-results/ | |
| - name: Generate HTML report | |
| run: | | |
| # Create simple report generation script | |
| cat > generate_report.py << 'EOF' | |
| import json | |
| from pathlib import Path | |
| from tests.performance.report_generator import HTMLBenchmarkReporter | |
| from tests.performance.utils import PerformanceMetrics | |
| # Load benchmark results | |
| results_file = Path("benchmark-results/benchmark-results.json") | |
| if not results_file.exists() or results_file.stat().st_size == 0: | |
| print("No benchmark results found; skipping HTML report") | |
| exit(0) | |
| try: | |
| with open(results_file) as f: | |
| data = json.load(f) | |
| except json.JSONDecodeError: | |
| print("Benchmark results are empty or invalid; skipping HTML report") | |
| exit(0) | |
| if not data.get("benchmarks"): | |
| print("No benchmark entries found; skipping HTML report") | |
| exit(0) | |
| # Convert to PerformanceMetrics | |
| reporter = HTMLBenchmarkReporter() | |
| for bench in data.get("benchmarks", []): | |
| stats = bench.get("stats", {}) | |
| metrics = PerformanceMetrics( | |
| name=bench.get("name", "unknown"), | |
| iterations=bench.get("params", {}).get("iterations", 1), | |
| ) | |
| # Add latency data | |
| if "mean" in stats: | |
| metrics.latencies_ms = [stats["mean"] * 1000] | |
| metrics.throughput = 1000 / (stats["mean"] * 1000) if stats["mean"] > 0 else 0 | |
| reporter.add_metrics(metrics) | |
| # Save report | |
| reporter.save("performance-report.html") | |
| print("Report generated: performance-report.html") | |
| EOF | |
| uv run python generate_report.py | |
| - name: Upload HTML report | |
| uses: actions/upload-artifact@v7 | |
| if: always() | |
| with: | |
| name: performance-report-${{ github.run_number }} | |
| path: performance-report.html | |
| retention-days: 90 | |
| - name: Deploy to GitHub Pages (main only) | |
| if: github.ref == 'refs/heads/main' | |
| uses: peaceiris/actions-gh-pages@v4 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| publish_dir: . | |
| destination_dir: performance-reports/${{ github.run_number }} | |
| keep_files: true | |
| publish_branch: gh-pages | |
| performance-regression-check: | |
| name: Check for Performance Regressions | |
| runs-on: ubuntu-latest | |
| needs: performance-tests | |
| if: github.event_name == 'pull_request' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| - name: Download benchmark results | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: benchmark-results-${{ github.run_number }} | |
| path: ./benchmark-results/ | |
| - name: Check for regressions | |
| id: regression-check | |
| run: | | |
| # Simple regression detection script | |
| cat > check_regression.py << 'EOF' | |
| import json | |
| import sys | |
| from pathlib import Path | |
| results_file = Path("benchmark-results/benchmark-results.json") | |
| if not results_file.exists(): | |
| print("No results file") | |
| sys.exit(0) | |
| # pytest-benchmark leaves the file empty when no benchmarks ran. | |
| raw = results_file.read_text().strip() | |
| if not raw: | |
| print("Results file is empty; no benchmarks to compare") | |
| sys.exit(0) | |
| try: | |
| data = json.loads(raw) | |
| except json.JSONDecodeError as exc: | |
| print(f"Results file is not valid JSON: {exc}") | |
| sys.exit(0) | |
| # Define thresholds | |
| LATENCY_THRESHOLD = 1.20 # 20% increase | |
| MEMORY_THRESHOLD = 1.50 # 50% increase | |
| regressions = [] | |
| warnings = [] | |
| for bench in data.get("benchmarks", []): | |
| name = bench.get("name", "unknown") | |
| stats = bench.get("stats", {}) | |
| # Check if we have baseline comparison | |
| if "baseline_mean" in stats and "mean" in stats: | |
| ratio = stats["mean"] / stats["baseline_mean"] | |
| if ratio > LATENCY_THRESHOLD: | |
| regressions.append(f"{name}: {ratio:.2%} slower") | |
| elif ratio > 1.10: | |
| warnings.append(f"{name}: {ratio:.2%} slower") | |
| if regressions: | |
| print("PERFORMANCE REGRESSIONS DETECTED:") | |
| for reg in regressions: | |
| print(f" {reg}") | |
| print(f"\nTotal: {len(regressions)} regressions") | |
| sys.exit(1) | |
| elif warnings: | |
| print("Performance warnings:") | |
| for warn in warnings: | |
| print(f" {warn}") | |
| print("\nNo critical regressions") | |
| else: | |
| print("No performance regressions detected") | |
| EOF | |
| # The script only uses the standard library, so it runs on the | |
| # runner's Python: this job installs neither uv nor the project. | |
| # Exit 1 means "regression found"; anything else is a real failure | |
| # and must not be reported as a performance regression. | |
| set +e | |
| python3 check_regression.py | |
| status=$? | |
| set -e | |
| if [ "$status" -eq 1 ]; then | |
| echo "regression_detected=true" >> "$GITHUB_OUTPUT" | |
| elif [ "$status" -ne 0 ]; then | |
| echo "::error::check_regression.py exited with status $status" | |
| exit "$status" | |
| fi | |
| - name: Fail on regression | |
| if: steps.regression-check.outputs.regression_detected == 'true' | |
| run: | | |
| echo "::error::Performance regression detected! Review the benchmark results." | |
| exit 1 |