Mock Experiment #131
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Mock Experiment | |
| permissions: | |
| contents: read | |
| on: | |
| workflow_run: | |
| workflows: ["Provider Analysis"] | |
| types: | |
| - completed | |
| push: | |
| tags: | |
| - mock-experiment* | |
| env: | |
| STACKQL_CORE_REPOSITORY: ${{ vars.STACKQL_CORE_REPOSITORY != '' && vars.STACKQL_CORE_REPOSITORY || 'stackql/stackql' }} | |
| STACKQL_CORE_REF: ${{ vars.STACKQL_CORE_REF != '' && vars.STACKQL_CORE_REF || 'main' }} | |
| GOLANG_VERSION: 1.25.3 | |
| PYTHON_VERSION: '3.12' | |
| ANALYSIS_RESULTS_BUCKET: 'stackql-provider-analysis-results' | |
| MOCK_PORT_BASE: 5050 | |
| MAX_TEST_COUNT: 20000 | |
| PARALLEL_JOBS: 20 | |
| DEFAULT_JOB_TIMEOUT_MIN: ${{ vars.DEFAULT_JOB_TIMEOUT_MIN == '' && 80 || vars.DEFAULT_JOB_TIMEOUT_MIN }} | |
| DEFAULT_MOCK_TIMEOUT_MIN: ${{ vars.DEFAULT_MOCK_TIMEOUT_MIN == '' && 60 || vars.DEFAULT_MOCK_TIMEOUT_MIN }} | |
| jobs: | |
| build-stackql: | |
| if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }} | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Set up golang | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: ^${{ env.GOLANG_VERSION }} | |
| id: go | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Download core | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ env.STACKQL_CORE_REPOSITORY }} | |
| ref: ${{ env.STACKQL_CORE_REF }} | |
| token: ${{ secrets.CI_STACKQL_PACKAGE_DOWNLOAD_TOKEN }} | |
| path: stackql-core | |
| - name: Build stackql from core source | |
| working-directory: stackql-core | |
| run: | | |
| go get ./... | |
| python3 cicd/python/build.py --build | |
| - name: Upload stackql binary | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: stackql_binary | |
| path: stackql-core/build/stackql | |
| mock-test: | |
| name: Mock E2E Tests | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: ${{ vars.DEFAULT_JOB_TIMEOUT_MIN == '' && 80 || vars.DEFAULT_JOB_TIMEOUT_MIN }} | |
| needs: | |
| - build-stackql | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install Flask | |
| run: pip install flask | |
| - name: Download stackql binary | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: stackql_binary | |
| path: build | |
| - name: Make stackql executable | |
| run: | | |
| chmod +x build/stackql | |
| echo "${{ github.workspace }}/build" >> $GITHUB_PATH | |
| - name: Download Provider Analysis artifacts | |
| run: | | |
| set -e | |
| RUN_ID=$(gh run list --workflow="Provider Analysis" --status=success --limit=1 --json databaseId --jq '.[0].databaseId') | |
| if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then | |
| echo "::error::No successful Provider Analysis run found. Run the Provider Analysis workflow first." | |
| exit 1 | |
| fi | |
| echo "Downloading artifacts from run: ${RUN_ID}" | |
| gh run download "$RUN_ID" --name auto-mocks --dir cicd/out/auto-mocks | |
| gh run download "$RUN_ID" --name mock-queries --dir cicd/out/mock-queries | |
| gh run download "$RUN_ID" --name mock-expectations --dir cicd/out/mock-expectations | |
| gh run download "$RUN_ID" --name closures --dir cicd/out/closures | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| - name: Verify downloaded artifacts | |
| run: | | |
| echo "Mocks: $(find cicd/out/auto-mocks -name 'mock_*.py' 2>/dev/null | wc -l)" | |
| echo "Queries: $(find cicd/out/mock-queries -name 'query_*.txt' 2>/dev/null | wc -l)" | |
| echo "Closures: $(find cicd/out/closures -name 'provider.yaml' 2>/dev/null | wc -l)" | |
| - name: Prepare test runner script | |
| run: | | |
| cat > /tmp/run_one_test.sh << 'SCRIPT' | |
| #!/usr/bin/env bash | |
| # Args: line_number provider_dir mock_filename provider service resource method | |
| set -f | |
| LINE_NUM="$1" | |
| PROVIDER_DIR="$2" | |
| MOCK_FILENAME="$3" | |
| PROVIDER="$4" | |
| SERVICE="$5" | |
| RESOURCE="$6" | |
| METHOD="$7" | |
| EVENTS_FILE="$EVENTS_DIR/events.jsonl" | |
| PORT=$(( MOCK_PORT_BASE + (LINE_NUM % 500) )) | |
| METHOD_KEY="${PROVIDER}_${SERVICE}_${RESOURCE}_${METHOD}" | |
| MOCK_FILE="cicd/out/auto-mocks/${PROVIDER_DIR}/${MOCK_FILENAME}" | |
| QUERY_FILE="cicd/out/mock-queries/${PROVIDER_DIR}/query_${METHOD_KEY}.txt" | |
| EXPECT_FILE="cicd/out/mock-expectations/${PROVIDER_DIR}/expect_${METHOD_KEY}.txt" | |
| CLOSURE_DIR="cicd/out/closures/${PROVIDER_DIR}/${PROVIDER}_${SERVICE}_${RESOURCE}" | |
| emit() { | |
| local status="$1" error_class="$2" detail="$3" http_code="$4" | |
| printf '{"method_key":"%s","provider":"%s","service":"%s","resource":"%s","method":"%s","sql_verb":"%s","status":"%s","error_class":"%s","http_code":%s,"detail":"%s"}\n' \ | |
| "$METHOD_KEY" "$PROVIDER" "$SERVICE" "$RESOURCE" "$METHOD" "$SQL_VERB" \ | |
| "$status" "$error_class" "${http_code:-null}" "$detail" \ | |
| | flock "$EVENTS_FILE.lock" tee -a "$EVENTS_FILE" > /dev/null | |
| } | |
| # Determine SQL verb from query | |
| SQL_VERB="unknown" | |
| if [ -f "$QUERY_FILE" ]; then | |
| first_word="$(head -c 20 "$QUERY_FILE" | awk '{print tolower($1)}')" | |
| case "$first_word" in | |
| select) SQL_VERB="select" ;; | |
| insert) SQL_VERB="insert" ;; | |
| delete) SQL_VERB="delete" ;; | |
| update) SQL_VERB="update" ;; | |
| exec) SQL_VERB="exec" ;; | |
| esac | |
| fi | |
| [ -f "$MOCK_FILE" ] || { emit "skip" "no_mock" "" ""; exit 0; } | |
| [ -f "$QUERY_FILE" ] || { emit "skip" "no_query" "" ""; exit 0; } | |
| [ -d "$CLOSURE_DIR" ] || { emit "skip" "no_closure" "" ""; exit 0; } | |
| QUERY="$(cat "$QUERY_FILE")" | |
| REGISTRY_DIR="$(pwd)/${CLOSURE_DIR}" | |
| # Start mock, retry port if busy | |
| for attempt in 1 2 3; do | |
| python3 "$MOCK_FILE" --port "$PORT" & | |
| MOCK_PID=$! | |
| sleep 0.3 | |
| if kill -0 $MOCK_PID 2>/dev/null; then | |
| break | |
| fi | |
| PORT=$(( PORT + 500 )) | |
| done | |
| if ! kill -0 $MOCK_PID 2>/dev/null; then | |
| emit "fail" "mock_start_failed" "could not start mock on any port" "" | |
| exit 0 | |
| fi | |
| STDERR_FILE="/tmp/sq_stderr_${LINE_NUM}.txt" | |
| RESPONSE="$(stackql \ | |
| --tls.allowInsecure \ | |
| --http.log.enabled \ | |
| --registry "{ \"url\": \"file://${REGISTRY_DIR}\", \"localDocRoot\": \"${REGISTRY_DIR}\", \"verifyConfig\": { \"nopVerify\": true } }" \ | |
| exec "${QUERY};" -o json 2>"$STDERR_FILE")" | |
| HTTP_CODE="$(grep 'http response status code:' "$STDERR_FILE" 2>/dev/null | head -1 | sed 's/.*status code: //' | sed 's/,.*//' | tr -d ' ')" | |
| kill $MOCK_PID 2>/dev/null | |
| wait $MOCK_PID 2>/dev/null | |
| if [ "$HTTP_CODE" = "200" ]; then | |
| # Check body match if expectation exists | |
| if [ -f "$EXPECT_FILE" ] && [ -s "$EXPECT_FILE" ]; then | |
| EXPECTED="$(cat "$EXPECT_FILE")" | |
| if [ "$RESPONSE" = "$EXPECTED" ]; then | |
| emit "pass" "status_ok_body_match" "" "$HTTP_CODE" | |
| else | |
| emit "fail" "body_mismatch" "" "$HTTP_CODE" | |
| fi | |
| else | |
| emit "pass" "status_ok" "" "$HTTP_CODE" | |
| fi | |
| elif [ -n "$HTTP_CODE" ]; then | |
| emit "fail" "status_${HTTP_CODE}" "" "$HTTP_CODE" | |
| else | |
| # No HTTP request made — classify from stderr | |
| STDERR_FIRST="$(head -1 "$STDERR_FILE" 2>/dev/null | tr '"' "'" | head -c 200)" | |
| if echo "$STDERR_FIRST" | grep -qi "syntax error"; then | |
| emit "fail" "query_syntax_error" "$STDERR_FIRST" "" | |
| elif echo "$STDERR_FIRST" | grep -qi "cannot resolve"; then | |
| emit "fail" "registry_resolve_error" "$STDERR_FIRST" "" | |
| elif echo "$STDERR_FIRST" | grep -qi "credentials"; then | |
| emit "fail" "credentials_error" "$STDERR_FIRST" "" | |
| else | |
| emit "fail" "no_http_request" "$STDERR_FIRST" "" | |
| fi | |
| fi | |
| rm -f "$STDERR_FILE" | |
| SCRIPT | |
| chmod +x /tmp/run_one_test.sh | |
| - name: Run Mock E2E Tests | |
| env: | |
| AWS_SECRET_ACCESS_KEY: fake | |
| AWS_ACCESS_KEY_ID: fake | |
| GOOGLE_CREDENTIALS: '{}' | |
| AZURE_CLIENT_ID: fake | |
| AZURE_CLIENT_SECRET: fake | |
| AZURE_TENANT_ID: fake | |
| MOCK_PORT_BASE: ${{ env.MOCK_PORT_BASE }} | |
| EVENTS_DIR: cicd/out | |
| run: | | |
| set +e | |
| mkdir -p cicd/out | |
| touch cicd/out/events.jsonl | |
| touch cicd/out/events.jsonl.lock | |
| MAX_TESTS="${MAX_TEST_COUNT:-20000}" | |
| # Unlimited for workflow_run trigger or mock-experiment-total* tags | |
| if [ "${{ github.event_name }}" = "workflow_run" ]; then | |
| MAX_TESTS=-1 | |
| elif echo "${{ github.ref }}" | grep -q "refs/tags/mock-experiment-total"; then | |
| MAX_TESTS=-1 | |
| fi | |
| TIMEOUT_MIN="${DEFAULT_MOCK_TIMEOUT_MIN:-60}" | |
| DEADLINE=$(($(date +%s) + TIMEOUT_MIN * 60)) | |
| # Build test list: prefer manifest, fall back to query files | |
| TEST_LIST="/tmp/test_list.txt" | |
| : > "$TEST_LIST" | |
| for manifest in cicd/out/auto-mocks/*/manifest.txt; do | |
| [ -f "$manifest" ] || continue | |
| provider_dir="$(basename "$(dirname "$manifest")")" | |
| while IFS='|' read -r mock_filename provider service resource method; do | |
| echo "${provider_dir}|${mock_filename}|${provider}|${service}|${resource}|${method}" | |
| done < "$manifest" | |
| done >> "$TEST_LIST" | |
| if [ ! -s "$TEST_LIST" ]; then | |
| echo "No manifests found, deriving from query files" | |
| for qf in cicd/out/mock-queries/*/query_*.txt; do | |
| [ -f "$qf" ] || continue | |
| provider_dir="$(basename "$(dirname "$qf")")" | |
| key="$(basename "$qf" .txt)"; key="${key#query_}" | |
| [ -f "cicd/out/auto-mocks/${provider_dir}/mock_${key}.py" ] || continue | |
| echo "${provider_dir}|mock_${key}.py|${key}||||" | |
| done >> "$TEST_LIST" | |
| fi | |
| TOTAL_AVAILABLE="$(wc -l < "$TEST_LIST")" | |
| if [ "$MAX_TESTS" -ge 0 ] 2>/dev/null && [ "$TOTAL_AVAILABLE" -gt "$MAX_TESTS" ]; then | |
| echo "Limiting to ${MAX_TESTS} of ${TOTAL_AVAILABLE} tests" | |
| head -n "$MAX_TESTS" "$TEST_LIST" > /tmp/test_list_limited.txt | |
| mv /tmp/test_list_limited.txt "$TEST_LIST" | |
| fi | |
| echo "Running $(wc -l < "$TEST_LIST") tests with ${PARALLEL_JOBS} parallel workers" | |
| export EVENTS_DIR MOCK_PORT_BASE DEADLINE | |
| echo "Deadline: $(date -u -d "@${DEADLINE}" +%H:%M:%SZ 2>/dev/null || date -u -r "${DEADLINE}" +%H:%M:%SZ) (${TIMEOUT_MIN}m from now)" | |
| timeout --signal=TERM "${TIMEOUT_MIN}m" bash -c ' | |
| nl -ba "$0" | tr "\t" "|" | \ | |
| xargs -P "${PARALLEL_JOBS}" -I {} bash -c '\'' | |
| IFS="|" read -r line_num provider_dir mock_filename provider service resource method <<< "{}" | |
| /tmp/run_one_test.sh "$line_num" "$provider_dir" "$mock_filename" "$provider" "$service" "$resource" "$method" | |
| '\'' | |
| ' "$TEST_LIST" || true | |
| echo "Test run complete (or timed out). Events: $(wc -l < cicd/out/events.jsonl)" | |
| - name: Download prior results for trend comparison | |
| if: always() | |
| continue-on-error: true | |
| run: | | |
| gcloud storage cp "gs://${ANALYSIS_RESULTS_BUCKET}/mock-experiments/latest.json" /tmp/prior_latest.json 2>/dev/null || true | |
| if [ -f /tmp/prior_latest.json ]; then | |
| PRIOR_DIR="$(jq -r '.latest' /tmp/prior_latest.json)" | |
| gcloud storage cp "${PRIOR_DIR}/mock-test-results.json" cicd/out/prior-results.json 2>/dev/null || true | |
| fi | |
| - name: Generate Summary | |
| if: always() | |
| run: | | |
| python3 << 'PYEOF' | |
| import json, os | |
| from collections import Counter | |
| events = [] | |
| with open("cicd/out/events.jsonl") as f: | |
| for line in f: | |
| line = line.strip() | |
| if line: | |
| try: | |
| events.append(json.loads(line)) | |
| except json.JSONDecodeError: | |
| pass | |
| total = len(events) | |
| status_counts = Counter(e["status"] for e in events) | |
| class_counts = Counter(e["error_class"] for e in events) | |
| # Score metrics | |
| pass_count = status_counts.get("pass", 0) | |
| fail_count = status_counts.get("fail", 0) | |
| skip_count = status_counts.get("skip", 0) | |
| tested = pass_count + fail_count | |
| scores = { | |
| "pass_rate": round(pass_count / tested, 4) if tested > 0 else 0, | |
| "fail_rate": round(fail_count / tested, 4) if tested > 0 else 0, | |
| "skip_rate": round(skip_count / total, 4) if total > 0 else 0, | |
| "tested": tested, | |
| } | |
| # Coverage: methods with mocks (have sample_response) — from events that aren't skips | |
| methods_tested = set() | |
| sql_verbs_seen = Counter() | |
| for e in events: | |
| if e["status"] != "skip": | |
| methods_tested.add(e["method_key"]) | |
| sql_verbs_seen[e.get("sql_verb", "unknown")] += 1 | |
| coverage = { | |
| "unique_methods_tested": len(methods_tested), | |
| "by_sql_verb": dict(sql_verbs_seen.most_common()), | |
| } | |
| # Intersection: sql_verb × error_class | |
| verb_class = Counter((e.get("sql_verb", "unknown"), e["error_class"]) for e in events) | |
| verb_class_summary = {f"{v}:{c}": n for (v, c), n in verb_class.most_common()} | |
| # Provider × status | |
| provider_status = Counter((e.get("provider", "unknown"), e["status"]) for e in events) | |
| provider_summary = {f"{p}:{s}": n for (p, s), n in provider_status.most_common()} | |
| summary = { | |
| "total": total, | |
| "scores": scores, | |
| "coverage": coverage, | |
| "by_status": dict(status_counts.most_common()), | |
| "by_error_class": dict(class_counts.most_common()), | |
| "by_verb_and_class": verb_class_summary, | |
| "by_provider_and_status": provider_summary, | |
| } | |
| # Trend comparison against prior run | |
| prior_path = "cicd/out/prior-results.json" | |
| if os.path.exists(prior_path): | |
| try: | |
| with open(prior_path) as f: | |
| prior = json.load(f) | |
| prior_scores = prior.get("scores", {}) | |
| prior_pass = prior_scores.get("pass_rate", 0) | |
| prior_tested = prior_scores.get("tested", 0) | |
| delta_pass_rate = round(scores["pass_rate"] - prior_pass, 4) | |
| delta_tested = scores["tested"] - prior_tested | |
| # Regressions: error classes that increased | |
| prior_classes = prior.get("by_error_class", {}) | |
| regressions = {} | |
| improvements = {} | |
| for cls, count in class_counts.items(): | |
| prior_count = prior_classes.get(cls, 0) | |
| if count > prior_count: | |
| regressions[cls] = {"current": count, "prior": prior_count, "delta": count - prior_count} | |
| elif count < prior_count: | |
| improvements[cls] = {"current": count, "prior": prior_count, "delta": prior_count - count} | |
| summary["trend"] = { | |
| "delta_pass_rate": delta_pass_rate, | |
| "delta_tested": delta_tested, | |
| "direction": "improving" if delta_pass_rate > 0 else "regressing" if delta_pass_rate < 0 else "stable", | |
| "regressions": regressions, | |
| "improvements": improvements, | |
| } | |
| except (json.JSONDecodeError, KeyError): | |
| summary["trend"] = {"error": "could not parse prior results"} | |
| else: | |
| summary["trend"] = {"note": "no prior run available for comparison"} | |
| with open("cicd/out/mock-test-results.json", "w") as f: | |
| json.dump(summary, f, indent=2) | |
| print(json.dumps(summary, indent=2)) | |
| PYEOF | |
| - name: Upload Mock Test Results | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: mock-test-results | |
| path: | | |
| cicd/out/mock-test-results.json | |
| cicd/out/events.jsonl | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| if: always() | |
| with: | |
| credentials_json: ${{ secrets.ANALYSIS_UPLOAD_GCS }} | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v2 | |
| if: always() | |
| - name: Upload Mock Test Results to GCS | |
| if: always() | |
| run: | | |
| RUN_EPOCH="$(date -u +%s)" | |
| RUN_TS="$(date -u -d "@${RUN_EPOCH}" +%Y-%m-%dT%H-%M-%SZ)" | |
| DATE_PATH="$(date -u -d "@${RUN_EPOCH}" +%Y/%m/%d)" | |
| DEST="gs://${ANALYSIS_RESULTS_BUCKET}/mock-experiments/${DATE_PATH}/${RUN_TS}" | |
| gcloud storage cp cicd/out/mock-test-results.json "${DEST}/mock-test-results.json" | |
| gcloud storage cp cicd/out/events.jsonl "${DEST}/events.jsonl" | |
| echo "{\"latest\": \"${DEST}\"}" > latest.json | |
| gcloud storage cp latest.json "gs://${ANALYSIS_RESULTS_BUCKET}/mock-experiments/latest.json" |