Skip to content

feat(ci): add text quality drift watch workflow #71

feat(ci): add text quality drift watch workflow

feat(ci): add text quality drift watch workflow #71

Workflow file for this run

name: Deploy Spark Runtime
on:
workflow_dispatch:
inputs:
use_pinned_images:
description: "Use GHCR pinned images for backend/worker"
required: true
type: choice
options:
- "true"
- "false"
default: "true"
image_tag:
description: "Pinned image tag override (defaults to commit SHA)"
required: false
type: string
verify_signatures:
description: "Verify cosign signatures for pinned images before deploy"
required: true
type: choice
options:
- "true"
- "false"
default: "true"
deploy_frontend:
description: "Deploy frontend service as part of this run"
required: true
type: choice
options:
- "false"
- "true"
default: "false"
remote_dir:
description: "Optional remote path override"
required: false
type: string
public_api_url:
description: "Optional API URL override used by frontend build"
required: false
type: string
smoke_base_url:
description: "Optional smoke test API URL override"
required: false
type: string
run_smoke:
description: "Run smoke tests after deploy"
required: true
type: choice
options:
- "true"
- "false"
default: "true"
rollback_on_smoke_failure:
description: "Attempt rollback to previous commit when smoke fails"
required: true
type: choice
options:
- "true"
- "false"
default: "true"
runner_type:
description: "Runner type for this deploy"
required: true
type: choice
options:
- "github-hosted"
- "self-hosted"
default: "github-hosted"
cost_override:
description: "Run deploy even when cost policy is in block state"
required: true
type: choice
options:
- "false"
- "true"
default: "false"
push:
branches: [main]
paths:
- "backend/**"
- "docker-compose.yml"
- "docker-compose.spark.yml"
- "docker-compose.spark.images.yml"
- "scripts/deploy_spark.sh"
- ".github/workflows/deploy-spark.yml"
permissions:
contents: read
packages: read
concurrency:
group: spark-runtime-production
cancel-in-progress: true
jobs:
cost-precheck:
runs-on: ubuntu-latest
outputs:
allow_run: ${{ steps.gate.outputs.allow_run }}
status: ${{ steps.gate.outputs.status }}
override_used: ${{ steps.gate.outputs.override_used }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
VERCEL_PROJECT_ID: ${{ vars.VERCEL_PROJECT_ID }}
VERCEL_TEAM_ID: ${{ vars.VERCEL_TEAM_ID }}
steps:
- uses: actions/checkout@v6
- name: Evaluate cost policy gate
id: gate
run: |
set -euo pipefail
mkdir -p ops/reports
python3 scripts/cost_governance_snapshot.py \
--repo "${GITHUB_REPOSITORY}" \
--window-days 30 \
--gh-token "${GH_TOKEN}" \
--vercel-token "${VERCEL_TOKEN}" \
--vercel-project-id "${VERCEL_PROJECT_ID}" \
--vercel-team-id "${VERCEL_TEAM_ID}" \
--policy-file config/cost_policy.yaml \
--workflow-name "${GITHUB_WORKFLOW}" \
--output-json ops/reports/deploy_cost_gate.json \
--output-md ops/reports/deploy_cost_gate.md \
--fail-on-alert-level none
status="$(python3 - <<'PY'
import json
payload = json.loads(open('ops/reports/deploy_cost_gate.json', encoding='utf-8').read())
print(payload.get('status', 'ok'))
PY
)"
override="false"
if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ github.event.inputs.cost_override || 'false' }}" = "true" ]; then
override="true"
fi
allow_run="true"
if [ "${status}" = "block" ] && [ "${override}" != "true" ]; then
allow_run="false"
echo "::warning::Cost policy is block. Deploy job is skipped. Re-dispatch with cost_override=true if operationally required."
fi
{
echo "status=${status}"
echo "override_used=${override}"
echo "allow_run=${allow_run}"
} >> "$GITHUB_OUTPUT"
- name: Upload cost gate artifact
uses: actions/upload-artifact@v7
with:
name: deploy-spark-cost-gate
path: ops/reports/deploy_cost_gate.*
retention-days: 14
deploy:
needs: cost-precheck
if: ${{ needs.cost-precheck.outputs.allow_run == 'true' }}
runs-on: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.runner_type == 'self-hosted' && 'self-hosted' || 'ubuntu-latest' }}
timeout-minutes: 35
env:
SPARK_SSH_HOST: ${{ secrets.SPARK_SSH_HOST }}
SPARK_SSH_USER: ${{ secrets.SPARK_SSH_USER }}
SPARK_SSH_KEY: ${{ secrets.SPARK_SSH_KEY }}
SPARK_SSH_PORT: ${{ secrets.SPARK_SSH_PORT }}
SPARK_REMOTE_DIR_VAR: ${{ vars.SPARK_REMOTE_DIR }}
SPARK_PUBLIC_API_URL_VAR: ${{ vars.SPARK_PUBLIC_API_URL }}
SPARK_DEPLOY_FRONTEND_VAR: ${{ vars.SPARK_DEPLOY_FRONTEND }}
SPARK_USE_PINNED_IMAGES_VAR: ${{ vars.SPARK_USE_PINNED_IMAGES }}
SPARK_RUNNER_TYPE_VAR: ${{ vars.SPARK_RUNNER_TYPE }}
SPARK_RUN_SMOKE_VAR: ${{ vars.SPARK_RUN_SMOKE }}
SPARK_ROLLBACK_ON_SMOKE_FAILURE_VAR: ${{ vars.SPARK_ROLLBACK_ON_SMOKE_FAILURE }}
SPARK_VERIFY_SIGNATURES_VAR: ${{ vars.SPARK_VERIFY_SIGNATURES }}
X_BEARER_TOKEN: ${{ secrets.X_BEARER_TOKEN }}
SPARK_SMOKE_API_KEY: ${{ secrets.SPARK_SMOKE_API_KEY }}
SPARK_SMOKE_API_KEY_HEADER: ${{ vars.SPARK_SMOKE_API_KEY_HEADER || 'X-API-Key' }}
OPS_ALERT_WEBHOOK_URL: ${{ secrets.OPS_ALERT_WEBHOOK_URL }}
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 2
- name: Validate required Spark secrets
run: |
set -euo pipefail
missing=0
for key in SPARK_SSH_HOST SPARK_SSH_USER SPARK_SSH_KEY; do
if [ -z "${!key:-}" ]; then
echo "Missing required repository secret: $key"
missing=1
fi
done
if [ "$missing" -ne 0 ]; then
exit 1
fi
- name: Resolve deploy settings
id: resolve
run: |
set -euo pipefail
remote_dir="${SPARK_REMOTE_DIR_VAR:-/home/weezboo/ogulcan/ai-provenance-tracker}"
public_api_url="${SPARK_PUBLIC_API_URL_VAR:-}"
deploy_frontend="${SPARK_DEPLOY_FRONTEND_VAR:-false}"
use_pinned_images="${SPARK_USE_PINNED_IMAGES_VAR:-}"
runner_type="${SPARK_RUNNER_TYPE_VAR:-github-hosted}"
smoke_base_url="${SPARK_PUBLIC_API_URL_VAR:-}"
run_smoke="${SPARK_RUN_SMOKE_VAR:-true}"
rollback_on_smoke_failure="${SPARK_ROLLBACK_ON_SMOKE_FAILURE_VAR:-true}"
verify_signatures="${SPARK_VERIFY_SIGNATURES_VAR:-true}"
image_tag="${GITHUB_SHA}"
ssh_port="${SPARK_SSH_PORT:-22}"
if [ -z "${use_pinned_images}" ]; then
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
use_pinned_images="true"
else
use_pinned_images="false"
fi
fi
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
if [ -n "${{ github.event.inputs.use_pinned_images }}" ]; then
use_pinned_images="${{ github.event.inputs.use_pinned_images }}"
fi
if [ -n "${{ github.event.inputs.image_tag }}" ]; then
image_tag="${{ github.event.inputs.image_tag }}"
fi
if [ -n "${{ github.event.inputs.verify_signatures }}" ]; then
verify_signatures="${{ github.event.inputs.verify_signatures }}"
fi
if [ -n "${{ github.event.inputs.remote_dir }}" ]; then
remote_dir="${{ github.event.inputs.remote_dir }}"
fi
if [ -n "${{ github.event.inputs.public_api_url }}" ]; then
public_api_url="${{ github.event.inputs.public_api_url }}"
fi
if [ -n "${{ github.event.inputs.smoke_base_url }}" ]; then
smoke_base_url="${{ github.event.inputs.smoke_base_url }}"
fi
if [ -n "${{ github.event.inputs.deploy_frontend }}" ]; then
deploy_frontend="${{ github.event.inputs.deploy_frontend }}"
fi
if [ -n "${{ github.event.inputs.run_smoke }}" ]; then
run_smoke="${{ github.event.inputs.run_smoke }}"
fi
if [ -n "${{ github.event.inputs.rollback_on_smoke_failure }}" ]; then
rollback_on_smoke_failure="${{ github.event.inputs.rollback_on_smoke_failure }}"
fi
if [ -n "${{ github.event.inputs.runner_type }}" ]; then
runner_type="${{ github.event.inputs.runner_type }}"
fi
fi
if [ "${use_pinned_images}" != "true" ] && [ "${use_pinned_images}" != "false" ]; then
echo "use_pinned_images must be true or false."
exit 1
fi
if [ "${verify_signatures}" != "true" ] && [ "${verify_signatures}" != "false" ]; then
echo "verify_signatures must be true or false."
exit 1
fi
if [ -z "${smoke_base_url}" ]; then
smoke_base_url="${public_api_url}"
fi
{
echo "remote_dir=${remote_dir}"
echo "public_api_url=${public_api_url}"
echo "deploy_frontend=${deploy_frontend}"
echo "use_pinned_images=${use_pinned_images}"
echo "image_tag=${image_tag}"
echo "runner_type=${runner_type}"
echo "smoke_base_url=${smoke_base_url}"
echo "run_smoke=${run_smoke}"
echo "rollback_on_smoke_failure=${rollback_on_smoke_failure}"
echo "verify_signatures=${verify_signatures}"
echo "ssh_port=${ssh_port}"
} >> "$GITHUB_OUTPUT"
- name: Resolve pinned image refs
id: image_refs
run: |
set -euo pipefail
use_pinned_images="${{ steps.resolve.outputs.use_pinned_images }}"
image_tag="${{ steps.resolve.outputs.image_tag }}"
api_ref=""
worker_ref=""
if [ "${use_pinned_images}" = "true" ]; then
api_ref="ghcr.io/${{ github.repository_owner }}/provenance-api:${image_tag}"
worker_ref="ghcr.io/${{ github.repository_owner }}/provenance-worker:${image_tag}"
fi
{
echo "use_pinned_images=${use_pinned_images}"
echo "image_tag=${image_tag}"
echo "api_ref=${api_ref}"
echo "worker_ref=${worker_ref}"
} >> "$GITHUB_OUTPUT"
- name: Install cosign for signature verification
if: ${{ steps.image_refs.outputs.use_pinned_images == 'true' && steps.resolve.outputs.verify_signatures == 'true' }}
uses: sigstore/cosign-installer@v4.0.0
- name: Verify pinned image signatures (cosign keyless)
if: ${{ steps.image_refs.outputs.use_pinned_images == 'true' && steps.resolve.outputs.verify_signatures == 'true' }}
env:
API_REF: ${{ steps.image_refs.outputs.api_ref }}
WORKER_REF: ${{ steps.image_refs.outputs.worker_ref }}
COSIGN_CERT_IDENTITY: https://github.com/${{ github.repository }}/.github/workflows/publish-images.yml@refs/heads/main
COSIGN_CERT_ISSUER: https://token.actions.githubusercontent.com
run: |
set -euo pipefail
cosign verify \
--certificate-identity "${COSIGN_CERT_IDENTITY}" \
--certificate-oidc-issuer "${COSIGN_CERT_ISSUER}" \
"${API_REF}"
cosign verify \
--certificate-identity "${COSIGN_CERT_IDENTITY}" \
--certificate-oidc-issuer "${COSIGN_CERT_ISSUER}" \
"${WORKER_REF}"
- name: Verify SBOM attestations (spdxjson)
if: ${{ steps.image_refs.outputs.use_pinned_images == 'true' && steps.resolve.outputs.verify_signatures == 'true' }}
env:
API_REF: ${{ steps.image_refs.outputs.api_ref }}
WORKER_REF: ${{ steps.image_refs.outputs.worker_ref }}
COSIGN_CERT_IDENTITY: https://github.com/${{ github.repository }}/.github/workflows/publish-images.yml@refs/heads/main
COSIGN_CERT_ISSUER: https://token.actions.githubusercontent.com
run: |
set -euo pipefail
cosign verify-attestation \
--type spdxjson \
--certificate-identity "${COSIGN_CERT_IDENTITY}" \
--certificate-oidc-issuer "${COSIGN_CERT_ISSUER}" \
"${API_REF}" > /tmp/api-attestations.jsonl
cosign verify-attestation \
--type spdxjson \
--certificate-identity "${COSIGN_CERT_IDENTITY}" \
--certificate-oidc-issuer "${COSIGN_CERT_ISSUER}" \
"${WORKER_REF}" > /tmp/worker-attestations.jsonl
test -s /tmp/api-attestations.jsonl
test -s /tmp/worker-attestations.jsonl
- name: Set up SSH access
id: ssh_prep
run: |
set -euo pipefail
keyscan_ok=true
mkdir -p ~/.ssh
printf '%s\n' "${SPARK_SSH_KEY}" > ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
if ! timeout 15 ssh-keyscan -H -p "${{ steps.resolve.outputs.ssh_port }}" "${SPARK_SSH_HOST}" >> ~/.ssh/known_hosts; then
keyscan_ok=false
echo "Host key scan failed. Spark host is not reachable from this runner; remote deploy will be skipped."
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
echo "Tip: if this host is private (for example Tailscale), run workflow_dispatch with runner_type=self-hosted from a reachable runner."
fi
fi
if [ "${keyscan_ok}" = "false" ]; then
echo "keyscan_ok=false" >> "$GITHUB_OUTPUT"
exit 0
fi
cat > ~/.ssh/config <<EOF
Host spark-ci
HostName ${SPARK_SSH_HOST}
User ${SPARK_SSH_USER}
Port ${{ steps.resolve.outputs.ssh_port }}
IdentityFile ~/.ssh/id_ed25519
IdentitiesOnly yes
StrictHostKeyChecking yes
EOF
chmod 600 ~/.ssh/config
echo "keyscan_ok=true" >> "$GITHUB_OUTPUT"
- name: Deploy to Spark
if: ${{ steps.ssh_prep.outputs.keyscan_ok == 'true' }}
run: |
set -euo pipefail
deploy_host="spark-ci"
if [ "${{ steps.resolve.outputs.runner_type }}" = "self-hosted" ]; then
deploy_host="local"
fi
SPARK_HOST="${deploy_host}" \
SPARK_REMOTE_DIR="${{ steps.resolve.outputs.remote_dir }}" \
SPARK_PUBLIC_API_URL="${{ steps.resolve.outputs.public_api_url }}" \
SPARK_DEPLOY_FRONTEND="${{ steps.resolve.outputs.deploy_frontend }}" \
SPARK_USE_PINNED_IMAGES="${{ steps.image_refs.outputs.use_pinned_images }}" \
SPARK_BACKEND_IMAGE="${{ steps.image_refs.outputs.api_ref }}" \
SPARK_WORKER_IMAGE="${{ steps.image_refs.outputs.worker_ref }}" \
X_BEARER_TOKEN="${X_BEARER_TOKEN:-}" \
./scripts/deploy_spark.sh
- name: Set up Python for smoke test
if: ${{ steps.ssh_prep.outputs.keyscan_ok == 'true' && steps.resolve.outputs.run_smoke == 'true' && steps.resolve.outputs.smoke_base_url != '' }}
uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Install backend dependencies for smoke test
if: ${{ steps.ssh_prep.outputs.keyscan_ok == 'true' && steps.resolve.outputs.run_smoke == 'true' && steps.resolve.outputs.smoke_base_url != '' }}
run: |
cd backend
pip install -e .
- name: Run Spark smoke test
id: smoke
if: ${{ steps.ssh_prep.outputs.keyscan_ok == 'true' && steps.resolve.outputs.run_smoke == 'true' && steps.resolve.outputs.smoke_base_url != '' }}
continue-on-error: true
run: |
set -euo pipefail
cd backend
python scripts/smoke_detect_prod.py \
--base-url "${{ steps.resolve.outputs.smoke_base_url }}" \
--api-key "${SPARK_SMOKE_API_KEY:-}" \
--api-key-header "${SPARK_SMOKE_API_KEY_HEADER}" \
--output ./evidence/smoke/spark_detect_smoke.json
- name: Roll back to previous commit after smoke failure
id: rollback
if: ${{ steps.ssh_prep.outputs.keyscan_ok == 'true' && steps.resolve.outputs.rollback_on_smoke_failure == 'true' && steps.smoke.outcome == 'failure' }}
run: |
set -euo pipefail
rollback_sha="$(git rev-list --max-count=2 HEAD | tail -n 1)"
if [ -z "${rollback_sha}" ]; then
echo "Rollback commit not found."
exit 1
fi
echo "Rolling back to ${rollback_sha}"
git checkout --detach "${rollback_sha}"
deploy_host="spark-ci"
if [ "${{ steps.resolve.outputs.runner_type }}" = "self-hosted" ]; then
deploy_host="local"
fi
SPARK_HOST="${deploy_host}" \
SPARK_REMOTE_DIR="${{ steps.resolve.outputs.remote_dir }}" \
SPARK_PUBLIC_API_URL="${{ steps.resolve.outputs.public_api_url }}" \
SPARK_DEPLOY_FRONTEND="${{ steps.resolve.outputs.deploy_frontend }}" \
SPARK_USE_PINNED_IMAGES=false \
SPARK_BACKEND_IMAGE="" \
SPARK_WORKER_IMAGE="" \
X_BEARER_TOKEN="${X_BEARER_TOKEN:-}" \
./scripts/deploy_spark.sh
git checkout --detach "${GITHUB_SHA}"
echo "rollback_sha=${rollback_sha}" >> "$GITHUB_OUTPUT"
- name: Re-run smoke after rollback
id: smoke_after_rollback
if: ${{ steps.ssh_prep.outputs.keyscan_ok == 'true' && steps.rollback.outcome == 'success' && steps.resolve.outputs.smoke_base_url != '' }}
continue-on-error: true
run: |
set -euo pipefail
cd backend
python scripts/smoke_detect_prod.py \
--base-url "${{ steps.resolve.outputs.smoke_base_url }}" \
--api-key "${SPARK_SMOKE_API_KEY:-}" \
--api-key-header "${SPARK_SMOKE_API_KEY_HEADER}" \
--output ./evidence/smoke/spark_detect_smoke_after_rollback.json
- name: Enforce smoke gate
if: ${{ steps.ssh_prep.outputs.keyscan_ok == 'true' && steps.resolve.outputs.run_smoke == 'true' && steps.resolve.outputs.smoke_base_url != '' }}
run: |
if [ "${{ steps.smoke.outcome }}" = "success" ]; then
echo "Smoke test passed."
exit 0
fi
if [ "${{ steps.resolve.outputs.rollback_on_smoke_failure }}" = "true" ] && [ "${{ steps.rollback.outcome }}" = "success" ]; then
echo "Smoke failed and rollback was attempted (sha: ${{ steps.rollback.outputs.rollback_sha }})."
if [ "${{ steps.smoke_after_rollback.outcome }}" = "success" ]; then
echo "Post-rollback smoke is healthy."
fi
fi
echo "Smoke gate failed."
exit 1
- name: Upload Spark smoke artifacts
if: ${{ always() && steps.resolve.outputs.run_smoke == 'true' }}
uses: actions/upload-artifact@v7
with:
name: spark-deploy-smoke
path: backend/evidence/smoke
retention-days: 14
- name: Send failure alert webhook
if: ${{ always() && failure() && env.OPS_ALERT_WEBHOOK_URL != '' }}
run: |
set -euo pipefail
payload="$(cat <<JSON
{
"event": "spark_deploy_failure",
"repository": "${{ github.repository }}",
"workflow": "${{ github.workflow }}",
"run_id": "${{ github.run_id }}",
"run_url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}",
"sha": "${{ github.sha }}",
"smoke_outcome": "${{ steps.smoke.outcome }}",
"rollback_outcome": "${{ steps.rollback.outcome }}"
}
JSON
)"
curl -fsS -X POST "${OPS_ALERT_WEBHOOK_URL}" \
-H "Content-Type: application/json" \
--data "${payload}"
- name: Write deployment summary
if: ${{ always() }}
run: |
{
echo "## Spark Deploy Summary"
echo
echo "- Commit: \`${{ github.sha }}\`"
echo "- Remote dir: \`${{ steps.resolve.outputs.remote_dir }}\`"
echo "- Deploy frontend: \`${{ steps.resolve.outputs.deploy_frontend }}\`"
echo "- Use pinned images: \`${{ steps.image_refs.outputs.use_pinned_images }}\`"
echo "- Image tag: \`${{ steps.image_refs.outputs.image_tag }}\`"
echo "- Verify signatures: \`${{ steps.resolve.outputs.verify_signatures }}\`"
if [ "${{ steps.resolve.outputs.verify_signatures }}" = "true" ] && [ "${{ steps.image_refs.outputs.use_pinned_images }}" = "true" ]; then
echo "- SBOM attestation check: \`spdxjson verified\`"
fi
if [ -n "${{ steps.image_refs.outputs.api_ref }}" ]; then
echo "- Backend image ref: \`${{ steps.image_refs.outputs.api_ref }}\`"
fi
if [ -n "${{ steps.image_refs.outputs.worker_ref }}" ]; then
echo "- Worker image ref: \`${{ steps.image_refs.outputs.worker_ref }}\`"
fi
echo "- Runner type: \`${{ steps.resolve.outputs.runner_type || 'github-hosted' }}\`"
echo "- Run smoke: \`${{ steps.resolve.outputs.run_smoke }}\`"
echo "- SSH reachable from runner: \`${{ steps.ssh_prep.outputs.keyscan_ok || 'false' }}\`"
echo "- Smoke outcome: \`${{ steps.smoke.outcome || 'skipped' }}\`"
echo "- Rollback outcome: \`${{ steps.rollback.outcome || 'not_triggered' }}\`"
if [ -n "${{ steps.resolve.outputs.public_api_url }}" ]; then
echo "- Frontend API URL: \`${{ steps.resolve.outputs.public_api_url }}\`"
else
echo "- Frontend API URL: \`(default from deploy script)\`"
fi
if [ -n "${{ steps.resolve.outputs.smoke_base_url }}" ]; then
echo "- Smoke base URL: \`${{ steps.resolve.outputs.smoke_base_url }}\`"
fi
} >> "$GITHUB_STEP_SUMMARY"
deploy-skipped:
needs: cost-precheck
if: ${{ needs.cost-precheck.outputs.allow_run != 'true' }}
runs-on: ubuntu-latest
steps:
- name: Report deploy skip reason
run: |
echo "Deploy Spark Runtime skipped by cost policy gate." >> "$GITHUB_STEP_SUMMARY"
echo "- status: ${{ needs.cost-precheck.outputs.status }}" >> "$GITHUB_STEP_SUMMARY"
echo "- override_used: ${{ needs.cost-precheck.outputs.override_used }}" >> "$GITHUB_STEP_SUMMARY"