Skip to content
Test Suite

feat(context): add evidence-aware execution and runtime contracts #1343

Sign in to view logs

feat(context): add evidence-aware execution and runtime contracts

feat(context): add evidence-aware execution and runtime contracts #1343

Workflow file for this run

.github/workflows/test.yml at ee0bd94
name: Test Suite
on:
push:
branches: [main, master]
pull_request:
branches: [main, master]
# Read-only. This workflow verifies the built artifact rather than committing it
# back, so it needs no write access — a CI job that can push to the repo is a
# supply-chain risk we avoid.
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12"]
steps:
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 (2026-07-16)
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 (2025-04-24)
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install pyyaml jsonschema pytest coverage hypothesis tiktoken
- name: Verify build artifact is in sync
# The committed perseus.py must match a fresh build from scripts/. We
# VERIFY it rather than auto-rebuilding and pushing back: a workflow that
# commits to the repo requires contents:write and a push-capable token,
# which is a supply-chain risk. Contributors run `python scripts/build.py`
# and commit perseus.py themselves; CI fails here if it drifted. (The
# Windows job already uses this same --check.)
run: python scripts/build.py --check
- name: Smoke test — render validation
run: |
set -e
export PERSEUS_ALLOW_DANGEROUS=1
python3 -c "
suffix = chr(112) + chr(101) + chr(114) + chr(115) + chr(101) + chr(117) + chr(115)
header = chr(64) + suffix + ' v1.0.8'
with open('/tmp/ci-smoke-context.md', 'w') as f:
f.write(header + '\n')
f.write(chr(64) + 'date\n')
f.write(chr(64) + 'query \"whoami\" fallback=\"ci-runner\"\n')
f.write(chr(64) + 'memory mode=search query=\"test\" k=1\n')
"
python perseus.py render /tmp/ci-smoke-context.md --output /tmp/ci-smoke-output.md
echo "SMOKE: render succeeded ($(wc -l < /tmp/ci-smoke-output.md) lines)"
python3 -c "
import sys, datetime
text = open('/tmp/ci-smoke-output.md').read()
errors = []
if text.strip().startswith(chr(64) + 'perseus') and 'v1.0.8' not in text:
errors.append('version pin not resolved')
# Assert @date resolved: its timestamp contains the current year. This
# is the environment-independent liveness signal — the old 'date' in
# text check failed because a rendered timestamp never contains the
# literal word 'date' (and no memory backend supplies it on a clean
# runner). @date needs no trust gate, unlike @query (shell) which is
# intentionally left unresolved without render.allow_query_shell.
at = chr(64)
year = str(datetime.date.today().year)
if at + 'date' in text:
errors.append('@date directive left unresolved')
if year not in text:
errors.append('@date did not resolve (no current year in output)')
if len(text) <= 20:
errors.append('output too short')
if errors:
print('SMOKE FAIL:', errors)
sys.exit(1)
print('SMOKE PASS: render pipeline functional')
"
- name: Run test suite with coverage
run: |
export PERSEUS_ALLOW_DANGEROUS=1
coverage run -m pytest tests/ -q
# Gate: baseline 56% (2026-08-12, 20503 stmts). fail-under 50 keeps
# a 6-point cushion so routine refactors don't trip it, while a real
# coverage regression still fails CI.
coverage report --fail-under=50
- name: Order-independence check — release/installer tests twice
# #953: these tests used to flip pass↔fail on shared dist/ artifacts and
# venv-escaped interpreters. They now isolate builds per-run; this
# back-to-back double run proves order-independence stays fixed.
if: matrix.python-version == '3.12'
run: |
python -m pytest tests/test_release.py tests/test_installer.py -q
python -m pytest tests/test_release.py -q
- name: Generate IP/patent evidence exhibits
if: matrix.python-version == '3.12'
run: |
export PERSEUS_ALLOW_DANGEROUS=1
python -m pytest tests/test_ip_evidence.py tests/test_benchmark_resolve_vs_agentic.py \
tests/test_ip_unified_grammar.py tests/test_ip_recursive_resolution.py \
-q --save-exhibits
- name: Upload IP evidence exhibits
if: matrix.python-version == '3.12'
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 (2025-03-19)
with:
name: ip-evidence-exhibits
path: docs/ip/exhibits/
if-no-files-found: warn
test-windows:
runs-on: windows-latest
# Windows runners are slow: the full suite plus setup exceeds 10 minutes
# (observed cancellation at 10m07s, 2026-08-12). 15 keeps a real ceiling
# without flaky mid-suite cancellations.
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# Windows-specific failures are OS-level (paths, fcntl, sockets, shells),
# not Python-version-specific, so one interpreter guards them. The Linux
# matrix above covers per-version behavior.
python-version: ["3.12"]
env:
PERSEUS_ALLOW_DANGEROUS: "1"
steps:
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 (2026-07-16)
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 (2025-04-24)
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install pyyaml jsonschema pytest coverage hypothesis
- name: Verify build artifact is in sync
run: python scripts/build.py --check
- name: Run test suite
shell: pwsh
timeout-minutes: 15
run: |
$ErrorActionPreference = "Stop"
python -m pytest tests/ -q --durations=20
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
adapters:
# #954: tests/test_adapters.py used to skip on missing langchain_core /
# llama_index and CI never installed them — the adapter SDK coverage never
# ran anywhere. This leg installs the declared `adapters` extra and runs
# the adapter tests for real.
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 (2026-07-16)
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 (2025-04-24)
with:
python-version: "3.12"
- name: Install dependencies (+ adapters extra)
run: |
python -m pip install --upgrade pip
pip install pyyaml jsonschema pytest ".[adapters]"
- name: Verify build artifact is in sync
run: python scripts/build.py --check
- name: Run adapter tests (optional deps installed)
run: python -m pytest tests/test_adapters.py -v
selection-eval:
# Offline, deterministic gate: tier-based context selection must be exact.
# Fails the run (non-zero exit) on any tier-gating regression. No network,
# no API key — imports the built artifact and checks the skip behavior
# against frozen ground truth in benchmark/selection/dataset.json.
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 (2026-07-16)
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 (2025-04-24)
with:
python-version: "3.12"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install pyyaml
- name: Verify build artifact is in sync
run: python scripts/build.py --check
- name: Context-selection eval (offline gate)
# No pipe — the script's non-zero exit must propagate to fail the job.
run: python benchmark/selection/run.py
trace-bench:
# Offline, deterministic gate: trajectory-mined failure attribution
# (#968) must score >= 70% attribution top-1 and >= 90% CREATE/UPDATE
# accuracy against the frozen planted-fault ground truth in
# benchmark/trace/dataset.json, with zero cross-layer verification
# errors. No network, no API key — imports the built artifact.
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 (2026-07-16)
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 (2025-04-24)
with:
python-version: "3.12"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install pyyaml
- name: Verify build artifact is in sync
run: python scripts/build.py --check
- name: TRACE attribution benchmark (offline gate)
run: python benchmark/trace/run.py
context-quality-bench:
# Offline, deterministic gate: the 7-criteria preflight scorer (#969)
# must degrade monotonically on planted defects and block every
# degraded sample while passing the healthy baseline, with replay-first
# verification. No network, no API key — imports the built artifact.
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 (2026-07-16)
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 (2025-04-24)
with:
python-version: "3.12"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install pyyaml
- name: Verify build artifact is in sync
run: python scripts/build.py --check
- name: Context-quality preflight benchmark (offline gate)
run: python benchmark/context-quality/run.py
pooled-selection-bench:
# Offline, deterministic gate: the pooled submodular selector (#970)
# must keep 100% of ground-truth-relevant candidates at <= 50% token
# budget, stay within budget, and re-verify every selection trace.
# No network, no API key — imports the built artifact.
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 (2026-07-16)
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 (2025-04-24)
with:
python-version: "3.12"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install pyyaml
- name: Verify build artifact is in sync
run: python scripts/build.py --check
- name: Pooled-selection benchmark (offline gate)
run: python benchmark/pooled-selection/run.py
context-codec-bench:
# Offline, deterministic gate: commitment-preserving compression (#971)
# must preserve every critical atom (CAR >= 0.99), lose zero safety
# boundaries, round-trip 1.0, and fall back to the original text when
# an injected lossy compressor breaks preservation. No network, no
# API key — imports the built artifact.
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 (2026-07-16)
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 (2025-04-24)
with:
python-version: "3.12"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install pyyaml
- name: Verify build artifact is in sync
run: python scripts/build.py --check
- name: Context Codec benchmark (offline gate)
run: python benchmark/context-codec/run.py
retry-isolation-bench:
# Offline, seeded gate: clean-restart attempt isolation (#972) must
# show the IID overestimate of pass@K >= 8pp at the ~7.1x cascade
# ratio, clean restart must recover it, the closed-form allocation
# must match exactly, and the fence demo must quarantine cleanly.
# No network, no API key — imports the built artifact.
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 (2026-07-16)
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 (2025-04-24)
with:
python-version: "3.12"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install pyyaml
- name: Verify build artifact is in sync
run: python scripts/build.py --check
- name: Retry-isolation benchmark (offline gate)
run: python benchmark/retry-isolation/run.py