'Audit' and pending change system, fluent API, better text model handling

.env.primary.example

@@ -22,7 +22,7 @@ HORDE_MODEL_REFERENCE_CACHE_TTL_SECONDS=60
 # - 'legacy': v1 API has CRUD, v2 API is read-only (converts from legacy to v2)
 # - 'v2': v2 API has CRUD, v1 API is read-only (converts from v2 to legacy)
 # Default is 'v2' but set to 'legacy' during transition period
-HORDE_MODEL_REFERENCE_CANONICAL_FORMAT=legacy
+HORDE_MODEL_REFERENCE_CANONICAL_FORMAT=LEGACY
 
 # ============================================================================
 # GitHub Seeding (First-Time Setup)
@@ -37,19 +37,19 @@ HORDE_MODEL_REFERENCE_GITHUB_SEED_ENABLED=false
 # ============================================================================
 
 # Enable Redis for distributed caching (required for multi-worker setups)
-# HORDE_MODEL_REFERENCE_REDIS_USE_REDIS=true
+# HORDE_MODEL_REFERENCE_REDIS__USE_REDIS=true
 
 # Redis connection URL
-# HORDE_MODEL_REFERENCE_REDIS_URL=redis://redis:6379/0
+# HORDE_MODEL_REFERENCE_REDIS__URL=redis://redis:6379/0
 
 # Redis connection pool size
-# HORDE_MODEL_REFERENCE_REDIS_POOL_SIZE=10
+# HORDE_MODEL_REFERENCE_REDIS__POOL_SIZE=10
 
 # Redis TTL (uses cache_ttl_seconds if not specified)
-# HORDE_MODEL_REFERENCE_REDIS_TTL_SECONDS=60
+# HORDE_MODEL_REFERENCE_REDIS__TTL_SECONDS=60
 
 # Enable pub/sub for cache invalidation across workers
-# HORDE_MODEL_REFERENCE_REDIS_USE_PUBSUB=true
+# HORDE_MODEL_REFERENCE_REDIS__USE_PUBSUB=true
 
 # ============================================================================
 # Data Directory

.github/workflows/docker-validation.yml

@@ -35,6 +35,14 @@ jobs:
           output-file: hadolint-results.sarif
           no-fail: false
 
+      - name: Ensure valid SARIF file
+        if: always()
+        run: |
+          if [ ! -s hadolint-results.sarif ] || ! python3 -c "import json,sys; json.load(open(sys.argv[1]))" hadolint-results.sarif 2>/dev/null; then
+            rm -f hadolint-results.sarif
+            echo '{"version":"2.1.0","$schema":"https://raw.githubusercontent.com/oasis-tcs/sarif-spec/main/sarif-2.1/schema/sarif-schema-2.1.0.json","runs":[{"tool":{"driver":{"name":"hadolint","rules":[]}},"results":[]}]}' > hadolint-results.sarif
+          fi
+
       - name: Upload hadolint SARIF results
         if: always()
         uses: github/codeql-action/upload-sarif@v3

.github/workflows/lint.yml

@@ -32,7 +32,7 @@ jobs:
         run: uv sync --locked --all-extras --dev
 
       - name: Run pre-commit
-        run: uv run pre-commit run --all-files
+        run: SKIP=mypy uv run pre-commit run --all-files --show-diff-on-failure
 
   lint:
     name: Lint (Ruff lint)
@@ -76,8 +76,6 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - "3.10"
-          - "3.11"
           - "3.12"
           - "3.13"
     steps:

.github/workflows/maintests.yml

@@ -24,8 +24,6 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - "3.10"
-          - "3.11"
           - "3.12"
           - "3.13"
 
@@ -57,9 +55,9 @@ jobs:
           - "3.13"
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - name: Install uv and set the python version
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v8.0.0
         with:
           python-version: ${{ matrix.python-version }}
           enable-cache: true
@@ -70,6 +68,7 @@ jobs:
       - name: Check that query fields have not drifted
         run: uv run python scripts/verify_query_fields.py
 
+
   build:
     env:
       AIWORKER_CACHE_HOME: ${{ github.workspace }}/.cache
@@ -79,8 +78,6 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - "3.10"
-          - "3.11"
           - "3.12"
           - "3.13"
 

.github/workflows/prtests.yml

@@ -27,8 +27,6 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - "3.10"
-          - "3.11"
           - "3.12"
           - "3.13"
 
@@ -60,11 +58,9 @@ jobs:
           - "3.13"
 
     steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
+      - uses: actions/checkout@v6
       - name: Install uv and set the python version
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v8.0.0
         with:
           python-version: ${{ matrix.python-version }}
           enable-cache: true
@@ -84,8 +80,6 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - "3.10"
-          - "3.11"
           - "3.12"
           - "3.13"
 
@@ -120,7 +114,7 @@ jobs:
       - name: Start fastapi server
         env:
           HORDE_MODEL_REFERENCE_REPLICATE_MODE: PRIMARY
-          HORDE_MODEL_REFERENCE_CANONICAL_FORMAT: legacy
+          HORDE_MODEL_REFERENCE_CANONICAL_FORMAT: LEGACY
           HORDE_MODEL_REFERENCE_PRIMARY_API_URL: http://localhost:19800/api
           HORDE_MODEL_REFERENCE_GITHUB_SEED_ENABLED: True
         run: |
@@ -169,8 +163,8 @@ jobs:
       - name: Start fastapi server with redis backend
         env:
           HORDE_MODEL_REFERENCE_REPLICATE_MODE: PRIMARY
-          HORDE_MODEL_REFERENCE_CANONICAL_FORMAT: legacy
-          HORDE_MODEL_REFERENCE_REDIS_URL: redis://localhost:6379/0
+          HORDE_MODEL_REFERENCE_CANONICAL_FORMAT: LEGACY
+          HORDE_MODEL_REFERENCE_REDIS__URL: redis://localhost:6379/0
           HORDE_MODEL_REFERENCE_PRIMARY_API_URL: http://localhost:19800/api
           HORDE_MODEL_REFERENCE_GITHUB_SEED_ENABLED: True
         run: |
@@ -192,4 +186,4 @@ jobs:
       - name: Run integration tests
         run: uv run pytest tests -m integration
         env:
-          HORDE_MODEL_REFERENCE_REDIS_URL: redis://localhost:6379/0
+          HORDE_MODEL_REFERENCE_REDIS__URL: redis://localhost:6379/0

.hadolint.yaml

@@ -0,0 +1,10 @@
+failure-threshold: error
+format: tty
+ignored:
+  - DL3008
+override:
+  error:
+    - DL3015
+trustedRegistries:
+  - docker.io
+  - ://registry.com

.pre-commit-config.yaml

@@ -13,7 +13,7 @@ repos:
     -    id: ruff-check
     -    id: ruff-format
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v1.18.2'
+    rev: 'v1.19.1'
     hooks:
     -   id: mypy
         args: [--strict, --disallow-untyped-defs, --warn-unused-configs]
@@ -35,6 +35,7 @@ repos:
             - redis[hiredis]
             - settings-doc
             - strenum
+            - tenacity
             - typing-extensions
             - ujson
             - types-aiofiles

.CONTRIBUTING.md → CONTRIBUTING.md

@@ -9,6 +9,9 @@ All pull requests, large or small, from anyone are welcome!
     - [Environment Management](#environment-management)
         - [First time setup](#first-time-setup)
     - [Code Quality Tools](#code-quality-tools)
+    - [Testing](#testing)
+        - [Writing Tests](#writing-tests)
+    - [Git Workflow](#git-workflow)
     - [Code Style and System Design](#code-style-and-system-design)
 
 ## Environment Management
@@ -18,12 +21,13 @@ All pull requests, large or small, from anyone are welcome!
 ### First time setup
 
 - Install uv, as described [in the uv installation guide](https://github.com/astral-sh/uv/#installation).
-- `uv python install 3.10 3.11 3.12 3.13` # We currently support these versions.
-- `uv set 3.12` # Set to your desired default version.
+- `uv python install 3.12` -- This project requires Python 3.12+.
+- `uv python pin 3.12` -- Pin the default version.
 - `uv self update`
 - `uv sync --all-groups`
 - The `.venv/` directory will now be created with all project, development and documentation dependencies installed.
     - Be sure to point your IDE to the python binary appropriate for your OS in this directory.
+- `pre-commit install` -- Set up pre-commit hooks for automatic formatting and linting on commit.
 
 ## Code Quality Tools
 
@@ -38,27 +42,55 @@ All pull requests, large or small, from anyone are welcome!
 > Note: Many of the tools below are run by `pre-commit` automatically, but can also be run manually if desired.
 
 - [**ruff**](https://github.com/astral-sh/ruff)
-    -  Also provides formatting (replacing `black`) with `ruff format . --fix`
-    - Linting rules from a wide variety of selectable rule sets
+    - Provides both formatting and linting
+    - Format: `ruff format .`
+    - Lint: `ruff check . --fix`
+    - Combined: `ruff format . && ruff check . --fix`
     - See `pyproject.toml` for the rules used.
     - See all rules (but not necessarily used in the project) [available in ruff here](https://beta.ruff.rs/docs/rules/).
-    - Run with `ruff check .`
-        - Note: When using autofixing (`ruff check . --fix`), changes may be made that require running black, which can then result in needing to run `ruff check . --fix` again.
-        - Consider running `black . && ruff check . --fix && black . && ruff check . --fix` to avoid this.
-- [**mypy**](https://mypy-lang.org/)
-    - Static type safety
-    - I recommending using the [mypy daemon](https://mypy.readthedocs.io/en/stable/mypy_daemon.html) instead of periodically running `pre-commit` (or `mypy` directly.).
+- [**ty**](https://github.com/astral-sh/ty)
+    - Type checker: `ty check .`
 - [**pyright**](https://github.com/microsoft/pyright)
     - Shipped with vscode by default (via the python extension `ms-python.vscode-pylance`)
     - Suggested settings:
         - `"python.analysis.typeCheckingMode": "off"`
-            - The pylance extension has certain opinionated type checking assertions which are clash with mypy.
+            - The pylance extension has certain opinionated type checking assertions which clash with other type checkers.
             - For example, overriding an optional field to be non-optional is considered by pylance to be a type error due to the field being invariant and the parent class potentially settings it to `None`. However, by convention in the SDK, this is a forbidden pattern.
         - `"python.analysis.languageServerMode": "full"`
         - `"python.testing.pytestEnabled": true`
 - [**tach**](https://github.com/gauge-sh/tach)
     - Enforces internal namespace dependency constraints. This helps avoid circular dependencies and helps ensure implementations are in a logical place.
 
+## Testing
+
+Tests require the `AI_HORDE_TESTING=True` environment variable to be set. This prevents interference from user-specific environment variables.
+
+```bash
+# Set the required env var and run tests
+export AI_HORDE_TESTING=True
+pytest
+
+# Run with coverage
+pytest --cov=horde_model_reference --cov-report=html
+
+# Run a specific test file
+pytest tests/test_model_reference_manager.py
+```
+
+### Writing Tests
+
+- **Singleton reset:** Use the `restore_manager_singleton` fixture to ensure `ModelReferenceManager` is restored to its pre-test state after your test. This is critical when testing with different manager configurations.
+- **Temp directories:** Use the `primary_base` fixture for tests that need an isolated temp directory for PRIMARY mode file operations.
+- **Test ordering:** Tests in `test_scripts`, `test_convert_legacy_database`, and `test_model_reference_manager` run in a fixed order (defined in `conftest.py`). Other tests run in arbitrary order.
+
+## Git Workflow
+
+1. Fork the repository and create a feature branch from `main`
+2. Make your changes
+3. Run the quality checks: `ruff format . && ruff check . --fix && ty check .`
+4. Run the tests: `AI_HORDE_TESTING=True pytest`
+5. Commit with a clear message and open a pull request
+
 ## Code Style and System Design
 
 - See the [python haidra style guide](docs/haidra-assets/docs/meta/python.md) for standards on code style, system design, testing, and documentation.

DEPLOYMENT.md

@@ -93,7 +93,7 @@ Create a `.env` file:
 ```bash
 HORDE_MODEL_REFERENCE_REPLICATE_MODE=PRIMARY
 HORDE_MODEL_REFERENCE_MAKE_FOLDERS=true
-HORDE_MODEL_REFERENCE_CANONICAL_FORMAT=legacy
+HORDE_MODEL_REFERENCE_CANONICAL_FORMAT=LEGACY
 ```
 
 Or use `.env.primary.example` as a template:
@@ -120,13 +120,13 @@ fastapi dev src/horde_model_reference/service/app.py --port 19800
 # Linux/macOS
 export HORDE_MODEL_REFERENCE_REPLICATE_MODE=PRIMARY
 export HORDE_MODEL_REFERENCE_MAKE_FOLDERS=true
-export HORDE_MODEL_REFERENCE_CANONICAL_FORMAT=legacy
+export HORDE_MODEL_REFERENCE_CANONICAL_FORMAT=LEGACY
 fastapi dev src/horde_model_reference/service/app.py --port 19800
 
 # Windows PowerShell
 $env:HORDE_MODEL_REFERENCE_REPLICATE_MODE="PRIMARY"
 $env:HORDE_MODEL_REFERENCE_MAKE_FOLDERS="true"
-$env:HORDE_MODEL_REFERENCE_CANONICAL_FORMAT="legacy"
+$env:HORDE_MODEL_REFERENCE_CANONICAL_FORMAT="LEGACY"
 fastapi dev src/horde_model_reference/service/app.py --port 19800
 ```
 
@@ -220,8 +220,8 @@ See `.env.example` for all available options, or `.env.primary.example` for PRIM
 | `HORDE_MODEL_REFERENCE_REPLICATE_MODE` | `REPLICA` | Set to `PRIMARY` for server mode |
 | `HORDE_MODEL_REFERENCE_MAKE_FOLDERS` | `false` | Auto-create directories |
 | `HORDE_MODEL_REFERENCE_CANONICAL_FORMAT` | `legacy` | Use legacy format (pre-v2 transition) |
-| `HORDE_MODEL_REFERENCE_REDIS_USE_REDIS` | `false` | Enable Redis (multi-worker) |
-| `HORDE_MODEL_REFERENCE_REDIS_URL` | `redis://localhost:6379/0` | Redis connection |
+| `HORDE_MODEL_REFERENCE_REDIS__USE_REDIS` | `false` | Enable Redis (multi-worker) |
+| `HORDE_MODEL_REFERENCE_REDIS__URL` | `redis://localhost:6379/0` | Redis connection |
 | `HORDE_MODEL_REFERENCE_CACHE_TTL_SECONDS` | `60` | Cache lifetime |
 | `HORDE_MODEL_REFERENCE_GITHUB_SEED_ENABLED` | `false` | Auto-seed on first start |
 
@@ -256,7 +256,7 @@ docker-compose -f docker-compose.redis.yml ps
 **Performance issues:**
 
 - Single-worker too slow? Switch to multi-worker with Redis
-- Multi-worker not faster? Verify `HORDE_MODEL_REFERENCE_REDIS_USE_REDIS=true` is set
+- Multi-worker not faster? Verify `HORDE_MODEL_REFERENCE_REDIS__USE_REDIS=true` is set
 - Increase `HORDE_MODEL_REFERENCE_CACHE_TTL_SECONDS` for longer caching
 
 ---

Dockerfile

@@ -55,8 +55,9 @@ LABEL org.opencontainers.image.title="Horde Model Reference" \
 
 # Install runtime dependencies
 # Note: git is required for the GitHub sync service
+# hadolint ignore=DL3008 -- curl is unpinned to avoid repeated CI breakage from Debian repo churn
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    curl=8.14.1-2 \
+    curl \
     git=1:2.47.3-0+deb13u1 \
     && rm -rf /var/lib/apt/lists/*