feat: did a couple things idk

Author: CarterPerez-dev

PROJECTS/advanced/ai-threat-detection/.gitignore

@@ -1,7 +1,7 @@
 # ©AngelaMos | 2026
 # .gitignore
 
-# Planning docs
+# dev docs
 .angelusvigil/
 
 # Environment

PROJECTS/advanced/ai-threat-detection/backend/app/api/models_api.py

@@ -52,7 +52,7 @@ async def _get_active_models(
     Query all active model metadata records
     """
     query = select(ModelMetadata).where(
-        ModelMetadata.is_active == True  # noqa: E712
+        ModelMetadata.is_active == True  # type: ignore[arg-type]  # noqa: E712
     )
     rows = (await session.execute(query)).scalars().all()
     return [{

PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/inference.py

@@ -6,13 +6,14 @@
 import json
 import logging
 from pathlib import Path
+from typing import Any
 
 import numpy as np
 
 try:
     import onnxruntime as ort
 except ImportError:
-    ort = None  # type: ignore[assignment]
+    ort = None
 
 logger = logging.getLogger(__name__)
 
@@ -33,9 +34,9 @@ class InferenceEngine:
     """
 
     def __init__(self, model_dir: str) -> None:
-        self._ae_session: ort.InferenceSession | None = None  # type: ignore[union-attr]
-        self._rf_session: ort.InferenceSession | None = None  # type: ignore[union-attr]
-        self._if_session: ort.InferenceSession | None = None  # type: ignore[union-attr]
+        self._ae_session: ort.InferenceSession | None = None
+        self._rf_session: ort.InferenceSession | None = None
+        self._if_session: ort.InferenceSession | None = None
         self._scaler_center: np.ndarray | None = None
         self._scaler_scale: np.ndarray | None = None
         self._threshold: float = 0.0
@@ -127,12 +128,12 @@ def _scale_for_ae(self, batch: np.ndarray) -> np.ndarray:
         """
         if self._scaler_center is None or self._scaler_scale is None:
             return batch
-        return (batch - self._scaler_center) / self._scaler_scale
+        return (batch - self._scaler_center) / self._scaler_scale  # type: ignore[no-any-return]
 
     @staticmethod
     def _extract_rf_proba(
-            ort_output: list | np.ndarray
-    ) -> np.ndarray:  # type: ignore[type-arg]
+            ort_output: list[Any] | np.ndarray
+    ) -> np.ndarray:
         """
         Extract attack probability from skl2onnx RF output format.
 

PROJECTS/advanced/ai-threat-detection/backend/app/core/features/mappings.py

@@ -93,6 +93,8 @@
     "file_extension": EXTENSION_MAP,
 }
 
+WINDOWED_FEATURE_NAMES: list[str] = FEATURE_ORDER[23:]
+
 BOOLEAN_FEATURES: frozenset[str] = frozenset({
     "has_encoded_chars",
     "has_double_encoding",

PROJECTS/advanced/ai-threat-detection/backend/app/factory.py

@@ -9,6 +9,7 @@
 from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 from fastapi import FastAPI
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
@@ -24,6 +25,9 @@
 from app.models import model_metadata as _model_metadata_reg  # noqa: F401
 from app.models import threat_event as _threat_event_reg  # noqa: F401
 
+if TYPE_CHECKING:
+    from app.core.detection.inference import InferenceEngine
+
 logger = logging.getLogger(__name__)
 
 
@@ -110,7 +114,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
     logger.info("AngelusVigil shut down cleanly")
 
 
-def _load_inference_engine() -> object | None:
+def _load_inference_engine() -> InferenceEngine | None:
     """
     Attempt to load the ONNX inference engine from the
     configured model directory, returning None if ML

PROJECTS/advanced/ai-threat-detection/backend/app/models/model_metadata.py

@@ -18,7 +18,6 @@ class ModelMetadata(TimestampedModel, table=True):
     __table_args__ = (Index(
         "idx_model_metadata_active",
         "model_type",
-        unique=True,
         postgresql_where=text("is_active = TRUE"),
     ), )
 

PROJECTS/advanced/ai-threat-detection/backend/cli/main.py

@@ -3,6 +3,8 @@
 main.py
 """
 
+import asyncio
+import dataclasses
 from pathlib import Path
 
 import typer
@@ -22,6 +24,44 @@
 DEFAULT_SERVER_URL = "http://localhost:8000"
 
 
+async def _write_metadata(
+    model_dir: Path,
+    training_samples: int,
+    metrics: dict[str, object],
+    mlflow_run_id: str | None,
+    threshold: float | None,
+) -> None:
+    """
+    Persist training metadata to the database
+    """
+    from app.config import settings
+    from ml.metadata import save_model_metadata
+    from sqlalchemy.ext.asyncio import (
+        AsyncSession,
+        async_sessionmaker,
+        create_async_engine,
+    )
+
+    engine = create_async_engine(settings.database_url)
+    try:
+        factory = async_sessionmaker(
+            engine,
+            class_=AsyncSession,
+            expire_on_commit=False,
+        )
+        async with factory() as session:
+            await save_model_metadata(
+                session,
+                model_dir=model_dir,
+                training_samples=training_samples,
+                metrics=metrics,
+                mlflow_run_id=mlflow_run_id,
+                threshold=threshold,
+            )
+    finally:
+        await engine.dispose()
+
+
 @app.command()
 def serve(
     host: str = typer.Option("0.0.0.0", help="Bind address"),
@@ -92,9 +132,10 @@ def train(
             )
             raise typer.Exit(code=1)
 
-        from ml.data_loader import load_csic_dataset
+        from ml.data_loader import load_csic_dataset, load_csic_normal
 
         normal_path = csic_dir / "normalTrafficTraining.txt"
+        normal_test_path = csic_dir / "normalTrafficTest.txt"
         attack_path = csic_dir / "anomalousTrafficTest.txt"
         typer.echo(f"Loading CSIC data from {csic_dir}")
         X_csic, y_csic = load_csic_dataset(
@@ -106,6 +147,14 @@ def train(
             f"  CSIC: {len(X_csic)} samples"
         )
 
+        if normal_test_path.exists():
+            X_extra, y_extra = load_csic_normal(normal_test_path)
+            X_parts.append(X_extra)
+            y_parts.append(y_extra)
+            typer.echo(
+                f"  CSIC normal test: {len(X_extra)} samples"
+            )
+
     if synthetic_normal > 0 or synthetic_attack > 0:
         from ml.synthetic import generate_mixed_dataset
 
@@ -143,6 +192,25 @@ def train(
     )
     result = orch.run(X, y)
 
+    try:
+        metrics: dict[str, object] = (
+            dataclasses.asdict(result.ensemble_metrics)
+            if result.ensemble_metrics else {}
+        )
+        asyncio.run(_write_metadata(
+            Path(output_dir),
+            int(len(X)),
+            metrics,
+            result.mlflow_run_id,
+            result.ae_metrics.get("ae_threshold"),
+        ))
+        typer.echo("  Model metadata saved to database")
+    except Exception as exc:
+        typer.echo(
+            f"  Warning: could not save metadata to DB: {exc}",
+            err=True,
+        )
+
     typer.echo(f"Models exported to {output_dir}")
     if result.ensemble_metrics is not None:
         typer.echo(

PROJECTS/advanced/ai-threat-detection/backend/ml/autoencoder.py

@@ -51,13 +51,13 @@ def encode(self, x: Tensor) -> Tensor:
         """
         Compress input through the encoder to the 6-dim bottleneck.
         """
-        return self.encoder(x)
+        return self.encoder(x)  # type: ignore[no-any-return]
 
     def decode(self, z: Tensor) -> Tensor:
         """
         Reconstruct input from the bottleneck representation.
         """
-        return self.decoder(z)
+        return self.decoder(z)  # type: ignore[no-any-return]
 
     def forward(self, x: Tensor) -> Tensor:
         """

PROJECTS/advanced/ai-threat-detection/backend/ml/data_loader.py

@@ -13,6 +13,7 @@
 
 from app.core.features.encoder import encode_for_inference
 from app.core.features.extractor import extract_request_features
+from app.core.features.mappings import WINDOWED_FEATURE_NAMES
 from app.core.ingestion.parsers import ParsedLogEntry
 
 logger = logging.getLogger(__name__)
@@ -26,21 +27,6 @@
 _DEFAULT_UA = ("Mozilla/5.0 (compatible; Konqueror/3.5; Linux)"
                " KHTML/3.5.8 (like Gecko)")
 
-_WINDOWED_FEATURE_NAMES: list[str] = [
-    "req_count_1m",
-    "req_count_5m",
-    "req_count_10m",
-    "error_rate_5m",
-    "unique_paths_5m",
-    "unique_uas_10m",
-    "method_entropy_5m",
-    "avg_response_size_5m",
-    "status_diversity_5m",
-    "path_depth_variance_5m",
-    "inter_request_time_mean",
-    "inter_request_time_std",
-]
-
 
 @dataclass
 class CSICRequest:
@@ -192,7 +178,7 @@ def load_csic_dataset(
         entry = csic_to_parsed_entry(req)
         features = extract_request_features(entry)
 
-        for name in _WINDOWED_FEATURE_NAMES:
+        for name in WINDOWED_FEATURE_NAMES:
             features[name] = 0.0
 
         vector = encode_for_inference(features)
@@ -211,3 +197,32 @@ def load_csic_dataset(
     )
 
     return X, y
+
+
+def load_csic_normal(
+    path: Path,
+) -> tuple[np.ndarray, np.ndarray]:
+    """
+    Load a CSIC 2010 normal traffic file and return (X, y) arrays
+    with all labels set to 0
+    """
+    reqs = parse_csic_file(path, label=0)
+
+    vectors: list[list[float]] = []
+    for req in reqs:
+        entry = csic_to_parsed_entry(req)
+        features = extract_request_features(entry)
+        for name in WINDOWED_FEATURE_NAMES:
+            features[name] = 0.0
+        vectors.append(encode_for_inference(features))
+
+    X = np.array(vectors, dtype=np.float32)
+    y = np.zeros(len(vectors), dtype=np.int32)
+
+    logger.info(
+        "Loaded %d normal samples from %s",
+        len(vectors),
+        path.name,
+    )
+
+    return X, y

PROJECTS/advanced/ai-threat-detection/backend/ml/download_csic.py

@@ -7,7 +7,8 @@
 import logging
 import sys
 from pathlib import Path
-from urllib.request import urlretrieve
+
+import httpx
 
 logger = logging.getLogger(__name__)
 
@@ -26,24 +27,6 @@
 MIN_FILE_BYTES = 1_000_000
 
 
-def _progress_hook(
-    block_num: int,
-    block_size: int,
-    total_size: int,
-) -> None:
-    """
-    Print download progress to stdout
-    """
-    downloaded = block_num * block_size
-    if total_size > 0:
-        pct = min(downloaded * 100 / total_size, 100)
-        sys.stdout.write(f"\r  {pct:.0f}%")
-    else:
-        mb = downloaded / 1_048_576
-        sys.stdout.write(f"\r  {mb:.1f} MB")
-    sys.stdout.flush()
-
-
 def _compute_sha256(path: Path) -> str:
     """
     Compute SHA-256 hash of a file
@@ -73,7 +56,31 @@ def download_csic(output_dir: Path = DATASET_DIR, ) -> None:
         print(f"Downloading {filename}...")
 
         try:
-            urlretrieve(url, dest, reporthook=_progress_hook)
+            with httpx.stream(
+                "GET",
+                url,
+                follow_redirects=True,
+            ) as response:
+                response.raise_for_status()
+                total = int(
+                    response.headers.get("content-length", 0)
+                )
+                downloaded = 0
+                with open(dest, "wb") as f:
+                    for chunk in response.iter_bytes(
+                        chunk_size=65536
+                    ):
+                        f.write(chunk)
+                        downloaded += len(chunk)
+                        if total > 0:
+                            pct = min(
+                                downloaded * 100 / total, 100
+                            )
+                            sys.stdout.write(f"\r  {pct:.0f}%")
+                        else:
+                            mb = downloaded / 1_048_576
+                            sys.stdout.write(f"\r  {mb:.1f} MB")
+                        sys.stdout.flush()
             print()
         except Exception as exc:
             logger.error(