Merge pull request #108 from CarterPerez-dev/project/ai-threat-detection

Project/ai threat detection phase 2-4

Author: CarterPerez-dev

PROJECTS/advanced/ai-threat-detection/.env.example

@@ -13,6 +13,7 @@ API_KEY=changeme-generate-a-real-key
 POSTGRES_HOST_PORT=16969
 REDIS_HOST_PORT=26969
 BACKEND_HOST_PORT=36969
+FRONTEND_HOST_PORT=46969
 
 # PostgreSQL
 POSTGRES_DB=angelusvigil

PROJECTS/advanced/ai-threat-detection/.gitignore

@@ -1,7 +1,7 @@
 # ©AngelaMos | 2026
 # .gitignore
 
-# Planning docs
+# dev docs
 .angelusvigil/
 
 # Environment

PROJECTS/advanced/ai-threat-detection/backend/app/api/deps.py

@@ -17,4 +17,3 @@ async def get_session(request: Request) -> AsyncIterator[AsyncSession]:
     factory = request.app.state.session_factory
     async with factory() as session:
         yield session
-        await session.commit()

PROJECTS/advanced/ai-threat-detection/backend/app/api/ingest.py

@@ -0,0 +1,42 @@
+"""
+©AngelaMos | 2026
+ingest.py
+"""
+
+import asyncio
+
+from fastapi import APIRouter, Request
+from pydantic import BaseModel
+
+router = APIRouter(prefix="/ingest", tags=["ingest"])
+
+
+class BatchIngestRequest(BaseModel):
+    """
+    Payload for bulk log line ingestion
+    """
+
+    lines: list[str]
+
+
+@router.post("/batch", status_code=200)
+async def ingest_batch(
+    body: BatchIngestRequest,
+    request: Request,
+) -> dict[str, int]:
+    """
+    Push a batch of raw log lines into the pipeline queue
+    """
+    pipeline = getattr(request.app.state, "pipeline", None)
+    if pipeline is None:
+        return {"queued": 0}
+
+    queued = 0
+    for line in body.lines:
+        try:
+            pipeline.raw_queue.put_nowait(line)
+            queued += 1
+        except asyncio.QueueFull:
+            break
+
+    return {"queued": queued}

PROJECTS/advanced/ai-threat-detection/backend/app/api/models_api.py

@@ -5,30 +5,60 @@
 
 import uuid
 
-from fastapi import APIRouter
+from fastapi import APIRouter, Request
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.models.model_metadata import ModelMetadata
 
 router = APIRouter(prefix="/models", tags=["models"])
 
 
 @router.get("/status")
-async def model_status() -> dict[str, object]:
+async def model_status(request: Request, ) -> dict[str, object]:
     """
-    Return the status of active ML models.
+    Return the status of active ML models
     """
+    models_loaded = getattr(request.app.state, "models_loaded", False)
+    detection_mode = getattr(request.app.state, "detection_mode", "rules")
+
+    active_models: list[dict[str, object]] = []
+    session_factory = getattr(request.app.state, "session_factory", None)
+    if session_factory is not None:
+        async with session_factory() as session:
+            active_models = await _get_active_models(session)
+
     return {
-        "active_models": [],
-        "detection_mode": "rules-only",
-        "note": "ML models available after Phase 2 training",
+        "models_loaded": models_loaded,
+        "detection_mode": detection_mode,
+        "active_models": active_models,
     }
 
 
 @router.post("/retrain", status_code=202)
 async def retrain() -> dict[str, object]:
     """
-    Trigger an async model retraining job.
+    Trigger an async model retraining job
     """
     return {
         "status": "accepted",
         "job_id": uuid.uuid4().hex,
-        "note": "Retraining not available in Phase 1",
     }
+
+
+async def _get_active_models(
+    session: AsyncSession, ) -> list[dict[str, object]]:
+    """
+    Query all active model metadata records
+    """
+    query = select(ModelMetadata).where(
+        ModelMetadata.is_active == True  # type: ignore[arg-type]  # noqa: E712
+    )
+    rows = (await session.execute(query)).scalars().all()
+    return [{
+        "model_type": row.model_type,
+        "version": row.version,
+        "training_samples": row.training_samples,
+        "metrics": row.metrics,
+        "threshold": row.threshold,
+    } for row in rows]

PROJECTS/advanced/ai-threat-detection/backend/app/api/stats.py

@@ -15,8 +15,8 @@
 
 @router.get("", response_model=StatsResponse)
 async def get_stats(
-    session: AsyncSession = Depends(get_session),
-    time_range: str = Query("24h", alias="range"),
+        session: AsyncSession = Depends(get_session),
+        time_range: str = Query("24h", alias="range"),
 ) -> StatsResponse:
     """
     Aggregate threat statistics for a given time window.

PROJECTS/advanced/ai-threat-detection/backend/app/api/threats.py

@@ -18,13 +18,13 @@
 
 @router.get("", response_model=ThreatListResponse)
 async def list_threats(
-    session: AsyncSession = Depends(get_session),
-    limit: int = Query(50, ge=1, le=100),
-    offset: int = Query(0, ge=0),
-    severity: str | None = Query(None),
-    source_ip: str | None = Query(None),
-    since: datetime | None = Query(None),
-    until: datetime | None = Query(None),
+        session: AsyncSession = Depends(get_session),
+        limit: int = Query(50, ge=1, le=100),
+        offset: int = Query(0, ge=0),
+        severity: str | None = Query(None),
+        source_ip: str | None = Query(None),
+        since: datetime | None = Query(None),
+        until: datetime | None = Query(None),
 ) -> ThreatListResponse:
     """
     List threat events with optional filters and pagination.
@@ -42,8 +42,8 @@ async def list_threats(
 
 @router.get("/{threat_id}", response_model=ThreatEventResponse)
 async def get_threat(
-    threat_id: uuid.UUID,
-    session: AsyncSession = Depends(get_session),
+        threat_id: uuid.UUID,
+        session: AsyncSession = Depends(get_session),
 ) -> ThreatEventResponse:
     """
     Fetch a single threat event by ID.

PROJECTS/advanced/ai-threat-detection/backend/app/core/alerts/dispatcher.py

@@ -6,9 +6,13 @@
 import logging
 
 import redis.asyncio as aioredis
-from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+from sqlalchemy.ext.asyncio import (
+    AsyncSession,
+    async_sessionmaker,
+)
 
 from app.core.alerts import ALERTS_CHANNEL
+from app.core.detection.ensemble import classify_severity
 from app.core.ingestion.pipeline import ScoredRequest
 from app.schemas.websocket import WebSocketAlert
 from app.services.threat_service import create_threat_event
@@ -18,11 +22,12 @@
 
 class AlertDispatcher:
     """
-    Routes scored threat events to storage, pub/sub, and structured logging.
+    Routes scored threat events to storage, pub/sub,
+    and structured logging
 
-    MEDIUM+ severity events are persisted to PostgreSQL and published
-    to the Redis alerts channel for WebSocket relay.
-    All events are logged to stdout.
+    MEDIUM+ severity events are persisted to PostgreSQL
+    and published to the Redis alerts channel for
+    WebSocket relay. All events are logged to stdout.
     """
 
     def __init__(
@@ -35,39 +40,49 @@ def __init__(
 
     async def dispatch(self, scored: ScoredRequest) -> None:
         """
-        Handle a scored request from the pipeline's dispatch stage.
+        Handle a scored request from the pipeline's
+        dispatch stage
         """
+        severity = classify_severity(scored.final_score)
+
         logger.info(
-            "threat_event severity=%s score=%.2f ip=%s path=%s rules=%s",
-            scored.rule_result.severity,
-            scored.rule_result.threat_score,
+            "threat_event severity=%s score=%.2f mode=%s ip=%s path=%s rules=%s",
+            severity,
+            scored.final_score,
+            scored.detection_mode,
             scored.entry.ip,
             scored.entry.path,
             scored.rule_result.matched_rules,
         )
 
-        if scored.rule_result.severity in ("HIGH", "MEDIUM"):
+        if severity in ("HIGH", "MEDIUM"):
             await self._store_event(scored)
-            await self._publish_alert(scored)
+            await self._publish_alert(scored, severity)
 
     async def _store_event(self, scored: ScoredRequest) -> None:
         """
-        Persist the scored request as a threat event in PostgreSQL.
+        Persist the scored request as a threat event
+        in PostgreSQL
         """
         async with self._session_factory() as session:
             await create_threat_event(session, scored)
             await session.commit()
 
-    async def _publish_alert(self, scored: ScoredRequest) -> None:
+    async def _publish_alert(
+        self,
+        scored: ScoredRequest,
+        severity: str,
+    ) -> None:
         """
-        Publish a real-time alert to the Redis pub/sub channel.
+        Publish a real-time alert to the Redis pub/sub
+        channel
         """
         alert = WebSocketAlert(
             timestamp=scored.entry.timestamp,
             source_ip=scored.entry.ip,
             request_path=scored.entry.path,
-            threat_score=scored.rule_result.threat_score,
-            severity=scored.rule_result.severity,
+            threat_score=scored.final_score,
+            severity=severity,
             component_scores=scored.rule_result.component_scores,
         )
         await self._redis.publish(ALERTS_CHANNEL, alert.model_dump_json())

PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/ensemble.py

@@ -0,0 +1,66 @@
+"""
+©AngelaMos | 2026
+ensemble.py
+"""
+
+
+def normalize_ae_score(error: float, threshold: float) -> float:
+    """
+    Normalize autoencoder reconstruction error to [0, 1]
+    """
+    if threshold <= 0:
+        return 0.0
+    return min(error / (threshold * 2), 1.0)
+
+
+def normalize_if_score(raw_score: float) -> float:
+    """
+    Normalize isolation forest score to [0, 1]
+
+    sklearn IF returns negative scores for anomalies,
+    positive for normal samples
+    """
+    return (1 - raw_score) / 2.0
+
+
+def fuse_scores(
+    scores: dict[str, float],
+    weights: dict[str, float],
+) -> float:
+    """
+    Weighted average of per-model normalized scores
+    """
+    total = 0.0
+    weight_sum = 0.0
+    for key, weight in weights.items():
+        if key in scores:
+            total += scores[key] * weight
+            weight_sum += weight
+    if weight_sum == 0:
+        return 0.0
+    return total / weight_sum
+
+
+def blend_scores(
+    ml_score: float,
+    rule_score: float,
+    ml_weight: float = 0.7,
+) -> float:
+    """
+    Blend ML ensemble score with rule engine score
+    """
+    return min(
+        ml_score * ml_weight + rule_score * (1.0 - ml_weight),
+        1.0,
+    )
+
+
+def classify_severity(score: float) -> str:
+    """
+    Map a unified threat score to a severity label
+    """
+    if score >= 0.7:
+        return "HIGH"
+    if score >= 0.5:
+        return "MEDIUM"
+    return "LOW"