Fix all mypy type errors and add types-PyYAML

Resolve 36 mypy errors: fix **kwargs typing, add missing defaults
to CheckResult fields, type check registries properly, and add
assert guards for lazy-loaded models.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: saidkaban

pyproject.toml

@@ -46,6 +46,7 @@ dev = [
     "pytest-mock>=3.12",
     "ruff>=0.4",
     "mypy>=1.10",
+    "types-PyYAML>=6.0",
 ]
 all = ["evalmedia[claude,openai,classical,cli]"]
 

src/evalmedia/checks/__init__.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+from typing import Any
+
 from evalmedia.checks.base import BaseCheck, ClassicalCheck, VLMCheck
 from evalmedia.checks.custom import CustomCheck
 
@@ -21,8 +23,8 @@ def _register_image_checks() -> None:
 
     from evalmedia.checks.image import ALL_CHECKS
 
-    for cls in ALL_CHECKS:
-        CHECK_REGISTRY[cls.name] = cls
+    for check_cls in ALL_CHECKS:
+        CHECK_REGISTRY[check_cls.name] = check_cls
 
 
 def register_check(check_class: type[BaseCheck]) -> type[BaseCheck]:
@@ -31,7 +33,7 @@ def register_check(check_class: type[BaseCheck]) -> type[BaseCheck]:
     return check_class
 
 
-def get_check(name: str, **kwargs: object) -> BaseCheck:
+def get_check(name: str, **kwargs: Any) -> BaseCheck:
     """Instantiate a check by its registered name."""
     _register_image_checks()
     if name not in CHECK_REGISTRY:

src/evalmedia/checks/base.py

@@ -42,7 +42,9 @@ async def evaluate(
 
     def run(self, image: ImageInput, prompt: str = "", **kwargs: object) -> CheckResult:
         """Synchronous entry point."""
-        return asyncio.run(self.arun(image, prompt, **kwargs))
+        judge = kwargs.get("judge")
+        resolved = judge if isinstance(judge, Judge) else None
+        return asyncio.run(self.arun(image, prompt, judge=resolved))
 
     async def arun(
         self,

src/evalmedia/checks/image/__init__.py

@@ -1,5 +1,6 @@
 """Image quality checks."""
 
+from evalmedia.checks.base import BaseCheck
 from evalmedia.checks.image.aesthetic_quality import AestheticQuality
 from evalmedia.checks.image.clip_similarity import CLIPSimilarity
 from evalmedia.checks.image.face_artifacts import FaceArtifacts
@@ -10,7 +11,7 @@
 from evalmedia.checks.image.style_consistency import StyleConsistency
 from evalmedia.checks.image.text_legibility import TextLegibility
 
-ALL_CHECKS: list[type] = [
+ALL_CHECKS: list[type[BaseCheck]] = [
     PromptAdherence,
     FaceArtifacts,
     HandArtifacts,
@@ -22,7 +23,7 @@
     ResolutionAdequacy,
 ]
 
-CHECK_REGISTRY: dict[str, type] = {cls.name: cls for cls in ALL_CHECKS}
+CHECK_REGISTRY: dict[str, type[BaseCheck]] = {cls.name: cls for cls in ALL_CHECKS}
 
 __all__ = [
     "ALL_CHECKS",

src/evalmedia/checks/image/clip_similarity.py

@@ -21,9 +21,10 @@ def __init__(
         self,
         model_name: str = "ViT-B-32",
         pretrained: str = "openai",
-        **kwargs: object,
+        threshold: float | None = None,
+        judge: str | None = None,
     ):
-        super().__init__(**kwargs)
+        super().__init__(threshold=threshold, judge=judge)
         self.model_name = model_name
         self.pretrained = pretrained
         self._model = None
@@ -61,6 +62,10 @@ async def evaluate(
 
         import torch
 
+        assert self._preprocess is not None
+        assert self._tokenizer is not None
+        assert self._model is not None
+
         # Preprocess image
         image_input = self._preprocess(image).unsqueeze(0).to(self._device)
 

src/evalmedia/checks/image/image_similarity.py

@@ -32,9 +32,10 @@ def __init__(
         model_name: str = "ViT-B-32",
         pretrained: str = "openai",
         backend: str = "clip",
-        **kwargs: object,
+        threshold: float | None = None,
+        judge: str | None = None,
     ):
-        super().__init__(**kwargs)
+        super().__init__(threshold=threshold, judge=judge)
         self.reference = reference
         self.model_name = model_name
         self.pretrained = pretrained
@@ -91,9 +92,11 @@ def _load_model(self) -> None:
         else:
             self._load_clip()
 
-    def _encode_image(self, img: Image.Image):
+    def _encode_image(self, img: Image.Image):  # type: ignore[no-untyped-def]
         import torch
 
+        assert self._preprocess is not None
+        assert self._model is not None
         image_input = self._preprocess(img).unsqueeze(0).to(self._device)
         with torch.no_grad():
             if self.backend == "dinov2":
@@ -114,6 +117,8 @@ async def evaluate(
             return CheckResult(
                 name=self.name,
                 status=CheckStatus.SKIPPED,
+                score=None,
+                confidence=None,
                 reasoning=(
                     "No reference image provided. "
                     "Provide a reference via ImageSimilarity(reference=...)."

src/evalmedia/checks/image/resolution_adequacy.py

@@ -31,9 +31,10 @@ def __init__(
         min_width: int = 512,
         min_height: int = 512,
         target: str | None = None,
-        **kwargs: object,
+        threshold: float | None = None,
+        judge: str | None = None,
     ):
-        super().__init__(**kwargs)
+        super().__init__(threshold=threshold, judge=judge)
         if target and target in self.TARGETS:
             self.min_width, self.min_height = self.TARGETS[target]
         else:

src/evalmedia/checks/image/style_consistency.py

@@ -46,9 +46,10 @@ class StyleConsistency(VLMCheck):
     def __init__(
         self,
         reference: ImageInput | None = None,
-        **kwargs: object,
+        threshold: float | None = None,
+        judge: str | None = None,
     ):
-        super().__init__(**kwargs)
+        super().__init__(threshold=threshold, judge=judge)
         self.reference = reference
 
     def get_check_prompt(self, prompt: str, **kwargs: object) -> str:
@@ -65,6 +66,8 @@ async def evaluate(
             return CheckResult(
                 name=self.name,
                 status=CheckStatus.SKIPPED,
+                score=None,
+                confidence=None,
                 reasoning=(
                     "No reference image provided. "
                     "Provide a reference via "

src/evalmedia/core.py

@@ -23,8 +23,8 @@ class CheckResult(BaseModel):
     name: str
     status: CheckStatus
     passed: bool | None = None
-    score: float | None = Field(None, ge=0.0, le=1.0)
-    confidence: float | None = Field(None, ge=0.0, le=1.0)
+    score: float | None = Field(default=None, ge=0.0, le=1.0)
+    confidence: float | None = Field(default=None, ge=0.0, le=1.0)
     reasoning: str = ""
     metadata: dict[str, Any] = Field(default_factory=dict)
     threshold: float = 0.5

src/evalmedia/eval.py

@@ -4,6 +4,7 @@
 
 import asyncio
 import time
+from collections.abc import Sequence
 from typing import TYPE_CHECKING
 
 from PIL import Image
@@ -32,6 +33,8 @@ async def _run_check_safe(
         return CheckResult(
             name=check.name,
             status=CheckStatus.ERROR,
+            score=None,
+            confidence=None,
             error=str(e),
             reasoning=f"Check failed with error: {e}",
         )
@@ -121,7 +124,7 @@ async def arun(
 
 
 async def compare(
-    images: list[ImageInput],
+    images: Sequence[ImageInput],
     prompt: str,
     checks: list[BaseCheck] | None = None,
     rubric: Rubric | None = None,