Release AbstractCore 2.13.29

Author: lpalbou

CHANGELOG.md

@@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.13.29] - 2026-05-26
+
+### Added
+- **Video generation through Core**: added Python `generate(..., output={"task":"text_to_video"|"image_to_video"})` callback forwarding for AbstractVision progress events, plus OpenAI-compatible `/v1/videos/generations`, `/v1/videos/edits`, and async `/v1/vision/jobs/videos/*` routes.
+- **Video job progress**: async video jobs now capture normalized backend progress events in `progress.last_event` while preserving step/frame counters for polling clients.
+
+### Changed
+- **Vision plugin floor**: raised AbstractVision integration requirements to `abstractvision>=0.3.16` so Core installs pick up MLX-Gen 0.18.6, exact model id routing, and text/image-to-video support.
+
+### Fixed
+- **Generated media callback boundary**: top-level progress callbacks supplied to multimodal `generate(...)` calls are attached to generated image/video output specs instead of leaking into the text-provider kwargs path.
+
+### Verified
+- `pytest tests/test_packaging_extras.py tests/test_output_specs.py tests/test_multimodal_generate_output.py tests/server/test_server_vision_image_endpoints.py tests/capabilities/test_vision_catalog_helper.py tests/server/test_server_model_residency_control_plane.py -q`
+
 ## [2.13.28] - 2026-05-26
 
 ### Changed

README.md

@@ -49,9 +49,12 @@ First-class support for:
 (**) Optional visual-text compression: render long text/PDFs into images and process them with a vision model to reduce token usage. See [Glyph Visual-Text Compression](docs/glyphs.md) (install `pip install "abstractcore[compression]"`; for PDFs also install `pip install "abstractcore[media]"`).
 
 Generative vision uses `abstractvision` when installed. In server mode, omit
-`model` only when the server has a configured image default, or use explicit
+`model` only when the server has a configured default, or use explicit
 provider/model ids such as `diffusers/default`, `diffusers/<huggingface-repo>`,
-`sdcpp/default`, or `openai-compatible/<model>`.
+`mlx-gen/AbstractFramework/qwen-image-2512-4bit`,
+`mlx-gen/Wan-AI/Wan2.2-TI2V-5B-Diffusers`, `sdcpp/default`, or
+`openai-compatible/<model>`. Quantized MLX-Gen models are selected by their
+published repo id; Core does not add a separate quant override.
 
 Docs: [Getting Started](docs/getting-started.md) · [FAQ](docs/faq.md) · [Docs Index](docs/README.md) · https://lpalbou.github.io/AbstractCore
 
@@ -337,6 +340,37 @@ png_bytes = image.outputs["image"][0].data
 # Image edit: image media + image output infers image-to-image.
 edited = llm.generate("Make the mug blue.", media="mug.png", output="image")
 
+def progress(event):
+    print("video progress", event)
+
+# Text-to-video via abstractvision. The callback is forwarded to the plugin.
+video = llm.generate(
+    "A red fox walking through a snowy forest, cinematic.",
+    on_progress=progress,
+    output={
+        "modality": "video",
+        "task": "text_to_video",
+        "provider": "mlx-gen",
+        "model": "Wan-AI/Wan2.2-TI2V-5B-Diffusers",
+        "num_frames": 121,
+        "fps": 24,
+    },
+)
+mp4_bytes = video.outputs["video"][0].data
+
+# Image-to-video uses image media plus task="image_to_video".
+i2v = llm.generate(
+    "Slow camera push-in.",
+    media={"type": "image", "path": "first-frame.png", "role": "source"},
+    output={
+        "task": "image_to_video",
+        "provider": "mlx-gen",
+        "model": "Wan-AI/Wan2.2-TI2V-5B-Diffusers",
+        "num_frames": 121,
+        "fps": 24,
+    },
+)
+
 # TTS via abstractvoice.
 speech = llm.generate(text="Hello from AbstractCore.", output="voice")
 wav_bytes = speech.outputs["voice"][0].data
@@ -377,6 +411,11 @@ The HTTP server exposes equivalent discovery at
 `/v1/audio/speech/models`, `/v1/audio/transcriptions/models`, and
 `/v1/voice/clone/providers`, plus `/v1/audio/music/providers` and
 `/v1/audio/music/models`.
+Generated media HTTP routes include `/v1/images/generations`,
+`/v1/images/edits`, `/v1/videos/generations`, `/v1/videos/edits`, and
+async polling routes under `/v1/vision/jobs/images/*` and
+`/v1/vision/jobs/videos/*`; video jobs include the latest backend progress event
+when the selected backend reports it.
 `/v1/models` remains focused on LLM and embedding provider models.
 
 ## HTTP server (OpenAI-compatible gateway)

abstractcore/capabilities/vision_catalog.py

@@ -247,12 +247,8 @@ def _cached_hf_snapshot(repo_id: str, cache_dirs: list[Path]) -> Optional[Path]:
 
 
 def _mlx_gen_selector_for_download(download: Any) -> str:
-    key = str(getattr(download, "key", "") or "").strip()
     repo_id = str(getattr(download, "repo_id", "") or "").strip()
-    bits = getattr(download, "bits", None)
-    if bits == 4 and key:
-        return key
-    return repo_id or key
+    return repo_id
 
 
 def _discover_cached_hf_diffusers_models(cache_dirs: list[Path]) -> list[str]:
@@ -438,7 +434,7 @@ def get_local_vision_cache_catalog() -> Dict[str, Any]:
     for model_id in model_ids:
         spec = registry.get(model_id)
         supported_tasks = sorted(spec.tasks.keys())
-        if "text_to_image" not in spec.tasks and "image_to_image" not in spec.tasks:
+        if not {"text_to_image", "image_to_image", "text_to_video", "image_to_video"}.intersection(spec.tasks):
             continue
 
         for download in list(getattr(spec, "downloads", []) or []):

abstractcore/core/output_specs.py

@@ -11,6 +11,12 @@
     "transcript",
     "transcription",
     "image",
+    "video",
+    "t2v",
+    "i2v",
+    "text_to_video",
+    "image_to_video",
+    "video_generation",
     "voice",
     "speech",
     "tts",
@@ -33,6 +39,12 @@
     "t2i",
     "i2i",
     "image_to_image",
+    "video",
+    "video_generation",
+    "t2v",
+    "i2v",
+    "text_to_video",
+    "image_to_video",
     "voice",
     "speech",
     "tts",
@@ -58,6 +70,12 @@
     "i2i": ("image", "image_edit"),
     "image_to_image": ("image", "image_edit"),
     "image_edit": ("image", "image_edit"),
+    "video": ("video", "video_generation"),
+    "video_generation": ("video", "video_generation"),
+    "t2v": ("video", "text_to_video"),
+    "text_to_video": ("video", "text_to_video"),
+    "i2v": ("video", "image_to_video"),
+    "image_to_video": ("video", "image_to_video"),
     "music": ("music", "music_generation"),
     "song": ("music", "music_generation"),
     "t2m": ("music", "music_generation"),
@@ -74,6 +92,8 @@
     "t2i": "image_generation",
     "i2i": "image_edit",
     "image_to_image": "image_edit",
+    "t2v": "text_to_video",
+    "i2v": "image_to_video",
     "song": "music_generation",
     "t2m": "music_generation",
     "text_to_music": "music_generation",
@@ -85,6 +105,9 @@
     "transcription": "text",
     "image_generation": "image",
     "image_edit": "image",
+    "video_generation": "video",
+    "text_to_video": "video",
+    "image_to_video": "video",
     "tts": "voice",
     "voice_clone": "voice",
     "music_generation": "music",

abstractcore/providers/base.py

@@ -1640,6 +1640,43 @@ def _normalize_output_specs(output: Any) -> List[Dict[str, Any]]:
     def _normalize_output_spec(output: Any) -> Dict[str, Any]:
         return normalize_output_spec(output)
 
+    @classmethod
+    def _output_request_with_progress_callbacks(
+        cls,
+        output: Any,
+        callbacks: Dict[str, Any],
+    ) -> Tuple[Any, bool]:
+        """Attach top-level generated-media progress callbacks to output specs."""
+        effective_callbacks = {
+            str(key): value
+            for key, value in callbacks.items()
+            if value is not None
+        }
+        if not effective_callbacks or not cls._is_acore_output_request(output):
+            return output, False
+
+        specs = cls._normalize_output_specs(output)
+        changed = False
+        patched_specs: List[Dict[str, Any]] = []
+        for spec in specs:
+            patched = dict(spec)
+            modality = str(patched.get("modality") or "").strip().lower()
+            if modality in {"image", "video"}:
+                nested_extra = patched.get("extra")
+                nested_keys = set(nested_extra) if isinstance(nested_extra, dict) else set()
+                for key, value in effective_callbacks.items():
+                    if key in patched or key in nested_keys:
+                        continue
+                    patched[key] = value
+                    changed = True
+            patched_specs.append(patched)
+
+        if not changed:
+            return output, False
+        if isinstance(output, (list, tuple)):
+            return patched_specs, True
+        return patched_specs[0], True
+
     @staticmethod
     def _coerce_media_items(media: Any) -> List[Any]:
         if media is None:
@@ -1948,6 +1985,9 @@ def _run_multimodal_spec(
         if modality == "image":
             self._run_image_output(result=result, spec=spec, prompt=prompt, media=media, artifact_store=artifact_store)
             return
+        if modality == "video":
+            self._run_video_output(result=result, spec=spec, prompt=prompt, media=media, artifact_store=artifact_store)
+            return
         if modality == "voice":
             self._run_voice_output(result=result, spec=spec, prompt=prompt, media=media, artifact_store=artifact_store)
             return
@@ -2046,6 +2086,86 @@ def _run_image_output(
         )
         result.add_output("image", item)
 
+    def _run_video_output(
+        self,
+        *,
+        result: MultimodalGenerateResponse,
+        spec: Dict[str, Any],
+        prompt: str,
+        media: Any,
+        artifact_store: Optional[Any],
+    ) -> None:
+        items = self._coerce_media_items(media)
+        images = [item for item in items if self._media_type(item, fallback="image" if isinstance(item, (bytes, bytearray)) else None) == "image"]
+        roles = [(item, self._media_role(item)) for item in images]
+        source_items = [item for item, role in roles if role == "source"]
+        mask_items = [item for item, role in roles if role == "mask"]
+        reference_like = [item for item, role in roles if role in {"reference", "style", "context"}]
+        unroled = [item for item, role in roles if role is None]
+        task = str(spec.get("task") or "").lower()
+
+        if mask_items:
+            raise ValueError("Video generation does not support mask media in v1.")
+        if len(source_items) > 1:
+            raise ValueError("Image-to-video supports at most one source image in v1.")
+
+        should_i2v = task == "image_to_video"
+        if task == "text_to_video":
+            should_i2v = False
+        elif not should_i2v:
+            if source_items:
+                should_i2v = True
+            elif len(unroled) == 1 and not reference_like:
+                should_i2v = True
+            elif len(unroled) > 1:
+                raise ValueError("Multiple image media items require explicit roles for image-to-video.")
+
+        kwargs = self._output_plugin_kwargs(
+            spec,
+            exclude={"format", "content_type", "mime_type", "provider", "response_format"},
+        )
+        if spec.get("provider") is not None:
+            kwargs["provider"] = spec.get("provider")
+        if artifact_store is not None:
+            kwargs["artifact_store"] = artifact_store
+
+        if should_i2v:
+            source = source_items[0] if source_items else (unroled[0] if unroled else None)
+            if source is None:
+                raise ValueError("Image-to-video requires one source image.")
+            raw = self.vision.i2v(self._media_payload(source), prompt=prompt, **kwargs)
+            task_name = "image_to_video"
+        else:
+            raw = self.vision.t2v(prompt, **kwargs)
+            task_name = "text_to_video"
+
+        data, artifact_ref, metadata = self._artifact_or_data(raw)
+        fmt = str(spec.get("format") or "mp4").strip().lower() or "mp4"
+        content_type = str(metadata.get("content_type") or metadata.get("mime_type") or f"video/{fmt}")
+        if artifact_ref is None:
+            data, stored_ref = self._store_generated_data(
+                data,
+                artifact_store=artifact_store,
+                content_type=content_type,
+                spec=spec,
+            )
+            artifact_ref = stored_ref
+        result.add_output(
+            "video",
+            GeneratedItem(
+                modality="video",
+                task=task_name,
+                data=data,
+                artifact_ref=artifact_ref,
+                content_type=content_type,
+                format=fmt,
+                backend_id=getattr(self.vision, "backend_id", None),
+                provider=str(spec.get("provider") or getattr(self.vision, "backend_id", None) or self.__class__.__name__),
+                model=str(spec.get("model") or self.model),
+                metadata=metadata,
+            ),
+        )
+
     def _run_voice_output(
         self,
         *,
@@ -2298,6 +2418,18 @@ def generate_with_telemetry(self,
 
         if is_acore_output:
             output_request = kwargs.pop("output")
+            output_request, progress_injected = self._output_request_with_progress_callbacks(
+                output_request,
+                {
+                    "on_progress": kwargs.get("on_progress"),
+                    "progress_event_callback": kwargs.get("progress_event_callback"),
+                    "progress_callback": kwargs.get("progress_callback"),
+                },
+            )
+            if progress_injected:
+                kwargs.pop("on_progress", None)
+                kwargs.pop("progress_event_callback", None)
+                kwargs.pop("progress_callback", None)
             artifact_store = kwargs.pop("artifact_store", None)
             partial = bool(kwargs.pop("partial", False))
             return self._generate_multimodal_response(
@@ -5471,6 +5603,18 @@ async def agenerate(self,
 
         if is_acore_output:
             output_request = kwargs.pop("output")
+            output_request, progress_injected = self._output_request_with_progress_callbacks(
+                output_request,
+                {
+                    "on_progress": kwargs.get("on_progress"),
+                    "progress_event_callback": kwargs.get("progress_event_callback"),
+                    "progress_callback": kwargs.get("progress_callback"),
+                },
+            )
+            if progress_injected:
+                kwargs.pop("on_progress", None)
+                kwargs.pop("progress_event_callback", None)
+                kwargs.pop("progress_callback", None)
             response_model = kwargs.pop("response_model", None)
             artifact_store = kwargs.pop("artifact_store", None)
             partial = bool(kwargs.pop("partial", False))