From cf3b5c5db0546e5e537bce5fb44e31b3f6dbbaa6 Mon Sep 17 00:00:00 2001 From: Andrii Novikov Date: Mon, 23 Mar 2026 11:16:58 +0200 Subject: [PATCH 1/3] feat: add ability to substitute media types for attachments --- CONFIGURATION.md | 6 +++++- docs/generated-app-schema.json | 8 ++++++++ src/quickapp/common/staged_base_tool.py | 5 ++++- src/quickapp/common/utils.py | 10 ++++++++++ src/quickapp/config/tools/base.py | 4 ++++ 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 79923660..6058e33f 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -781,6 +781,7 @@ Do not show parameter with value in stage while tool call: |---------------------------|----------|---------------|----------------------------------------------------------------------------------------------------------------------|----------------------------------------------|---------------| | supported_types | No | Array[String] | List of supported attachment MIME types | `*/*`(all), `image/png`, `image/jpeg`, etc.. | `[*/*]` | | propagate_types_to_choice | No | Array[String] | List of supported attachment MIME types that will be shown in main chat (propagated from tool call result to choice) | `*/*`(all), `image/png`, `image/jpeg`, etc.. | `[]` | +| media_type_substitution | No | dict[str, str] | Maps original MIME type to substitute. Key is a original mime_type, value is desired mime type. | `*/*`(all), `image/png`, `image/jpeg`, etc.. | `{}` |
Parameter info configuration JSON sample @@ -794,7 +795,10 @@ Do not show parameter with value in stage while tool call: "image/png", "image/jpeg", "application/vnd.plotly.v1+json" - ] + ], + "media_type_substitution": { + "application/json": "application/vnd.plotly.v1+json" + } } ``` diff --git a/docs/generated-app-schema.json b/docs/generated-app-schema.json index d2300943..503913a6 100644 --- a/docs/generated-app-schema.json +++ b/docs/generated-app-schema.json @@ -61,6 +61,14 @@ }, "title": "Propagate Types To Choice", "type": "array" + }, + "media_type_substitution": { + "additionalProperties": { + "type": "string" + }, + "description": "Maps original attachment types to substitution types for custom visualizers.", + "title": "Media Type Substitution", + "type": "object" } }, "title": "AttachmentConfig", diff --git a/src/quickapp/common/staged_base_tool.py b/src/quickapp/common/staged_base_tool.py index b5472780..a0ec06c6 100644 --- a/src/quickapp/common/staged_base_tool.py +++ b/src/quickapp/common/staged_base_tool.py @@ -14,7 +14,7 @@ from .exceptions import InvalidToolCallParameterException from .perf_timer.perf_timer import PerformanceTimer from .tool_fallback.processor import FallbackProcessor -from .utils import matches_type +from .utils import matches_type, substitute_media_type logger = logging.getLogger(__name__) @@ -114,6 +114,9 @@ async def _run_in_stage_report_success( filtered: list = [] for a in result.attachments: if matches_type(a.type, attachment_cfg.supported_types): + a.type = substitute_media_type( + a.type, attachment_cfg.media_type_substitution + ) filtered.append(a) if matches_type(a.type, attachment_cfg.propagate_types_to_choice): result.propagate_to_choice.append(a) diff --git a/src/quickapp/common/utils.py b/src/quickapp/common/utils.py index 52383c9e..65e003f8 100644 --- a/src/quickapp/common/utils.py +++ b/src/quickapp/common/utils.py @@ -32,6 +32,16 @@ def matches_type(mime_type: str | None, allowed_mime_types: list[str] | None) -> return False +def substitute_media_type( + mime_type: str | None, + mapping: dict[str, str], +) -> str | None: + """Apply media type substitution""" + if mime_type is None: + return None + return mapping.get(mime_type, mime_type) + + def sanitize_toolname(input_str: str) -> str: """ Sanitizes a string to match the pattern ^[a-zA-Z0-9_-]{1,64}$ diff --git a/src/quickapp/config/tools/base.py b/src/quickapp/config/tools/base.py index 0cb938f1..346e8276 100644 --- a/src/quickapp/config/tools/base.py +++ b/src/quickapp/config/tools/base.py @@ -173,6 +173,10 @@ class AttachmentConfig(BaseModel): default_factory=lambda: DEFAULT_PROPAGATE_TO_CHOICE, description="List of attachment types to propagate from stage to choice.", ) + media_type_substitution: dict[str, str] = Field( + default_factory=dict, + description="Maps original attachment types to substitution types for custom visualizers.", + ) @field_validator("supported_types", mode="before") @classmethod From d2f2f12f2b849653658da0813547fed273b15456 Mon Sep 17 00:00:00 2001 From: Andrii Novikov Date: Mon, 23 Mar 2026 15:10:09 +0200 Subject: [PATCH 2/3] test: substitute media type --- .../common/test_staged_base_tool.py | 105 ++++++++++++++++++ .../common/test_substitute_media_type.py | 24 ++++ 2 files changed, 129 insertions(+) create mode 100644 src/tests/unit_tests/common/test_substitute_media_type.py diff --git a/src/tests/unit_tests/common/test_staged_base_tool.py b/src/tests/unit_tests/common/test_staged_base_tool.py index 290cbf28..2433e705 100644 --- a/src/tests/unit_tests/common/test_staged_base_tool.py +++ b/src/tests/unit_tests/common/test_staged_base_tool.py @@ -126,3 +126,108 @@ async def test_propagation_only_for_surviving_attachments(mock_stage_wrapper_fac # so it should NOT appear in propagate_to_choice assert len(result.propagate_to_choice) == 1 assert result.propagate_to_choice[0].type == "image/png" + + +@pytest.mark.asyncio +async def test_media_type_substitution_applied(mock_stage_wrapper_factory): + """Attachments that pass supported_types should have their type substituted + according to media_type_substitution mapping.""" + mock_config = Mock() + mock_config.display = None + mock_config.fallback_configuration = ToolFallbackConfig(display_error_in_stage=True) + mock_config.attachment = AttachmentConfig( + supported_types=["image/*"], + propagate_types_to_choice=["image/*"], + media_type_substitution={"image/png": "image/webp"}, + ) + + attachment = Attachment(type="image/png", title="photo.png", data="img_data") + + result_to_return = CompletionResult( + content="result", + content_type="text/plain", + attachments=[attachment], + ) + + tool = CustomTestStagedBaseTool( + stage_wrapper_builder=mock_stage_wrapper_factory, + tool_config=mock_config, + perf_timer=Mock(), + result_to_return=result_to_return, + ) + + result = await tool.arun("call-1") + + assert len(result.attachments) == 1 + assert result.attachments[0].type == "image/webp" + + +@pytest.mark.asyncio +async def test_media_type_substitution_not_applied_when_no_match(mock_stage_wrapper_factory): + """Attachments whose type is not in the substitution mapping keep their original type.""" + mock_config = Mock() + mock_config.display = None + mock_config.fallback_configuration = ToolFallbackConfig(display_error_in_stage=True) + mock_config.attachment = AttachmentConfig( + supported_types=["image/*"], + propagate_types_to_choice=["image/*"], + media_type_substitution={"image/png": "image/webp"}, + ) + + attachment = Attachment(type="image/jpeg", title="photo.jpg", data="img_data") + + result_to_return = CompletionResult( + content="result", + content_type="text/plain", + attachments=[attachment], + ) + + tool = CustomTestStagedBaseTool( + stage_wrapper_builder=mock_stage_wrapper_factory, + tool_config=mock_config, + perf_timer=Mock(), + result_to_return=result_to_return, + ) + + result = await tool.arun("call-1") + + assert len(result.attachments) == 1 + assert result.attachments[0].type == "image/jpeg" + + +@pytest.mark.asyncio +async def test_propagation_uses_substituted_type(mock_stage_wrapper_factory): + """After substitution, propagate_types_to_choice should match against the NEW type.""" + mock_config = Mock() + mock_config.display = None + mock_config.fallback_configuration = ToolFallbackConfig(display_error_in_stage=True) + mock_config.attachment = AttachmentConfig( + supported_types=["image/*"], + propagate_types_to_choice=["application/custom"], + media_type_substitution={"image/png": "application/custom"}, + ) + + attachment = Attachment(type="image/png", title="chart.png", data="img_data") + + result_to_return = CompletionResult( + content="result", + content_type="text/plain", + attachments=[attachment], + ) + + tool = CustomTestStagedBaseTool( + stage_wrapper_builder=mock_stage_wrapper_factory, + tool_config=mock_config, + perf_timer=Mock(), + result_to_return=result_to_return, + ) + + result = await tool.arun("call-1") + + # The attachment survives (image/png passes supported_types=["image/*"]) + assert len(result.attachments) == 1 + # Its type was substituted + assert result.attachments[0].type == "application/custom" + # Propagation check uses the substituted type, which matches propagate_types_to_choice + assert len(result.propagate_to_choice) == 1 + assert result.propagate_to_choice[0].type == "application/custom" diff --git a/src/tests/unit_tests/common/test_substitute_media_type.py b/src/tests/unit_tests/common/test_substitute_media_type.py new file mode 100644 index 00000000..7cf93f63 --- /dev/null +++ b/src/tests/unit_tests/common/test_substitute_media_type.py @@ -0,0 +1,24 @@ +from quickapp.common.utils import substitute_media_type + + +class TestSubstituteMediaType: + def test_none_mime_type_returns_none(self): + assert substitute_media_type(None, {"image/png": "image/webp"}) is None + + def test_empty_mapping_returns_original(self): + assert substitute_media_type("image/png", {}) == "image/png" + + def test_no_match_returns_original(self): + assert substitute_media_type("image/png", {"text/plain": "text/html"}) == "image/png" + + def test_exact_match_substitutes(self): + assert substitute_media_type("image/png", {"image/png": "image/webp"}) == "image/webp" + + def test_multiple_mappings(self): + mapping = { + "image/png": "image/webp", + "text/plain": "text/html", + } + assert substitute_media_type("image/png", mapping) == "image/webp" + assert substitute_media_type("text/plain", mapping) == "text/html" + assert substitute_media_type("application/pdf", mapping) == "application/pdf" From 08a34bc4ab21d3a27277b9274dbd55da7c8816b9 Mon Sep 17 00:00:00 2001 From: Andrii Novikov Date: Mon, 23 Mar 2026 15:45:40 +0200 Subject: [PATCH 3/3] fix: coerce None media_type_substitution to empty dict in validator --- src/quickapp/config/tools/base.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/quickapp/config/tools/base.py b/src/quickapp/config/tools/base.py index 346e8276..4ca0c1fd 100644 --- a/src/quickapp/config/tools/base.py +++ b/src/quickapp/config/tools/base.py @@ -192,6 +192,13 @@ def coerce_none_propagate_types(cls, v: list[str] | None) -> list[str]: return DEFAULT_PROPAGATE_TO_CHOICE return v + @field_validator("media_type_substitution", mode="before") + @classmethod + def coerce_none_type_substitution(cls, v: dict[str, str] | None) -> dict[str, str]: + if v is None: + return {} + return v + class OpenAiToolConfig( BaseModel,