[Multimodal] Expose mm hash algothrim selection to cli args (vllm-project#49686)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>

Author: Isotr0py

docs/usage/security.md

@@ -396,7 +396,7 @@ FIPS compliance depends on many factors, so a vLLM deployment is not automatical
 
 Operators running vLLM on FIPS-enabled hosts should select FIPS-approved algorithms via the following knobs:
 
-- **Multimodal input hashing** — `VLLM_MM_HASHER_ALGORITHM` defaults to `blake3`, which is not FIPS-approved. Set it to `sha256` or `sha512` in FIPS-enabled environments.
+- **Multimodal input hashing** — `--mm-hasher-algorithm` (config field `mm_hasher_algorithm`) defaults to `blake3`, which is not FIPS-approved. Set it to `sha256` or `sha512` in FIPS-enabled environments.
 - **Prefix-cache hashing** — set `--prefix-caching-hash-algo` (config field `prefix_caching_hash_algo`) to `sha256` or `sha256_cbor`. The `xxhash` and `xxhash_cbor` options are not FIPS-approved.
 - **TLS ciphers** — use `--ssl-ciphers` to restrict the API server's TLS handshake to FIPS-approved cipher suites that match your environment's policy.
 
@@ -408,7 +408,13 @@ vLLM uses MD5 in a few places to derive non-security cache keys (for example, co
 
 Some dependencies expose hash implementations that are not FIPS-approved. vLLM only invokes them when the corresponding algorithm is selected, but operators with strict cryptographic controls may want to ensure the code paths are not exercised — and, where policy requires, that the packages themselves are absent:
 
-- `blake3` — currently listed in `requirements/common.txt`, so a standard install pulls it in. It is imported lazily and only used when `VLLM_MM_HASHER_ALGORITHM=blake3` (the default). Setting `VLLM_MM_HASHER_ALGORITHM` to `sha256` or `sha512` is sufficient to keep the non-FIPS code path dormant. If your policy additionally forbids the package being present, uninstall it after `pip install` (`pip uninstall blake3`); vLLM will continue to function as long as `VLLM_MM_HASHER_ALGORITHM` is set to a non-blake3 value.
+- `blake3` — currently listed in `requirements/common.txt`, so a standard
+  install pulls it in. It is imported lazily and only used when
+  `mm_hasher_algorithm=blake3` (the default). Setting
+  `--mm-hasher-algorithm sha256` or `--mm-hasher-algorithm sha512` is sufficient
+  to keep the non-FIPS code path dormant. If your policy additionally forbids
+  the package being present, uninstall it after installation; vLLM will
+  continue to function as long as a non-blake3 algorithm is selected.
 - `xxhash` — a true optional dependency (not in `requirements/common.txt`). It is only imported when an `xxhash`-based prefix-cache algorithm is selected. Leave it uninstalled and select a `sha256`-based prefix-cache algorithm.
 
 ### Beyond hashing: other FIPS considerations

tests/config/test_multimodal_config.py

@@ -24,6 +24,11 @@ def test_mm_encoder_attn_backend_invalid():
         MultiModalConfig(mm_encoder_attn_backend="not_a_backend")  # type: ignore[arg-type]
 
 
+def test_mm_hasher_algorithm_invalid():
+    with pytest.raises(ValueError, match="mm_hasher_algorithm"):
+        MultiModalConfig(mm_hasher_algorithm="md5")  # type: ignore[arg-type]
+
+
 def test_mm_encoder_attn_backend_hash_updates():
     base_hash = MultiModalConfig().compute_hash()
     overridden_hash = MultiModalConfig(

tests/models/multimodal/processing/test_moss_audio.py

@@ -50,6 +50,7 @@ def batch_decode(self, batch_token_ids, **kwargs):
 class _MMConfig:
     enable_mm_embeds = False
     mm_processor_cache_gb = 1
+    mm_hasher_algorithm = "blake3"
 
     def merge_mm_processor_kwargs(self, kwargs):
         return dict(kwargs)

tests/multimodal/test_cache.py

@@ -144,7 +144,8 @@ def _compare_caches(
         for _ in range(int(item_capacity / hit_rate))
     ]
     all_hashes = [
-        MultiModalHasher.hash_kwargs(item=item.get_data()) for item in all_items
+        MultiModalHasher.hash_kwargs("blake3", item=item.get_data())
+        for item in all_items
     ]
 
     prompt_update = PromptInsertion("dummy", "target", "insertion").resolve(0)

tests/multimodal/test_hasher.py

@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import hashlib
 import uuid
 from io import BytesIO
 from pathlib import Path
@@ -9,6 +10,7 @@
 import torch
 from PIL import Image, ImageDraw
 
+from vllm.config.multimodal import MMHasherAlgorithm
 from vllm.multimodal.hasher import MultiModalHasher
 from vllm.multimodal.media.base import MediaWithBytes
 from vllm.multimodal.media.image import ImageMediaIO
@@ -20,20 +22,36 @@
 assert ASSETS_DIR.exists()
 
 
+@pytest.mark.parametrize("algorithm", ["sha256", "sha512"])
+def test_hash_algorithm(algorithm: MMHasherAlgorithm):
+    hasher = getattr(hashlib, algorithm)()
+    for bytes_ in MultiModalHasher.iter_item_to_bytes("value", "test"):
+        hasher.update(bytes_)
+
+    assert MultiModalHasher.hash_kwargs(algorithm, value="test") == hasher.hexdigest()
+
+
+def test_hash_algorithm_required():
+    with pytest.raises(TypeError, match="algorithm"):
+        MultiModalHasher.hash_kwargs(value="test")  # type: ignore[call-arg]
+
+
 def test_hash_single_item_different_shape():
     x1 = torch.zeros(())
     x2 = torch.zeros((1,))
 
     hasher = MultiModalHasher
-    assert hasher.hash_kwargs(x=x1) != hasher.hash_kwargs(x=x2)
+    assert hasher.hash_kwargs("blake3", x=x1) != hasher.hash_kwargs("blake3", x=x2)
 
 
 def test_hash_key_order_invariant():
     x = torch.zeros((5, 10))
     y = torch.ones((5, 10))
 
     hasher = MultiModalHasher
-    assert hasher.hash_kwargs(x=x, y=y) == hasher.hash_kwargs(y=y, x=x)
+    assert hasher.hash_kwargs("blake3", x=x, y=y) == hasher.hash_kwargs(
+        "blake3", y=y, x=x
+    )
 
 
 # NOTE: Images that are the same visually are allowed to have the same hash
@@ -44,7 +62,9 @@ def test_hash_collision_image_mode(mode_pair):
     image2 = Image.new(mode2, size=(10, 10), color=1)
 
     hasher = MultiModalHasher
-    assert hasher.hash_kwargs(image=image1) != hasher.hash_kwargs(image=image2)
+    assert hasher.hash_kwargs("blake3", image=image1) != hasher.hash_kwargs(
+        "blake3", image=image2
+    )
 
 
 def test_hash_collision_image_palette():
@@ -53,7 +73,9 @@ def test_hash_collision_image_palette():
     image2 = Image.open(ASSETS_DIR / "image2.png")
 
     hasher = MultiModalHasher
-    assert hasher.hash_kwargs(image=image1) != hasher.hash_kwargs(image=image2)
+    assert hasher.hash_kwargs("blake3", image=image1) != hasher.hash_kwargs(
+        "blake3", image=image2
+    )
 
 
 def test_hash_collision_image_transpose():
@@ -64,7 +86,9 @@ def test_hash_collision_image_transpose():
     ImageDraw.Draw(image2).line([(0, 0), (0, 10)])
 
     hasher = MultiModalHasher
-    assert hasher.hash_kwargs(image=image1) != hasher.hash_kwargs(image=image2)
+    assert hasher.hash_kwargs("blake3", image=image1) != hasher.hash_kwargs(
+        "blake3", image=image2
+    )
 
 
 @pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
@@ -74,7 +98,9 @@ def test_hash_collision_tensor_shape(dtype):
     arr2 = torch.zeros((10, 20, 5, 3), dtype=dtype)
 
     hasher = MultiModalHasher
-    assert hasher.hash_kwargs(data=arr1) != hasher.hash_kwargs(data=arr2)
+    assert hasher.hash_kwargs("blake3", data=arr1) != hasher.hash_kwargs(
+        "blake3", data=arr2
+    )
 
 
 def test_hash_collision_array_shape():
@@ -83,7 +109,9 @@ def test_hash_collision_array_shape():
     arr2 = np.zeros((10, 20, 5, 3))
 
     hasher = MultiModalHasher
-    assert hasher.hash_kwargs(data=arr1) != hasher.hash_kwargs(data=arr2)
+    assert hasher.hash_kwargs("blake3", data=arr1) != hasher.hash_kwargs(
+        "blake3", data=arr2
+    )
 
 
 def test_hash_collision_video_num_frames():
@@ -104,8 +132,8 @@ def item_for_hash(num_frames: int):
         return items.get_all_items_for_hash()[0]
 
     hasher = MultiModalHasher
-    assert hasher.hash_kwargs(video=item_for_hash(2)) != hasher.hash_kwargs(
-        video=item_for_hash(4)
+    assert hasher.hash_kwargs("blake3", video=item_for_hash(2)) != hasher.hash_kwargs(
+        "blake3", video=item_for_hash(4)
     )
 
 
@@ -118,7 +146,9 @@ def test_hash_non_contiguous_array():
 
     hasher = MultiModalHasher
     # Both should be hashable and produce the same hashes
-    assert hasher.hash_kwargs(data=arr) == hasher.hash_kwargs(data=arr_c)
+    assert hasher.hash_kwargs("blake3", data=arr) == hasher.hash_kwargs(
+        "blake3", data=arr_c
+    )
 
 
 def test_hash_image_exif_id():
@@ -133,9 +163,13 @@ def test_hash_image_exif_id():
 
     hasher = MultiModalHasher
     # first image has UUID in ImageID, so it should hash to that UUID
-    assert hasher.hash_kwargs(image=image1) == hasher.hash_kwargs(image=id.bytes)
+    assert hasher.hash_kwargs("blake3", image=image1) == hasher.hash_kwargs(
+        "blake3", image=id.bytes
+    )
     # second image has non-UUID in ImageID, so it should hash to the image data
-    assert hasher.hash_kwargs(image=image2) == hasher.hash_kwargs(image=image2a)
+    assert hasher.hash_kwargs("blake3", image=image2) == hasher.hash_kwargs(
+        "blake3", image=image2a
+    )
 
 
 def _rgba_png_bytes() -> bytes:
@@ -153,9 +187,15 @@ def test_hash_collision_media_io_config():
     keep = ImageMediaIO(image_mode=None).load_bytes(data)
 
     hasher = MultiModalHasher
-    assert hasher.hash_kwargs(image=white) != hasher.hash_kwargs(image=black)
-    assert hasher.hash_kwargs(image=white) != hasher.hash_kwargs(image=keep)
-    assert hasher.hash_kwargs(image=white) == hasher.hash_kwargs(image=white2)
+    assert hasher.hash_kwargs("blake3", image=white) != hasher.hash_kwargs(
+        "blake3", image=black
+    )
+    assert hasher.hash_kwargs("blake3", image=white) != hasher.hash_kwargs(
+        "blake3", image=keep
+    )
+    assert hasher.hash_kwargs("blake3", image=white) == hasher.hash_kwargs(
+        "blake3", image=white2
+    )
 
 
 def test_hash_media_io_noop_config_preserves_hash():
@@ -169,4 +209,6 @@ def test_hash_media_io_noop_config_preserves_hash():
 
     plain = MediaWithBytes(loaded.media, data)
     hasher = MultiModalHasher
-    assert hasher.hash_kwargs(image=loaded) == hasher.hash_kwargs(image=plain)
+    assert hasher.hash_kwargs("blake3", image=loaded) == hasher.hash_kwargs(
+        "blake3", image=plain
+    )

tests/multimodal/test_utils.py

@@ -199,6 +199,31 @@ def test_group_and_batch_mm_items_split_by_fieldset():
     assert [num_items for num_items, _ in res] == [2, 1, 1, 1]
 
 
+def test_group_and_batch_mm_items_splits_shared_data_by_dtype():
+    elem1 = MultiModalFieldElem(
+        data=torch.zeros(1, dtype=torch.int32),
+        field=MultiModalSharedField(batch_size=1),
+    )
+    elem2 = MultiModalFieldElem(
+        data=torch.zeros(1, dtype=torch.float32),
+        field=MultiModalSharedField(batch_size=1),
+    )
+    elem3 = MultiModalFieldElem(
+        data=[torch.zeros(1, dtype=torch.int32), torch.zeros(1, dtype=torch.float32)],
+        field=MultiModalSharedField(batch_size=1),
+    )
+
+    res = group_and_batch_mm_items(
+        [
+            MultiModalKwargsItem({"x": elem1}),
+            MultiModalKwargsItem({"x": elem2}),
+            MultiModalKwargsItem({"x": elem3}),
+        ]
+    )
+
+    assert [num_items for num_items, _ in res] == [1, 1, 1]
+
+
 def test_group_and_batch_mm_items_split_by_shared_data():
     elem1 = MultiModalFieldElem(
         data=torch.zeros(1, dtype=torch.uint8),

vllm/config/model.py

@@ -17,6 +17,7 @@
 from vllm.config.multimodal import (
     MMCacheType,
     MMEncoderTPMode,
+    MMHasherAlgorithm,
     MMTensorIPC,
     MultiModalConfig,
 )
@@ -374,6 +375,7 @@ class ModelConfig:
     mm_processor_kwargs: InitVar[dict[str, Any] | None] = None
     mm_processor_cache_gb: InitVar[float | None] = None
     mm_processor_cache_type: InitVar[MMCacheType | None] = None
+    mm_hasher_algorithm: InitVar[MMHasherAlgorithm | None] = None
     mm_shm_cache_max_object_size_mb: InitVar[int | None] = None
     mm_encoder_only: InitVar[bool | None] = None
     mm_encoder_tp_mode: InitVar[MMEncoderTPMode | None] = None
@@ -503,6 +505,7 @@ def __post_init__(
         mm_processor_kwargs: dict[str, Any] | None,
         mm_processor_cache_gb: float | None,
         mm_processor_cache_type: MMCacheType | None,
+        mm_hasher_algorithm: MMHasherAlgorithm | None,
         mm_shm_cache_max_object_size_mb: int | None,
         mm_encoder_only: bool | None,
         mm_encoder_tp_mode: MMEncoderTPMode | None,
@@ -737,6 +740,7 @@ def __post_init__(
                 mm_processor_kwargs=mm_processor_kwargs,
                 mm_processor_cache_gb=mm_processor_cache_gb,
                 mm_processor_cache_type=mm_processor_cache_type,
+                mm_hasher_algorithm=mm_hasher_algorithm,
                 mm_shm_cache_max_object_size_mb=mm_shm_cache_max_object_size_mb,
                 mm_encoder_only=mm_encoder_only,
                 mm_encoder_tp_mode=mm_encoder_tp_mode,

vllm/config/multimodal.py

@@ -3,13 +3,13 @@
 
 from collections.abc import Mapping
 from pathlib import Path
-from typing import Any, Literal, TypeAlias, TypedDict, final
+from typing import Any, Literal, TypeAlias, TypedDict, cast, final
 
 from pydantic import ConfigDict, Field, field_validator, model_validator
 from pydantic.dataclasses import dataclass
 
 import vllm.envs as envs
-from vllm.config.utils import config
+from vllm.config.utils import config, get_from_deprecated_env_if_set
 from vllm.utils.hashing import safe_hash
 from vllm.v1.attention.backends.registry import AttentionBackendEnum
 
@@ -63,6 +63,19 @@ class MultiModalDummyOptionsBuiltins(TypedDict, total=False):
 MMCacheType = Literal["shm", "lru"]
 VideoPruningMethod = Literal["evs", "vidcom2"]
 MMTensorIPC = Literal["direct_rpc", "torch_shm"]
+MMHasherAlgorithm = Literal["blake3", "sha256", "sha512"]
+
+
+def _get_mm_hasher_algorithm() -> MMHasherAlgorithm:
+    env_value = get_from_deprecated_env_if_set(
+        "VLLM_MM_HASHER_ALGORITHM",
+        "v0.27",
+        "mm_hasher_algorithm",
+    )
+    env_value = "blake3" if env_value is None else env_value
+    return cast(MMHasherAlgorithm, env_value.lower())
+
+
 MMDummyOptions: TypeAlias = dict[str, BaseDummyOptions]
 """
 A dictionary containing an entry for each modality type of dummy data.
@@ -134,6 +147,11 @@ class MultiModalConfig:
     mm_processor_cache_type: MMCacheType = "lru"
     """Type of cache to use for the multi-modal preprocessor/mapper. If `shm`,
     use shared memory FIFO cache. If `lru`, use mirrored LRU cache."""
+    mm_hasher_algorithm: MMHasherAlgorithm = Field(
+        default_factory=_get_mm_hasher_algorithm
+    )
+    """Hash algorithm to use for multi-modal input caching. Use `"sha256"` or
+    `"sha512"` for FIPS-compliant deployments."""
     mm_shm_cache_max_object_size_mb: int = Field(default=128, ge=0)
     """Size limit (in MiB) for each object stored in the multi-modal processor
     shared memory cache. Only effective when `mm_processor_cache_type` is

vllm/engine/arg_utils.py

@@ -86,7 +86,12 @@
     RunnerOption,
     TokenizerMode,
 )
-from vllm.config.multimodal import MMCacheType, MMEncoderTPMode, MMTensorIPC
+from vllm.config.multimodal import (
+    MMCacheType,
+    MMEncoderTPMode,
+    MMHasherAlgorithm,
+    MMTensorIPC,
+)
 from vllm.config.observability import DetailedTraceModules
 from vllm.config.parallel import (
     All2AllBackend,
@@ -566,6 +571,9 @@ class EngineArgs:
     mm_processor_cache_type: MMCacheType | None = (
         MultiModalConfig.mm_processor_cache_type
     )
+    mm_hasher_algorithm: MMHasherAlgorithm = get_field(
+        MultiModalConfig, "mm_hasher_algorithm"
+    )
     mm_shm_cache_max_object_size_mb: int = (
         MultiModalConfig.mm_shm_cache_max_object_size_mb
     )
@@ -1295,6 +1303,9 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
         multimodal_group.add_argument(
             "--mm-processor-cache-type", **multimodal_kwargs["mm_processor_cache_type"]
         )
+        multimodal_group.add_argument(
+            "--mm-hasher-algorithm", **multimodal_kwargs["mm_hasher_algorithm"]
+        )
         multimodal_group.add_argument(
             "--mm-shm-cache-max-object-size-mb",
             **multimodal_kwargs["mm_shm_cache_max_object_size_mb"],
@@ -1712,6 +1723,7 @@ def create_model_config(self) -> ModelConfig:
             mm_processor_cache_gb=self.mm_processor_cache_gb,
             mm_processor_cache_type=self.mm_processor_cache_type,
             mm_shm_cache_max_object_size_mb=self.mm_shm_cache_max_object_size_mb,
+            mm_hasher_algorithm=self.mm_hasher_algorithm,
             mm_encoder_only=self.mm_encoder_only,
             mm_encoder_tp_mode=self.mm_encoder_tp_mode,
             mm_encoder_attn_backend=self.mm_encoder_attn_backend,

vllm/model_executor/models/terratorch.py

@@ -217,7 +217,10 @@ def apply(
         )
 
         with timing_ctx.record("get_mm_hashes"):
-            mm_hashes = inputs.get_mm_hashes(self.info.model_id)
+            mm_hashes = inputs.get_mm_hashes(
+                self.info.model_id,
+                self.info.ctx.get_mm_config().mm_hasher_algorithm,
+            )
 
         mm_placeholders = {"image": [PlaceholderRange(offset=0, length=0)]}