fix: filter benchmarks with None benchmark_id before validation

Filter out benchmark entries with None/empty benchmark_id to support
conditional env var syntax like ${env.VAR:+value}.

Signed-off-by: Mustafa Elbehery <melbeher@redhat.com>

Author: Elbehery

src/llama_stack/core/configure.py

@@ -200,4 +200,13 @@ def parse_and_maybe_upgrade_config(config_dict: dict[str, Any]) -> StackConfig:
     config_dict["version"] = LLAMA_STACK_RUN_CONFIG_VERSION
 
     processed_config_dict = replace_env_vars(config_dict)
+
+    if (
+        "registered_resources" in processed_config_dict
+        and "benchmarks" in processed_config_dict["registered_resources"]
+    ):
+        processed_config_dict["registered_resources"]["benchmarks"] = [
+            b for b in processed_config_dict["registered_resources"]["benchmarks"] if b.get("benchmark_id")
+        ]
+
     return StackConfig(**cast_image_name_to_string(processed_config_dict))

tests/unit/cli/test_stack_config.py

@@ -312,3 +312,32 @@ def test_providers_flag_generates_config_with_api_keys():
     assert openai_provider["config"], "Provider config should not be empty"
     assert "api_key" in openai_provider["config"], "API key should be in provider config"
     assert "base_url" in openai_provider["config"], "Base URL should be in provider config"
+
+
+def test_parse_and_maybe_upgrade_config_filters_disabled_benchmarks(up_to_date_config):
+    """Test that parse_and_maybe_upgrade_config filters out benchmarks with None benchmark_id.
+
+    This tests the full integration of the fix for conditional env vars like
+    ${env.VAR:+value} that resolve to None.
+    """
+    up_to_date_config["registered_resources"] = {
+        "benchmarks": [
+            {
+                "benchmark_id": None,
+                "dataset_id": "garak",
+                "scoring_functions": ["garak_scoring"],
+                "provider_id": "trustyai_garak_remote",
+            },
+            {
+                "benchmark_id": "valid-benchmark",
+                "dataset_id": "test",
+                "scoring_functions": ["test_scoring"],
+                "provider_id": "test_provider",
+            },
+        ]
+    }
+
+    result = parse_and_maybe_upgrade_config(up_to_date_config)
+
+    assert len(result.registered_resources.benchmarks) == 1
+    assert result.registered_resources.benchmarks[0].benchmark_id == "valid-benchmark"