ci: add no extra flag (#653)

* ci: refactor extra grouping by adding no-extra flag

* ci: add default markers

Author: begumcig

.github/workflows/tests.yaml

@@ -70,7 +70,7 @@ jobs:
         include:
         - name: base
           extras: ""
-          mark_filter: "cpu and not slow and not style and not requires_intel and not requires_lmharness and not requires_rapidata"
+          mark_filter: "no_extras"
         - name: lmharness
           extras: "--extra lmharness"
           mark_filter: "requires_lmharness"
@@ -83,6 +83,7 @@ jobs:
       HF_HOME: ${{ github.workspace }}/.cache/huggingface
       HF_DATASETS_CACHE: ${{ github.workspace }}/.cache/huggingface/datasets
       HUGGINGFACE_HUB_CACHE: ${{ github.workspace }}/.cache/huggingface/hub
+      DEFAULT_MARK_FILTER: "cpu and not slow and not style"
 
     steps:
       - name: Checkout code
@@ -138,4 +139,4 @@ jobs:
       - name: Run tests with pytest-rerunfailures
         run: |
           echo "Running tests with up to 3 reruns on failure using $PYTEST_WORKERS workers..."
-          uv run pytest -n $PYTEST_WORKERS -m "${{ matrix.mark_filter }}" --reruns 3 --reruns-delay 10 --maxfail=1
+          uv run pytest -n $PYTEST_WORKERS -m "${{ env.DEFAULT_MARK_FILTER }} and ${{ matrix.mark_filter }}" --reruns 3 --reruns-delay 10 --maxfail=1

tests/conftest.py

@@ -5,40 +5,53 @@
 # import all fixtures to make them avaliable for pytest
 from .fixtures import *  # noqa: F403, F401
 
-HARDWARE_MARKS = {"cpu", "cuda", "multi_gpu"}
+DEVICE_MARKS = {
+    "cpu": "mark test to run on CPU",
+    "cuda": "mark test to run only on GPU machines",
+    "multi_gpu": "mark test to run only on multi-GPU machines",
+}
+EXTRA_MARKS = {
+    "requires_gptq": "mark test that needs pruna[gptq]",
+    "requires_awq": "mark test that needs pruna[awq]",
+    "requires_stable_fast": "mark test that needs pruna[stable-fast]",
+    "requires_vllm": "mark test that needs pruna[vllm]",
+    "requires_intel": "mark test that needs pruna[intel]",
+    "requires_lmharness": "mark test that needs pruna[lmharness]",
+    "requires_whisper": "mark test that needs pruna[whisper]",
+    "requires_upscale": "mark test that needs pruna[upscale]",
+    "requires_rapidata": "mark test that needs pruna[rapidata]",
+}
 
 
 def pytest_configure(config: Any) -> None:
     """Configure the pytest markers."""
-    # Hardware marks
-    config.addinivalue_line("markers", "cpu: mark test to run on CPU")
-    config.addinivalue_line("markers", "cuda: mark test to run only on GPU machines")
-    config.addinivalue_line("markers", "multi_gpu: mark test to run only on multi-GPU machines")
-    config.addinivalue_line("markers", "high_gpu: mark test to run only on large GPUs")  # e.g. H100
+    # Device marks
+    for mark, description in DEVICE_MARKS.items():
+        config.addinivalue_line("markers", f"{mark}: {description}")
+    config.addinivalue_line("markers", "high_gpu: mark test to run only on large GPUs")
     # Dependency marks for external dependencies
-    config.addinivalue_line("markers", "requires_gptq: mark test that needs pruna[gptq]")
-    config.addinivalue_line("markers", "requires_awq: mark test that needs pruna[awq]")
-    config.addinivalue_line("markers", "requires_stable_fast: mark test that needs pruna[stable-fast]")
-    config.addinivalue_line("markers", "requires_vllm: mark test that needs pruna[vllm]")
-    config.addinivalue_line("markers", "requires_intel: mark test that needs pruna[intel]")
-    config.addinivalue_line("markers", "requires_lmharness: mark test that needs pruna[lmharness]")
-    config.addinivalue_line("markers", "requires_whisper: mark test that needs pruna[whisper]")
-    config.addinivalue_line("markers", "requires_upscale: mark test that needs pruna[upscale]")
-    config.addinivalue_line("markers", "requires_rapidata: mark test that needs pruna[rapidata]")
+    for mark, description in EXTRA_MARKS.items():
+        config.addinivalue_line("markers", f"{mark}: {description}")
+    config.addinivalue_line("markers", "no_extras: mark test that runs without optional dependency extras")
     # Category marks
     config.addinivalue_line("markers", "slow: mark test that run rather long")
     config.addinivalue_line("markers", "style: mark test that only check style")
     config.addinivalue_line("markers", "integration: mark test that is an integration test")
 
 
+@pytest.hookimpl(tryfirst=True)
 def pytest_collection_modifyitems(session: Any, config: Any, items: list) -> None:
     """Hook that is called after test collection."""
     selected = []
     deselected = []
     for item in items:
         # Auto-tag unmarked tests as CPU
-        if not any(mark in item.keywords for mark in HARDWARE_MARKS):
+        if not any(mark in item.keywords for mark in DEVICE_MARKS):
             item.add_marker(pytest.mark.cpu)
+        # Auto-tag tests that do not require optional dependency extras. This
+        # keeps the default CI selection positive as new extras are added.
+        if not any(mark.name in EXTRA_MARKS for mark in item.iter_markers()):
+            item.add_marker(pytest.mark.no_extras)
         # device_parametrized generates cpu/cuda/accelerate variants for every
         # algorithm test, even when the algorithm's runs_on excludes that device.
         # The incompatible variants get collected by the CI (e.g. -m "cpu") and