Skip to content

Commit b752484

Browse files
ddevin96ddevin96
authored
chore: improve integration test skip on network failure (#282)
1 parent 3f50762 commit b752484

6 files changed

Lines changed: 148 additions & 89 deletions

File tree

‎.github/workflows/ci.yaml‎

Lines changed: 78 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ name: CI
22

33
on:
44
push:
5-
branches: [ main ]
5+
branches: [main]
66
pull_request:
77

88
permissions:
@@ -13,125 +13,125 @@ jobs:
1313
code_quality:
1414
runs-on: ubuntu-latest
1515
steps:
16-
- uses: actions/checkout@v6
16+
- uses: actions/checkout@v6
1717

18-
- name: Install uv
19-
uses: astral-sh/setup-uv@v7
20-
with:
21-
python-version: "3.14"
18+
- name: Install uv
19+
uses: astral-sh/setup-uv@v7
20+
with:
21+
python-version: "3.14"
2222

23-
- name: Install dependencies
24-
run: |
25-
make build
23+
- name: Install dependencies
24+
run: |
25+
make build
2626
27-
- name: Check docstrings
28-
continue-on-error: true # TODO: remove when all docstring issues are fixed
29-
run: |
30-
make docstring-check
27+
- name: Check docstrings
28+
continue-on-error: true # TODO: remove when all docstring issues are fixed
29+
run: |
30+
make docstring-check
3131
32-
- name: Type check
33-
run: |
34-
make typecheck
32+
- name: Type check
33+
run: |
34+
make typecheck
3535
36-
- name: Lint code with ruff
37-
run: |
38-
make lint
36+
- name: Lint code with ruff
37+
run: |
38+
make lint
3939
40-
- name: Format code with ruff
41-
run: |
42-
make format
40+
- name: Format code with ruff
41+
run: |
42+
make format
4343
4444
4545
test:
4646
runs-on: ${{ matrix.os }}
4747
strategy:
4848
matrix:
49-
os: [ ubuntu-latest, macos-latest, windows-latest]
50-
python-version: [ "3.10", "3.11", "3.12", "3.13", "3.14" ]
49+
os: [ubuntu-latest, macos-latest, windows-latest]
50+
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
5151

5252
steps:
53-
- uses: actions/checkout@v6
53+
- uses: actions/checkout@v6
5454

55-
- name: Install uv with Python ${{ matrix.python-version }}
56-
uses: astral-sh/setup-uv@v7
57-
with:
58-
python-version: ${{ matrix.python-version }}
55+
- name: Install uv with Python ${{ matrix.python-version }}
56+
uses: astral-sh/setup-uv@v7
57+
with:
58+
python-version: ${{ matrix.python-version }}
5959

60-
- name: Install dependencies
61-
run: |
62-
make build
60+
- name: Install dependencies
61+
run: |
62+
make build
6363
64-
- name: Run tests with pytest
65-
run: |
66-
make test
64+
- name: Run tests with pytest
65+
run: |
66+
make test
6767
6868
test-arm:
6969
runs-on: ${{ matrix.os }}
7070
strategy:
7171
matrix:
7272
os: [ubuntu-24.04-arm, macos-26]
73-
python-version: [ "3.10", "3.11", "3.12", "3.13", "3.14" ]
73+
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
7474

7575
steps:
76-
- uses: actions/checkout@v6
76+
- uses: actions/checkout@v6
7777

78-
- name: Install uv with Python ${{ matrix.python-version }}
79-
uses: astral-sh/setup-uv@v7
80-
with:
81-
python-version: ${{ matrix.python-version }}
78+
- name: Install uv with Python ${{ matrix.python-version }}
79+
uses: astral-sh/setup-uv@v7
80+
with:
81+
python-version: ${{ matrix.python-version }}
8282

83-
- name: Install dependencies
84-
run: |
85-
make build
83+
- name: Install dependencies
84+
run: |
85+
make build
8686
87-
- name: Run tests with pytest
88-
run: |
89-
make test
87+
- name: Run tests with pytest
88+
run: |
89+
make test
9090
9191
test-x86:
9292
runs-on: ${{ matrix.os }}
9393
strategy:
9494
matrix:
95-
os: [ubuntu-24.04, windows-2025]
96-
python-version: [ "3.10", "3.11", "3.12", "3.13", "3.14" ]
95+
os: [ubuntu-24.04, windows-2025-vs2026]
96+
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
9797

9898
steps:
99-
- uses: actions/checkout@v6
99+
- uses: actions/checkout@v6
100100

101-
- name: Install uv with Python ${{ matrix.python-version }}
102-
uses: astral-sh/setup-uv@v7
103-
with:
104-
python-version: ${{ matrix.python-version }}
101+
- name: Install uv with Python ${{ matrix.python-version }}
102+
uses: astral-sh/setup-uv@v7
103+
with:
104+
python-version: ${{ matrix.python-version }}
105105

106-
- name: Install dependencies
107-
run: |
108-
make build
106+
- name: Install dependencies
107+
run: |
108+
make build
109109
110-
- name: Run tests with pytest
111-
run: |
112-
make test
110+
- name: Run tests with pytest
111+
run: |
112+
make test
113113
114114
integration-test:
115115
runs-on: ${{ matrix.os }}
116116
strategy:
117117
matrix:
118-
os: [ ubuntu-latest]
119-
python-version: [ "3.10", "3.14"]
118+
os: [ubuntu-latest]
119+
python-version: ["3.10", "3.14"]
120120

121121
steps:
122-
- uses: actions/checkout@v6
123-
124-
- name: Install uv with Python ${{ matrix.python-version }}
125-
uses: astral-sh/setup-uv@v7
126-
with:
127-
python-version: ${{ matrix.python-version }}
128-
129-
- name: Install dependencies
130-
run: |
131-
make build
132-
133-
- name: Run integration tests with pytest
134-
env:
135-
HF_DOWNLOAD_TOKEN: ${{ secrets.HF_DOWNLOAD_TOKEN }}
136-
run: |
137-
make i-test
122+
- uses: actions/checkout@v6
123+
124+
- name: Install uv with Python ${{ matrix.python-version }}
125+
uses: astral-sh/setup-uv@v7
126+
with:
127+
python-version: ${{ matrix.python-version }}
128+
129+
- name: Install dependencies
130+
run: |
131+
make build
132+
133+
- name: Run integration tests with pytest
134+
env:
135+
HF_DOWNLOAD_TOKEN: ${{ secrets.HF_DOWNLOAD_TOKEN }}
136+
run: |
137+
make i-test

‎.github/workflows/daily_ci.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ jobs:
9191
runs-on: ${{ matrix.os }}
9292
strategy:
9393
matrix:
94-
os: [ubuntu-24.04, windows-2025]
94+
os: [ubuntu-24.04, windows-2025-vs2026]
9595
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
9696

9797
steps:

‎docs/development/policies.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ Our CI relies on [GitHub-hosted runners](https://docs.github.com/en/actions/refe
2222
The current CI matrix tests these runner labels:
2323
- `ubuntu-latest`, `ubuntu-24.04`, and `ubuntu-24.04-arm`.
2424
- `macos-latest` and `macos-26`.
25-
- `windows-latest` and `windows-2025`.
25+
- `windows-latest` and `windows-2025-vs2026`.
2626

2727
We do not support these platforms:
2828
- `macos-26-intel` for incompatibility with PyTorch.
@@ -32,7 +32,7 @@ We do not support these platforms:
3232
|---|---:|---:|---:|---|---|:--:|
3333
| Linux | 1 | 5 GB | 14 GB | x64 | ubuntu-slim | :x: |
3434
| Linux | 4 | 16 GB | 14 GB | x64 | ubuntu-latest, ubuntu-24.04 | :heavy_check_mark: |
35-
| Windows | 4 | 16 GB | 14 GB | x64 | windows-latest, windows-2025 | :heavy_check_mark: |
35+
| Windows | 4 | 16 GB | 14 GB | x64 | windows-latest, windows-2025-vs2026 | :heavy_check_mark: |
3636
| Linux | 4 | 16 GB | 14 GB | arm64 | ubuntu-24.04-arm | :heavy_check_mark: |
3737
| Windows | 4 | 16 GB | 14 GB | arm64 | windows-11-arm | :x: |
3838
| macOS | 4 | 14 GB | 14 GB | Intel | macos-26-intel | :x: |

‎hyperbench/integration_tests/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
from .common import (
2+
is_ci,
3+
warn_ci,
24
common_metrics,
35
loaders,
46
model_configs,
@@ -16,10 +18,12 @@
1618
"common_metrics",
1719
"enrich_datasets",
1820
"extract_state_dict",
21+
"is_ci",
1922
"loaders",
2023
"model_configs",
2124
"model_configs_with_single_model",
2225
"split_dataset",
2326
"train_test_loop",
27+
"warn_ci",
2428
"zero_model_parameters",
2529
]

‎hyperbench/integration_tests/common.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
import lightning as L
2+
23
import torch
4+
import os
5+
import sys
36

47
from collections.abc import Sequence
58
from functools import cache
@@ -288,3 +291,12 @@ def zero_models_parameters(model_configs: Sequence[ModelConfig]) -> None:
288291
"""
289292
for config in model_configs:
290293
zero_model_parameters(config.model)
294+
295+
296+
def is_ci() -> bool:
297+
return os.getenv("GITHUB_ACTIONS") == "true"
298+
299+
300+
def warn_ci(message: str) -> None:
301+
if is_ci():
302+
print(f"::warning::{message}", file=sys.stderr)

‎hyperbench/integration_tests/data/supported_datasets_integration_test.py‎

Lines changed: 51 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,46 @@
77
list_datasets,
88
)
99

10+
from hyperbench.integration_tests.common import warn_ci
1011

11-
RATE_LIMIT_TERMS = ["429", "rate limit", "too many requests"]
12+
13+
NETWORK_ERROR_TERMS = [
14+
"429",
15+
"rate limit",
16+
"too many requests",
17+
"connection error",
18+
"connection refused",
19+
"connection reset",
20+
"network error",
21+
"name resolution",
22+
"remote disconnected",
23+
"service unavailable",
24+
"temporary failure",
25+
"timed out",
26+
"unable to locate the file on the hub",
27+
"unable to find the requested files in the local cache",
28+
"nodename nor servname provided",
29+
"cannot send a request",
30+
"client has been closed",
31+
]
32+
33+
34+
def _exception_chain_text(exception: BaseException) -> str:
35+
parts: list[str] = []
36+
current: BaseException | None = exception
37+
seen_ids: set[int] = set()
38+
39+
while current is not None and id(current) not in seen_ids:
40+
seen_ids.add(id(current))
41+
parts.append(str(current).lower())
42+
current = current.__cause__ or current.__context__
43+
44+
return "\n".join(parts)
45+
46+
47+
def _is_network_download_failure(exception: BaseException) -> bool:
48+
message = _exception_chain_text(exception)
49+
return any(term in message for term in NETWORK_ERROR_TERMS)
1250

1351

1452
@pytest.mark.flaky(reruns=3, reruns_delay=10, rerun_show_tracebacks=True)
@@ -26,7 +64,7 @@ def test_all_supported_datasets_load(dataset_name):
2664
assert len(dataset) > 0
2765

2866

29-
@pytest.mark.flaky(reruns=1, reruns_delay=10 * 60, rerun_show_tracebacks=True)
67+
@pytest.mark.flaky(reruns=3, reruns_delay=5 * 60, rerun_show_tracebacks=True)
3068
@pytest.mark.integration
3169
def test_all_supported_datasets_load_from_hf(request):
3270
datasets = list_datasets()
@@ -46,14 +84,19 @@ def test_all_supported_datasets_load_from_hf(request):
4684
dataset = get_dataset_by_name(dataset_name)
4785
except Exception as e:
4886
message = str(e)
49-
5087
execution_count = getattr(request.node, "execution_count", 1)
51-
max_attempts = request.node.get_closest_marker("flaky").kwargs.get("reruns", 0) + 1
5288

53-
if execution_count == max_attempts and any(
54-
term in message.lower() for term in RATE_LIMIT_TERMS
55-
):
56-
pytest.skip(f"Skipping {dataset_name} due to Hugging Face rate limit: {message}")
89+
if _is_network_download_failure(e):
90+
warn_ci(
91+
f"Skipping 'test_all_supported_datasets_load_from_hf' for "
92+
f"{dataset_name!r} because an upstream download failed due to network issues."
93+
)
94+
95+
if execution_count > 2:
96+
pytest.skip(
97+
f"Skipping {dataset_name!r} due to "
98+
f"Hugging Face network failure: {message!r}"
99+
)
57100

58101
raise
59102

0 commit comments

Comments
 (0)