fix: file exists on consecutive metric evaluations (#699) #1773
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Code Quality & Tests | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| concurrency: | |
| group: ci-${{ github.repository }}-tests-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| linting: | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| success: ${{ steps.lint-check.outputs.success }} | |
| strategy: | |
| matrix: | |
| python-version: ["3.11"] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-uv-project | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Get changed Python files | |
| id: changed-files | |
| uses: tj-actions/changed-files@v47 | |
| with: | |
| files: | | |
| **/*.py | |
| files_ignore: | | |
| tests/** | |
| - name: Run ruff on changed Pruna code | |
| if: steps.changed-files.outputs.any_changed == 'true' | |
| uses: astral-sh/ruff-action@v3 | |
| with: | |
| version: "latest" | |
| src: ${{ steps.changed-files.outputs.all_changed_files }} | |
| - name: Run Ty type checker on Pruna code | |
| run: uv run ty check src/pruna | |
| - name: Run docstring checks with pytest | |
| run: uv run pytest -m "style" | |
| - name: Set lint check output | |
| id: lint-check | |
| run: echo "success=true" >> $GITHUB_OUTPUT | |
| test: | |
| needs: linting | |
| runs-on: pruna-cpu | |
| strategy: | |
| matrix: | |
| python-version: ["3.11"] | |
| name: ["base", 'lmharness', 'rapidata', 'llamacpp'] | |
| include: | |
| - name: base | |
| extras: "dev" | |
| mark_filter: "no_extras" | |
| - name: lmharness | |
| extras: "dev,lmharness" | |
| mark_filter: "requires_lmharness" | |
| - name: rapidata | |
| extras: "dev,rapidata" | |
| mark_filter: "requires_rapidata" | |
| - name: llamacpp | |
| extras: "dev,llamacpp" | |
| mark_filter: "requires_llamacpp" | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| HF_HOME: ${{ github.workspace }}/.cache/huggingface | |
| HF_DATASETS_CACHE: ${{ github.workspace }}/.cache/huggingface/datasets | |
| HUGGINGFACE_HUB_CACHE: ${{ github.workspace }}/.cache/huggingface/hub | |
| DEFAULT_MARK_FILTER: "cpu and not slow and not style" | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-uv-project | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| extras: ${{ matrix.extras }} | |
| - name: Rebuild llama-cpp-python for runner CPU | |
| if: matrix.name == 'llamacpp' | |
| env: | |
| FORCE_CMAKE: "1" | |
| # Prebuilt wheels is using code not compatible with cpu runners. | |
| # Build from source against the runner's actual CPU feature set. | |
| CMAKE_ARGS: "-DGGML_NATIVE=ON" | |
| run: | | |
| uv pip install --no-cache --reinstall-package llama-cpp-python --no-binary llama-cpp-python "llama-cpp-python>=0.2.78" | |
| - name: Cache Hugging Face datasets and models | |
| uses: actions/cache@v5 | |
| with: | |
| path: | | |
| .cache/huggingface/datasets | |
| .cache/huggingface/hub | |
| .cache/huggingface/models | |
| key: hf-cache-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }} | |
| restore-keys: | | |
| hf-cache-${{ runner.os }}-${{ matrix.python-version }}- | |
| hf-cache-${{ runner.os }}- | |
| - name: Export HF token for subprocesses | |
| run: | | |
| echo "Exporting HUGGINGFACE_HUB_TOKEN so subprocesses inherit auth" | |
| echo "HUGGINGFACE_HUB_TOKEN=$HF_TOKEN" >> "$GITHUB_ENV" | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| - name: Clean incomplete Hugging Face cache files (if any) | |
| run: | | |
| echo "Removing any *.incomplete directories from HF caches..." | |
| set -eux | |
| # datasets incomplete folders | |
| if [ -d "${{ github.workspace }}/.cache/huggingface/datasets" ]; then | |
| find "${{ github.workspace }}/.cache/huggingface/datasets" -type d -name "*.incomplete" -prune -exec rm -rf {} + || true | |
| fi | |
| # hub incomplete folders | |
| if [ -d "${{ github.workspace }}/.cache/huggingface/hub" ]; then | |
| find "${{ github.workspace }}/.cache/huggingface/hub" -type d -name "*.incomplete" -prune -exec rm -rf {} + || true | |
| fi | |
| # models incomplete folders | |
| if [ -d "${{ github.workspace }}/.cache/huggingface/models" ]; then | |
| find "${{ github.workspace }}/.cache/huggingface/models" -type d -name "*.incomplete" -prune -exec rm -rf {} + || true | |
| fi | |
| - name: Set test worker count | |
| run: echo "PYTEST_WORKERS=$(( $(nproc) - 1 ))" >> "$GITHUB_ENV" # leave 1 for the system process | |
| - name: Run tests with pytest-rerunfailures | |
| run: | | |
| echo "Running tests with up to 3 reruns on failure using $PYTEST_WORKERS workers..." | |
| uv run pytest -n $PYTEST_WORKERS -m "${{ env.DEFAULT_MARK_FILTER }} and ${{ matrix.mark_filter }}" --reruns 3 --reruns-delay 10 --maxfail=1 |