BioNeMo Recipes CI #2634
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "BioNeMo Recipes CI" | |
| on: | |
| push: | |
| branches: | |
| - "pull-request/[0-9]+" | |
| - "dependabot/**" | |
| merge_group: | |
| types: [checks_requested] | |
| schedule: | |
| - cron: "0 9 * * *" # Runs at 9 AM UTC daily (2 AM MST) | |
| defaults: | |
| run: | |
| shell: bash -x -e -u -o pipefail {0} | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| pre-commit: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 | |
| with: | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| - name: Setup Python | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 | |
| with: | |
| python-version: "3.13" | |
| - name: Setup uv | |
| uses: astral-sh/setup-uv@d0d8abe699bfb85fec6de9f7adb5ae17292296ff | |
| with: | |
| enable-cache: true | |
| - name: Install pre-commit | |
| run: | | |
| uv tool install pre-commit --with pre-commit-uv --force-reinstall | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| - name: Run pre-commit | |
| env: | |
| PRE_COMMIT_HOME: ${{ runner.temp }}/pre-commit | |
| run: pre-commit run --all-files --show-diff-on-failure | |
| changed-dirs: | |
| needs: | |
| - pre-commit | |
| runs-on: ubuntu-latest | |
| outputs: | |
| any_changed: ${{ steps.changed-files.outputs.any_changed }} | |
| all_changed_files: ${{ steps.changed-files.outputs.all_changed_files }} | |
| dirs: ${{ steps.set-dirs.outputs.dirs }} | |
| labels: ${{ steps.set-dirs.outputs.labels }} | |
| steps: | |
| - id: get-pr-info | |
| if: ${{ startsWith(github.ref_name, 'pull-request/') }} | |
| uses: nv-gha-runners/get-pr-info@090577647b8ddc4e06e809e264f7881650ecdccf | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 | |
| with: | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| - name: Get merge-base commit | |
| id: merge-base | |
| run: | | |
| # Get the merge-base between current branch and main | |
| MERGE_BASE=$(git merge-base HEAD origin/main) | |
| echo "merge-base=$MERGE_BASE" >> $GITHUB_OUTPUT | |
| echo "Merge-base commit: $MERGE_BASE" | |
| - name: Get changed files | |
| id: changed-files | |
| uses: step-security/changed-files@95b56dadb92a30ca9036f16423fd3c088a71ee94 | |
| with: | |
| json: true | |
| matrix: true | |
| base_sha: ${{ steps.merge-base.outputs.merge-base }} | |
| dir_names: true | |
| dir_names_max_depth: 3 | |
| files: | | |
| models/** | |
| recipes/** | |
| !recipes/vllm_inference/** | |
| - id: set-dirs | |
| name: Determine which directories to run | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| PR_INFO: ${{ steps.get-pr-info.outputs.pr-info }} | |
| CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} | |
| run: | | |
| # Get all recipe and model directories | |
| ALL_DIRS=$(ls -d models/*/ recipes/*/ 2>/dev/null | grep -v -E "vllm_inference" | jq -R -s -c 'split("\n")[:-1] | map(rtrimstr("/"))') | |
| # Helper to check for a PR label | |
| has_label() { | |
| [[ "$PR_INFO" != "null" && "$PR_INFO" != "" ]] && \ | |
| echo "$PR_INFO" | jq -e ".labels[]? | select(.name == \"$1\")" > /dev/null 2>&1 | |
| } | |
| # Determine which directories to run | |
| if [[ "$EVENT_NAME" == "schedule" ]]; then | |
| echo "Scheduled run - running all directories" | |
| DIRS="$ALL_DIRS" | |
| elif has_label "ciflow:skip"; then | |
| echo "Found 'ciflow:skip' label - skipping all recipe tests" | |
| DIRS="[]" | |
| elif has_label "ciflow:all"; then | |
| echo "Found 'ciflow:all' label - running all directories" | |
| DIRS="$ALL_DIRS" | |
| else | |
| # Filter directories to only those that have changed files | |
| DIRS=$(echo "$ALL_DIRS" | jq -c --argjson changed "$CHANGED_FILES" ' | |
| map(select(. as $dir | $changed | index($dir) != null)) | |
| ') | |
| fi | |
| # Assign Docker images to the selected directories | |
| # Currently, AMPLIFY is the only folder that needs a custom base image, since we have to support both TE and | |
| # xformers-based models for golden value testing. The rest of the models use the default pytorch image. | |
| DIRS_WITH_IMAGES=$(echo "$DIRS" | jq -c ' | |
| map({ | |
| dir: ., | |
| name: ., | |
| image: ( | |
| if . == "models/amplify" then | |
| "svcbionemo023/bionemo-framework:amplify-model-devcontainer-082025" | |
| else | |
| "nvcr.io/nvidia/pytorch:26.06-py3" | |
| end | |
| ) | |
| }) | |
| ') | |
| echo "dirs=$DIRS_WITH_IMAGES" >> $GITHUB_OUTPUT | |
| # Emit PR labels as a JSON array so downstream jobs can gate on ciflow:* labels. | |
| if [[ "$PR_INFO" != "null" && "$PR_INFO" != "" ]]; then | |
| LABELS=$(echo "$PR_INFO" | jq -c '[.labels[]?.name]' 2>/dev/null || echo "[]") | |
| else | |
| LABELS="[]" | |
| fi | |
| echo "labels=$LABELS" >> $GITHUB_OUTPUT | |
| - name: Show output | |
| run: | | |
| echo "=== Changed Files Analysis ===" | |
| echo "Current branch: ${{ github.ref_name }}" | |
| echo "Merge-base commit: ${{ steps.merge-base.outputs.merge-base }}" | |
| echo "Changed files compared to merge-base:" | |
| echo '${{ steps.changed-files.outputs.all_changed_files }}' | jq -r '.[]' | sed 's/^/ - /' | |
| echo "Total changed files: $(echo '${{ steps.changed-files.outputs.all_changed_files }}' | jq '. | length')" | |
| echo '${{ toJSON(steps.changed-files.outputs) }}' | |
| echo '${{ toJSON(steps.set-dirs.outputs) }}' | |
| shell: bash | |
| unit-tests: | |
| needs: | |
| - changed-dirs | |
| - pre-commit | |
| runs-on: linux-amd64-gpu-l4-latest-1 | |
| if: ${{ needs.changed-dirs.outputs.dirs != '[]' }} | |
| name: "unit-tests (${{ matrix.recipe.name }})" | |
| container: | |
| image: ${{ matrix.recipe.image }} | |
| options: --shm-size=16G | |
| env: | |
| CI: true | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| HF_HOME: /cache/huggingface | |
| strategy: | |
| matrix: | |
| recipe: ${{ fromJson(needs.changed-dirs.outputs.dirs) }} | |
| fail-fast: false | |
| steps: | |
| - name: Show GPU info | |
| run: nvidia-smi | |
| - name: Setup proxy cache | |
| uses: nv-gha-runners/setup-proxy-cache@14229018fe157c83e03c008f27d183d8e99bc67c | |
| - name: Checkout repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 | |
| with: | |
| sparse-checkout: "${{ matrix.recipe.dir }}" | |
| sparse-checkout-cone-mode: false | |
| persist-credentials: false | |
| - name: Cache Hugging Face models | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 | |
| with: | |
| path: /cache/huggingface | |
| key: ${{ runner.os }}-huggingface-${{ matrix.recipe.name }}-${{ github.sha }} | |
| restore-keys: | | |
| ${{ runner.os }}-huggingface-${{ matrix.recipe.name }}- | |
| ${{ runner.os }}-huggingface- | |
| - name: Setup uv | |
| uses: astral-sh/setup-uv@d0d8abe699bfb85fec6de9f7adb5ae17292296ff | |
| with: | |
| enable-cache: true | |
| - name: Install dependencies | |
| working-directory: ${{ matrix.recipe.dir }} | |
| run: | | |
| if [ -f .ci_build.sh ]; then | |
| bash .ci_build.sh | |
| elif [ -f pyproject.toml ] || [ -f setup.py ]; then | |
| PIP_CONSTRAINT= pip install -e . | |
| echo "Installed ${{ matrix.recipe.dir }} as editable package" | |
| elif [ -f requirements.txt ]; then | |
| PIP_CONSTRAINT= pip install -r requirements.txt | |
| echo "Installed ${{ matrix.recipe.dir }} from requirements.txt" | |
| else | |
| echo "No pyproject.toml, setup.py, or requirements.txt found in ${{ matrix.recipe.dir }}" | |
| exit 1 | |
| fi | |
| - name: Run tests | |
| working-directory: ${{ matrix.recipe.dir }} | |
| run: | | |
| if [ -f .ci_test_env.sh ]; then | |
| source .ci_test_env.sh | |
| fi | |
| pytest -v . | |
| run-tests-notebooks: | |
| needs: | |
| - changed-dirs | |
| - pre-commit | |
| runs-on: linux-amd64-gpu-l4-latest-1 | |
| permissions: | |
| contents: read | |
| # Label-only on PRs, automatic on merge_group and nightly schedule. Scoped to evo2_megatron, | |
| # the only recipe in this suite with example notebooks. | |
| if: | | |
| contains(needs.changed-dirs.outputs.dirs, 'recipes/evo2_megatron') && | |
| ( | |
| (github.event_name == 'schedule') || | |
| (github.event_name == 'merge_group') || | |
| contains(fromJSON(needs.changed-dirs.outputs.labels || '[]'), 'ciflow:all') || | |
| contains(fromJSON(needs.changed-dirs.outputs.labels || '[]'), 'ciflow:notebooks') | |
| ) | |
| name: "notebook-tests (evo2_megatron)" | |
| container: | |
| image: nvcr.io/nvidia/pytorch:26.06-py3 | |
| options: --shm-size=16G | |
| env: | |
| CI: true | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| HF_HOME: /cache/huggingface | |
| BIONEMO_DATA_SOURCE: ngc | |
| steps: | |
| - name: Show GPU info | |
| run: nvidia-smi | |
| - name: Setup proxy cache | |
| uses: nv-gha-runners/setup-proxy-cache@14229018fe157c83e03c008f27d183d8e99bc67c | |
| - name: Checkout repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 | |
| with: | |
| sparse-checkout: recipes/evo2_megatron | |
| sparse-checkout-cone-mode: false | |
| persist-credentials: false | |
| - name: Cache Hugging Face models | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 | |
| with: | |
| path: /cache/huggingface | |
| key: ${{ runner.os }}-huggingface-evo2_megatron-notebooks-${{ github.sha }} | |
| restore-keys: | | |
| ${{ runner.os }}-huggingface-evo2_megatron-notebooks- | |
| ${{ runner.os }}-huggingface-evo2_megatron- | |
| ${{ runner.os }}-huggingface- | |
| - name: Setup uv | |
| uses: astral-sh/setup-uv@d0d8abe699bfb85fec6de9f7adb5ae17292296ff | |
| with: | |
| enable-cache: true | |
| - name: Install dependencies | |
| working-directory: recipes/evo2_megatron | |
| run: | | |
| bash .ci_build.sh | |
| source .ci_test_env.sh | |
| pip install nbval | |
| - name: Run notebook tests | |
| working-directory: recipes/evo2_megatron | |
| run: | | |
| source .ci_test_env.sh | |
| FAST_CI_MODE=1 pytest -v -s --nbval-lax -x -p no:python \ | |
| examples/lora-fine-tuning-tutorial.ipynb | |
| verify-recipe-tests: | |
| # This job checks the status of the test jobs and fails if any failed or were cancelled. | |
| # Use this job as the required check for PRs. | |
| needs: | |
| - pre-commit | |
| - unit-tests | |
| - run-tests-notebooks | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - name: Check unit-tests matrix status | |
| run: | | |
| if [[ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" == "true" ]]; then | |
| echo "Some recipe test jobs have failed or been cancelled!" | |
| exit 1 | |
| else | |
| echo "All recipe test jobs have completed successfully or were skipped!" | |
| exit 0 | |
| fi |