Collect and publish catalogs #17894
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Collect and publish catalogs | |
| run-name: Collect and publish catalogs${{ inputs.staging && ' (staging)' || '' }} | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| staging: | |
| description: Staging | |
| type: boolean | |
| default: true | |
| schedule: | |
| - cron: '5 * * * *' # Run every hour at HH:05 | |
| env: | |
| PIP_DISABLE_PIP_VERSION_CHECK: on | |
| PIP_DEFAULT_TIMEOUT: 10 | |
| PIP_PROGRESS_BAR: off | |
| PYTHON_VERSION: 3.13 | |
| jobs: | |
| catalog-aws: | |
| name: Collect AWS catalog | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies | |
| run: | | |
| pip install pip -U | |
| pip install -e '.[aws]' | |
| - name: Collect catalog | |
| working-directory: src | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| run: python -m gpuhunt aws --output ../aws.csv | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: catalogs-aws | |
| path: aws.csv | |
| retention-days: 1 | |
| catalog-azure: | |
| name: Collect Azure catalog | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: azure/login@v2 | |
| with: | |
| creds: '{"clientId":"${{ secrets.AZURE_CLIENT_ID }}","clientSecret":"${{ secrets.AZURE_CLIENT_SECRET }}","subscriptionId":"${{ secrets.AZURE_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.AZURE_TENANT_ID }}"}' | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies | |
| run: | | |
| pip install pip -U | |
| pip install -e '.[azure]' | |
| - name: Collect catalog | |
| working-directory: src | |
| env: | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| run: python -m gpuhunt azure --output ../azure.csv | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: catalogs-azure | |
| path: azure.csv | |
| retention-days: 1 | |
| catalog-verda: | |
| name: Collect Verda catalog | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies | |
| run: | | |
| pip install pip -U | |
| pip install -e '.[verda]' | |
| - name: Collect catalog | |
| working-directory: src | |
| env: | |
| VERDA_CLIENT_ID: ${{ secrets.VERDA_CLIENT_ID }} | |
| VERDA_CLIENT_SECRET: ${{ secrets.VERDA_CLIENT_SECRET }} | |
| run: | | |
| python -m gpuhunt verda --output ../verda.csv | |
| # Copy for backward compatibility | |
| cp ../verda.csv ../datacrunch.csv | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: catalogs-verda | |
| path: | | |
| verda.csv | |
| datacrunch.csv | |
| retention-days: 1 | |
| catalog-gcp: | |
| name: Collect GCP catalog | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: google-github-actions/auth@v2 | |
| with: | |
| workload_identity_provider: 'projects/531508670106/locations/global/workloadIdentityPools/github-identity-pool/providers/github-id-provider' | |
| service_account: 'dstack-gpu-pricing-ci@dstack.iam.gserviceaccount.com' | |
| create_credentials_file: true | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies | |
| run: | | |
| pip install pip -U | |
| pip install -e '.[gcp]' | |
| - name: Collect catalog | |
| working-directory: src | |
| env: | |
| GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} | |
| run: python -m gpuhunt gcp --output ../gcp.csv | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: catalogs-gcp | |
| path: gcp.csv | |
| retention-days: 1 | |
| catalog-lambdalabs: | |
| name: Collect LambdaLabs catalog | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies | |
| run: | | |
| pip install pip -U | |
| pip install -e '.' | |
| - name: Collect catalog | |
| working-directory: src | |
| env: | |
| LAMBDALABS_TOKEN: ${{ secrets.LAMBDALABS_TOKEN }} | |
| run: python -m gpuhunt lambdalabs --output ../lambdalabs.csv | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: catalogs-lambdalabs | |
| path: lambdalabs.csv | |
| retention-days: 1 | |
| catalog-nebius: | |
| name: Collect Nebius catalog | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies | |
| run: | | |
| pip install pip -U | |
| pip install -e '.[nebius]' | |
| - name: Collect catalog | |
| working-directory: src | |
| run: | | |
| echo "${{ secrets.NEBIUS_PRIVATE_KEY }}" > "${NEBIUS_PRIVATE_KEY_FILE}" | |
| python -m gpuhunt nebius --output ../nebius.csv | |
| env: | |
| NEBIUS_SERVICE_ACCOUNT_ID: ${{ secrets.NEBIUS_SERVICE_ACCOUNT_ID }} | |
| NEBIUS_PUBLIC_KEY_ID: ${{ secrets.NEBIUS_PUBLIC_KEY_ID }} | |
| NEBIUS_PRIVATE_KEY_FILE: /tmp/nebius.key | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: catalogs-nebius | |
| path: nebius.csv | |
| retention-days: 1 | |
| catalog-oci: | |
| name: Collect OCI catalog | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies | |
| run: | | |
| pip install pip -U | |
| pip install -e '.[oci]' | |
| - name: Collect catalog | |
| working-directory: src | |
| run: python -m gpuhunt oci --output ../oci.csv | |
| env: | |
| OCI_CLI_USER: ${{ secrets.OCI_CLI_USER }} | |
| OCI_CLI_KEY_CONTENT: ${{ secrets.OCI_CLI_KEY_CONTENT }} | |
| OCI_CLI_FINGERPRINT: ${{ secrets.OCI_CLI_FINGERPRINT }} | |
| OCI_CLI_TENANCY: ${{ secrets.OCI_CLI_TENANCY }} | |
| OCI_CLI_REGION: ${{ secrets.OCI_CLI_REGION }} | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: catalogs-oci | |
| path: oci.csv | |
| retention-days: 1 | |
| catalog-runpod: | |
| name: Collect Runpod catalog | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies | |
| run: | | |
| pip install pip -U | |
| pip install -e '.' | |
| - name: Collect catalog | |
| working-directory: src | |
| run: python -m gpuhunt runpod --output ../runpod.csv | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: catalogs-runpod | |
| path: runpod.csv | |
| retention-days: 1 | |
| catalog-cloudrift: | |
| name: Collect CloudRift catalog | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies | |
| run: | | |
| pip install pip -U | |
| pip install -e '.' | |
| - name: Collect catalog | |
| working-directory: src | |
| run: python -m gpuhunt cloudrift --output ../cloudrift.csv | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: catalogs-cloudrift | |
| path: cloudrift.csv | |
| retention-days: 1 | |
| test-catalog: | |
| name: Test catalogs integrity | |
| needs: | |
| - catalog-aws | |
| - catalog-azure | |
| - catalog-verda | |
| - catalog-gcp | |
| - catalog-lambdalabs | |
| - catalog-nebius | |
| - catalog-oci | |
| - catalog-runpod | |
| - catalog-cloudrift | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies | |
| run: | | |
| pip install pip -U | |
| pip install '.[all]' | |
| pip install pytest | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: catalogs-* | |
| merge-multiple: true | |
| - name: Run integrity tests | |
| env: | |
| HOTAISLE_API_KEY: ${{ secrets.HOTAISLE_API_KEY }} | |
| HOTAISLE_TEAM_HANDLE: ${{ secrets.HOTAISLE_TEAM_HANDLE }} | |
| CATALOG_DIR: . | |
| run: pytest src/integrity_tests | |
| publish-catalog: | |
| name: Publish catalogs | |
| needs: [ test-catalog ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install gpuhunt | |
| run: pip install . | |
| - name: Install AWS CLI | |
| run: pip install awscli | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: catalogs-* | |
| merge-multiple: true | |
| path: v2/ | |
| - name: Build legacy v1 catalogs | |
| run: | | |
| mkdir v1 | |
| for catalog_path in $(find v2/*.csv); do | |
| file=$(basename "$catalog_path") | |
| python -m gpuhunt.scripts.catalog_v1 --input "v2/$file" --output "v1/$file" | |
| done | |
| - name: Write version | |
| run: echo "$(date +%Y%m%d)-${{ github.run_number }}" | tee v2/version | tee v1/version | |
| - name: Package catalogs | |
| run: | | |
| zip -j v2/catalog.zip v2/*.csv v2/version | |
| zip -j v1/catalog.zip v1/*.csv v1/version | |
| - name: Upload to S3 | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| BUCKET: s3://dstack-gpu-pricing${{ github.event_name == 'workflow_dispatch' && inputs.staging && '/stgn' || '' }} | |
| run: | | |
| VERSION=$(cat v2/version) | |
| aws s3 cp v2/catalog.zip "$BUCKET/v2/$VERSION/catalog.zip" --acl public-read | |
| aws s3 cp v1/catalog.zip "$BUCKET/v1/$VERSION/catalog.zip" --acl public-read | |
| echo $VERSION | aws s3 cp - "$BUCKET/v2/version" --acl public-read | |
| echo $VERSION | aws s3 cp - "$BUCKET/v1/version" --acl public-read | |
| aws s3 cp "$BUCKET/v2/$VERSION/catalog.zip" "$BUCKET/v2/latest/catalog.zip" --acl public-read | |
| aws s3 cp "$BUCKET/v1/$VERSION/catalog.zip" "$BUCKET/v1/latest/catalog.zip" --acl public-read |