Feature/262 reuse databricks cluster across benchmark iterations #168
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Code Quality and Docker Image Validation | |
| concurrency: | |
| group: code-quality-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| pull_request: | |
| branches: [main] | |
| types: | |
| - opened | |
| - synchronize | |
| - reopened | |
| - ready_for_review | |
| paths: | |
| - '**/*.py' | |
| - '**/*.yml' | |
| - '**/*.yaml' | |
| - '.docker/**' | |
| - 'Dockerfile*' | |
| - 'docker-compose.yml' | |
| - 'requirements.txt' | |
| - '.github/workflows/pull-request-tests.yml' | |
| jobs: | |
| detect-changes: | |
| name: Detect changed paths | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| outputs: | |
| orchestrator: ${{ steps.filter.outputs.orchestrator }} | |
| api: ${{ steps.filter.outputs.api }} | |
| benchmarks: ${{ steps.filter.outputs.benchmarks }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| filters: | | |
| orchestrator: | |
| - 'main.py' | |
| - 'benchmarks.yml' | |
| - 'src/__init__.py' | |
| - 'src/config.py' | |
| - 'src/application/common/**' | |
| - 'src/domain/enums/storage_container.py' | |
| - '.docker/Orchestrator.Dockerfile' | |
| - 'requirements.txt' | |
| - 'docker-compose.yml' | |
| benchmarks: | |
| - 'src/**' | |
| - 'benchmark_runner.py' | |
| - '.docker/Query.Dockerfile' | |
| - '.docker/Setup.Dockerfile' | |
| - 'requirements.txt' | |
| - 'docker-compose.yml' | |
| api: | |
| - 'src/**' | |
| - '!src/presentation/entrypoints/**' | |
| - '!src/presentation/databricks/**' | |
| - '.docker/Api.Dockerfile' | |
| - 'requirements.txt' | |
| - 'docker-compose.yml' | |
| compile: | |
| name: Check Python syntax | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install system dependencies (GDAL) | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y gdal-bin libgdal-dev | |
| - name: Install dependencies | |
| run: | | |
| pip install gdal==$(gdal-config --version) | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| - name: Compile all Python files | |
| run: python -m compileall src main.py benchmark_runner.py | |
| build-orchestrator-image: | |
| name: Build Container Orchestrator | |
| runs-on: ubuntu-latest | |
| needs: | |
| - compile | |
| - detect-changes | |
| if: needs.detect-changes.outputs.orchestrator == 'true' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build Container Orchestrator from docker-compose | |
| run: docker compose build container-orchestrator | |
| build-api-image: | |
| name: Build VMT API Server | |
| runs-on: ubuntu-latest | |
| needs: | |
| - compile | |
| - detect-changes | |
| if: needs.detect-changes.outputs.api == 'true' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build VMT API Server from docker-compose | |
| run: docker compose build vmt-api-server | |
| build-benchmark-images: | |
| name: Build ${{ matrix.display_name }} | |
| runs-on: ubuntu-latest | |
| needs: | |
| - compile | |
| - detect-changes | |
| if: needs.detect-changes.outputs.benchmarks == 'true' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - service: db-scan-blob-storage | |
| display_name: Blob Storage DB Scan | |
| - service: db-scan-postgis | |
| display_name: PostGIS DB Scan | |
| - service: bbox-filtering-advanced-duckdb | |
| display_name: Advanced DuckDB Bounding Box Filtering | |
| - service: bbox-filtering-advanced-postgis | |
| display_name: Advanced PostGIS Bounding Box Filtering | |
| - service: bbox-filtering-simple-local | |
| display_name: Simple Shapefile Bounding Box Filtering | |
| - service: bbox-filtering-simple-blob-storage | |
| display_name: Simple GeoParquet Bounding Box Filtering | |
| - service: bbox-filtering-result-set-sizes-neighborhood-duckdb | |
| display_name: Neighborhood DuckDB Bounding Box Filtering Result Set Sizes | |
| - service: bbox-filtering-result-set-sizes-municipality-duckdb | |
| display_name: Municipality DuckDB Bounding Box Filtering Result Set Sizes | |
| - service: bbox-filtering-result-set-sizes-county-duckdb | |
| display_name: County DuckDB Bounding Box Filtering Result Set Sizes | |
| - service: bbox-filtering-result-set-sizes-neighborhood-postgis | |
| display_name: Neighborhood PostGIS Bounding Box Filtering Result Set Sizes | |
| - service: bbox-filtering-result-set-sizes-municipality-postgis | |
| display_name: Municipality PostGIS Bounding Box Filtering Result Set Sizes | |
| - service: bbox-filtering-result-set-sizes-county-postgis | |
| display_name: County PostGIS Bounding Box Filtering Result Set Sizes | |
| - service: bbox-filtering-result-set-sizes-neighborhood-local | |
| display_name: Neighborhood Local Bounding Box Filtering Result Set Sizes | |
| - service: bbox-filtering-result-set-sizes-municipality-local | |
| display_name: Municipality Local Bounding Box Filtering Result Set Sizes | |
| - service: bbox-filtering-result-set-sizes-county-local | |
| display_name: County Local Bounding Box Filtering Result Set Sizes | |
| - service: vector-tiles-single-tile-pmtiles | |
| display_name: PMTiles Fetch Single Tile | |
| - service: vector-tiles-single-tile-vmt | |
| display_name: VMT Fetch Single Tile | |
| - service: vector-tiles-100k-pmtiles | |
| display_name: PMTiles Fetch 100 000 Tiles | |
| - service: vector-tiles-100k-vmt | |
| display_name: VMT Fetch 100 000 Tiles | |
| - service: spatial-aggregation-grid-duckdb | |
| display_name: DuckDB Spatial Aggregation by Grid Cell | |
| - service: spatial-aggregation-grid-postgis | |
| display_name: PostGIS Spatial Aggregation by Grid Cell | |
| - service: attribute-spatial-compound-filter-duckdb | |
| display_name: DuckDB Attribute + Spatial Compound Filter | |
| - service: attribute-spatial-compound-filter-postgis | |
| display_name: PostGIS Attribute + Spatial Compound Filter | |
| - service: ordered-range-query-duckdb | |
| display_name: DuckDB Ordered Range Query with LIMIT | |
| - service: ordered-range-query-postgis | |
| display_name: PostGIS Ordered Range Query with LIMIT | |
| - service: point-in-polygon-lookup-duckdb | |
| display_name: DuckDB Point-in-Polygon Lookup | |
| - service: point-in-polygon-lookup-postgis | |
| display_name: PostGIS Point-in-Polygon Lookup | |
| - service: national-scale-spatial-join-duckdb | |
| display_name: DuckDB National Scale Spatial Join | |
| - service: national-scale-spatial-join-postgis | |
| display_name: PostGIS National Scale Spatial Join | |
| - service: national-scale-spatial-join-databricks-2-nodes | |
| display_name: Databricks National Scale Spatial Join - 2 Nodes | |
| - service: national-scale-spatial-join-databricks-4-nodes | |
| display_name: Databricks National Scale Spatial Join - 4 Nodes | |
| - service: national-scale-spatial-join-databricks-8-nodes | |
| display_name: Databricks National Scale Spatial Join - 8 Nodes | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build ${{ matrix.display_name }} from docker-compose | |
| run: docker compose build ${{ matrix.service }} |