Skip to content

Commit 2010cd1

Browse files
hancocbhancocb
authored
Merge pull request #384 from hancocb/feat/uv-docker-migration
[Chore] Replace the conda backend environment with uv and a geo base image
2 parents 30f97ba + 4e51089 commit 2010cd1

14 files changed

Lines changed: 3643 additions & 1058 deletions

‎.github/workflows/geo-base.yml‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
name: Verify geo base image
2+
3+
# Builds the native geospatial stack (GDAL, PDAL, PROJ, GEOS, Untwine) that
4+
# backend.dockerfile builds on top of, then checks the built image against the
5+
# declarations in backend/pyproject.toml and smoke-tests both trees it ships.
6+
# This takes upwards of an hour, so it runs only when one of the inputs below
7+
# changes -- the application image build does not compile any of it.
8+
#
9+
# This workflow does not publish. gdslab/d2s-geo-base:latest is pushed by hand by
10+
# a maintainer: the tag is mutable and every backend build consumes it, so
11+
# anything published reaches production images immediately, and that is not a
12+
# decision to automate for one image ahead of a strategy for the rest.
13+
#
14+
# It runs on pull requests because that is where the answer is worth having. The
15+
# geospatial declarations in backend/pyproject.toml describe the same libraries
16+
# this image provides, since the Python bindings are compiled against them, and a
17+
# mismatch breaks every backend build. Nothing else checks that: no workflow here
18+
# builds the application image, where the equivalent guard lives. Learning about a
19+
# bad pin after the merge is learning too late.
20+
21+
on:
22+
pull_request:
23+
paths:
24+
- backend/geobase.dockerfile
25+
# geobase.dockerfile.dockerignore decides what the build can see,
26+
# fetch-source.sh is copied into it, and verify_geo_stack.sh is the smoke test
27+
# below -- each one changes what this workflow produces or what it proves.
28+
# Without them a change that quietly defeated the checksum guard would merge
29+
# unbuilt and unverified. These are literal paths rather than prefixes:
30+
# geobase.dockerfile does not match geobase.dockerfile.dockerignore.
31+
- backend/geobase.dockerfile.dockerignore
32+
- backend/scripts/fetch-source.sh
33+
- backend/scripts/verify_geo_stack.sh
34+
- backend/scripts/check_geo_pins.sh
35+
# pyproject.toml is not a build input; it is what the declaration check below
36+
# compares the built image against, so a pin-only change has to run this too.
37+
- backend/pyproject.toml
38+
- .github/workflows/geo-base.yml
39+
# For verifying the stack outside a pull request -- checking a rebuild of a
40+
# published image, or confirming main is sound before pushing the tag by hand.
41+
workflow_dispatch:
42+
43+
permissions:
44+
contents: read
45+
46+
# An hour of compiling per push to a pull request branch is worth spending once.
47+
# A new commit makes the run in flight answer a question nobody is asking any more.
48+
concurrency:
49+
group: ${{ github.workflow }}-${{ github.ref }}
50+
cancel-in-progress: true
51+
52+
env:
53+
# Local to the runner. Nothing here pushes it; the name matches what
54+
# backend.dockerfile expects so the smoke tests exercise the real thing.
55+
IMAGE: gdslab/d2s-geo-base:latest
56+
57+
jobs:
58+
verify:
59+
runs-on: ubuntu-latest
60+
timeout-minutes: 180
61+
steps:
62+
- uses: actions/checkout@v4
63+
64+
- uses: docker/setup-buildx-action@v3
65+
66+
# Cache scope on a pull request is the pull request, so the first run here
67+
# compiles everything and later pushes to the same branch reuse it. That is
68+
# the case worth optimising: reviewing a geo change usually means several
69+
# commits, and only the first should cost an hour.
70+
#
71+
# mode=max is what makes that true. mode=min exports only the final stage,
72+
# which is a COPY on top of ubuntu, so the expensive build stage would not be
73+
# cached at all. Nothing else in this repo uses the Actions cache, so the
74+
# 10 GB budget is not contended.
75+
- name: Build
76+
uses: docker/build-push-action@v6
77+
with:
78+
context: backend
79+
file: backend/geobase.dockerfile
80+
tags: ${{ env.IMAGE }}
81+
load: true
82+
cache-from: type=gha
83+
cache-to: type=gha,mode=max
84+
85+
# The application build refuses to compile bindings against a geo base whose
86+
# versions disagree with pyproject.toml -- but nothing builds the application
87+
# image in CI, so that guard never runs here. Without this step, bumping
88+
# GDAL_VERSION and forgetting the pin passes every check above and breaks
89+
# every backend build from the merge onwards.
90+
#
91+
# Same script the application build runs, so the two cannot disagree about what
92+
# the invariant is. It takes seconds; it runs before the smoke tests so a
93+
# mismatch is reported without waiting for them.
94+
- name: Check pyproject declarations against the built image
95+
run: |
96+
docker run --rm \
97+
-v "$PWD/backend/scripts/check_geo_pins.sh:/check-geo-pins:ro" \
98+
-v "$PWD/backend/pyproject.toml:/pyproject.toml:ro" \
99+
${{ env.IMAGE }} sh /check-geo-pins /opt/geo/VERSIONS /pyproject.toml
100+
101+
# The raster fixture has to be mounted for the COG, JPEG and hillshade
102+
# checks to run; without it the script fails rather than skipping past them.
103+
- name: Smoke test (build tree)
104+
run: |
105+
docker run --rm \
106+
-v "$PWD/backend/scripts:/scripts:ro" \
107+
-v "$PWD/backend/app/tests/data:/app/app/tests/data:ro" \
108+
${{ env.IMAGE }} bash /scripts/verify_geo_stack.sh --cli-only
109+
110+
# The step above checks /opt/geo, the unstripped tree this image puts on PATH
111+
# and the application's build stage compiles against. It is not what ships:
112+
# backend.dockerfile copies /opt/geo-runtime, stripped and pruned, into the
113+
# final image. Checking only the first would let a bad strip or an over-broad
114+
# rm -rf in the prune step through undetected. GEO_PREFIX has to move with
115+
# PATH, or the linkage sweep would read the tree that was already checked.
116+
- name: Smoke test (runtime tree)
117+
run: |
118+
docker run --rm \
119+
-e PATH=/opt/geo-runtime/bin:/usr/local/bin:/usr/bin:/bin \
120+
-e LD_LIBRARY_PATH=/opt/geo-runtime/lib \
121+
-e GDAL_DATA=/opt/geo-runtime/share/gdal \
122+
-e PROJ_DATA=/opt/geo-runtime/share/proj \
123+
-e PROJ_LIB=/opt/geo-runtime/share/proj \
124+
-e PDAL_DRIVER_PATH=/opt/geo-runtime/lib \
125+
-e GEO_PREFIX=/opt/geo-runtime \
126+
-v "$PWD/backend/scripts:/scripts:ro" \
127+
-v "$PWD/backend/app/tests/data:/app/app/tests/data:ro" \
128+
${{ env.IMAGE }} bash /scripts/verify_geo_stack.sh --cli-only

‎backend/.dockerignore‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,19 @@
1-
**./.pytest_cache
1+
# potree/ and pc-gltf-viewer/ are served as static files at runtime, and app/tests
2+
# runs inside the dev image, so all three stay in the build context.
3+
.git
4+
.gitignore
5+
.dockerignore
6+
backend.dockerfile
7+
geobase.dockerfile
8+
geobase.dockerfile.dockerignore
9+
.mypy_cache
10+
**/.mypy_cache
11+
.pytest_cache
12+
**/.pytest_cache
13+
**/__pycache__
14+
**/*.pyc
15+
.venv
216
**/.venv
17+
.coverage
18+
htmlcov
19+
logs

0 commit comments

Comments
 (0)