Fixes, improvements, 6x build speed up, update repos

.github/workflows/publish-docker.yml

@@ -0,0 +1,108 @@
+name: Publish Docker
+
+on:
+  push:
+    branches:
+    - master
+    tags:
+    - v*
+  workflow_dispatch:
+jobs:
+  build-amd64:
+    runs-on: ubuntu-latest
+    timeout-minutes: 1440
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v5
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+      with:
+        config-inline: |
+          [worker.oci]
+            max-parallelism = 1
+    - name: Login to DockerHub
+      uses: docker/login-action@v3 
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - name: Docker meta
+      id: docker_meta
+      uses: crazy-max/ghaction-docker-meta@v1
+      with:
+        images: webodm/odm
+        tag-semver: |
+          {{version}}
+    - name: Set Swap Space
+      uses: pierotofy/set-swap-space@master
+      with:
+        swap-size-gb: 10
+    - name: Build and push Docker image (AMD64)
+      id: docker_build_amd64
+      uses: docker/build-push-action@v6
+      with:
+        context: .
+        file: ./portable.Dockerfile
+        platforms: linux/amd64
+        push: true
+        no-cache: true
+        tags: |
+          webodm/odm:latest-amd64
+        provenance: false
+  build-arm64:
+    runs-on: ubuntu-24.04-arm
+    timeout-minutes: 1440
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v5
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+      with:
+        config-inline: |
+          [worker.oci]
+            max-parallelism = 1
+    - name: Login to DockerHub
+      uses: docker/login-action@v3 
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - name: Set Swap Space
+      uses: pierotofy/set-swap-space@master
+      with:
+        swap-size-gb: 10
+    - name: Build and push Docker image (ARM64)
+      id: docker_build_arm64
+      uses: docker/build-push-action@v6
+      with:
+        context: .
+        file: ./portable.Dockerfile
+        platforms: linux/arm64
+        push: true
+        no-cache: true
+        tags: |
+          webodm/odm:latest-arm64
+        provenance: false
+
+  create-manifest:
+    runs-on: ubuntu-latest
+    needs: [build-amd64, build-arm64]
+    steps:
+    - name: Login to DockerHub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - name: Create and push multi-platform manifest
+      run: |
+        docker pull --platform linux/amd64 webodm/odm:latest-amd64
+        docker pull --platform linux/arm64 webodm/odm:latest-arm64
+        docker manifest create webodm/odm:latest \
+          webodm/odm:latest-amd64 \
+          webodm/odm:latest-arm64
+        docker manifest push webodm/odm:latest
+
+    # Trigger NodeODM build, OATS test
+    - name: Dispatch Build Event
+      id: build_dispatch
+      run: |
+        curl -X POST -u "${{secrets.PAT_USERNAME}}:${{secrets.PAT_TOKEN}}" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" https://api.github.com/repos/WebODM/NodeODM/actions/workflows/publish-docker.yml/dispatches --data '{"ref": "master"}'
+        curl -X POST -u "${{secrets.PAT_USERNAME}}:${{secrets.PAT_TOKEN}}" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" https://api.github.com/repos/WebODM/oats/actions/workflows/test.yaml/dispatches --data '{"ref": "master"}'

.github/workflows/publish-windows.yml

@@ -14,7 +14,6 @@ permissions:
 jobs:
   build:
     runs-on: windows-2022
-    environment: azuresigning
     steps:
     - name: Checkout
       uses: actions/checkout@v2
@@ -42,21 +41,9 @@ jobs:
         rmdir SuperBuild\download /s /q
         rmdir SuperBuild\build /s /q
       shell: cmd
-    - name: Retrieve the metadata and decode it to a file
-      env:
-        AZURESIGNING_METADATA: ${{ secrets.AZURESIGNING_METADATA }}
-      run: |
-        echo $AZURESIGNING_METADATA | base64 --decode > "$RUNNER_TEMP\metadata.json"
-      shell: bash
-    - name: Azure login
-      uses: azure/login@v1
-      with:
-        client-id: ${{ secrets.AZURE_CLIENT_ID }}
-        tenant-id: ${{ secrets.AZURE_TENANT_ID }}
-        subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
     - name: Create setup
       run: |
-        python configure.py dist --signtool-path "C:\Program Files (x86)\Windows Kits\10\bin\10.0.26100.0\x64\signtool.exe" --azure-signing-metadata "%RUNNER_TEMP%\metadata.json"
+        python configure.py dist
       shell: cmd
     - name: Upload Setup File
       uses: actions/upload-artifact@v4

.github/workflows/test-build-prs.yaml → .github/workflows/test-build-prs.yml

@@ -24,30 +24,26 @@ jobs:
         platforms: linux/amd64
         push: false
 
-  # snapcraft:
-  #   runs-on: ubuntu-latest
-  #   strategy:
-  #     matrix:
-  #       architecture:
-  #       - amd64
-  #   steps:
-  #   - name: Checkout
-  #     uses: actions/checkout@v2
-  #   - name: Set Swap Space
-  #     uses: pierotofy/set-swap-space@master
-  #     with:
-  #       swap-size-gb: 12
-  #   - name: Build
-  #     id: build
-  #     uses: diddlesnaps/snapcraft-multiarch-action@v1
-  #     with:
-  #       architecture: ${{ matrix.architecture }}
-  #   - name: Review
-  #     uses: diddlesnaps/snapcraft-review-tools-action@v1
-  #     with:
-  #       snap: ${{ steps.build.outputs.snap }}
-  #       isClassic: 'false'
-
+  docker_gpu:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Set Swap Space
+      uses: pierotofy/set-swap-space@master
+      with:
+        swap-size-gb: 12
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+    - name: Build
+      uses: docker/build-push-action@v2
+      with:
+        file: ./gpu.Dockerfile
+        platforms: linux/amd64
+        push: false
+
   windows:
     runs-on: windows-2022
     steps:

Dockerfile

@@ -2,7 +2,7 @@ FROM ubuntu:24.04 AS builder
 
 # Env variables
 ENV DEBIAN_FRONTEND=noninteractive \
-    PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/local/lib/python3.12/dist-packages:/code/SuperBuild/install/lib/python3.12/dist-packages:/code/SuperBuild/install/bin/opensfm" \
+    PYTHONPATH="/code/SuperBuild/install/lib/python3.12/dist-packages:/code/SuperBuild/install/bin/opensfm" \
     LD_LIBRARY_PATH="/code/SuperBuild/install/lib"
 
 # Prepare directories
@@ -14,10 +14,6 @@ COPY . ./
 # Run the build
 RUN bash configure.sh install
 
-# Run the tests
-ENV PATH="/code/venv/bin:$PATH"
-RUN bash test.sh
-
 # Clean Superbuild
 RUN bash configure.sh clean
 
@@ -29,7 +25,7 @@ FROM ubuntu:24.04
 
 # Env variables
 ENV DEBIAN_FRONTEND=noninteractive \
-    PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/local/lib/python3.12/dist-packages:/code/SuperBuild/install/lib/python3.12/dist-packages:/code/SuperBuild/install/bin/opensfm" \
+    PYTHONPATH="/code/SuperBuild/install/lib/python3.12:/code/SuperBuild/install/bin/opensfm" \
     LD_LIBRARY_PATH="/code/SuperBuild/install/lib" \
     PDAL_DRIVER_PATH="/code/SuperBuild/install/bin"
 
@@ -38,12 +34,13 @@ WORKDIR /code
 # Copy everything we built from the builder
 COPY --from=builder /code /code
 
-ENV PATH="/code/venv/bin:$PATH"
+# Copy the Python libraries installed via pip from the builder
+COPY --from=builder /usr/local /usr/local
 
 # Install shared libraries that we depend on via APT, but *not*
 # the -dev packages to save space!
 # Also run a smoke test on ODM and OpenSfM
-RUN bash configure.sh installruntimedepsonly \
+RUN bash configure.sh installruntimedeps \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
   && bash run.sh --help \