e2e test runtime reproducibility #110
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: e2e test runtime reproducibility | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 19 * * 0" # 7pm UTC on Sundays | |
| jobs: | |
| os-matrix: | |
| strategy: | |
| matrix: | |
| os: [ubuntu-22.04, ubuntu-24.04] | |
| # As we do not use the Cachix cache containing the artifacts built by developers in this workflow, | |
| # building the node-installer-image ensures that the whole transitive closure of the packages we build (i.e. everything | |
| # except what's in cache.nixos.org). Therefore, building this ensures that the kernel, image, IGVM, runtime and the image itself | |
| # is reproducible across individual builds (as the --rebuild flag is used, causing Nix to rebuild the node-installer-image derivation) | |
| # and across independent builds on Ubuntu 24.04 and 22.04 (which also test the reproducibility of the transitive closure of our packages, as no shared | |
| # cache is present between the two machines) | |
| # | |
| # If adjusting the build-target, remember to also adjust the matrix for the collect-checksums job | |
| build-target: | |
| - "kata.contrast-node-installer-image" | |
| - "kata.contrast-node-installer-image.gpu" | |
| fail-fast: false | |
| # Usually we would define the matrix outputs here, but as GitHub Actions don't seem to allow per-combination outputs, | |
| # we'll write the outputs without defining them here. See https://github.com/orgs/community/discussions/17245#discussioncomment-3814009. | |
| runs-on: ${{ matrix.os }} | |
| permissions: | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| persist-credentials: false | |
| - uses: ./.github/actions/setup_nix | |
| with: | |
| githubToken: ${{ secrets.GITHUB_TOKEN }} | |
| cachixToken: "" # Don't use the cachix cache | |
| - uses: nicknovitski/nix-develop@9be7cfb4b10451d3390a75dc18ad0465bed4932a # v1.2.1 | |
| - name: Allow unrestricted user namespaces | |
| # Ubuntu 24.04 ships strict apparmor defaults, so we have to disable them to be able to call unshare in the Nix sansbox. | |
| run: | | |
| sudo sysctl --ignore --write kernel.apparmor_restrict_unprivileged_unconfined=0 | |
| sudo sysctl --ignore --write kernel.apparmor_restrict_unprivileged_userns=0 | |
| - name: Build | |
| id: build | |
| run: | | |
| nix build .#${{ matrix.build-target }} --option substituters https://cache.nixos.org --builders "" | |
| reference_checksum="$(jq -r '.manifests[0].digest' result/index.json)" | |
| echo "$reference_checksum" > ${{ matrix.build-target }}-${{ matrix.os }}-reference_checksum.txt | |
| nix build .#${{ matrix.build-target }} --rebuild --option substituters https://cache.nixos.org --builders "" -o rebuild | |
| rebuild_checksum="$(jq -r '.manifests[0].digest' rebuild/index.json)" | |
| echo "$rebuild_checksum" > ${{ matrix.build-target }}-${{ matrix.os }}-rebuild_checksum.txt | |
| - name: Upload Build Artifacts | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: ${{ matrix.build-target }}-${{ matrix.os }} | |
| path: result | |
| - name: Upload Build Artifacts (Rebuild) | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: ${{ matrix.build-target }}-${{ matrix.os }}-rebuild | |
| path: rebuild | |
| - name: Upload checksums | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: ${{ matrix.build-target }}-${{ matrix.os }}-checksums | |
| path: ${{ matrix.build-target }}-${{ matrix.os }}-*_checksum.txt | |
| - name: Notify teams channel of failure | |
| if: failure() && github.event_name == 'schedule' && github.run_attempt == 1 | |
| uses: ./.github/actions/post_to_teams | |
| with: | |
| webhook: ${{ secrets.TEAMS_CI_WEBHOOK }} | |
| title: "Runtime reproducibility test failed" | |
| message: "Build failed on ${{ matrix.build-target}}-${{matrix.os }}" | |
| additionalFields: '[{"title": "Job ID", "value": "${{ github.job }}"},{"title": "Build target", "value": "${{ matrix.build-target }}"}, {"title": "OS", "value": "${{ matrix.os }}"}]' | |
| collect-checksums: | |
| runs-on: ubuntu-24.04 | |
| permissions: | |
| contents: read | |
| needs: os-matrix | |
| strategy: | |
| matrix: | |
| build-target: | |
| - "kata.contrast-node-installer-image" | |
| - "kata.contrast-node-installer-image.gpu" | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Download all checksum artifacts | |
| uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 | |
| with: | |
| pattern: "${{matrix.build-target}}-*-checksums" | |
| path: "./checksums" | |
| merge-multiple: true | |
| - name: Collect checksums | |
| id: collect | |
| shell: python | |
| run: | | |
| import json, os, pathlib | |
| seen = {} | |
| for file in os.listdir("./checksums"): | |
| checksum = pathlib.Path("./checksums", file).read_text() | |
| if not checksum in seen: | |
| seen[checksum] = [] | |
| seen[checksum].append(file) | |
| assert len(seen) > 0 | |
| if len(seen) > 1: | |
| print("At least one checksum mismatched:") | |
| print(json.dumps(seen, indent=2)) | |
| exit(1) | |
| print("All checksums were equal") | |
| - name: Notify teams channel of failure | |
| if: failure() && github.event_name == 'schedule' && github.run_attempt == 1 | |
| uses: ./.github/actions/post_to_teams | |
| with: | |
| webhook: ${{ secrets.TEAMS_CI_WEBHOOK }} | |
| title: "Runtime reproducibility test failed" | |
| message: "Reproducibility issue found on ${{ matrix.build-target }}" | |
| additionalFields: '[{"title": "Job ID", "value": "${{ github.job }}"},{"title": "Build target", "value": "${{matrix.build-target}}"}]' |