Add support for custom certificate packages (#2105)

- Extend installCertificates to collect certificate files from installed
APK packages that provide a configured value, in addition to inline
certificates from the image config
- Certificates from qualifying packages are read from the filesystem,
parsed, and appended to the CA bundle (ca-certificates.crt) and Java
truststores, replacing what update-ca-certificates post-install scripts
would normally do
- When no existing CA bundle is present, the primary bundle is now
created automatically if certificates need to be installed

To enable, set `certificates.providers = ["custom-ca-certificates"]` in
config and install packages that provide the given string
("custom-ca-certificates" in this example) containing .pem/.crt files.

Author: codysoyland

.github/workflows/build-samples.yml

@@ -237,64 +237,33 @@ jobs:
           check-latest: true
       - name: test additional certificates
         run: |
-          set -euo pipefail
-
-          # Gets all certificate fingerprints from a PEM stream, sorted, so two outputs can
-          # be semantically compared.
-          get_fingerprints() {
-              local cert=""
-              while IFS= read -r line; do
-                  case "$line" in
-                      "-----BEGIN CERTIFICATE-----")
-                          cert="$line"$'\n' ;;
-                      "-----END CERTIFICATE-----")
-                          cert+="$line"
-                          echo "$cert" | openssl x509 -noout -fingerprint -sha256 2>/dev/null
-                          cert="" ;;
-                      *)
-                          [[ -n "$cert" ]] && cert+="$line"$'\n' || true ;;
-                  esac
-              done | sort
-          }
-
-          # Verifies that fingerprints file contains at least N certs and the two additional test certs.
-          verify_fingerprints() {
-              local file="$1"
-              local min_count="$2"
-              local name="$3"
-
-              local count=$(wc -l < "$file")
-              if [ "$count" -lt "$min_count" ]; then
-                  echo "Expected at least $min_count certificates in $name, found $count"
-                  exit 1
-              fi
-
-              grep "E7:05:70:A9:89:F8:56:5A:AB:DF:7C:AE:27:AB:D1:62:18:72:D6:A3:F8:11:E3:FE:F2:7E:3D:BA:02:91:21:98" "$file"
-              grep "9B:2A:33:9F:E6:A3:E8:55:85:C4:CD:75:53:6C:B8:C1:CF:7C:D6:03:B9:A6:4B:EC:25:21:85:8A:E4:8D:A8:5D" "$file"
-              echo "$name contains the additional certificates."
-          }
+          ./hack/test-certificates.sh ./examples/certificates.yaml \
+            "E7:05:70:A9:89:F8:56:5A:AB:DF:7C:AE:27:AB:D1:62:18:72:D6:A3:F8:11:E3:FE:F2:7E:3D:BA:02:91:21:98" \
+            "9B:2A:33:9F:E6:A3:E8:55:85:C4:CD:75:53:6C:B8:C1:CF:7C:D6:03:B9:A6:4B:EC:25:21:85:8A:E4:8D:A8:5D"
 
-          make apko
-
-          # Build the certificates image.
-          ./apko build ./examples/certificates.yaml certificates:build /tmp/certificates.tar --arch amd64
-          docker load < /tmp/certificates.tar
-
-          # Get fingerprints from CA bundle and Java truststore before update-ca-certificates.
-          docker run --rm certificates:build-amd64 "cat /etc/ssl/certs/ca-certificates.crt" | get_fingerprints > /tmp/ca-bundle-fingerprints.txt
-          docker run --rm certificates:build-amd64 "trust extract --filter=ca-anchors --purpose=server-auth --format=pem-bundle /tmp/certs.pem && cat /tmp/certs.pem" | get_fingerprints > /tmp/java-truststore-fingerprints.txt
-
-          # Verify both stores contain base certs and the additional certificates.
-          verify_fingerprints /tmp/ca-bundle-fingerprints.txt 10 "CA bundle"
-          verify_fingerprints /tmp/java-truststore-fingerprints.txt 10 "Java truststore"
+  package-certificates:
+    name: package-certificates
+    runs-on: ubuntu-latest
 
-          # Run update-ca-certificates and get fingerprints from both stores after.
-          docker run --rm certificates:build-amd64 "apk add ca-certificates && update-ca-certificates && cat /etc/ssl/certs/ca-certificates.crt" | get_fingerprints > /tmp/ca-bundle-updated-fingerprints.txt
-          docker run --rm certificates:build-amd64 "apk add ca-certificates && update-ca-certificates && trust extract --filter=ca-anchors --purpose=server-auth --format=pem-bundle /tmp/certs.pem && cat /tmp/certs.pem" | get_fingerprints > /tmp/java-truststore-updated-fingerprints.txt
+    permissions:
+      contents: read
 
-          # Verify that the stores are semantically identical before and after update-ca-certificates.
-          diff /tmp/ca-bundle-fingerprints.txt /tmp/ca-bundle-updated-fingerprints.txt
-          echo "CA bundles before and after update-ca-certificates are identical."
+    steps:
+      - uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0
+        with:
+          egress-policy: audit
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
 
-          diff /tmp/java-truststore-fingerprints.txt /tmp/java-truststore-updated-fingerprints.txt
-          echo "Java truststores before and after update-ca-certificates are identical."
+      - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+      - name: test package-provided certificates
+        run: |
+          ./hack/test-certificates.sh ./examples/package-certificates.yaml \
+            "E7:05:70:A9:89:F8:56:5A:AB:DF:7C:AE:27:AB:D1:62:18:72:D6:A3:F8:11:E3:FE:F2:7E:3D:BA:02:91:21:98" \
+            "9B:2A:33:9F:E6:A3:E8:55:85:C4:CD:75:53:6C:B8:C1:CF:7C:D6:03:B9:A6:4B:EC:25:21:85:8A:E4:8D:A8:5D" \
+            "34:75:37:AF:7A:09:D4:03:F1:9F:58:F8:3C:35:68:91:2A:F2:4B:7C:12:E7:45:F1:D5:55:70:79:70:8C:91:AD" \
+            "12:AE:34:99:9A:A6:4D:CD:1A:69:47:E8:38:A5:3A:AB:AB:FC:FA:CA:45:AB:CA:8D:C0:CB:B8:DC:B7:BD:06:3C"

examples/package-certificates.yaml

@@ -0,0 +1,24 @@
+contents:
+  keyring:
+    - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
+    - ./internal/cli/testdata/melange.rsa.pub
+  repositories:
+    - https://packages.wolfi.dev/os
+    - ./internal/cli/testdata/packages
+  packages:
+    - ca-certificates-bundle
+    - busybox
+    - apk-tools
+    - java-cacerts
+    - custom-ca-certs-1
+    - custom-ca-certs-2
+
+certificates:
+  providers:
+    - custom-ca-certificates
+
+entrypoint:
+  command: /bin/sh -c
+
+archs:
+  - amd64

hack/test-certificates.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+# Copyright 2025 Chainguard, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+# Tests that an apko-built image contains expected certificates in both the CA
+# bundle and Java truststore, and that running update-ca-certificates doesn't
+# change them.
+#
+# Usage: hack/test-certificates.sh <yaml> <fingerprint> [<fingerprint> ...]
+#
+# Example:
+#   hack/test-certificates.sh ./examples/certificates.yaml \
+#     "E7:05:70:A9:..." "9B:2A:33:9F:..."
+
+set -euo pipefail
+
+if [ $# -lt 2 ]; then
+    echo "Usage: $0 <yaml> <fingerprint> [<fingerprint> ...]"
+    exit 1
+fi
+
+yaml="$1"
+shift
+fingerprints=("$@")
+
+name=$(basename "${yaml}" .yaml)
+image="${name}:build"
+image_arch="${name}:build-amd64"
+tarball="/tmp/${name}.tar"
+
+# Gets all certificate fingerprints from a PEM stream, sorted, so two outputs can
+# be semantically compared.
+get_fingerprints() {
+    local cert=""
+    while IFS= read -r line; do
+        case "$line" in
+            "-----BEGIN CERTIFICATE-----")
+                cert="$line"$'\n' ;;
+            "-----END CERTIFICATE-----")
+                cert+="$line"
+                echo "$cert" | openssl x509 -noout -fingerprint -sha256 2>/dev/null
+                cert="" ;;
+            *)
+                [[ -n "$cert" ]] && cert+="$line"$'\n' || true ;;
+        esac
+    done | sort
+}
+
+# Verifies that fingerprints file contains at least N certs and all expected fingerprints.
+verify_fingerprints() {
+    local file="$1"
+    local min_count="$2"
+    local store_name="$3"
+
+    local count
+    count=$(wc -l < "$file")
+    if [ "$count" -lt "$min_count" ]; then
+        echo "Expected at least $min_count certificates in $store_name, found $count"
+        exit 1
+    fi
+
+    for fp in "${fingerprints[@]}"; do
+        grep "$fp" "$file"
+    done
+    echo "$store_name contains all expected certificates."
+}
+
+# Build the image.
+make apko
+./apko build "${yaml}" "${image}" "${tarball}" --arch amd64
+docker load < "${tarball}"
+
+# Get fingerprints from CA bundle and Java truststore before update-ca-certificates.
+docker run --rm "${image_arch}" "cat /etc/ssl/certs/ca-certificates.crt" | get_fingerprints > /tmp/ca-bundle-fingerprints.txt
+docker run --rm "${image_arch}" "trust extract --filter=ca-anchors --purpose=server-auth --format=pem-bundle /tmp/certs.pem && cat /tmp/certs.pem" | get_fingerprints > /tmp/java-truststore-fingerprints.txt
+
+# Verify both stores contain base certs and all expected certificates.
+verify_fingerprints /tmp/ca-bundle-fingerprints.txt 10 "CA bundle"
+verify_fingerprints /tmp/java-truststore-fingerprints.txt 10 "Java truststore"
+
+# Run update-ca-certificates and get fingerprints from both stores after.
+docker run --rm "${image_arch}" "apk add ca-certificates && update-ca-certificates && cat /etc/ssl/certs/ca-certificates.crt" | get_fingerprints > /tmp/ca-bundle-updated-fingerprints.txt
+docker run --rm "${image_arch}" "apk add ca-certificates && update-ca-certificates && trust extract --filter=ca-anchors --purpose=server-auth --format=pem-bundle /tmp/certs.pem && cat /tmp/certs.pem" | get_fingerprints > /tmp/java-truststore-updated-fingerprints.txt
+
+# Verify that the stores are semantically identical before and after update-ca-certificates.
+diff /tmp/ca-bundle-fingerprints.txt /tmp/ca-bundle-updated-fingerprints.txt
+echo "CA bundles before and after update-ca-certificates are identical."
+
+diff /tmp/java-truststore-fingerprints.txt /tmp/java-truststore-updated-fingerprints.txt
+echo "Java truststores before and after update-ca-certificates are identical."

hack/update-packages.sh

@@ -7,6 +7,8 @@ set -ex
 
 (cd internal/cli/testdata && \
   melange build --arch arm64 --arch amd64 -r https://packages.wolfi.dev/os -k https://packages.wolfi.dev/os/wolfi-signing.rsa.pub --signing-key ./melange.rsa pretend-baselayout.melange.yaml && \
-  melange build --arch arm64 --arch amd64 -r https://packages.wolfi.dev/os -k https://packages.wolfi.dev/os/wolfi-signing.rsa.pub --signing-key ./melange.rsa replayout.melange.yaml)
+  melange build --arch arm64 --arch amd64 -r https://packages.wolfi.dev/os -k https://packages.wolfi.dev/os/wolfi-signing.rsa.pub --signing-key ./melange.rsa replayout.melange.yaml && \
+  melange build --arch arm64 --arch amd64 -r https://packages.wolfi.dev/os -k https://packages.wolfi.dev/os/wolfi-signing.rsa.pub --signing-key ./melange.rsa custom-ca-certs-1.melange.yaml && \
+  melange build --arch arm64 --arch amd64 -r https://packages.wolfi.dev/os -k https://packages.wolfi.dev/os/wolfi-signing.rsa.pub --signing-key ./melange.rsa custom-ca-certs-2.melange.yaml)
 (cd internal/cli &&
   apko lock ./testdata/apko.yaml)

internal/cli/testdata/apko-certs.yaml

@@ -0,0 +1,17 @@
+contents:
+  keyring:
+    - ./testdata/melange.rsa.pub
+  repositories:
+    - ./testdata/packages
+  packages:
+    - pretend-baselayout
+    - custom-ca-certs-1
+    - custom-ca-certs-2
+
+certificates:
+  providers:
+    - custom-ca-certificates
+
+archs:
+- x86_64
+- aarch64

internal/cli/testdata/custom-ca-certs-1.melange.yaml

@@ -0,0 +1,71 @@
+package:
+  name: custom-ca-certs-1
+  version: 1.0.0
+  epoch: 0
+  description: "custom CA certificates package 1 (test)"
+  copyright:
+    - license: MIT
+  dependencies:
+    provides:
+      - custom-ca-certificates
+
+environment:
+  contents:
+    packages:
+      - busybox
+
+pipeline:
+  - name: Install certificates
+    runs: |
+      mkdir -p ${{targets.destdir}}/usr/local/share/ca-certificates
+      cat >${{targets.destdir}}/usr/local/share/ca-certificates/cert-a.crt <<'CERTEOF'
+      -----BEGIN CERTIFICATE-----
+      MIIFmDCCA4CgAwIBAgIQU9C87nMpOIFKYpfvOHFHFDANBgkqhkiG9w0BAQsFADBm
+      MQswCQYDVQQGEwJVUzEzMDEGA1UEChMqKFNUQUdJTkcpIEludGVybmV0IFNlY3Vy
+      aXR5IFJlc2VhcmNoIEdyb3VwMSIwIAYDVQQDExkoU1RBR0lORykgUHJldGVuZCBQ
+      ZWFyIFgxMB4XDTE1MDYwNDExMDQzOFoXDTM1MDYwNDExMDQzOFowZjELMAkGA1UE
+      BhMCVVMxMzAxBgNVBAoTKihTVEFHSU5HKSBJbnRlcm5ldCBTZWN1cml0eSBSZXNl
+      YXJjaCBHcm91cDEiMCAGA1UEAxMZKFNUQUdJTkcpIFByZXRlbmQgUGVhciBYMTCC
+      AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALbagEdDTa1QgGBWSYkyMhsc
+      ZXENOBaVRTMX1hceJENgsL0Ma49D3MilI4KS38mtkmdF6cPWnL++fgehT0FbRHZg
+      jOEr8UAN4jH6omjrbTD++VZneTsMVaGamQmDdFl5g1gYaigkkmx8OiCO68a4QXg4
+      wSyn6iDipKP8utsE+x1E28SA75HOYqpdrk4HGxuULvlr03wZGTIf/oRt2/c+dYmD
+      oaJhge+GOrLAEQByO7+8+vzOwpNAPEx6LW+crEEZ7eBXih6VP19sTGy3yfqK5tPt
+      TdXXCOQMKAp+gCj/VByhmIr+0iNDC540gtvV303WpcbwnkkLYC0Ft2cYUyHtkstO
+      fRcRO+K2cZozoSwVPyB8/J9RpcRK3jgnX9lujfwA/pAbP0J2UPQFxmWFRQnFjaq6
+      rkqbNEBgLy+kFL1NEsRbvFbKrRi5bYy2lNms2NJPZvdNQbT/2dBZKmJqxHkxCuOQ
+      FjhJQNeO+Njm1Z1iATS/3rts2yZlqXKsxQUzN6vNbD8KnXRMEeOXUYvbV4lqfCf8
+      mS14WEbSiMy87GB5S9ucSV1XUrlTG5UGcMSZOBcEUpisRPEmQWUOTWIoDQ5FOia/
+      GI+Ki523r2ruEmbmG37EBSBXdxIdndqrjy+QVAmCebyDx9eVEGOIpn26bW5LKeru
+      mJxa/CFBaKi4bRvmdJRLAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB
+      Af8EBTADAQH/MB0GA1UdDgQWBBS182Xy/rAKkh/7PH3zRKCsYyXDFDANBgkqhkiG
+      9w0BAQsFAAOCAgEAncDZNytDbrrVe68UT6py1lfF2h6Tm2p8ro42i87WWyP2LK8Y
+      nLHC0hvNfWeWmjZQYBQfGC5c7aQRezak+tHLdmrNKHkn5kn+9E9LCjCaEsyIIn2j
+      qdHlAkepu/C3KnNtVx5tW07e5bvIjJScwkCDbP3akWQixPpRFAsnP+ULx7k0aO1x
+      qAeaAhQ2rgo1F58hcflgqKTXnpPM02intVfiVVkX5GXpJjK5EoQtLceyGOrkxlM/
+      sTPq4UrnypmsqSagWV3HcUlYtDinc+nukFk6eR4XkzXBbwKajl0YjztfrCIHOn5Q
+      CJL6TERVDbM/aAPly8kJ1sWGLuvvWYzMYgLzDul//rUF10gEMWaXVZV51KpS9DY/
+      5CunuvCXmEQJHo7kGcViT7sETn6Jz9KOhvYcXkJ7po6d93A/jy4GKPIPnsKKNEmR
+      xUuXY4xRdh45tMJnLTUDdC9FIU0flTeO9/vNpVA8OPU1i14vCz+MU8KX1bV3GXm/
+      fxlB7VBBjX9v5oUep0o/j68R/iDlCOM4VVfRa8gX6T2FU7fNdatvGro7uQzIvWof
+      gN9WUwCbEMBy/YhBSrXycKA8crgGg3x1mIsopn88JKwmMBa68oS7EHM9w7C4y71M
+      7DiA+/9Qdp9RBWJpTS9i/mDnJg1xvo8Xz49mrrgfmcAXTCJqXi24NatI3Oc=
+      -----END CERTIFICATE-----
+      CERTEOF
+      cat >${{targets.destdir}}/usr/local/share/ca-certificates/cert-b.crt <<'CERTEOF'
+      -----BEGIN CERTIFICATE-----
+      MIICTjCCAdSgAwIBAgIRAIPgc3k5LlLVLtUUvs4K/QcwCgYIKoZIzj0EAwMwaDEL
+      MAkGA1UEBhMCVVMxMzAxBgNVBAoTKihTVEFHSU5HKSBJbnRlcm5ldCBTZWN1cml0
+      eSBSZXNlYXJjaCBHcm91cDEkMCIGA1UEAxMbKFNUQUdJTkcpIEJvZ3VzIEJyb2Nj
+      b2xpIFgyMB4XDTIwMDkwNDAwMDAwMFoXDTQwMDkxNzE2MDAwMFowaDELMAkGA1UE
+      BhMCVVMxMzAxBgNVBAoTKihTVEFHSU5HKSBJbnRlcm5ldCBTZWN1cml0eSBSZXNl
+      YXJjaCBHcm91cDEkMCIGA1UEAxMbKFNUQUdJTkcpIEJvZ3VzIEJyb2Njb2xpIFgy
+      MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEOvS+w1kCzAxYOJbA06Aw0HFP2tLBLKPo
+      FQqR9AMskl1nC2975eQqycR+ACvYelA8rfwFXObMHYXJ23XLB+dAjPJVOJ2OcsjT
+      VqO4dcDWu+rQ2VILdnJRYypnV1MMThVxo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYD
+      VR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU3tGjWWQOwZo2o0busBB2766XlWYwCgYI
+      KoZIzj0EAwMDaAAwZQIwRcp4ZKBsq9XkUuN8wfX+GEbY1N5nmCRc8e80kUkuAefo
+      uc2j3cICeXo1cOybQ1iWAjEA3Ooawl8eQyR4wrjCofUE8h44p0j7Yl/kBlJZT8+9
+      vbtH7QiVzeKCOTQPINyRql6P
+      -----END CERTIFICATE-----
+      CERTEOF

internal/cli/testdata/custom-ca-certs-2.melange.yaml

@@ -0,0 +1,48 @@
+package:
+  name: custom-ca-certs-2
+  version: 1.0.0
+  epoch: 0
+  description: "custom CA certificates package 2 (test)"
+  copyright:
+    - license: MIT
+  dependencies:
+    provides:
+      - custom-ca-certificates
+
+environment:
+  contents:
+    packages:
+      - busybox
+
+pipeline:
+  - name: Install certificates
+    runs: |
+      mkdir -p ${{targets.destdir}}/usr/local/share/ca-certificates
+      cat >${{targets.destdir}}/usr/local/share/ca-certificates/cert-c.crt <<'CERTEOF'
+      -----BEGIN CERTIFICATE-----
+      MIIBwjCCAWegAwIBAgIUBKZDifzRAz30jwlcoQLIOxkBPLMwCgYIKoZIzj0EAwIw
+      NTEeMBwGA1UEAwwVVGVzdCBDQSBDZXJ0aWZpY2F0ZSAzMRMwEQYDVQQKDApUZXN0
+      IE9yZyAzMCAXDTI2MDIyNzIwMzk1OVoYDzIxMjYwMjAzMjAzOTU5WjA1MR4wHAYD
+      VQQDDBVUZXN0IENBIENlcnRpZmljYXRlIDMxEzARBgNVBAoMClRlc3QgT3JnIDMw
+      WTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARx/10O/q2rOnQtpBXHjARAUryfNWjD
+      UXeshzFk44hrv45loTsGQcyb5vAL6h3FSdBN91njUch4eF1NEYLKoR3Qo1MwUTAd
+      BgNVHQ4EFgQUhLbWEa0IUIixKPBVvuKxhK6UMnMwHwYDVR0jBBgwFoAUhLbWEa0I
+      UIixKPBVvuKxhK6UMnMwDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAgNJADBG
+      AiEAqgTlOPOiNJLPJhMjRl9Zpaq6TTGfh+awe7N3fcEdHVICIQDfgVRRkuv1KTWk
+      44YBh2/IaTSFwFo8cd39Fnv7CYi/2g==
+      -----END CERTIFICATE-----
+      CERTEOF
+      cat >${{targets.destdir}}/usr/local/share/ca-certificates/cert-d.crt <<'CERTEOF'
+      -----BEGIN CERTIFICATE-----
+      MIIBwTCCAWegAwIBAgIUPrm4YvABD98JhdU93qPsAgryo0UwCgYIKoZIzj0EAwIw
+      NTEeMBwGA1UEAwwVVGVzdCBDQSBDZXJ0aWZpY2F0ZSA0MRMwEQYDVQQKDApUZXN0
+      IE9yZyA0MCAXDTI2MDIyNzIwNDAwMFoYDzIxMjYwMjAzMjA0MDAwWjA1MR4wHAYD
+      VQQDDBVUZXN0IENBIENlcnRpZmljYXRlIDQxEzARBgNVBAoMClRlc3QgT3JnIDQw
+      WTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQbR9hBg7/IeSBYJzUvBUxnnaNmoOJj
+      ESG5CiOa2980CC5aixcLof5kk/9K16B+OLIGSUE+Ya98N0vNP8KmDmvBo1MwUTAd
+      BgNVHQ4EFgQU6ZlpZtkvodhxZX1aRsM44dY0SJ8wHwYDVR0jBBgwFoAU6ZlpZtkv
+      odhxZX1aRsM44dY0SJ8wDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAgNIADBF
+      AiARCNSY4WZ7Tl1oAmWghJz0Sxzi57JY4pdrvzyzYQNrhgIhAPMAzTOf33fVRhaX
+      wB7TKj2HAGTDpoliTH80SMWJN3jK
+      -----END CERTIFICATE-----
+      CERTEOF