Fix pipeline issues by multiple small changes (#1094)

## Proposed Changes

* Pin ko version it know it's compatible with used go version
* Supply _CLOUDSDK_IMAGE variable to pipelines to force alpine with
docker
* Add _REF_NAME as a substitution only if it's required

Author: jakweg

ci/cloudbuild/release-and-test.yaml

@@ -66,8 +66,8 @@ steps:
     name: "${_CLOUDSDK_IMAGE}"
     args: ["./hack/prow/e2e-test.sh"]
     env:
+      - "_CLOUDSDK_IMAGE=${_CLOUDSDK_IMAGE}"
       - "GCP_PROJECT_ID=$PROJECT_ID"
-      - "REF_NAME=$REF_NAME"
       - "BUILD_ID=$BUILD_ID"
       - "RELEASE_BUCKET=$_RELEASE_BUCKET"
       - "EXPORT_BUCKET=$_EXPORT_BUCKET"

ci/cloudbuild/scheduled/reap-gke-clusters/build-image.yaml

@@ -17,7 +17,7 @@
 #
 # ```sh
 # gcloud builds submit ./ci/cloudbuild/scheduled \
-# --config=./ci/cloudbuild/scheduled/reap-gke-clusters/cloudbuild.yaml
+# --config=./ci/cloudbuild/scheduled/reap-gke-clusters/build-image.yaml
 # ```
 
 images: ["gcr.io/${PROJECT_ID}/reap-gke-clusters"]

ci/cloudbuild/test.yaml

@@ -257,10 +257,31 @@ steps:
       echo -e "import random\n\nimport time\ntime.sleep(random.randint(30,90))" | python3
     done
 
-        gcloud builds submit \
-            --no-source \
-            --config=/workspace/${BUILD_ID}/fresh-cluster.yaml \
-            --substitutions=_CLOUDSDK_COMPUTE_ZONE=$(cat /workspace/${BUILD_ID}/_CLOUDSDK_COMPUTE_ZONE),_CLOUDSDK_CONTAINER_CLUSTER=$(cat /workspace/${BUILD_ID}/_CLOUDSDK_CONTAINER_CLUSTER),_NODE_COUNT=${_NODE_COUNT},_MACHINE_TYPE=${_MACHINE_TYPE},_RELEASE_CHANNEL=${_RELEASE_CHANNEL},_REF_NAME=${_REF_NAME},_ASM_MANAGED=${_ASM_MANAGED}
+    CONFIG_FILE="/workspace/${BUILD_ID}/fresh-cluster.yaml"
+
+    ZONE=$(cat "/workspace/${BUILD_ID}/_CLOUDSDK_COMPUTE_ZONE")
+    CLUSTER=$(cat "/workspace/${BUILD_ID}/_CLOUDSDK_CONTAINER_CLUSTER")
+
+    VARS="_CLOUDSDK_COMPUTE_ZONE=$${ZONE}"
+    VARS="$${VARS},_CLOUDSDK_CONTAINER_CLUSTER=$${CLUSTER}"
+    VARS="$${VARS},_NODE_COUNT=${_NODE_COUNT}"
+    VARS="$${VARS},_MACHINE_TYPE=${_MACHINE_TYPE}"
+    VARS="$${VARS},_RELEASE_CHANNEL=${_RELEASE_CHANNEL}"
+    VARS="$${VARS},_ASM_MANAGED=${_ASM_MANAGED}"
+
+    # Older versions of kf don't use _REF_NAME, therefore check if it exists in the config file before adding it
+    # This is related to https://github.com/google/kf/pull/1092
+    # The if statement below can be safely removed after two versions following v2.11.29.
+    # The reason is that we run tests for previous releases as well. Newer versions will always have _REF_NAME.
+    if grep -q "_REF_NAME" "$$CONFIG_FILE"; then
+        echo "Found _REF_NAME in config, adding to substitutions..."
+        VARS="$${VARS},_REF_NAME=${_REF_NAME}"
+    fi
+
+    gcloud builds submit \
+        --no-source \
+        --config="$$CONFIG_FILE" \
+        --substitutions="$$VARS"
 
     if [ ! -z "${_EXTRA_CERTS_URL}" ]; then
       gsutil cp "${_EXTRA_CERTS_URL}" /workspace/certs.pem

cmd/generate-release/cloudbuild.yaml

@@ -33,7 +33,7 @@ substitutions:
   _CLOUDSDK_IMAGE: 'gcr.io/google.com/cloudsdktool/cloud-sdk:528.0.0-alpine'
   _GOLANG_IMAGE: 'golang:1.25'
   _BUILDPACK_WRAPPER_GOLANG_IMAGE: 'golang:1.17'
-  _KO_VERSION: 'github.com/google/ko@latest'
+  _KO_VERSION: 'github.com/google/ko@v0.18.1'
 
 timeout: '1800s'
 

hack/prow/e2e-test.sh

@@ -25,7 +25,6 @@ _ASM_MANAGED="${ASM_MANAGED:-false}"
 _RELEASE_CHANNEL="${RELEASE_CHANNEL:-REGULAR}"
 _SKIP_UNIT_TESTS="${SKIP_UNIT_TESTS:-true}"
 _EXTRA_CERTS_URL="${_EXTRA_CERTS_URL:-}"
-_REF_NAME="${REF_NAME:-}"
 
 if ! [ -x "$(command -v jq)" ]; then
   apk add --update --no-cache jq
@@ -80,7 +79,7 @@ trap finish EXIT
 gcloud builds submit . \
     --project ${_GCP_PROJECT_ID} \
     --config=cmd/generate-release/cloudbuild.yaml \
-    --substitutions=_RELEASE_BUCKET=${_RELEASE_BUCKET},_GIT_SHA=${git_sha},_VERSION=${release_id} \
+    --substitutions=_RELEASE_BUCKET=${_RELEASE_BUCKET},_GIT_SHA=${git_sha},_VERSION=${release_id},_CLOUDSDK_IMAGE=${_CLOUDSDK_IMAGE} \
     | tee -a ./build-log.txt
 
 # Run integration tests.
@@ -90,7 +89,7 @@ full_release_bucket=gs://${_RELEASE_BUCKET}/${release_id}
 gcloud builds submit . \
     --project ${_GCP_PROJECT_ID} \
     --config=ci/cloudbuild/test.yaml \
-    --substitutions="_CLOUDSDK_COMPUTE_ZONE=random,_CLOUDSDK_CONTAINER_CLUSTER=${cluster_name},_NODE_COUNT=6,_FULL_RELEASE_BUCKET=${full_release_bucket},_DELETE_CLUSTER=${_DELETE_CLUSTER},_MACHINE_TYPE=n1-highmem-4,_RELEASE_CHANNEL=${_RELEASE_CHANNEL},_REF_NAME=${_REF_NAME},_SKIP_UNIT_TESTS=${_SKIP_UNIT_TESTS},_ASM_MANAGED=${_ASM_MANAGED}",_EXTRA_CERTS_URL=${_EXTRA_CERTS_URL} \
+    --substitutions="_CLOUDSDK_COMPUTE_ZONE=random,_CLOUDSDK_CONTAINER_CLUSTER=${cluster_name},_NODE_COUNT=6,_FULL_RELEASE_BUCKET=${full_release_bucket},_DELETE_CLUSTER=${_DELETE_CLUSTER},_MACHINE_TYPE=n1-highmem-4,_RELEASE_CHANNEL=${_RELEASE_CHANNEL},_REF_NAME=${git_sha},_SKIP_UNIT_TESTS=${_SKIP_UNIT_TESTS},_ASM_MANAGED=${_ASM_MANAGED},_CLOUDSDK_IMAGE=${_CLOUDSDK_IMAGE}",_EXTRA_CERTS_URL=${_EXTRA_CERTS_URL} \
     | tee -a ./build-log.txt
 
 # Update result to success if we reach the end, elsewise we'll report a failure

hack/prow/reap.sh

@@ -19,6 +19,8 @@ _GCP_PROJECT_ID="${GCP_PROJECT_ID}"
 
 cd ci/cloudbuild/scheduled/reap-gke-clusters
 
+# Make sure to rebuild reap image after any modifications in following files, see build-image.yaml
+
 python3 delete_im_deployments.py ${_GCP_PROJECT_ID}
 python3 delete_gke_clusters.py ${_GCP_PROJECT_ID}
 python3 cleanup_load_balancers.py ${_GCP_PROJECT_ID}