Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 42 additions & 21 deletions .github/workflows/cre-regression-system-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@
workflow_dispatch:
inputs:
chainlink_image_repository_path:
description: "ECR repository name used to compose image with chainlink_image_tag (for example: chainlink-integration-tests or chainlink or chainlink/chainlink)."
description: "ECR repository name used to compose image with chainlink_image_tag
(for example: chainlink-integration-tests or chainlink or
chainlink/chainlink)."
required: true
type: string
chainlink_image_tag:
Expand All @@ -18,16 +20,20 @@
- "sdlc"
- "public"
default: "sdlc"
description: "Whether to use the SDLC or public ECR registry for the Chainlink image."
description: "Whether to use the SDLC or public ECR registry for the Chainlink
image."
chainlink_version:
required: false
type: string
description: "The version of Chainlink repository to use for the tests. If empty, defaults to github.sha."
description: "The version of Chainlink repository to use for the tests. If
empty, defaults to github.sha."
default: ""
workflow_call:
inputs:
chainlink_image_repository_path:
description: "ECR repository name used to compose image with chainlink_image_tag (for example: chainlink-integration-tests or chainlink or chainlink/chainlink)."
description: "ECR repository name used to compose image with chainlink_image_tag
(for example: chainlink-integration-tests or chainlink or
chainlink/chainlink)."
required: true
type: string
chainlink_image_tag:
Expand All @@ -37,17 +43,19 @@
ecr:
type: string
required: true
default: "sdlc"
description: "Whether to use the SDLC (sdlc) or public ECR registry (public) for the Chainlink image."
description: "Whether to use the SDLC (sdlc) or public ECR registry (public) for
the Chainlink image."
chainlink_version:
required: false
type: string
description: "The version of Chainlink repository to use for the tests. If empty, defaults to github.sha."
description: "The version of Chainlink repository to use for the tests. If
empty, defaults to github.sha."
default: ""

jobs:
define-test-matrix:
runs-on: runs-on=${{ github.run_id }}/cpu=8/ram=32/family=m6i+m5.*/spot=false/image=ubuntu24-full-x64/extras=s3-cache+tmpfs
runs-on: runs-on=${{ github.run_id
}}/cpu=8/ram=32/family=m6i+m5.*/spot=false/image=ubuntu24-full-x64/extras=s3-cache+tmpfs
outputs:
matrix: ${{ steps.define-matrix.outputs.matrix }}
permissions:
Expand Down Expand Up @@ -102,10 +110,12 @@
fail-fast: false
matrix:
tests: ${{fromJson(needs.define-test-matrix.outputs.matrix)}}
needs: [define-test-matrix]
needs: [ define-test-matrix ]
# we need a `test_id` and `run_attempt` here to stop runner stealing
# see: https://runs-on.com/guides/troubleshoot/#runner-stealing-and-matrix-jobs
runs-on: runs-on=${{ github.run_id }}-${{ matrix.tests.test_id }}-${{ github.run_attempt }}/cpu=16/ram=64/family=m6i+m5.*/spot=false/image=ubuntu24-full-x64/extras=s3-cache+tmpfs
runs-on: runs-on=${{ github.run_id }}-${{ matrix.tests.test_id }}-${{
github.run_attempt
}}/cpu=16/ram=64/family=m6i+m5.*/spot=false/image=ubuntu24-full-x64/extras=s3-cache+tmpfs
environment:
# http://docs.github.com/en/actions/how-tos/deploy/configure-and-manage-deployments/control-deployments#using-environments-without-deployments
name: integration
Expand All @@ -115,10 +125,16 @@
ENABLE_AUTO_QUARANTINE: "true"
# override Chip Ingress and Chip Config images with remote images. We have added this env var here, instead of the "Start local CRE" step, because
# Beholder stack will be started only for the Beholder tests
CHIP_INGRESS_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/atlas-chip-ingress:da84cb72d3a160e02896247d46ab4b9806ebee2f
CHIP_CONFIG_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/atlas-chip-config:7b4e9ee68fd1c737dd3480b5a3ced0188f29b969
CTF_CHIP_ROUTER_IMAGE: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/local-cre-chip-router:v1.0.1"
BILLING_PLATFORM_SERVICE_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/billing-platform-service:v1.45.0
CTF_CHIP_INGRESS_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
secrets.QA_AWS_REGION
}}.amazonaws.com/atlas-chip-ingress:da84cb72d3a160e02896247d46ab4b9806ebee2f
CTF_CHIP_CONFIG_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
secrets.QA_AWS_REGION
}}.amazonaws.com/atlas-chip-config:7b4e9ee68fd1c737dd3480b5a3ced0188f29b969
CTF_CHIP_ROUTER_IMAGE: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{
secrets.QA_AWS_REGION }}.amazonaws.com/local-cre-chip-router:v1.0.1"
BILLING_PLATFORM_SERVICE_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
secrets.QA_AWS_REGION }}.amazonaws.com/billing-platform-service:v1.45.0

steps:
- name: Enable S3 Cache for Self-Hosted Runners
Expand Down Expand Up @@ -153,7 +169,8 @@
id: login-ecr
uses: aws-actions/amazon-ecr-login@183a1442edf41672e66566b7fc560e297a290896 # v2.1.1
with:
registries: ${{ format('{0},{1}', secrets.QA_AWS_ACCOUNT_NUMBER, secrets.AWS_ACCOUNT_ID_PROD) }}
registries: ${{ format('{0},{1}', secrets.QA_AWS_ACCOUNT_NUMBER,
secrets.AWS_ACCOUNT_ID_PROD) }}
env:
AWS_REGION: ${{ secrets.QA_AWS_REGION }}

Expand Down Expand Up @@ -192,7 +209,8 @@
shell: bash
id: start-local-cre
env:
CTF_JD_IMAGE: "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/job-distributor:0.22.1"
CTF_JD_IMAGE: "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
secrets.QA_AWS_REGION }}.amazonaws.com/job-distributor:0.22.1"
CTF_CHAINLINK_IMAGE: "${{ steps.resolve-chainlink-image.outputs.resolved_image }}"
CTF_CONFIGS: configs/workflow-gateway-capabilities-don.toml
CRE_VERSION: ${{ matrix.tests.cre_version }}
Expand Down Expand Up @@ -225,7 +243,7 @@
TEST_TIMEOUT: 7m # let's leave 3 minutes for other steps (the whole job times out after 10 minutes)
PARALLEL_COUNT: "10"
CRE_TEST_PARALLEL_ENABLED: "true"
run: |

Check warning on line 246 in .github/workflows/cre-regression-system-tests.yaml

View workflow job for this annotation

GitHub Actions / Validate Github Action Workflows

[actionlint] reported by reviewdog 🐶 shellcheck reported issue in this script: SC2086:info:14:41: Double quote to prevent globbing and word splitting [shellcheck] Raw Output: i:.github/workflows/cre-regression-system-tests.yaml:246:9: shellcheck reported issue in this script: SC2086:info:14:41: Double quote to prevent globbing and word splitting [shellcheck]
echo "Starting test: '${TEST_NAME}'"
echo "⚠️⚠️⚠️ Add 'skip-e2e-regression' label to skip this step if necessary ⚠️⚠️⚠️"

Expand All @@ -234,17 +252,20 @@
--junitfile=/tmp/junit-report-regression.xml \
--format=github-actions \
-- \
-v -run "^(${TEST_NAME})$" -timeout ${TEST_TIMEOUT} -count=1 -parallel=${PARALLEL_COUNT} \
-v -run "^(${TEST_NAME})$" -timeout "${TEST_TIMEOUT}" -count=1 -parallel="${PARALLEL_COUNT}" \
github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre

echo "⚠️⚠️⚠️ Add 'skip-e2e-regression' label to skip this step if necessary ⚠️⚠️⚠️"
exit_code=$?
if [ $exit_code -eq 0 ]; then
if [ "$exit_code" -eq 0 ]; then
echo "tests_result=✅ Tests passed" >> $GITHUB_OUTPUT
fi
exit $exit_code

- name: Analyze and upload test results (${{ steps.run-regression-tests.outputs.tests_result }})
echo "⚠️⚠️⚠️ Add 'skip-e2e-regression' label to skip this step if necessary ⚠️⚠️⚠️"

exit "$exit_code"

- name: Analyze and upload test results (${{
steps.run-regression-tests.outputs.tests_result }})
if: ${{ !cancelled() }}
uses: smartcontractkit/.github/actions/branch-out-upload@branch-out-upload/v1
with:
Expand Down
30 changes: 20 additions & 10 deletions .github/workflows/cre-soak-memory-leak.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,19 +28,30 @@
environment:
name: integration
deployment: false
runs-on: runs-on=${{ github.run_id }}/cpu=32/ram=128/family=m6id+m6idn/spot=false/image=ubuntu24-full-x64/extras=s3-cache
runs-on: runs-on=${{ github.run_id
}}/cpu=32/ram=128/family=m6id+m6idn/spot=false/image=ubuntu24-full-x64/extras=s3-cache
timeout-minutes: 270 # 4h30m — test timeout is 4h20m
permissions:
contents: read
id-token: write
env:
CTF_CONFIGS: configs/workflow-gateway-capabilities-don.toml
CRE_SOAK_DURATION: "2h"
CTF_JD_IMAGE: "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/job-distributor:0.22.1"
CTF_CHAINLINK_IMAGE: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/${{ inputs.ecr_name || 'chainlink' }}:${{ inputs.chainlink_image_tag }}"
CHIP_INGRESS_IMAGE: "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/atlas-chip-ingress:da84cb72d3a160e02896247d46ab4b9806ebee2f"
CHIP_CONFIG_IMAGE: "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/atlas-chip-config:7b4e9ee68fd1c737dd3480b5a3ced0188f29b969"
CTF_CHIP_ROUTER_IMAGE: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/local-cre-chip-router:v1.0.1"
CTF_JD_IMAGE: "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
secrets.QA_AWS_REGION }}.amazonaws.com/job-distributor:0.22.1"
CTF_CHAINLINK_IMAGE: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{
secrets.QA_AWS_REGION }}.amazonaws.com/${{ inputs.ecr_name ||
'chainlink' }}:${{ inputs.chainlink_image_tag }}"
CTF_CHIP_INGRESS_IMAGE: "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
secrets.QA_AWS_REGION
}}.amazonaws.com/atlas-chip-ingress:da84cb72d3a160e02896247d46ab4b9806e\
bee2f"
CTF_CHIP_CONFIG_IMAGE: "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
secrets.QA_AWS_REGION
}}.amazonaws.com/atlas-chip-config:7b4e9ee68fd1c737dd3480b5a3ced0188f29\
b969"
CTF_CHIP_ROUTER_IMAGE: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{
secrets.QA_AWS_REGION }}.amazonaws.com/local-cre-chip-router:v1.0.1"

steps:
- name: Enable S3 Cache for Self-Hosted Runners
Expand Down Expand Up @@ -74,7 +85,8 @@
id: login-ecr
uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1
with:
registries: ${{ format('{0},{1}', secrets.QA_AWS_ACCOUNT_NUMBER, secrets.AWS_ACCOUNT_ID_PROD) }}
registries: ${{ format('{0},{1}', secrets.QA_AWS_ACCOUNT_NUMBER,
secrets.AWS_ACCOUNT_ID_PROD) }}
env:
AWS_REGION: ${{ secrets.QA_AWS_REGION }}
- name: Start observability stack
Expand All @@ -101,8 +113,6 @@
id: run-soak
shell: bash
working-directory: system-tests/tests
env:
GITHUB_TOKEN: ${{ steps.github-token.outputs.access-token || '' }}
run: |
gotestsum \
--jsonfile=/tmp/gotest.log \
Expand Down Expand Up @@ -133,8 +143,8 @@
notify-test-failure:
name: Notify about test Failure
#if: failure()
if: false # TODO: Silence for now

Check failure on line 146 in .github/workflows/cre-soak-memory-leak.yml

View workflow job for this annotation

GitHub Actions / Validate Github Action Workflows

[actionlint] reported by reviewdog 🐶 constant expression "false" in condition. remove the if: section [if-cond] Raw Output: e:.github/workflows/cre-soak-memory-leak.yml:146:9: constant expression "false" in condition. remove the if: section [if-cond]
needs: [soak]
needs: [ soak ]
environment:
name: integration
deployment: false
Expand Down
53 changes: 36 additions & 17 deletions .github/workflows/cre-system-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@
workflow_dispatch:
inputs:
chainlink_image_repository_path:
description: "ECR repository name used to compose image with chainlink_image_tag (for example: chainlink-integration-tests or chainlink or chainlink/chainlink)."
description: "ECR repository name used to compose image with chainlink_image_tag
(for example: chainlink-integration-tests or chainlink or
chainlink/chainlink)."
required: true
type: string
chainlink_image_tag:
Expand All @@ -18,16 +20,20 @@
- "sdlc"
- "public"
default: "sdlc"
description: "Whether to use the SDLC or public ECR registry for the Chainlink image."
description: "Whether to use the SDLC or public ECR registry for the Chainlink
image."
chainlink_version:
required: false
type: string
description: "The version of Chainlink repository to use for the tests. If empty, defaults to github.sha."
description: "The version of Chainlink repository to use for the tests. If
empty, defaults to github.sha."
default: ""
workflow_call:
inputs:
chainlink_image_repository_path:
description: "ECR repository name used to compose image with chainlink_image_tag (for example: chainlink-integration-tests or chainlink or chainlink/chainlink)."
description: "ECR repository name used to compose image with chainlink_image_tag
(for example: chainlink-integration-tests or chainlink or
chainlink/chainlink)."
required: true
type: string
chainlink_image_tag:
Expand All @@ -37,17 +43,19 @@
ecr:
type: string
required: true
default: "sdlc"
description: "Whether to use the SDLC (sdlc) or public ECR registry (public) for the Chainlink image."
description: "Whether to use the SDLC (sdlc) or public ECR registry (public) for
the Chainlink image."
chainlink_version:
required: false
type: string
description: "The version of Chainlink repository to use for the tests. If empty, defaults to github.sha."
description: "The version of Chainlink repository to use for the tests. If
empty, defaults to github.sha."
default: ""

jobs:
define-test-matrix:
runs-on: runs-on=${{ github.run_id }}/cpu=8/ram=32/family=m6i+m5.*/spot=false/image=ubuntu24-full-x64/extras=s3-cache+tmpfs
runs-on: runs-on=${{ github.run_id
}}/cpu=8/ram=32/family=m6i+m5.*/spot=false/image=ubuntu24-full-x64/extras=s3-cache+tmpfs
outputs:
matrix: ${{ steps.define-matrix.outputs.matrix }}
permissions:
Expand Down Expand Up @@ -156,10 +164,12 @@
fail-fast: false
matrix:
tests: ${{fromJson(needs.define-test-matrix.outputs.matrix)}}
needs: [define-test-matrix]
needs: [ define-test-matrix ]
# we need a `test_id` and `run_attempt` here to stop runner stealing
# see: https://runs-on.com/guides/troubleshoot/#runner-stealing-and-matrix-jobs
runs-on: runs-on=${{ github.run_id }}-${{ matrix.tests.test_id }}-${{ github.run_attempt }}/cpu=16/ram=64/family=m6i+m5.*/spot=false/image=ubuntu24-full-x64/extras=s3-cache+tmpfs
runs-on: runs-on=${{ github.run_id }}-${{ matrix.tests.test_id }}-${{
github.run_attempt
}}/cpu=16/ram=64/family=m6i+m5.*/spot=false/image=ubuntu24-full-x64/extras=s3-cache+tmpfs
environment:
# http://docs.github.com/en/actions/how-tos/deploy/configure-and-manage-deployments/control-deployments#using-environments-without-deployments
name: integration
Expand All @@ -169,9 +179,14 @@
ENABLE_AUTO_QUARANTINE: "true"
# override Chip Ingress and Chip Config images with remote images. We have added this env var here, instead of the "Start local CRE" step, because
# Beholder stack will be started only for the Beholder tests
CHIP_INGRESS_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/atlas-chip-ingress:da84cb72d3a160e02896247d46ab4b9806ebee2f
CHIP_CONFIG_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/atlas-chip-config:7b4e9ee68fd1c737dd3480b5a3ced0188f29b969
BILLING_PLATFORM_SERVICE_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/billing-platform-service:v1.57.1
CTF_CHIP_INGRESS_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
secrets.QA_AWS_REGION
}}.amazonaws.com/atlas-chip-ingress:da84cb72d3a160e02896247d46ab4b9806ebee2f
CTF_CHIP_CONFIG_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
secrets.QA_AWS_REGION
}}.amazonaws.com/atlas-chip-config:7b4e9ee68fd1c737dd3480b5a3ced0188f29b969
BILLING_PLATFORM_SERVICE_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
secrets.QA_AWS_REGION }}.amazonaws.com/billing-platform-service:v1.57.1

steps:
- name: Enable S3 Cache for Self-Hosted Runners
Expand Down Expand Up @@ -205,7 +220,8 @@
id: login-ecr
uses: aws-actions/amazon-ecr-login@183a1442edf41672e66566b7fc560e297a290896 # v2.1.1
with:
registries: ${{ format('{0},{1}', secrets.QA_AWS_ACCOUNT_NUMBER, secrets.AWS_ACCOUNT_ID_PROD) }}
registries: ${{ format('{0},{1}', secrets.QA_AWS_ACCOUNT_NUMBER,
secrets.AWS_ACCOUNT_ID_PROD) }}
env:
AWS_REGION: ${{ secrets.QA_AWS_REGION }}

Expand Down Expand Up @@ -250,13 +266,15 @@
shell: bash
id: start-local-cre
env:
CTF_JD_IMAGE: "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/job-distributor:0.22.1"
CTF_JD_IMAGE: "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
secrets.QA_AWS_REGION }}.amazonaws.com/job-distributor:0.22.1"
CTF_CHAINLINK_IMAGE: "${{ steps.resolve-chainlink-image.outputs.resolved_image }}"
CTF_CHIP_ROUTER_IMAGE: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/local-cre-chip-router:v1.0.1"
CTF_CHIP_ROUTER_IMAGE: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{
secrets.QA_AWS_REGION }}.amazonaws.com/local-cre-chip-router:v1.0.1"
CTF_CONFIGS: ${{ matrix.tests.configs }}
CRE_VERSION: ${{ matrix.tests.cre_version }}
TEST_NAME: ${{ matrix.tests.test_name }}
run: |

Check warning on line 277 in .github/workflows/cre-system-tests.yaml

View workflow job for this annotation

GitHub Actions / Validate Github Action Workflows

[actionlint] reported by reviewdog 🐶 shellcheck reported issue in this script: SC2034:warning:17:3: i appears unused. Verify use (or export if used externally) [shellcheck] Raw Output: w:.github/workflows/cre-system-tests.yaml:277:9: shellcheck reported issue in this script: SC2034:warning:17:3: i appears unused. Verify use (or export if used externally) [shellcheck]
cd core/scripts/cre/environment

# Check if test requires observability stack
Expand Down Expand Up @@ -338,7 +356,7 @@
GITHUB_TOKEN: ${{ steps.github-token.outputs.access-token || '' }} # to avoid rate limiting when downloading protobuf files from GitHub
PARALLEL_COUNT: "10"
CRE_TEST_PARALLEL_ENABLED: "true"
run: |

Check warning on line 359 in .github/workflows/cre-system-tests.yaml

View workflow job for this annotation

GitHub Actions / Validate Github Action Workflows

[actionlint] reported by reviewdog 🐶 shellcheck reported issue in this script: SC2086:info:12:41: Double quote to prevent globbing and word splitting [shellcheck] Raw Output: i:.github/workflows/cre-system-tests.yaml:359:9: shellcheck reported issue in this script: SC2086:info:12:41: Double quote to prevent globbing and word splitting [shellcheck]
echo "Starting test: '${TEST_NAME}'"
gotestsum \
--jsonfile=/tmp/gotest.log \
Expand All @@ -353,7 +371,8 @@
echo "tests_result=✅ Tests passed" >> $GITHUB_OUTPUT
fi

- name: Analyze and upload test results (${{ steps.run-tests.outputs.tests_result }})
- name: Analyze and upload test results (${{ steps.run-tests.outputs.tests_result
}})
if: ${{ !cancelled() }}
uses: smartcontractkit/.github/actions/branch-out-upload@branch-out-upload/v1
with:
Expand Down
Loading
Loading