Skip to content

Commit 95c8740

Browse files
tandereggtanderegg
authored
Merge pull request #136 from resource-watch/dev
Update Staging from Dev
2 parents e6ef27f + a979d05 commit 95c8740

File tree

90 files changed

+6714
-211
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

90 files changed

+6714
-211
lines changed

‎.github/workflows/terraform_build.yaml‎

Lines changed: 119 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,136 @@
11
name: Run tests and apply terraform changes for current branch
22

3+
concurrency:
4+
group: deploy-terraform
5+
cancel-in-progress: false
6+
37
on:
48
push:
59
branches: [dev, staging]
610

11+
permissions:
12+
id-token: write
13+
contents: read
14+
pull-requests: read
15+
actions: read
16+
717
jobs:
8-
build_dev:
18+
build_eks_cluster:
919
runs-on: ubuntu-latest
1020

1121
env:
12-
ENV: ${{ github.ref_name }}
13-
AWS_ACCESS_KEY_ID: >-
14-
${{ github.ref_name == 'production' && secrets.aws_key_production ||
15-
github.ref_name == 'staging' && secrets.aws_key_staging ||
16-
secrets.aws_key_dev }}
17-
AWS_SECRET_ACCESS_KEY: >-
18-
${{ github.ref_name == 'production' && secrets.aws_secret_production ||
19-
github.ref_name == 'staging' && secrets.aws_secret_staging ||
20-
secrets.aws_secret_dev }}
21-
AWS_REGION: >-
22-
${{ github.ref_name == 'production' && secrets.aws_region_production ||
23-
github.ref_name == 'staging' && secrets.aws_region_staging ||
24-
secrets.aws_region_dev }}
25-
TF_VAR_cloudflare_api_key: ${{ secrets.cloudflare_api_key }}
26-
TF_VAR_cloudflare_email: ${{ secrets.cloudflare_email }}
27-
TF_VAR_sparkpost_api_key: ${{ secrets.sparkpost_api_key }}
22+
ENV: ${{ github.ref_name }}
23+
AWS_ACCESS_KEY_ID: >-
24+
${{ github.ref_name == 'production' && secrets.aws_key_production ||
25+
github.ref_name == 'staging' && secrets.aws_key_staging ||
26+
secrets.aws_key_dev }}
27+
AWS_SECRET_ACCESS_KEY: >-
28+
${{ github.ref_name == 'production' && secrets.aws_secret_production ||
29+
github.ref_name == 'staging' && secrets.aws_secret_staging ||
30+
secrets.aws_secret_dev }}
31+
AWS_REGION: >-
32+
${{ github.ref_name == 'production' && secrets.aws_region_production ||
33+
github.ref_name == 'staging' && secrets.aws_region_staging ||
34+
secrets.aws_region_dev }}
35+
TF_VAR_cloudflare_api_key: ${{ secrets.cloudflare_api_key }}
36+
TF_VAR_cloudflare_email: ${{ secrets.cloudflare_email }}
37+
TF_VAR_sparkpost_api_key: ${{ secrets.sparkpost_api_key }}
2838

2939
steps:
30-
- uses: actions/checkout@v1
40+
- uses: actions/checkout@v4
41+
42+
- name: Setup terraform
43+
uses: hashicorp/setup-terraform@v3
44+
with:
45+
terraform_version: 1.3.6
3146

3247
- name: TF Init
33-
run: ./scripts/infra -chdir=terraform init -backend-config=vars/backend-$ENV.tfvars
48+
run: terraform -chdir=terraform init -backend-config=vars/backend-$ENV.tfvars
49+
50+
#- name: TF Plan
51+
# run: |
52+
# terraform -chdir=terraform plan -var-file=vars/terraform-$ENV.tfvars \
53+
# -var "cloudflare_api_key=${TF_VAR_cloudflare_api_key}" \
54+
# -var "cloudflare_email=${TF_VAR_cloudflare_email}" \
55+
# -var "sparkpost_api_key=${TF_VAR_sparkpost_api_key}" \
56+
# -out tf.plan
57+
58+
- name: Get PR Number
59+
uses: jwalton/gh-find-current-pr@master
60+
id: findpr
61+
with:
62+
state: closed
63+
64+
- name: Download TF EKS Cluster Plan
65+
uses: dawidd6/action-download-artifact@v3
66+
with:
67+
github_token: ${{ secrets.GITHUB_TOKEN }}
68+
workflow: terraform_plan.yaml
69+
pr: ${{ steps.findpr.outputs.pr }}
70+
name: tf_eks-${{ steps.findpr.outputs.pr }}.plan
71+
path: terraform/
72+
check_artifacts: true
3473

35-
- name: TF Plan
36-
run: |
37-
./scripts/infra -chdir=terraform plan -var-file=vars/terraform-$ENV.tfvars \
38-
-var "cloudflare_api_key=${TF_VAR_cloudflare_api_key}" \
39-
-var "cloudflare_email=${TF_VAR_cloudflare_email}" \
40-
-var "sparkpost_api_key=${TF_VAR_sparkpost_api_key}"
4174
- name: TF Apply
4275
run: |
43-
./scripts/infra -chdir=terraform apply -auto-approve -var-file=vars/terraform-$ENV.tfvars \
44-
-var "cloudflare_api_key=${TF_VAR_cloudflare_api_key}" \
45-
-var "cloudflare_email=${TF_VAR_cloudflare_email}" \
46-
-var "sparkpost_api_key=${TF_VAR_sparkpost_api_key}"
76+
terraform -chdir=terraform apply tf_eks-${{ steps.findpr.outputs.pr }}.plan
77+
78+
build_k8s_infra:
79+
runs-on: ubuntu-latest
80+
needs: build_eks_cluster
81+
82+
env:
83+
ENV: ${{ github.ref_name }}
84+
AWS_ROLE: >-
85+
${{ github.base_ref == 'production' && 'TBD' ||
86+
github.base_ref == 'staging' && 'TBD' ||
87+
'arn:aws:iam::842534099497:role/wri-api-dev-githubactions-role' }}
88+
AWS_REGION: >-
89+
${{ github.ref_name == 'production' && secrets.aws_region_production ||
90+
github.ref_name == 'staging' && secrets.aws_region_staging ||
91+
secrets.aws_region_dev }}
92+
TF_VAR_cloudflare_api_key: ${{ secrets.cloudflare_api_key }}
93+
TF_VAR_cloudflare_email: ${{ secrets.cloudflare_email }}
94+
TF_VAR_sparkpost_api_key: ${{ secrets.sparkpost_api_key }}
95+
96+
steps:
97+
- name: Checkout repository
98+
uses: actions/checkout@v4
99+
100+
- name: Configure AWS Credentials
101+
uses: aws-actions/configure-aws-credentials@main
102+
with:
103+
role-to-assume: ${{ env.AWS_ROLE }}
104+
aws-region: ${{ env.AWS_REGION }}
105+
106+
- name: Setup terraform
107+
uses: hashicorp/setup-terraform@v3
108+
with:
109+
terraform_version: 1.3.6
110+
111+
- name: Configure Kubeconfig
112+
run: aws eks update-kubeconfig --region us-east-1 --name core-k8s-cluster-$ENV
113+
114+
- name: TF Init K8s Infra
115+
run: terraform -chdir=terraform-k8s-infrastructure init -backend-config=vars/backend-$ENV.tfvars
116+
117+
- name: Get PR Number
118+
uses: jwalton/gh-find-current-pr@master
119+
id: findpr
120+
with:
121+
state: closed
122+
123+
- name: Download TF k8s Infra Plan
124+
uses: dawidd6/action-download-artifact@v3
125+
with:
126+
github_token: ${{ secrets.GITHUB_TOKEN }}
127+
workflow: terraform_plan.yaml
128+
pr: ${{ steps.findpr.outputs.pr }}
129+
name: tf_k8s_infra-${{ steps.findpr.outputs.pr }}.plan
130+
path: terraform-k8s-infrastructure/
131+
check_artifacts: true
132+
133+
- name: TF Apply
134+
run: |
135+
terraform -chdir=terraform-k8s-infrastructure apply \
136+
tf_k8s_infra-${{ steps.findpr.outputs.pr }}.plan

‎.github/workflows/terraform_plan.yaml‎

Lines changed: 84 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,18 @@
1-
name: Plan terraform changes for base branch
1+
name: Terraform Plan
2+
3+
concurrency:
4+
group: deploy-terraform
5+
cancel-in-progress: false
26

37
on:
48
pull_request:
59
branches: [dev, staging]
610

11+
permissions:
12+
id-token: write
13+
714
jobs:
8-
plan:
15+
plan_eks_cluster:
916
runs-on: ubuntu-latest
1017

1118
env:
@@ -27,14 +34,83 @@ jobs:
2734
TF_VAR_sparkpost_api_key: ${{ secrets.sparkpost_api_key }}
2835

2936
steps:
30-
- uses: actions/checkout@v1
37+
- name: Checkout repository
38+
uses: actions/checkout@v4
39+
40+
- name: Setup terraform
41+
uses: hashicorp/setup-terraform@v3
42+
with:
43+
terraform_version: 1.3.6
44+
45+
- name: TF Init EKS Cluster
46+
run: terraform -chdir=terraform init -backend-config=vars/backend-$ENV.tfvars
47+
48+
- name: TF Plan EKS Cluster
49+
run: |
50+
terraform -chdir=terraform plan -var-file=vars/terraform-$ENV.tfvars \
51+
-var "cloudflare_api_key=${TF_VAR_cloudflare_api_key}" \
52+
-var "cloudflare_email=${TF_VAR_cloudflare_email}" \
53+
-var "sparkpost_api_key=${TF_VAR_sparkpost_api_key}" \
54+
-out tf_eks-${{ github.event.pull_request.number }}.plan
55+
56+
- name: Upload EKS Cluster Plan File
57+
uses: actions/upload-artifact@v4
58+
with:
59+
name: tf_eks-${{ github.event.pull_request.number }}.plan
60+
path: "terraform/tf_eks-${{ github.event.pull_request.number }}.plan"
61+
if-no-files-found: 'error'
62+
overwrite: true
63+
64+
plan_k8s_infra:
65+
runs-on: ubuntu-latest
66+
needs: plan_eks_cluster
67+
68+
env:
69+
ENV: ${{ github.base_ref }}
70+
AWS_ROLE: >-
71+
${{ github.base_ref == 'production' && 'TBD' ||
72+
github.base_ref == 'staging' && 'arn:aws:iam::843801476059:role/wri-api-staging-githubactions-role' ||
73+
'arn:aws:iam::842534099497:role/wri-api-dev-githubactions-role' }}
74+
AWS_REGION: >-
75+
${{ github.base_ref == 'production' && secrets.aws_region_production ||
76+
github.base_ref == 'staging' && secrets.aws_region_staging ||
77+
secrets.aws_region_dev }}
78+
TF_VAR_cloudflare_api_key: ${{ secrets.cloudflare_api_key }}
79+
TF_VAR_cloudflare_email: ${{ secrets.cloudflare_email }}
80+
TF_VAR_sparkpost_api_key: ${{ secrets.sparkpost_api_key }}
81+
82+
steps:
83+
- name: Checkout repository
84+
uses: actions/checkout@v4
85+
86+
- name: Configure AWS Credentials
87+
uses: aws-actions/configure-aws-credentials@main
88+
with:
89+
role-to-assume: ${{ env.AWS_ROLE }}
90+
aws-region: ${{ env.AWS_REGION }}
91+
92+
- name: Setup terraform
93+
uses: hashicorp/setup-terraform@v3
94+
with:
95+
terraform_version: 1.3.6
96+
97+
- name: Configure Kubeconfig
98+
run: aws eks update-kubeconfig --region us-east-1 --name core-k8s-cluster-$ENV
3199

32-
- name: TF Init
33-
run: ./scripts/infra -chdir=terraform init -backend-config=vars/backend-$ENV.tfvars
100+
- name: TF Init K8s Infra
101+
run: terraform -chdir=terraform-k8s-infrastructure init -backend-config=vars/backend-$ENV.tfvars
34102

35-
- name: TF Plan
103+
- name: TF Plan K8s Infra
36104
run: |
37-
./scripts/infra -chdir=terraform plan -var-file=vars/terraform-$ENV.tfvars \
105+
terraform -chdir=terraform-k8s-infrastructure plan -var-file=vars/terraform-$ENV.tfvars \
38106
-var "cloudflare_api_key=${TF_VAR_cloudflare_api_key}" \
39107
-var "cloudflare_email=${TF_VAR_cloudflare_email}" \
40-
-var "sparkpost_api_key=${TF_VAR_sparkpost_api_key}"
108+
-out tf_k8s_infra-${{ github.event.pull_request.number }}.plan
109+
110+
- name: Upload K8s Infrastructure Plan File
111+
uses: actions/upload-artifact@v4
112+
with:
113+
name: tf_k8s_infra-${{ github.event.pull_request.number }}.plan
114+
path: "terraform-k8s-infrastructure/tf_k8s_infra-${{ github.event.pull_request.number }}.plan"
115+
if-no-files-found: 'error'
116+
overwrite: true

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ crash.log
2525
# version control.
2626
#
2727
# example.tfvars
28+
*.auto.tfvars
2829

2930
# Ignore override files as they are usually used to override resources locally and so
3031
# are not checked in
@@ -52,3 +53,6 @@ terraform-k8s-infrastructure/vars/private.tfvars
5253

5354
terraform.tfstate
5455
terraform.tfstate.backup
56+
57+
# Snyk Security Extension - AI Rules (auto-generated)
58+
.github/instructions/snyk_rules.instructions.md

‎README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@
44

55
For a description of the setup, see the infrastructure [section](https://resource-watch.github.io/doc-api/developer.html#infrastructure-configuration) of the developer documentation.
66

7+
# Github Actions
8+
Github Actions (GHA) has been setup to run `terraform plan` when a PR is opened to either the `dev`, `staging`, or `production` (TODO) branches, and `terraform apply` when the PR is merged. This makes use of an OIDC role as described here: https://docs.github.com/en/actions/how-tos/secure-your-work/security-harden-deployments/oidc-in-aws. The role for each environment was created manually, and is specified using the `gha_role_arn` Terraform variable.
9+
710
## Setting up the AWS resources
811

912
To setup the cluster cloud resources, use the following command:

‎scripts/README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# PVC Debugger
2+
3+
export PVC_NAME=<my-pvc-name>
4+
envsubst < pvc-debugger.yaml | kubectl apply -n <namespace> -f -

‎scripts/pvc-debugger.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
apiVersion: v1
2+
kind: Pod
3+
metadata:
4+
name: volume-debugger
5+
spec:
6+
volumes:
7+
- name: volume-to-debug
8+
persistentVolumeClaim:
9+
claimName: ${PVC_NAME} # Replace with your PVC name
10+
containers:
11+
- name: debugger
12+
image: busybox:stable
13+
command: ['sleep', '3600'] # Keeps the pod running
14+
volumeMounts:
15+
- mountPath: "/data" # The path where the volume will be mounted
16+
name: volume-to-debug
17+
restartPolicy: Never

0 commit comments

Comments
 (0)