Skip to content

Commit cc9fc2b

Browse files
brent-anyscalebrent-anyscale
authored
Merge pull request #67 from anyscale/brent/eks-upd
upd: aws/eks-existing, aws/eks-private, aws/eks-public examples
2 parents 13ace70 + 79d54cf commit cc9fc2b

38 files changed

Lines changed: 2843 additions & 431 deletions

‎.pre-commit-config.yaml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
default_stages: [pre-commit]
33
repos:
44
- repo: https://github.com/antonbabenko/pre-commit-terraform
5-
rev: v1.97.4
5+
rev: v1.105.0
66
hooks:
77
- id: terraform_fmt
88
- id: terraform_validate
@@ -39,11 +39,12 @@ repos:
3939
# - --args=provider google
4040
# - --args=--version "~> 5.0"
4141
- repo: https://github.com/pre-commit/pre-commit-hooks
42-
rev: v5.0.0
42+
rev: v6.0.0
4343
hooks:
4444
- id: trailing-whitespace
4545
- id: end-of-file-fixer
4646
- id: check-yaml
47+
args: [--allow-multiple-documents]
4748
- id: check-added-large-files
4849
- id: detect-private-key
4950
- id: check-merge-conflict

‎examples/aws/eks-existing/README.md‎

Lines changed: 179 additions & 52 deletions
Large diffs are not rendered by default.

‎examples/aws/eks-existing/deploy.sh.tftpl‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
#!/usr/bin/env bash
2+
# Anyscale on an existing EKS cluster — post-terraform deployment commands.
3+
#
4+
# Substitute <eks_cluster_name>, <anyscale_cloud_name>, and
5+
# <node_IAM_role_arn> below before running. Steps 5-8 also depend on
6+
# $CLOUD_DEPLOYMENT_ID (captured from step 1) and $GATEWAY_HOSTNAME
7+
# (captured from step 5).
8+
set -euo pipefail
9+
10+
# ------------------------------------------------------------------
11+
# 1. Register the Anyscale cloud (returns a cldrsrc_... id)
12+
# ------------------------------------------------------------------
13+
echo "==> Step 1/8: Registering the Anyscale cloud..."
14+
# Replace <anyscale_cloud_name> and <node_IAM_role_arn> below:
15+
register_output=$(${registration_command} 2>&1)
16+
echo "$register_output"
17+
export CLOUD_DEPLOYMENT_ID=$(echo "$register_output" | grep -oE 'cldrsrc_[a-zA-Z0-9]+' | head -1 || true)
18+
: "$${CLOUD_DEPLOYMENT_ID:?failed to capture cldrsrc_ id from registration output}"
19+
echo " Captured CLOUD_DEPLOYMENT_ID=$CLOUD_DEPLOYMENT_ID"
20+
21+
# ------------------------------------------------------------------
22+
# 2. Authenticate to the EKS cluster
23+
# ------------------------------------------------------------------
24+
echo "==> Step 2/8: Updating kubeconfig for cluster <eks_cluster_name>..."
25+
aws eks update-kubeconfig --region ${aws_region} --name <eks_cluster_name>
26+
27+
# ------------------------------------------------------------------
28+
# 3. Cluster Autoscaler
29+
# ------------------------------------------------------------------
30+
echo "==> Step 3/8: Installing Cluster Autoscaler..."
31+
helm repo add autoscaler https://kubernetes.github.io/autoscaler 2>/dev/null || true
32+
helm repo update
33+
helm upgrade cluster-autoscaler autoscaler/cluster-autoscaler \
34+
--version 9.46.0 \
35+
--namespace kube-system \
36+
--set awsRegion=${aws_region} \
37+
--set autoDiscovery.clusterName=<eks_cluster_name> \
38+
--install
39+
40+
# ------------------------------------------------------------------
41+
# 4. AWS Load Balancer Controller (region + vpcId explicit so it
42+
# does not need IMDSv2 access to introspect them)
43+
# ------------------------------------------------------------------
44+
echo "==> Step 4/8: Installing AWS Load Balancer Controller..."
45+
helm repo add eks https://aws.github.io/eks-charts 2>/dev/null || true
46+
helm repo update
47+
helm upgrade aws-load-balancer-controller eks/aws-load-balancer-controller \
48+
--version 1.13.2 \
49+
--namespace kube-system \
50+
--set clusterName=<eks_cluster_name> \
51+
--set region=${aws_region} \
52+
--set vpcId=${existing_vpc_id} \
53+
--install
54+
%{ if enable_s3_pvc ~}
55+
56+
# ------------------------------------------------------------------
57+
# 4a. (enable_s3_pvc = true) Install the Mountpoint-S3 CSI managed
58+
# addon and bind it to the IAM role Terraform created.
59+
# ------------------------------------------------------------------
60+
echo "==> Step 4a/8: Installing Mountpoint-S3 CSI managed addon + Pod Identity Association..."
61+
aws eks create-addon \
62+
--cluster-name <eks_cluster_name> \
63+
--addon-name aws-mountpoint-s3-csi-driver \
64+
--resolve-conflicts OVERWRITE
65+
aws eks create-pod-identity-association \
66+
--cluster-name <eks_cluster_name> \
67+
--namespace kube-system \
68+
--service-account s3-csi-driver-sa \
69+
--role-arn ${s3_csi_driver_role_arn}
70+
%{ endif ~}
71+
72+
# ------------------------------------------------------------------
73+
# 5. Envoy Gateway + Anyscale gateway manifests
74+
# (substitutes the cldrsrc slug into the gateway YAML on the fly
75+
# so the TLS Secret refs are correct from the first apply)
76+
#
77+
# CRDs are applied separately because `helm upgrade --install`
78+
# skips `chart/crds/` on subsequent runs by design — so reruns
79+
# with a newer chart version would otherwise leave Gateway API
80+
# and Envoy Gateway CRDs stale.
81+
# ------------------------------------------------------------------
82+
echo "==> Step 5/8: Installing Envoy Gateway v1.7.0 and applying Anyscale gateway manifests..."
83+
EG_CRD_DIR=$(mktemp -d)
84+
trap 'rm -rf "$${EG_CRD_DIR}"' EXIT
85+
helm pull oci://docker.io/envoyproxy/gateway-helm --version v1.7.0 \
86+
--untar --untardir "$${EG_CRD_DIR}"
87+
kubectl apply --server-side --force-conflicts -R -f "$${EG_CRD_DIR}/gateway-helm/crds/"
88+
helm upgrade eg oci://docker.io/envoyproxy/gateway-helm \
89+
--version v1.7.0 \
90+
--namespace envoy-gateway-system \
91+
--create-namespace \
92+
--skip-crds \
93+
--install
94+
kubectl wait --for=condition=available deployment/envoy-gateway \
95+
-n envoy-gateway-system --timeout=120s
96+
97+
SECRET_SLUG="$${CLOUD_DEPLOYMENT_ID//_/-}" # cldrsrc_xxx → cldrsrc-xxx
98+
echo " Substituting SECRET_SLUG=$SECRET_SLUG into sample-values_gateway.yaml..."
99+
sed "s/<cldrsrc-with-dashes>/$${SECRET_SLUG}/g" sample-values_gateway.yaml | kubectl apply -f -
100+
101+
echo " Waiting for Gateway to be Programmed (up to 5 min)..."
102+
kubectl wait -n anyscale-operator --for=condition=Programmed gateway/gateway --timeout=300s
103+
export GATEWAY_HOSTNAME=$(kubectl get gateway gateway -n anyscale-operator \
104+
-o jsonpath='{.status.addresses[0].value}')
105+
echo " Captured GATEWAY_HOSTNAME=$GATEWAY_HOSTNAME"
106+
%{ if enable_s3_pvc ~}
107+
108+
# ------------------------------------------------------------------
109+
# 6. Apply the Mountpoint-S3 PV/PVC (enable_s3_pvc = true)
110+
# ------------------------------------------------------------------
111+
echo "==> Step 6/8: Applying S3 PersistentVolume + PersistentVolumeClaim..."
112+
kubectl apply -f ./generated/pv-pvc.yaml
113+
echo " Waiting for PVC anyscale-shared-fuse to bind (up to 2 min)..."
114+
kubectl wait -n anyscale-operator --for=jsonpath='{.status.phase}'=Bound \
115+
pvc/anyscale-shared-fuse --timeout=120s
116+
%{ endif ~}
117+
118+
# ------------------------------------------------------------------
119+
# 7. Anyscale Operator
120+
# ------------------------------------------------------------------
121+
echo "==> Step 7/8: Installing the Anyscale Operator helm chart..."
122+
helm repo add anyscale https://anyscale.github.io/helm-charts 2>/dev/null || true
123+
helm repo update
124+
helm upgrade anyscale-operator anyscale/anyscale-operator \
125+
--set-string global.cloudDeploymentId="$CLOUD_DEPLOYMENT_ID" \
126+
--set-string global.cloudProvider=aws \
127+
--set-string global.aws.region=${aws_region} \
128+
--set-string workloads.serviceAccount.name=anyscale-operator \
129+
--set networking.gateway.enabled=true \
130+
--set-string networking.gateway.name=gateway \
131+
--set-string networking.gateway.namespace=anyscale-operator \
132+
--set-string networking.gateway.apiVersion=gateway.networking.k8s.io/v1 \
133+
--set-string networking.gateway.hostname="$GATEWAY_HOSTNAME" \
134+
--namespace anyscale-operator \
135+
--install \
136+
--wait --timeout 10m
137+
138+
echo " Waiting for the operator to create the head-node TLS Secret (up to 5 min)..."
139+
kubectl wait --for=create secret/anyscale-$${SECRET_SLUG}-certificate \
140+
-n anyscale-operator --timeout=300s
141+
142+
# ------------------------------------------------------------------
143+
# 8. Verify
144+
# ------------------------------------------------------------------
145+
echo "==> Step 8/8: Verifying the deployment..."
146+
echo " Cert Secret:"
147+
kubectl get secret "anyscale-$${SECRET_SLUG}-certificate" -n anyscale-operator
148+
echo " Gateway listener status:"
149+
kubectl get gateway gateway -n anyscale-operator \
150+
-o jsonpath='{range .status.listeners[*]}{.name}: ResolvedRefs={.conditions[?(@.type=="ResolvedRefs")].status}{"\n"}{end}'
151+
echo " HTTPRoutes (operator-managed, none until workloads launch):"
152+
kubectl get httproutes -n anyscale-operator
153+
echo "==> Done. Anyscale cloud $CLOUD_DEPLOYMENT_ID is ready."

‎examples/aws/eks-existing/main.tf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ module "anyscale_s3" {
1717
module_enabled = true
1818

1919
anyscale_bucket_name = "anyscale-eks-existing-${var.aws_region}"
20+
bucket_namespace = "global"
2021

2122
tags = var.tags
2223
}

‎examples/aws/eks-existing/memorydb.tf‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#################################################################
2+
# Optional Anyscale-managed MemoryDB (Redis) cluster.
3+
#
4+
# When enable_memorydb = true, this provisions a MemoryDB cluster
5+
# via the upstream `aws-anyscale-memorydb` submodule. The cluster
6+
# endpoint is exposed via the `memorydb_endpoint` output and gets
7+
# appended to the Anyscale registration command as
8+
# `--memorydb-cluster-id`, so Anyscale Services head-node fault
9+
# tolerance is wired at registration time (Anyscale CLI/SDK
10+
# >= 0.26.99 required).
11+
#
12+
# Since this example brings its own EKS cluster, you must tell us
13+
# which security group(s) are allowed to reach the MemoryDB SG via
14+
# `memorydb_allowed_security_group_ids` — typically that's your
15+
# EKS managed-node-group security group.
16+
#################################################################
17+
18+
#trivy:ignore:avd-aws-0104
19+
resource "aws_security_group" "memorydb" {
20+
#checkov:skip=CKV2_AWS_5: "Ensure that Security Groups are attached to another resource" - attached via aws_anyscale_memorydb module.
21+
count = var.enable_memorydb ? 1 : 0
22+
23+
name = "anyscale-eks-existing-memorydb"
24+
description = "Anyscale MemoryDB ingress from user-provided EKS node security groups."
25+
vpc_id = var.existing_vpc_id
26+
27+
dynamic "ingress" {
28+
for_each = var.memorydb_allowed_security_group_ids
29+
content {
30+
description = "Redis from caller-allowed SG"
31+
from_port = var.memorydb_port
32+
to_port = var.memorydb_port
33+
protocol = "tcp"
34+
security_groups = [ingress.value]
35+
}
36+
}
37+
38+
tags = var.tags
39+
}
40+
41+
module "anyscale_memorydb" {
42+
#checkov:skip=CKV_TF_1: Example code should use the latest version of the module
43+
#checkov:skip=CKV_TF_2: Example code should use the latest version of the module
44+
source = "github.com/anyscale/terraform-aws-anyscale-cloudfoundation-modules//modules/aws-anyscale-memorydb"
45+
46+
module_enabled = var.enable_memorydb
47+
48+
anyscale_memorydb_name_prefix = "anyscale-mdb-"
49+
50+
memorydb_subnet_ids = var.existing_subnet_ids
51+
memorydb_security_group_ids = var.enable_memorydb ? [aws_security_group.memorydb[0].id] : []
52+
memorydb_port = var.memorydb_port
53+
memorydb_node_type = var.memorydb_node_type
54+
memorydb_num_shards = var.memorydb_num_shards
55+
memorydb_num_replicas_per_shard = var.memorydb_num_replicas_per_shard
56+
57+
tags = var.tags
58+
}

‎examples/aws/eks-existing/outputs.tf‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ locals {
1717
"--kubernetes-zones ${local.kubernetes_zones}",
1818
"--s3-bucket-id ${module.anyscale_s3.s3_bucket_id}",
1919
var.enable_efs ? "--efs-id ${module.anyscale_efs.efs_id}" : null,
20+
var.enable_s3_pvc ? "--persistent-volume-claim anyscale-shared-fuse" : null,
21+
var.enable_memorydb ? "--memorydb-cluster-id ${module.anyscale_memorydb.memorydb_cluster_id}" : null,
2022
"--anyscale-operator-iam-identity <node_IAM_role_arn>",
2123
])
2224

@@ -26,10 +28,62 @@ locals {
2628
"--set-string global.cloudProvider=aws",
2729
"--set-string global.aws.region=${var.aws_region}",
2830
"--set-string workloads.serviceAccount.name=anyscale-operator",
31+
"--set networking.gateway.enabled=true",
32+
"--set-string networking.gateway.name=gateway",
33+
"--set-string networking.gateway.namespace=anyscale-operator",
34+
"--set-string networking.gateway.apiVersion=gateway.networking.k8s.io/v1",
35+
"--set-string networking.gateway.hostname=<gateway-nlb-hostname>",
2936
"--namespace anyscale-operator",
3037
"--create-namespace",
3138
"-i"
3239
])
40+
41+
pv_pvc_yaml = <<-YAML
42+
apiVersion: v1
43+
kind: PersistentVolume
44+
metadata:
45+
name: anyscale-shared-s3
46+
spec:
47+
accessModes:
48+
- ReadWriteMany
49+
capacity:
50+
storage: 1200Gi
51+
storageClassName: ""
52+
claimRef:
53+
namespace: anyscale-operator
54+
name: anyscale-shared-fuse
55+
mountOptions:
56+
- allow-other
57+
- region ${var.aws_region}
58+
- prefix anyscale-shared/
59+
csi:
60+
driver: s3.csi.aws.com
61+
volumeHandle: anyscale-shared-s3-volume
62+
volumeAttributes:
63+
bucketName: ${module.anyscale_s3.s3_bucket_id}
64+
authenticationSource: driver
65+
---
66+
apiVersion: v1
67+
kind: PersistentVolumeClaim
68+
metadata:
69+
name: anyscale-shared-fuse
70+
namespace: anyscale-operator
71+
spec:
72+
accessModes:
73+
- ReadWriteMany
74+
storageClassName: ""
75+
resources:
76+
requests:
77+
storage: 1200Gi
78+
volumeName: anyscale-shared-s3
79+
YAML
80+
}
81+
82+
resource "local_file" "pv_pvc_yaml" {
83+
count = var.enable_s3_pvc ? 1 : 0
84+
85+
filename = "${path.module}/generated/pv-pvc.yaml"
86+
content = local.pv_pvc_yaml
3387
}
3488

3589
output "anyscale_registration_command" {
@@ -41,3 +95,44 @@ output "helm_upgrade_command" {
4195
description = "The helm upgrade command."
4296
value = join(" \\\n\t", local.helm_upgrade_command_parts)
4397
}
98+
99+
output "s3_pvc_bucket_name" {
100+
description = "Name of the S3 bucket exposed as a PVC via the Mountpoint-for-S3 CSI driver. Only set when `enable_s3_pvc = true`."
101+
value = var.enable_s3_pvc ? module.anyscale_s3.s3_bucket_id : null
102+
}
103+
104+
output "s3_pvc_csi_driver_role_arn" {
105+
description = "IAM role ARN that the Mountpoint-for-S3 CSI driver pods should assume via EKS Pod Identity. Pass this to `aws eks create-pod-identity-association --role-arn`. Only set when `enable_s3_pvc = true`."
106+
value = var.enable_s3_pvc ? aws_iam_role.s3_csi_driver[0].arn : null
107+
}
108+
109+
output "memorydb_endpoint" {
110+
description = "MemoryDB cluster configuration endpoint as host:port. Only set when `enable_memorydb = true`."
111+
value = var.enable_memorydb ? "${module.anyscale_memorydb.memorydb_cluster_endpoint_address}:${module.anyscale_memorydb.memorydb_cluster_endpoint_port}" : null
112+
}
113+
114+
#####################################################################
115+
# Rendered post-terraform deployment script — ordered list of every
116+
# helm/kubectl/anyscale command to run after `terraform apply`.
117+
# Because this example uses an existing EKS cluster, you'll need to
118+
# substitute <eks_cluster_name>, <anyscale_cloud_name>, and
119+
# <node_IAM_role_arn> placeholders before running. The script body
120+
# lives in `deploy.sh.tftpl` so it stays plain bash (shellcheck-friendly).
121+
#####################################################################
122+
123+
resource "local_file" "deploy_script" {
124+
filename = "${path.module}/generated/deploy.sh"
125+
content = templatefile("${path.module}/deploy.sh.tftpl", {
126+
aws_region = var.aws_region
127+
existing_vpc_id = var.existing_vpc_id
128+
enable_s3_pvc = var.enable_s3_pvc
129+
s3_csi_driver_role_arn = var.enable_s3_pvc ? aws_iam_role.s3_csi_driver[0].arn : ""
130+
registration_command = join(" \\\n ", local.registration_command_parts)
131+
})
132+
file_permission = "0755"
133+
}
134+
135+
output "deploy_script_path" {
136+
description = "Path to a rendered shell script containing every post-terraform step in order (autoscaler, AWS LBC, optional S3 CSI addon, Envoy Gateway + manifests, PVC, Anyscale Operator, verify). Open it to copy-paste steps after substituting the BYO placeholders (<eks_cluster_name>, <anyscale_cloud_name>, <node_IAM_role_arn>)."
137+
value = local_file.deploy_script.filename
138+
}

0 commit comments

Comments
 (0)