feat: Cluster Autoscaler #1345

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft

hemasenthil13 wants to merge 26 commits into main from cas

pod-configs/module/eks-cas/main.tf

-Original file line number
+Diff line change
@@ -0,0 +1,56 @@
+    # creating service account for cas controller
+    resource "kubernetes_service_account" "cluster_autoscaler" {
+      metadata {
+        name      = var.cas_service_account
+        namespace = var.cas_namespace
+        annotations = {
+          "eks.amazonaws.com/role-arn" = var.cas_controller_arn
+        }
+      }
+    }
+    resource "helm_release" "cluster_autoscaler" {
+      name       = "cluster-autoscaler"
+      repository = "https://kubernetes.github.io/autoscaler"
+      chart      = "cluster-autoscaler"
+      namespace  = var.cas_namespace
+      depends_on = [
+        kubernetes_service_account.cluster_autoscaler
+      ]
+      set = [
+      {
+        name  = "priorityClassName"
+        value = "system-cluster-critical"
+      },
+      {
+        name  = "autoDiscovery.clusterName"
+        value = var.cluster_name
+      },
+      {
+        name  = "awsRegion"
+        value = var.aws_region
+      },
+      {
+        name  = "rbac.serviceAccount.create"
+        value = "false"
+      },
+      {
+        name  = "rbac.serviceAccount.name"
+        value = var.cas_service_account
+      },
+      {
+        name  = "extraArgs.balance-similar-node-groups"
+        value = "true"
+      },
+      {
+        name  = "extraArgs.expander"
+        value = "least-waste"
+      }
+    ]
+    }

pod-configs/module/eks-cas/output.tf

Empty file.

pod-configs/module/eks-cas/variable.tf

-Original file line number
+Diff line change
@@ -0,0 +1,21 @@
+    variable "cluster_name" {
+      type = string
+    }
+    variable "cas_namespace" {
+      description = "The Kubernetes namespace for cluster autoscaler"
+      default     = "kube-system"
+    }
+    variable "cas_service_account" {
+      description = "The Kubernetes service account name of cluster autoscaler"
+      default     = "cluster-autoscaler"
+    }
+    variable "cas_controller_arn" {
+      type = string
+    }
+    variable "aws_region" {
+      type = string
+    }

pod-configs/module/eks/main.tf

-Original file line number
+Diff line change
@@ Expand Up / @@ -310,6 +310,12 @@ resource "aws_eks_node_group" "nodegroup" { @@
         aws_iam_role_policy_attachment.AmazonEBSCSIDriverPolicy,
         aws_launch_template.eks_launch_template
       ]
+      tags = {
+        "k8s.io/cluster-autoscaler/enabled"               = "true"
+        "k8s.io/cluster-autoscaler/${var.cluster_name}"  = "owned"
+      }
     }
     resource "aws_eks_node_group" "additional_node_group" {
@@ Expand Down Expand Up / @@ -344,6 +350,14 @@ resource "aws_eks_node_group" "additional_node_group" { @@
         }
       }
+      tags = merge(
+        {
+          "k8s.io/cluster-autoscaler/enabled"              = "true"
+          "k8s.io/cluster-autoscaler/${var.cluster_name}" = "owned"
+        },
+        lookup(each.value, "tags", {})
+      )
       depends_on = [
         aws_iam_role_policy_attachment.AmazonEKSWorkerNodePolicy,
         aws_iam_role_policy_attachment.AmazonEKS_CNI_Policy,
@@ Expand Down Expand Up / @@ -425,7 +439,8 @@ resource "aws_iam_policy" "cas_controller" { @@
             "ec2:DescribeInstanceTypes",
             "ec2:DescribeLaunchTemplateVersions",
             "ec2:GetInstanceTypesFromInstanceRequirements",
-            "eks:DescribeNodegroup"
+            "eks:DescribeNodegroup",
+            "eks:DescribeCluster"
           ],
           "Resource": ["*"]
         },
@@ Expand Down @@

pod-configs/module/eks/output.tf

-Original file line number
+Diff line change
@@ Expand Up / @@ -35,3 +35,8 @@ output "eks_security_group_id" { @@
       value = aws_security_group.eks_cluster.id
       description = "The major security group ID for the EKS cluster"
     }
+    output "cas_controller_role_arn" {
+      description = "IAM role ARN used by Cluster Autoscaler (IRSA)"
+      value       = aws_iam_role.cas_controller.arn
+    }

pod-configs/orchestrator/cluster/main.tf

-Original file line number
+Diff line change
@@ Expand Up / @@ -228,3 +228,11 @@ module "gitea" { @@
       # gitea_database_password    = module.aurora_database.user_password["gitea-gitea_user"].result
       # gitea_database             = "gitea-gitea"  # See aurora_database module
     }
+    module "eks-cas" {
+      depends_on    = [module.eks, module.aws_lb_controller]
+      source        = "../../module/eks-cas"
+      aws_region    = var.aws_region
+      cluster_name  = var.eks_cluster_name
+      cas_controller_arn = module.eks.cas_controller_role_arn
+    }

Provide feedback