From 2938447a30ad41941f4638bdfb747ff650f58a0b Mon Sep 17 00:00:00 2001 From: "Cody W. Eilar" Date: Fri, 20 Mar 2026 14:24:56 -0700 Subject: [PATCH 1/5] Allow workers on an older K8s version to join newer CP co-author: Wei-Chen Chen --- bootstrap/kubeadm/config/rbac/role.yaml | 9 + .../internal/cloudinit/cloudinit_test.go | 2 + bootstrap/kubeadm/internal/cloudinit/node.go | 18 + .../kubeadm/internal/cloudinit/node_test.go | 6 +- .../controllers/kubeadmconfig_controller.go | 37 +- .../kubeadmconfig_controller_test.go | 59 ++- .../kubeadm/internal/ignition/ignition.go | 8 + test/e2e/config/docker.yaml | 2 + ...ter-template-topology-kubeadm-version.yaml | 55 +++ ...sterclass-quick-start-kubeadm-version.yaml | 341 ++++++++++++++++++ test/e2e/kubeadm_version_on_join.go | 328 +++++++++++++++++ test/e2e/kubeadm_version_on_join_test.go | 39 ++ 12 files changed, 884 insertions(+), 20 deletions(-) create mode 100644 test/e2e/data/infrastructure-docker/main/cluster-template-topology-kubeadm-version.yaml create mode 100644 test/e2e/data/infrastructure-docker/main/clusterclass-quick-start-kubeadm-version.yaml create mode 100644 test/e2e/kubeadm_version_on_join.go create mode 100644 test/e2e/kubeadm_version_on_join_test.go diff --git a/bootstrap/kubeadm/config/rbac/role.yaml b/bootstrap/kubeadm/config/rbac/role.yaml index fe485cf2c5cf..c302ff0922a5 100644 --- a/bootstrap/kubeadm/config/rbac/role.yaml +++ b/bootstrap/kubeadm/config/rbac/role.yaml @@ -93,3 +93,12 @@ rules: - get - list - watch +- apiGroups: + - controlplane.cluster.x-k8s.io + resources: + - kubeadmcontrolplanes + - kubeadmcontrolplanes/status + verbs: + - get + - list + - watch diff --git a/bootstrap/kubeadm/internal/cloudinit/cloudinit_test.go b/bootstrap/kubeadm/internal/cloudinit/cloudinit_test.go index a66e83a11ce7..418753a7e5ce 100644 --- a/bootstrap/kubeadm/internal/cloudinit/cloudinit_test.go +++ b/bootstrap/kubeadm/internal/cloudinit/cloudinit_test.go @@ -416,6 +416,8 @@ func TestNewJoinNodeCommands(t *testing.T) { - "echo $(date) ': hello PostKubeadmCommands!'"` g.Expect(out).To(ContainSubstring(expectedRunCmd)) + + g.Expect(out).To(ContainSubstring("path: " + KubeadmVersionPath)) } func TestOmittableFields(t *testing.T) { diff --git a/bootstrap/kubeadm/internal/cloudinit/node.go b/bootstrap/kubeadm/internal/cloudinit/node.go index 474b3d08b93e..8e2a51e8af47 100644 --- a/bootstrap/kubeadm/internal/cloudinit/node.go +++ b/bootstrap/kubeadm/internal/cloudinit/node.go @@ -16,6 +16,16 @@ limitations under the License. package cloudinit +import ( + bootstrapv1 "sigs.k8s.io/cluster-api/api/bootstrap/kubeadm/v1beta2" +) + +const ( + // KubeadmVersionPath is the path where the control plane Kubernetes version is written for worker nodes. + // It must exist before kubeadm join runs. + KubeadmVersionPath = "/run/cluster-api/kubeadm-version" +) + const ( nodeCloudInit = `{{.Header}} {{template "files" .WriteFiles}} @@ -52,5 +62,13 @@ type NodeInput struct { func NewNode(input *NodeInput) ([]byte, error) { input.prepare() input.Header = cloudConfigHeader + // Write control plane version to KubeadmVersionPath so it exists before kubeadm join. + versionFile := bootstrapv1.File{ + Path: KubeadmVersionPath, + Owner: "root:root", + Permissions: "0644", + Content: input.KubernetesVersion.String(), + } + input.WriteFiles = append([]bootstrapv1.File{versionFile}, input.WriteFiles...) return generate("Node", nodeCloudInit, input) } diff --git a/bootstrap/kubeadm/internal/cloudinit/node_test.go b/bootstrap/kubeadm/internal/cloudinit/node_test.go index 4fca5b2398b3..2c2cd4d4d2ea 100644 --- a/bootstrap/kubeadm/internal/cloudinit/node_test.go +++ b/bootstrap/kubeadm/internal/cloudinit/node_test.go @@ -46,13 +46,13 @@ func TestNewNode(t *testing.T) { }, }, }, - checkWriteFiles("/etc/foo.conf", "/run/kubeadm/kubeadm-join-config.yaml", "/run/cluster-api/placeholder"), + checkWriteFiles(KubeadmVersionPath, "/etc/foo.conf", "/run/kubeadm/kubeadm-join-config.yaml", "/run/cluster-api/placeholder"), false, }, { - "check for existence of /run/kubeadm/kubeadm-join-config.yaml and /run/cluster-api/placeholder", + "check for existence of kubeadm-version path, /run/kubeadm/kubeadm-join-config.yaml and /run/cluster-api/placeholder", &NodeInput{}, - checkWriteFiles("/run/kubeadm/kubeadm-join-config.yaml", "/run/cluster-api/placeholder"), + checkWriteFiles(KubeadmVersionPath, "/run/kubeadm/kubeadm-join-config.yaml", "/run/cluster-api/placeholder"), false, }, } diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go index 5634dca1c7c7..1fc74eebf7bf 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go @@ -51,7 +51,9 @@ import ( "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/upstream" bsutil "sigs.k8s.io/cluster-api/bootstrap/util" "sigs.k8s.io/cluster-api/controllers/clustercache" + "sigs.k8s.io/cluster-api/controllers/external" "sigs.k8s.io/cluster-api/feature" + "sigs.k8s.io/cluster-api/internal/contract" "sigs.k8s.io/cluster-api/internal/util/taints" "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/cluster-api/util/conditions" @@ -77,6 +79,7 @@ type InitLocker interface { // +kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=kubeadmconfigs;kubeadmconfigs/status;kubeadmconfigs/finalizers,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status;machinesets;machines;machines/status;machinepools;machinepools/status,verbs=get;list;watch +// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=kubeadmcontrolplanes;kubeadmcontrolplanes/status,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=secrets;configmaps,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch @@ -685,7 +688,13 @@ func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope) return res, nil } - kubernetesVersion := scope.ConfigOwner.KubernetesVersion() + // Use the control plane (cluster) version for worker join so that e.g. a 1.34 node uses kubeadm 1.35 + // when the control plane is at 1.35. Fall back to the joining machine's version if the control plane + // version cannot be determined. + kubernetesVersion := r.getControlPlaneVersionForJoin(ctx, scope) + if kubernetesVersion == "" { + kubernetesVersion = scope.ConfigOwner.KubernetesVersion() + } parsedVersion, err := semver.ParseTolerant(kubernetesVersion) if err != nil { return ctrl.Result{}, errors.Wrapf(err, "failed to parse kubernetes version %q", kubernetesVersion) @@ -807,6 +816,32 @@ func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope) return ctrl.Result{RequeueAfter: r.tokenCheckRefreshOrRotationInterval()}, nil } +// getControlPlaneVersionForJoin returns the control plane (cluster) version from the cluster's ControlPlaneRef, +// e.g. KubeadmControlPlane.spec.version. Returns empty string if the cluster has no ControlPlaneRef or the version +// cannot be read (e.g. control plane not found or does not support version). Used for worker join so that +// a 1.34 node uses kubeadm 1.35 when the control plane is at 1.35, for example. +func (r *KubeadmConfigReconciler) getControlPlaneVersionForJoin(ctx context.Context, scope *Scope) string { + if !scope.Cluster.Spec.ControlPlaneRef.IsDefined() { + return "" + } + controlPlane, err := external.GetObjectFromContractVersionedRef(ctx, r.Client, scope.Cluster.Spec.ControlPlaneRef, scope.Cluster.Namespace) + if err != nil { + scope.V(4).Info("Could not get control plane for version, falling back to machine version", "error", err) + return "" + } + cpVersion, err := contract.ControlPlane().Version().Get(controlPlane) + if err != nil { + if !errors.Is(err, contract.ErrFieldNotFound) { + scope.V(4).Info("Could not get control plane version, falling back to machine version", "error", err) + } + return "" + } + if cpVersion == nil { + return "" + } + return *cpVersion +} + func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *Scope) (ctrl.Result, error) { scope.Info("Creating BootstrapData for the joining control plane") diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_test.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_test.go index ca514f47cea6..084164e6a915 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_test.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_test.go @@ -27,6 +27,7 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" bootstrapapi "k8s.io/cluster-bootstrap/token/api" utilfeature "k8s.io/component-base/featuregate/testing" @@ -51,6 +52,11 @@ import ( utilyaml "sigs.k8s.io/cluster-api/util/yaml" ) +const ( + testK8sVersion = "v1.30.1" + testSkewK8sVersion = "v1.31.0" +) + // MachineToBootstrapMapFunc return kubeadm bootstrap configref name when configref exists. func TestKubeadmConfigReconciler_MachineToBootstrapMapFuncReturn(t *testing.T) { g := NewWithT(t) @@ -61,7 +67,7 @@ func TestKubeadmConfigReconciler_MachineToBootstrapMapFuncReturn(t *testing.T) { for i := range 3 { configName := fmt.Sprintf("my-config-%d", i) m := builder.Machine(metav1.NamespaceDefault, fmt.Sprintf("my-machine-%d", i)). - WithVersion("v1.23.1"). + WithVersion(testK8sVersion). WithClusterName(cluster.Name). WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "").Unstructured()). Build() @@ -134,7 +140,7 @@ func TestKubeadmConfigReconciler_TestSecretOwnerReferenceReconciliation(t *testi clusterName := "my-cluster" cluster := builder.Cluster(metav1.NamespaceDefault, clusterName).Build() machine := builder.Machine(metav1.NamespaceDefault, "machine"). - WithVersion("v1.23.1"). + WithVersion(testK8sVersion). WithClusterName(clusterName). WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "cfg").Unstructured()). Build() @@ -234,7 +240,7 @@ func TestKubeadmConfigReconciler_Reconcile_ReturnNilIfReferencedMachineIsNotFoun machine := builder.Machine(metav1.NamespaceDefault, "machine"). WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "cfg").Unstructured()). - WithVersion("v1.23.1"). + WithVersion(testK8sVersion). Build() config := newKubeadmConfig(metav1.NamespaceDefault, "cfg") addKubeadmConfigToMachine(config, machine) @@ -266,7 +272,7 @@ func TestKubeadmConfigReconciler_Reconcile_ReturnEarlyIfMachineHasDataSecretName cluster.Status.Initialization.InfrastructureProvisioned = ptr.To(true) machine := builder.Machine(metav1.NamespaceDefault, "machine"). - WithVersion("v1.23.1"). + WithVersion(testK8sVersion). WithClusterName("cluster1"). WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "cfg").Unstructured()). Build() @@ -305,7 +311,7 @@ func TestKubeadmConfigReconciler_ReturnEarlyIfClusterInfraNotReady(t *testing.T) cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build() machine := builder.Machine(metav1.NamespaceDefault, "machine"). - WithVersion("v1.23.1"). + WithVersion(testK8sVersion). WithClusterName(cluster.Name). WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "cfg").Unstructured()). Build() @@ -345,7 +351,7 @@ func TestKubeadmConfigReconciler_ReturnEarlyIfClusterInfraNotReady(t *testing.T) func TestKubeadmConfigReconciler_Reconcile_ReturnEarlyIfMachineHasNoCluster(t *testing.T) { g := NewWithT(t) machine := builder.Machine(metav1.NamespaceDefault, "machine"). - WithVersion("v1.23.1"). + WithVersion(testK8sVersion). WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "cfg").Unstructured()). Build() config := newKubeadmConfig(metav1.NamespaceDefault, "cfg") @@ -378,7 +384,7 @@ func TestKubeadmConfigReconciler_Reconcile_ReturnNilIfAssociatedClusterIsNotFoun cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build() machine := builder.Machine(metav1.NamespaceDefault, "machine"). - WithVersion("v1.23.1"). + WithVersion(testK8sVersion). WithClusterName(cluster.Name). WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "cfg").Unstructured()). Build() @@ -804,6 +810,7 @@ func TestBootstrapDataFormat(t *testing.T) { isWorker bool format bootstrapv1.Format clusterInitialized bool + withClusterClass bool }{ { name: "cloud-config init config", @@ -828,6 +835,13 @@ func TestBootstrapDataFormat(t *testing.T) { name: "Empty format field", format: bootstrapv1.CloudConfig, }, + { + name: "cloudinit worker with clusterclass on different k8s version", + isWorker: true, + format: bootstrapv1.CloudConfig, + clusterInitialized: true, + withClusterClass: true, + }, } for _, tc := range testcases { @@ -835,6 +849,19 @@ func TestBootstrapDataFormat(t *testing.T) { g := NewWithT(t) cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build() + clusterClass := builder.ClusterClass(metav1.NamespaceDefault, "cluster-class"). + WithControlPlaneInfrastructureMachineTemplate(&unstructured.Unstructured{}). + Build() + // intentionally use a different version from configOwner, where 1.31.0 will be parsed to upstreamv1beta4 + // and <1.30.0 will be parsed to upstreamv1beta3, to test the compatibility of clusterclass and kubeadmconfig versions. + topology := builder.ClusterTopology(). + WithClass(clusterClass.Name). + WithClassNamespace(clusterClass.Namespace). + WithVersion(testSkewK8sVersion). + Build() + if tc.withClusterClass { + cluster.Spec.Topology = *topology + } cluster.Status.Initialization.InfrastructureProvisioned = ptr.To(true) cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443} if tc.clusterInitialized { @@ -1838,7 +1865,7 @@ func TestKubeadmConfigReconciler_computeClusterConfigurationAndAdditionalData(t }, machine: &clusterv1.Machine{ Spec: clusterv1.MachineSpec{ - Version: "v1.23.0", + Version: testK8sVersion, }, }, initConfiguration: &bootstrapv1.InitConfiguration{ @@ -1856,7 +1883,7 @@ func TestKubeadmConfigReconciler_computeClusterConfigurationAndAdditionalData(t clusterConfiguration := &bootstrapv1.ClusterConfiguration{} gotData := k.computeClusterConfigurationAndAdditionalData(tc.cluster, tc.machine, clusterConfiguration, tc.initConfiguration) g.Expect(clusterConfiguration.ControlPlaneEndpoint).To(Equal("myControlPlaneEndpoint:6443")) - g.Expect(gotData.KubernetesVersion).To(Equal(ptr.To("v1.23.0"))) + g.Expect(gotData.KubernetesVersion).To(Equal(ptr.To(testK8sVersion))) g.Expect(gotData.ClusterName).To(Equal(ptr.To("mycluster"))) g.Expect(gotData.PodSubnet).To(Equal(ptr.To("myPodSubnet"))) g.Expect(gotData.ServiceSubnet).To(Equal(ptr.To("myServiceSubnet"))) @@ -1882,13 +1909,13 @@ func TestKubeadmConfigReconciler_Reconcile_AlwaysCheckCAVerificationUnlessReques controlPlaneMachineName := "my-machine" machine := builder.Machine(metav1.NamespaceDefault, controlPlaneMachineName). - WithVersion("v1.23.1"). + WithVersion(testK8sVersion). WithClusterName(cluster.Name). Build() workerMachineName := "my-worker" workerMachine := builder.Machine(metav1.NamespaceDefault, workerMachineName). - WithVersion("v1.23.1"). + WithVersion(testK8sVersion). WithClusterName(cluster.Name). Build() @@ -1969,7 +1996,7 @@ func TestKubeadmConfigReconciler_ClusterToKubeadmConfigs(t *testing.T) { for i := range 3 { configName := fmt.Sprintf("my-config-%d", i) m := builder.Machine(metav1.NamespaceDefault, fmt.Sprintf("my-machine-%d", i)). - WithVersion("v1.23.1"). + WithVersion(testK8sVersion). WithClusterName(cluster.Name). WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, configName).Unstructured()). Build() @@ -2524,7 +2551,7 @@ func TestKubeadmConfigReconciler_ResolveUsers(t *testing.T) { // newWorkerMachineForCluster returns a Machine with the passed Cluster's information and a pre-configured name. func newWorkerMachineForCluster(cluster *clusterv1.Cluster) *clusterv1.Machine { return builder.Machine(cluster.Namespace, "worker-machine"). - WithVersion("v1.23.1"). + WithVersion(testK8sVersion). WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(cluster.Namespace, "conf1").Unstructured()). WithInfrastructureMachine(builder.InfrastructureMachine(cluster.Namespace, "inframachine").Build()). WithClusterName(cluster.Name). @@ -2534,7 +2561,7 @@ func newWorkerMachineForCluster(cluster *clusterv1.Cluster) *clusterv1.Machine { // newControlPlaneMachine returns a Machine with the passed Cluster information and a MachineControlPlaneLabel. func newControlPlaneMachine(cluster *clusterv1.Cluster, name string) *clusterv1.Machine { m := builder.Machine(cluster.Namespace, name). - WithVersion("v1.23.1"). + WithVersion(testK8sVersion). WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "cfg").Unstructured()). WithClusterName(cluster.Name). WithLabels(map[string]string{clusterv1.MachineControlPlaneLabel: ""}). @@ -2548,7 +2575,7 @@ func newMachinePool(cluster *clusterv1.Cluster, name string) *clusterv1.MachineP WithClusterName(cluster.Name). WithLabels(map[string]string{clusterv1.ClusterNameLabel: cluster.Name}). WithBootstrap(bootstrapbuilder.KubeadmConfig(cluster.Namespace, "conf1").Unstructured()). - WithVersion("v1.23.1"). + WithVersion(testK8sVersion). Build() return m } @@ -2691,7 +2718,7 @@ func TestKubeadmConfigReconciler_Reconcile_v1beta2_conditions(t *testing.T) { } machine := builder.Machine(metav1.NamespaceDefault, "my-machine"). - WithVersion("v1.23.1"). + WithVersion(testK8sVersion). WithClusterName(cluster.Name). WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "").Unstructured()). Build() diff --git a/bootstrap/kubeadm/internal/ignition/ignition.go b/bootstrap/kubeadm/internal/ignition/ignition.go index 68e236a69c7c..7ee051dad191 100644 --- a/bootstrap/kubeadm/internal/ignition/ignition.go +++ b/bootstrap/kubeadm/internal/ignition/ignition.go @@ -63,6 +63,14 @@ func NewNode(input *NodeInput) ([]byte, string, error) { return nil, "", fmt.Errorf("node input can't be nil") } + // Write control plane version to KubeadmVersionPath so it exists before kubeadm join. + versionFile := bootstrapv1.File{ + Path: cloudinit.KubeadmVersionPath, + Owner: "root:root", + Permissions: "0644", + Content: input.KubernetesVersion.String(), + } + input.WriteFiles = append([]bootstrapv1.File{versionFile}, input.WriteFiles...) input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...) input.KubeadmCommand = fmt.Sprintf(kubeadmCommandTemplate, joinSubcommand, input.KubeadmVerbosity) diff --git a/test/e2e/config/docker.yaml b/test/e2e/config/docker.yaml index da2922a18842..4fd5a5df6b6a 100644 --- a/test/e2e/config/docker.yaml +++ b/test/e2e/config/docker.yaml @@ -197,12 +197,14 @@ providers: - sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-autoscaler.yaml" - sourcePath: "../data/infrastructure-docker/main/cluster-template-topology.yaml" - sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-taints.yaml" + - sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-kubeadm-version.yaml" - sourcePath: "../data/infrastructure-docker/main/cluster-template-ignition.yaml" - sourcePath: "../data/infrastructure-docker/main/cluster-template-in-memory.yaml" - sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start.yaml" - sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start-kcp-only.yaml" - sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start-runtimesdk.yaml" - sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start-runtimesdk-v1beta1.yaml" + - sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start-kubeadm-version.yaml" - sourcePath: "../data/infrastructure-docker/main/clusterclass-in-memory.yaml" - sourcePath: "../data/shared/main/metadata.yaml" diff --git a/test/e2e/data/infrastructure-docker/main/cluster-template-topology-kubeadm-version.yaml b/test/e2e/data/infrastructure-docker/main/cluster-template-topology-kubeadm-version.yaml new file mode 100644 index 000000000000..9b503f54667d --- /dev/null +++ b/test/e2e/data/infrastructure-docker/main/cluster-template-topology-kubeadm-version.yaml @@ -0,0 +1,55 @@ +apiVersion: v1 +binaryData: null +data: ${CNI_RESOURCES} +kind: ConfigMap +metadata: + name: cni-${CLUSTER_NAME}-crs-0 +--- +apiVersion: addons.cluster.x-k8s.io/v1beta2 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-crs-0 +spec: + clusterSelector: + matchLabels: + cni: ${CLUSTER_NAME}-crs-0 + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-crs-0 + strategy: ApplyOnce +--- +apiVersion: cluster.x-k8s.io/v1beta2 +kind: Cluster +metadata: + labels: + cni: ${CLUSTER_NAME}-crs-0 + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - ${DOCKER_POD_CIDRS} + serviceDomain: ${DOCKER_SERVICE_DOMAIN} + services: + cidrBlocks: + - ${DOCKER_SERVICE_CIDRS} + topology: + classRef: + name: quick-start-kubeadm-version + namespace: ${CLUSTER_CLASS_NAMESPACE:-${NAMESPACE}} + controlPlane: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + variables: + - name: etcdImageTag + value: "" + - name: coreDNSImageTag + value: "" + - name: preLoadImages + value: ${DOCKER_PRELOAD_IMAGES:-[]} + version: ${KUBERNETES_VERSION} + workers: + machineDeployments: + - class: default-worker + name: md-0 + replicas: ${WORKER_MACHINE_COUNT} diff --git a/test/e2e/data/infrastructure-docker/main/clusterclass-quick-start-kubeadm-version.yaml b/test/e2e/data/infrastructure-docker/main/clusterclass-quick-start-kubeadm-version.yaml new file mode 100644 index 000000000000..b4c03dc157fa --- /dev/null +++ b/test/e2e/data/infrastructure-docker/main/clusterclass-quick-start-kubeadm-version.yaml @@ -0,0 +1,341 @@ +apiVersion: cluster.x-k8s.io/v1beta2 +kind: ClusterClass +metadata: + name: quick-start-kubeadm-version +spec: + controlPlane: + templateRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + name: kubeadm-version-control-plane + machineInfrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: DockerMachineTemplate + name: kubeadm-version-control-plane + infrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: DockerClusterTemplate + name: kubeadm-version-cluster + workers: + machineDeployments: + - class: default-worker + bootstrap: + templateRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KubeadmConfigTemplate + name: kubeadm-version-default-worker-bootstraptemplate + infrastructure: + templateRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: DockerMachineTemplate + name: kubeadm-version-default-worker-machinetemplate + variables: + - name: lbImageRepository + required: true + schema: + openAPIV3Schema: + type: string + default: kindest + - name: etcdImageTag + required: true + schema: + openAPIV3Schema: + type: string + default: "" + example: "3.5.3-0" + description: "etcdImageTag sets the tag for the etcd image." + - name: coreDNSImageTag + required: true + schema: + openAPIV3Schema: + type: string + default: "" + example: "v1.8.5" + description: "coreDNSImageTag sets the tag for the coreDNS image." + - name: preLoadImages + required: false + schema: + openAPIV3Schema: + default: [] + type: array + items: + type: string + description: "preLoadImages sets the images for the docker machines to preload." + patches: + - name: lbImageRepository + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: DockerClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/loadBalancer" + valueFrom: + template: | + imageRepository: {{ .lbImageRepository }} + - name: etcdImageTag + enabledIf: '{{ ne .etcdImageTag "" }}' + description: "Sets tag to use for the etcd image in the KubeadmControlPlane." + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/etcd" + valueFrom: + template: | + local: + imageTag: {{ .etcdImageTag }} + - name: coreDNSImageTag + enabledIf: '{{ ne .coreDNSImageTag "" }}' + description: "Sets tag to use for the coreDNS image in the KubeadmControlPlane." + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/dns" + valueFrom: + template: | + imageTag: {{ .coreDNSImageTag }} + - name: customImage + description: "Sets the container image that is used for running dockerMachines for the controlPlane and default-worker machineDeployments." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: DockerMachineTemplate + matchResources: + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/customImage" + valueFrom: + template: | + kindest/node:{{ .builtin.machineDeployment.version | replace "+" "_" }} + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: DockerMachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/customImage" + valueFrom: + template: | + kindest/node:{{ .builtin.controlPlane.version | replace "+" "_" }} + - name: preloadImages + description: "Sets the container images to preload to the node that is used for running dockerMachines." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: DockerMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/preLoadImages" + valueFrom: + variable: preLoadImages + - name: podSecurityStandard + description: "Adds an admission configuration for PodSecurity to the kube-apiserver." + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs" + value: + - name: admission-control-config-file + value: "/etc/kubernetes/kube-apiserver-admission-pss.yaml" + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraVolumes" + value: + - name: admission-pss + hostPath: /etc/kubernetes/kube-apiserver-admission-pss.yaml + mountPath: /etc/kubernetes/kube-apiserver-admission-pss.yaml + readOnly: true + pathType: "File" + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/files" + valueFrom: + template: | + - content: | + apiVersion: apiserver.config.k8s.io/v1 + kind: AdmissionConfiguration + plugins: + - name: PodSecurity + configuration: + apiVersion: pod-security.admission.config.k8s.io/v1 + kind: PodSecurityConfiguration + defaults: + enforce: "baseline" + enforce-version: "latest" + audit: "baseline" + audit-version: "latest" + warn: "baseline" + warn-version: "latest" + exemptions: + usernames: [] + runtimeClasses: [] + namespaces: [kube-system] + path: /etc/kubernetes/kube-apiserver-admission-pss.yaml +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: DockerClusterTemplate +metadata: + name: kubeadm-version-cluster +spec: + template: + spec: + failureDomains: + - name: fd1 + controlPlane: true + - name: fd2 + controlPlane: true + - name: fd3 + controlPlane: true + - name: fd4 + controlPlane: false +--- +kind: KubeadmControlPlaneTemplate +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +metadata: + name: kubeadm-version-control-plane +spec: + template: + spec: + machineTemplate: + spec: + deletion: + nodeDrainTimeoutSeconds: 1 + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + certSANs: [localhost, host.docker.internal, "::", "::1", "127.0.0.1", "0.0.0.0"] + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: eviction-hard + value: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: eviction-hard + value: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: DockerMachineTemplate +metadata: + name: kubeadm-version-control-plane +spec: + template: + spec: + extraMounts: + - containerPath: "/var/run/docker.sock" + hostPath: "/var/run/docker.sock" + preLoadImages: ${DOCKER_PRELOAD_IMAGES:-[]} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: DockerMachineTemplate +metadata: + name: kubeadm-version-default-worker-machinetemplate +spec: + template: + spec: + extraMounts: + - containerPath: "/var/run/docker.sock" + hostPath: "/var/run/docker.sock" + preLoadImages: ${DOCKER_PRELOAD_IMAGES:-[]} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: KubeadmConfigTemplate +metadata: + name: kubeadm-version-default-worker-bootstraptemplate +spec: + template: + spec: + files: + - path: /run/cluster-api/fetch-kubeadm.sh + owner: "root:root" + permissions: "0755" + content: | + #!/bin/sh + set -e + VERSION_FILE="/run/cluster-api/kubeadm-version" + KUBEADM_INSTALL="/usr/bin/kubeadm" + BASE_URL="https://dl.k8s.io/release" + echo "fetch-kubeadm.sh: starting" + echo "fetch-kubeadm.sh: VERSION_FILE=$$VERSION_FILE KUBEADM_INSTALL=$$KUBEADM_INSTALL" + if [ ! -f "$$VERSION_FILE" ]; then + echo "fetch-kubeadm.sh: ERROR: $$VERSION_FILE not found, aborting" + exit 1 + fi + version=$$(tr -d '[:space:]' <"$$VERSION_FILE") + echo "fetch-kubeadm.sh: raw version from file: '$$version'" + if [ -z "$$version" ]; then + echo "fetch-kubeadm.sh: ERROR: empty version in $$VERSION_FILE, aborting" + exit 1 + fi + case "$$version" in + v*) ;; + *) version="v$$version" ;; + esac + current_version="" + if [ -x "$$KUBEADM_INSTALL" ]; then + current_version=$$($$KUBEADM_INSTALL version -o short 2>/dev/null || true) + fi + if [ "$$current_version" = "$$version" ]; then + echo "fetch-kubeadm.sh: kubeadm $$version already installed, skipping download" + exit 0 + fi + echo "fetch-kubeadm.sh: installed=$$current_version desired=$$version, fetching" + arch=$$(uname -m) + case "$$arch" in + x86_64) arch="amd64" ;; + aarch64|arm64) arch="arm64" ;; + *) echo "Unsupported arch: $$arch"; exit 1 ;; + esac + url="$${BASE_URL}/$${version}/bin/linux/$${arch}/kubeadm" + echo "Fetching kubeadm $${version} ($${arch}) from $${url}" + echo "fetch-kubeadm.sh: curl -fLsS -o $$url" + tmp=$$(mktemp -p /tmp kubeadm.XXXXXX) + if curl -fLsS -o "$$tmp" "$$url"; then + echo "fetch-kubeadm.sh: download succeeded, installing to $$KUBEADM_INSTALL" + chmod 755 "$$tmp" + mv -f "$$tmp" "$$KUBEADM_INSTALL" + echo "fetch-kubeadm.sh: done, kubeadm installed" + else + rc=$$? + echo "fetch-kubeadm.sh: curl failed with exit code $$rc" + echo "fetch-kubeadm.sh: trying curl -v to diagnose..." + curl -v -o /dev/null "$$url" 2>&1 || true + rm -f "$$tmp" + exit 1 + fi + preKubeadmCommands: + - | + command -v curl >/dev/null 2>&1 || (apt-get update && apt-get install -y --no-install-recommends curl ca-certificates && rm -rf /var/lib/apt/lists/*) + - 'sh /run/cluster-api/kubeadm-version/fetch-kubeadm.sh 2>&1 | tee /var/log/fetch-kubeadm.log || true' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: eviction-hard + value: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' diff --git a/test/e2e/kubeadm_version_on_join.go b/test/e2e/kubeadm_version_on_join.go new file mode 100644 index 000000000000..ffb5eac5d2db --- /dev/null +++ b/test/e2e/kubeadm_version_on_join.go @@ -0,0 +1,328 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/cluster-api/test/e2e/internal/log" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/test/infrastructure/container" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/patch" +) + +// KubeadmVersionOnJoinSpecInput is the input for KubeadmVersionOnJoinSpec. +type KubeadmVersionOnJoinSpecInput struct { + E2EConfig *clusterctl.E2EConfig + ClusterctlConfigPath string + BootstrapClusterProxy framework.ClusterProxy + ArtifactFolder string + SkipCleanup bool + ControlPlaneWaiters clusterctl.ControlPlaneWaiters + + // Flavor to use when creating the cluster for testing. + Flavor string + + // InfrastructureProvider specifies the infrastructure to use for clusterctl + // operations (Example: get cluster templates). + InfrastructureProvider *string + + // Allows to inject a function to be run after test namespace is created. + // If not specified, this is a no-op. + PostNamespaceCreated func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string) +} + +// KubeadmVersionOnJoinSpec verifies that when a worker Machine joins a cluster during an +// upgrade, the bootstrap controller uses the control plane's Kubernetes version for the +// kubeadm version file. A fetch-kubeadm.sh preKubeadmCommand then downloads the matching +// kubeadm binary so that kubeadm join succeeds against the upgraded control plane, even +// though the worker's spec version is still the old version. +func KubeadmVersionOnJoinSpec(ctx context.Context, inputGetter func() KubeadmVersionOnJoinSpecInput) { + const specName = "kubeadm-version-on-join" + + var ( + input KubeadmVersionOnJoinSpecInput + namespace *corev1.Namespace + cancelWatches context.CancelFunc + clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult + ) + + BeforeEach(func() { + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + input = inputGetter() + Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName) + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersionUpgradeFrom)) + Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersionUpgradeTo)) + + namespace, cancelWatches = framework.SetupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, input.PostNamespaceCreated) + clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult) + }) + + It("Should use the control plane kubeadm version when a worker joins during an upgrade", func() { + infrastructureProvider := clusterctl.DefaultInfrastructureProvider + if input.InfrastructureProvider != nil { + infrastructureProvider = *input.InfrastructureProvider + } + + kubernetesVersionUpgradeFrom := input.E2EConfig.MustGetVariable(KubernetesVersionUpgradeFrom) + kubernetesVersionUpgradeTo := input.E2EConfig.MustGetVariable(KubernetesVersionUpgradeTo) + clusterName := fmt.Sprintf("%s-%s", specName, util.RandomString(6)) + + By("Creating a workload cluster") + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: infrastructureProvider, + Flavor: input.Flavor, + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: kubernetesVersionUpgradeFrom, + ControlPlaneMachineCount: ptr.To[int64](1), + WorkerMachineCount: ptr.To[int64](1), + }, + ControlPlaneWaiters: input.ControlPlaneWaiters, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }, clusterResources) + + Expect(clusterResources.Cluster).ToNot(BeNil()) + Expect(clusterResources.Cluster.Spec.Topology.IsDefined()).To(BeTrue(), "Cluster must use ClusterClass topology") + Expect(clusterResources.MachineDeployments).To(HaveLen(1)) + + mgmtClient := input.BootstrapClusterProxy.GetClient() + cluster := clusterResources.Cluster + + By("Adding defer-upgrade and skip-preflight-checks annotations to the MachineDeployment topology") + patchHelper, err := patch.NewHelper(cluster, mgmtClient) + Expect(err).ToNot(HaveOccurred()) + + mdTopology := cluster.Spec.Topology.Workers.MachineDeployments[0] + if mdTopology.Metadata.Annotations == nil { + mdTopology.Metadata.Annotations = map[string]string{} + } + mdTopology.Metadata.Annotations[clusterv1.ClusterTopologyDeferUpgradeAnnotation] = "" + mdTopology.Metadata.Annotations[clusterv1.MachineSetSkipPreflightChecksAnnotation] = string(clusterv1.MachineSetPreflightCheckAll) + cluster.Spec.Topology.Workers.MachineDeployments[0] = mdTopology + Eventually(func() error { + return patchHelper.Patch(ctx, cluster) + }, 1*time.Minute, 10*time.Second).Should(Succeed(), "Failed to add annotations to MachineDeployment topology") + + By("Waiting for the skip-preflight-checks annotation to propagate to the MachineSet") + Eventually(func() bool { + msList := &clusterv1.MachineSetList{} + if err := mgmtClient.List(ctx, msList, + client.InNamespace(cluster.Namespace), + client.MatchingLabels{ + clusterv1.ClusterNameLabel: cluster.Name, + clusterv1.ClusterTopologyMachineDeploymentNameLabel: "md-0", + }, + ); err != nil { + return false + } + for i := range msList.Items { + if v, ok := msList.Items[i].Annotations[clusterv1.MachineSetSkipPreflightChecksAnnotation]; ok && v != "" { + log.Logf("MachineSet %s has skip-preflight-checks annotation: %s", msList.Items[i].Name, v) + return true + } + } + return false + }, 2*time.Minute, 5*time.Second).Should(BeTrue(), + "Timed out waiting for skip-preflight-checks annotation to propagate to MachineSet") + + By("Upgrading the Cluster topology version to trigger CP upgrade") + patchHelper, err = patch.NewHelper(cluster, mgmtClient) + Expect(err).ToNot(HaveOccurred()) + cluster.Spec.Topology.Version = kubernetesVersionUpgradeTo + Eventually(func() error { + return patchHelper.Patch(ctx, cluster) + }, 1*time.Minute, 10*time.Second).Should(Succeed(), "Failed to patch Cluster topology version") + + By("Waiting for control plane machines to be upgraded") + framework.WaitForControlPlaneMachinesToBeUpgraded(ctx, framework.WaitForControlPlaneMachinesToBeUpgradedInput{ + Lister: mgmtClient, + Cluster: cluster, + MachineCount: 1, + KubernetesUpgradeVersion: kubernetesVersionUpgradeTo, + }, input.E2EConfig.GetIntervals(specName, "wait-control-plane-upgrade")...) + + By("Verifying worker machines have NOT been upgraded (deferred)") + md := clusterResources.MachineDeployments[0] + machines := framework.GetMachinesByMachineDeployments(ctx, framework.GetMachinesByMachineDeploymentsInput{ + Lister: mgmtClient, + ClusterName: cluster.Name, + Namespace: cluster.Namespace, + MachineDeployment: *md, + }) + Expect(machines).To(HaveLen(1), "Expected exactly 1 worker machine") + originalMachine := machines[0] + Expect(originalMachine.Spec.Version).To(Equal(kubernetesVersionUpgradeFrom), + "Worker machine %s should still be at old version %s, got %s", originalMachine.Name, kubernetesVersionUpgradeFrom, originalMachine.Spec.Version) + log.Logf("Original worker machine: %s (version %s)", originalMachine.Name, originalMachine.Spec.Version) + + By("Scaling the MachineDeployment directly to 2 replicas") + // The topology controller skips MachineDeployment reconciliation while the upgrade is + // deferred, so we scale the underlying MachineDeployment object directly. + Expect(mgmtClient.Get(ctx, client.ObjectKeyFromObject(md), md)).To(Succeed()) + framework.ScaleAndWaitMachineDeployment(ctx, framework.ScaleAndWaitMachineDeploymentInput{ + ClusterProxy: input.BootstrapClusterProxy, + Cluster: cluster, + MachineDeployment: md, + Replicas: 2, + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }) + + By("Identifying the new worker Machine") + currentMachines := framework.GetMachinesByMachineDeployments(ctx, framework.GetMachinesByMachineDeploymentsInput{ + Lister: mgmtClient, + ClusterName: cluster.Name, + Namespace: cluster.Namespace, + MachineDeployment: *md, + }) + var newMachine *clusterv1.Machine + for i := range currentMachines { + if currentMachines[i].Name != originalMachine.Name { + newMachine = ¤tMachines[i] + break + } + } + Expect(newMachine).ToNot(BeNil(), "Could not find new Machine (original: %s)", originalMachine.Name) + log.Logf("New worker machine: %s, nodeRef: %s", newMachine.Name, newMachine.Status.NodeRef.Name) + + By("Verifying the kubeadm version file and fetch-kubeadm.sh on the new node") + containerName := newMachine.Status.NodeRef.Name + verifyKubeadmVersionOnNode(ctx, containerName, kubernetesVersionUpgradeTo) + + By("Removing the defer-upgrade and skip-preflight-checks annotations and syncing topology replicas") + // Update the topology replicas to 2 to match the direct MD scaling we did above, + // so the topology controller doesn't scale the MachineDeployment back down. + Expect(mgmtClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster)).To(Succeed()) + patchHelper, err = patch.NewHelper(cluster, mgmtClient) + Expect(err).ToNot(HaveOccurred()) + mdTopology = cluster.Spec.Topology.Workers.MachineDeployments[0] + delete(mdTopology.Metadata.Annotations, clusterv1.ClusterTopologyDeferUpgradeAnnotation) + delete(mdTopology.Metadata.Annotations, clusterv1.MachineSetSkipPreflightChecksAnnotation) + mdTopology.Replicas = ptr.To[int32](2) + cluster.Spec.Topology.Workers.MachineDeployments[0] = mdTopology + Eventually(func() error { + return patchHelper.Patch(ctx, cluster) + }, 1*time.Minute, 10*time.Second).Should(Succeed(), "Failed to remove annotations and sync replicas") + + By("Waiting for all worker machines to be upgraded") + mdList := &clusterv1.MachineDeploymentList{} + Expect(mgmtClient.List(ctx, mdList, + client.InNamespace(cluster.Namespace), + client.MatchingLabels{ + clusterv1.ClusterNameLabel: cluster.Name, + clusterv1.ClusterTopologyMachineDeploymentNameLabel: "md-0", + }, + )).To(Succeed()) + Expect(mdList.Items).To(HaveLen(1)) + framework.WaitForMachineDeploymentMachinesToBeUpgraded(ctx, framework.WaitForMachineDeploymentMachinesToBeUpgradedInput{ + Lister: mgmtClient, + Cluster: cluster, + MachineCount: 2, + KubernetesUpgradeVersion: kubernetesVersionUpgradeTo, + MachineDeployment: mdList.Items[0], + }, input.E2EConfig.GetIntervals(specName, "wait-worker-nodes")...) + + Byf("Verify Cluster Available condition is true") + framework.VerifyClusterAvailable(ctx, framework.VerifyClusterAvailableInput{ + Getter: mgmtClient, + Name: cluster.Name, + Namespace: cluster.Namespace, + }) + + By("PASSED!") + }) + + AfterEach(func() { + framework.DumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ClusterctlConfigPath, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + }) +} + +// verifyKubeadmVersionOnNode verifies the kubeadm version file and kubeadm binary +// version on the CAPD container match the expected version. +func verifyKubeadmVersionOnNode(ctx context.Context, containerName, expectedVersion string) { + containerRuntime, err := container.NewDockerClient() + Expect(err).ToNot(HaveOccurred(), "Failed to create container runtime client") + + // The version file is written by the bootstrap controller using semver.String(), + // which strips the "v" prefix (e.g. "1.35.0" not "v1.35.0"). + expectedVersionNoPfx := strings.TrimPrefix(expectedVersion, "v") + + // Verify the version file was written with the expected content. + log.Logf("Checking /run/cluster-api/kubeadm-version on container %s", containerName) + out, err := execInContainer(ctx, containerRuntime, containerName, "cat", "/run/cluster-api/kubeadm-version") + Expect(err).ToNot(HaveOccurred(), "Failed to read kubeadm version file: %s", out) + versionFileContent := strings.TrimSpace(out) + Expect(versionFileContent).To(Equal(expectedVersionNoPfx), + "Version file content %q does not match expected %q", versionFileContent, expectedVersionNoPfx) + log.Logf("Version file contains: %s", versionFileContent) + + // Verify that fetch-kubeadm.sh ran and found the version file. This proves the + // version file was present before kubeadm join (i.e. before preKubeadmCommands ran). + log.Logf("Checking fetch-kubeadm.log on container %s", containerName) + out, err = execInContainer(ctx, containerRuntime, containerName, "cat", "/var/log/fetch-kubeadm.log") + Expect(err).ToNot(HaveOccurred(), "Failed to read fetch-kubeadm.log: %s", out) + log.Logf("fetch-kubeadm.log:\n%s", out) + Expect(out).To(ContainSubstring("raw version from file:"), + "fetch-kubeadm.sh must find the version file; log:\n%s", out) + + // Log the kubeadm binary version (soft check -- the curl download may fail in + // environments without internet access, so we don't fail on a version mismatch). + log.Logf("Checking kubeadm version on container %s", containerName) + out, err = execInContainer(ctx, containerRuntime, containerName, "kubeadm", "version", "-o", "short") + if err == nil { + kubeadmVersion := strings.TrimSpace(out) + log.Logf("kubeadm version: %s (expected %s)", kubeadmVersion, expectedVersion) + } else { + log.Logf("Could not get kubeadm version: %s", out) + } +} + +// execInContainer runs a command in a container and returns the combined stdout/stderr output. +func execInContainer(ctx context.Context, cr container.Runtime, containerName, command string, args ...string) (string, error) { + var stdout, stderr bytes.Buffer + err := cr.ExecContainer(ctx, containerName, &container.ExecContainerInput{ + OutputBuffer: &stdout, + ErrorBuffer: &stderr, + }, command, args...) + combined := stdout.String() + stderr.String() + return combined, err +} diff --git a/test/e2e/kubeadm_version_on_join_test.go b/test/e2e/kubeadm_version_on_join_test.go new file mode 100644 index 000000000000..7eac661718a2 --- /dev/null +++ b/test/e2e/kubeadm_version_on_join_test.go @@ -0,0 +1,39 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + . "github.com/onsi/ginkgo/v2" + "k8s.io/utils/ptr" +) + +var _ = Describe("When a worker joins during a control plane upgrade [ClusterClass]", Label("ClusterClass"), func() { + KubeadmVersionOnJoinSpec(ctx, func() KubeadmVersionOnJoinSpecInput { + return KubeadmVersionOnJoinSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + InfrastructureProvider: ptr.To("docker"), + Flavor: "topology-kubeadm-version", + } + }) +}) From dbc7e687dafd64423ff1d4468de137e7f6ca0c0d Mon Sep 17 00:00:00 2001 From: "Cody W. Eilar" Date: Fri, 20 Mar 2026 14:55:46 -0700 Subject: [PATCH 2/5] Use template rendering of file --- .../kubeadm/v1beta1/kubeadmconfig_types.go | 28 +++++++++++++ .../v1beta1/zz_generated.conversion.go | 2 + .../kubeadm/v1beta2/kubeadmconfig_types.go | 28 +++++++++++++ ...strap.cluster.x-k8s.io_kubeadmconfigs.yaml | 20 ++++++++++ ...uster.x-k8s.io_kubeadmconfigtemplates.yaml | 20 ++++++++++ .../internal/cloudinit/cloudinit_test.go | 2 - bootstrap/kubeadm/internal/cloudinit/node.go | 18 --------- .../kubeadm/internal/cloudinit/node_test.go | 6 +-- .../controllers/kubeadmconfig_controller.go | 36 +++++++++++++++++ .../kubeadm/internal/ignition/ignition.go | 8 ---- .../internal/webhooks/kubeadmconfig_test.go | 35 ++++++++++++++++ ...cluster.x-k8s.io_kubeadmcontrolplanes.yaml | 20 ++++++++++ ...x-k8s.io_kubeadmcontrolplanetemplates.yaml | 20 ++++++++++ ...sterclass-quick-start-kubeadm-version.yaml | 15 +++---- test/e2e/kubeadm_version_on_join.go | 40 ++++++++----------- 15 files changed, 234 insertions(+), 64 deletions(-) diff --git a/api/bootstrap/kubeadm/v1beta1/kubeadmconfig_types.go b/api/bootstrap/kubeadm/v1beta1/kubeadmconfig_types.go index 1095b1dc4384..32e3813a5be8 100644 --- a/api/bootstrap/kubeadm/v1beta1/kubeadmconfig_types.go +++ b/api/bootstrap/kubeadm/v1beta1/kubeadmconfig_types.go @@ -46,6 +46,7 @@ var ( missingSecretNameMsg = "secret file source must specify non-empty secret name" missingSecretKeyMsg = "secret file source must specify non-empty secret key" pathConflictMsg = "path property must be unique among all files" + invalidFileContentFormatMsg = "contentFormat must be empty or go-template" ) // KubeadmConfigSpec defines the desired state of KubeadmConfig. @@ -223,6 +224,16 @@ func (c *KubeadmConfigSpec) validateFiles(pathPrefix *field.Path) field.ErrorLis // n.b.: if we ever add types besides Secret as a ContentFrom // Source, we must add webhook validation here for one of the // sources being non-nil. + if file.ContentFormat != "" && file.ContentFormat != FileContentFormatGoTemplate { + allErrs = append( + allErrs, + field.Invalid( + pathPrefix.Child("files").Index(i).Child("contentFormat"), + file.ContentFormat, + invalidFileContentFormatMsg, + ), + ) + } if file.ContentFrom != nil { if file.ContentFrom.Secret.Name == "" { allErrs = append( @@ -575,6 +586,16 @@ func init() { // +kubebuilder:validation:Enum=base64;gzip;gzip+base64 type Encoding string +// FileContentFormat specifies how file content is interpreted after resolving content/contentFrom and before writing bootstrap data. +// +kubebuilder:validation:Enum="";go-template +type FileContentFormat string + +const ( + // FileContentFormatGoTemplate means content is rendered as a Go text/template with KubernetesVersion and other + // fields documented by the kubeadm bootstrap provider. The default empty value means content is used verbatim. + FileContentFormatGoTemplate FileContentFormat = "go-template" +) + const ( // Base64 implies the contents of the file are encoded as base64. Base64 Encoding = "base64" @@ -608,6 +629,13 @@ type File struct { // +optional Encoding Encoding `json:"encoding,omitempty"` + // contentFormat specifies how to interpret content after it is loaded (inline or from contentFrom). + // When set to "go-template", content is rendered as a Go text/template with data including KubernetesVersion + // (semver.String(), no "v" prefix). For a worker joining a cluster, that version is the control plane + // Kubernetes version when the controller can read it; otherwise the Machine's version. + // +optional + ContentFormat FileContentFormat `json:"contentFormat,omitempty"` + // append specifies whether to append Content to existing file if Path exists. // +optional Append bool `json:"append,omitempty"` diff --git a/api/bootstrap/kubeadm/v1beta1/zz_generated.conversion.go b/api/bootstrap/kubeadm/v1beta1/zz_generated.conversion.go index 65c91898475a..7db0a78dffff 100644 --- a/api/bootstrap/kubeadm/v1beta1/zz_generated.conversion.go +++ b/api/bootstrap/kubeadm/v1beta1/zz_generated.conversion.go @@ -885,6 +885,7 @@ func autoConvert_v1beta1_File_To_v1beta2_File(in *File, out *v1beta2.File, s con out.Owner = in.Owner out.Permissions = in.Permissions out.Encoding = v1beta2.Encoding(in.Encoding) + out.ContentFormat = v1beta2.FileContentFormat(in.ContentFormat) if err := v1.Convert_bool_To_Pointer_bool(&in.Append, &out.Append, s); err != nil { return err } @@ -898,6 +899,7 @@ func autoConvert_v1beta2_File_To_v1beta1_File(in *v1beta2.File, out *File, s con out.Owner = in.Owner out.Permissions = in.Permissions out.Encoding = Encoding(in.Encoding) + out.ContentFormat = FileContentFormat(in.ContentFormat) if err := v1.Convert_Pointer_bool_To_bool(&in.Append, &out.Append, s); err != nil { return err } diff --git a/api/bootstrap/kubeadm/v1beta2/kubeadmconfig_types.go b/api/bootstrap/kubeadm/v1beta2/kubeadmconfig_types.go index b353195558e5..4297924ac482 100644 --- a/api/bootstrap/kubeadm/v1beta2/kubeadmconfig_types.go +++ b/api/bootstrap/kubeadm/v1beta2/kubeadmconfig_types.go @@ -47,6 +47,7 @@ var ( missingSecretNameMsg = "secret file source must specify non-empty secret name" missingSecretKeyMsg = "secret file source must specify non-empty secret key" pathConflictMsg = "path property must be unique among all files" + invalidFileContentFormatMsg = "contentFormat must be empty or go-template" ) // KubeadmConfigSpec defines the desired state of KubeadmConfig. @@ -220,6 +221,16 @@ func (c *KubeadmConfigSpec) validateFiles(pathPrefix *field.Path) field.ErrorLis // n.b.: if we ever add types besides Secret as a ContentFrom // Source, we must add webhook validation here for one of the // sources being non-nil. + if file.ContentFormat != "" && file.ContentFormat != FileContentFormatGoTemplate { + allErrs = append( + allErrs, + field.Invalid( + pathPrefix.Child("files").Index(i).Child("contentFormat"), + file.ContentFormat, + invalidFileContentFormatMsg, + ), + ) + } if file.ContentFrom.IsDefined() { if file.ContentFrom.Secret.Name == "" { allErrs = append( @@ -624,6 +635,16 @@ func init() { // +kubebuilder:validation:Enum=base64;gzip;gzip+base64 type Encoding string +// FileContentFormat specifies how file content is interpreted after resolving content/contentFrom and before writing bootstrap data. +// +kubebuilder:validation:Enum="";go-template +type FileContentFormat string + +const ( + // FileContentFormatGoTemplate means content is rendered as a Go text/template (see kubeadm bootstrap provider docs). + // The default empty value means content is used verbatim. + FileContentFormatGoTemplate FileContentFormat = "go-template" +) + const ( // Base64 implies the contents of the file are encoded as base64. Base64 Encoding = "base64" @@ -657,6 +678,13 @@ type File struct { // +optional Encoding Encoding `json:"encoding,omitempty"` + // contentFormat specifies how to interpret content after it is loaded (inline or from contentFrom). + // When set to "go-template", content is rendered as a Go text/template with data including KubernetesVersion + // (semver.String(), no "v" prefix). For a worker joining a cluster, that version is the control plane + // Kubernetes version when the controller can read it; otherwise the Machine's version. + // +optional + ContentFormat FileContentFormat `json:"contentFormat,omitempty"` + // append specifies whether to append Content to existing file if Path exists. // +optional Append *bool `json:"append,omitempty"` diff --git a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml index 62ae44bc321d..3ae61cb20a55 100644 --- a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml +++ b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml @@ -1280,6 +1280,16 @@ spec: maxLength: 10240 minLength: 1 type: string + contentFormat: + description: |- + contentFormat specifies how to interpret content after it is loaded (inline or from contentFrom). + When set to "go-template", content is rendered as a Go text/template with data including KubernetesVersion + (semver.String(), no "v" prefix). For a worker joining a cluster, that version is the control plane + Kubernetes version when the controller can read it; otherwise the Machine's version. + enum: + - "" + - go-template + type: string contentFrom: description: contentFrom is a referenced source of content to populate the file. @@ -3669,6 +3679,16 @@ spec: maxLength: 10240 minLength: 1 type: string + contentFormat: + description: |- + contentFormat specifies how to interpret content after it is loaded (inline or from contentFrom). + When set to "go-template", content is rendered as a Go text/template with data including KubernetesVersion + (semver.String(), no "v" prefix). For a worker joining a cluster, that version is the control plane + Kubernetes version when the controller can read it; otherwise the Machine's version. + enum: + - "" + - go-template + type: string contentFrom: description: contentFrom is a referenced source of content to populate the file. diff --git a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml index 40f86233fdcf..72408762f677 100644 --- a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml +++ b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml @@ -1337,6 +1337,16 @@ spec: maxLength: 10240 minLength: 1 type: string + contentFormat: + description: |- + contentFormat specifies how to interpret content after it is loaded (inline or from contentFrom). + When set to "go-template", content is rendered as a Go text/template with data including KubernetesVersion + (semver.String(), no "v" prefix). For a worker joining a cluster, that version is the control plane + Kubernetes version when the controller can read it; otherwise the Machine's version. + enum: + - "" + - go-template + type: string contentFrom: description: contentFrom is a referenced source of content to populate the file. @@ -3650,6 +3660,16 @@ spec: maxLength: 10240 minLength: 1 type: string + contentFormat: + description: |- + contentFormat specifies how to interpret content after it is loaded (inline or from contentFrom). + When set to "go-template", content is rendered as a Go text/template with data including KubernetesVersion + (semver.String(), no "v" prefix). For a worker joining a cluster, that version is the control plane + Kubernetes version when the controller can read it; otherwise the Machine's version. + enum: + - "" + - go-template + type: string contentFrom: description: contentFrom is a referenced source of content to populate the file. diff --git a/bootstrap/kubeadm/internal/cloudinit/cloudinit_test.go b/bootstrap/kubeadm/internal/cloudinit/cloudinit_test.go index 418753a7e5ce..a66e83a11ce7 100644 --- a/bootstrap/kubeadm/internal/cloudinit/cloudinit_test.go +++ b/bootstrap/kubeadm/internal/cloudinit/cloudinit_test.go @@ -416,8 +416,6 @@ func TestNewJoinNodeCommands(t *testing.T) { - "echo $(date) ': hello PostKubeadmCommands!'"` g.Expect(out).To(ContainSubstring(expectedRunCmd)) - - g.Expect(out).To(ContainSubstring("path: " + KubeadmVersionPath)) } func TestOmittableFields(t *testing.T) { diff --git a/bootstrap/kubeadm/internal/cloudinit/node.go b/bootstrap/kubeadm/internal/cloudinit/node.go index 8e2a51e8af47..474b3d08b93e 100644 --- a/bootstrap/kubeadm/internal/cloudinit/node.go +++ b/bootstrap/kubeadm/internal/cloudinit/node.go @@ -16,16 +16,6 @@ limitations under the License. package cloudinit -import ( - bootstrapv1 "sigs.k8s.io/cluster-api/api/bootstrap/kubeadm/v1beta2" -) - -const ( - // KubeadmVersionPath is the path where the control plane Kubernetes version is written for worker nodes. - // It must exist before kubeadm join runs. - KubeadmVersionPath = "/run/cluster-api/kubeadm-version" -) - const ( nodeCloudInit = `{{.Header}} {{template "files" .WriteFiles}} @@ -62,13 +52,5 @@ type NodeInput struct { func NewNode(input *NodeInput) ([]byte, error) { input.prepare() input.Header = cloudConfigHeader - // Write control plane version to KubeadmVersionPath so it exists before kubeadm join. - versionFile := bootstrapv1.File{ - Path: KubeadmVersionPath, - Owner: "root:root", - Permissions: "0644", - Content: input.KubernetesVersion.String(), - } - input.WriteFiles = append([]bootstrapv1.File{versionFile}, input.WriteFiles...) return generate("Node", nodeCloudInit, input) } diff --git a/bootstrap/kubeadm/internal/cloudinit/node_test.go b/bootstrap/kubeadm/internal/cloudinit/node_test.go index 2c2cd4d4d2ea..4fca5b2398b3 100644 --- a/bootstrap/kubeadm/internal/cloudinit/node_test.go +++ b/bootstrap/kubeadm/internal/cloudinit/node_test.go @@ -46,13 +46,13 @@ func TestNewNode(t *testing.T) { }, }, }, - checkWriteFiles(KubeadmVersionPath, "/etc/foo.conf", "/run/kubeadm/kubeadm-join-config.yaml", "/run/cluster-api/placeholder"), + checkWriteFiles("/etc/foo.conf", "/run/kubeadm/kubeadm-join-config.yaml", "/run/cluster-api/placeholder"), false, }, { - "check for existence of kubeadm-version path, /run/kubeadm/kubeadm-join-config.yaml and /run/cluster-api/placeholder", + "check for existence of /run/kubeadm/kubeadm-join-config.yaml and /run/cluster-api/placeholder", &NodeInput{}, - checkWriteFiles(KubeadmVersionPath, "/run/kubeadm/kubeadm-join-config.yaml", "/run/cluster-api/placeholder"), + checkWriteFiles("/run/kubeadm/kubeadm-join-config.yaml", "/run/cluster-api/placeholder"), false, }, } diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go index 1fc74eebf7bf..af9bd4cd5988 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go @@ -44,6 +44,7 @@ import ( bootstrapv1 "sigs.k8s.io/cluster-api/api/bootstrap/kubeadm/v1beta2" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/bootstrapfiles" "sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/cloudinit" "sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/ignition" "sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/locking" @@ -579,6 +580,17 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex }) return ctrl.Result{}, err } + files, err = bootstrapfiles.RenderTemplates(files, bootstrapfiles.DataFromVersion(parsedVersion)) + if err != nil { + v1beta1conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableV1Beta1Condition, bootstrapv1.DataSecretGenerationFailedV1Beta1Reason, clusterv1.ConditionSeverityWarning, "%s", err.Error()) + conditions.Set(scope.Config, metav1.Condition{ + Type: bootstrapv1.KubeadmConfigDataSecretAvailableCondition, + Status: metav1.ConditionFalse, + Reason: bootstrapv1.KubeadmConfigDataSecretNotAvailableReason, + Message: "Failed to render go-template in spec.files", + }) + return ctrl.Result{}, err + } users, err := r.resolveUsers(ctx, scope.Config) if err != nil { @@ -765,6 +777,18 @@ func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope) files = append(files, *kubeconfig) } + files, err = bootstrapfiles.RenderTemplates(files, bootstrapfiles.DataFromVersion(parsedVersion)) + if err != nil { + v1beta1conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableV1Beta1Condition, bootstrapv1.DataSecretGenerationFailedV1Beta1Reason, clusterv1.ConditionSeverityWarning, "%s", err.Error()) + conditions.Set(scope.Config, metav1.Condition{ + Type: bootstrapv1.KubeadmConfigDataSecretAvailableCondition, + Status: metav1.ConditionFalse, + Reason: bootstrapv1.KubeadmConfigDataSecretNotAvailableReason, + Message: "Failed to render go-template in spec.files", + }) + return ctrl.Result{}, err + } + nodeInput := &cloudinit.NodeInput{ BaseUserData: cloudinit.BaseUserData{ AdditionalFiles: files, @@ -951,6 +975,18 @@ func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *S files = append(files, *kubeconfig) } + files, err = bootstrapfiles.RenderTemplates(files, bootstrapfiles.DataFromVersion(parsedVersion)) + if err != nil { + v1beta1conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableV1Beta1Condition, bootstrapv1.DataSecretGenerationFailedV1Beta1Reason, clusterv1.ConditionSeverityWarning, "%s", err.Error()) + conditions.Set(scope.Config, metav1.Condition{ + Type: bootstrapv1.KubeadmConfigDataSecretAvailableCondition, + Status: metav1.ConditionFalse, + Reason: bootstrapv1.KubeadmConfigDataSecretNotAvailableReason, + Message: "Failed to render go-template in spec.files", + }) + return ctrl.Result{}, err + } + controlPlaneJoinInput := &cloudinit.ControlPlaneJoinInput{ JoinConfiguration: joinData, Certificates: certificates, diff --git a/bootstrap/kubeadm/internal/ignition/ignition.go b/bootstrap/kubeadm/internal/ignition/ignition.go index 7ee051dad191..68e236a69c7c 100644 --- a/bootstrap/kubeadm/internal/ignition/ignition.go +++ b/bootstrap/kubeadm/internal/ignition/ignition.go @@ -63,14 +63,6 @@ func NewNode(input *NodeInput) ([]byte, string, error) { return nil, "", fmt.Errorf("node input can't be nil") } - // Write control plane version to KubeadmVersionPath so it exists before kubeadm join. - versionFile := bootstrapv1.File{ - Path: cloudinit.KubeadmVersionPath, - Owner: "root:root", - Permissions: "0644", - Content: input.KubernetesVersion.String(), - } - input.WriteFiles = append([]bootstrapv1.File{versionFile}, input.WriteFiles...) input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...) input.KubeadmCommand = fmt.Sprintf(kubeadmCommandTemplate, joinSubcommand, input.KubeadmVerbosity) diff --git a/bootstrap/kubeadm/internal/webhooks/kubeadmconfig_test.go b/bootstrap/kubeadm/internal/webhooks/kubeadmconfig_test.go index b688fb9bacae..241ca2b90f93 100644 --- a/bootstrap/kubeadm/internal/webhooks/kubeadmconfig_test.go +++ b/bootstrap/kubeadm/internal/webhooks/kubeadmconfig_test.go @@ -154,6 +154,41 @@ func TestKubeadmConfigValidate(t *testing.T) { }, expectErr: true, }, + "valid go-template contentFormat": { + in: &bootstrapv1.KubeadmConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "baz", + Namespace: metav1.NamespaceDefault, + }, + Spec: bootstrapv1.KubeadmConfigSpec{ + Files: []bootstrapv1.File{ + { + Path: "/x", + Content: "{{ .KubernetesVersion }}", + ContentFormat: bootstrapv1.FileContentFormatGoTemplate, + }, + }, + }, + }, + }, + "invalid contentFormat": { + in: &bootstrapv1.KubeadmConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "baz", + Namespace: metav1.NamespaceDefault, + }, + Spec: bootstrapv1.KubeadmConfigSpec{ + Files: []bootstrapv1.File{ + { + Path: "/x", + Content: "foo", + ContentFormat: bootstrapv1.FileContentFormat("helm"), + }, + }, + }, + }, + expectErr: true, + }, "valid passwd": { in: &bootstrapv1.KubeadmConfig{ ObjectMeta: metav1.ObjectMeta{ diff --git a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml index 1441421b45ba..d5d0b42f0b98 100644 --- a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml +++ b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml @@ -1336,6 +1336,16 @@ spec: maxLength: 10240 minLength: 1 type: string + contentFormat: + description: |- + contentFormat specifies how to interpret content after it is loaded (inline or from contentFrom). + When set to "go-template", content is rendered as a Go text/template with data including KubernetesVersion + (semver.String(), no "v" prefix). For a worker joining a cluster, that version is the control plane + Kubernetes version when the controller can read it; otherwise the Machine's version. + enum: + - "" + - go-template + type: string contentFrom: description: contentFrom is a referenced source of content to populate the file. @@ -4231,6 +4241,16 @@ spec: maxLength: 10240 minLength: 1 type: string + contentFormat: + description: |- + contentFormat specifies how to interpret content after it is loaded (inline or from contentFrom). + When set to "go-template", content is rendered as a Go text/template with data including KubernetesVersion + (semver.String(), no "v" prefix). For a worker joining a cluster, that version is the control plane + Kubernetes version when the controller can read it; otherwise the Machine's version. + enum: + - "" + - go-template + type: string contentFrom: description: contentFrom is a referenced source of content to populate the file. diff --git a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml index 279faf177e3a..27ed3376d838 100644 --- a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml +++ b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml @@ -1360,6 +1360,16 @@ spec: maxLength: 10240 minLength: 1 type: string + contentFormat: + description: |- + contentFormat specifies how to interpret content after it is loaded (inline or from contentFrom). + When set to "go-template", content is rendered as a Go text/template with data including KubernetesVersion + (semver.String(), no "v" prefix). For a worker joining a cluster, that version is the control plane + Kubernetes version when the controller can read it; otherwise the Machine's version. + enum: + - "" + - go-template + type: string contentFrom: description: contentFrom is a referenced source of content to populate the file. @@ -3946,6 +3956,16 @@ spec: maxLength: 10240 minLength: 1 type: string + contentFormat: + description: |- + contentFormat specifies how to interpret content after it is loaded (inline or from contentFrom). + When set to "go-template", content is rendered as a Go text/template with data including KubernetesVersion + (semver.String(), no "v" prefix). For a worker joining a cluster, that version is the control plane + Kubernetes version when the controller can read it; otherwise the Machine's version. + enum: + - "" + - go-template + type: string contentFrom: description: contentFrom is a referenced source of content to populate the file. diff --git a/test/e2e/data/infrastructure-docker/main/clusterclass-quick-start-kubeadm-version.yaml b/test/e2e/data/infrastructure-docker/main/clusterclass-quick-start-kubeadm-version.yaml index b4c03dc157fa..6f246f9ee18b 100644 --- a/test/e2e/data/infrastructure-docker/main/clusterclass-quick-start-kubeadm-version.yaml +++ b/test/e2e/data/infrastructure-docker/main/clusterclass-quick-start-kubeadm-version.yaml @@ -276,22 +276,17 @@ spec: - path: /run/cluster-api/fetch-kubeadm.sh owner: "root:root" permissions: "0755" + contentFormat: go-template content: | #!/bin/sh set -e - VERSION_FILE="/run/cluster-api/kubeadm-version" KUBEADM_INSTALL="/usr/bin/kubeadm" BASE_URL="https://dl.k8s.io/release" + version="{{ .KubernetesVersion }}" echo "fetch-kubeadm.sh: starting" - echo "fetch-kubeadm.sh: VERSION_FILE=$$VERSION_FILE KUBEADM_INSTALL=$$KUBEADM_INSTALL" - if [ ! -f "$$VERSION_FILE" ]; then - echo "fetch-kubeadm.sh: ERROR: $$VERSION_FILE not found, aborting" - exit 1 - fi - version=$$(tr -d '[:space:]' <"$$VERSION_FILE") - echo "fetch-kubeadm.sh: raw version from file: '$$version'" + echo "fetch-kubeadm.sh: raw version from template: '$$version'" if [ -z "$$version" ]; then - echo "fetch-kubeadm.sh: ERROR: empty version in $$VERSION_FILE, aborting" + echo "fetch-kubeadm.sh: ERROR: empty KubernetesVersion in script, aborting" exit 1 fi case "$$version" in @@ -333,7 +328,7 @@ spec: preKubeadmCommands: - | command -v curl >/dev/null 2>&1 || (apt-get update && apt-get install -y --no-install-recommends curl ca-certificates && rm -rf /var/lib/apt/lists/*) - - 'sh /run/cluster-api/kubeadm-version/fetch-kubeadm.sh 2>&1 | tee /var/log/fetch-kubeadm.log || true' + - 'sh /run/cluster-api/fetch-kubeadm.sh 2>&1 | tee /var/log/fetch-kubeadm.log || true' joinConfiguration: nodeRegistration: kubeletExtraArgs: diff --git a/test/e2e/kubeadm_version_on_join.go b/test/e2e/kubeadm_version_on_join.go index ffb5eac5d2db..eb3a8f74738d 100644 --- a/test/e2e/kubeadm_version_on_join.go +++ b/test/e2e/kubeadm_version_on_join.go @@ -1,5 +1,5 @@ /* -Copyright 2025 The Kubernetes Authors. +Copyright 2026 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -62,10 +62,10 @@ type KubeadmVersionOnJoinSpecInput struct { } // KubeadmVersionOnJoinSpec verifies that when a worker Machine joins a cluster during an -// upgrade, the bootstrap controller uses the control plane's Kubernetes version for the -// kubeadm version file. A fetch-kubeadm.sh preKubeadmCommand then downloads the matching -// kubeadm binary so that kubeadm join succeeds against the upgraded control plane, even -// though the worker's spec version is still the old version. +// upgrade, the bootstrap controller renders spec.files go-template content with the control +// plane's Kubernetes version. A fetch-kubeadm.sh preKubeadmCommand (installed from that +// templated file) downloads the matching kubeadm binary so that kubeadm join succeeds against +// the upgraded control plane, even though the worker's spec version is still the old version. func KubeadmVersionOnJoinSpec(ctx context.Context, inputGetter func() KubeadmVersionOnJoinSpecInput) { const specName = "kubeadm-version-on-join" @@ -224,7 +224,7 @@ func KubeadmVersionOnJoinSpec(ctx context.Context, inputGetter func() KubeadmVer Expect(newMachine).ToNot(BeNil(), "Could not find new Machine (original: %s)", originalMachine.Name) log.Logf("New worker machine: %s, nodeRef: %s", newMachine.Name, newMachine.Status.NodeRef.Name) - By("Verifying the kubeadm version file and fetch-kubeadm.sh on the new node") + By("Verifying fetch-kubeadm.sh was rendered with the control plane version on the new node") containerName := newMachine.Status.NodeRef.Name verifyKubeadmVersionOnNode(ctx, containerName, kubernetesVersionUpgradeTo) @@ -276,33 +276,27 @@ func KubeadmVersionOnJoinSpec(ctx context.Context, inputGetter func() KubeadmVer }) } -// verifyKubeadmVersionOnNode verifies the kubeadm version file and kubeadm binary -// version on the CAPD container match the expected version. +// verifyKubeadmVersionOnNode verifies fetch-kubeadm.sh was rendered with the expected +// control plane Kubernetes version and ran before kubeadm join. func verifyKubeadmVersionOnNode(ctx context.Context, containerName, expectedVersion string) { containerRuntime, err := container.NewDockerClient() Expect(err).ToNot(HaveOccurred(), "Failed to create container runtime client") - // The version file is written by the bootstrap controller using semver.String(), - // which strips the "v" prefix (e.g. "1.35.0" not "v1.35.0"). + // Bootstrap go-template data uses semver.String() (no "v" prefix). expectedVersionNoPfx := strings.TrimPrefix(expectedVersion, "v") - // Verify the version file was written with the expected content. - log.Logf("Checking /run/cluster-api/kubeadm-version on container %s", containerName) - out, err := execInContainer(ctx, containerRuntime, containerName, "cat", "/run/cluster-api/kubeadm-version") - Expect(err).ToNot(HaveOccurred(), "Failed to read kubeadm version file: %s", out) - versionFileContent := strings.TrimSpace(out) - Expect(versionFileContent).To(Equal(expectedVersionNoPfx), - "Version file content %q does not match expected %q", versionFileContent, expectedVersionNoPfx) - log.Logf("Version file contains: %s", versionFileContent) - - // Verify that fetch-kubeadm.sh ran and found the version file. This proves the - // version file was present before kubeadm join (i.e. before preKubeadmCommands ran). + log.Logf("Checking /run/cluster-api/fetch-kubeadm.sh on container %s", containerName) + out, err := execInContainer(ctx, containerRuntime, containerName, "cat", "/run/cluster-api/fetch-kubeadm.sh") + Expect(err).ToNot(HaveOccurred(), "Failed to read fetch-kubeadm.sh: %s", out) + Expect(out).To(ContainSubstring(`version="`+expectedVersionNoPfx+`"`), + "fetch script should contain rendered KubernetesVersion %q; script:\n%s", expectedVersionNoPfx, out) + log.Logf("Checking fetch-kubeadm.log on container %s", containerName) out, err = execInContainer(ctx, containerRuntime, containerName, "cat", "/var/log/fetch-kubeadm.log") Expect(err).ToNot(HaveOccurred(), "Failed to read fetch-kubeadm.log: %s", out) log.Logf("fetch-kubeadm.log:\n%s", out) - Expect(out).To(ContainSubstring("raw version from file:"), - "fetch-kubeadm.sh must find the version file; log:\n%s", out) + Expect(out).To(ContainSubstring("raw version from template:"), + "fetch-kubeadm.sh must log embedded version; log:\n%s", out) // Log the kubeadm binary version (soft check -- the curl download may fail in // environments without internet access, so we don't fail on a version mismatch). From 01dd23deb778a4f7cd2e5125cffcb7a7ce63e164 Mon Sep 17 00:00:00 2001 From: "Cody W. Eilar" Date: Fri, 20 Mar 2026 15:27:32 -0700 Subject: [PATCH 3/5] Use a subset of preflight checks --- test/e2e/kubeadm_version_on_join.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/test/e2e/kubeadm_version_on_join.go b/test/e2e/kubeadm_version_on_join.go index eb3a8f74738d..e98a9572f6a0 100644 --- a/test/e2e/kubeadm_version_on_join.go +++ b/test/e2e/kubeadm_version_on_join.go @@ -128,16 +128,24 @@ func KubeadmVersionOnJoinSpec(ctx context.Context, inputGetter func() KubeadmVer mgmtClient := input.BootstrapClusterProxy.GetClient() cluster := clusterResources.Cluster - By("Adding defer-upgrade and skip-preflight-checks annotations to the MachineDeployment topology") + By("Adding defer-upgrade and targeted skip-preflight-checks annotations to the MachineDeployment topology") patchHelper, err := patch.NewHelper(cluster, mgmtClient) Expect(err).ToNot(HaveOccurred()) + // Scale-out uses a worker template still at upgradeFrom while the control plane is already at + // upgradeTo. KubeadmVersionSkew and ControlPlaneVersionSkew would block that; other preflights + // (e.g. KubernetesVersionSkew, ControlPlaneIsStable) should still run. + skippedMachineSetPreflights := strings.Join([]string{ + string(clusterv1.MachineSetPreflightCheckKubeadmVersionSkew), + string(clusterv1.MachineSetPreflightCheckControlPlaneVersionSkew), + }, ",") + mdTopology := cluster.Spec.Topology.Workers.MachineDeployments[0] if mdTopology.Metadata.Annotations == nil { mdTopology.Metadata.Annotations = map[string]string{} } mdTopology.Metadata.Annotations[clusterv1.ClusterTopologyDeferUpgradeAnnotation] = "" - mdTopology.Metadata.Annotations[clusterv1.MachineSetSkipPreflightChecksAnnotation] = string(clusterv1.MachineSetPreflightCheckAll) + mdTopology.Metadata.Annotations[clusterv1.MachineSetSkipPreflightChecksAnnotation] = skippedMachineSetPreflights cluster.Spec.Topology.Workers.MachineDeployments[0] = mdTopology Eventually(func() error { return patchHelper.Patch(ctx, cluster) @@ -228,7 +236,7 @@ func KubeadmVersionOnJoinSpec(ctx context.Context, inputGetter func() KubeadmVer containerName := newMachine.Status.NodeRef.Name verifyKubeadmVersionOnNode(ctx, containerName, kubernetesVersionUpgradeTo) - By("Removing the defer-upgrade and skip-preflight-checks annotations and syncing topology replicas") + By("Removing the defer-upgrade and skip-preflight annotations and syncing topology replicas") // Update the topology replicas to 2 to match the direct MD scaling we did above, // so the topology controller doesn't scale the MachineDeployment back down. Expect(mgmtClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster)).To(Succeed()) From 0b40256129e355839e0cc2a2b4850bd2988f2368 Mon Sep 17 00:00:00 2001 From: "Cody W. Eilar" Date: Mon, 23 Mar 2026 17:20:15 -0700 Subject: [PATCH 4/5] Add conditions for fetching control plane --- .../kubeadm/v1beta1/kubeadmconfig_types.go | 2 +- .../v1beta1/v1beta2_condition_consts.go | 13 ++ .../kubeadm/v1beta2/kubeadm_types.go | 16 +++ .../kubeadm/v1beta2/kubeadmconfig_types.go | 6 +- .../v1beta2/v1beta1_condition_consts.go | 8 ++ ...strap.cluster.x-k8s.io_kubeadmconfigs.yaml | 4 +- .../internal/bootstrapfiles/template.go | 63 ++++++++++ .../internal/bootstrapfiles/template_test.go | 62 ++++++++++ .../controllers/kubeadmconfig_controller.go | 53 +++++--- ...ig_controller_controlplane_version_test.go | 116 ++++++++++++++++++ .../kubeadmconfig_controller_test.go | 2 +- 11 files changed, 323 insertions(+), 22 deletions(-) create mode 100644 bootstrap/kubeadm/internal/bootstrapfiles/template.go create mode 100644 bootstrap/kubeadm/internal/bootstrapfiles/template_test.go create mode 100644 bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_controlplane_version_test.go diff --git a/api/bootstrap/kubeadm/v1beta1/kubeadmconfig_types.go b/api/bootstrap/kubeadm/v1beta1/kubeadmconfig_types.go index 32e3813a5be8..f3e7711f93a4 100644 --- a/api/bootstrap/kubeadm/v1beta1/kubeadmconfig_types.go +++ b/api/bootstrap/kubeadm/v1beta1/kubeadmconfig_types.go @@ -508,7 +508,7 @@ type KubeadmConfigStatus struct { // See https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20240916-improve-status-in-CAPI-resources.md for more context. type KubeadmConfigV1Beta2Status struct { // conditions represents the observations of a KubeadmConfig's current state. - // Known condition types are Ready, DataSecretAvailable, CertificatesAvailable. + // Known condition types are Ready, DataSecretAvailable, CertificatesAvailable, ControlPlaneKubernetesVersionAvailable. // +optional // +listType=map // +listMapKey=type diff --git a/api/bootstrap/kubeadm/v1beta1/v1beta2_condition_consts.go b/api/bootstrap/kubeadm/v1beta1/v1beta2_condition_consts.go index 58e121519e55..1f741e780508 100644 --- a/api/bootstrap/kubeadm/v1beta1/v1beta2_condition_consts.go +++ b/api/bootstrap/kubeadm/v1beta1/v1beta2_condition_consts.go @@ -61,3 +61,16 @@ const ( // KubeadmConfigDataSecretNotAvailableV1Beta2Reason surfaces when the bootstrap secret is not available. KubeadmConfigDataSecretNotAvailableV1Beta2Reason = clusterv1beta1.NotAvailableV1Beta2Reason ) + +// KubeadmConfig's ControlPlaneKubernetesVersionAvailable condition and corresponding reasons that will be used in v1Beta2 API version. +const ( + // KubeadmConfigControlPlaneKubernetesVersionAvailableV1Beta2Condition documents whether join Kubernetes version + // could be resolved from the Cluster's control plane reference when applicable. + KubeadmConfigControlPlaneKubernetesVersionAvailableV1Beta2Condition = "ControlPlaneKubernetesVersionAvailable" + + // KubeadmConfigControlPlaneKubernetesVersionAvailableV1Beta2Reason surfaces when join version resolution succeeded. + KubeadmConfigControlPlaneKubernetesVersionAvailableV1Beta2Reason = clusterv1beta1.AvailableV1Beta2Reason + + // KubeadmConfigControlPlaneKubernetesVersionResolutionFailedV1Beta2Reason surfaces when resolution failed. + KubeadmConfigControlPlaneKubernetesVersionResolutionFailedV1Beta2Reason = "ControlPlaneKubernetesVersionResolutionFailed" +) diff --git a/api/bootstrap/kubeadm/v1beta2/kubeadm_types.go b/api/bootstrap/kubeadm/v1beta2/kubeadm_types.go index 17e3bfc7840f..33073c115684 100644 --- a/api/bootstrap/kubeadm/v1beta2/kubeadm_types.go +++ b/api/bootstrap/kubeadm/v1beta2/kubeadm_types.go @@ -72,6 +72,22 @@ const ( KubeadmConfigDataSecretNotAvailableReason = clusterv1.NotAvailableReason ) +// KubeadmConfig's ControlPlaneKubernetesVersionAvailable condition and corresponding reasons. +// This condition is set when reconciling worker join bootstrap data: it reflects whether the controller +// could resolve the Kubernetes version from the Cluster's control plane reference (when present). +const ( + // KubeadmConfigControlPlaneKubernetesVersionAvailableCondition is true when the control plane Kubernetes + // version for join could be resolved or is not required (no controlPlaneRef or control plane has no spec.version). + KubeadmConfigControlPlaneKubernetesVersionAvailableCondition = "ControlPlaneKubernetesVersionAvailable" + + // KubeadmConfigControlPlaneKubernetesVersionAvailableReason surfaces when join version resolution succeeded. + KubeadmConfigControlPlaneKubernetesVersionAvailableReason = clusterv1.AvailableReason + + // KubeadmConfigControlPlaneKubernetesVersionResolutionFailedReason surfaces when the controller could not + // read the control plane object or its Kubernetes version while a controlPlaneRef is set. + KubeadmConfigControlPlaneKubernetesVersionResolutionFailedReason = "ResolutionFailed" +) + // EncryptionAlgorithmType can define an asymmetric encryption algorithm type. // +kubebuilder:validation:Enum=ECDSA-P256;ECDSA-P384;RSA-2048;RSA-3072;RSA-4096 type EncryptionAlgorithmType string diff --git a/api/bootstrap/kubeadm/v1beta2/kubeadmconfig_types.go b/api/bootstrap/kubeadm/v1beta2/kubeadmconfig_types.go index 4297924ac482..0855b6023276 100644 --- a/api/bootstrap/kubeadm/v1beta2/kubeadmconfig_types.go +++ b/api/bootstrap/kubeadm/v1beta2/kubeadmconfig_types.go @@ -486,7 +486,7 @@ func (r *ContainerLinuxConfig) IsDefined() bool { // +kubebuilder:validation:MinProperties=1 type KubeadmConfigStatus struct { // conditions represents the observations of a KubeadmConfig's current state. - // Known condition types are Ready, DataSecretAvailable, CertificatesAvailable. + // Known condition types are Ready, DataSecretAvailable, CertificatesAvailable, ControlPlaneKubernetesVersionAvailable. // +optional // +listType=map // +listMapKey=type @@ -640,8 +640,8 @@ type Encoding string type FileContentFormat string const ( - // FileContentFormatGoTemplate means content is rendered as a Go text/template (see kubeadm bootstrap provider docs). - // The default empty value means content is used verbatim. + // FileContentFormatGoTemplate means content is rendered as a Go text/template with KubernetesVersion and other + // fields documented by the kubeadm bootstrap provider. The default empty value means content is used verbatim. FileContentFormatGoTemplate FileContentFormat = "go-template" ) diff --git a/api/bootstrap/kubeadm/v1beta2/v1beta1_condition_consts.go b/api/bootstrap/kubeadm/v1beta2/v1beta1_condition_consts.go index 7ccc263e927c..550f91184c44 100644 --- a/api/bootstrap/kubeadm/v1beta2/v1beta1_condition_consts.go +++ b/api/bootstrap/kubeadm/v1beta2/v1beta1_condition_consts.go @@ -39,6 +39,14 @@ const ( // an error while generating a data secret; those kind of errors are usually due to misconfigurations // and user intervention is required to get them fixed. DataSecretGenerationFailedV1Beta1Reason = "DataSecretGenerationFailed" + + // ControlPlaneKubernetesVersionAvailableV1Beta1Condition documents whether the controller could resolve the + // Kubernetes version used for worker join from the Cluster's control plane reference (when applicable). + ControlPlaneKubernetesVersionAvailableV1Beta1Condition clusterv1.ConditionType = "ControlPlaneKubernetesVersionAvailable" + + // ControlPlaneKubernetesVersionResolutionFailedV1Beta1Reason (Severity=Warning) documents a failure to read + // the control plane object or its Kubernetes version while resolving join bootstrap data. + ControlPlaneKubernetesVersionResolutionFailedV1Beta1Reason = "ControlPlaneKubernetesVersionResolutionFailed" ) const ( diff --git a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml index 3ae61cb20a55..a557a4febc00 100644 --- a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml +++ b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml @@ -2257,7 +2257,7 @@ spec: conditions: description: |- conditions represents the observations of a KubeadmConfig's current state. - Known condition types are Ready, DataSecretAvailable, CertificatesAvailable. + Known condition types are Ready, DataSecretAvailable, CertificatesAvailable, ControlPlaneKubernetesVersionAvailable. items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -4738,7 +4738,7 @@ spec: conditions: description: |- conditions represents the observations of a KubeadmConfig's current state. - Known condition types are Ready, DataSecretAvailable, CertificatesAvailable. + Known condition types are Ready, DataSecretAvailable, CertificatesAvailable, ControlPlaneKubernetesVersionAvailable. items: description: Condition contains details for one aspect of the current state of this API Resource. diff --git a/bootstrap/kubeadm/internal/bootstrapfiles/template.go b/bootstrap/kubeadm/internal/bootstrapfiles/template.go new file mode 100644 index 000000000000..02ea62161eec --- /dev/null +++ b/bootstrap/kubeadm/internal/bootstrapfiles/template.go @@ -0,0 +1,63 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package bootstrapfiles contains helpers for KubeadmConfig spec.files processing. +package bootstrapfiles + +import ( + "bytes" + "text/template" + + "github.com/blang/semver/v4" + "github.com/pkg/errors" + + bootstrapv1 "sigs.k8s.io/cluster-api/api/bootstrap/kubeadm/v1beta2" +) + +// TemplateData is passed to Go text/template when a KubeadmConfig spec.files entry uses contentFormat "go-template". +type TemplateData struct { + // KubernetesVersion is the effective Kubernetes version for bootstrap data (semver.String(), no "v" prefix). + // For a worker Machine joining a cluster, this is the control plane Kubernetes version when the controller + // can read it; otherwise the Machine's version. + KubernetesVersion string +} + +// DataFromVersion returns template data using semver.Version.String() (no "v" prefix). +func DataFromVersion(v semver.Version) TemplateData { + return TemplateData{KubernetesVersion: v.String()} +} + +// RenderTemplates renders go-template file contents and clears contentFormat on those entries. +func RenderTemplates(files []bootstrapv1.File, data TemplateData) ([]bootstrapv1.File, error) { + out := make([]bootstrapv1.File, len(files)) + copy(out, files) + for i := range out { + if out[i].ContentFormat != bootstrapv1.FileContentFormatGoTemplate { + continue + } + tpl, err := template.New(out[i].Path).Option("missingkey=error").Parse(out[i].Content) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse go-template for file %q", out[i].Path) + } + var buf bytes.Buffer + if err := tpl.Execute(&buf, data); err != nil { + return nil, errors.Wrapf(err, "failed to execute go-template for file %q", out[i].Path) + } + out[i].Content = buf.String() + out[i].ContentFormat = "" + } + return out, nil +} diff --git a/bootstrap/kubeadm/internal/bootstrapfiles/template_test.go b/bootstrap/kubeadm/internal/bootstrapfiles/template_test.go new file mode 100644 index 000000000000..0f44920dc999 --- /dev/null +++ b/bootstrap/kubeadm/internal/bootstrapfiles/template_test.go @@ -0,0 +1,62 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrapfiles + +import ( + "testing" + + "github.com/blang/semver/v4" + . "github.com/onsi/gomega" + + bootstrapv1 "sigs.k8s.io/cluster-api/api/bootstrap/kubeadm/v1beta2" +) + +func TestRenderTemplates(t *testing.T) { + v := semver.MustParse("1.29.0") + data := DataFromVersion(v) + + t.Run("plain files unchanged", func(t *testing.T) { + g := NewWithT(t) + in := []bootstrapv1.File{ + {Path: "/a", Content: "hello {{ .KubernetesVersion }}"}, + } + out, err := RenderTemplates(in, data) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(out[0].Content).To(Equal("hello {{ .KubernetesVersion }}")) + g.Expect(out[0].ContentFormat).To(BeEmpty()) + }) + + t.Run("go-template renders and clears format", func(t *testing.T) { + g := NewWithT(t) + in := []bootstrapv1.File{ + {Path: "/b", ContentFormat: bootstrapv1.FileContentFormatGoTemplate, Content: "v={{ .KubernetesVersion }}"}, + } + out, err := RenderTemplates(in, data) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(out[0].Content).To(Equal("v=1.29.0")) + g.Expect(out[0].ContentFormat).To(BeEmpty()) + }) + + t.Run("bad template errors", func(t *testing.T) { + g := NewWithT(t) + in := []bootstrapv1.File{ + {Path: "/c", ContentFormat: bootstrapv1.FileContentFormatGoTemplate, Content: "{{ .KubernetesVersion "}, + } + _, err := RenderTemplates(in, data) + g.Expect(err).To(HaveOccurred()) + }) +} diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go index af9bd4cd5988..9c08273f15d4 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go @@ -229,6 +229,10 @@ func (r *KubeadmConfigReconciler) Reconcile(ctx context.Context, req ctrl.Reques conditions.ForConditionTypes{ bootstrapv1.KubeadmConfigDataSecretAvailableCondition, bootstrapv1.KubeadmConfigCertificatesAvailableCondition, + bootstrapv1.KubeadmConfigControlPlaneKubernetesVersionAvailableCondition, + }, + conditions.IgnoreTypesIfMissing{ + bootstrapv1.KubeadmConfigControlPlaneKubernetesVersionAvailableCondition, }, // Using a custom merge strategy to override reasons applied during merge and to ignore some // info message so the ready condition aggregation in other resources is less noisy. @@ -251,12 +255,14 @@ func (r *KubeadmConfigReconciler) Reconcile(ctx context.Context, req ctrl.Reques clusterv1.ReadyV1Beta1Condition, bootstrapv1.DataSecretAvailableV1Beta1Condition, bootstrapv1.CertificatesAvailableV1Beta1Condition, + bootstrapv1.ControlPlaneKubernetesVersionAvailableV1Beta1Condition, }}, patch.WithOwnedConditions{Conditions: []string{ clusterv1.PausedCondition, bootstrapv1.KubeadmConfigReadyCondition, bootstrapv1.KubeadmConfigDataSecretAvailableCondition, bootstrapv1.KubeadmConfigCertificatesAvailableCondition, + bootstrapv1.KubeadmConfigControlPlaneKubernetesVersionAvailableCondition, }}, } if rerr == nil { @@ -701,12 +707,29 @@ func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope) } // Use the control plane (cluster) version for worker join so that e.g. a 1.34 node uses kubeadm 1.35 - // when the control plane is at 1.35. Fall back to the joining machine's version if the control plane - // version cannot be determined. - kubernetesVersion := r.getControlPlaneVersionForJoin(ctx, scope) + // when the control plane is at 1.35. Fall back to the joining machine's version only when the cluster has + // no control plane ref or the referenced object does not expose spec.version (see getControlPlaneVersionForJoin). + kubernetesVersion, err := r.getControlPlaneVersionForJoin(ctx, scope) + if err != nil { + scope.Error(err, "Failed to resolve control plane Kubernetes version for worker join") + v1beta1conditions.MarkFalse(scope.Config, bootstrapv1.ControlPlaneKubernetesVersionAvailableV1Beta1Condition, bootstrapv1.ControlPlaneKubernetesVersionResolutionFailedV1Beta1Reason, clusterv1.ConditionSeverityWarning, "%s", err.Error()) + conditions.Set(scope.Config, metav1.Condition{ + Type: bootstrapv1.KubeadmConfigControlPlaneKubernetesVersionAvailableCondition, + Status: metav1.ConditionFalse, + Reason: bootstrapv1.KubeadmConfigControlPlaneKubernetesVersionResolutionFailedReason, + Message: err.Error(), + }) + return ctrl.Result{}, err + } if kubernetesVersion == "" { kubernetesVersion = scope.ConfigOwner.KubernetesVersion() } + v1beta1conditions.MarkTrue(scope.Config, bootstrapv1.ControlPlaneKubernetesVersionAvailableV1Beta1Condition) + conditions.Set(scope.Config, metav1.Condition{ + Type: bootstrapv1.KubeadmConfigControlPlaneKubernetesVersionAvailableCondition, + Status: metav1.ConditionTrue, + Reason: bootstrapv1.KubeadmConfigControlPlaneKubernetesVersionAvailableReason, + }) parsedVersion, err := semver.ParseTolerant(kubernetesVersion) if err != nil { return ctrl.Result{}, errors.Wrapf(err, "failed to parse kubernetes version %q", kubernetesVersion) @@ -841,29 +864,29 @@ func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope) } // getControlPlaneVersionForJoin returns the control plane (cluster) version from the cluster's ControlPlaneRef, -// e.g. KubeadmControlPlane.spec.version. Returns empty string if the cluster has no ControlPlaneRef or the version -// cannot be read (e.g. control plane not found or does not support version). Used for worker join so that -// a 1.34 node uses kubeadm 1.35 when the control plane is at 1.35, for example. -func (r *KubeadmConfigReconciler) getControlPlaneVersionForJoin(ctx context.Context, scope *Scope) string { +// e.g. KubeadmControlPlane.spec.version. Returns ("", nil) if the cluster has no ControlPlaneRef or the referenced +// control plane does not expose spec.version (ErrFieldNotFound or unset); callers should fall back to the machine's +// Kubernetes version only in those cases. Returns an error if the control plane object cannot be fetched or if +// the version field cannot be read for any other reason. +func (r *KubeadmConfigReconciler) getControlPlaneVersionForJoin(ctx context.Context, scope *Scope) (string, error) { if !scope.Cluster.Spec.ControlPlaneRef.IsDefined() { - return "" + return "", nil } controlPlane, err := external.GetObjectFromContractVersionedRef(ctx, r.Client, scope.Cluster.Spec.ControlPlaneRef, scope.Cluster.Namespace) if err != nil { - scope.V(4).Info("Could not get control plane for version, falling back to machine version", "error", err) - return "" + return "", errors.Wrap(err, "failed to get control plane for join version") } cpVersion, err := contract.ControlPlane().Version().Get(controlPlane) if err != nil { - if !errors.Is(err, contract.ErrFieldNotFound) { - scope.V(4).Info("Could not get control plane version, falling back to machine version", "error", err) + if errors.Is(err, contract.ErrFieldNotFound) { + return "", nil } - return "" + return "", errors.Wrap(err, "failed to read control plane version") } if cpVersion == nil { - return "" + return "", nil } - return *cpVersion + return *cpVersion, nil } func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *Scope) (ctrl.Result, error) { diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_controlplane_version_test.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_controlplane_version_test.go new file mode 100644 index 000000000000..a6b255c869bc --- /dev/null +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_controlplane_version_test.go @@ -0,0 +1,116 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + . "github.com/onsi/gomega" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/cluster-api/util/test/builder" +) + +func TestKubeadmConfigReconciler_getControlPlaneVersionForJoin(t *testing.T) { + ctx := context.Background() + + buildScope := func(cluster *clusterv1.Cluster) *Scope { + return &Scope{ + Logger: logr.Discard(), + Cluster: cluster, + } + } + + t.Run("returns empty when ControlPlaneRef is not defined", func(t *testing.T) { + g := NewWithT(t) + cluster := builder.Cluster(metav1.NamespaceDefault, "c").Build() + r := &KubeadmConfigReconciler{Client: fake.NewClientBuilder().Build()} + v, err := r.getControlPlaneVersionForJoin(ctx, buildScope(cluster)) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(v).To(BeEmpty()) + }) + + t.Run("returns version when control plane exists", func(t *testing.T) { + g := NewWithT(t) + scheme := runtime.NewScheme() + g.Expect(apiextensionsv1.AddToScheme(scheme)).To(Succeed()) + g.Expect(clusterv1.AddToScheme(scheme)).To(Succeed()) + + cp := builder.TestControlPlane(metav1.NamespaceDefault, "cp").WithVersion("v1.30.0").Build() + crd := builder.TestControlPlaneCRD.DeepCopy() + cluster := builder.Cluster(metav1.NamespaceDefault, "c").Build() + cluster.Spec.ControlPlaneRef = clusterv1.ContractVersionedObjectReference{ + APIGroup: builder.ControlPlaneGroupVersion.Group, + Kind: builder.TestControlPlaneKind, + Name: "cp", + } + + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(crd, cp, cluster).Build() + r := &KubeadmConfigReconciler{Client: c} + v, err := r.getControlPlaneVersionForJoin(ctx, buildScope(cluster)) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(v).To(Equal("v1.30.0")) + }) + + t.Run("returns error when control plane object is missing", func(t *testing.T) { + g := NewWithT(t) + scheme := runtime.NewScheme() + g.Expect(apiextensionsv1.AddToScheme(scheme)).To(Succeed()) + g.Expect(clusterv1.AddToScheme(scheme)).To(Succeed()) + + crd := builder.TestControlPlaneCRD.DeepCopy() + cluster := builder.Cluster(metav1.NamespaceDefault, "c").Build() + cluster.Spec.ControlPlaneRef = clusterv1.ContractVersionedObjectReference{ + APIGroup: builder.ControlPlaneGroupVersion.Group, + Kind: builder.TestControlPlaneKind, + Name: "missing", + } + + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(crd, cluster).Build() + r := &KubeadmConfigReconciler{Client: c} + _, err := r.getControlPlaneVersionForJoin(ctx, buildScope(cluster)) + g.Expect(err).To(HaveOccurred()) + }) + + t.Run("returns empty when control plane has no spec.version", func(t *testing.T) { + g := NewWithT(t) + scheme := runtime.NewScheme() + g.Expect(apiextensionsv1.AddToScheme(scheme)).To(Succeed()) + g.Expect(clusterv1.AddToScheme(scheme)).To(Succeed()) + + cp := builder.TestControlPlane(metav1.NamespaceDefault, "cp").Build() + crd := builder.TestControlPlaneCRD.DeepCopy() + cluster := builder.Cluster(metav1.NamespaceDefault, "c").Build() + cluster.Spec.ControlPlaneRef = clusterv1.ContractVersionedObjectReference{ + APIGroup: builder.ControlPlaneGroupVersion.Group, + Kind: builder.TestControlPlaneKind, + Name: "cp", + } + + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(crd, cp, cluster).Build() + r := &KubeadmConfigReconciler{Client: c} + v, err := r.getControlPlaneVersionForJoin(ctx, buildScope(cluster)) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(v).To(BeEmpty()) + }) +} diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_test.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_test.go index 084164e6a915..9aa39c34bd0c 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_test.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_test.go @@ -2783,7 +2783,7 @@ func TestKubeadmConfigReconciler_Reconcile_v1beta2_conditions(t *testing.T) { newConfig := &bootstrapv1.KubeadmConfig{} g.Expect(myclient.Get(ctx, key, newConfig)).To(Succeed()) - for _, conditionType := range []string{bootstrapv1.KubeadmConfigReadyCondition, bootstrapv1.KubeadmConfigCertificatesAvailableCondition, bootstrapv1.KubeadmConfigDataSecretAvailableCondition} { + for _, conditionType := range []string{bootstrapv1.KubeadmConfigReadyCondition, bootstrapv1.KubeadmConfigCertificatesAvailableCondition, bootstrapv1.KubeadmConfigDataSecretAvailableCondition, bootstrapv1.KubeadmConfigControlPlaneKubernetesVersionAvailableCondition} { condition := conditions.Get(newConfig, conditionType) g.Expect(condition).ToNot(BeNil(), "condition %s is missing", conditionType) g.Expect(condition.Status).To(Equal(metav1.ConditionTrue)) From 27988ea103794d31f901ea5ec26bfd4a4ce87c6b Mon Sep 17 00:00:00 2001 From: "Cody W. Eilar" Date: Tue, 24 Mar 2026 12:49:55 -0700 Subject: [PATCH 5/5] Update tests and condition messaging --- .../kubeadm/v1beta1/kubeadmconfig_types.go | 6 +- .../v1beta1/v1beta2_condition_consts.go | 7 +- .../kubeadm/v1beta2/kubeadm_types.go | 18 ++-- .../kubeadm/v1beta2/kubeadmconfig_types.go | 6 +- .../v1beta2/v1beta1_condition_consts.go | 8 ++ .../internal/bootstrapfiles/template_test.go | 10 +++ .../controllers/kubeadmconfig_controller.go | 25 ++++-- .../kubeadmconfig_controller_test.go | 88 ++++++++++++++++++- 8 files changed, 148 insertions(+), 20 deletions(-) diff --git a/api/bootstrap/kubeadm/v1beta1/kubeadmconfig_types.go b/api/bootstrap/kubeadm/v1beta1/kubeadmconfig_types.go index f3e7711f93a4..b2df320c6fb7 100644 --- a/api/bootstrap/kubeadm/v1beta1/kubeadmconfig_types.go +++ b/api/bootstrap/kubeadm/v1beta1/kubeadmconfig_types.go @@ -221,9 +221,6 @@ func (c *KubeadmConfigSpec) validateFiles(pathPrefix *field.Path) field.ErrorLis ), ) } - // n.b.: if we ever add types besides Secret as a ContentFrom - // Source, we must add webhook validation here for one of the - // sources being non-nil. if file.ContentFormat != "" && file.ContentFormat != FileContentFormatGoTemplate { allErrs = append( allErrs, @@ -234,6 +231,9 @@ func (c *KubeadmConfigSpec) validateFiles(pathPrefix *field.Path) field.ErrorLis ), ) } + // n.b.: if we ever add types besides Secret as a ContentFrom + // Source, we must add webhook validation here for one of the + // sources being non-nil. if file.ContentFrom != nil { if file.ContentFrom.Secret.Name == "" { allErrs = append( diff --git a/api/bootstrap/kubeadm/v1beta1/v1beta2_condition_consts.go b/api/bootstrap/kubeadm/v1beta1/v1beta2_condition_consts.go index 1f741e780508..61fedb6d13c8 100644 --- a/api/bootstrap/kubeadm/v1beta1/v1beta2_condition_consts.go +++ b/api/bootstrap/kubeadm/v1beta1/v1beta2_condition_consts.go @@ -68,8 +68,11 @@ const ( // could be resolved from the Cluster's control plane reference when applicable. KubeadmConfigControlPlaneKubernetesVersionAvailableV1Beta2Condition = "ControlPlaneKubernetesVersionAvailable" - // KubeadmConfigControlPlaneKubernetesVersionAvailableV1Beta2Reason surfaces when join version resolution succeeded. - KubeadmConfigControlPlaneKubernetesVersionAvailableV1Beta2Reason = clusterv1beta1.AvailableV1Beta2Reason + // KubeadmConfigControlPlaneKubernetesVersionFromControlPlaneV1Beta2Reason surfaces when join version was read from the control plane. + KubeadmConfigControlPlaneKubernetesVersionFromControlPlaneV1Beta2Reason = "ControlPlaneKubernetesVersionFromControlPlane" + + // KubeadmConfigControlPlaneKubernetesVersionFromMachineV1Beta2Reason surfaces when join version uses the Machine. + KubeadmConfigControlPlaneKubernetesVersionFromMachineV1Beta2Reason = "ControlPlaneKubernetesVersionFromMachine" // KubeadmConfigControlPlaneKubernetesVersionResolutionFailedV1Beta2Reason surfaces when resolution failed. KubeadmConfigControlPlaneKubernetesVersionResolutionFailedV1Beta2Reason = "ControlPlaneKubernetesVersionResolutionFailed" diff --git a/api/bootstrap/kubeadm/v1beta2/kubeadm_types.go b/api/bootstrap/kubeadm/v1beta2/kubeadm_types.go index 33073c115684..724c605ce8ba 100644 --- a/api/bootstrap/kubeadm/v1beta2/kubeadm_types.go +++ b/api/bootstrap/kubeadm/v1beta2/kubeadm_types.go @@ -73,15 +73,21 @@ const ( ) // KubeadmConfig's ControlPlaneKubernetesVersionAvailable condition and corresponding reasons. -// This condition is set when reconciling worker join bootstrap data: it reflects whether the controller -// could resolve the Kubernetes version from the Cluster's control plane reference (when present). +// This condition is set when reconciling worker join bootstrap data: it reflects how the controller chose +// the Kubernetes version for join (from the control plane reference when possible, otherwise from the Machine). const ( - // KubeadmConfigControlPlaneKubernetesVersionAvailableCondition is true when the control plane Kubernetes - // version for join could be resolved or is not required (no controlPlaneRef or control plane has no spec.version). + // KubeadmConfigControlPlaneKubernetesVersionAvailableCondition is true when the Kubernetes version for worker + // join was chosen successfully (from the control plane or from the Machine). KubeadmConfigControlPlaneKubernetesVersionAvailableCondition = "ControlPlaneKubernetesVersionAvailable" - // KubeadmConfigControlPlaneKubernetesVersionAvailableReason surfaces when join version resolution succeeded. - KubeadmConfigControlPlaneKubernetesVersionAvailableReason = clusterv1.AvailableReason + // KubeadmConfigControlPlaneKubernetesVersionFromControlPlaneReason surfaces when the Kubernetes version for + // worker join was read from the referenced control plane's spec.version. + KubeadmConfigControlPlaneKubernetesVersionFromControlPlaneReason = "FromControlPlane" + + // KubeadmConfigControlPlaneKubernetesVersionFromMachineReason surfaces when the Kubernetes version for worker + // join uses the Machine's spec.version because the Cluster has no controlPlaneRef or the referenced control + // plane does not expose a version. + KubeadmConfigControlPlaneKubernetesVersionFromMachineReason = "FromMachine" // KubeadmConfigControlPlaneKubernetesVersionResolutionFailedReason surfaces when the controller could not // read the control plane object or its Kubernetes version while a controlPlaneRef is set. diff --git a/api/bootstrap/kubeadm/v1beta2/kubeadmconfig_types.go b/api/bootstrap/kubeadm/v1beta2/kubeadmconfig_types.go index 0855b6023276..a9c2a537a79d 100644 --- a/api/bootstrap/kubeadm/v1beta2/kubeadmconfig_types.go +++ b/api/bootstrap/kubeadm/v1beta2/kubeadmconfig_types.go @@ -218,9 +218,6 @@ func (c *KubeadmConfigSpec) validateFiles(pathPrefix *field.Path) field.ErrorLis ), ) } - // n.b.: if we ever add types besides Secret as a ContentFrom - // Source, we must add webhook validation here for one of the - // sources being non-nil. if file.ContentFormat != "" && file.ContentFormat != FileContentFormatGoTemplate { allErrs = append( allErrs, @@ -231,6 +228,9 @@ func (c *KubeadmConfigSpec) validateFiles(pathPrefix *field.Path) field.ErrorLis ), ) } + // n.b.: if we ever add types besides Secret as a ContentFrom + // Source, we must add webhook validation here for one of the + // sources being non-nil. if file.ContentFrom.IsDefined() { if file.ContentFrom.Secret.Name == "" { allErrs = append( diff --git a/api/bootstrap/kubeadm/v1beta2/v1beta1_condition_consts.go b/api/bootstrap/kubeadm/v1beta2/v1beta1_condition_consts.go index 550f91184c44..16bb05091daf 100644 --- a/api/bootstrap/kubeadm/v1beta2/v1beta1_condition_consts.go +++ b/api/bootstrap/kubeadm/v1beta2/v1beta1_condition_consts.go @@ -47,6 +47,14 @@ const ( // ControlPlaneKubernetesVersionResolutionFailedV1Beta1Reason (Severity=Warning) documents a failure to read // the control plane object or its Kubernetes version while resolving join bootstrap data. ControlPlaneKubernetesVersionResolutionFailedV1Beta1Reason = "ControlPlaneKubernetesVersionResolutionFailed" + + // ControlPlaneKubernetesVersionFromControlPlaneV1Beta1Reason documents that the Kubernetes version for worker + // join was read from the Cluster's control plane reference. + ControlPlaneKubernetesVersionFromControlPlaneV1Beta1Reason = "ControlPlaneKubernetesVersionFromControlPlane" + + // ControlPlaneKubernetesVersionFromMachineV1Beta1Reason documents that the Kubernetes version for worker join + // uses the Machine's spec.version because the control plane reference is unset or does not expose a version. + ControlPlaneKubernetesVersionFromMachineV1Beta1Reason = "ControlPlaneKubernetesVersionFromMachine" ) const ( diff --git a/bootstrap/kubeadm/internal/bootstrapfiles/template_test.go b/bootstrap/kubeadm/internal/bootstrapfiles/template_test.go index 0f44920dc999..54b1a13b880a 100644 --- a/bootstrap/kubeadm/internal/bootstrapfiles/template_test.go +++ b/bootstrap/kubeadm/internal/bootstrapfiles/template_test.go @@ -59,4 +59,14 @@ func TestRenderTemplates(t *testing.T) { _, err := RenderTemplates(in, data) g.Expect(err).To(HaveOccurred()) }) + + t.Run("template execution errors on missing field", func(t *testing.T) { + g := NewWithT(t) + in := []bootstrapv1.File{ + {Path: "/d", ContentFormat: bootstrapv1.FileContentFormatGoTemplate, Content: "{{ .NonExistentField }}"}, + } + _, err := RenderTemplates(in, data) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(`failed to execute go-template for file "/d"`)) + }) } diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go index 9c08273f15d4..6cd699dfb3b7 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go @@ -721,14 +721,29 @@ func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope) }) return ctrl.Result{}, err } + var cpVersionReasonV1Beta1, cpVersionReason string + var cpVersionMsg string if kubernetesVersion == "" { kubernetesVersion = scope.ConfigOwner.KubernetesVersion() - } - v1beta1conditions.MarkTrue(scope.Config, bootstrapv1.ControlPlaneKubernetesVersionAvailableV1Beta1Condition) + cpVersionReasonV1Beta1 = bootstrapv1.ControlPlaneKubernetesVersionFromMachineV1Beta1Reason + cpVersionReason = bootstrapv1.KubeadmConfigControlPlaneKubernetesVersionFromMachineReason + cpVersionMsg = "Kubernetes version for join uses the Machine's spec.version because the control plane reference is unset or does not expose a version." + } else { + cpVersionReasonV1Beta1 = bootstrapv1.ControlPlaneKubernetesVersionFromControlPlaneV1Beta1Reason + cpVersionReason = bootstrapv1.KubeadmConfigControlPlaneKubernetesVersionFromControlPlaneReason + cpVersionMsg = "Kubernetes version for join was read from the Cluster's control plane reference." + } + v1beta1conditions.Set(scope.Config, &clusterv1.Condition{ + Type: bootstrapv1.ControlPlaneKubernetesVersionAvailableV1Beta1Condition, + Status: corev1.ConditionTrue, + Reason: cpVersionReasonV1Beta1, + Message: cpVersionMsg, + }) conditions.Set(scope.Config, metav1.Condition{ - Type: bootstrapv1.KubeadmConfigControlPlaneKubernetesVersionAvailableCondition, - Status: metav1.ConditionTrue, - Reason: bootstrapv1.KubeadmConfigControlPlaneKubernetesVersionAvailableReason, + Type: bootstrapv1.KubeadmConfigControlPlaneKubernetesVersionAvailableCondition, + Status: metav1.ConditionTrue, + Reason: cpVersionReason, + Message: cpVersionMsg, }) parsedVersion, err := semver.ParseTolerant(kubernetesVersion) if err != nil { diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_test.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_test.go index 9aa39c34bd0c..57d278942117 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_test.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_test.go @@ -25,8 +25,10 @@ import ( ignition "github.com/flatcar/ignition/config/v2_3" . "github.com/onsi/gomega" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" bootstrapapi "k8s.io/cluster-bootstrap/token/api" @@ -2787,8 +2789,23 @@ func TestKubeadmConfigReconciler_Reconcile_v1beta2_conditions(t *testing.T) { condition := conditions.Get(newConfig, conditionType) g.Expect(condition).ToNot(BeNil(), "condition %s is missing", conditionType) g.Expect(condition.Status).To(Equal(metav1.ConditionTrue)) - g.Expect(condition.Message).To(BeEmpty()) + if conditionType == bootstrapv1.KubeadmConfigControlPlaneKubernetesVersionAvailableCondition { + g.Expect(condition.Reason).To(Equal(bootstrapv1.KubeadmConfigControlPlaneKubernetesVersionFromMachineReason)) + g.Expect(condition.Message).To(ContainSubstring("Machine")) + } else { + g.Expect(condition.Message).To(BeEmpty()) + } } + foundCPKubeVer := false + for _, c := range newConfig.GetV1Beta1Conditions() { + if c.Type == bootstrapv1.ControlPlaneKubernetesVersionAvailableV1Beta1Condition { + g.Expect(c.Status).To(Equal(corev1.ConditionTrue)) + g.Expect(c.Reason).To(Equal(bootstrapv1.ControlPlaneKubernetesVersionFromMachineV1Beta1Reason)) + foundCPKubeVer = true + break + } + } + g.Expect(foundCPKubeVer).To(BeTrue()) for _, conditionType := range []string{clusterv1.PausedCondition} { condition := conditions.Get(newConfig, conditionType) g.Expect(condition).ToNot(BeNil(), "condition %s is missing", conditionType) @@ -2798,3 +2815,72 @@ func TestKubeadmConfigReconciler_Reconcile_v1beta2_conditions(t *testing.T) { }) } } + +func TestKubeadmConfigReconciler_Reconcile_v1beta2_conditions_ControlPlaneKubernetesVersionFromControlPlaneRef(t *testing.T) { + g := NewWithT(t) + scheme := runtime.NewScheme() + g.Expect(apiextensionsv1.AddToScheme(scheme)).To(Succeed()) + g.Expect(clusterv1.AddToScheme(scheme)).To(Succeed()) + g.Expect(bootstrapv1.AddToScheme(scheme)).To(Succeed()) + + clusterName := "my-cluster-cp" + cluster := builder.Cluster(metav1.NamespaceDefault, clusterName).Build() + cluster.Status.Conditions = []metav1.Condition{{Type: clusterv1.ClusterControlPlaneInitializedCondition, Status: metav1.ConditionTrue}} + cluster.Status.Initialization.InfrastructureProvisioned = ptr.To(true) + cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "example.com", Port: 6443} + cluster.Spec.ControlPlaneRef = clusterv1.ContractVersionedObjectReference{ + APIGroup: builder.ControlPlaneGroupVersion.Group, + Kind: builder.TestControlPlaneKind, + Name: "cp", + } + cp := builder.TestControlPlane(metav1.NamespaceDefault, "cp").WithVersion(testSkewK8sVersion).Build() + crd := builder.TestControlPlaneCRD.DeepCopy() + + machine := builder.Machine(metav1.NamespaceDefault, "my-machine"). + WithVersion(testK8sVersion). + WithClusterName(cluster.Name). + WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "").Unstructured()). + Build() + kubeadmConfig := newKubeadmConfig(metav1.NamespaceDefault, "kubeadmconfig") + kubeadmConfig.SetOwnerReferences([]metav1.OwnerReference{{ + APIVersion: clusterv1.GroupVersion.String(), + Kind: "Machine", + Name: machine.Name, + }}) + + objects := []client.Object{cluster, machine, kubeadmConfig, cp, crd} + objects = append(objects, createSecrets(t, cluster, kubeadmConfig)...) + + myclient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build() + + r := &KubeadmConfigReconciler{ + Client: myclient, + SecretCachingClient: myclient, + ClusterCache: clustercache.NewFakeClusterCache(myclient, client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}), + KubeadmInitLock: &myInitLocker{}, + } + + key := client.ObjectKeyFromObject(kubeadmConfig) + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + g.Expect(err).ToNot(HaveOccurred()) + + newConfig := &bootstrapv1.KubeadmConfig{} + g.Expect(myclient.Get(ctx, key, newConfig)).To(Succeed()) + + c := conditions.Get(newConfig, bootstrapv1.KubeadmConfigControlPlaneKubernetesVersionAvailableCondition) + g.Expect(c).ToNot(BeNil()) + g.Expect(c.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(c.Reason).To(Equal(bootstrapv1.KubeadmConfigControlPlaneKubernetesVersionFromControlPlaneReason)) + g.Expect(c.Message).To(ContainSubstring("control plane reference")) + + foundCPKubeVer := false + for _, cond := range newConfig.GetV1Beta1Conditions() { + if cond.Type == bootstrapv1.ControlPlaneKubernetesVersionAvailableV1Beta1Condition { + g.Expect(cond.Status).To(Equal(corev1.ConditionTrue)) + g.Expect(cond.Reason).To(Equal(bootstrapv1.ControlPlaneKubernetesVersionFromControlPlaneV1Beta1Reason)) + foundCPKubeVer = true + break + } + } + g.Expect(foundCPKubeVer).To(BeTrue()) +}