chore: improve ck memberjoin (#2431)

Co-authored-by: loomts <loomts@users.noreply.github.com>

Author: loomts

addons-cluster/clickhouse-cluster/templates/cluster-tls-secrets.yaml

@@ -1,6 +1,18 @@
 {{- if and .Values.tls.enabled (eq .Values.tls.issuer "UserProvided") }}
+{{- $clusterName := include "kblib.clusterName" . }}
+{{- $namespace := .Release.Namespace }}
+{{- $svcNames := list (printf "%s-clickhouse" $clusterName) (printf "%s-ch-keeper" $clusterName) }}
+{{- $clusterDomain := "cluster.local" }}
+{{- $dnsNames := list "localhost" }}
+{{- range $svc := $svcNames }}
+{{- $dnsNames = concat $dnsNames (list
+  $svc
+  (printf "%s.%s.svc" $svc $namespace)
+  (printf "*.%s-headless.%s.svc.%s" $svc $namespace $clusterDomain)
+  ) }}
+{{- end }}
 {{- $ca := genCA "KubeBlocks" 36500 }}
-{{- $cert := genSignedCert "clickhouse" (list "127.0.0.1" "::1") (list "localhost" "*.cluster.local") 36500 $ca }}
+{{- $cert := genSignedCert "clickhouse" (list "127.0.0.1" "::1") $dnsNames 36500 $ca }}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -12,4 +24,4 @@ stringData:
   ca.crt: {{ $ca.Cert | quote }}
   tls.crt: {{ $cert.Cert | quote }}
   tls.key: {{ $cert.Key | quote }}
-{{- end }}
+{{- end }}

addons/clickhouse/configs/00_default_overrides.xml.tpl

@@ -100,6 +100,9 @@
     <server>
       <certificateFile>{{$CERT_FILE}}</certificateFile>
       <privateKeyFile>{{$KEY_FILE}}</privateKeyFile>
+      <!-- 
+        Use relaxed verification for ClickHouse to skip hostname check, while still supporting TLS encryption.
+      -->
       <verificationMode>relaxed</verificationMode>
       <caConfig>{{$CA_FILE}}</caConfig>
       <cacheSessions>true</cacheSessions>
@@ -114,7 +117,7 @@
       <cacheSessions>true</cacheSessions>
       <disableProtocols>sslv2,sslv3</disableProtocols>
       <preferServerCiphers>true</preferServerCiphers>
-      <verificationMode>relaxed</verificationMode>
+      <verificationMode>strict</verificationMode>
       <invalidCertificateHandler>
         <name>RejectCertificateHandler</name>
       </invalidCertificateHandler>

addons/clickhouse/configs/ch-keeper_00_default_overrides.xml.tpl

@@ -83,6 +83,9 @@
     <server>
       <certificateFile>{{$CERT_FILE}}</certificateFile>
       <privateKeyFile>{{$KEY_FILE}}</privateKeyFile>
+      <!-- 
+        Use relaxed verification for Keeper to skip hostname check, while still supporting TLS encryption.
+      -->
       <verificationMode>relaxed</verificationMode>
       <caConfig>{{$CA_FILE}}</caConfig>
       <cacheSessions>true</cacheSessions>
@@ -97,7 +100,7 @@
       <cacheSessions>true</cacheSessions>
       <disableProtocols>sslv2,sslv3</disableProtocols>
       <preferServerCiphers>true</preferServerCiphers>
-      <verificationMode>relaxed</verificationMode>
+      <verificationMode>strict</verificationMode>
       <invalidCertificateHandler>
         <name>RejectCertificateHandler</name>
       </invalidCertificateHandler>

addons/clickhouse/configs/client.xml.tpl

@@ -1,12 +1,15 @@
 <config>
-  <user>admin</user>
+  <user from_env="CLICKHOUSE_ADMIN_USER"/>
   <password from_env="CLICKHOUSE_ADMIN_PASSWORD"/>
   {{- if $.component.tlsConfig -}}
   {{- $CA_FILE := getCAFile -}}
   <secure>true</secure>
+  <port from_env="CLICKHOUSE_TCP_SECURE_PORT"/>
   <openSSL>
     <client>
       <caConfig>{{$CA_FILE}}</caConfig>
+      <certificateFile>{{$CERT_FILE}}</certificateFile>
+      <privateKeyFile>{{$KEY_FILE}}</privateKeyFile>
     </client>
   </openSSL>
   {{- end }}

addons/clickhouse/scripts/clickhouse-ping.sh

@@ -1,26 +1,26 @@
 #!/bin/bash
 set -euo pipefail
 
-HOST="127.0.0.1"
-SCHEME="http"
 PORT="${CLICKHOUSE_HTTP_PORT:-8123}"
-
-wget_args=(
-	-O /dev/null
-	-q
-	-T 3
-	--tries=1
+CURL_ARGS=(
+	-sf
+	--max-time 3
+	"http://127.0.0.1:${PORT}/ping"
 )
 
 if [[ "${TLS_ENABLED:-false}" == "true" ]]; then
-	SCHEME="https"
 	PORT="${CLICKHOUSE_HTTPS_PORT:-8443}"
-	wget_args+=(--no-check-certificate)
+	CURL_ARGS=(
+		-sf
+		--max-time 3
+		--cacert /etc/pki/tls/ca.pem
+		--cert /etc/pki/tls/cert.pem
+		--key /etc/pki/tls/key.pem
+		"https://127.0.0.1:${PORT}/ping"
+	)
 fi
 
-endpoint="${SCHEME}://${HOST}:${PORT}/ping"
-
-if ! /shared-tools/wget "${wget_args[@]}" "${endpoint}"; then
-	echo "Readiness probe failed accessing ${endpoint}" >&2
+if ! /shared-tools/curl "${CURL_ARGS[@]}" >/dev/null; then
+	echo "Readiness probe failed" >&2
 	exit 1
 fi

addons/clickhouse/scripts/common.sh

@@ -43,10 +43,9 @@ function keeper_run() {
 			--query "$query"
 		)
 		if [[ "${TLS_ENABLED:-false}" == "true" ]]; then
-			keeper_args+=(--secure --tls-ca-file "$CLICKHOUSE_TLS_CA" --tls-cert-file "$CLICKHOUSE_TLS_CERT" --tls-key-file "$CLICKHOUSE_TLS_KEY")
+			keeper_args+=(--tls-ca-file "$CLICKHOUSE_TLS_CA" --tls-cert-file "$CLICKHOUSE_TLS_CERT" --tls-key-file "$CLICKHOUSE_TLS_KEY")
 		fi
 		if output=$(clickhouse-keeper-client "${keeper_args[@]}" 2>&1); then
-
 			if [[ "$output" != *"Coordination error"* ]] &&
 				[[ "$output" != *"Connection refused"* ]] &&
 				[[ "$output" != *"Timeout"* ]]; then
@@ -130,15 +129,29 @@ function get_mode_by_keeper() {
 	echo "$mode" | awk '{print $2}'
 }
 
-# Find leader node from member addresses
+# Get mode with retry to tolerate some network failures
+function get_mode_with_retry() {
+	local host="$1"
+	for _ in {1..5}; do
+		local mode
+		if mode=$(get_mode "$host") && [[ -n "$mode" ]]; then
+			echo "$mode"
+			return 0
+		fi
+		sleep 6
+	done
+	return 1
+}
+
+# Find leader node from member addresses with retry mechanism
 function find_leader() {
 	local member_addresses="$1"
 	[[ -z "$member_addresses" ]] && return 1
 
 	while IFS=',' read -ra members; do
 		for member_addr in "${members[@]}"; do
 			local member_fqdn="${member_addr%:*}"
-			mode=$(get_mode "$member_fqdn")
+			local mode=$(get_mode_with_retry "$member_fqdn")
 			if [[ "$mode" == "leader" || "$mode" == "standalone" ]]; then
 				echo "$member_fqdn"
 				return 0

addons/clickhouse/scripts/keeper-member-join.sh

@@ -8,21 +8,11 @@ if [[ "${TLS_ENABLED:-false}" == "true" ]]; then
 	keeper_raft_port=${CLICKHOUSE_KEEPER_RAFT_TLS_PORT:-9444}
 fi
 
-function check_is_leader() {
-	local mode=$(get_mode 127.0.0.1)
-	if [[ "$mode" == "leader" ]]; then
-		echo "INFO: This member is the leader, no need to join."
-		return 0
-	fi
-}
-
 # 1. Find leader from existing members
 leader_fqdn=$(find_leader "$KB_MEMBER_ADDRESSES")
 if [[ -z "$leader_fqdn" ]]; then
-	if ! check_is_leader; then
-		echo "ERROR: Could not find cluster leader."
-		exit 1
-	fi
+	echo "ERROR: Could not find keeper leader"
+	exit 1
 fi
 
 # 2. Extract ordinal from pod name and calculate server ID

addons/clickhouse/templates/cmpd-ch.yaml

@@ -24,8 +24,8 @@ spec:
           - sh
           - -c
           - |
-            cp /bin/wget /shared-tools/wget
-            chmod +x /shared-tools/wget
+            cp /bin/curl /shared-tools/curl
+            chmod +x /shared-tools/curl
         volumeMounts:
           - name: shared-tools
             mountPath: /shared-tools

addons/clickhouse/values.yaml

@@ -33,25 +33,24 @@ clickhouseVersions:
   - version: 22.3.18
     imageTag: 22.3.18-debian-11-r3
 
-busyboxImage:
-  # if the value of busyboxImage.registry is not specified using `--set`, it will be set to the value of 'image.registry' by default
-  registry: ""
-  repository: apecloud/busybox
-  pullPolicy: IfNotPresent
-  # Overrides the image tag whose default is the chart appVersion.
-  tag: 1.37.0-musl
-
 roleProbe:
   initialDelaySeconds: 15
   failureThreshold: 3
   periodSeconds: 3
   timeoutSeconds: 3
 
+busyboxImage:
+  registry: ""
+  repository: apecloud/bash-busybox
+  pullPolicy: IfNotPresent
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: 1.37.0-musl-curl
+
 backupImage:
   registry: ""
   repository: apecloud/clickhouse-backup-full
   pullPolicy: IfNotPresent
-  tag: 2.6.14
+  tag: 2.6.42
 
 restore:
   schemaReadyTimeoutSeconds: 1800

addons/etcd/values.yaml

@@ -49,4 +49,4 @@ images:
   pullPolicy: IfNotPresent
   bashBusybox:
     repository: apecloud/bash-busybox
-    tag: 1.37.0-musl
+    tag: 1.37.0-musl-curl