Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,6 +1,18 @@
{{- if and .Values.tls.enabled (eq .Values.tls.issuer "UserProvided") }}
{{- $clusterName := include "kblib.clusterName" . }}
{{- $namespace := .Release.Namespace }}
{{- $svcNames := list (printf "%s-clickhouse" $clusterName) (printf "%s-ch-keeper" $clusterName) }}
{{- $clusterDomain := "cluster.local" }}
{{- $dnsNames := list "localhost" }}
{{- range $svc := $svcNames }}
{{- $dnsNames = concat $dnsNames (list
$svc
(printf "%s.%s.svc" $svc $namespace)
(printf "*.%s-headless.%s.svc.%s" $svc $namespace $clusterDomain)
) }}
{{- end }}
{{- $ca := genCA "KubeBlocks" 36500 }}
{{- $cert := genSignedCert "clickhouse" (list "127.0.0.1" "::1") (list "localhost" "*.cluster.local") 36500 $ca }}
{{- $cert := genSignedCert "clickhouse" (list "127.0.0.1" "::1") $dnsNames 36500 $ca }}
apiVersion: v1
kind: Secret
metadata:
Expand All @@ -12,4 +24,4 @@ stringData:
ca.crt: {{ $ca.Cert | quote }}
tls.crt: {{ $cert.Cert | quote }}
tls.key: {{ $cert.Key | quote }}
{{- end }}
{{- end }}
5 changes: 4 additions & 1 deletion addons/clickhouse/configs/00_default_overrides.xml.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,9 @@
<server>
<certificateFile>{{$CERT_FILE}}</certificateFile>
<privateKeyFile>{{$KEY_FILE}}</privateKeyFile>
<!--
Use relaxed verification for ClickHouse to skip hostname check, while still supporting TLS encryption.
-->
<verificationMode>relaxed</verificationMode>
<caConfig>{{$CA_FILE}}</caConfig>
<cacheSessions>true</cacheSessions>
Expand All @@ -114,7 +117,7 @@
<cacheSessions>true</cacheSessions>
<disableProtocols>sslv2,sslv3</disableProtocols>
<preferServerCiphers>true</preferServerCiphers>
<verificationMode>relaxed</verificationMode>
<verificationMode>strict</verificationMode>
<invalidCertificateHandler>
<name>RejectCertificateHandler</name>
</invalidCertificateHandler>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,9 @@
<server>
<certificateFile>{{$CERT_FILE}}</certificateFile>
<privateKeyFile>{{$KEY_FILE}}</privateKeyFile>
<!--
Use relaxed verification for Keeper to skip hostname check, while still supporting TLS encryption.
-->
<verificationMode>relaxed</verificationMode>
<caConfig>{{$CA_FILE}}</caConfig>
<cacheSessions>true</cacheSessions>
Expand All @@ -97,7 +100,7 @@
<cacheSessions>true</cacheSessions>
<disableProtocols>sslv2,sslv3</disableProtocols>
<preferServerCiphers>true</preferServerCiphers>
<verificationMode>relaxed</verificationMode>
<verificationMode>strict</verificationMode>
<invalidCertificateHandler>
<name>RejectCertificateHandler</name>
</invalidCertificateHandler>
Expand Down
5 changes: 4 additions & 1 deletion addons/clickhouse/configs/client.xml.tpl
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
<config>
<user>admin</user>
<user from_env="CLICKHOUSE_ADMIN_USER"/>
<password from_env="CLICKHOUSE_ADMIN_PASSWORD"/>
{{- if $.component.tlsConfig -}}
{{- $CA_FILE := getCAFile -}}
<secure>true</secure>
<port from_env="CLICKHOUSE_TCP_SECURE_PORT"/>
<openSSL>
<client>
<caConfig>{{$CA_FILE}}</caConfig>
<certificateFile>{{$CERT_FILE}}</certificateFile>
<privateKeyFile>{{$KEY_FILE}}</privateKeyFile>
</client>
</openSSL>
{{- end }}
Expand Down
28 changes: 14 additions & 14 deletions addons/clickhouse/scripts/clickhouse-ping.sh
Original file line number Diff line number Diff line change
@@ -1,26 +1,26 @@
#!/bin/bash
set -euo pipefail

HOST="127.0.0.1"
SCHEME="http"
PORT="${CLICKHOUSE_HTTP_PORT:-8123}"

wget_args=(
-O /dev/null
-q
-T 3
--tries=1
CURL_ARGS=(
-sf
--max-time 3
"http://127.0.0.1:${PORT}/ping"
)

if [[ "${TLS_ENABLED:-false}" == "true" ]]; then
SCHEME="https"
PORT="${CLICKHOUSE_HTTPS_PORT:-8443}"
wget_args+=(--no-check-certificate)
CURL_ARGS=(
-sf
--max-time 3
--cacert /etc/pki/tls/ca.pem
--cert /etc/pki/tls/cert.pem
--key /etc/pki/tls/key.pem
"https://127.0.0.1:${PORT}/ping"
)
fi

endpoint="${SCHEME}://${HOST}:${PORT}/ping"

if ! /shared-tools/wget "${wget_args[@]}" "${endpoint}"; then
echo "Readiness probe failed accessing ${endpoint}" >&2
if ! /shared-tools/curl "${CURL_ARGS[@]}" >/dev/null; then
echo "Readiness probe failed" >&2
exit 1
fi
21 changes: 17 additions & 4 deletions addons/clickhouse/scripts/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,9 @@ function keeper_run() {
--query "$query"
)
if [[ "${TLS_ENABLED:-false}" == "true" ]]; then
keeper_args+=(--secure --tls-ca-file "$CLICKHOUSE_TLS_CA" --tls-cert-file "$CLICKHOUSE_TLS_CERT" --tls-key-file "$CLICKHOUSE_TLS_KEY")
keeper_args+=(--tls-ca-file "$CLICKHOUSE_TLS_CA" --tls-cert-file "$CLICKHOUSE_TLS_CERT" --tls-key-file "$CLICKHOUSE_TLS_KEY")
fi
if output=$(clickhouse-keeper-client "${keeper_args[@]}" 2>&1); then

if [[ "$output" != *"Coordination error"* ]] &&
[[ "$output" != *"Connection refused"* ]] &&
[[ "$output" != *"Timeout"* ]]; then
Expand Down Expand Up @@ -130,15 +129,29 @@ function get_mode_by_keeper() {
echo "$mode" | awk '{print $2}'
}

# Find leader node from member addresses
# Get mode with retry to tolerate some network failures
function get_mode_with_retry() {
local host="$1"
for _ in {1..5}; do
local mode
if mode=$(get_mode "$host") && [[ -n "$mode" ]]; then
echo "$mode"
return 0
fi
sleep 6
done
return 1
}

# Find leader node from member addresses with retry mechanism
function find_leader() {
local member_addresses="$1"
[[ -z "$member_addresses" ]] && return 1

while IFS=',' read -ra members; do
for member_addr in "${members[@]}"; do
local member_fqdn="${member_addr%:*}"
mode=$(get_mode "$member_fqdn")
local mode=$(get_mode_with_retry "$member_fqdn")
if [[ "$mode" == "leader" || "$mode" == "standalone" ]]; then
echo "$member_fqdn"
return 0
Expand Down
14 changes: 2 additions & 12 deletions addons/clickhouse/scripts/keeper-member-join.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,11 @@ if [[ "${TLS_ENABLED:-false}" == "true" ]]; then
keeper_raft_port=${CLICKHOUSE_KEEPER_RAFT_TLS_PORT:-9444}
fi

function check_is_leader() {
local mode=$(get_mode 127.0.0.1)
if [[ "$mode" == "leader" ]]; then
echo "INFO: This member is the leader, no need to join."
return 0
fi
}

# 1. Find leader from existing members
leader_fqdn=$(find_leader "$KB_MEMBER_ADDRESSES")
if [[ -z "$leader_fqdn" ]]; then
if ! check_is_leader; then
echo "ERROR: Could not find cluster leader."
exit 1
fi
echo "ERROR: Could not find keeper leader"
exit 1
fi

# 2. Extract ordinal from pod name and calculate server ID
Expand Down
4 changes: 2 additions & 2 deletions addons/clickhouse/templates/cmpd-ch.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ spec:
- sh
- -c
- |
cp /bin/wget /shared-tools/wget
chmod +x /shared-tools/wget
cp /bin/curl /shared-tools/curl
chmod +x /shared-tools/curl
volumeMounts:
- name: shared-tools
mountPath: /shared-tools
Expand Down
17 changes: 8 additions & 9 deletions addons/clickhouse/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -33,25 +33,24 @@ clickhouseVersions:
- version: 22.3.18
imageTag: 22.3.18-debian-11-r3

busyboxImage:
# if the value of busyboxImage.registry is not specified using `--set`, it will be set to the value of 'image.registry' by default
registry: ""
repository: apecloud/busybox
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: 1.37.0-musl

roleProbe:
initialDelaySeconds: 15
failureThreshold: 3
periodSeconds: 3
timeoutSeconds: 3

busyboxImage:
registry: ""
repository: apecloud/bash-busybox
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: 1.37.0-musl-curl

backupImage:
registry: ""
repository: apecloud/clickhouse-backup-full
pullPolicy: IfNotPresent
tag: 2.6.14
tag: 2.6.42

restore:
schemaReadyTimeoutSeconds: 1800
Expand Down
2 changes: 1 addition & 1 deletion addons/etcd/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -49,4 +49,4 @@ images:
pullPolicy: IfNotPresent
bashBusybox:
repository: apecloud/bash-busybox
tag: 1.37.0-musl
tag: 1.37.0-musl-curl
Loading