diff --git a/addons-cluster/clickhouse-cluster/templates/cluster-tls-secrets.yaml b/addons-cluster/clickhouse-cluster/templates/cluster-tls-secrets.yaml index e1f3c5939..a7c410c6d 100644 --- a/addons-cluster/clickhouse-cluster/templates/cluster-tls-secrets.yaml +++ b/addons-cluster/clickhouse-cluster/templates/cluster-tls-secrets.yaml @@ -1,6 +1,18 @@ {{- if and .Values.tls.enabled (eq .Values.tls.issuer "UserProvided") }} +{{- $clusterName := include "kblib.clusterName" . }} +{{- $namespace := .Release.Namespace }} +{{- $svcNames := list (printf "%s-clickhouse" $clusterName) (printf "%s-ch-keeper" $clusterName) }} +{{- $clusterDomain := "cluster.local" }} +{{- $dnsNames := list "localhost" }} +{{- range $svc := $svcNames }} +{{- $dnsNames = concat $dnsNames (list + $svc + (printf "%s.%s.svc" $svc $namespace) + (printf "*.%s-headless.%s.svc.%s" $svc $namespace $clusterDomain) + ) }} +{{- end }} {{- $ca := genCA "KubeBlocks" 36500 }} -{{- $cert := genSignedCert "clickhouse" (list "127.0.0.1" "::1") (list "localhost" "*.cluster.local") 36500 $ca }} +{{- $cert := genSignedCert "clickhouse" (list "127.0.0.1" "::1") $dnsNames 36500 $ca }} apiVersion: v1 kind: Secret metadata: @@ -12,4 +24,4 @@ stringData: ca.crt: {{ $ca.Cert | quote }} tls.crt: {{ $cert.Cert | quote }} tls.key: {{ $cert.Key | quote }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/addons/clickhouse/configs/00_default_overrides.xml.tpl b/addons/clickhouse/configs/00_default_overrides.xml.tpl index ea250befe..6a4503932 100644 --- a/addons/clickhouse/configs/00_default_overrides.xml.tpl +++ b/addons/clickhouse/configs/00_default_overrides.xml.tpl @@ -100,6 +100,9 @@ {{$CERT_FILE}} {{$KEY_FILE}} + relaxed {{$CA_FILE}} true @@ -114,7 +117,7 @@ true sslv2,sslv3 true - relaxed + strict RejectCertificateHandler diff --git a/addons/clickhouse/configs/ch-keeper_00_default_overrides.xml.tpl b/addons/clickhouse/configs/ch-keeper_00_default_overrides.xml.tpl index 84a915882..88aceb473 100644 --- a/addons/clickhouse/configs/ch-keeper_00_default_overrides.xml.tpl +++ b/addons/clickhouse/configs/ch-keeper_00_default_overrides.xml.tpl @@ -83,6 +83,9 @@ {{$CERT_FILE}} {{$KEY_FILE}} + relaxed {{$CA_FILE}} true @@ -97,7 +100,7 @@ true sslv2,sslv3 true - relaxed + strict RejectCertificateHandler diff --git a/addons/clickhouse/configs/client.xml.tpl b/addons/clickhouse/configs/client.xml.tpl index 8676d90b5..b5b2326e3 100644 --- a/addons/clickhouse/configs/client.xml.tpl +++ b/addons/clickhouse/configs/client.xml.tpl @@ -1,12 +1,15 @@ - admin + {{- if $.component.tlsConfig -}} {{- $CA_FILE := getCAFile -}} true + {{$CA_FILE}} + {{$CERT_FILE}} + {{$KEY_FILE}} {{- end }} diff --git a/addons/clickhouse/scripts/clickhouse-ping.sh b/addons/clickhouse/scripts/clickhouse-ping.sh index 26fa83f2f..152769edd 100644 --- a/addons/clickhouse/scripts/clickhouse-ping.sh +++ b/addons/clickhouse/scripts/clickhouse-ping.sh @@ -1,26 +1,26 @@ #!/bin/bash set -euo pipefail -HOST="127.0.0.1" -SCHEME="http" PORT="${CLICKHOUSE_HTTP_PORT:-8123}" - -wget_args=( - -O /dev/null - -q - -T 3 - --tries=1 +CURL_ARGS=( + -sf + --max-time 3 + "http://127.0.0.1:${PORT}/ping" ) if [[ "${TLS_ENABLED:-false}" == "true" ]]; then - SCHEME="https" PORT="${CLICKHOUSE_HTTPS_PORT:-8443}" - wget_args+=(--no-check-certificate) + CURL_ARGS=( + -sf + --max-time 3 + --cacert /etc/pki/tls/ca.pem + --cert /etc/pki/tls/cert.pem + --key /etc/pki/tls/key.pem + "https://127.0.0.1:${PORT}/ping" + ) fi -endpoint="${SCHEME}://${HOST}:${PORT}/ping" - -if ! /shared-tools/wget "${wget_args[@]}" "${endpoint}"; then - echo "Readiness probe failed accessing ${endpoint}" >&2 +if ! /shared-tools/curl "${CURL_ARGS[@]}" >/dev/null; then + echo "Readiness probe failed" >&2 exit 1 fi diff --git a/addons/clickhouse/scripts/common.sh b/addons/clickhouse/scripts/common.sh index 77f0c5bcf..28f69fd75 100644 --- a/addons/clickhouse/scripts/common.sh +++ b/addons/clickhouse/scripts/common.sh @@ -43,10 +43,9 @@ function keeper_run() { --query "$query" ) if [[ "${TLS_ENABLED:-false}" == "true" ]]; then - keeper_args+=(--secure --tls-ca-file "$CLICKHOUSE_TLS_CA" --tls-cert-file "$CLICKHOUSE_TLS_CERT" --tls-key-file "$CLICKHOUSE_TLS_KEY") + keeper_args+=(--tls-ca-file "$CLICKHOUSE_TLS_CA" --tls-cert-file "$CLICKHOUSE_TLS_CERT" --tls-key-file "$CLICKHOUSE_TLS_KEY") fi if output=$(clickhouse-keeper-client "${keeper_args[@]}" 2>&1); then - if [[ "$output" != *"Coordination error"* ]] && [[ "$output" != *"Connection refused"* ]] && [[ "$output" != *"Timeout"* ]]; then @@ -130,7 +129,21 @@ function get_mode_by_keeper() { echo "$mode" | awk '{print $2}' } -# Find leader node from member addresses +# Get mode with retry to tolerate some network failures +function get_mode_with_retry() { + local host="$1" + for _ in {1..5}; do + local mode + if mode=$(get_mode "$host") && [[ -n "$mode" ]]; then + echo "$mode" + return 0 + fi + sleep 6 + done + return 1 +} + +# Find leader node from member addresses with retry mechanism function find_leader() { local member_addresses="$1" [[ -z "$member_addresses" ]] && return 1 @@ -138,7 +151,7 @@ function find_leader() { while IFS=',' read -ra members; do for member_addr in "${members[@]}"; do local member_fqdn="${member_addr%:*}" - mode=$(get_mode "$member_fqdn") + local mode=$(get_mode_with_retry "$member_fqdn") if [[ "$mode" == "leader" || "$mode" == "standalone" ]]; then echo "$member_fqdn" return 0 diff --git a/addons/clickhouse/scripts/keeper-member-join.sh b/addons/clickhouse/scripts/keeper-member-join.sh index 3057fbdb0..893ca064f 100644 --- a/addons/clickhouse/scripts/keeper-member-join.sh +++ b/addons/clickhouse/scripts/keeper-member-join.sh @@ -8,21 +8,11 @@ if [[ "${TLS_ENABLED:-false}" == "true" ]]; then keeper_raft_port=${CLICKHOUSE_KEEPER_RAFT_TLS_PORT:-9444} fi -function check_is_leader() { - local mode=$(get_mode 127.0.0.1) - if [[ "$mode" == "leader" ]]; then - echo "INFO: This member is the leader, no need to join." - return 0 - fi -} - # 1. Find leader from existing members leader_fqdn=$(find_leader "$KB_MEMBER_ADDRESSES") if [[ -z "$leader_fqdn" ]]; then - if ! check_is_leader; then - echo "ERROR: Could not find cluster leader." - exit 1 - fi + echo "ERROR: Could not find keeper leader" + exit 1 fi # 2. Extract ordinal from pod name and calculate server ID diff --git a/addons/clickhouse/templates/cmpd-ch.yaml b/addons/clickhouse/templates/cmpd-ch.yaml index 7a52b669d..966ebb47c 100644 --- a/addons/clickhouse/templates/cmpd-ch.yaml +++ b/addons/clickhouse/templates/cmpd-ch.yaml @@ -24,8 +24,8 @@ spec: - sh - -c - | - cp /bin/wget /shared-tools/wget - chmod +x /shared-tools/wget + cp /bin/curl /shared-tools/curl + chmod +x /shared-tools/curl volumeMounts: - name: shared-tools mountPath: /shared-tools diff --git a/addons/clickhouse/values.yaml b/addons/clickhouse/values.yaml index dd1dc6b1e..4164e23ca 100644 --- a/addons/clickhouse/values.yaml +++ b/addons/clickhouse/values.yaml @@ -33,25 +33,24 @@ clickhouseVersions: - version: 22.3.18 imageTag: 22.3.18-debian-11-r3 -busyboxImage: - # if the value of busyboxImage.registry is not specified using `--set`, it will be set to the value of 'image.registry' by default - registry: "" - repository: apecloud/busybox - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: 1.37.0-musl - roleProbe: initialDelaySeconds: 15 failureThreshold: 3 periodSeconds: 3 timeoutSeconds: 3 +busyboxImage: + registry: "" + repository: apecloud/bash-busybox + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: 1.37.0-musl-curl + backupImage: registry: "" repository: apecloud/clickhouse-backup-full pullPolicy: IfNotPresent - tag: 2.6.14 + tag: 2.6.42 restore: schemaReadyTimeoutSeconds: 1800 diff --git a/addons/etcd/values.yaml b/addons/etcd/values.yaml index bac6e69ff..bc7c05912 100644 --- a/addons/etcd/values.yaml +++ b/addons/etcd/values.yaml @@ -49,4 +49,4 @@ images: pullPolicy: IfNotPresent bashBusybox: repository: apecloud/bash-busybox - tag: 1.37.0-musl \ No newline at end of file + tag: 1.37.0-musl-curl \ No newline at end of file diff --git a/examples/clickhouse/cluster-tls.yaml b/examples/clickhouse/cluster-tls.yaml index 467ff4808..aaa1e0fef 100644 --- a/examples/clickhouse/cluster-tls.yaml +++ b/examples/clickhouse/cluster-tls.yaml @@ -91,7 +91,14 @@ type: Opaque data: password: cGFzc3dvcmQxMjM= # 'password123' in base64 --- -# pre generated tls secret +# pre generated tls secret using helm template +# Generated by: helm template clickhouse-tls addons-cluster/clickhouse --namespace demo --set tls.enabled=true --set tls.issuer=UserProvided --set tls.secretName=clickhouse-cluster-tls +# Certificate SANs include: +# DNS: localhost, clickhouse-tls-clickhouse, clickhouse-tls-clickhouse.demo.svc, +# *.clickhouse-tls-clickhouse-headless.demo.svc.cluster.local, +# clickhouse-tls-ch-keeper, clickhouse-tls-ch-keeper.demo.svc, +# *.clickhouse-tls-ch-keeper-headless.demo.svc.cluster.local +# IP: 127.0.0.1, ::1 apiVersion: v1 kind: Secret metadata: @@ -101,74 +108,75 @@ type: Opaque stringData: ca.crt: | -----BEGIN CERTIFICATE----- - MIIDCzCCAfOgAwIBAgIUBjoE02lIYlEl2RDjOp9T9wmP1TUwDQYJKoZIhvcNAQEL - BQAwFTETMBEGA1UEAwwKS3ViZUJsb2NrczAeFw0yNTEyMjMwMzQ5NDhaFw0zNTEy - MjEwMzQ5NDhaMBUxEzARBgNVBAMMCkt1YmVCbG9ja3MwggEiMA0GCSqGSIb3DQEB - AQUAA4IBDwAwggEKAoIBAQDhkMhIKhwKFi5xtK5dXVrucJ23ABqeoDTq9uBoCIV6 - hAcfvsv9AMBGWqn7NbcdKN8eYQ97M4qBRsFxR5FAfq2F5ecfgFWVElWd3IAc1RRD - E9sLeVGbhdwk91OwG41Mo0BuSvBYZXT0wHz8EIYGoB5B5vx2kpQC7mGWqeonNlBJ - 4uFdKy1oL+5lWHVK1DBGqB+h9X3nH317ERNCQuOnvrho3Hs6SajOHv25MROUIcTg - 4WiESCY+SX8MVyDnJjw4+qlMl9fdxSH+s56FrF0MzcgpB1rcIwd67sdElj0abBeo - llaMNv3asEyvNJJR20qGPLonznLPN8mqhzsPKW30qwQrAgMBAAGjUzBRMB0GA1Ud - DgQWBBR1By3uWSRlgL4ABYM5vVRbKTNakjAfBgNVHSMEGDAWgBR1By3uWSRlgL4A - BYM5vVRbKTNakjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAa - cmhCy+8xaMIn+icqqVL9mzUThMtYEPul6GjGrUqad9FC9/K9P+FDUbgPHre775IE - Sgb0LdpBATCgR+MZLWXtq7p7rnfgCpsrPhV5Trf/AF/qhmyhS4M3M2f1i+AID043 - l/VcFUVKPQm3jtjs3klGhTgg/KtE7jD6wFSa8f3LzrNWBI0Ls/yrwgZ4cEx1ur+I - iVUpC6n2+5UIBY1mm81w4TWSn6dr0kqQmxJJ4kPxrTU4XKhs+YOJ9XOiF7nUT6iy - OeR4mm4Mv97rkMoykZ+ntuRIfYrgpFIIndaiD1856K5vsWtZTDQ7GIwdaBrI/xu6 - A7TZ/BvGOKuYuKhgZcwA + MIIDFzCCAf+gAwIBAgIQflqmC1o0WGc+uBpiwIMx2zANBgkqhkiG9w0BAQsFADAV + MRMwEQYDVQQDEwpLdWJlQmxvY2tzMCAXDTI2MDIwNTA5MDc1MFoYDzIxMjYwMTEy + MDkwNzUwWjAVMRMwEQYDVQQDEwpLdWJlQmxvY2tzMIIBIjANBgkqhkiG9w0BAQEF + AAOCAQ8AMIIBCgKCAQEAz4WGIlQ0pvwSVD4mcAy+W2uUlNjRLf+LhKY1xWG9ei8R + 06xX4LIieNYZ1K8gjnH8eKcn7SaVztnp6mIsK33zTOu/cg1gLBXZArl5rBj4U79U + VMNR1dgYuLDYACv8fjL1ONxQD29hXIM+riBFQ0SH45HePsYzVN/kbd6zz4wObF/p + KMtB6HpOar/C2xTwbgUKxlCLMY7pJA0R0+7k3Q/Hl0VZi56iOCOpT0PZ3dZRMG8h + vlAaGj3gKYVFvt3ZVTQtynyNiuZsqyd7rnEFaLXgCQ6N33IpfHpbLOfhw4B/GHwx + UsyPgQNZRV3djSioM/wKVk9sOYaOxzpN4q2qlUGJowIDAQABo2EwXzAOBgNVHQ8B + Af8EBAMCAqQwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMA8GA1UdEwEB + /wQFMAMBAf8wHQYDVR0OBBYEFOQZ2TSN6S6vslPhLrX+6DR1KJ6HMA0GCSqGSIb3 + DQEBCwUAA4IBAQDH1VyjKYaWbZM0MyXp7vF9qAumimjp2I3+nvwUlgM85FJznYSZ + QK2HQosPal3SCJ7fdtx+hbPUhZc+Hq99Jlx69qfHMzeDhqdC395v4fFA0h/N9y8L + YAZcBsdR6kt5B3jculbGOHEJkAnRfuBqKF3WSuIBWoOthDOORksiQOBSL1K5A+1/ + YTA/f9xCF6y7iWD0PuW028rtgKVXVO4/pt9UZqqyduo+bhrknPGlhmwaF5fr4nvp + 5FyLBc7ufG66TAhcSkdVUm8bUd4aq4qGqmNE38HKbEatQ2cvnDWQdYQLFB4C/rW0 + nu6402RpntXT6hOeD9at6K3OWK1+vAcPes1/ -----END CERTIFICATE----- tls.crt: | -----BEGIN CERTIFICATE----- - MIID2jCCAsKgAwIBAgIUO4YobiiQDKHCIAItX/nBspRGeoUwDQYJKoZIhvcNAQEL - BQAwFTETMBEGA1UEAwwKS3ViZUJsb2NrczAeFw0yNTEyMjMwMzQ5NDlaFw0zNTEy - MjEwMzQ5NDlaMBkxFzAVBgNVBAMMDmNsaWNraG91c2UtdGxzMIIBIjANBgkqhkiG - 9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1gpO4/cVUYVkG3zs1HQ8fSh4pOrbrgIQtQQ/ - YTT6GsgaJnWaXJqhBQKQMedkhRss2OeEeKzIT+ZzqZjn+tT3OsktekS2Ll0yY+xx - b6O8iBoKrjJq8O2Fot6FwzZUxePeJl5YzUZQvvo6H1ZsMnukPZX3ZMGmo8wijUNS - pFn/R8kIGKzl3ve0r7xe75vu5Vu2lq/vrFAoPhkx8CaRfrEtVc8P9CsKLZMsUWON - 9IkzUKrO5tFwG2KMxsfNOgt0W9jN9v3dfgdrbqxCS8gkBETrZf6GGFRqfcEYHjrk - 4m+kbHpbjlh674EMqV5jFFCYF14w/CouTT/1BM0E3nhxMBU+TQIDAQABo4IBHDCC - ARgwCQYDVR0TBAIwADALBgNVHQ8EBAMCBeAwgb0GA1UdEQSBtTCBsoIJbG9jYWxo - b3N0gg8qLmNsdXN0ZXIubG9jYWyCGyouY2xpY2tob3VzZS10bHMtY2xpY2tob3Vz - ZYIaKi5jbGlja2hvdXNlLXRscy1jaC1rZWVwZXKCHyouY2xpY2tob3VzZS1jbHVz - dGVyLWNsaWNraG91c2WCHiouY2xpY2tob3VzZS1jbHVzdGVyLWNoLWtlZXBlcoIK - Y2xpY2tob3VzZYIOY2xpY2tob3VzZS10bHMwHQYDVR0OBBYEFGrrWGptkWpmPQje - BhT8uTxAEKlZMB8GA1UdIwQYMBaAFHUHLe5ZJGWAvgAFgzm9VFspM1qSMA0GCSqG - SIb3DQEBCwUAA4IBAQA9hqiYW4EGMdxNq0ODc8J2gwoIk0ancfid+7hhz5J8R8qh - Wd+dNJhXXfNAAZGC2w2xa0MleFsNRJ0roFBRVqIB/Z+ZSjiLtXcDjL8ZqLpU1Jw1 - H0WzGnuJqh97hJ6KYF5XTb2Aa2AIZ30Q4RBAQCQmd3N/i+PgHucks7/V5HF29Uw7 - AnLtBC1tfZ2uf2fBluFJieUoBtUr0R1S35sQsywMdXhzerzxPkeQauvvFZrjdnG/ - vWKVqXK1SAht7TsQ12cTGGCe81O0R3rWKXRzW7htacWUSgZuRHpPJQTZOUJbUn4k - PTzoiR+7y/3Z3N5H0wlLIkyP4qrweJTOs6JIB9XH + MIIEQzCCAyugAwIBAgIQNjtqrPaIjdrpgEu97ckGhDANBgkqhkiG9w0BAQsFADAV + MRMwEQYDVQQDEwpLdWJlQmxvY2tzMCAXDTI2MDIwNTA5MDc1MVoYDzIxMjYwMTEy + MDkwNzUxWjAVMRMwEQYDVQQDEwpjbGlja2hvdXNlMIIBIjANBgkqhkiG9w0BAQEF + AAOCAQ8AMIIBCgKCAQEAnp+yoP+StcTx0pGtwlB21ykBEf7CnJLFcZPshR8xD6Fp + EthmixjBVdaFWU414+Z/6rRkry3zccrt0D6f1b0FAaIsYgqg8kehOl94iZf7h4qq + e3aGMHHtfoqW2t+IGu+XPjyJUZHmBHPJnuWwwAlL1vN0nK/Ffp2YKAFSXm4xiyws + /atAsJscUlJIPnYdg1uYjkokdMW0CmuImN3oYsapQqh3aucbplS5/7GqSM6EheoS + EO5E8xQrDv1b18EM/m0QIxReA2zuWmlgByp6psRtwVJU9f+/es6vMkE3L/4e0WOH + 1QemDqdBPTXXCfDKpuoV0D94oS2oDl2/Crh00rLM1QIDAQABo4IBizCCAYcwDgYD + VR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNV + HRMBAf8EAjAAMB8GA1UdIwQYMBaAFOQZ2TSN6S6vslPhLrX+6DR1KJ6HMIIBJQYD + VR0RBIIBHDCCARiCCWxvY2FsaG9zdIIZY2xpY2tob3VzZS10bHMtY2xpY2tob3Vz + ZYIiY2xpY2tob3VzZS10bHMtY2xpY2tob3VzZS5kZW1vLnN2Y4I7Ki5jbGlja2hv + dXNlLXRscy1jbGlja2hvdXNlLWhlYWRsZXNzLmRlbW8uc3ZjLmNsdXN0ZXIubG9j + YWyCGGNsaWNraG91c2UtdGxzLWNoLWtlZXBlcoIhY2xpY2tob3VzZS10bHMtY2gt + a2VlcGVyLmRlbW8uc3ZjgjoqLmNsaWNraG91c2UtdGxzLWNoLWtlZXBlci1oZWFk + bGVzcy5kZW1vLnN2Yy5jbHVzdGVyLmxvY2FshwR/AAABhxAAAAAAAAAAAAAAAAAA + AAABMA0GCSqGSIb3DQEBCwUAA4IBAQAMxS8Oyqa/EXS1f8qU1VzD4R2I0DH+13vM + JjVL840HJTAjubKBkFDP7tKdLw6r1iCapk6DpXyPipBmZgpjNgV9qT9f/edmTxw5 + 661wz+H8lVPW6uFRdu7h+QV1pY224nV2riX7Vt+1zEg5XcBKTLccE/IR+/yYclS4 + F91XPwLD8m3orQCAbA0vlLTlKTs0htjBeD8jotY/AveeZUUqgwUWbhjLwPhy8hZZ + 08xkWfM97EbNy62LQgtd+rqElmFQ47xVkFA98lp928yF2J7kmUO8D3F8Ce9willW + YxR3GQRRVS5XXWPde1ew35f8532RxatuZ7WGC5B9o1PRpFPGCLyO -----END CERTIFICATE----- tls.key: | - -----BEGIN PRIVATE KEY----- - MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDWCk7j9xVRhWQb - fOzUdDx9KHik6tuuAhC1BD9hNPoayBomdZpcmqEFApAx52SFGyzY54R4rMhP5nOp - mOf61Pc6yS16RLYuXTJj7HFvo7yIGgquMmrw7YWi3oXDNlTF494mXljNRlC++jof - Vmwye6Q9lfdkwaajzCKNQ1KkWf9HyQgYrOXe97SvvF7vm+7lW7aWr++sUCg+GTHw - JpF+sS1Vzw/0KwotkyxRY430iTNQqs7m0XAbYozGx806C3Rb2M32/d1+B2turEJL - yCQEROtl/oYYVGp9wRgeOuTib6RseluOWHrvgQypXmMUUJgXXjD8Ki5NP/UEzQTe - eHEwFT5NAgMBAAECggEAKgXeFU3WhqnczLTLPq8PjTcb8K0XsmM/anrKAsjG7ekp - kTF3vASz5mrpapLWnneGZ5OU46hwr5c8UCjwKsQTQhxrbFz/M70ifpHWd6e7BTGv - tSG681B+80ojEv+gxzWE0R2m666JfeVc8fgiyAqUZW8DImoO0IvsoLV+DTyKLUqD - 4Vv+r1N0JhW8mfc8qTbnWccs+8e+C9eg/jrMDWv8UGkFLi25ayYS8XFEOYKdppqz - M5/hCSEKPkFz6+/uj1J4UX2jh6mwGKsJoFiueBWuC+ooz9QtgHAhXpifhIF2J8vu - u21Qf6ZlWWPh/20J305xBhL2IU5iqyuAE2Jp2TGLFQKBgQD1pNOowzK6FCR0EHSQ - xkONJWLHDkEEQjyOq+XkXZqzypm9vvVXWv1clNVq1fctS0sLShzp8Z9J48hRLKGR - o/yozTsss++/Bp2qtrdX2wz6eyqbmclJoxhl/Z6XaBcnTKFGYtDHpkCo1dZIDw8g - 2yP31t+ODY6yKYajB8vSGiHMowKBgQDfEGRFI9grQyibPAfmW/v/fRG8BtYQE4td - 0B/sDjSzq+4patnT1RUQRLfipDG0WBgVTE3tira4L9JqvDi2i1oxTOW/JfKDRk+b - Hblg937tSM/NkOM9kk4YChauB6qV+B08QhxnSOAnWN58UzT8MUIx5LCgwsGObgsW - b4EqC9YITwKBgQCV+6jcfyqm4QuM7kst5lByiuQv4+0gu4ycFCsO73Q42LhcWY8V - YlIWSC2yyKfeOP2+C+dxk/0NMY4quhSAh18KdhzuY4M74L8978gsVWwsOC3AyfpU - Asgv5dYCXiTc8vX5svYFIOaT79ShNMio6ASjG8htxKte7uns+yKgyyHd/wKBgH6a - Aw7qxSnouAdDDwjDdEcRaRtalewR66uXEEcd2POQxV9kcbU03vuYxPUxU7STuzd7 - U09ax1HKcpZ5tYaFmO8aQds3Ymj3Yv8a47gRQEzUYny9mvu7Ke+i6jRjzYHIjG9C - 5nQIfJBYdA4D+7KXEobW0Ris8MYx1sEpEBoZFaUpAoGARaUzWbazlSPGwA4PpV1a - A7iBBGxrr7R6itswp9C+3rM6zNboDKfA/jgivnVZkH+vEF3Etm+1ic4PTHYxuH8V - MkndnzemaQ5sPLEM/adgInVc/o8WwNiHZ7aFUKduhbpVyUiCCo+e4ILC+XHNJm14 - toNEdKxJTK9cBU3TTVma2uk= - -----END PRIVATE KEY----- + -----BEGIN RSA PRIVATE KEY----- + MIIEogIBAAKCAQEAnp+yoP+StcTx0pGtwlB21ykBEf7CnJLFcZPshR8xD6FpEthm + ixjBVdaFWU414+Z/6rRkry3zccrt0D6f1b0FAaIsYgqg8kehOl94iZf7h4qqe3aG + MHHtfoqW2t+IGu+XPjyJUZHmBHPJnuWwwAlL1vN0nK/Ffp2YKAFSXm4xiyws/atA + sJscUlJIPnYdg1uYjkokdMW0CmuImN3oYsapQqh3aucbplS5/7GqSM6EheoSEO5E + 8xQrDv1b18EM/m0QIxReA2zuWmlgByp6psRtwVJU9f+/es6vMkE3L/4e0WOH1Qem + DqdBPTXXCfDKpuoV0D94oS2oDl2/Crh00rLM1QIDAQABAoIBAAW6yIHCwYqhysdb + nj5k2CoU79Y10NNFhO7BlFtJoWTKZIh6+xc+pWsjlM0BQ3aQMqIocnOWchLz7Mun + G3Blw7rHoBlJf39I4ZTBcpHrxLjkZBySDy5MSzMVL/ZHeTvZ2Cl2F9Kzta6G/Ssp + ak1BeLUSta1cNwLIEAEv7Qj+q3j2qLVyG8Ps1HQVEuCM0zOwsmW7oUXhjW4NG2uz + hGlKxVrf6r7xNewLEqF95ltPa3hwc5rfxbwgakmwf3A96UxbtH5l5k5QzKA8yv5T + +Z17Fe8m2MyLskjb/1wOs8TLlxLMl00s4xG+Ku7UcE1ujKrpv1YHvVoj/v3GlyqX + VjwH28ECgYEA0zO+nSp51XQvZwo19iwQ0kiMDv+XRP0gQ9TcVX2SskjGaZrSkGms + 2JG9QiqclIlNvNCl/sZd2T5fjD3iTlKNzIaX9scfZN/GQNf4xUmeUXIrqzieDDzP + R/8uKF/15uXc2MDBbhH9DgsriFyowCcqpmI4gVVU5Hc6laPLrNmDt8ECgYEAwET0 + wO9ehovIbEnwNY9f9f9ycAaZAGJRI2ypLqsAsHqEF51rlgz+GvRKjvjv6cioHaF2 + JEhvOakq7EKIy1lVBSXdhSQDd0DKMLelbUqwWk2bTManxIn21hEHquqYqCEzdRz1 + mWHVW1f1QmPCt79Deijxbm2Tl7x931T5EvwDOhUCgYAigGd5IUE533sG6CIcjuJI + l9VZdeNuP7OPoSxFQvg966mOAt62/KxhzJ0QPAnMMgni+GrFjf4yyP+u10Uq6k2D + xdD5XVoBjpTCbwWSpQ4Z2/7KP7uB4EU0S7lsmxB+obpMJmDy7Dlcm/KGmixvB3bu + K0lzx07Q67FEVLenCvl7gQKBgGl6Ks9hQfkL3ELT+SxY4GsC3VPpuqwEQ8DsTo/k + jfdC7w5JdQkXTZuZ4wE2Pd+CDgBbYIWdGy+Fx59fDM6JzmOJl1IAJMqaR5GcXets + Kv0PUCA5ZzYh/cEIDK3ODztFI4afAXlIu5Rl1425Tswg6DKvHWvYPzzh0iff5Nhu + WpLVAoGAEwqBym1eP4lxdeIfbM7CfTTQWgStr9pJE+hc7v8pd/wrrHQ12/Q3Ih0+ + bwChBJ4y18Hm150+5iVp743Bw94gp/1Kuo/ZLAW18uWAlT51lVl15XuiJCYjFOZt + 3Zkx9rnDQpZSZRELXJXhRTem/62a1cqnGXC6WEFfTUlTaPaQ1eo= + -----END RSA PRIVATE KEY-----