Skip to content

Commit 908a331

Browse files
kizuna-lekkizuna-lek
authored
chore: support es dump (#2446)
1 parent 79c445c commit 908a331

File tree

5 files changed

+216
-0
lines changed

5 files changed

+216
-0
lines changed

‎addons/elasticsearch/dataprotection/es-dump-backup.sh‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
#!/usr/bin/env bash
2+
3+
# This script performs a full logical backup of Elasticsearch using elasticsearch-dump (multielasticdump).
4+
# It dumps all indices' data, mappings, analyzers, aliases, settings, and templates,
5+
# packages them into a tar archive, and pushes to backup storage via datasafed.
6+
7+
set -e
8+
set -o errexit
9+
set -x
10+
11+
export PATH="$PATH:$DP_DATASAFED_BIN_PATH"
12+
export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH"
13+
14+
ES_ENDPOINT="http://${DP_DB_HOST}.${KB_NAMESPACE}.svc.cluster.local:9200"
15+
16+
# Exit handler: write backup info on success, or touch exit file on failure
17+
handle_exit() {
18+
exit_code=$?
19+
if [ $exit_code -ne 0 ]; then
20+
echo "failed with exit code $exit_code"
21+
touch "${DP_BACKUP_INFO_FILE}.exit"
22+
exit 1
23+
else
24+
echo "{}" >"${DP_BACKUP_INFO_FILE}"
25+
exit 0
26+
fi
27+
}
28+
trap handle_exit EXIT
29+
30+
# Build authenticated endpoint URL for elasticdump
31+
if [ -n "${ELASTIC_USER_PASSWORD}" ]; then
32+
ES_AUTH_ENDPOINT="http://elastic:${ELASTIC_USER_PASSWORD}@${DP_DB_HOST}.${KB_NAMESPACE}.svc.cluster.local:9200"
33+
else
34+
ES_AUTH_ENDPOINT="${ES_ENDPOINT}"
35+
fi
36+
37+
# Create temporary backup directory
38+
BACKUP_DIR=/tmp/es-dump-backup
39+
rm -rf ${BACKUP_DIR}
40+
mkdir -p ${BACKUP_DIR}
41+
42+
echo "INFO: Starting elasticsearch-dump full backup"
43+
echo "INFO: Elasticsearch endpoint: ${ES_ENDPOINT}"
44+
45+
# Default match pattern: only backup user indices (exclude system indices starting with ".")
46+
# System indices (.kibana, .kibana_task_manager, .security, .tasks, .apm, etc.)
47+
# are managed internally by Elasticsearch and Kibana. Restoring them from a backup
48+
# will overwrite their internal migration/state tracking and cause errors (e.g. Kibana
49+
# migration lock). Override with the MATCH env variable if needed.
50+
MATCH_PATTERN="${MATCH:-^[^\.]}"
51+
echo "INFO: Index match pattern: ${MATCH_PATTERN}"
52+
53+
# Set elasticdump options
54+
DUMP_OPTS=""
55+
if [ -n "${SCROLL_TIME}" ]; then
56+
DUMP_OPTS="${DUMP_OPTS} --scrollTime=${SCROLL_TIME}"
57+
fi
58+
if [ -n "${LIMIT}" ]; then
59+
DUMP_OPTS="${DUMP_OPTS} --limit=${LIMIT}"
60+
fi
61+
62+
# Use multielasticdump to dump all matched indices
63+
# Types: data (documents), mapping (index mappings), analyzer (custom analyzers),
64+
# alias (index aliases), settings (index settings), template (index templates)
65+
multielasticdump \
66+
--direction=dump \
67+
--input="${ES_AUTH_ENDPOINT}" \
68+
--output="${BACKUP_DIR}" \
69+
--match="${MATCH_PATTERN}" \
70+
--includeType=data,mapping,analyzer,alias,settings,template \
71+
"${DUMP_OPTS}"
72+
73+
echo "INFO: elasticsearch-dump completed, packaging backup data"
74+
75+
# Tar and push to backup storage via datasafed
76+
cd ${BACKUP_DIR}
77+
tar -cf - . | datasafed push -z zstd-fastest - "/${DP_BACKUP_NAME}.tar.zst"
78+
79+
echo "INFO: Backup data pushed to storage successfully"
80+
81+
# Cleanup
82+
rm -rf ${BACKUP_DIR}

‎addons/elasticsearch/dataprotection/es-dump-restore.sh‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
#!/usr/bin/env bash
2+
3+
# This script performs a full logical restore of Elasticsearch using elasticsearch-dump (multielasticdump).
4+
# It pulls the backup archive from storage, extracts it, and restores all indices'
5+
# data, mappings, analyzers, aliases, settings, and templates.
6+
7+
set -e
8+
set -o errexit
9+
set -x
10+
11+
export PATH="$PATH:$DP_DATASAFED_BIN_PATH"
12+
export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH"
13+
14+
ES_ENDPOINT="http://${DP_DB_HOST}.${KB_NAMESPACE}.svc.cluster.local:9200"
15+
16+
# Exit handler
17+
handle_exit() {
18+
exit_code=$?
19+
if [ $exit_code -ne 0 ]; then
20+
echo "failed with exit code $exit_code"
21+
touch "${DP_BACKUP_INFO_FILE}.exit"
22+
exit 1
23+
fi
24+
}
25+
trap handle_exit EXIT
26+
27+
# Build authenticated endpoint URL for elasticdump
28+
if [ -n "${ELASTIC_USER_PASSWORD}" ]; then
29+
ES_AUTH_ENDPOINT="http://elastic:${ELASTIC_USER_PASSWORD}@${DP_DB_HOST}.${KB_NAMESPACE}.svc.cluster.local:9200"
30+
else
31+
ES_AUTH_ENDPOINT="${ES_ENDPOINT}"
32+
fi
33+
34+
# Create temporary restore directory
35+
RESTORE_DIR=/tmp/es-dump-restore
36+
rm -rf ${RESTORE_DIR}
37+
mkdir -p ${RESTORE_DIR}
38+
39+
echo "INFO: Pulling backup data from storage"
40+
41+
# Pull and extract backup data
42+
datasafed pull -d zstd-fastest "${DP_BACKUP_NAME}.tar.zst" - | tar -xf - -C ${RESTORE_DIR}
43+
44+
echo "INFO: Backup data extracted to ${RESTORE_DIR}"
45+
ls -la ${RESTORE_DIR}
46+
47+
# Safety measure: remove any system index dump files (starting with ".") from restore directory.
48+
# System indices (.kibana, .kibana_task_manager, .security, .tasks, etc.) are managed internally
49+
# by Elasticsearch and Kibana. Restoring them overwrites their migration/state tracking and causes
50+
# errors such as Kibana migration lock loops.
51+
echo "INFO: Removing system index dump files (starting with '.') from restore directory"
52+
for f in "${RESTORE_DIR}"/.*; do
53+
case "$(basename "$f")" in
54+
.|..) continue ;;
55+
*)
56+
echo "INFO: Removing system index dump file: $f"
57+
rm -f "$f"
58+
;;
59+
esac
60+
done
61+
62+
echo "INFO: Starting elasticsearch-dump restore"
63+
64+
# Set elasticdump options
65+
DUMP_OPTS=""
66+
if [ -n "${SCROLL_TIME}" ]; then
67+
DUMP_OPTS="${DUMP_OPTS} --scrollTime=${SCROLL_TIME}"
68+
fi
69+
if [ -n "${LIMIT}" ]; then
70+
DUMP_OPTS="${DUMP_OPTS} --limit=${LIMIT}"
71+
fi
72+
73+
# Use multielasticdump to restore all indices
74+
multielasticdump \
75+
--direction=load \
76+
--input="${RESTORE_DIR}" \
77+
--output="${ES_AUTH_ENDPOINT}" \
78+
"${DUMP_OPTS}"
79+
80+
echo "INFO: elasticsearch-dump restore completed"
81+
82+
# Cleanup
83+
rm -rf ${RESTORE_DIR}
84+
85+
echo "INFO: Elasticsearch restore finished successfully"

‎addons/elasticsearch/templates/actionset-es-dump.yaml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
apiVersion: dataprotection.kubeblocks.io/v1alpha1
2+
kind: ActionSet
3+
metadata:
4+
name: elasticsearch-es-dump
5+
labels:
6+
{{- include "elasticsearch.labels" . | nindent 4 }}
7+
spec:
8+
backupType: Full
9+
env:
10+
- name: ES_DUMP_IMAGE_TAG
11+
value: {{ .Values.image.esDump.tag | default "latest" }}
12+
backup:
13+
preBackup: []
14+
postBackup: []
15+
backupData:
16+
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.esDump.repository }}:{{ .Values.image.esDump.tag }}
17+
runOnTargetPodNode: false
18+
command:
19+
- sh
20+
- -c
21+
- |
22+
{{- .Files.Get "dataprotection/es-dump-backup.sh" | nindent 8 }}
23+
syncProgress:
24+
enabled: true
25+
intervalSeconds: 5
26+
restore:
27+
postReady:
28+
- job:
29+
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.esDump.repository }}:{{ .Values.image.esDump.tag }}
30+
runOnTargetPodNode: false
31+
command:
32+
- sh
33+
- -c
34+
- |
35+
{{- .Files.Get "dataprotection/es-dump-restore.sh" | nindent 10 }}

‎addons/elasticsearch/templates/backuppolicytemplate.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,18 @@ spec:
1616
strategy: Any
1717
snapshotVolumes: false
1818
actionSetName: elasticsearch-physical-br
19+
- name: es-dump
20+
target:
21+
role: ""
22+
strategy: Any
23+
snapshotVolumes: false
24+
actionSetName: elasticsearch-es-dump
1925
schedules:
2026
- backupMethod: full-backup
2127
enabled: false
2228
cronExpression: "0 18 * * *"
2329
retentionPeriod: 7d
30+
- backupMethod: es-dump
31+
enabled: false
32+
cronExpression: "0 18 * * *"
33+
retentionPeriod: 7d

‎addons/elasticsearch/values.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ image:
3434
agent:
3535
repository: apecloud/elasticsearch-agent
3636
tag: "0.1.0"
37+
# elasticsearch-dump: https://github.com/elasticsearch-dump/elasticsearch-dump
38+
esDump:
39+
repository: elasticdump/elasticsearch-dump
40+
tag: "v6.124.2"
3741

3842
## @param supported elasticsearch versions with detailed information
3943
esVersions:

0 commit comments

Comments
 (0)