fix: nebula backup and restore failed (#2435)

Author: wangyelei

addons/nebula/dataprotection/br-backup.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-set -eo pipefail
 
 toolConfig=/etc/datasafed/datasafed.conf
 
@@ -31,6 +30,7 @@ if [ -n "$region" ]; then
 fi
 
 meta_ep=$(echo $NEBULA_METAD_SVC | cut -d',' -f1)
+set -eo pipefail
 br backup full --meta ${meta_ep}  --s3.endpoint "${endpoint}" \
   --storage="s3://${bucket}${DP_BACKUP_BASE_PATH}" --s3.access_key="${access_key_id}" \
   --s3.secret_key="${secret_access_key}" ${region_flag}

addons/nebula/dataprotection/br-restore.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-set -eo pipefail
 
 toolConfig=/etc/datasafed/datasafed.conf
 
@@ -29,6 +28,7 @@ backup_name=$(datasafed list -d / | sort -Vr | head -n 1)
 backup_name=$(basename $backup_name)
 
 meta_ep=$(echo $NEBULA_METAD_SVC | cut -d',' -f1)
+set -eo pipefail
 br restore full --meta ${meta_ep} --s3.endpoint "${endpoint}" \
   --storage="s3://${bucket}/${DP_BACKUP_BASE_PATH}" --s3.access_key="${access_key_id}" \
   --s3.secret_key="${secret_access_key}" --name ${backup_name} ${region_flag}

addons/nebula/scripts/common.sh

@@ -36,6 +36,26 @@ function nebula_service_start() {
   cp ${root_dir}/config/nebula-$1.conf ${root_dir}/etc/nebula-$1.conf
   printf "\n--local_ip=${POD_FQDN}" >> ${root_dir}/etc/nebula-$1.conf
   ${root_dir}/scripts/nebula.service -c ${root_dir}/etc/nebula-$1.conf start $1
+  wait_service_ready
+}
+
+function wait_service_ready() {
+  count=0
+  set +e
+  while true; do
+      if [ $count -gt 10 ]; then
+          echo "Service is not ready after waiting for a long time"
+          exit 1
+      fi
+      count=$((count+1))
+      response=$(curl -s http://127.0.0.1:${HTTP_PORT}/status)
+      if echo "$response" | grep "running"; then
+          echo "Service is ready"
+          break
+      fi
+      sleep 3
+  done
+  set -e
 }
 
 function start_nebula_agent() {
@@ -51,4 +71,12 @@ function end_restore() {
   ${root_dir}/scripts/nebula.service stop $1
   check_service_is_stopped $1
   rm -f ${root_dir}/logs/.kb_agent
+}
+
+function check_agent() {
+  pid=`ps -eo pid,args | grep -F "/usr/local/nebula/console/agent" | grep -v "grep" | tail -1 | awk '{print $1}'`
+  if [ -z "$pid" ]; then
+    echo "$(date): Nebula agent process is not running, exit..."
+    exit 1
+  fi
 }

addons/nebula/scripts/start-agent.sh

@@ -18,6 +18,11 @@ if [ -f "/usr/local/nebula/logs/.kb_restore" ]; then
       echo "$(date): Nebula restoration completed."
       break
     fi
+    pid=`ps -eo pid,args | grep -F "restore-agent" | grep -v "grep" | tail -1 | awk '{print $1}'`
+    if [ -z "$pid" ]; then
+      echo "restore-agent is not running, exit..."
+      exit 1
+    fi
     echo "$(date): Waiting for Nebula restoration to complete..."
   done
   # kill restore-agent if it is still running

addons/nebula/scripts/start-graphd.sh

@@ -20,6 +20,7 @@ if [ -f "${root_dir}/logs/.kb_restore" ]; then
       end_restore graphd
       break
     fi
+    check_agent
     echo "$(date): Waiting for Nebula restoration to complete..."
   done
 fi

addons/nebula/scripts/start-metad.sh

@@ -24,6 +24,7 @@ if [ -f "${root_dir}/logs/.kb_restore" ]; then
       end_restore metad
       break
     fi
+    check_agent
     echo "$(date): Waiting for Nebula restoration to complete..."
   done
 fi

addons/nebula/scripts/start-storaged.sh

@@ -47,6 +47,7 @@ if [ -f "${root_dir}/logs/.kb_restore" ]; then
       end_restore storaged
       break
     fi
+    check_agent
     echo "$(date): Waiting for Nebula restoration to complete..."
   done
 fi