Merge pull request #10 from swisscom/master

bump

Author: JamesClonk

Dockerfile

@@ -5,7 +5,13 @@ RUN echo "debconf debconf/frontend select noninteractive" | debconf-set-selectio
   export DEBIAN_FRONTEND=noninteractive && \
   apt-get -y $package_args update && \
   apt-get -y $package_args dist-upgrade && \
-  apt-get -y $package_args install curl ca-certificates gnupg tzdata golang git
+  apt-get -y $package_args install curl ca-certificates gnupg tzdata git
+RUN curl --location --output go.tar.gz "https://golang.org/dl/go1.16.3.linux-amd64.tar.gz" && \
+  echo "951a3c7c6ce4e56ad883f97d9db74d3d6d80d5fec77455c6ada6c1f7ac4776d2  go.tar.gz" | sha256sum -c  && \
+  tar -C /usr/local -xzf go.tar.gz && \
+  rm go.tar.gz 
+
+ENV PATH=$PATH:/usr/local/go/bin
 
 WORKDIR /go/src/github.com/swisscom/backman
 COPY . .
@@ -27,7 +33,7 @@ RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \
   echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.4 multiverse" > /etc/apt/sources.list.d/mongodb-org-4.4.list
 RUN curl -sL https://deb.nodesource.com/setup_lts.x | bash -
 RUN apt-get -y $package_args update && \
-  apt-get -y $package_args install mysql-client postgresql-client-12 mongodb-org-tools=4.4.0 mongodb-org-shell=4.4.0 redis-tools nodejs openssh-server bash vim && \
+  apt-get -y $package_args install mysql-client postgresql-client-12 mongodb-org-tools=4.4.5 mongodb-org-shell=4.4.5 redis-tools nodejs openssh-server bash vim && \
   apt-get clean && \
   find /usr/share/doc/*/* ! -name copyright | xargs rm -rf && \
   rm -rf \

config/config.go

@@ -44,9 +44,11 @@ type ServiceConfig struct {
 		Days  int
 		Files int
 	}
-	DisableColumnStatistics bool   `json:"disable_column_statistics"`
-	ForceImport             bool   `json:"force_import"`
-	LocalBackupPath         string `json:"local_backup_path"`
+	DisableColumnStatistics bool     `json:"disable_column_statistics"`
+	ForceImport             bool     `json:"force_import"`
+	LocalBackupPath         string   `json:"local_backup_path"`
+	BackupOptions           []string `json:"backup_options"`
+	RestoreOptions          []string `json:"restore_options"`
 }
 
 type TimeoutDuration struct {
@@ -170,6 +172,12 @@ func Get() *Config {
 				if len(serviceConfig.LocalBackupPath) > 0 {
 					mergedServiceConfig.LocalBackupPath = serviceConfig.LocalBackupPath
 				}
+				if len(serviceConfig.BackupOptions) > 0 {
+					mergedServiceConfig.BackupOptions = serviceConfig.BackupOptions
+				}
+				if len(serviceConfig.RestoreOptions) > 0 {
+					mergedServiceConfig.RestoreOptions = serviceConfig.RestoreOptions
+				}
 				config.Services[serviceName] = mergedServiceConfig
 			}
 		}

go.mod

@@ -1,17 +1,28 @@
 module github.com/swisscom/backman
 
-go 1.14
+go 1.16
 
 require (
 	github.com/cloudfoundry-community/go-cfenv v1.18.0
 	github.com/dustin/go-humanize v1.0.0
-	github.com/hako/durafmt v0.0.0-20190612201238-650ed9f29a84
-	github.com/labstack/echo/v4 v4.1.6
-	github.com/minio/minio-go/v6 v6.0.33
-	github.com/minio/sio v0.2.0
-	github.com/prometheus/client_golang v1.1.0
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd
+	github.com/klauspost/cpuid/v2 v2.0.6 // indirect
+	github.com/labstack/echo/v4 v4.2.2
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/minio/minio-go/v6 v6.0.57
+	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/minio/sio v0.2.1
+	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/prometheus/client_golang v1.10.0
+	github.com/prometheus/common v0.20.0 // indirect
 	github.com/robfig/cron v1.2.0
-	github.com/sirupsen/logrus v1.4.2
+	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.6.1 // indirect
-	golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b
+	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
+	golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1 // indirect
+	golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750 // indirect
+	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
+	gopkg.in/ini.v1 v1.62.0 // indirect
 )

go.sum

@@ -23,7 +23,7 @@ applications:
 
   # ### push either as docker image
   docker:
-    image: jamesclonk/backman:1.26.0 # choose version from https://hub.docker.com/r/jamesclonk/backman/tags, or 'latest'
+    image: jamesclonk/backman:1.27.2 # choose version from https://hub.docker.com/r/jamesclonk/backman/tags, or 'latest'
   # ### or as buildpack/src
   # buildpacks:
   # - https://github.com/cloudfoundry/apt-buildpack

manifest.yml

@@ -107,7 +107,7 @@ func (h *Handler) GetBackup(c echo.Context) error {
 // schemes: http, https
 //
 // responses:
-//   202:
+//   202: service
 func (h *Handler) CreateBackup(c echo.Context) error {
 	serviceType := c.Param("service_type")
 	serviceName, err := url.QueryUnescape(c.Param("service_name"))
@@ -132,7 +132,7 @@ func (h *Handler) CreateBackup(c echo.Context) error {
 			log.Errorf("requested backup for service [%s] failed: %v", serviceName, err)
 		}
 	}()
-	return c.JSON(http.StatusAccepted, nil)
+	return c.JSON(http.StatusAccepted, cfService)
 }
 
 // swagger:route GET /api/v1/backup/{service_type}/{service_name}/{filename}/download backup downloadBackup

router/api/backup.go

@@ -10,7 +10,7 @@ import (
 	"github.com/swisscom/backman/service/util"
 )
 
-// swagger:route POST /api/v1/restore/{service_type}/{service_name} restore restoreBackup
+// swagger:route POST /api/v1/restore/{service_type}/{service_name}/{filename} restore restoreBackup
 // Triggers a restore for given service.
 //
 // produces:
@@ -19,7 +19,7 @@ import (
 // schemes: http, https
 //
 // responses:
-//   202:
+//   202: service
 func (h *Handler) RestoreBackup(c echo.Context) error {
 	serviceType := c.Param("service_type")
 	serviceName, err := url.QueryUnescape(c.Param("service_name"))
@@ -64,5 +64,8 @@ func (h *Handler) RestoreBackup(c echo.Context) error {
 			log.Errorf("requested restore for service [%s] failed: %v", serviceName, err)
 		}
 	}()
-	return c.JSON(http.StatusAccepted, nil)
+	if len(targetService.Name) > 0 {
+		return c.JSON(http.StatusAccepted, targetService)
+	}
+	return c.JSON(http.StatusAccepted, cfService)
 }

router/api/restore.go

@@ -29,7 +29,7 @@ func Backup(ctx context.Context, s3 *s3.Client, service util.Service, binding *c
 	esMutex.Lock()
 	defer esMutex.Unlock()
 
-	state.BackupStart(service)
+	state.BackupStart(service, filename)
 
 	host, _ := binding.CredentialString("host")
 	username, _ := binding.CredentialString("full_access_username")
@@ -58,7 +58,7 @@ func Backup(ctx context.Context, s3 *s3.Client, service util.Service, binding *c
 	outPipe, err := cmd.StdoutPipe()
 	if err != nil {
 		log.Errorf("could not get stdout pipe for elasticdump: %v", err)
-		state.BackupFailure(service)
+		state.BackupFailure(service, filename)
 		return err
 	}
 	defer outPipe.Close()
@@ -94,7 +94,7 @@ func Backup(ctx context.Context, s3 *s3.Client, service util.Service, binding *c
 		err = s3.UploadWithContext(uploadCtx, objectPath, pr, -1)
 		if err != nil {
 			log.Errorf("could not upload service backup [%s] to S3: %v", service.Name, err)
-			state.BackupFailure(service)
+			state.BackupFailure(service, filename)
 		}
 	}()
 	time.Sleep(2 * time.Second) // wait for upload goroutine to be ready
@@ -105,12 +105,12 @@ func Backup(ctx context.Context, s3 *s3.Client, service util.Service, binding *c
 
 	if err := cmd.Start(); err != nil {
 		log.Errorf("could not run elasticdump: %v", err)
-		state.BackupFailure(service)
+		state.BackupFailure(service, filename)
 		return err
 	}
 
 	if err := cmd.Wait(); err != nil {
-		state.BackupFailure(service)
+		state.BackupFailure(service, filename)
 		// check for timeout error
 		if ctx.Err() == context.DeadlineExceeded {
 			return fmt.Errorf("elasticdump: timeout: %v", ctx.Err())
@@ -122,7 +122,7 @@ func Backup(ctx context.Context, s3 *s3.Client, service util.Service, binding *c
 
 	uploadWait.Wait() // wait for upload to have finished
 	if err == nil {
-		state.BackupSuccess(service)
+		state.BackupSuccess(service, filename)
 	}
 	return err
 }

service/elasticsearch/backup.go

@@ -8,6 +8,7 @@ import (
 	"net/url"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 
 	"github.com/cloudfoundry-community/go-cfenv"
@@ -24,7 +25,8 @@ func Restore(ctx context.Context, s3 *s3.Client, service util.Service, binding *
 	esMutex.Lock()
 	defer esMutex.Unlock()
 
-	state.RestoreStart(service)
+	filename := filepath.Base(objectPath)
+	state.RestoreStart(service, filename)
 
 	host, _ := binding.CredentialString("host")
 	username, _ := binding.CredentialString("full_access_username")
@@ -56,14 +58,14 @@ func Restore(ctx context.Context, s3 *s3.Client, service util.Service, binding *
 	reader, err := s3.DownloadWithContext(downloadCtx, objectPath)
 	if err != nil {
 		log.Errorf("could not download service backup [%s] from S3: %v", service.Name, err)
-		state.RestoreFailure(service)
+		state.RestoreFailure(service, filename)
 		return err
 	}
 	defer reader.Close()
 	gr, err := gzip.NewReader(reader)
 	if err != nil {
 		log.Errorf("could not open gzip reader: %v", err)
-		state.RestoreFailure(service)
+		state.RestoreFailure(service, filename)
 		return err
 	}
 	defer gr.Close()
@@ -75,12 +77,12 @@ func Restore(ctx context.Context, s3 *s3.Client, service util.Service, binding *
 
 	if err := cmd.Start(); err != nil {
 		log.Errorf("could not run elasticsearch restore: %v", err)
-		state.RestoreFailure(service)
+		state.RestoreFailure(service, filename)
 		return err
 	}
 
 	if err := cmd.Wait(); err != nil {
-		state.RestoreFailure(service)
+		state.RestoreFailure(service, filename)
 		// check for timeout error
 		if ctx.Err() == context.DeadlineExceeded {
 			return fmt.Errorf("elasticsearch restore: timeout: %v", ctx.Err())
@@ -89,7 +91,7 @@ func Restore(ctx context.Context, s3 *s3.Client, service util.Service, binding *
 	}
 
 	if err == nil {
-		state.RestoreSuccess(service)
+		state.RestoreSuccess(service, filename)
 	}
 	return err
 }

service/elasticsearch/restore.go

@@ -28,7 +28,7 @@ func Backup(ctx context.Context, s3 *s3.Client, service util.Service, binding *c
 	mongoMutex.Lock()
 	defer mongoMutex.Unlock()
 
-	state.BackupStart(service)
+	state.BackupStart(service, filename)
 
 	uri, _ := binding.CredentialString("uri")
 
@@ -49,7 +49,7 @@ func Backup(ctx context.Context, s3 *s3.Client, service util.Service, binding *c
 	outPipe, err := cmd.StdoutPipe()
 	if err != nil {
 		log.Errorf("could not get stdout pipe for mongodump: %v", err)
-		state.BackupFailure(service)
+		state.BackupFailure(service, filename)
 		return err
 	}
 	defer outPipe.Close()
@@ -75,7 +75,7 @@ func Backup(ctx context.Context, s3 *s3.Client, service util.Service, binding *c
 		err = s3.UploadWithContext(uploadCtx, objectPath, pr, -1)
 		if err != nil {
 			log.Errorf("could not upload service backup [%s] to S3: %v", service.Name, err)
-			state.BackupFailure(service)
+			state.BackupFailure(service, filename)
 		}
 	}()
 	time.Sleep(2 * time.Second) // wait for upload goroutine to be ready
@@ -86,12 +86,12 @@ func Backup(ctx context.Context, s3 *s3.Client, service util.Service, binding *c
 
 	if err := cmd.Start(); err != nil {
 		log.Errorf("could not run mongodump: %v", err)
-		state.BackupFailure(service)
+		state.BackupFailure(service, filename)
 		return err
 	}
 
 	if err := cmd.Wait(); err != nil {
-		state.BackupFailure(service)
+		state.BackupFailure(service, filename)
 		// check for timeout error
 		if ctx.Err() == context.DeadlineExceeded {
 			return fmt.Errorf("mongodump: timeout: %v", ctx.Err())
@@ -103,7 +103,7 @@ func Backup(ctx context.Context, s3 *s3.Client, service util.Service, binding *c
 
 	uploadWait.Wait() // wait for upload to have finished
 	if err == nil {
-		state.BackupSuccess(service)
+		state.BackupSuccess(service, filename)
 	}
 	return err
 }