Skip to content

Commit 8994772

Browse files
authored
Merge pull request #9 from swisscom/master
bump
2 parents e7f1d0e + d97af30 commit 8994772

File tree

7 files changed

+109
-63
lines changed

7 files changed

+109
-63
lines changed

README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,8 @@ Possible JSON properties:
8585
- `disable_web`: optional, disable web interface and api
8686
- `disable_metrics`: optional, disable Prometheus metrics endpoint
8787
- `unprotected_metrics`: optional, disable HTTP basic auth protection for Prometheus metrics endpoint
88+
- `s3.disable_ssl`: optional, S3 client connections will use HTTP instead of HTTPS
89+
- `s3.skip_ssl_verification`: optional, S3 client will still use HTTPS but skips certificate verification
8890
- `s3.service_label`: optional, defines which service label backman will look for to find the S3-compatible object storage
8991
- `s3.bucket_name`: optional, bucket to use on S3 storage, backman will use service-instance/binding-name if not configured
9092
- `s3.encryption_key`: optional, defines the key which will be used to encrypt and decrypt backups as they are stored on the S3 can also be passed as an environment variable with the name `BACKMAN_ENCRYPTION_KEY`

config/config.go

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,12 @@ type Config struct {
2929
}
3030

3131
type S3Config struct {
32-
DisableSSL bool `json:"disable_ssl"`
33-
ServiceLabel string `json:"service_label"`
34-
ServiceName string `json:"service_name"`
35-
BucketName string `json:"bucket_name"`
36-
EncryptionKey string `json:"encryption_key"`
32+
DisableSSL bool `json:"disable_ssl"`
33+
SkipSSLVerification bool `json:"skip_ssl_verification"`
34+
ServiceLabel string `json:"service_label"`
35+
ServiceName string `json:"service_name"`
36+
BucketName string `json:"bucket_name"`
37+
EncryptionKey string `json:"encryption_key"`
3738
}
3839

3940
type ServiceConfig struct {
@@ -131,6 +132,9 @@ func Get() *Config {
131132
if envConfig.S3.DisableSSL {
132133
config.S3.DisableSSL = envConfig.S3.DisableSSL
133134
}
135+
if envConfig.S3.SkipSSLVerification {
136+
config.S3.SkipSSLVerification = envConfig.S3.SkipSSLVerification
137+
}
134138
if len(envConfig.S3.ServiceLabel) > 0 {
135139
config.S3.ServiceLabel = envConfig.S3.ServiceLabel
136140
}

manifest.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ applications:
2323

2424
# ### push either as docker image
2525
docker:
26-
image: jamesclonk/backman:1.24.1 # choose version from https://hub.docker.com/r/jamesclonk/backman/tags, or 'latest'
26+
image: jamesclonk/backman:1.26.0 # choose version from https://hub.docker.com/r/jamesclonk/backman/tags, or 'latest'
2727
# ### or as buildpack/src
2828
# buildpacks:
2929
# - https://github.com/cloudfoundry/apt-buildpack

s3/client.go

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
package s3
22

33
import (
4+
"crypto/tls"
5+
"net"
6+
"net/http"
7+
"time"
8+
49
cfenv "github.com/cloudfoundry-community/go-cfenv"
510
"github.com/minio/minio-go/v6"
611
"github.com/swisscom/backman/config"
@@ -53,6 +58,24 @@ func New(app *cfenv.App) *Client {
5358
log.Fatalf("%v", err)
5459
}
5560

61+
if config.Get().S3.SkipSSLVerification {
62+
log.Debugln("disabling S3 client SSL verification ...")
63+
minioClient.SetCustomTransport(&http.Transport{
64+
Proxy: http.ProxyFromEnvironment,
65+
TLSClientConfig: &tls.Config{
66+
InsecureSkipVerify: true,
67+
},
68+
DialContext: (&net.Dialer{
69+
Timeout: 30 * time.Second,
70+
KeepAlive: 30 * time.Second,
71+
}).DialContext,
72+
MaxIdleConns: 100,
73+
IdleConnTimeout: 90 * time.Second,
74+
TLSHandshakeTimeout: 10 * time.Second,
75+
ExpectContinueTimeout: 1 * time.Second,
76+
})
77+
}
78+
5679
// check if bucket exists and is accessible and if not create it, or fail
5780
exists, errBucketExists := minioClient.BucketExists(bucketName)
5881
if errBucketExists == nil && exists {

s3/objects.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@ func (s *Client) UploadWithContext(ctx context.Context, object string, reader io
4646
size = -1
4747
}
4848

49-
var err error
5049
uploadReader := reader
5150
if len(config.Get().S3.EncryptionKey) != 0 {
5251
hdr := newHeader(sio.AES_256_GCM, kdfScrypt)
@@ -63,6 +62,7 @@ func (s *Client) UploadWithContext(ctx context.Context, object string, reader io
6362
}
6463
uploadReader = io.MultiReader(bytes.NewBuffer(hdr[:]), uploadReader)
6564
}
65+
6666
n, err := s.Client.PutObjectWithContext(ctx, s.BucketName, object, uploadReader, size, minio.PutObjectOptions{ContentType: "application/gzip"})
6767
if err != nil {
6868
return err
@@ -90,13 +90,13 @@ func (s *Client) DownloadWithContext(ctx context.Context, object string) (io.Rea
9090
if err != nil {
9191
return nil, err
9292
}
93-
masterKey := config.Get().S3.EncryptionKey
94-
if len(masterKey) > 0 {
93+
94+
if len(config.Get().S3.EncryptionKey) > 0 {
9595
hdr, err := readHeader(reader)
9696
if err != nil {
9797
return nil, err
9898
}
99-
key, err := getKey(masterKey, object, hdr, reader)
99+
key, err := getKey(config.Get().S3.EncryptionKey, object, hdr, reader)
100100
if err != nil {
101101
return nil, fmt.Errorf("could not derive key: %v", err)
102102
}

service/service.go

Lines changed: 67 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -57,63 +57,80 @@ func Get() *Service {
5757
func (s *Service) parseServices() {
5858
s.Services = make([]util.Service, 0)
5959

60-
for label, services := range s.App.Services {
61-
if util.IsValidServiceType(label) || label == "user-provided" {
62-
for _, service := range services {
63-
// try to figure out if user-provided service binding can be handled
64-
if service.Label == "user-provided" {
65-
// can it be identified as a custom postgres binding?
66-
if postgres.IsPostgresBinding(&service) {
67-
service.Label = "postgres"
68-
} else if mysql.IsMySQLBinding(&service) { // or a mysql binding?
69-
service.Label = "mysql"
70-
} else if redis.IsRedisBinding(&service) { // or a redis binding?
71-
service.Label = "redis"
72-
} else {
73-
continue // cannot handle service binding
60+
for _, services := range s.App.Services {
61+
for _, service := range services {
62+
// exclude S3 storage service, don't try to parse it as a service for backups
63+
if service.Name == config.Get().S3.ServiceName ||
64+
(service.Label == config.Get().S3.ServiceLabel && service.Label != "user-provided") {
65+
continue
66+
}
67+
68+
// if it is an unrecognized type/label or user-provided service
69+
// then try to figure out if it can be identified as a supported service type
70+
if !util.IsValidServiceType(service.Label) || service.Label == "user-provided" {
71+
// can it be identified as a custom postgres binding?
72+
if postgres.IsPostgresBinding(&service) {
73+
service.Label = "postgres"
74+
} else if mysql.IsMySQLBinding(&service) { // or a mysql binding?
75+
service.Label = "mysql"
76+
} else if redis.IsRedisBinding(&service) { // or a redis binding?
77+
service.Label = "redis"
78+
} else {
79+
// try to guess it via service tags as a last resort
80+
var identified bool
81+
for _, tag := range service.Tags {
82+
if util.IsValidServiceType(tag) {
83+
identified = true
84+
service.Label = tag
85+
break
86+
}
87+
}
88+
if !identified {
89+
log.Errorf("unsupported service type [%s]: could not identify [%s]", service.Label, service.Name)
90+
continue // cannot handle this service binding
7491
}
7592
}
93+
}
7694

77-
// read timeout for service
78-
timeout := config.Get().Services[service.Name].Timeout
79-
if timeout.Seconds() <= 1 {
80-
timeout.Duration = 1 * time.Hour // default
81-
}
95+
// read timeout for service
96+
timeout := config.Get().Services[service.Name].Timeout
97+
if timeout.Seconds() <= 1 {
98+
timeout.Duration = 1 * time.Hour // default
99+
}
82100

83-
// read crontab schedule for service
84-
schedule := config.Get().Services[service.Name].Schedule
85-
if len(schedule) == 0 {
86-
// create a random schedule for daily backup as a fallback
87-
schedule = fmt.Sprintf("%d %d %d * * *", rand.Intn(59), rand.Intn(59), rand.Intn(23))
88-
}
101+
// read crontab schedule for service
102+
schedule := config.Get().Services[service.Name].Schedule
103+
if len(schedule) == 0 {
104+
// create a random schedule for daily backup as a fallback
105+
schedule = fmt.Sprintf("%d %d %d * * *", rand.Intn(59), rand.Intn(59), rand.Intn(23))
106+
}
89107

90-
// read retention days & files, with defaults as fallback
91-
retentionDays := config.Get().Services[service.Name].Retention.Days
92-
retentionFiles := config.Get().Services[service.Name].Retention.Files
93-
if retentionDays <= 0 {
94-
retentionDays = 31 // default
95-
}
96-
if retentionFiles <= 0 {
97-
retentionFiles = 100 // default
98-
}
108+
// read retention days & files, with defaults as fallback
109+
retentionDays := config.Get().Services[service.Name].Retention.Days
110+
retentionFiles := config.Get().Services[service.Name].Retention.Files
111+
if retentionDays <= 0 {
112+
retentionDays = 31 // default
113+
}
114+
if retentionFiles <= 0 {
115+
retentionFiles = 100 // default
116+
}
99117

100-
newService := util.Service{
101-
Name: service.Name,
102-
Label: service.Label,
103-
Plan: service.Plan,
104-
Tags: service.Tags,
105-
Timeout: timeout.Duration,
106-
Schedule: schedule,
107-
Retention: util.Retention{
108-
Days: retentionDays,
109-
Files: retentionFiles,
110-
},
111-
DisableColumnStatistics: config.Get().Services[service.Name].DisableColumnStatistics,
112-
ForceImport: config.Get().Services[service.Name].ForceImport,
113-
LocalBackupPath: config.Get().Services[service.Name].LocalBackupPath,
114-
}
115-
s.Services = append(s.Services, newService)
118+
newService := util.Service{
119+
Name: service.Name,
120+
Label: service.Label,
121+
Plan: service.Plan,
122+
Tags: service.Tags,
123+
Timeout: timeout.Duration,
124+
Schedule: schedule,
125+
Retention: util.Retention{
126+
Days: retentionDays,
127+
Files: retentionFiles,
128+
},
129+
DisableColumnStatistics: config.Get().Services[service.Name].DisableColumnStatistics,
130+
ForceImport: config.Get().Services[service.Name].ForceImport,
131+
LocalBackupPath: config.Get().Services[service.Name].LocalBackupPath,
116132
}
133+
s.Services = append(s.Services, newService)
117134
}
118135
}
119136

service/util/service_type.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,13 @@ func ParseServiceType(serviceType string) ServiceType {
1414
switch serviceType {
1515
case "postgres", "pg", "psql", "postgresql", "elephantsql", "citusdb":
1616
return Postgres
17-
case "mysql", "mariadb", "mariadbent", "pxc", "galera", "mysql-database", "mariadb-k8s-database":
17+
case "mysql", "mariadb", "mariadbent", "pxc", "galera", "mysql-database", "mariadb-k8s-database", "mysql-k8s", "mariadb-k8s":
1818
return MySQL
19-
case "mongo", "mongodb", "mongodb-2", "mongodbent", "mongodbent-database", "mangodb":
19+
case "mongo", "mongodb", "mongodb-2", "mongodbent", "mongodbent-database", "mangodb", "mongodb-k8s":
2020
return MongoDB
2121
case "redis", "redis-2", "redisent", "redis-enterprise", "redis-ha", "redis-k8s":
2222
return Redis
23-
case "elastic", "es", "elasticsearch":
23+
case "elastic", "es", "elasticsearch", "ece":
2424
return Elasticsearch
2525
}
2626
return -1

0 commit comments

Comments
 (0)