Merge branch 'dev' into main

Author: sandraromanchenko

pbm/backup/physical.go

@@ -511,16 +511,32 @@ func (b *Backup) uploadPhysical(
 	l log.LogEvent,
 ) error {
 	l.Info("uploading data")
-	dataFiles, err := uploadFiles(ctx, data, bcp.Name+"/"+rsMeta.Name, dbpath,
-		b.typ == defs.IncrementalBackup, stg, bcp.Compression, bcp.CompressionLevel, l)
+	dataFiles, err := uploadFiles(
+		ctx,
+		data,
+		bcp.Name+"/"+rsMeta.Name,
+		dbpath,
+		b.typ == defs.IncrementalBackup,
+		stg,
+		bcp.Compression,
+		bcp.CompressionLevel,
+	)
 	if err != nil {
 		return errors.Wrap(err, "upload data files")
 	}
 	l.Info("uploading data done")
 
 	l.Info("uploading journals")
-	ju, err := uploadFiles(ctx, jrnls, bcp.Name+"/"+rsMeta.Name, dbpath,
-		false, stg, bcp.Compression, bcp.CompressionLevel, l)
+	ju, err := uploadFiles(
+		ctx,
+		jrnls,
+		bcp.Name+"/"+rsMeta.Name,
+		dbpath,
+		false,
+		stg,
+		bcp.Compression,
+		bcp.CompressionLevel,
+	)
 	if err != nil {
 		return errors.Wrap(err, "upload journal files")
 	}
@@ -533,7 +549,12 @@ func (b *Backup) uploadPhysical(
 	sizeUncompressed := int64(0)
 	for _, f := range filelist {
 		size += f.StgSize
-		sizeUncompressed += f.Size
+		if f.StgSize != 0 {
+			// maintain uncompressed size just for the files that have a disk footprint,
+			// backup meta might contains files which are unchanged from the previous
+			// inc/base backup
+			sizeUncompressed += f.Size
+		}
 	}
 
 	filelistPath := path.Join(bcp.Name, rsMeta.Name, FilelistName)
@@ -633,7 +654,6 @@ func uploadFiles(
 	stg storage.Storage,
 	comprT compress.CompressionType,
 	comprL *int,
-	l log.LogEvent,
 ) ([]File, error) {
 	if len(files) == 0 {
 		return nil, nil
@@ -675,7 +695,7 @@ func uploadFiles(
 			continue
 		}
 
-		fw, err := writeFile(ctx, wfile, path.Join(subdir, trim(wfile.Name)), stg, comprT, comprL, l)
+		fw, err := writeFile(ctx, wfile, path.Join(subdir, trim(wfile.Name)), stg, comprT, comprL)
 		if err != nil {
 			return data, errors.Wrapf(err, "upload file `%s`", wfile.Name)
 		}
@@ -690,7 +710,7 @@ func uploadFiles(
 		return data, nil
 	}
 
-	f, err := writeFile(ctx, wfile, path.Join(subdir, trim(wfile.Name)), stg, comprT, comprL, l)
+	f, err := writeFile(ctx, wfile, path.Join(subdir, trim(wfile.Name)), stg, comprT, comprL)
 	if err != nil {
 		return data, errors.Wrapf(err, "upload file `%s`", wfile.Name)
 	}
@@ -708,7 +728,6 @@ func writeFile(
 	stg storage.Storage,
 	compression compress.CompressionType,
 	compressLevel *int,
-	l log.LogEvent,
 ) (*File, error) {
 	fstat, err := os.Stat(src.Name)
 	if err != nil {
@@ -739,7 +758,7 @@ func writeFile(
 
 	return &File{
 		Name:    src.Name,
-		Size:    fstat.Size(),
+		Size:    sz,
 		Fmode:   fstat.Mode(),
 		StgSize: finf.Size,
 		Off:     src.Off,

pbm/config/config.go

@@ -542,6 +542,7 @@ func SetConfig(ctx context.Context, m connect.Client, cfg *Config) error {
 	if err := cfg.Storage.Cast(); err != nil {
 		return errors.Wrap(err, "cast storage")
 	}
+	sanitizeStoragePaths(&cfg.Storage)
 
 	if cfg.Storage.Type == storage.S3 {
 		// call the function for notification purpose.
@@ -613,6 +614,10 @@ func SetConfigVar(ctx context.Context, m connect.Client, key, val string) error
 		return errors.Wrapf(err, "casting value of %s", key)
 	}
 
+	if isStoragePathKey(key) {
+		v = storage.TrimSlashes(v.(string))
+	}
+
 	// TODO: how to be with special case options like pitr.enabled
 	switch key {
 	case "pitr.enabled":
@@ -635,6 +640,50 @@ func SetConfigVar(ctx context.Context, m connect.Client, key, val string) error
 	return errors.Wrap(err, "write to db")
 }
 
+// sanitizeStoragePaths trims leading/trailing slashes from bucket, container,
+// and prefix values in the storage config. Extra slashes in these values cause
+// S3-compatible API signature errors or backup discovery failures.
+func sanitizeStoragePaths(s *StorageConf) {
+	switch s.Type {
+	case storage.S3:
+		if s.S3 != nil {
+			s.S3.Bucket = storage.TrimSlashes(s.S3.Bucket)
+			s.S3.Prefix = storage.TrimSlashes(s.S3.Prefix)
+		}
+	case storage.Minio:
+		if s.Minio != nil {
+			s.Minio.Bucket = storage.TrimSlashes(s.Minio.Bucket)
+			s.Minio.Prefix = storage.TrimSlashes(s.Minio.Prefix)
+		}
+	case storage.GCS:
+		if s.GCS != nil {
+			s.GCS.Bucket = storage.TrimSlashes(s.GCS.Bucket)
+			s.GCS.Prefix = storage.TrimSlashes(s.GCS.Prefix)
+		}
+	case storage.Azure:
+		if s.Azure != nil {
+			s.Azure.Container = storage.TrimSlashes(s.Azure.Container)
+			s.Azure.Prefix = storage.TrimSlashes(s.Azure.Prefix)
+		}
+	case storage.OSS:
+		if s.OSS != nil {
+			s.OSS.Bucket = storage.TrimSlashes(s.OSS.Bucket)
+			s.OSS.Prefix = storage.TrimSlashes(s.OSS.Prefix)
+		}
+	}
+}
+
+// isStoragePathKey reports whether the config key refers to a storage
+// bucket, container, or prefix field that should have slashes trimmed.
+func isStoragePathKey(key string) bool {
+	if !strings.HasPrefix(key, "storage.") {
+		return false
+	}
+	return strings.HasSuffix(key, ".bucket") ||
+		strings.HasSuffix(key, ".container") ||
+		strings.HasSuffix(key, ".prefix")
+}
+
 func confSetPITR(ctx context.Context, m connect.Client, value bool) error {
 	ct, err := topo.GetClusterTime(ctx, m)
 	if err != nil {

pbm/config/config_test.go

@@ -9,7 +9,8 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/percona/percona-backup-mongodb/pbm/storage/oss"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"github.com/testcontainers/testcontainers-go"
 	"github.com/testcontainers/testcontainers-go/modules/mongodb"
 	"go.mongodb.org/mongo-driver/mongo"
@@ -22,6 +23,7 @@ import (
 	"github.com/percona/percona-backup-mongodb/pbm/storage/fs"
 	"github.com/percona/percona-backup-mongodb/pbm/storage/gcs"
 	"github.com/percona/percona-backup-mongodb/pbm/storage/mio"
+	"github.com/percona/percona-backup-mongodb/pbm/storage/oss"
 	"github.com/percona/percona-backup-mongodb/pbm/storage/s3"
 )
 
@@ -458,6 +460,147 @@ func TestConfig(t *testing.T) {
 	})
 }
 
+func TestSanitizeStoragePaths(t *testing.T) {
+	tests := []struct {
+		name       string
+		conf       StorageConf
+		wantBucket string
+		wantPrefix string
+	}{
+		{
+			"s3 trailing slash on bucket",
+			StorageConf{Type: storage.S3, S3: &s3.Config{Bucket: "bcp/", Prefix: "data"}},
+			"bcp", "data",
+		},
+		{
+			"s3 leading slash on prefix",
+			StorageConf{Type: storage.S3, S3: &s3.Config{Bucket: "bcp", Prefix: "/data/pbm"}},
+			"bcp", "data/pbm",
+		},
+		{
+			"s3 both slashes",
+			StorageConf{Type: storage.S3, S3: &s3.Config{Bucket: "bcp/", Prefix: "/data/"}},
+			"bcp", "data",
+		},
+		{
+			"s3 multiple slashes",
+			StorageConf{Type: storage.S3, S3: &s3.Config{Bucket: "///bcp///", Prefix: "///data///"}},
+			"bcp", "data",
+		},
+		{
+			"s3 clean values",
+			StorageConf{Type: storage.S3, S3: &s3.Config{Bucket: "bcp", Prefix: "data"}},
+			"bcp", "data",
+		},
+		{
+			"minio trailing slash",
+			StorageConf{Type: storage.Minio, Minio: &mio.Config{Bucket: "bcp/", Prefix: "/pfx/"}},
+			"bcp", "pfx",
+		},
+		{
+			"gcs leading slash on prefix",
+			StorageConf{Type: storage.GCS, GCS: &gcs.Config{Bucket: "bcp/", Prefix: "/pfx"}},
+			"bcp", "pfx",
+		},
+		{
+			"oss trailing slash",
+			StorageConf{Type: storage.OSS, OSS: &oss.Config{Bucket: "bcp/", Prefix: "/pfx/"}},
+			"bcp", "pfx",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sanitizeStoragePaths(&tt.conf)
+			switch tt.conf.Type {
+			case storage.S3:
+				assert.Equal(t, tt.wantBucket, tt.conf.S3.Bucket)
+				assert.Equal(t, tt.wantPrefix, tt.conf.S3.Prefix)
+			case storage.Minio:
+				assert.Equal(t, tt.wantBucket, tt.conf.Minio.Bucket)
+				assert.Equal(t, tt.wantPrefix, tt.conf.Minio.Prefix)
+			case storage.GCS:
+				assert.Equal(t, tt.wantBucket, tt.conf.GCS.Bucket)
+				assert.Equal(t, tt.wantPrefix, tt.conf.GCS.Prefix)
+			case storage.OSS:
+				assert.Equal(t, tt.wantBucket, tt.conf.OSS.Bucket)
+				assert.Equal(t, tt.wantPrefix, tt.conf.OSS.Prefix)
+			}
+		})
+	}
+}
+
+func TestSanitizeStoragePathsAzure(t *testing.T) {
+	conf := StorageConf{Type: storage.Azure, Azure: &azure.Config{Container: "cnt/", Prefix: "/pfx/"}}
+	sanitizeStoragePaths(&conf)
+	assert.Equal(t, "cnt", conf.Azure.Container)
+	assert.Equal(t, "pfx", conf.Azure.Prefix)
+}
+
+func TestIsStoragePathKey(t *testing.T) {
+	// Storage keys that should match.
+	for _, key := range []string{
+		"storage.s3.bucket",
+		"storage.s3.prefix",
+		"storage.minio.bucket",
+		"storage.minio.prefix",
+		"storage.gcs.bucket",
+		"storage.gcs.prefix",
+		"storage.azure.container",
+		"storage.azure.prefix",
+		"storage.oss.bucket",
+		"storage.oss.prefix",
+	} {
+		assert.True(t, isStoragePathKey(key), "expected true for %q", key)
+	}
+
+	// Non-storage keys must not match.
+	for _, key := range []string{
+		"pitr.enabled",
+		"pitr.compression",
+		"storage.s3.region",
+		"storage.s3.debugLogLevels",
+		"storage.filesystem.path",
+		"bucket",
+		"prefix",
+	} {
+		assert.False(t, isStoragePathKey(key), "expected false for %q", key)
+	}
+}
+
+func TestSetConfigVarTrimsSlashes(t *testing.T) {
+	ctx := context.Background()
+
+	// Set up initial config so SetConfigVar works
+	emptyCfg := &Config{
+		Storage: StorageConf{Type: storage.S3, S3: &s3.Config{Bucket: "init"}},
+	}
+	err := SetConfig(ctx, connClient, emptyCfg)
+	require.NoError(t, err)
+
+	tests := []struct {
+		key     string
+		val     string
+		wantVal string
+	}{
+		{"storage.s3.bucket", "bcp/", "bcp"},
+		{"storage.s3.prefix", "/data/pbm/", "data/pbm"},
+		{"storage.s3.bucket", "///bcp///", "bcp"},
+		{"storage.s3.prefix", "clean", "clean"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.key+"="+tt.val, func(t *testing.T) {
+			err := SetConfigVar(ctx, connClient, tt.key, tt.val)
+			require.NoError(t, err)
+
+			got, err := GetConfigVar(ctx, connClient, tt.key)
+			require.NoError(t, err)
+			assert.Equal(t, tt.wantVal, got)
+		})
+	}
+}
+
 func boolPtr(b bool) *bool {
 	return &b
 }

pbm/config/profile.go

@@ -64,6 +64,7 @@ func AddProfile(ctx context.Context, m connect.Client, profile *Config) error {
 	if err := profile.Storage.Cast(); err != nil {
 		return errors.Wrap(err, "cast storage")
 	}
+	sanitizeStoragePaths(&profile.Storage)
 
 	if profile.Storage.Type == storage.S3 {
 		// call the function for notification purpose.

pbm/storage/storage.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"strings"
 
 	"github.com/percona/percona-backup-mongodb/pbm/compress"
 	"github.com/percona/percona-backup-mongodb/pbm/defs"
@@ -393,3 +394,11 @@ func (s MaskedString) MarshalYAML() (any, error) {
 	}
 	return "***", nil
 }
+
+// TrimSlashes removes leading and trailing '/' characters from a storage path
+// component (bucket, prefix, container). This prevents issues with S3-compatible
+// APIs where extra slashes in bucket names or prefixes cause signature errors
+// or object discovery failures.
+func TrimSlashes(s string) string {
+	return strings.Trim(s, "/")
+}