Skip to content
This repository was archived by the owner on Jul 4, 2022. It is now read-only.

Commit 05eb9f7

Browse files
taku-ktaku-k
authored
Merge pull request #64 from taku-k/feat/add-test-1
Change meta key format and ddd more tests
2 parents f346eb9 + 6364f6a commit 05eb9f7

File tree

9 files changed

+393
-118
lines changed

9 files changed

+393
-118
lines changed

‎pkg/keys/keys.go‎

Lines changed: 62 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"github.com/pkg/errors"
88

99
"github.com/taku-k/polymerase/pkg/polypb"
10+
"github.com/taku-k/polymerase/pkg/utils"
1011
)
1112

1213
var (
@@ -27,6 +28,19 @@ func unescapeSlash(s []byte) []byte {
2728
return bytes.Replace(s, []byte(`_`), []byte("/"), -1)
2829
}
2930

31+
func encodeUint32(b []byte, v uint32) []byte {
32+
return append(b, byte(v>>24), byte(v>>16), byte(v>>8), byte(v))
33+
}
34+
35+
func decodeUint32(b []byte) ([]byte, uint32, error) {
36+
if len(b) < 4 {
37+
return nil, 0, errors.Errorf("insufficient bytes to decode uint32 int value")
38+
}
39+
v := (uint32(b[0]) << 24) | (uint32(b[1]) << 16) |
40+
(uint32(b[2]) << 8) | uint32(b[3])
41+
return b[4:], v, nil
42+
}
43+
3044
type BackupKeyItem struct {
3145
Db string
3246
StoredTime time.Time
@@ -35,8 +49,9 @@ type BackupKeyItem struct {
3549
func makeMetaPrefixWithDB(
3650
db polypb.DatabaseID, baseTime, backupTime polypb.TimePoint,
3751
) polypb.BackupMetaKey {
38-
buf := make(polypb.BackupMetaKey, 0, len(backupMetaPrefix)+len(db)+1)
52+
buf := make(polypb.BackupMetaKey, 0, len(backupMetaPrefix)+4+len(db)+1)
3953
buf = append(buf, backupMetaPrefix...)
54+
buf = encodeUint32(buf, uint32(len(db)))
4055
buf = append(buf, db...)
4156
buf = append(buf, baseTime...)
4257
buf = append(buf, backupTime...)
@@ -67,8 +82,11 @@ func MakeNodeMetaPrefix() polypb.NodeMetaKey {
6782
}
6883

6984
func MakeNodeMetaKey(node polypb.NodeID) polypb.NodeMetaKey {
70-
return polypb.NodeMetaKey(
71-
makeKey(MakeNodeMetaPrefix(), node))
85+
buf := make(polypb.NodeMetaKey, 0, len(nodeMetaPrefix)+4+len(node)+1)
86+
buf = append(buf, nodeMetaPrefix...)
87+
buf = encodeUint32(buf, uint32(len(node)))
88+
buf = append(buf, node...)
89+
return buf
7290
}
7391

7492
func MakeBackupPrefix(
@@ -88,19 +106,56 @@ func MakeBackupKey(
88106
}
89107

90108
func MakeBackupMetaKeyFromKey(key polypb.Key) polypb.BackupMetaKey {
91-
db, base, backup, err := decodeKey(key)
109+
db, base, backup, err := DecodeKey(key)
92110
if err != nil {
93111
panic(err)
94112
}
95113
return makeMetaPrefixWithDB(db, base, backup)
96114
}
97115

98-
func decodeKey(
116+
func DecodeMetaKey(
117+
key polypb.BackupMetaKey,
118+
) (db polypb.DatabaseID, baseTime, backupTime polypb.TimePoint, err error) {
119+
if !bytes.HasPrefix(key, backupMetaPrefix) {
120+
return nil, nil, nil, errors.Errorf("key %s does not have %s prefix", key, backupMetaPrefix)
121+
}
122+
b := key[len(backupMetaPrefix):]
123+
b, n, err := decodeUint32(b)
124+
if err != nil {
125+
return nil, nil, nil, err
126+
}
127+
if uint32(len(b)) < n {
128+
return nil, nil, nil, errors.Errorf("key does not contain DatabaseID")
129+
}
130+
db = unescapeSlash(b[:n])
131+
b = b[n:]
132+
if len(b) < utils.TimeFormatByteLen {
133+
return
134+
}
135+
baseTime = polypb.TimePoint(b[:utils.TimeFormatByteLen])
136+
b = b[utils.TimeFormatByteLen:]
137+
if len(b) < utils.TimeFormatByteLen {
138+
return
139+
}
140+
backupTime = polypb.TimePoint(b[:utils.TimeFormatByteLen])
141+
return
142+
}
143+
144+
func DecodeKey(
99145
key polypb.Key,
100146
) (db polypb.DatabaseID, baseTime, backupTime polypb.TimePoint, err error) {
101147
sp := bytes.Split(key, []byte("/"))
102-
if len(sp) != 3 {
148+
if len(sp) > 3 {
103149
return nil, nil, nil, errors.Errorf("key (%s) is invalid", key)
104150
}
105-
return unescapeSlash(sp[0]), sp[1], sp[2], nil
151+
db = unescapeSlash(sp[0])
152+
if len(sp) == 1 {
153+
return
154+
}
155+
baseTime = sp[1]
156+
if len(sp) == 2 {
157+
return
158+
}
159+
backupTime = sp[2]
160+
return
106161
}

‎pkg/keys/keys_test.go‎

Lines changed: 67 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@ package keys
33
import (
44
"bytes"
55
"testing"
6+
"time"
67

78
"github.com/taku-k/polymerase/pkg/polypb"
9+
"github.com/taku-k/polymerase/pkg/utils/testutil"
810
)
911

1012
func TestMakeBackupMetaKey(t *testing.T) {
@@ -18,13 +20,13 @@ func TestMakeBackupMetaKey(t *testing.T) {
1820
db: polypb.DatabaseID("db1"),
1921
baseTP: polypb.TimePoint("time1"),
2022
backTP: polypb.TimePoint("time1"),
21-
exp: polypb.BackupMetaKey("meta-backup-db1time1time1"),
23+
exp: polypb.BackupMetaKey("meta-backup-\x00\x00\x00\x03db1time1time1"),
2224
},
2325
{
2426
db: polypb.DatabaseID("db/2"),
2527
baseTP: polypb.TimePoint("time1"),
2628
backTP: polypb.TimePoint("time2"),
27-
exp: polypb.BackupMetaKey(`meta-backup-db/2time1time2`),
29+
exp: polypb.BackupMetaKey("meta-backup-\x00\x00\x00\x04db/2time1time2"),
2830
},
2931
}
3032

@@ -44,11 +46,11 @@ func TestMakeNodeMetaKey(t *testing.T) {
4446
}{
4547
{
4648
node: polypb.NodeID("node1"),
47-
exp: polypb.NodeMetaKey("meta-node-node1"),
49+
exp: polypb.NodeMetaKey("meta-node-\x00\x00\x00\x05node1"),
4850
},
4951
{
5052
node: polypb.NodeID("node/2"),
51-
exp: polypb.NodeMetaKey(`meta-node-node/2`),
53+
exp: polypb.NodeMetaKey("meta-node-\x00\x00\x00\x06node/2"),
5254
},
5355
}
5456

@@ -98,11 +100,11 @@ func TestMakeBackupMetaKeyFromKey(t *testing.T) {
98100
}{
99101
{
100102
key: polypb.Key("db/time/time"),
101-
exp: polypb.BackupMetaKey("meta-backup-dbtimetime"),
103+
exp: polypb.BackupMetaKey("meta-backup-\x00\x00\x00\x02dbtimetime"),
102104
},
103105
{
104-
key: polypb.Key(`db_1/time/time`),
105-
exp: polypb.BackupMetaKey(`meta-backup-db/1timetime`),
106+
key: polypb.Key("db_1/time/time"),
107+
exp: polypb.BackupMetaKey("meta-backup-\x00\x00\x00\x04db/1timetime"),
106108
},
107109
}
108110

@@ -114,3 +116,61 @@ func TestMakeBackupMetaKeyFromKey(t *testing.T) {
114116
}
115117
}
116118
}
119+
120+
func TestDecodeMetaKey(t *testing.T) {
121+
testCases := []struct {
122+
key polypb.BackupMetaKey
123+
db polypb.DatabaseID
124+
baseTime polypb.TimePoint
125+
backupTime polypb.TimePoint
126+
errStr string
127+
}{
128+
{
129+
db: polypb.DatabaseID("db"),
130+
},
131+
{
132+
db: polypb.DatabaseID("db-id"),
133+
baseTime: polypb.NewTimePoint(time.Now()),
134+
},
135+
{
136+
db: polypb.DatabaseID("long-db-id"),
137+
baseTime: polypb.NewTimePoint(time.Now()),
138+
backupTime: polypb.NewTimePoint(time.Now()),
139+
},
140+
{
141+
key: polypb.BackupMetaKey(makeKey(backupMetaPrefix, []byte("\x00\x00\x00\x02d"))), // Len=2, DB=d
142+
errStr: "key does not contain DatabaseID",
143+
},
144+
}
145+
146+
for i, tc := range testCases {
147+
key := tc.key
148+
if key == nil {
149+
key = MakeBackupMetaKey(tc.db, tc.baseTime, tc.backupTime)
150+
}
151+
db, base, backup, err := DecodeMetaKey(key)
152+
if tc.errStr != "" {
153+
if !testutil.IsError(err, tc.errStr) {
154+
t.Errorf("#%d: expected error %q, but found %q",
155+
i, tc.errStr, err)
156+
}
157+
} else {
158+
if err != nil {
159+
t.Errorf("#%d(key=%q): got error %q; want success",
160+
i, key, err)
161+
}
162+
if !bytes.Equal(db, tc.db) {
163+
t.Errorf("#%d(key=%q): got wrong DatabaseID %q; want %q",
164+
i, key, db, tc.db)
165+
}
166+
if !bytes.Equal(base, tc.baseTime) {
167+
t.Errorf("#%d(key=%q): got wrong Base TimePoint %q; want %q",
168+
i, key, base, tc.baseTime)
169+
}
170+
if !bytes.Equal(backup, tc.backupTime) {
171+
t.Errorf("#%d(key=%q): got wrong Backup TimePoint %q; want %q",
172+
i, key, backup, tc.backupTime)
173+
}
174+
}
175+
}
176+
}

‎pkg/polypb/metadata.pb.go‎

Lines changed: 43 additions & 57 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/polypb/metadata.proto‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ message BackupMeta {
3535

3636
// StorageType
3737
enum StorageType {
38-
LOCAL = 0;
38+
LOCAL_DISK = 0;
39+
LOCAL_MEM = 1;
3940
}
4041

4142
// BackupType

0 commit comments

Comments
 (0)