diff --git a/blobstore/access/controller/shard.go b/blobstore/access/controller/shard.go index 7e598433c..6a2edb842 100644 --- a/blobstore/access/controller/shard.go +++ b/blobstore/access/controller/shard.go @@ -48,7 +48,7 @@ var ( ) type IShardController interface { - GetShard(ctx context.Context, shardKeys [][]byte) (Shard, error) + GetShard(ctx context.Context, shardKeys []string) (Shard, error) GetShardByID(ctx context.Context, shardID proto.ShardID) (Shard, error) GetShardByRange(ctx context.Context, shardRange sharding.Range) (Shard, error) GetFisrtShard(ctx context.Context) (Shard, error) @@ -129,7 +129,7 @@ type shardControllerImpl struct { stopCh <-chan struct{} } -func (s *shardControllerImpl) GetShard(ctx context.Context, shardKeys [][]byte) (Shard, error) { +func (s *shardControllerImpl) GetShard(ctx context.Context, shardKeys []string) (Shard, error) { // shard_1 ranges [1, 100) , shard 2: [100, 200), shard 3: [200, 300) ... // if compare shard keys=20, it belong to shard 1 ; if keys=100, it belong to shard 2 ; keys=220, belong to shard 3 // if keys=120, will walk [shard 2, shard end] diff --git a/blobstore/access/controller/shard_test.go b/blobstore/access/controller/shard_test.go index 39d37970a..de6e338fd 100644 --- a/blobstore/access/controller/shard_test.go +++ b/blobstore/access/controller/shard_test.go @@ -56,8 +56,8 @@ func TestShardController(t *testing.T) { require.ErrorIs(t, err, errcode.ErrAccessNotFoundShard) require.Nil(t, s) - blobName := []byte("blob1") - shardKeys := [][]byte{blobName} + blobName := "blob1" + shardKeys := []string{blobName} // empty tree // _, err = s.GetShard(ctx, shardKeys) // require.NotNil(t, err) @@ -126,8 +126,8 @@ func TestShardController(t *testing.T) { svr.version = 8 // ret, err := svr.GetShard(ctx, []byte("blob1__xxx")) // expect 2 keys - ret, err := svr.GetShard(ctx, [][]byte{[]byte("blob1__xxx")}) // expect 2 keys - sk := [][]byte{[]byte("blob1__xxx")} + ret, err := svr.GetShard(ctx, []string{"blob1__xxx"}) // expect 2 keys + sk := []string{"blob1__xxx"} bd := sharding.NewCompareItem(sharding.RangeType_RangeTypeHash, sk).GetBoundary() t.Logf("shard key 1, key boundary=%d, shardBounary=%d, range=%s, treeLen=%d", bd, ret.(*shard).rangeExt.MaxBoundary(), ret.(*shard).String(), svr.ranges.Len()) // for i := range shards { @@ -138,8 +138,8 @@ func TestShardController(t *testing.T) { require.Equal(t, proto.ShardID(2), ret.(*shard).shardID) // ret, err = svr.GetShard(ctx, []byte("{blob2__yy}{11}")) - ret, err = svr.GetShard(ctx, [][]byte{[]byte("blob2__yy"), []byte("11")}) - bd = sharding.NewCompareItem(sharding.RangeType_RangeTypeHash, [][]byte{[]byte("blob2__yy"), []byte("11")}).GetBoundary() + ret, err = svr.GetShard(ctx, []string{"blob2__yy", "11"}) + bd = sharding.NewCompareItem(sharding.RangeType_RangeTypeHash, []string{"blob2__yy", "11"}).GetBoundary() t.Logf("shard key 2, get boundary=%d", bd) require.Nil(t, err) require.Equal(t, proto.ShardID(7), ret.(*shard).shardID) @@ -786,12 +786,12 @@ func TestShardGetShard(t *testing.T) { { // sd, err := svr.GetShard(ctx, []byte("{blob1}{1}")) - sd, err := svr.GetShard(ctx, [][]byte{[]byte("blob1"), []byte("1")}) + sd, err := svr.GetShard(ctx, []string{"blob1", "1"}) require.NoError(t, err) require.Equal(t, proto.ShardID(2), sd.GetShardID()) // sd, err = svr.GetShard(ctx, []byte("{blob1}{}")) - sd, err = svr.GetShard(ctx, [][]byte{[]byte("blob1"), {}}) + sd, err = svr.GetShard(ctx, []string{"blob1", ""}) require.NoError(t, err) require.Equal(t, proto.ShardID(2), sd.GetShardID()) diff --git a/blobstore/access/stream/controller_mock_test.go b/blobstore/access/stream/controller_mock_test.go index cd09bbf3a..11a0bca48 100644 --- a/blobstore/access/stream/controller_mock_test.go +++ b/blobstore/access/stream/controller_mock_test.go @@ -429,7 +429,7 @@ func (mr *MockShardControllerMockRecorder) GetNextShard(arg0, arg1 interface{}) } // GetShard mocks base method. -func (m *MockShardController) GetShard(arg0 context.Context, arg1 [][]byte) (controller.Shard, error) { +func (m *MockShardController) GetShard(arg0 context.Context, arg1 []string) (controller.Shard, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetShard", arg0, arg1) ret0, _ := ret[0].(controller.Shard) diff --git a/blobstore/access/stream/stream_blob.go b/blobstore/access/stream/stream_blob.go index 15cc8e655..c438b3674 100644 --- a/blobstore/access/stream/stream_blob.go +++ b/blobstore/access/stream/stream_blob.go @@ -396,7 +396,7 @@ func (h *Handler) getShardOpHeader(ctx context.Context, args acapi.GetShardCommo return shardnode.ShardOpHeader{}, err } - shardKeys := shardnode.ParseShardKeys(args.BlobName, shardMgr.GetShardSubRangeCount(ctx)) + shardKeys := shardnode.DecodeShardKeys(args.BlobName, shardMgr.GetShardSubRangeCount(ctx)) shard, err := shardMgr.GetShard(ctx, shardKeys) if err != nil { return shardnode.ShardOpHeader{}, err diff --git a/blobstore/api/shardnode/parse.go b/blobstore/api/shardnode/parse.go index 393591030..d855f1a47 100644 --- a/blobstore/api/shardnode/parse.go +++ b/blobstore/api/shardnode/parse.go @@ -16,17 +16,19 @@ package shardnode import ( "bytes" + "fmt" + "strings" "github.com/cubefs/cubefs/blobstore/common/proto" ) -func ParseShardKeys(keyStr string, tagNum int) [][]byte { +func DecodeShardKeys(keyStr string, tagNum int) []string { key := []byte(keyStr) if len(key) == 0 || tagNum < 1 { - return [][]byte{} + return []string{} } - var tags [][]byte + var tags []string start := 0 for i := 0; i < tagNum; i++ { left := bytes.IndexByte(key[start:], proto.ShardingTagLeft) @@ -41,14 +43,14 @@ func ParseShardKeys(keyStr string, tagNum int) [][]byte { } right += left + 1 - tags = append(tags, key[left+1:right]) + tags = append(tags, string(key[left+1:right])) start = right + 1 } // no tags, use key as tags if len(tags) < 1 { for i := 0; i < tagNum; i++ { - tags = append(tags, key) + tags = append(tags, string(key)) } return tags } @@ -56,9 +58,57 @@ func ParseShardKeys(keyStr string, tagNum int) [][]byte { // tags not enough, add empty tags if len(tags) < tagNum { for i := 0; i < tagNum-len(tags); i++ { - tags = append(tags, []byte("")) + tags = append(tags, "") } return tags } return tags } + +// EncodeName encode shardkeys into formatStr, for example: +// formatStr = "s%s%sname1%sname2" keys=["key1", "key2", "key3"] +// encode to "s{key1}{key2}name1{key3}name2", and len(shardkeys) must equal with tagNum +func EncodeName(formatStr string, shardkeys []string, tagNum int) string { + if len(shardkeys) != tagNum { + panic("len(shardkeys) not equal with tagNum") + } + + if formatStr == "" || len(shardkeys) == 0 || tagNum <= 0 { + return formatStr + } + + // Validate that formatStr only contains %s placeholders, and has no '{' or '}' + validateFormatString(formatStr) + containsUnsupportedBytes(formatStr) + + count := strings.Count(formatStr, "%s") + if count != len(shardkeys) { + panic("num of '%s' in formatStr no equal with len(shardkeys)") + } + + encodedKeys := make([]interface{}, len(shardkeys)) + for i := range shardkeys { + encodedKeys[i] = fmt.Sprintf("{%s}", shardkeys[i]) + } + return fmt.Sprintf(formatStr, encodedKeys...) +} + +// validateFormatString ensures that formatStr only contains %s placeholders +func validateFormatString(formatStr string) { + for i := 0; i < len(formatStr); i++ { + if i == len(formatStr)-1 && formatStr[i] == '%' { + panic("unsupported format specifier: found '%' in last character") + } + if formatStr[i] == '%' && formatStr[i+1] != 's' { + panic(fmt.Sprintf("unsupported format specifier: only %%s is allowed, found '%%%c'", formatStr[i+1])) + } + } +} + +func containsUnsupportedBytes(s string) { + raw := []byte(s) + idx0, idx1 := bytes.IndexByte(raw, '{'), bytes.IndexByte(raw, '}') + if idx0 > -1 || idx1 > -1 { + panic("'{' or '}' is not supported") + } +} diff --git a/blobstore/api/shardnode/parse_test.go b/blobstore/api/shardnode/parse_test.go index b4af296c1..31a8944e0 100644 --- a/blobstore/api/shardnode/parse_test.go +++ b/blobstore/api/shardnode/parse_test.go @@ -14,9 +14,14 @@ package shardnode -import "testing" +import ( + "bytes" + "testing" -func TestParseShardKeys(t *testing.T) { + "github.com/stretchr/testify/require" +) + +func TestDecodeShardKeys(t *testing.T) { tests := []struct { name string input string @@ -111,7 +116,7 @@ func TestParseShardKeys(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := ParseShardKeys(tt.input, 2) + got := DecodeShardKeys(tt.input, 2) if len(got) != len(tt.expected) { t.Errorf("%s: expected %d keys, got %d", tt.name, len(tt.expected), len(got)) return @@ -125,3 +130,195 @@ func TestParseShardKeys(t *testing.T) { }) } } + +func TestEncodeName(t *testing.T) { + tests := []struct { + name string + format string + keys []string + tagNum int + expected string + shouldPanic bool + }{ + { + name: "basic encoding with 3 placeholders", + format: "s%s%sname1%sname2", + keys: []string{"key1", "key2", "key3"}, + tagNum: 3, + expected: "s{key1}{key2}name1{key3}name2", + shouldPanic: false, + }, + { + name: "only placeholders", + format: "%s%s%s", + keys: []string{"key1", "key2", "key3"}, + tagNum: 3, + expected: "{key1}{key2}{key3}", + shouldPanic: false, + }, + { + name: "placeholders at start and end", + format: "%sname1%s", + keys: []string{"key1", "key2"}, + tagNum: 2, + expected: "{key1}name1{key2}", + shouldPanic: false, + }, + { + name: "single placeholder", + format: "%sname", + keys: []string{"key1"}, + tagNum: 1, + expected: "{key1}name", + shouldPanic: false, + }, + { + name: "complex path with placeholders", + format: "%s%s%spath%sfile", + keys: []string{"user", "project", "version", "name"}, + tagNum: 4, + expected: "{user}{project}{version}path{name}file", + shouldPanic: false, + }, + { + name: "empty format string", + format: "", + keys: []string{}, + tagNum: 0, + expected: "", + shouldPanic: false, + }, + { + name: "no placeholders", + format: "static/path", + keys: []string{}, + tagNum: 0, + expected: "static/path", + shouldPanic: false, + }, + { + name: "empty keys array", + format: "%sname", + keys: []string{}, + tagNum: 1, + expected: "", + shouldPanic: true, + }, + { + name: "zero tagNum", + format: "%sname", + keys: []string{"key1"}, + tagNum: 0, + expected: "", + shouldPanic: true, + }, + { + name: "mismatch: more keys than placeholders", + format: "%sname", + keys: []string{"key1", "key2"}, + tagNum: 2, + expected: "", + shouldPanic: true, + }, + { + name: "mismatch: fewer keys than placeholders", + format: "%s%sname", + keys: []string{"key1"}, + tagNum: 2, + expected: "", + shouldPanic: true, + }, + { + name: "mismatch: tagNum not equal to placeholders", + format: "%s%sname", + keys: []string{"key1", "key2"}, + tagNum: 3, + expected: "", + shouldPanic: true, + }, + { + name: "special characters in keys", + format: "%s%sname", + keys: []string{"key/1", "key\\2"}, + tagNum: 2, + expected: "{key/1}{key\\2}name", + shouldPanic: false, + }, + { + name: "special characters in keys 2", + format: "%s%daa%sname", + keys: []string{"key1", "key2"}, + tagNum: 2, + expected: "", + shouldPanic: true, + }, + { + name: "special characters in keys 3", + format: "%s%sname%", + keys: []string{"key1", "key2"}, + tagNum: 2, + expected: "", + shouldPanic: true, + }, + { + name: "empty keys", + format: "%s%sname", + keys: []string{"", ""}, + tagNum: 2, + expected: "{}{}name", + shouldPanic: false, + }, + { + name: "too much '%s' in format", + format: "%ssss%s%sss%sname", + keys: []string{"key1", "key2"}, + tagNum: 2, + expected: "", + shouldPanic: true, + }, + { + name: "illegal format string 1", + format: "/%s/%s/name{", + keys: []string{"key1", "key2"}, + tagNum: 2, + expected: "", + shouldPanic: true, + }, + { + name: "illegal format string 2", + format: "/%s/%s/name}", + keys: []string{"key1", "key2"}, + tagNum: 2, + expected: "", + shouldPanic: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defer func() { + if r := recover(); r != nil { + if !tt.shouldPanic { + t.Errorf("Test %s: unexpected panic: %v", tt.name, r) + } + } else if tt.shouldPanic { + t.Errorf("Test %s: expected panic but none occurred", tt.name) + } + }() + + result := EncodeName(tt.format, tt.keys, tt.tagNum) + + if !tt.shouldPanic && result != tt.expected { + t.Errorf("EncodeName(%q, %v, %d) = %q, want %q", + tt.format, tt.keys, tt.tagNum, result, tt.expected) + } + + if !tt.shouldPanic && len(tt.keys) > 0 { + shardkeys := DecodeShardKeys(result, tt.tagNum) + for i := range shardkeys { + require.True(t, bytes.Equal([]byte(shardkeys[i]), []byte(tt.keys[i]))) + } + } + }) + } +} diff --git a/blobstore/common/sharding/hashrange.go b/blobstore/common/sharding/hashrange.go index 69ccfb6d9..942ddbb21 100644 --- a/blobstore/common/sharding/hashrange.go +++ b/blobstore/common/sharding/hashrange.go @@ -32,7 +32,7 @@ func (h *hashRange) Belong(ci *CompareItem) bool { if ci.context == nil { hashValues := make([]uint64, len(ci.keys)) for i := range ci.keys { - hashValues[i] = Hash(ci.keys[i]) + hashValues[i] = Hash([]byte(ci.keys[i])) } ci.context = hashValues } diff --git a/blobstore/common/sharding/proto.go b/blobstore/common/sharding/proto.go index 28ff9fa11..b51354cbe 100644 --- a/blobstore/common/sharding/proto.go +++ b/blobstore/common/sharding/proto.go @@ -65,13 +65,13 @@ func InitShardingRange(RangeType RangeType, subRangeCount, shardCount int) []*Ra // NewCompareItem return compare item for range belong compare // key can be integer type or bytes for different sharding range type -func NewCompareItem(rt RangeType, keys [][]byte) *CompareItem { +func NewCompareItem(rt RangeType, keys []string) *CompareItem { return &CompareItem{rt: rt, keys: keys} } type CompareItem struct { rt RangeType - keys [][]byte + keys []string context interface{} } @@ -81,7 +81,7 @@ func (c *CompareItem) GetBoundary() Boundary { if c.context == nil { values := make([]uint64, len(c.keys)) for i := range c.keys { - values[i] = Hash(c.keys[i]) + values[i] = Hash([]byte(c.keys[i])) } c.context = values } diff --git a/blobstore/common/sharding/proto_test.go b/blobstore/common/sharding/proto_test.go index d722bbd71..00a19768e 100644 --- a/blobstore/common/sharding/proto_test.go +++ b/blobstore/common/sharding/proto_test.go @@ -33,13 +33,13 @@ func TestInitRange(t *testing.T) { } func TestCompareItem(t *testing.T) { - inputs2 := [][][][]byte{ - {{[]byte{1}}, {[]byte{1}}}, - {{[]byte{1}}, {[]byte{2}}}, - {{[]byte{2}}, {[]byte{1}}}, - {{[]byte{1}, []byte{1}}, {[]byte{1}, []byte{1}}}, - {{[]byte{1}, []byte{1}}, {[]byte{1}, []byte{2}}}, - {{[]byte{1}, []byte{2}}, {[]byte{1}, []byte{1}}}, + inputs2 := [][][]string{ + {{"1"}, {"1"}}, + {{"1"}, {"2"}}, + {{"2"}, {"1"}}, + {{"1", "1"}, {"1", "1"}}, + {{"1", "1"}, {"1", "2"}}, + {{"1", "2"}, {"1", "1"}}, } outputs2 := []bool{ false, diff --git a/blobstore/common/sharding/rangeext_test.go b/blobstore/common/sharding/rangeext_test.go index 208cadbee..bba9e13d7 100644 --- a/blobstore/common/sharding/rangeext_test.go +++ b/blobstore/common/sharding/rangeext_test.go @@ -21,9 +21,9 @@ import ( ) func TestRange(t *testing.T) { - subRangeCountMap := map[int][][]byte{ - 1: {[]byte{1}}, - 2: {[]byte{1}, []byte{2}}, + subRangeCountMap := map[int][]string{ + 1: {"1"}, + 2: {"1", "2"}, } rt := RangeType_RangeTypeHash diff --git a/blobstore/shardnode/blob.go b/blobstore/shardnode/blob.go index fc6a8e93d..241242d7f 100644 --- a/blobstore/shardnode/blob.go +++ b/blobstore/shardnode/blob.go @@ -56,7 +56,7 @@ func (s *service) deleteBlob(ctx context.Context, req *shardnode.DeleteBlobArgs) if err != nil { return } - shardKeys := shardnode.ParseShardKeys(req.Name, tagNum) + shardKeys := shardnode.DecodeShardKeys(req.Name, tagNum) items, err := s.blobDelMgr.SlicesToDeleteMsgItems(ctx, blob.Blob.Location.Slices, shardKeys) if err != nil { diff --git a/blobstore/shardnode/blobdeleter/blob.go b/blobstore/shardnode/blobdeleter/blob.go index 2bd3e9c9b..af0d4fa76 100644 --- a/blobstore/shardnode/blobdeleter/blob.go +++ b/blobstore/shardnode/blobdeleter/blob.go @@ -21,7 +21,7 @@ import ( "github.com/cubefs/cubefs/blobstore/common/proto" ) -func (m *BlobDeleteMgr) SlicesToDeleteMsgItems(ctx context.Context, slices []proto.Slice, shardKeys [][]byte) ([]snapi.Item, error) { +func (m *BlobDeleteMgr) SlicesToDeleteMsgItems(ctx context.Context, slices []proto.Slice, shardKeys []string) ([]snapi.Item, error) { return m.slicesToDeleteMsgItems(ctx, slices, shardKeys) } diff --git a/blobstore/shardnode/blobdeleter/blob_deleter.go b/blobstore/shardnode/blobdeleter/blob_deleter.go index aeb0a3060..7bb0ef0f8 100644 --- a/blobstore/shardnode/blobdeleter/blob_deleter.go +++ b/blobstore/shardnode/blobdeleter/blob_deleter.go @@ -649,7 +649,7 @@ func (m *BlobDeleteMgr) insertDeleteMsg(ctx context.Context, req *snapi.DeleteBl return shard.InsertItem(ctx, oph, []byte(itm.ID), itm) } -func (m *BlobDeleteMgr) slicesToDeleteMsgItems(ctx context.Context, slices []proto.Slice, shardKeys [][]byte) ([]snapi.Item, error) { +func (m *BlobDeleteMgr) slicesToDeleteMsgItems(ctx context.Context, slices []proto.Slice, shardKeys []string) ([]snapi.Item, error) { span := trace.SpanFromContextSafe(ctx) items := make([]snapi.Item, len(slices)) for i := range slices { @@ -671,7 +671,7 @@ func (m *BlobDeleteMgr) slicesToDeleteMsgItems(ctx context.Context, slices []pro return items, nil } -func (m *BlobDeleteMgr) sliceToDeleteMsgItemRaw(ctx context.Context, slices proto.Slice, tagNum int) (snapi.Item, [][]byte, error) { +func (m *BlobDeleteMgr) sliceToDeleteMsgItemRaw(ctx context.Context, slices proto.Slice, tagNum int) (snapi.Item, []string, error) { span := trace.SpanFromContextSafe(ctx) ts := m.tsGen.GenerateTs() key, shardKeys := encodeRawDelMsgKey(ts, slices.Vid, slices.MinSliceID, tagNum) diff --git a/blobstore/shardnode/blobdeleter/blob_deleter_test.go b/blobstore/shardnode/blobdeleter/blob_deleter_test.go index a7d72f5e2..242ad96e9 100644 --- a/blobstore/shardnode/blobdeleter/blob_deleter_test.go +++ b/blobstore/shardnode/blobdeleter/blob_deleter_test.go @@ -792,7 +792,7 @@ func TestBlobDeleteMgr_InsertDeleteMsg(t *testing.T) { } func TestBlobDeleteMgr_SlicesToDeleteMsgItems(t *testing.T) { - shardKeys := [][]byte{[]byte("abc"), []byte("def")} + shardKeys := []string{"abc", "def"} mgr := newTestBlobDeleteMgr(t, nil, nil, nil) diff --git a/blobstore/shardnode/blobdeleter/msg.go b/blobstore/shardnode/blobdeleter/msg.go index e9588147b..daab2ac06 100644 --- a/blobstore/shardnode/blobdeleter/msg.go +++ b/blobstore/shardnode/blobdeleter/msg.go @@ -29,7 +29,7 @@ const minMsgKeyLen = 21 // 1(prefix) + 8(ts) + 4(vid) + 8(bid) var errInvalidMsgKey = errors.New("invalid msg key") -func encodeDelMsgKey(ts base.Ts, vid proto.Vid, bid proto.BlobID, shardKeys [][]byte) []byte { +func encodeDelMsgKey(ts base.Ts, vid proto.Vid, bid proto.BlobID, shardKeys []string) []byte { shardKeyLen := 0 for _, sk := range shardKeys { shardKeyLen += len(sk) @@ -56,15 +56,15 @@ func encodeDelMsgKey(ts base.Ts, vid proto.Vid, bid proto.BlobID, shardKeys [][] return buf } -func encodeRawDelMsgKey(ts base.Ts, vid proto.Vid, bid proto.BlobID, tagNum int) (key []byte, shardKeys [][]byte) { - key1 := []byte(fmt.Sprintf("%d", uint32(vid))) - key2 := []byte(fmt.Sprintf("%d", uint64(bid))) +func encodeRawDelMsgKey(ts base.Ts, vid proto.Vid, bid proto.BlobID, tagNum int) (key []byte, shardKeys []string) { + key1 := fmt.Sprintf("%d", uint32(vid)) + key2 := fmt.Sprintf("%d", uint64(bid)) // raw msg shardKeys = {"vid"}{"bid"}{...} - shardKeys = [][]byte{key1, key2} + shardKeys = []string{key1, key2} if tagNum > 2 { for i := 0; i < tagNum-2; i++ { - shardKeys = append(shardKeys, []byte("")) + shardKeys = append(shardKeys, "") } } @@ -72,12 +72,12 @@ func encodeRawDelMsgKey(ts base.Ts, vid proto.Vid, bid proto.BlobID, tagNum int) return encodeDelMsgKey(ts, vid, bid, shardKeys), shardKeys } -func decodeDelMsgKey(key []byte, tagNum int) (base.Ts, proto.Vid, proto.BlobID, [][]byte, error) { +func decodeDelMsgKey(key []byte, tagNum int) (base.Ts, proto.Vid, proto.BlobID, []string, error) { if len(key) < minMsgKeyLen+2*tagNum { return base.Ts(0), proto.InvalidVid, proto.InValidBlobID, nil, errInvalidMsgKey } ts := base.Ts(binary.BigEndian.Uint64(key[1:9])) vid := proto.Vid(binary.BigEndian.Uint32(key[9:13])) bid := proto.BlobID(binary.BigEndian.Uint64(key[13:21])) - return ts, vid, bid, snapi.ParseShardKeys(string(key[minMsgKeyLen:]), tagNum), nil + return ts, vid, bid, snapi.DecodeShardKeys(string(key[minMsgKeyLen:]), tagNum), nil } diff --git a/blobstore/shardnode/blobdeleter/msg_test.go b/blobstore/shardnode/blobdeleter/msg_test.go index bf2c32e6f..9f10af3a6 100644 --- a/blobstore/shardnode/blobdeleter/msg_test.go +++ b/blobstore/shardnode/blobdeleter/msg_test.go @@ -37,7 +37,7 @@ func TestEncodeDecodeDelMsgKey(t *testing.T) { ts := g.GenerateTs() vid := proto.Vid(123) bid := proto.BlobID(456) - shardKeys := [][]byte{[]byte("part1"), []byte("part2")} + shardKeys := []string{"part1", "part2"} key := encodeDelMsgKey(ts, vid, bid, shardKeys) require.True(t, ts > 0) @@ -80,10 +80,10 @@ func TestEncodeDecodeDelMsgKey(t *testing.T) { t.Run("CompositeOrdering", func(t *testing.T) { g := base.NewTsGenerator(0) ts1 := g.GenerateTs() - key1 := encodeDelMsgKey(ts1, 200, 50, [][]byte{[]byte("part1"), []byte("z")}) + key1 := encodeDelMsgKey(ts1, 200, 50, []string{"part1", "z"}) time.Sleep(time.Millisecond) ts2 := g.GenerateTs() - key2 := encodeDelMsgKey(ts2, 100, 40, [][]byte{[]byte("part1"), []byte("a")}) + key2 := encodeDelMsgKey(ts2, 100, 40, []string{"part1", "a"}) require.True(t, ts1 < ts2) require.Equal(t, -1, bytes.Compare([]byte(key1), []byte(key2))) diff --git a/blobstore/shardnode/catalog/space.go b/blobstore/shardnode/catalog/space.go index bb75b93de..8a5d48429 100644 --- a/blobstore/shardnode/catalog/space.go +++ b/blobstore/shardnode/catalog/space.go @@ -104,7 +104,7 @@ func (s *Space) InsertItem(ctx context.Context, h shardnode.ShardOpHeader, i sha return shard.InsertItem(ctx, storage.OpHeader{ RouteVersion: h.RouteVersion, - ShardKeys: shardnode.ParseShardKeys(i.ID, shard.ShardingSubRangeCount()), + ShardKeys: shardnode.DecodeShardKeys(i.ID, shard.ShardingSubRangeCount()), }, s.generateSpaceKey([]byte(i.ID)), i) } @@ -119,7 +119,7 @@ func (s *Space) UpdateItem(ctx context.Context, h shardnode.ShardOpHeader, i sha return shard.UpdateItem(ctx, storage.OpHeader{ RouteVersion: h.RouteVersion, - ShardKeys: shardnode.ParseShardKeys(i.ID, shard.ShardingSubRangeCount()), + ShardKeys: shardnode.DecodeShardKeys(i.ID, shard.ShardingSubRangeCount()), }, s.generateSpaceKey([]byte(i.ID)), i) } @@ -131,7 +131,7 @@ func (s *Space) DeleteItem(ctx context.Context, h shardnode.ShardOpHeader, id st return shard.DeleteItem(ctx, storage.OpHeader{ RouteVersion: h.RouteVersion, - ShardKeys: shardnode.ParseShardKeys(id, shard.ShardingSubRangeCount()), + ShardKeys: shardnode.DecodeShardKeys(id, shard.ShardingSubRangeCount()), }, s.generateSpaceKey([]byte(id))) } @@ -143,7 +143,7 @@ func (s *Space) GetItem(ctx context.Context, h shardnode.ShardOpHeader, id strin return shard.GetItem(ctx, storage.OpHeader{ RouteVersion: h.RouteVersion, - ShardKeys: shardnode.ParseShardKeys(id, shard.ShardingSubRangeCount()), + ShardKeys: shardnode.DecodeShardKeys(id, shard.ShardingSubRangeCount()), }, s.generateSpaceKey([]byte(id))) } @@ -227,7 +227,7 @@ INSERT: start = time.Now() cb, _err := sd.CreateBlob(ctx, storage.OpHeader{ RouteVersion: h.RouteVersion, - ShardKeys: shardnode.ParseShardKeys(req.Name, sd.ShardingSubRangeCount()), + ShardKeys: shardnode.DecodeShardKeys(req.Name, sd.ShardingSubRangeCount()), }, s.generateSpaceKey([]byte(req.Name)), b) span.AppendTrackLog(opInsert, start, _err, trace.OptSpanDurationUs()) if _err != nil { @@ -273,7 +273,7 @@ func (s *Space) DeleteBlob(ctx context.Context, req *shardnode.DeleteBlobArgs, i start := time.Now() err = sd.DeleteBlob(ctx, storage.OpHeader{ RouteVersion: h.RouteVersion, - ShardKeys: shardnode.ParseShardKeys(req.Name, sd.ShardingSubRangeCount()), + ShardKeys: shardnode.DecodeShardKeys(req.Name, sd.ShardingSubRangeCount()), }, s.generateSpaceKey([]byte(req.Name)), items) span.AppendTrackLog(opDelete, start, err, trace.OptSpanDurationUs()) return err @@ -368,7 +368,7 @@ func (s *Space) SealBlob(ctx context.Context, req *shardnode.SealBlobArgs) (err start := time.Now() err = sd.UpdateBlob(ctx, storage.OpHeader{ RouteVersion: h.RouteVersion, - ShardKeys: shardnode.ParseShardKeys(req.Name, sd.ShardingSubRangeCount()), + ShardKeys: shardnode.DecodeShardKeys(req.Name, sd.ShardingSubRangeCount()), }, s.generateSpaceKey([]byte(req.Name)), b) span.AppendTrackLog(opUpdate, start, err, trace.OptSpanDurationUs()) if err != nil { @@ -492,7 +492,7 @@ func (s *Space) AllocSlice(ctx context.Context, req *shardnode.AllocSliceArgs) ( start = time.Now() err = sd.UpdateBlob(ctx, storage.OpHeader{ RouteVersion: h.RouteVersion, - ShardKeys: shardnode.ParseShardKeys(req.Name, sd.ShardingSubRangeCount()), + ShardKeys: shardnode.DecodeShardKeys(req.Name, sd.ShardingSubRangeCount()), }, s.generateSpaceKey([]byte(req.Name)), b) span.AppendTrackLog(opUpdate, start, err, trace.OptSpanDurationUs()) if err != nil { @@ -518,7 +518,7 @@ func (s *Space) getBlob(ctx context.Context, sd storage.ShardHandler, h shardnod start := time.Now() b, err = sd.GetBlob(ctx, storage.OpHeader{ RouteVersion: h.RouteVersion, - ShardKeys: shardnode.ParseShardKeys(name, sd.ShardingSubRangeCount()), + ShardKeys: shardnode.DecodeShardKeys(name, sd.ShardingSubRangeCount()), }, key) withErr := err diff --git a/blobstore/shardnode/storage/disk_test.go b/blobstore/shardnode/storage/disk_test.go index 258348d3b..9fc24fc81 100644 --- a/blobstore/shardnode/storage/disk_test.go +++ b/blobstore/shardnode/storage/disk_test.go @@ -388,7 +388,7 @@ func TestServerDisk_RaftData(t *testing.T) { blobName := "test_blob" h := OpHeader{ RouteVersion: version, - ShardKeys: [][]byte{[]byte(blobName)}, + ShardKeys: []string{blobName}, } b := proto.Blob{ Name: blobName, diff --git a/blobstore/shardnode/storage/shard.go b/blobstore/shardnode/storage/shard.go index 4e6fd65be..62af91f26 100644 --- a/blobstore/shardnode/storage/shard.go +++ b/blobstore/shardnode/storage/shard.go @@ -119,7 +119,7 @@ type ( } OpHeader struct { RouteVersion proto.RouteVersion - ShardKeys [][]byte + ShardKeys []string } ShardBaseConfig struct { diff --git a/blobstore/shardnode/storage/shard_sm_test.go b/blobstore/shardnode/storage/shard_sm_test.go index 0c88a2b55..b88f9dd94 100644 --- a/blobstore/shardnode/storage/shard_sm_test.go +++ b/blobstore/shardnode/storage/shard_sm_test.go @@ -79,13 +79,13 @@ func TestServerShardSM_Item(t *testing.T) { err = mockShard.shardSM.applyDeleteRaw(ctx, sk.encodeItemKey(newID)) require.Nil(t, err) _, err = mockShard.shard.GetItem(ctx, OpHeader{ - ShardKeys: [][]byte{newID}, + ShardKeys: []string{string(newID)}, }, newID) require.ErrorIs(t, err, errors.ErrKeyNotFound) err = mockShard.shardSM.applyDeleteRaw(ctx, sk.encodeItemKey(newID)) require.Nil(t, err) _, err = mockShard.shard.GetItem(ctx, OpHeader{ - ShardKeys: [][]byte{newID}, + ShardKeys: []string{string(newID)}, }, newID) require.ErrorIs(t, err, errors.ErrKeyNotFound) @@ -110,7 +110,7 @@ func TestServerShardSM_Item(t *testing.T) { ids[i] = []byte(protoItem.ID) } rets, marker, err := mockShard.shard.ListItem(ctx, OpHeader{ - ShardKeys: [][]byte{[]byte(items[0].ID)}, + ShardKeys: []string{items[0].ID}, }, nil, []byte(items[0].ID), uint64(n-1)) require.Nil(t, err) require.Equal(t, items[n-1].ID, string(marker)) @@ -120,7 +120,7 @@ func TestServerShardSM_Item(t *testing.T) { } _, marker, err = mockShard.shard.ListItem(ctx, OpHeader{ - ShardKeys: [][]byte{[]byte(items[0].ID)}, + ShardKeys: []string{items[0].ID}, }, nil, []byte(items[0].ID), uint64(n)) require.Nil(t, err) require.Nil(t, marker) @@ -137,7 +137,7 @@ func TestServerShardSM_Item(t *testing.T) { for i := 0; i < n-1; i++ { _, err := mockShard.shard.GetItem(ctx, OpHeader{ - ShardKeys: [][]byte{[]byte(items[0].ID)}, + ShardKeys: []string{items[0].ID}, }, ids[i]) require.Equal(t, errors.ErrKeyNotFound, err) } @@ -380,7 +380,7 @@ func TestShardInfo(t *testing.T) { func checkItemEqual(t *testing.T, shard *mockShard, id []byte, item *proto.Item) { ret, err := shard.shard.GetItem(ctx, OpHeader{ - ShardKeys: [][]byte{id}, + ShardKeys: []string{string(id)}, }, id) if err != nil { require.ErrorIs(t, err, kvstore.ErrNotFound) diff --git a/blobstore/shardnode/storage/shard_test.go b/blobstore/shardnode/storage/shard_test.go index 763045eb9..66dfe5405 100644 --- a/blobstore/shardnode/storage/shard_test.go +++ b/blobstore/shardnode/storage/shard_test.go @@ -207,8 +207,8 @@ func TestServerShard_Item(t *testing.T) { }, } - oldShardOpHeader := OpHeader{ShardKeys: [][]byte{[]byte(oldProtoItem.ID)}} - newShardOpHeader := OpHeader{ShardKeys: [][]byte{[]byte(newProtoItem.ID)}} + oldShardOpHeader := OpHeader{ShardKeys: []string{oldProtoItem.ID}} + newShardOpHeader := OpHeader{ShardKeys: []string{newProtoItem.ID}} oldID := []byte(oldProtoItem.ID) newID := []byte(newProtoItem.ID) diff --git a/blobstore/testing/blobnode/base_qos.go b/blobstore/testing/blobnode/base_qos.go index 901fe082f..199576627 100644 --- a/blobstore/testing/blobnode/base_qos.go +++ b/blobstore/testing/blobnode/base_qos.go @@ -9,8 +9,6 @@ import ( io "io" reflect "reflect" - blobnode "github.com/cubefs/cubefs/blobstore/api/blobnode" - qos "github.com/cubefs/cubefs/blobstore/blobnode/base/qos" gomock "github.com/golang/mock/gomock" ) @@ -117,18 +115,6 @@ func (mr *MockQosAPIMockRecorder) ReleaseBid(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReleaseBid", reflect.TypeOf((*MockQosAPI)(nil).ReleaseBid), arg0) } -// ResetQosLimit mocks base method. -func (m *MockQosAPI) ResetQosLimit(arg0 blobnode.IOType, arg1 qos.Config) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "ResetQosLimit", arg0, arg1) -} - -// ResetQosLimit indicates an expected call of ResetQosLimit. -func (mr *MockQosAPIMockRecorder) ResetQosLimit(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResetQosLimit", reflect.TypeOf((*MockQosAPI)(nil).ResetQosLimit), arg0, arg1) -} - // Writer mocks base method. func (m *MockQosAPI) Writer(arg0 context.Context, arg1 io.Writer) io.Writer { m.ctrl.T.Helper()