diff --git a/blobstore/access/server.go b/blobstore/access/server.go index abc8f6d2e..2992de4da 100644 --- a/blobstore/access/server.go +++ b/blobstore/access/server.go @@ -391,9 +391,9 @@ func (s *Service) Get(c *rpc.Context) { w.Header().Set(rpc.HeaderContentType, rpc.MIMEStream) w.Header().Set(rpc.HeaderContentLength, strconv.FormatInt(int64(args.ReadSize), 10)) - if args.ReadSize > 0 && args.ReadSize != args.Location.Size { + if args.ReadSize > 0 && args.ReadSize != args.Location.Size_ { w.Header().Set(rpc.HeaderContentRange, fmt.Sprintf("bytes %d-%d/%d", - args.Offset, args.Offset+args.ReadSize-1, args.Location.Size)) + args.Offset, args.Offset+args.ReadSize-1, args.Location.Size_)) c.RespondStatus(http.StatusPartialContent) } else { c.RespondStatus(http.StatusOK) @@ -456,14 +456,14 @@ func (s *Service) Delete(c *rpc.Context) { err = errcode.ErrIllegalArguments return } - clusterBlobsN[loc.ClusterID] += len(loc.Blobs) + clusterBlobsN[loc.ClusterID] += len(loc.Slices) } if len(args.Locations) == 1 { loc := args.Locations[0] if err := s.streamHandler.Delete(ctx, &loc); err != nil { span.Error("stream delete failed", errors.Detail(err)) - resp.FailedLocations = []access.Location{loc} + resp.FailedLocations = []proto.Location{loc} } return } @@ -475,12 +475,12 @@ func (s *Service) Delete(c *rpc.Context) { // max delete locations is 1024, one location is max to 5G, // merged message max size about 40MB. - merged := make(map[proto.ClusterID][]access.SliceInfo, len(clusterBlobsN)) + merged := make(map[proto.ClusterID][]proto.Slice, len(clusterBlobsN)) for id, n := range clusterBlobsN { - merged[id] = make([]access.SliceInfo, 0, n) + merged[id] = make([]proto.Slice, 0, n) } for _, loc := range args.Locations { - merged[loc.ClusterID] = append(merged[loc.ClusterID], loc.Blobs...) + merged[loc.ClusterID] = append(merged[loc.ClusterID], loc.Slices...) } var wg sync.WaitGroup @@ -489,7 +489,7 @@ func (s *Service) Delete(c *rpc.Context) { go func() { for id := range failedCh { if resp.FailedLocations == nil { - resp.FailedLocations = make([]access.Location, 0, len(args.Locations)) + resp.FailedLocations = make([]proto.Location, 0, len(args.Locations)) } for _, loc := range args.Locations { if loc.ClusterID == id { @@ -503,10 +503,10 @@ func (s *Service) Delete(c *rpc.Context) { wg.Add(len(merged)) for id := range merged { go func(id proto.ClusterID) { - if err := s.streamHandler.Delete(ctx, &access.Location{ + if err := s.streamHandler.Delete(ctx, &proto.Location{ ClusterID: id, - BlobSize: 1, - Blobs: merged[id], + SliceSize: 1, + Slices: merged[id], }); err != nil { span.Error("stream delete failed", id, errors.Detail(err)) failedCh <- id @@ -551,13 +551,13 @@ func (s *Service) DeleteBlob(c *rpc.Context) { return } - if err := s.streamHandler.Delete(ctx, &access.Location{ + if err := s.streamHandler.Delete(ctx, &proto.Location{ ClusterID: args.ClusterID, - BlobSize: 1, - Blobs: []access.SliceInfo{{ - MinBid: args.BlobID, - Vid: args.Vid, - Count: 1, + SliceSize: 1, + Slices: []proto.Slice{{ + MinSliceID: args.BlobID, + Vid: args.Vid, + Count: 1, }}, }); err != nil { span.Error("stream delete blob failed", errors.Detail(err)) diff --git a/blobstore/access/server_test.go b/blobstore/access/server_test.go index a6e9d8ec8..74cdfb1f1 100644 --- a/blobstore/access/server_test.go +++ b/blobstore/access/server_test.go @@ -44,15 +44,15 @@ import ( var ( ctx = context.Background() _blobSize uint32 = 1 << 20 - location = &access.Location{ + location = &proto.Location{ ClusterID: 1, CodeMode: 1, - BlobSize: _blobSize, + SliceSize: _blobSize, Crc: 0, - Blobs: []access.SliceInfo{{ - MinBid: 111, - Vid: 1111, - Count: 11, + Slices: []proto.Slice{{ + MinSliceID: 111, + Vid: 1111, + Count: 11, }}, } @@ -74,12 +74,12 @@ func newService() *Service { s.EXPECT().Alloc(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( func(ctx context.Context, size uint64, blobSize uint32, assignClusterID proto.ClusterID, codeMode codemode.CodeMode, - ) (*access.Location, error) { + ) (*proto.Location, error) { if size < 1024 { return nil, errors.New("fake alloc location") } loc := location.Copy() - loc.Size = uint64(size) + loc.Size_ = uint64(size) stream.LocationCrcFill(&loc) return &loc, nil }) @@ -96,28 +96,28 @@ func newService() *Service { }) s.EXPECT().Put(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( - func(ctx context.Context, rc io.Reader, size int64, hasherMap access.HasherMap) (*access.Location, error) { + func(ctx context.Context, rc io.Reader, size int64, hasherMap access.HasherMap) (*proto.Location, error) { if size < 1024 { return nil, errors.New("fake put nil body") } loc := location.Copy() - loc.Size = uint64(size) + loc.Size_ = uint64(size) stream.LocationCrcFill(&loc) return &loc, nil }) s.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( - func(ctx context.Context, w io.Writer, location access.Location, readSize, offset uint64) (func() error, error) { + func(ctx context.Context, w io.Writer, location proto.Location, readSize, offset uint64) (func() error, error) { if readSize < 1024 { return nil, errors.New("fake get nil body") } return func() error { return nil }, nil }) s.EXPECT().Delete(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( - func(ctx context.Context, location *access.Location) error { + func(ctx context.Context, location *proto.Location) error { if location.ClusterID >= 10 { return errors.New("fake delete error with cluster") - } else if location.ClusterID == 1 && location.Crc > 0 && location.Size < 1024 { + } else if location.ClusterID == 1 && location.Crc > 0 && location.Size_ < 1024 { return errors.New("fake delete error") } return nil @@ -172,21 +172,21 @@ func TestAccessServiceAlloc(t *testing.T) { resp := &access.AllocResp{} err := cli.PostWith(ctx, url(), resp, args) require.NoError(t, err) - require.Equal(t, uint64(1024), resp.Location.Size) + require.Equal(t, uint64(1024), resp.Location.Size_) } { args.Size = uint64(_blobSize) resp := &access.AllocResp{} err := cli.PostWith(ctx, url(), resp, args) require.NoError(t, err) - require.Equal(t, uint64(_blobSize), resp.Location.Size) + require.Equal(t, uint64(_blobSize), resp.Location.Size_) } { args.Size = uint64(_blobSize) + 1 resp := &access.AllocResp{} err := cli.PostWith(ctx, url(), resp, args) require.NoError(t, err) - require.Equal(t, uint64(_blobSize)+1, resp.Location.Size) + require.Equal(t, uint64(_blobSize)+1, resp.Location.Size_) } } @@ -280,7 +280,7 @@ func TestAccessServicePut(t *testing.T) { resp := &access.PutResp{} err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode()) require.NoError(t, err) - require.Equal(t, uint64(1024), resp.Location.Size) + require.Equal(t, uint64(1024), resp.Location.Size_) } } } @@ -305,7 +305,7 @@ func TestAccessServiceGet(t *testing.T) { require.Equal(t, 400, resp.StatusCode, resp.Status) } { - args.Location.Size = 1023 + args.Location.Size_ = 1023 args.ReadSize = 1023 stream.LocationCrcFill(&args.Location) resp, err := cli.Post(ctx, url(), args) @@ -314,7 +314,7 @@ func TestAccessServiceGet(t *testing.T) { require.Equal(t, 500, resp.StatusCode, resp.Status) } { - args.Location.Size = 1024 + args.Location.Size_ = 1024 args.ReadSize = 1024 stream.LocationCrcFill(&args.Location) resp, err := cli.Post(ctx, url(), args) @@ -323,7 +323,7 @@ func TestAccessServiceGet(t *testing.T) { require.Equal(t, 200, resp.StatusCode, resp.Status) } { - args.Location.Size = 10240 + args.Location.Size_ = 10240 args.Offset = 1000 args.ReadSize = 1024 stream.LocationCrcFill(&args.Location) @@ -366,7 +366,7 @@ func TestAccessServiceDelete(t *testing.T) { } args := access.DeleteArgs{ - Locations: []access.Location{location.Copy()}, + Locations: []proto.Location{location.Copy()}, } { code, _, err := deleteRequest(access.DeleteArgs{}) @@ -387,7 +387,7 @@ func TestAccessServiceDelete(t *testing.T) { } { loc := &args.Locations[0] - loc.Size = 1024 + loc.Size_ = 1024 stream.LocationCrcFill(loc) code, _, err := deleteRequest(args) require.NoError(t, err) @@ -395,9 +395,9 @@ func TestAccessServiceDelete(t *testing.T) { } { loc := location.Copy() - loc.Size = 1024 + loc.Size_ = 1024 stream.LocationCrcFill(&loc) - locs := make([]access.Location, access.MaxDeleteLocations) + locs := make([]proto.Location, access.MaxDeleteLocations) for idx := range locs { locs[idx] = loc } @@ -408,9 +408,9 @@ func TestAccessServiceDelete(t *testing.T) { } { loc := location.Copy() - loc.Size = 1024 + loc.Size_ = 1024 stream.LocationCrcFill(&loc) - locs := make([]access.Location, access.MaxDeleteLocations+1) + locs := make([]proto.Location, access.MaxDeleteLocations+1) for idx := range locs { locs[idx] = loc } @@ -420,20 +420,20 @@ func TestAccessServiceDelete(t *testing.T) { } { loc := location.Copy() - loc.Size = 1024 + loc.Size_ = 1024 loc.ClusterID = proto.ClusterID(11) stream.LocationCrcFill(&loc) - code, resp, err := deleteRequest(access.DeleteArgs{Locations: []access.Location{loc}}) + code, resp, err := deleteRequest(access.DeleteArgs{Locations: []proto.Location{loc}}) require.NoError(t, err) require.Equal(t, 226, code) require.Equal(t, 1, len(resp.FailedLocations)) require.Equal(t, proto.ClusterID(11), resp.FailedLocations[0].ClusterID) } { - locs := make([]access.Location, access.MaxDeleteLocations) + locs := make([]proto.Location, access.MaxDeleteLocations) for idx := range locs { loc := location.Copy() - loc.Size = 1024 + loc.Size_ = 1024 loc.ClusterID = proto.ClusterID(idx % 11) stream.LocationCrcFill(&loc) locs[idx] = loc @@ -506,7 +506,7 @@ func TestAccessServiceSign(t *testing.T) { return fmt.Sprintf("%s/sign", host) } args := access.SignArgs{ - Locations: []access.Location{location.Copy()}, + Locations: []proto.Location{location.Copy()}, Location: location.Copy(), } { @@ -535,149 +535,149 @@ func assertErrorCode(t *testing.T, code int, err error) { func TestAccessServiceTokens(t *testing.T) { skey := stream.TokenSecretKeys()[0][:] - checker := func(loc *access.Location, tokens []string) { - if loc.Size == 0 { + checker := func(loc *proto.Location, tokens []string) { + if loc.Size_ == 0 { require.Equal(t, 0, len(tokens)) return } - hasMultiBlobs := loc.Size >= uint64(loc.BlobSize) - lastSize := uint32(loc.Size % uint64(loc.BlobSize)) + hasMultiBlobs := loc.Size_ >= uint64(loc.SliceSize) + lastSize := uint32(loc.Size_ % uint64(loc.SliceSize)) if !hasMultiBlobs { require.Equal(t, 1, len(tokens)) token := uptoken.DecodeToken(tokens[0]) - blob := loc.Blobs[0] - for bid := blob.MinBid - 100; bid < blob.MinBid+100; bid++ { - require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey)) + blob := loc.Slices[0] + for bid := blob.MinSliceID - 100; bid < blob.MinSliceID+100; bid++ { + require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey)) } - require.True(t, token.IsValid(loc.ClusterID, blob.Vid, blob.MinBid, lastSize, skey)) + require.True(t, token.IsValid(loc.ClusterID, blob.Vid, blob.MinSliceID, lastSize, skey)) return } if lastSize == 0 { - require.Equal(t, len(loc.Blobs), len(tokens)) - for idx, blob := range loc.Blobs { + require.Equal(t, len(loc.Slices), len(tokens)) + for idx, blob := range loc.Slices { token := uptoken.DecodeToken(tokens[idx]) for ii := uint32(0); ii < 100; ii++ { - bid := blob.MinBid - proto.BlobID(ii) - 1 - require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey)) - bid = blob.MinBid + proto.BlobID(blob.Count+ii) - require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey)) + bid := blob.MinSliceID - proto.BlobID(ii) - 1 + require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey)) + bid = blob.MinSliceID + proto.BlobID(blob.Count+ii) + require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey)) } for ii := uint32(0); ii < blob.Count; ii++ { - bid := blob.MinBid + proto.BlobID(ii) - require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey)) + bid := blob.MinSliceID + proto.BlobID(ii) + require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey)) } } return } - require.Equal(t, len(loc.Blobs)+1, len(tokens)) - for ii := 0; ii < len(loc.Blobs)-1; ii++ { + require.Equal(t, len(loc.Slices)+1, len(tokens)) + for ii := 0; ii < len(loc.Slices)-1; ii++ { token := uptoken.DecodeToken(tokens[ii]) - blob := loc.Blobs[ii] + blob := loc.Slices[ii] for ii := uint32(0); ii < blob.Count; ii++ { - bid := blob.MinBid + proto.BlobID(ii) - require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey)) + bid := blob.MinSliceID + proto.BlobID(ii) + require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey)) } } - token := uptoken.DecodeToken(tokens[len(loc.Blobs)-1]) - blob := loc.Blobs[len(loc.Blobs)-1] + token := uptoken.DecodeToken(tokens[len(loc.Slices)-1]) + blob := loc.Slices[len(loc.Slices)-1] for ii := uint32(0); ii < 100; ii++ { - bid := blob.MinBid - proto.BlobID(ii) - 1 - require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey)) - bid = blob.MinBid + proto.BlobID(blob.Count+ii) - 1 - require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey)) + bid := blob.MinSliceID - proto.BlobID(ii) - 1 + require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey)) + bid = blob.MinSliceID + proto.BlobID(blob.Count+ii) - 1 + require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey)) } for ii := uint32(0); ii < blob.Count-1; ii++ { - bid := blob.MinBid + proto.BlobID(ii) - require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey)) + bid := blob.MinSliceID + proto.BlobID(ii) + require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey)) } - token = uptoken.DecodeToken(tokens[len(loc.Blobs)]) - lastbid := blob.MinBid + proto.BlobID(blob.Count) - 1 + token = uptoken.DecodeToken(tokens[len(loc.Slices)]) + lastbid := blob.MinSliceID + proto.BlobID(blob.Count) - 1 require.True(t, token.IsValid(loc.ClusterID, blob.Vid, lastbid, lastSize, skey)) } { - loc := &access.Location{ - Size: 0, - BlobSize: 333, - Blobs: []access.SliceInfo{}, + loc := &proto.Location{ + Size_: 0, + SliceSize: 333, + Slices: []proto.Slice{}, } checker(loc, stream.StreamGenTokens(loc)) } { - loc := &access.Location{ - Size: 1, - BlobSize: 1024, - Blobs: []access.SliceInfo{ - {MinBid: 100, Vid: 1000, Count: 1}, + loc := &proto.Location{ + Size_: 1, + SliceSize: 1024, + Slices: []proto.Slice{ + {MinSliceID: 100, Vid: 1000, Count: 1}, }, } checker(loc, stream.StreamGenTokens(loc)) } { - loc := &access.Location{ - Size: 1024, - BlobSize: 1024, - Blobs: []access.SliceInfo{ - {MinBid: 100, Vid: 1000, Count: 1}, + loc := &proto.Location{ + Size_: 1024, + SliceSize: 1024, + Slices: []proto.Slice{ + {MinSliceID: 100, Vid: 1000, Count: 1}, }, } checker(loc, stream.StreamGenTokens(loc)) } { - loc := &access.Location{ - Size: 1025, - BlobSize: 1024, - Blobs: []access.SliceInfo{ - {MinBid: 100, Vid: 1000, Count: 2}, + loc := &proto.Location{ + Size_: 1025, + SliceSize: 1024, + Slices: []proto.Slice{ + {MinSliceID: 100, Vid: 1000, Count: 2}, }, } checker(loc, stream.StreamGenTokens(loc)) } { - loc := &access.Location{ - Size: 2048, - BlobSize: 1024, - Blobs: []access.SliceInfo{ - {MinBid: 100, Vid: 1000, Count: 2}, + loc := &proto.Location{ + Size_: 2048, + SliceSize: 1024, + Slices: []proto.Slice{ + {MinSliceID: 100, Vid: 1000, Count: 2}, }, } checker(loc, stream.StreamGenTokens(loc)) } { - loc := &access.Location{ - Size: 10240, - BlobSize: 1024, - Blobs: []access.SliceInfo{ - {MinBid: 100, Vid: 1000, Count: 4}, - {MinBid: 200, Vid: 1000, Count: 6}, + loc := &proto.Location{ + Size_: 10240, + SliceSize: 1024, + Slices: []proto.Slice{ + {MinSliceID: 100, Vid: 1000, Count: 4}, + {MinSliceID: 200, Vid: 1000, Count: 6}, }, } checker(loc, stream.StreamGenTokens(loc)) } { - loc := &access.Location{ - Size: 1025, - BlobSize: 1024, - Blobs: []access.SliceInfo{ - {MinBid: 100, Vid: 1000, Count: 1}, - {MinBid: 200, Vid: 1000, Count: 1}, + loc := &proto.Location{ + Size_: 1025, + SliceSize: 1024, + Slices: []proto.Slice{ + {MinSliceID: 100, Vid: 1000, Count: 1}, + {MinSliceID: 200, Vid: 1000, Count: 1}, }, } checker(loc, stream.StreamGenTokens(loc)) } { - loc := &access.Location{ - Size: 10242, - BlobSize: 1024, - Blobs: []access.SliceInfo{ - {MinBid: 100, Vid: 1000, Count: 5}, - {MinBid: 200, Vid: 1000, Count: 6}, + loc := &proto.Location{ + Size_: 10242, + SliceSize: 1024, + Slices: []proto.Slice{ + {MinSliceID: 100, Vid: 1000, Count: 5}, + {MinSliceID: 200, Vid: 1000, Count: 6}, }, } checker(loc, stream.StreamGenTokens(loc)) @@ -702,7 +702,7 @@ func TestAccessServiceLimited(t *testing.T) { if err != nil { assertErrorCode(t, errcode.CodeAccessLimited, err) } else { - require.Equal(t, uint64(1024), resp.Location.Size) + require.Equal(t, uint64(1024), resp.Location.Size_) } }() } diff --git a/blobstore/access/stream/server_location.go b/blobstore/access/stream/server_location.go index e8a0a72d7..73a233765 100644 --- a/blobstore/access/stream/server_location.go +++ b/blobstore/access/stream/server_location.go @@ -20,7 +20,6 @@ import ( "sync" "time" - "github.com/cubefs/cubefs/blobstore/api/access" "github.com/cubefs/cubefs/blobstore/common/proto" "github.com/cubefs/cubefs/blobstore/common/uptoken" "github.com/cubefs/cubefs/blobstore/util/bytespool" @@ -82,7 +81,7 @@ func TokenSecretKeys() [][20]byte { return tokenSecretKeys[:] } -func calcCrc(loc *access.Location) (uint32, error) { +func calcCrc(loc *proto.Location) (uint32, error) { crcWriter := crc32.New(_crcTable) buf := bytespool.Alloc(1024) @@ -103,7 +102,7 @@ func calcCrc(loc *access.Location) (uint32, error) { return crcWriter.Sum32(), nil } -func fillCrc(loc *access.Location) error { +func fillCrc(loc *proto.Location) error { crc, err := calcCrc(loc) if err != nil { return err @@ -112,7 +111,7 @@ func fillCrc(loc *access.Location) error { return nil } -func verifyCrc(loc *access.Location) bool { +func verifyCrc(loc *proto.Location) bool { crc, err := calcCrc(loc) if err != nil { return false @@ -120,13 +119,13 @@ func verifyCrc(loc *access.Location) bool { return loc.Crc == crc } -func signCrc(loc *access.Location, locs []access.Location) error { +func signCrc(loc *proto.Location, locs []proto.Location) error { first := locs[0] bids := make(map[proto.BlobID]struct{}, 64) if loc.ClusterID != first.ClusterID || loc.CodeMode != first.CodeMode || - loc.BlobSize != first.BlobSize { + loc.SliceSize != first.SliceSize { return fmt.Errorf("not equal in constant field") } @@ -138,20 +137,20 @@ func signCrc(loc *access.Location, locs []access.Location) error { // assert if l.ClusterID != first.ClusterID || l.CodeMode != first.CodeMode || - l.BlobSize != first.BlobSize { + l.SliceSize != first.SliceSize { return fmt.Errorf("not equal in constant field") } - for _, blob := range l.Blobs { + for _, blob := range l.Slices { for c := 0; c < int(blob.Count); c++ { - bids[blob.MinBid+proto.BlobID(c)] = struct{}{} + bids[blob.MinSliceID+proto.BlobID(c)] = struct{}{} } } } - for _, blob := range loc.Blobs { + for _, blob := range loc.Slices { for c := 0; c < int(blob.Count); c++ { - bid := blob.MinBid + proto.BlobID(c) + bid := blob.MinSliceID + proto.BlobID(c) if _, ok := bids[bid]; !ok { return fmt.Errorf("not equal in blob_id(%d)", bid) } @@ -169,27 +168,27 @@ func signCrc(loc *access.Location, locs []access.Location) error { // will be used by the last blob, even if the last slice blobs' size // less than blobsize. // 5. Each segment blob has its specified token include the last blob. -func genTokens(location *access.Location) []string { - tokens := make([]string, 0, len(location.Blobs)+1) +func genTokens(location *proto.Location) []string { + tokens := make([]string, 0, len(location.Slices)+1) - hasMultiBlobs := location.Size >= uint64(location.BlobSize) - lastSize := uint32(location.Size % uint64(location.BlobSize)) - for idx, blob := range location.Blobs { + hasMultiBlobs := location.Size_ >= uint64(location.SliceSize) + lastSize := uint32(location.Size_ % uint64(location.SliceSize)) + for idx, blob := range location.Slices { // returns one token if size < blobsize if hasMultiBlobs { count := blob.Count - if idx == len(location.Blobs)-1 && lastSize > 0 { + if idx == len(location.Slices)-1 && lastSize > 0 { count-- } tokens = append(tokens, uptoken.EncodeToken(uptoken.NewUploadToken(location.ClusterID, - blob.Vid, blob.MinBid, count, - location.BlobSize, _tokenExpiration, tokenSecretKeys[0][:]))) + blob.Vid, blob.MinSliceID, count, + location.SliceSize, _tokenExpiration, tokenSecretKeys[0][:]))) } // token of the last blob - if idx == len(location.Blobs)-1 && lastSize > 0 { + if idx == len(location.Slices)-1 && lastSize > 0 { tokens = append(tokens, uptoken.EncodeToken(uptoken.NewUploadToken(location.ClusterID, - blob.Vid, blob.MinBid+proto.BlobID(blob.Count)-1, 1, + blob.Vid, blob.MinSliceID+proto.BlobID(blob.Count)-1, 1, lastSize, _tokenExpiration, tokenSecretKeys[0][:]))) } } diff --git a/blobstore/access/stream/server_location_test.go b/blobstore/access/stream/server_location_test.go index f5c58723e..cf99b9a40 100644 --- a/blobstore/access/stream/server_location_test.go +++ b/blobstore/access/stream/server_location_test.go @@ -22,7 +22,6 @@ import ( "github.com/stretchr/testify/require" - "github.com/cubefs/cubefs/blobstore/api/access" "github.com/cubefs/cubefs/blobstore/common/codemode" "github.com/cubefs/cubefs/blobstore/common/proto" "github.com/cubefs/cubefs/blobstore/common/uptoken" @@ -30,21 +29,21 @@ import ( ) var ( - testMaxBlob = access.SliceInfo{ - MinBid: proto.BlobID(math.MaxUint64), - Vid: proto.Vid(math.MaxInt32), - Count: math.MaxUint32, + testMaxBlob = proto.Slice{ + MinSliceID: proto.BlobID(math.MaxUint64), + Vid: proto.Vid(math.MaxInt32), + Count: math.MaxUint32, } - testMaxLoc = access.Location{ + testMaxLoc = proto.Location{ ClusterID: proto.ClusterID(math.MaxUint32), CodeMode: codemode.CodeMode(math.MaxInt8), - Size: math.MaxUint64, - BlobSize: math.MaxUint32, + Size_: math.MaxUint64, + SliceSize: math.MaxUint32, Crc: math.MaxUint32, } - testMinBlob = access.SliceInfo{} - testMinLoc = access.Location{} + testMinBlob = proto.Slice{} + testMinLoc = proto.Location{} ) func TestAccessServiceLocationCrc(t *testing.T) { @@ -64,7 +63,7 @@ func TestAccessServiceLocationCrc(t *testing.T) { } { loc := testMinLoc.Copy() - loc.Size = 1 << 30 + loc.Size_ = 1 << 30 err := fillCrc(&loc) require.NoError(t, err) @@ -72,7 +71,7 @@ func TestAccessServiceLocationCrc(t *testing.T) { } { loc := testMinLoc.Copy() - loc.Size = 1 << 30 + loc.Size_ = 1 << 30 require.False(t, verifyCrc(&loc)) loc.Crc = 0x9e17bc9e @@ -108,15 +107,15 @@ func TestAccessServiceTokenSecret(t *testing.T) { tokenSecretKeys = keys }() b := [...]byte{1: 2, 7: 8} - loc := &access.Location{ + loc := &proto.Location{ ClusterID: 1, CodeMode: 1, - Size: 1023, - BlobSize: 1024, - Blobs: []access.SliceInfo{{ - MinBid: 11, - Vid: 199, - Count: 1, + Size_: 1023, + SliceSize: 1024, + Slices: []proto.Slice{{ + MinSliceID: 11, + Vid: 199, + Count: 1, }}, } @@ -131,16 +130,16 @@ func TestAccessServiceTokenSecret(t *testing.T) { } func TestAccessServiceLocationSignCrc(t *testing.T) { - loc := &access.Location{ + loc := &proto.Location{ ClusterID: 1, CodeMode: 1, - Size: 1023, - BlobSize: 6, + Size_: 1023, + SliceSize: 6, Crc: 0, - Blobs: []access.SliceInfo{{ - MinBid: 11, - Vid: 199, - Count: 10, + Slices: []proto.Slice{{ + MinSliceID: 11, + Vid: 199, + Count: 10, }}, } fillCrc(loc) @@ -148,42 +147,42 @@ func TestAccessServiceLocationSignCrc(t *testing.T) { { loc1, loc2 := loc.Copy(), loc.Copy() - require.NoError(t, signCrc(loc, []access.Location{loc1, loc2})) + require.NoError(t, signCrc(loc, []proto.Location{loc1, loc2})) } { loc1, loc2 := loc.Copy(), loc.Copy() - loc1.BlobSize = 100 + loc1.SliceSize = 100 fillCrc(&loc1) - require.Error(t, signCrc(loc, []access.Location{loc1, loc2})) + require.Error(t, signCrc(loc, []proto.Location{loc1, loc2})) } { loc1, loc2 := loc.Copy(), loc.Copy() loc2.Crc = 0 - require.Error(t, signCrc(loc, []access.Location{loc1, loc2})) + require.Error(t, signCrc(loc, []proto.Location{loc1, loc2})) } { loc1, loc2 := loc.Copy(), loc.Copy() loc2.ClusterID = 2 fillCrc(&loc2) - require.Error(t, signCrc(loc, []access.Location{loc1, loc2})) + require.Error(t, signCrc(loc, []proto.Location{loc1, loc2})) } { loc1, loc2 := loc.Copy(), loc.Copy() loc2.CodeMode = 100 fillCrc(&loc2) - require.Error(t, signCrc(loc, []access.Location{loc1, loc2})) + require.Error(t, signCrc(loc, []proto.Location{loc1, loc2})) } { loc1, loc2 := loc.Copy(), loc.Copy() - loc1.Blobs = nil - loc2.Blobs[0].Count = 5 + loc1.Slices = nil + loc2.Slices[0].Count = 5 fillCrc(&loc1) fillCrc(&loc2) - require.Error(t, signCrc(loc, []access.Location{loc1, loc2})) + require.Error(t, signCrc(loc, []proto.Location{loc1, loc2})) } } -func calcCrcWithoutMagic(loc *access.Location) (uint32, error) { +func calcCrcWithoutMagic(loc *proto.Location) (uint32, error) { crcWriter := crc32.New(_crcTable) buf := bytespool.Alloc(1024) @@ -196,17 +195,17 @@ func calcCrcWithoutMagic(loc *access.Location) (uint32, error) { } func benchmarkCrc(b *testing.B, key string, - location access.Location, blob access.SliceInfo, - run func(loc *access.Location) (uint32, error), + location proto.Location, blob proto.Slice, + run func(loc *proto.Location) (uint32, error), ) { cases := []int{0, 2, 4, 8, 16, 32} for _, l := range cases { b.ResetTimer() b.Run(fmt.Sprintf(key+"-%d", l), func(b *testing.B) { loc := location.Copy() - loc.Blobs = make([]access.SliceInfo, l) - for idx := range loc.Blobs { - loc.Blobs[idx] = blob + loc.Slices = make([]proto.Slice, l) + for idx := range loc.Slices { + loc.Slices[idx] = blob } b.ResetTimer() for ii := 0; ii <= b.N; ii++ { diff --git a/blobstore/access/stream/stream.go b/blobstore/access/stream/stream.go index 12583eda1..0e0c8681c 100644 --- a/blobstore/access/stream/stream.go +++ b/blobstore/access/stream/stream.go @@ -56,7 +56,7 @@ type StreamHandler interface { // codeMode > 0, alloc in this codemode // return: a location of file Alloc(ctx context.Context, size uint64, blobSize uint32, - assignClusterID proto.ClusterID, codeMode codemode.CodeMode) (*access.Location, error) + assignClusterID proto.ClusterID, codeMode codemode.CodeMode) (*proto.Location, error) // PutAt access interface /putat, put one blob // required: rc file reader @@ -69,7 +69,7 @@ type StreamHandler interface { // Put put one object // required: size, file size // optional: hasher map to calculate hash.Hash - Put(ctx context.Context, rc io.Reader, size int64, hasherMap access.HasherMap) (*access.Location, error) + Put(ctx context.Context, rc io.Reader, size int64, hasherMap access.HasherMap) (*proto.Location, error) // Get read file // required: location, readSize @@ -93,10 +93,10 @@ type StreamHandler interface { //... //read-9 [d4 p5] //failed - Get(ctx context.Context, w io.Writer, location access.Location, readSize, offset uint64) (func() error, error) + Get(ctx context.Context, w io.Writer, location proto.Location, readSize, offset uint64) (func() error, error) // Delete delete all blobs in this location - Delete(ctx context.Context, location *access.Location) error + Delete(ctx context.Context, location *proto.Location) error // Admin returns internal admin interface. Admin() interface{} @@ -317,7 +317,7 @@ func NewStreamHandler(cfg *StreamConfig, stopCh <-chan struct{}) (h StreamHandle } // Delete delete all blobs in this location -func (h *Handler) Delete(ctx context.Context, location *access.Location) error { +func (h *Handler) Delete(ctx context.Context, location *proto.Location) error { span := trace.SpanFromContextSafe(ctx) span.Debugf("to delete %+v", location) return h.clearGarbage(ctx, location) @@ -384,7 +384,7 @@ func (h *Handler) sendRepairMsg(ctx context.Context, blob blobIdent, badIdxes [] span.Infof("send repair message(%+v)", repairArgs) } -func (h *Handler) clearGarbage(ctx context.Context, location *access.Location) error { +func (h *Handler) clearGarbage(ctx context.Context, location *proto.Location) error { span := trace.SpanFromContextSafe(ctx) serviceController, err := h.clusterController.GetServiceController(location.ClusterID) if err != nil { diff --git a/blobstore/access/stream/stream_alloc.go b/blobstore/access/stream/stream_alloc.go index 0f2164f27..a41c299a5 100644 --- a/blobstore/access/stream/stream_alloc.go +++ b/blobstore/access/stream/stream_alloc.go @@ -41,7 +41,7 @@ var errAllocatePunishedVolume = errors.New("allocate punished volume") // return: a location of file func (h *Handler) Alloc(ctx context.Context, size uint64, blobSize uint32, assignClusterID proto.ClusterID, codeMode codemode.CodeMode, -) (*access.Location, error) { +) (*proto.Location, error) { span := trace.SpanFromContextSafe(ctx) span.Debugf("alloc request with size:%d blobsize:%d cluster:%d codemode:%d", size, blobSize, assignClusterID, codeMode) @@ -72,12 +72,12 @@ func (h *Handler) Alloc(ctx context.Context, size uint64, blobSize uint32, } span.Debugf("allocated from %d %+v", clusterID, blobs) - location := &access.Location{ + location := &proto.Location{ ClusterID: clusterID, CodeMode: codeMode, - Size: size, - BlobSize: blobSize, - Blobs: blobs, + Size_: size, + SliceSize: blobSize, + Slices: blobs, } span.Debugf("alloc ok %+v", location) return location, nil @@ -85,7 +85,7 @@ func (h *Handler) Alloc(ctx context.Context, size uint64, blobSize uint32, func (h *Handler) allocFromAllocatorWithHystrix(ctx context.Context, codeMode codemode.CodeMode, size uint64, blobSize uint32, clusterID proto.ClusterID, -) (cid proto.ClusterID, bidRets []access.SliceInfo, err error) { +) (cid proto.ClusterID, bidRets []proto.Slice, err error) { err = hystrix.Do(allocCommand, func() error { cid, bidRets, err = h.allocFromAllocator(ctx, codeMode, size, blobSize, clusterID) return err @@ -95,7 +95,7 @@ func (h *Handler) allocFromAllocatorWithHystrix(ctx context.Context, func (h *Handler) allocFromAllocator(ctx context.Context, codeMode codemode.CodeMode, size uint64, blobSize uint32, clusterID proto.ClusterID, -) (proto.ClusterID, []access.SliceInfo, error) { +) (proto.ClusterID, []proto.Slice, error) { span := trace.SpanFromContextSafe(ctx) if blobSize == 0 { @@ -183,7 +183,7 @@ func (h *Handler) allocFromAllocator(ctx context.Context, } blobN := blobCount(size, blobSize) - blobs := make([]access.SliceInfo, 0, blobN) + blobs := make([]proto.Slice, 0, blobN) for _, bidRet := range allocRets { if blobN <= 0 { break @@ -192,10 +192,10 @@ func (h *Handler) allocFromAllocator(ctx context.Context, count := minU64(blobN, uint64(bidRet.BidEnd)-uint64(bidRet.BidStart)+1) blobN -= count - blobs = append(blobs, access.SliceInfo{ - MinBid: bidRet.BidStart, - Vid: bidRet.Vid, - Count: uint32(count), + blobs = append(blobs, proto.Slice{ + MinSliceID: bidRet.BidStart, + Vid: bidRet.Vid, + Count: uint32(count), }) } if blobN > 0 { diff --git a/blobstore/access/stream/stream_alloc_test.go b/blobstore/access/stream/stream_alloc_test.go index f17c1d492..7f61e17f4 100644 --- a/blobstore/access/stream/stream_alloc_test.go +++ b/blobstore/access/stream/stream_alloc_test.go @@ -32,26 +32,26 @@ func TestAccessStreamAllocBase(t *testing.T) { require.NoError(t, err) require.Equal(t, clusterID, loc.ClusterID) require.Equal(t, codemode.EC6P6, loc.CodeMode) - require.Equal(t, uint64(1<<30), loc.Size) - require.Equal(t, uint32(1<<22), loc.BlobSize) - require.Equal(t, 2, len(loc.Blobs)) - require.Equal(t, uint32(1), loc.Blobs[0].Count) - require.Equal(t, uint32((1<<8)-1), loc.Blobs[1].Count) + require.Equal(t, uint64(1<<30), loc.Size_) + require.Equal(t, uint32(1<<22), loc.SliceSize) + require.Equal(t, 2, len(loc.Slices)) + require.Equal(t, uint32(1), loc.Slices[0].Count) + require.Equal(t, uint32((1<<8)-1), loc.Slices[1].Count) } { loc, err := streamer.Alloc(ctx(), (1<<30)+1, 0, 0, 0) require.NoError(t, err) - require.Equal(t, 2, len(loc.Blobs)) - require.Equal(t, uint32(1), loc.Blobs[0].Count) - require.Equal(t, uint32(1<<8), loc.Blobs[1].Count) + require.Equal(t, 2, len(loc.Slices)) + require.Equal(t, uint32(1), loc.Slices[0].Count) + require.Equal(t, uint32(1<<8), loc.Slices[1].Count) } // 1M blobsize { loc, err := streamer.Alloc(ctx(), 1<<30, 1<<20, 0, 0) require.NoError(t, err) - require.Equal(t, 2, len(loc.Blobs)) - require.Equal(t, uint32(1), loc.Blobs[0].Count) - require.Equal(t, uint32((1<<10)-1), loc.Blobs[1].Count) + require.Equal(t, 2, len(loc.Slices)) + require.Equal(t, uint32(1), loc.Slices[0].Count) + require.Equal(t, uint32((1<<10)-1), loc.Slices[1].Count) } // max size + 1 { diff --git a/blobstore/access/stream/stream_get.go b/blobstore/access/stream/stream_get.go index 241197c63..b08bd0fc4 100644 --- a/blobstore/access/stream/stream_get.go +++ b/blobstore/access/stream/stream_get.go @@ -27,7 +27,6 @@ import ( "github.com/afex/hystrix-go/hystrix" "github.com/cubefs/cubefs/blobstore/access/controller" - "github.com/cubefs/cubefs/blobstore/api/access" "github.com/cubefs/cubefs/blobstore/api/blobnode" "github.com/cubefs/cubefs/blobstore/common/codemode" "github.com/cubefs/cubefs/blobstore/common/ec" @@ -112,7 +111,7 @@ type pipeBuffer struct { // ... // read-9 [d4 p5] // failed -func (h *Handler) Get(ctx context.Context, w io.Writer, location access.Location, readSize, offset uint64) (func() error, error) { +func (h *Handler) Get(ctx context.Context, w io.Writer, location proto.Location, readSize, offset uint64) (func() error, error) { span := trace.SpanFromContextSafe(ctx) span.Debugf("get request cluster:%d size:%d offset:%d", location.ClusterID, readSize, offset) @@ -758,14 +757,14 @@ func (h *Handler) getOneShardFromHost(ctx context.Context, serviceController con return } -func genLocationBlobs(location *access.Location, readSize uint64, offset uint64) ([]blobGetArgs, error) { - if readSize > location.Size || offset > location.Size || offset+readSize > location.Size { - return nil, fmt.Errorf("FileSize:%d ReadSize:%d Offset:%d", location.Size, readSize, offset) +func genLocationBlobs(location *proto.Location, readSize uint64, offset uint64) ([]blobGetArgs, error) { + if readSize > location.Size_ || offset > location.Size_ || offset+readSize > location.Size_ { + return nil, fmt.Errorf("FileSize:%d ReadSize:%d Offset:%d", location.Size_, readSize, offset) } - blobSize := uint64(location.BlobSize) + blobSize := uint64(location.SliceSize) if blobSize <= 0 { - return nil, fmt.Errorf("BlobSize:%d", blobSize) + return nil, fmt.Errorf("SliceSize:%d", blobSize) } remainSize := readSize @@ -776,8 +775,8 @@ func genLocationBlobs(location *access.Location, readSize uint64, offset uint64) idx := uint64(0) blobs := make([]blobGetArgs, 0, 1+(readSize+blobOffset)/blobSize) - for _, blob := range location.Blobs { - currBlobID := blob.MinBid + for _, blob := range location.Slices { + currBlobID := blob.MinSliceID for ii := uint32(0); ii < blob.Count; ii++ { if remainSize <= 0 { @@ -788,7 +787,7 @@ func genLocationBlobs(location *access.Location, readSize uint64, offset uint64) toReadSize := minU64(remainSize, blobSize-blobOffset) if toReadSize > 0 { // update the last blob size - fixedBlobSize := minU64(location.Size-idx*blobSize, blobSize) + fixedBlobSize := minU64(location.Size_-idx*blobSize, blobSize) sizes, _ := ec.GetBufferSizes(int(fixedBlobSize), tactic) shardSize := sizes.ShardSize diff --git a/blobstore/access/stream/stream_get_test.go b/blobstore/access/stream/stream_get_test.go index 7d81c7eff..9c6db5847 100644 --- a/blobstore/access/stream/stream_get_test.go +++ b/blobstore/access/stream/stream_get_test.go @@ -24,7 +24,6 @@ import ( "github.com/stretchr/testify/require" - "github.com/cubefs/cubefs/blobstore/api/access" "github.com/cubefs/cubefs/blobstore/common/codemode" "github.com/cubefs/cubefs/blobstore/common/proto" ) @@ -530,21 +529,21 @@ func TestAccessStreamGenLocationBlobs(t *testing.T) { firstSliceStart := proto.BlobID(100) secondSliceStart := proto.BlobID(200) - loc := access.Location{ + loc := proto.Location{ ClusterID: 0, CodeMode: codemode.EC6P6, - Size: 1024*4 + 37 + 1024*2, // 5 fine blobs and 2 missing blobs - BlobSize: 1024, - Blobs: []access.SliceInfo{ + Size_: 1024*4 + 37 + 1024*2, // 5 fine blobs and 2 missing blobs + SliceSize: 1024, + Slices: []proto.Slice{ { - MinBid: firstSliceStart, - Vid: proto.Vid(1001), - Count: 3, + MinSliceID: firstSliceStart, + Vid: proto.Vid(1001), + Count: 3, }, { - MinBid: secondSliceStart, - Vid: proto.Vid(2001), - Count: 2, + MinSliceID: secondSliceStart, + Vid: proto.Vid(2001), + Count: 2, }, }, } diff --git a/blobstore/access/stream/stream_put.go b/blobstore/access/stream/stream_put.go index 3ecb4180f..a049e2376 100644 --- a/blobstore/access/stream/stream_put.go +++ b/blobstore/access/stream/stream_put.go @@ -30,6 +30,7 @@ import ( "github.com/cubefs/cubefs/blobstore/api/blobnode" "github.com/cubefs/cubefs/blobstore/common/ec" errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" "github.com/cubefs/cubefs/blobstore/common/rpc" "github.com/cubefs/cubefs/blobstore/common/trace" "github.com/cubefs/cubefs/blobstore/util/errors" @@ -45,7 +46,7 @@ import ( // optional: hasher map to calculate hash.Hash func (h *Handler) Put(ctx context.Context, rc io.Reader, size int64, hasherMap access.HasherMap, -) (*access.Location, error) { +) (*proto.Location, error) { span := trace.SpanFromContextSafe(ctx) span.Debugf("put request size:%d hashes:b(%b)", size, hasherMap.ToHashAlgorithm()) @@ -76,12 +77,12 @@ func (h *Handler) Put(ctx context.Context, // 3.read body and split, alloc from mem pool;ec encode and put into data node limitReader := io.LimitReader(rc, int64(size)) - location := &access.Location{ + location := &proto.Location{ ClusterID: clusterID, CodeMode: selectedCodeMode, - Size: uint64(size), - BlobSize: blobSize, - Blobs: blobs, + Size_: uint64(size), + SliceSize: blobSize, + Slices: blobs, } uploadSucc := false diff --git a/blobstore/access/stream/stream_put_test.go b/blobstore/access/stream/stream_put_test.go index 8723ae6f2..7200a57eb 100644 --- a/blobstore/access/stream/stream_put_test.go +++ b/blobstore/access/stream/stream_put_test.go @@ -50,8 +50,8 @@ func TestAccessStreamPutBase(t *testing.T) { size := 1 loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil) require.NoError(t, err) - require.Equal(t, 1, len(loc.Blobs)) - require.Equal(t, uint32(1), loc.Blobs[0].Count) + require.Equal(t, 1, len(loc.Slices)) + require.Equal(t, uint32(1), loc.Slices[0].Count) // time wait the punished services time.Sleep(time.Second * time.Duration(punishServiceS)) } @@ -60,8 +60,8 @@ func TestAccessStreamPutBase(t *testing.T) { size := 1 << 18 loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil) require.NoError(t, err) - require.Equal(t, 1, len(loc.Blobs)) - require.Equal(t, uint32(1), loc.Blobs[0].Count) + require.Equal(t, 1, len(loc.Slices)) + require.Equal(t, uint32(1), loc.Slices[0].Count) time.Sleep(time.Second * time.Duration(punishServiceS)) } // 8M + 1k @@ -69,8 +69,8 @@ func TestAccessStreamPutBase(t *testing.T) { size := (1 << 23) + 1024 loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil) require.NoError(t, err) - require.Equal(t, 2, len(loc.Blobs)) - require.Equal(t, uint32(2), loc.Blobs[1].Count) + require.Equal(t, 2, len(loc.Slices)) + require.Equal(t, uint32(2), loc.Slices[1].Count) time.Sleep(time.Second * time.Duration(punishServiceS)) } // max size + 1 diff --git a/blobstore/api/access/client.go b/blobstore/api/access/client.go index 1120f6950..2527f6f63 100644 --- a/blobstore/api/access/client.go +++ b/blobstore/api/access/client.go @@ -212,7 +212,7 @@ type API interface { // // If PutArgs' body is of type *bytes.Buffer, *bytes.Reader, or *strings.Reader, // GetBody is populated, then the Put once request has retry ability. - Put(ctx context.Context, args *PutArgs) (location Location, hashSumMap HashSumMap, err error) + Put(ctx context.Context, args *PutArgs) (location proto.Location, hashSumMap HashSumMap, err error) // Get object, range is supported. Get(ctx context.Context, args *GetArgs) (body io.ReadCloser, err error) // Delete all blobs in these locations. @@ -221,7 +221,7 @@ type API interface { // - (nil, nil): all blobs deleted successfully. // - (nil, ErrIllegalArguments): when args is invalid. // - (failedLocations, err): returns the list of locations that have not yet been deleted. - Delete(ctx context.Context, args *DeleteArgs) (failedLocations []Location, err error) + Delete(ctx context.Context, args *DeleteArgs) (failedLocations []proto.Location, err error) } var _ API = (*client)(nil) @@ -391,13 +391,13 @@ func getClient(cfg *Config, hosts []string) rpc.Client { return rpc.NewLbClient(lbConfig, nil) } -func (c *client) Put(ctx context.Context, args *PutArgs) (location Location, hashSumMap HashSumMap, err error) { +func (c *client) Put(ctx context.Context, args *PutArgs) (location proto.Location, hashSumMap HashSumMap, err error) { if args.Size == 0 { hashSumMap := args.Hashes.ToHashSumMap() for alg := range hashSumMap { hashSumMap[alg] = alg.ToHasher().Sum(nil) } - return Location{Blobs: make([]SliceInfo, 0)}, hashSumMap, nil + return proto.Location{Slices: make([]proto.Slice, 0)}, hashSumMap, nil } ctx = withReqidContext(ctx) @@ -407,7 +407,7 @@ func (c *client) Put(ctx context.Context, args *PutArgs) (location Location, has return c.putParts(ctx, args) } -func (c *client) putObject(ctx context.Context, args *PutArgs) (location Location, hashSumMap HashSumMap, err error) { +func (c *client) putObject(ctx context.Context, args *PutArgs) (location proto.Location, hashSumMap HashSumMap, err error) { rpcClient := c.rpcClient.Load().(rpc.Client) urlStr := fmt.Sprintf("/put?size=%d&hashes=%d", args.Size, args.Hashes) @@ -510,7 +510,7 @@ func (c *client) readerPipeline(span trace.Span, reqBody io.Reader, return ch } -func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSumMap, error) { +func (c *client) putParts(ctx context.Context, args *PutArgs) (proto.Location, HashSumMap, error) { span := trace.SpanFromContextSafe(ctx) rpcClient := c.rpcClient.Load().(rpc.Client) @@ -526,7 +526,7 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum } var ( - loc Location + loc proto.Location tokens []string ) @@ -544,7 +544,7 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum signArgs.Location = loc.Copy() signResp := &SignResp{} if err := rpcClient.PostWith(newCtx, "/sign", signResp, signArgs); err == nil { - locations = []Location{signResp.Location.Copy()} + locations = []proto.Location{signResp.Location.Copy()} } } if len(locations) > 0 { @@ -565,7 +565,7 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum // buffer pipeline closeCh := make(chan struct{}) - bufferPipe := c.readerPipeline(span, reqBody, closeCh, int(loc.Size), int(loc.BlobSize)) + bufferPipe := c.readerPipeline(span, reqBody, closeCh, int(loc.Size_), int(loc.SliceSize)) defer func() { close(closeCh) // waiting pipeline close if has error @@ -584,17 +584,17 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum currBlobIdx := 0 currBlobCount := uint32(0) - remainSize := loc.Size + remainSize := loc.Size_ restPartsLoc := loc readSize := 0 - for readSize < int(loc.Size) { + for readSize < int(loc.Size_) { parts := make([]blobPart, 0, c.config.PartConcurrence) // waiting at least one blob buf, ok := <-bufferPipe - if !ok && readSize < int(loc.Size) { - return Location{}, nil, errcode.ErrAccessReadRequestBody + if !ok && readSize < int(loc.Size_) { + return proto.Location{}, nil, errcode.ErrAccessReadRequestBody } readSize += len(buf) parts = append(parts, blobPart{size: len(buf), buf: buf}) @@ -604,9 +604,9 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum select { case buf, ok := <-bufferPipe: if !ok { - if readSize < int(loc.Size) { + if readSize < int(loc.Size_) { releaseBuffer(parts) - return Location{}, nil, errcode.ErrAccessReadRequestBody + return proto.Location{}, nil, errcode.ErrAccessReadRequestBody } more = false } else { @@ -620,9 +620,9 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum tryTimes := c.config.MaxPartRetry for { - if len(loc.Blobs) > MaxLocationBlobs { + if len(loc.Slices) > MaxLocationBlobs { releaseBuffer(parts) - return Location{}, nil, errcode.ErrUnexpected + return proto.Location{}, nil, errcode.ErrUnexpected } // feed new params @@ -630,16 +630,16 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum currCount := currBlobCount for i := range parts { token := tokens[currIdx] - if restPartsLoc.Size > uint64(loc.BlobSize) && parts[i].size < int(loc.BlobSize) { + if restPartsLoc.Size_ > uint64(loc.SliceSize) && parts[i].size < int(loc.SliceSize) { token = tokens[currIdx+1] } parts[i].token = token parts[i].cid = loc.ClusterID - parts[i].vid = loc.Blobs[currIdx].Vid - parts[i].bid = loc.Blobs[currIdx].MinBid + proto.BlobID(currCount) + parts[i].vid = loc.Slices[currIdx].Vid + parts[i].bid = loc.Slices[currIdx].MinSliceID + proto.BlobID(currCount) currCount++ - if loc.Blobs[currIdx].Count == currCount { + if loc.Slices[currIdx].Count == currCount { currIdx++ currCount = 0 } @@ -651,7 +651,7 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum remainSize -= uint64(part.size) currBlobCount++ // next blobs - if loc.Blobs[currBlobIdx].Count == currBlobCount { + if loc.Slices[currBlobIdx].Count == currBlobCount { currBlobIdx++ currBlobCount = 0 } @@ -665,7 +665,7 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum if tryTimes == 1 { releaseBuffer(parts) span.Error("exceed the max retry limit", c.config.MaxPartRetry) - return Location{}, nil, errcode.ErrUnexpected + return proto.Location{}, nil, errcode.ErrUnexpected } tryTimes-- } @@ -676,14 +676,14 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum resp := &AllocResp{} if err := rpcClient.PostWith(ctx, "/alloc", resp, AllocArgs{ Size: remainSize, - BlobSize: loc.BlobSize, + BlobSize: loc.SliceSize, CodeMode: loc.CodeMode, AssignClusterID: loc.ClusterID, }); err != nil { return true, err } - if len(resp.Location.Blobs) > 0 { - if newVid := resp.Location.Blobs[0].Vid; newVid == loc.Blobs[currBlobIdx].Vid { + if len(resp.Location.Slices) > 0 { + if newVid := resp.Location.Slices[0].Vid; newVid == loc.Slices[currBlobIdx].Vid { return false, fmt.Errorf("alloc the same vid %d", newVid) } } @@ -693,17 +693,17 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum if err != nil { releaseBuffer(parts) span.Error("alloc another parts to put", err) - return Location{}, nil, errcode.ErrUnexpected + return proto.Location{}, nil, errcode.ErrUnexpected } restPartsLoc = restPartsResp.Location signArgs.Locations = append(signArgs.Locations, restPartsLoc.Copy()) if currBlobCount > 0 { - loc.Blobs[currBlobIdx].Count = currBlobCount + loc.Slices[currBlobIdx].Count = currBlobCount currBlobIdx++ } - loc.Blobs = append(loc.Blobs[:currBlobIdx], restPartsLoc.Blobs...) + loc.Slices = append(loc.Slices[:currBlobIdx], restPartsLoc.Slices...) tokens = append(tokens[:currBlobIdx], restPartsResp.Tokens...) currBlobCount = 0 @@ -718,7 +718,7 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum signResp := &SignResp{} if err := rpcClient.PostWith(ctx, "/sign", signResp, signArgs); err != nil { span.Error("sign location with crc", err) - return Location{}, nil, errcode.ErrUnexpected + return proto.Location{}, nil, errcode.ErrUnexpected } loc = signResp.Location } @@ -737,7 +737,7 @@ func (c *client) Get(ctx context.Context, args *GetArgs) (body io.ReadCloser, er rpcClient := c.rpcClient.Load().(rpc.Client) ctx = withReqidContext(ctx) - if args.Location.Size == 0 || args.ReadSize == 0 { + if args.Location.Size_ == 0 || args.ReadSize == 0 { return noopBody{}, nil } @@ -752,7 +752,7 @@ func (c *client) Get(ctx context.Context, args *GetArgs) (body io.ReadCloser, er return resp.Body, nil } -func (c *client) Delete(ctx context.Context, args *DeleteArgs) ([]Location, error) { +func (c *client) Delete(ctx context.Context, args *DeleteArgs) ([]proto.Location, error) { if !args.IsValid() { if args == nil { return nil, errcode.ErrIllegalArguments @@ -762,9 +762,9 @@ func (c *client) Delete(ctx context.Context, args *DeleteArgs) ([]Location, erro rpcClient := c.rpcClient.Load().(rpc.Client) ctx = withReqidContext(ctx) - locations := make([]Location, 0, len(args.Locations)) + locations := make([]proto.Location, 0, len(args.Locations)) for _, loc := range args.Locations { - if loc.Size > 0 { + if loc.Size_ > 0 { locations = append(locations, loc.Copy()) } } diff --git a/blobstore/api/access/client_test.go b/blobstore/api/access/client_test.go index 70b9b7a28..29e6ca0cc 100644 --- a/blobstore/api/access/client_test.go +++ b/blobstore/api/access/client_test.go @@ -157,60 +157,60 @@ func handleAlloc(c *rpc.Context) { return } - loc := access.Location{ + loc := proto.Location{ ClusterID: 1, - Size: args.Size, - BlobSize: blobSize, - Blobs: []access.SliceInfo{ + Size_: args.Size, + SliceSize: blobSize, + Slices: []proto.Slice{ { - MinBid: proto.BlobID(mrand.Int()), - Vid: proto.Vid(mrand.Int()), - Count: uint32((args.Size + blobSize - 1) / blobSize), + MinSliceID: proto.BlobID(mrand.Int()), + Vid: proto.Vid(mrand.Int()), + Count: uint32((args.Size + blobSize - 1) / blobSize), }, }, } // split to two blobs if large enough - if loc.Blobs[0].Count > 2 { - loc.Blobs[0].Count = 2 - loc.Blobs = append(loc.Blobs, []access.SliceInfo{ + if loc.Slices[0].Count > 2 { + loc.Slices[0].Count = 2 + loc.Slices = append(loc.Slices, []proto.Slice{ { - MinBid: proto.BlobID(mrand.Int()), - Vid: proto.Vid(mrand.Int()), - Count: uint32((args.Size - 2*blobSize + blobSize - 1) / blobSize), + MinSliceID: proto.BlobID(mrand.Int()), + Vid: proto.Vid(mrand.Int()), + Count: uint32((args.Size - 2*blobSize + blobSize - 1) / blobSize), }, }...) } // alloc the rest parts if args.AssignClusterID > 0 { - loc.Blobs = []access.SliceInfo{ + loc.Slices = []proto.Slice{ { - MinBid: proto.BlobID(mrand.Int()), - Vid: proto.Vid(mrand.Int()), - Count: uint32((args.Size + blobSize - 1) / blobSize), + MinSliceID: proto.BlobID(mrand.Int()), + Vid: proto.Vid(mrand.Int()), + Count: uint32((args.Size + blobSize - 1) / blobSize), }, } } - tokens := make([]string, 0, len(loc.Blobs)+1) + tokens := make([]string, 0, len(loc.Slices)+1) - hasMultiBlobs := loc.Size >= uint64(loc.BlobSize) - lastSize := uint32(loc.Size % uint64(loc.BlobSize)) - for idx, blob := range loc.Blobs { + hasMultiBlobs := loc.Size_ >= uint64(loc.SliceSize) + lastSize := uint32(loc.Size_ % uint64(loc.SliceSize)) + for idx, blob := range loc.Slices { // returns one token if size < blobsize if hasMultiBlobs { count := blob.Count - if idx == len(loc.Blobs)-1 && lastSize > 0 { + if idx == len(loc.Slices)-1 && lastSize > 0 { count-- } tokens = append(tokens, uptoken.EncodeToken(uptoken.NewUploadToken(loc.ClusterID, - blob.Vid, blob.MinBid, count, - loc.BlobSize, 0, tokenAlloc[:]))) + blob.Vid, blob.MinSliceID, count, + loc.SliceSize, 0, tokenAlloc[:]))) } // token of the last blob - if idx == len(loc.Blobs)-1 && lastSize > 0 { + if idx == len(loc.Slices)-1 && lastSize > 0 { tokens = append(tokens, uptoken.EncodeToken(uptoken.NewUploadToken(loc.ClusterID, - blob.Vid, blob.MinBid+proto.BlobID(blob.Count)-1, 1, + blob.Vid, blob.MinSliceID+proto.BlobID(blob.Count)-1, 1, lastSize, 0, tokenAlloc[:]))) } } @@ -251,7 +251,7 @@ func handlePut(c *rpc.Context) { hashSumMap[alg] = hasher.Sum(nil) } - loc := access.Location{Size: uint64(args.Size)} + loc := proto.Location{Size_: uint64(args.Size)} fillCrc(&loc) c.RespondJSON(access.PutResp{ Location: loc, @@ -298,7 +298,7 @@ func handleGet(c *rpc.Context) { return } - if args.Location.Size == 100 { + if args.Location.Size_ == 100 { c.RespondStatus(http.StatusBadRequest) return } @@ -359,7 +359,7 @@ func handleSign(c *rpc.Context) { c.RespondJSON(access.SignResp{Location: args.Location}) } -func calcCrc(loc *access.Location) (uint32, error) { +func calcCrc(loc *proto.Location) (uint32, error) { crcWriter := crc32.New(crc32.IEEETable) buf := bytespool.Alloc(1024) @@ -377,7 +377,7 @@ func calcCrc(loc *access.Location) (uint32, error) { return crcWriter.Sum32(), nil } -func fillCrc(loc *access.Location) error { +func fillCrc(loc *proto.Location) error { crc, err := calcCrc(loc) if err != nil { return err @@ -386,7 +386,7 @@ func fillCrc(loc *access.Location) error { return nil } -func verifyCrc(loc *access.Location) bool { +func verifyCrc(loc *proto.Location) bool { crc, err := calcCrc(loc) if err != nil { return false @@ -394,13 +394,13 @@ func verifyCrc(loc *access.Location) bool { return loc.Crc == crc } -func signCrc(loc *access.Location, locs []access.Location) error { +func signCrc(loc *proto.Location, locs []proto.Location) error { first := locs[0] bids := make(map[proto.BlobID]struct{}, 64) if loc.ClusterID != first.ClusterID || loc.CodeMode != first.CodeMode || - loc.BlobSize != first.BlobSize { + loc.SliceSize != first.SliceSize { return fmt.Errorf("not equal in constant field") } @@ -412,20 +412,20 @@ func signCrc(loc *access.Location, locs []access.Location) error { // assert if l.ClusterID != first.ClusterID || l.CodeMode != first.CodeMode || - l.BlobSize != first.BlobSize { + l.SliceSize != first.SliceSize { return fmt.Errorf("not equal in constant field") } - for _, blob := range l.Blobs { + for _, blob := range l.Slices { for c := 0; c < int(blob.Count); c++ { - bids[blob.MinBid+proto.BlobID(c)] = struct{}{} + bids[blob.MinSliceID+proto.BlobID(c)] = struct{}{} } } } - for _, blob := range loc.Blobs { + for _, blob := range loc.Slices { for c := 0; c < int(blob.Count); c++ { - bid := blob.MinBid + proto.BlobID(c) + bid := blob.MinSliceID + proto.BlobID(c) if _, ok := bids[bid]; !ok { return fmt.Errorf("not equal in blob_id(%d)", bid) } @@ -500,10 +500,10 @@ func TestAccessClientConnectionMode(t *testing.T) { if cs.size <= 0 { continue } - loc := access.Location{Size: uint64(mrand.Int63n(cs.size))} + loc := proto.Location{Size_: uint64(mrand.Int63n(cs.size))} fillCrc(&loc) _, err = cli.Delete(randCtx(), &access.DeleteArgs{ - Locations: []access.Location{loc}, + Locations: []proto.Location{loc}, }) require.NoError(t, err) } @@ -545,7 +545,7 @@ func TestAccessClientPutGet(t *testing.T) { } // test code 400 - _, err := client.Get(randCtx(), &access.GetArgs{Location: access.Location{Size: 100}, ReadSize: uint64(100)}) + _, err := client.Get(randCtx(), &access.GetArgs{Location: proto.Location{Size_: 100}, ReadSize: uint64(100)}) require.Error(t, err) } @@ -786,31 +786,31 @@ func TestAccessClientDelete(t *testing.T) { } { locs, err := client.Delete(randCtx(), &access.DeleteArgs{ - Locations: make([]access.Location, 1), + Locations: make([]proto.Location, 1), }) require.Nil(t, locs) require.NoError(t, err) } { args := &access.DeleteArgs{ - Locations: make([]access.Location, 1000), + Locations: make([]proto.Location, 1000), } _, err := client.Delete(randCtx(), args) require.NoError(t, err) } { args := &access.DeleteArgs{ - Locations: make([]access.Location, access.MaxDeleteLocations+1), + Locations: make([]proto.Location, access.MaxDeleteLocations+1), } locs, err := client.Delete(randCtx(), args) require.Equal(t, args.Locations, locs) require.ErrorIs(t, errcode.ErrIllegalArguments, err) } { - loc := access.Location{Size: 100, Blobs: make([]access.SliceInfo, 0)} + loc := proto.Location{Size_: 100, Slices: make([]proto.Slice, 0)} fillCrc(&loc) args := &access.DeleteArgs{ - Locations: make([]access.Location, 0, access.MaxDeleteLocations), + Locations: make([]proto.Location, 0, access.MaxDeleteLocations), } for i := 1; i < access.MaxDeleteLocations/10; i++ { args.Locations = append(args.Locations, loc) diff --git a/blobstore/api/access/proto.go b/blobstore/api/access/proto.go index a3dd95dbc..f05d02153 100644 --- a/blobstore/api/access/proto.go +++ b/blobstore/api/access/proto.go @@ -18,10 +18,7 @@ import ( "crypto/md5" "crypto/sha1" "crypto/sha256" - "encoding/base64" - "encoding/binary" "encoding/hex" - "fmt" "hash" "hash/crc32" "io" @@ -196,291 +193,6 @@ func (h HashSumMap) All() map[string]interface{} { return m } -// Location file location, 4 + 1 + 8 + 4 + 4 + len*16 bytes -// | | -// | ClusterID(4) | CodeMode(1) | -// | Size(8) | -// | BlobSize(4) | Crc(4) | -// | len*SliceInfo(16) | -// -// ClusterID which cluster file is in -// CodeMode is ec encode mode, see defined in "common/lib/codemode" -// Size is file size -// BlobSize is every blob's size but the last one which's size=(Size mod BlobSize) -// Crc is the checksum, change anything of the location, crc will mismatch -// Blobs all blob information -type Location struct { - _ [0]byte - ClusterID proto.ClusterID `json:"cluster_id"` - CodeMode codemode.CodeMode `json:"code_mode"` - Size uint64 `json:"size"` - BlobSize uint32 `json:"blob_size"` - Crc uint32 `json:"crc"` - Blobs []SliceInfo `json:"blobs"` -} - -// SliceInfo blobs info, 8 + 4 + 4 bytes -// -// MinBid is the first blob id -// Vid is which volume all blobs in -// Count is num of consecutive blob ids, count=1 just has one blob -// -// blob ids = [MinBid, MinBid+count) -type SliceInfo struct { - _ [0]byte - MinBid proto.BlobID `json:"min_bid"` - Vid proto.Vid `json:"vid"` - Count uint32 `json:"count"` -} - -// Blob is one piece of data in a location -// -// Bid is the blob id -// Vid is which volume the blob in -// Size is real size of the blob -type Blob struct { - Bid proto.BlobID - Vid proto.Vid - Size uint32 -} - -// Copy returns a new same Location -func (loc *Location) Copy() Location { - dst := Location{ - ClusterID: loc.ClusterID, - CodeMode: loc.CodeMode, - Size: loc.Size, - BlobSize: loc.BlobSize, - Crc: loc.Crc, - Blobs: make([]SliceInfo, len(loc.Blobs)), - } - copy(dst.Blobs, loc.Blobs) - return dst -} - -// Encode transfer Location to slice byte -// Returns the buf created by me -// -// (n) means max-n bytes -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -// | field | crc | clusterid | codemode | size | blobsize | -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -// | n-bytes | 4 | uvarint(5) | 1 | uvarint(10) | uvarint(5) | -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -// 25 + (5){len(blobs)} + len(Blobs) * 20 -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -// | blobs | minbid | vid | count | ... | -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -// | n-bytes | (10) | (5) | (5) | (20) | (20) | ... | -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -func (loc *Location) Encode() []byte { - if loc == nil { - return nil - } - n := 25 + 5 + len(loc.Blobs)*20 - buf := make([]byte, n) - n = loc.Encode2(buf) - return buf[:n] -} - -// Encode2 transfer Location to the buf, the buf reuse by yourself -// Returns the number of bytes read -// If the buffer is too small, Encode2 will panic -func (loc *Location) Encode2(buf []byte) int { - if loc == nil { - return 0 - } - - n := 0 - binary.BigEndian.PutUint32(buf[n:], loc.Crc) - n += 4 - n += binary.PutUvarint(buf[n:], uint64(loc.ClusterID)) - buf[n] = byte(loc.CodeMode) - n++ - n += binary.PutUvarint(buf[n:], uint64(loc.Size)) - n += binary.PutUvarint(buf[n:], uint64(loc.BlobSize)) - - n += binary.PutUvarint(buf[n:], uint64(len(loc.Blobs))) - for _, blob := range loc.Blobs { - n += binary.PutUvarint(buf[n:], uint64(blob.MinBid)) - n += binary.PutUvarint(buf[n:], uint64(blob.Vid)) - n += binary.PutUvarint(buf[n:], uint64(blob.Count)) - } - - return n -} - -// Decode parse location from buf -// Returns the number of bytes read -// Error is not nil when parsing failed -func (loc *Location) Decode(buf []byte) (int, error) { - if loc == nil { - return 0, fmt.Errorf("location receiver is nil") - } - - location, n, err := DecodeLocation(buf) - if err != nil { - return n, err - } - - *loc = location - return n, nil -} - -// ToString transfer location to hex string -func (loc *Location) ToString() string { - return loc.HexString() -} - -// HexString transfer location to hex string -func (loc *Location) HexString() string { - return hex.EncodeToString(loc.Encode()) -} - -// Base64String transfer location to base64 string -func (loc *Location) Base64String() string { - return base64.StdEncoding.EncodeToString(loc.Encode()) -} - -// Spread location blobs to slice -func (loc *Location) Spread() []Blob { - count := 0 - for _, blob := range loc.Blobs { - count += int(blob.Count) - } - - blobs := make([]Blob, 0, count) - for _, blob := range loc.Blobs { - for offset := uint32(0); offset < blob.Count; offset++ { - blobs = append(blobs, Blob{ - Bid: blob.MinBid + proto.BlobID(offset), - Vid: blob.Vid, - Size: loc.BlobSize, - }) - } - } - if len(blobs) > 0 && loc.BlobSize > 0 { - if lastSize := loc.Size % uint64(loc.BlobSize); lastSize > 0 { - blobs[len(blobs)-1].Size = uint32(lastSize) - } - } - return blobs -} - -// DecodeLocation parse location from buf -// Returns Location and the number of bytes read -// Error is not nil when parsing failed -func DecodeLocation(buf []byte) (Location, int, error) { - var ( - loc Location - n int - - val uint64 - nn int - ) - next := func() (uint64, int) { - val, nn := binary.Uvarint(buf) - if nn <= 0 { - return 0, nn - } - n += nn - buf = buf[nn:] - return val, nn - } - - if len(buf) < 4 { - return loc, n, fmt.Errorf("bytes crc %d", len(buf)) - } - loc.Crc = binary.BigEndian.Uint32(buf) - n += 4 - buf = buf[4:] - - if val, nn = next(); nn <= 0 { - return loc, n, fmt.Errorf("bytes cluster_id %d", nn) - } - loc.ClusterID = proto.ClusterID(val) - - if len(buf) < 1 { - return loc, n, fmt.Errorf("bytes codemode %d", len(buf)) - } - loc.CodeMode = codemode.CodeMode(buf[0]) - n++ - buf = buf[1:] - - if val, nn = next(); nn <= 0 { - return loc, n, fmt.Errorf("bytes size %d", nn) - } - loc.Size = val - - if val, nn = next(); nn <= 0 { - return loc, n, fmt.Errorf("bytes blob_size %d", nn) - } - loc.BlobSize = uint32(val) - - if val, nn = next(); nn <= 0 { - return loc, n, fmt.Errorf("bytes length blobs %d", nn) - } - length := int(val) - - if length > 0 { - loc.Blobs = make([]SliceInfo, 0, length) - } - for index := 0; index < length; index++ { - var blob SliceInfo - if val, nn = next(); nn <= 0 { - return loc, n, fmt.Errorf("bytes %dth-blob min_bid %d", index, nn) - } - blob.MinBid = proto.BlobID(val) - - if val, nn = next(); nn <= 0 { - return loc, n, fmt.Errorf("bytes %dth-blob vid %d", index, nn) - } - blob.Vid = proto.Vid(val) - - if val, nn = next(); nn <= 0 { - return loc, n, fmt.Errorf("bytes %dth-blob count %d", index, nn) - } - blob.Count = uint32(val) - - loc.Blobs = append(loc.Blobs, blob) - } - - return loc, n, nil -} - -// DecodeLocationFrom decode location from hex string -func DecodeLocationFrom(s string) (Location, error) { - return DecodeLocationFromHex(s) -} - -// DecodeLocationFromHex decode location from hex string -func DecodeLocationFromHex(s string) (Location, error) { - var loc Location - src, err := hex.DecodeString(s) - if err != nil { - return loc, err - } - _, err = loc.Decode(src) - if err != nil { - return loc, err - } - return loc, nil -} - -// DecodeLocationFromBase64 decode location from base64 string -func DecodeLocationFromBase64(s string) (Location, error) { - var loc Location - src, err := base64.StdEncoding.DecodeString(s) - if err != nil { - return loc, err - } - _, err = loc.Decode(src) - if err != nil { - return loc, err - } - return loc, nil -} - // PutArgs for service /put // Hashes means how to calculate check sum, // HashAlgCRC32 | HashAlgMD5 equal 2 + 4 = 6 @@ -507,8 +219,8 @@ func (args *PutArgs) IsValid() bool { // PutResp put response result type PutResp struct { - Location Location `json:"location"` - HashSumMap HashSumMap `json:"hashsum"` + Location proto.Location `json:"location"` + HashSumMap HashSumMap `json:"hashsum"` } // PutAtArgs for service /putat @@ -562,16 +274,16 @@ func (args *AllocArgs) IsValid() bool { // if size mod blobsize == 0, length of tokens equal length of location blobs // otherwise additional token for the last blob uploading type AllocResp struct { - Location Location `json:"location"` - Tokens []string `json:"tokens"` + Location proto.Location `json:"location"` + Tokens []string `json:"tokens"` } // GetArgs for service /get type GetArgs struct { - Location Location `json:"location"` - Offset uint64 `json:"offset"` - ReadSize uint64 `json:"read_size"` - Writer io.Writer `json:"-"` + Location proto.Location `json:"location"` + Offset uint64 `json:"offset"` + ReadSize uint64 `json:"read_size"` + Writer io.Writer `json:"-"` } // IsValid is valid get args @@ -579,14 +291,14 @@ func (args *GetArgs) IsValid() bool { if args == nil { return false } - return args.Offset <= args.Location.Size && - args.ReadSize <= args.Location.Size && - args.Offset+args.ReadSize <= args.Location.Size + return args.Offset <= args.Location.Size_ && + args.ReadSize <= args.Location.Size_ && + args.Offset+args.ReadSize <= args.Location.Size_ } // DeleteArgs for service /delete type DeleteArgs struct { - Locations []Location `json:"locations"` + Locations []proto.Location `json:"locations"` } // IsValid is valid delete args @@ -599,7 +311,7 @@ func (args *DeleteArgs) IsValid() bool { // DeleteResp delete response with failed locations type DeleteResp struct { - FailedLocations []Location `json:"failed_locations,omitempty"` + FailedLocations []proto.Location `json:"failed_locations,omitempty"` } // DeleteBlobArgs for service /deleteblob @@ -626,8 +338,8 @@ func (args *DeleteBlobArgs) IsValid() bool { // Locations are signed location getting from /alloc // Location is to be signed location which merged by yourself type SignArgs struct { - Locations []Location `json:"locations"` - Location Location `json:"location"` + Locations []proto.Location `json:"locations"` + Location proto.Location `json:"location"` } // IsValid is valid sign args @@ -640,5 +352,5 @@ func (args *SignArgs) IsValid() bool { // SignResp sign response location with crc type SignResp struct { - Location Location `json:"location"` + Location proto.Location `json:"location"` } diff --git a/blobstore/api/access/proto_test.go b/blobstore/api/access/proto_test.go index fba17c8eb..4227f5844 100644 --- a/blobstore/api/access/proto_test.go +++ b/blobstore/api/access/proto_test.go @@ -259,192 +259,6 @@ func TestHashAlgorithm2HashSumMap(t *testing.T) { } } -func TestLocationEncodeDecodeNil(t *testing.T) { - var loc *access.Location - require.Nil(t, loc.Encode()) - require.Equal(t, 0, loc.Encode2(nil)) - require.Equal(t, "", loc.ToString()) - require.Equal(t, "", loc.HexString()) - require.Equal(t, "", loc.Base64String()) - - n, err := loc.Decode(nil) - require.Error(t, err) - require.Equal(t, 0, n) - - locx, n, err := access.DecodeLocation(nil) - require.Error(t, err) - require.Equal(t, 0, n) - require.Equal(t, access.Location{}, locx) - - locx, err = access.DecodeLocationFrom("") - require.Error(t, err) - require.Equal(t, access.Location{}, locx) - - locx, err = access.DecodeLocationFrom("xxx") - require.Error(t, err) - require.Equal(t, access.Location{}, locx) - - locx, err = access.DecodeLocationFromHex("xxx") - require.Error(t, err) - require.Equal(t, access.Location{}, locx) - - locx, err = access.DecodeLocationFromBase64("xxx") - require.Error(t, err) - require.Equal(t, access.Location{}, locx) -} - -func TestLocationEncodeDecode(t *testing.T) { - for ii := 0; ii < 100; ii++ { - loc := &access.Location{ - ClusterID: proto.ClusterID(mrand.Uint32()), - CodeMode: codemode.CodeMode(mrand.Intn(0xff)), - Size: mrand.Uint64(), - BlobSize: mrand.Uint32(), - Crc: mrand.Uint32(), - } - - num := mrand.Intn(5) - for i := 0; i < num; i++ { - loc.Blobs = append(loc.Blobs, access.SliceInfo{ - MinBid: proto.BlobID(mrand.Uint64()), - Vid: proto.Vid(mrand.Uint32()), - Count: mrand.Uint32(), - }) - } - - buf := loc.Encode() - bufx := make([]byte, len(buf)) - n := loc.Encode2(bufx) - require.Equal(t, len(buf), n) - require.Equal(t, buf, bufx) - - require.Panics(t, func() { loc.Encode2(nil) }) - require.Panics(t, func() { loc.Encode2(bufx[:3]) }) - require.Panics(t, func() { loc.Encode2(bufx[:n/2]) }) - require.Panics(t, func() { loc.Encode2(bufx[:n-1]) }) - - locx := access.Location{} - locx.Decode(bufx) - require.Equal(t, loc.ToString(), locx.ToString()) - require.Equal(t, loc.HexString(), locx.HexString()) - require.Equal(t, loc.Base64String(), locx.Base64String()) - - str := loc.ToString() - locx, err := access.DecodeLocationFrom(str) - require.NoError(t, err) - require.Equal(t, *loc, locx) - - str = loc.HexString() - locx, err = access.DecodeLocationFromHex(str) - require.NoError(t, err) - require.Equal(t, *loc, locx) - - str = loc.Base64String() - locx, err = access.DecodeLocationFromBase64(str) - require.NoError(t, err) - require.Equal(t, *loc, locx) - } -} - -func TestLocationDecodeError(t *testing.T) { - loc := &access.Location{ - ClusterID: proto.ClusterID(math.MaxUint32), - CodeMode: codemode.CodeMode(math.MaxInt8), - Size: math.MaxUint64, - BlobSize: math.MaxUint32, - Crc: math.MaxUint32, - } - - buf := loc.Encode() - require.Equal(t, 25+1, len(buf)) - for _, n := range []int{3, 8, 9, 19, 24} { - _, _, err := access.DecodeLocation(buf[:n]) - require.Error(t, err) - t.Log(err) - } - - loc.Blobs = append(loc.Blobs, access.SliceInfo{ - MinBid: proto.BlobID(math.MaxUint64), - Vid: proto.Vid(math.MaxUint32), - Count: math.MaxUint32, - }) - - buf = loc.Encode() - require.Equal(t, 25+1+20, len(buf)) - for _, n := range []int{25, 35, 40, 45} { - _, _, err := access.DecodeLocation(buf[:n]) - require.Error(t, err) - t.Log(err) - } -} - -func TestLocationSpread(t *testing.T) { - { - var loc access.Location - blobs := loc.Spread() - require.NotNil(t, blobs) - require.Equal(t, 0, len(blobs)) - } - { - loc := &access.Location{ - Size: 10, - BlobSize: 1 << 22, - Blobs: []access.SliceInfo{{ - MinBid: 100, - Vid: 4, - Count: 1, - }}, - } - blobs := loc.Spread() - require.Equal(t, 1, len(blobs)) - require.Equal(t, proto.BlobID(100), blobs[0].Bid) - require.Equal(t, proto.Vid(4), blobs[0].Vid) - require.Equal(t, uint32(10), blobs[0].Size) - } - { - loc := &access.Location{ - Size: (1 << 22) + 10, - BlobSize: 1 << 22, - Blobs: []access.SliceInfo{{ - MinBid: 100, - Vid: 4, - Count: 2, - }}, - } - blobs := loc.Spread() - require.Equal(t, 2, len(blobs)) - require.Equal(t, proto.BlobID(100), blobs[0].Bid) - require.Equal(t, proto.Vid(4), blobs[0].Vid) - require.Equal(t, uint32(1<<22), blobs[0].Size) - require.Equal(t, proto.BlobID(101), blobs[1].Bid) - require.Equal(t, proto.Vid(4), blobs[1].Vid) - require.Equal(t, uint32(10), blobs[1].Size) - } - { - loc := &access.Location{ - Size: 1 << 23, - BlobSize: 1 << 22, - Blobs: []access.SliceInfo{{ - MinBid: 100, - Vid: 4, - Count: 1, - }, { - MinBid: 200, - Vid: 4, - Count: 1, - }}, - } - blobs := loc.Spread() - require.Equal(t, 2, len(blobs)) - require.Equal(t, proto.BlobID(100), blobs[0].Bid) - require.Equal(t, proto.Vid(4), blobs[0].Vid) - require.Equal(t, uint32(1<<22), blobs[0].Size) - require.Equal(t, proto.BlobID(200), blobs[1].Bid) - require.Equal(t, proto.Vid(4), blobs[1].Vid) - require.Equal(t, uint32(1<<22), blobs[1].Size) - } -} - func TestPutArgs(t *testing.T) { cases := []struct { size int64 @@ -538,7 +352,7 @@ func TestGetArgs(t *testing.T) { Offset: cs.offset, ReadSize: cs.readSize, } - args.Location.Size = cs.size + args.Location.Size_ = cs.size require.Equal(t, cs.valid, args.IsValid()) } } @@ -547,11 +361,11 @@ func TestDeleteArgs(t *testing.T) { args := access.DeleteArgs{} require.False(t, args.IsValid()) require.False(t, (*access.DeleteArgs)(nil).IsValid()) - args.Locations = []access.Location{{}} + args.Locations = []proto.Location{{}} require.True(t, args.IsValid()) - args.Locations = make([]access.Location, access.MaxDeleteLocations) + args.Locations = make([]proto.Location, access.MaxDeleteLocations) require.True(t, args.IsValid()) - args.Locations = make([]access.Location, access.MaxDeleteLocations+1) + args.Locations = make([]proto.Location, access.MaxDeleteLocations+1) require.False(t, args.IsValid()) } @@ -573,7 +387,7 @@ func TestSignArgs(t *testing.T) { args := access.SignArgs{} require.False(t, args.IsValid()) require.False(t, (*access.DeleteArgs)(nil).IsValid()) - args.Locations = []access.Location{{}} + args.Locations = []proto.Location{{}} require.True(t, args.IsValid()) } diff --git a/blobstore/api/shardnode/shardnode.pb.go b/blobstore/api/shardnode/shardnode.pb.go index e50235ab3..77817e44f 100644 --- a/blobstore/api/shardnode/shardnode.pb.go +++ b/blobstore/api/shardnode/shardnode.pb.go @@ -6,7 +6,9 @@ package shardnode import ( fmt "fmt" clustermgr "github.com/cubefs/cubefs/blobstore/api/clustermgr" + github_com_cubefs_cubefs_blobstore_common_codemode "github.com/cubefs/cubefs/blobstore/common/codemode" github_com_cubefs_cubefs_blobstore_common_proto "github.com/cubefs/cubefs/blobstore/common/proto" + proto1 "github.com/cubefs/cubefs/blobstore/common/proto" sharding "github.com/cubefs/cubefs/blobstore/common/sharding" _ "github.com/gogo/protobuf/gogoproto" proto "github.com/gogo/protobuf/proto" @@ -1017,10 +1019,10 @@ func (m *GetShardRequest) GetSuid() github_com_cubefs_cubefs_blobstore_common_pr } type GetShardResponse struct { - Shard clustermgr.Shard `protobuf:"bytes,1,opt,name=shard,proto3" json:"shard"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Shard clustermgr.ShardUnitInfo `protobuf:"bytes,1,opt,name=shard,proto3" json:"shard"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *GetShardResponse) Reset() { *m = GetShardResponse{} } @@ -1056,11 +1058,538 @@ func (m *GetShardResponse) XXX_DiscardUnknown() { var xxx_messageInfo_GetShardResponse proto.InternalMessageInfo -func (m *GetShardResponse) GetShard() clustermgr.Shard { +func (m *GetShardResponse) GetShard() clustermgr.ShardUnitInfo { if m != nil { return m.Shard } - return clustermgr.Shard{} + return clustermgr.ShardUnitInfo{} +} + +type CreateBlobRequest struct { + Header ShardOpHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header"` + Name []byte `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + CodeMode github_com_cubefs_cubefs_blobstore_common_codemode.CodeMode `protobuf:"varint,3,opt,name=codemode,proto3,casttype=github.com/cubefs/cubefs/blobstore/common/codemode.CodeMode" json:"codemode,omitempty"` + Size_ uint64 `protobuf:"varint,4,opt,name=size,proto3" json:"size,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CreateBlobRequest) Reset() { *m = CreateBlobRequest{} } +func (m *CreateBlobRequest) String() string { return proto.CompactTextString(m) } +func (*CreateBlobRequest) ProtoMessage() {} +func (*CreateBlobRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9d3815ca0e5f30f0, []int{19} +} +func (m *CreateBlobRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *CreateBlobRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_CreateBlobRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *CreateBlobRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateBlobRequest.Merge(m, src) +} +func (m *CreateBlobRequest) XXX_Size() int { + return m.Size() +} +func (m *CreateBlobRequest) XXX_DiscardUnknown() { + xxx_messageInfo_CreateBlobRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_CreateBlobRequest proto.InternalMessageInfo + +func (m *CreateBlobRequest) GetHeader() ShardOpHeader { + if m != nil { + return m.Header + } + return ShardOpHeader{} +} + +func (m *CreateBlobRequest) GetName() []byte { + if m != nil { + return m.Name + } + return nil +} + +func (m *CreateBlobRequest) GetCodeMode() github_com_cubefs_cubefs_blobstore_common_codemode.CodeMode { + if m != nil { + return m.CodeMode + } + return 0 +} + +func (m *CreateBlobRequest) GetSize_() uint64 { + if m != nil { + return m.Size_ + } + return 0 +} + +type CreateBlobResponse struct { + Blob proto1.Blob `protobuf:"bytes,1,opt,name=blob,proto3" json:"blob"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CreateBlobResponse) Reset() { *m = CreateBlobResponse{} } +func (m *CreateBlobResponse) String() string { return proto.CompactTextString(m) } +func (*CreateBlobResponse) ProtoMessage() {} +func (*CreateBlobResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9d3815ca0e5f30f0, []int{20} +} +func (m *CreateBlobResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *CreateBlobResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_CreateBlobResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *CreateBlobResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateBlobResponse.Merge(m, src) +} +func (m *CreateBlobResponse) XXX_Size() int { + return m.Size() +} +func (m *CreateBlobResponse) XXX_DiscardUnknown() { + xxx_messageInfo_CreateBlobResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_CreateBlobResponse proto.InternalMessageInfo + +func (m *CreateBlobResponse) GetBlob() proto1.Blob { + if m != nil { + return m.Blob + } + return proto1.Blob{} +} + +type GetBlobRequest struct { + Header ShardOpHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header"` + Name []byte `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetBlobRequest) Reset() { *m = GetBlobRequest{} } +func (m *GetBlobRequest) String() string { return proto.CompactTextString(m) } +func (*GetBlobRequest) ProtoMessage() {} +func (*GetBlobRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9d3815ca0e5f30f0, []int{21} +} +func (m *GetBlobRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *GetBlobRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_GetBlobRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *GetBlobRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetBlobRequest.Merge(m, src) +} +func (m *GetBlobRequest) XXX_Size() int { + return m.Size() +} +func (m *GetBlobRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GetBlobRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_GetBlobRequest proto.InternalMessageInfo + +func (m *GetBlobRequest) GetHeader() ShardOpHeader { + if m != nil { + return m.Header + } + return ShardOpHeader{} +} + +func (m *GetBlobRequest) GetName() []byte { + if m != nil { + return m.Name + } + return nil +} + +type GetBlobResponse struct { + Blob proto1.Blob `protobuf:"bytes,1,opt,name=blob,proto3" json:"blob"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetBlobResponse) Reset() { *m = GetBlobResponse{} } +func (m *GetBlobResponse) String() string { return proto.CompactTextString(m) } +func (*GetBlobResponse) ProtoMessage() {} +func (*GetBlobResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9d3815ca0e5f30f0, []int{22} +} +func (m *GetBlobResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *GetBlobResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_GetBlobResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *GetBlobResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetBlobResponse.Merge(m, src) +} +func (m *GetBlobResponse) XXX_Size() int { + return m.Size() +} +func (m *GetBlobResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetBlobResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetBlobResponse proto.InternalMessageInfo + +func (m *GetBlobResponse) GetBlob() proto1.Blob { + if m != nil { + return m.Blob + } + return proto1.Blob{} +} + +type ListBlobRequest struct { + Header ShardOpHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header"` + Prefix []byte `protobuf:"bytes,2,opt,name=prefix,proto3" json:"prefix,omitempty"` + Marker []byte `protobuf:"bytes,3,opt,name=marker,proto3" json:"marker,omitempty"` + Count uint64 `protobuf:"varint,4,opt,name=count,proto3" json:"count,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ListBlobRequest) Reset() { *m = ListBlobRequest{} } +func (m *ListBlobRequest) String() string { return proto.CompactTextString(m) } +func (*ListBlobRequest) ProtoMessage() {} +func (*ListBlobRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9d3815ca0e5f30f0, []int{23} +} +func (m *ListBlobRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ListBlobRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ListBlobRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ListBlobRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListBlobRequest.Merge(m, src) +} +func (m *ListBlobRequest) XXX_Size() int { + return m.Size() +} +func (m *ListBlobRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ListBlobRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ListBlobRequest proto.InternalMessageInfo + +func (m *ListBlobRequest) GetHeader() ShardOpHeader { + if m != nil { + return m.Header + } + return ShardOpHeader{} +} + +func (m *ListBlobRequest) GetPrefix() []byte { + if m != nil { + return m.Prefix + } + return nil +} + +func (m *ListBlobRequest) GetMarker() []byte { + if m != nil { + return m.Marker + } + return nil +} + +func (m *ListBlobRequest) GetCount() uint64 { + if m != nil { + return m.Count + } + return 0 +} + +type ListBlobResponse struct { + Blobs []proto1.Blob `protobuf:"bytes,1,rep,name=blobs,proto3" json:"blobs"` + NextMarker []byte `protobuf:"bytes,2,opt,name=nextMarker,proto3" json:"nextMarker,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ListBlobResponse) Reset() { *m = ListBlobResponse{} } +func (m *ListBlobResponse) String() string { return proto.CompactTextString(m) } +func (*ListBlobResponse) ProtoMessage() {} +func (*ListBlobResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9d3815ca0e5f30f0, []int{24} +} +func (m *ListBlobResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ListBlobResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ListBlobResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ListBlobResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListBlobResponse.Merge(m, src) +} +func (m *ListBlobResponse) XXX_Size() int { + return m.Size() +} +func (m *ListBlobResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ListBlobResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ListBlobResponse proto.InternalMessageInfo + +func (m *ListBlobResponse) GetBlobs() []proto1.Blob { + if m != nil { + return m.Blobs + } + return nil +} + +func (m *ListBlobResponse) GetNextMarker() []byte { + if m != nil { + return m.NextMarker + } + return nil +} + +type DeleteBlobRequest struct { + Header ShardOpHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header"` + Name []byte `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DeleteBlobRequest) Reset() { *m = DeleteBlobRequest{} } +func (m *DeleteBlobRequest) String() string { return proto.CompactTextString(m) } +func (*DeleteBlobRequest) ProtoMessage() {} +func (*DeleteBlobRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9d3815ca0e5f30f0, []int{25} +} +func (m *DeleteBlobRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *DeleteBlobRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_DeleteBlobRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *DeleteBlobRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteBlobRequest.Merge(m, src) +} +func (m *DeleteBlobRequest) XXX_Size() int { + return m.Size() +} +func (m *DeleteBlobRequest) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteBlobRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteBlobRequest proto.InternalMessageInfo + +func (m *DeleteBlobRequest) GetHeader() ShardOpHeader { + if m != nil { + return m.Header + } + return ShardOpHeader{} +} + +func (m *DeleteBlobRequest) GetName() []byte { + if m != nil { + return m.Name + } + return nil +} + +type RetainBlobRequest struct { + Header ShardOpHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header"` + Name []byte `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Lease uint64 `protobuf:"varint,3,opt,name=lease,proto3" json:"lease,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *RetainBlobRequest) Reset() { *m = RetainBlobRequest{} } +func (m *RetainBlobRequest) String() string { return proto.CompactTextString(m) } +func (*RetainBlobRequest) ProtoMessage() {} +func (*RetainBlobRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9d3815ca0e5f30f0, []int{26} +} +func (m *RetainBlobRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *RetainBlobRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_RetainBlobRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *RetainBlobRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_RetainBlobRequest.Merge(m, src) +} +func (m *RetainBlobRequest) XXX_Size() int { + return m.Size() +} +func (m *RetainBlobRequest) XXX_DiscardUnknown() { + xxx_messageInfo_RetainBlobRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_RetainBlobRequest proto.InternalMessageInfo + +func (m *RetainBlobRequest) GetHeader() ShardOpHeader { + if m != nil { + return m.Header + } + return ShardOpHeader{} +} + +func (m *RetainBlobRequest) GetName() []byte { + if m != nil { + return m.Name + } + return nil +} + +func (m *RetainBlobRequest) GetLease() uint64 { + if m != nil { + return m.Lease + } + return 0 +} + +type SealBlobRequest struct { + Header ShardOpHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header"` + Size_ uint64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` + Name []byte `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SealBlobRequest) Reset() { *m = SealBlobRequest{} } +func (m *SealBlobRequest) String() string { return proto.CompactTextString(m) } +func (*SealBlobRequest) ProtoMessage() {} +func (*SealBlobRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9d3815ca0e5f30f0, []int{27} +} +func (m *SealBlobRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SealBlobRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SealBlobRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SealBlobRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_SealBlobRequest.Merge(m, src) +} +func (m *SealBlobRequest) XXX_Size() int { + return m.Size() +} +func (m *SealBlobRequest) XXX_DiscardUnknown() { + xxx_messageInfo_SealBlobRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_SealBlobRequest proto.InternalMessageInfo + +func (m *SealBlobRequest) GetHeader() ShardOpHeader { + if m != nil { + return m.Header + } + return ShardOpHeader{} +} + +func (m *SealBlobRequest) GetSize_() uint64 { + if m != nil { + return m.Size_ + } + return 0 +} + +func (m *SealBlobRequest) GetName() []byte { + if m != nil { + return m.Name + } + return nil } func init() { @@ -1083,64 +1612,86 @@ func init() { proto.RegisterType((*UpdateShardResponse)(nil), "cubefs.blobstore.api.shardnode.UpdateShardResponse") proto.RegisterType((*GetShardRequest)(nil), "cubefs.blobstore.api.shardnode.GetShardRequest") proto.RegisterType((*GetShardResponse)(nil), "cubefs.blobstore.api.shardnode.GetShardResponse") + proto.RegisterType((*CreateBlobRequest)(nil), "cubefs.blobstore.api.shardnode.CreateBlobRequest") + proto.RegisterType((*CreateBlobResponse)(nil), "cubefs.blobstore.api.shardnode.CreateBlobResponse") + proto.RegisterType((*GetBlobRequest)(nil), "cubefs.blobstore.api.shardnode.GetBlobRequest") + proto.RegisterType((*GetBlobResponse)(nil), "cubefs.blobstore.api.shardnode.GetBlobResponse") + proto.RegisterType((*ListBlobRequest)(nil), "cubefs.blobstore.api.shardnode.ListBlobRequest") + proto.RegisterType((*ListBlobResponse)(nil), "cubefs.blobstore.api.shardnode.ListBlobResponse") + proto.RegisterType((*DeleteBlobRequest)(nil), "cubefs.blobstore.api.shardnode.DeleteBlobRequest") + proto.RegisterType((*RetainBlobRequest)(nil), "cubefs.blobstore.api.shardnode.RetainBlobRequest") + proto.RegisterType((*SealBlobRequest)(nil), "cubefs.blobstore.api.shardnode.SealBlobRequest") } func init() { proto.RegisterFile("shardnode.proto", fileDescriptor_9d3815ca0e5f30f0) } var fileDescriptor_9d3815ca0e5f30f0 = []byte{ - // 830 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x55, 0xdd, 0x6e, 0xe3, 0x44, - 0x14, 0xc6, 0x8e, 0xe3, 0xc2, 0x69, 0x4a, 0xba, 0x43, 0x41, 0x56, 0x25, 0xe2, 0xca, 0xfc, 0x55, - 0xfc, 0xd8, 0x52, 0x41, 0x70, 0x81, 0x04, 0xbb, 0xd9, 0xa8, 0x10, 0x2d, 0x08, 0xad, 0xbb, 0xdb, - 0x0b, 0x24, 0x54, 0x39, 0xf6, 0x34, 0x1d, 0x25, 0xb1, 0x8d, 0x67, 0xbc, 0x6a, 0xef, 0x78, 0x03, - 0x24, 0x9e, 0x80, 0x0b, 0x9e, 0x05, 0xed, 0x25, 0x4f, 0x60, 0x21, 0xdf, 0xf1, 0x0a, 0xbd, 0x42, - 0x73, 0x66, 0x9c, 0x9f, 0x2e, 0xb0, 0xa4, 0xa2, 0x68, 0xf7, 0x2e, 0x73, 0x72, 0xbe, 0x73, 0xbe, - 0xf3, 0x9d, 0xf1, 0x37, 0xd0, 0xe5, 0x67, 0x51, 0x91, 0xa4, 0x59, 0x42, 0xfd, 0xbc, 0xc8, 0x44, - 0x46, 0x7a, 0x71, 0x39, 0xa2, 0xa7, 0xdc, 0x1f, 0x4d, 0xb3, 0x11, 0x17, 0x59, 0x41, 0xfd, 0x28, - 0x67, 0xfe, 0x3c, 0x6b, 0x77, 0x67, 0x9c, 0x8d, 0x33, 0x4c, 0x0d, 0xe4, 0x2f, 0x85, 0xda, 0x7d, - 0x5f, 0xa1, 0x82, 0x39, 0x2a, 0x88, 0xb3, 0xd9, 0x2c, 0x4b, 0x03, 0x04, 0xb2, 0x74, 0x1c, 0x14, - 0x51, 0x3a, 0xd6, 0x3d, 0x76, 0xdf, 0x7b, 0x22, 0x3b, 0xca, 0x59, 0x10, 0x4f, 0x4b, 0x2e, 0x68, - 0x31, 0x1b, 0x17, 0x0a, 0xa5, 0x92, 0xbd, 0x18, 0xac, 0xa1, 0xa0, 0x33, 0xf2, 0x1a, 0x98, 0x2c, - 0x71, 0x8c, 0x3d, 0x63, 0xbf, 0xd3, 0xb7, 0xeb, 0xca, 0x35, 0x87, 0x83, 0xd0, 0x64, 0x09, 0xb9, - 0x0b, 0xf6, 0x29, 0xa3, 0xd3, 0x84, 0x3b, 0xe6, 0x5e, 0x6b, 0x7f, 0xf3, 0xe0, 0x2d, 0xff, 0x9f, - 0x27, 0xf0, 0x0f, 0x65, 0x76, 0xdf, 0x7a, 0x5c, 0xb9, 0x2f, 0x84, 0x1a, 0xea, 0xe5, 0xd0, 0xc6, - 0x30, 0xb9, 0x3f, 0xef, 0xb2, 0xd5, 0xbf, 0xa3, 0xba, 0x5c, 0x56, 0xee, 0x27, 0x63, 0x26, 0xce, - 0xca, 0x91, 0x1f, 0x67, 0xb3, 0x40, 0x73, 0xff, 0xbb, 0x81, 0x91, 0xb4, 0xea, 0xa1, 0x09, 0xee, - 0x40, 0xfb, 0x51, 0x34, 0x2d, 0xa9, 0x63, 0x4a, 0xee, 0xa1, 0x3a, 0x78, 0x3f, 0xb6, 0x60, 0xeb, - 0x48, 0x72, 0xfa, 0x26, 0xff, 0x92, 0x46, 0x09, 0x2d, 0x48, 0x04, 0x2f, 0xf2, 0x3c, 0x8a, 0xe9, - 0x89, 0x26, 0x60, 0xf5, 0x0f, 0xeb, 0xca, 0xdd, 0x38, 0x92, 0xb1, 0xeb, 0xb1, 0xd0, 0xd0, 0x70, - 0x03, 0xeb, 0x0e, 0x13, 0xf2, 0x1d, 0x6c, 0x24, 0x8c, 0x4f, 0x64, 0x07, 0x13, 0x47, 0x1c, 0xd4, - 0x95, 0x6b, 0x0f, 0x18, 0x9f, 0x60, 0x83, 0x8f, 0xd7, 0x6d, 0xa0, 0x90, 0xa1, 0x2d, 0x8b, 0x0e, - 0x13, 0xf2, 0x00, 0x2c, 0x5e, 0xb2, 0xc4, 0x69, 0x61, 0xed, 0xdb, 0x75, 0xe5, 0x5a, 0x47, 0x25, - 0x4b, 0x2e, 0x2b, 0xf7, 0xa3, 0xb5, 0xa9, 0x97, 0x2c, 0x09, 0xb1, 0x1a, 0xf1, 0xa0, 0x83, 0xfc, - 0x8f, 0x69, 0xc1, 0x59, 0x96, 0x3a, 0x96, 0xd4, 0x26, 0x5c, 0x89, 0x91, 0x37, 0x60, 0xab, 0xc8, - 0x4a, 0x41, 0x4f, 0x1e, 0xe9, 0xa4, 0xb6, 0x4a, 0xc2, 0x60, 0x93, 0xf4, 0x3a, 0x00, 0xde, 0x82, - 0x93, 0x09, 0xbd, 0xe0, 0x8e, 0xbd, 0xd7, 0xda, 0xef, 0x84, 0x2f, 0x61, 0xe4, 0x1e, 0xbd, 0xe0, - 0xde, 0xcf, 0x06, 0xdc, 0x1a, 0xa6, 0x9c, 0x16, 0x42, 0xde, 0xb7, 0x90, 0x7e, 0x5f, 0x52, 0x2e, - 0xc8, 0x3d, 0xb0, 0xcf, 0x70, 0x3f, 0xb8, 0x93, 0xcd, 0x83, 0x0f, 0x9e, 0x76, 0xbd, 0x56, 0x96, - 0xda, 0x5c, 0x33, 0x55, 0x82, 0x7c, 0x06, 0x16, 0x13, 0x74, 0x86, 0xe2, 0x6f, 0x1e, 0xbc, 0xf9, - 0xb4, 0x52, 0x92, 0x87, 0xae, 0x80, 0x38, 0x6f, 0x07, 0xc8, 0x32, 0x43, 0x9e, 0x67, 0x29, 0xa7, - 0x48, 0xfc, 0x61, 0x9e, 0x44, 0x82, 0x3e, 0xcb, 0xc4, 0x97, 0x19, 0x6a, 0xe2, 0xe7, 0x70, 0x6b, - 0x40, 0xa7, 0xf4, 0x06, 0x79, 0x2b, 0xd3, 0x30, 0xaf, 0x9a, 0x86, 0xe4, 0xb3, 0xdc, 0x59, 0xf3, - 0x29, 0xe1, 0xe5, 0x2f, 0xa8, 0xf8, 0xdf, 0xc9, 0xdc, 0x87, 0xee, 0xbc, 0xad, 0x62, 0x32, 0xd7, - 0xdb, 0xb8, 0xa6, 0xde, 0xbf, 0x18, 0xd0, 0xfd, 0x8a, 0xf1, 0x9b, 0x9c, 0xc5, 0xce, 0x0b, 0x7a, - 0xca, 0xce, 0xb5, 0xab, 0xe9, 0x93, 0x8c, 0xcf, 0xa2, 0x62, 0x42, 0x0b, 0x34, 0x81, 0x4e, 0xa8, - 0x4f, 0xd2, 0x04, 0xe3, 0xac, 0x4c, 0x85, 0xfe, 0x7a, 0xd5, 0xc1, 0x13, 0xb0, 0xbd, 0x60, 0xa9, - 0x47, 0xbf, 0x0d, 0x6d, 0x39, 0x02, 0x77, 0x0c, 0xb4, 0xf3, 0x75, 0x66, 0x57, 0x40, 0xd2, 0x03, - 0x48, 0xe9, 0xb9, 0xf8, 0x5a, 0xf1, 0x50, 0xfc, 0x96, 0x22, 0xde, 0x1f, 0x26, 0x74, 0xef, 0x24, - 0x09, 0x8e, 0xd7, 0x88, 0xb3, 0xe4, 0x8c, 0xc6, 0x0d, 0x3a, 0xa3, 0x89, 0xbe, 0xfe, 0x5f, 0x39, - 0xe3, 0x5d, 0x68, 0xe3, 0xb3, 0x8a, 0x5a, 0x6f, 0x1e, 0xbc, 0xf3, 0xa4, 0x54, 0x0a, 0xe9, 0x37, - 0xaf, 0xb0, 0x1f, 0xca, 0xf4, 0x46, 0x2d, 0xc4, 0x92, 0x43, 0x68, 0x97, 0x29, 0x13, 0xdc, 0xb1, - 0x50, 0xef, 0x77, 0xff, 0x5a, 0xef, 0xc5, 0xe3, 0xac, 0xae, 0xc5, 0xc3, 0x94, 0x89, 0xa6, 0x0e, - 0xc2, 0xe5, 0x86, 0x69, 0x9e, 0xc5, 0x67, 0xda, 0x7a, 0xd5, 0xc1, 0x23, 0xb0, 0xbd, 0x90, 0x5a, - 0x7f, 0x66, 0x3f, 0xb5, 0x1a, 0x37, 0x78, 0xfe, 0x57, 0xf0, 0x83, 0x01, 0x5d, 0x25, 0x08, 0x0e, - 0xf4, 0xe0, 0x22, 0xa7, 0xfa, 0xf9, 0x3b, 0xae, 0x2b, 0xf7, 0xea, 0x5f, 0x97, 0x95, 0xfb, 0xf9, - 0xda, 0xcd, 0x56, 0x4b, 0x84, 0x57, 0x6b, 0x92, 0x01, 0x58, 0x72, 0x03, 0xf8, 0x65, 0x5d, 0x67, - 0x7f, 0x88, 0xf6, 0x5e, 0x85, 0x57, 0x56, 0x76, 0xa2, 0x77, 0xf5, 0xab, 0x81, 0xe6, 0xf4, 0xdc, - 0x2f, 0xca, 0x3b, 0x86, 0xed, 0xc5, 0x1c, 0xda, 0x6a, 0xfa, 0xd0, 0xc6, 0x2f, 0x43, 0x1b, 0xe2, - 0xdb, 0xff, 0x4e, 0xba, 0xe6, 0xda, 0x23, 0xb4, 0xbf, 0xfb, 0xb8, 0xee, 0x19, 0xbf, 0xd5, 0x3d, - 0xe3, 0xf7, 0xba, 0x67, 0x7c, 0xdb, 0xf1, 0x83, 0x4f, 0xe7, 0xd6, 0x34, 0xb2, 0x91, 0xc5, 0x87, - 0x7f, 0x06, 0x00, 0x00, 0xff, 0xff, 0xc8, 0x79, 0xde, 0x01, 0x65, 0x0b, 0x00, 0x00, + // 1025 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x56, 0xcf, 0x6f, 0x1b, 0xc5, + 0x17, 0xff, 0xee, 0x7a, 0xed, 0xe4, 0xfb, 0x92, 0xe0, 0x64, 0x09, 0xc8, 0x8a, 0x84, 0x37, 0xda, + 0x82, 0x88, 0xf8, 0xb1, 0x96, 0x02, 0x82, 0x43, 0x05, 0x6d, 0x1d, 0x2b, 0x60, 0x42, 0x05, 0xdd, + 0xb4, 0x3d, 0x20, 0x41, 0xb4, 0xf6, 0x4c, 0x9c, 0x51, 0xec, 0x9d, 0xed, 0xce, 0x6c, 0x95, 0x70, + 0xe2, 0xc4, 0x81, 0x0b, 0x12, 0x7f, 0x01, 0x07, 0xfe, 0x16, 0xd4, 0x23, 0x7f, 0xc1, 0x0a, 0xf9, + 0xc6, 0x91, 0x6b, 0x4e, 0x68, 0xde, 0xcc, 0xfa, 0x47, 0x4a, 0x9b, 0x3a, 0x8a, 0x23, 0x7a, 0xdb, + 0x19, 0xbf, 0xcf, 0x7b, 0x9f, 0xf7, 0x63, 0x3e, 0xcf, 0x50, 0x15, 0x47, 0x51, 0x4a, 0x62, 0x4e, + 0x68, 0x90, 0xa4, 0x5c, 0x72, 0xb7, 0xde, 0xcd, 0x3a, 0xf4, 0x50, 0x04, 0x9d, 0x3e, 0xef, 0x08, + 0xc9, 0x53, 0x1a, 0x44, 0x09, 0x0b, 0x46, 0x56, 0x1b, 0xeb, 0x3d, 0xde, 0xe3, 0x68, 0xda, 0x50, + 0x5f, 0x1a, 0xb5, 0xf1, 0x9e, 0x46, 0x35, 0x46, 0xa8, 0x46, 0x97, 0x0f, 0x06, 0x3c, 0x6e, 0x20, + 0x90, 0xc5, 0xbd, 0x46, 0x1a, 0xc5, 0x3d, 0x13, 0x63, 0xe3, 0xdd, 0xa7, 0xac, 0xa3, 0x84, 0x35, + 0xba, 0xfd, 0x4c, 0x48, 0x9a, 0x0e, 0x7a, 0xa9, 0x46, 0x19, 0xe3, 0xad, 0x67, 0xb9, 0xd6, 0x24, + 0xd4, 0xb5, 0xb6, 0xf4, 0xbb, 0xe0, 0xb4, 0x25, 0x1d, 0xb8, 0xaf, 0x83, 0xcd, 0x48, 0xcd, 0xda, + 0xb4, 0xb6, 0x96, 0x9b, 0x95, 0x61, 0xee, 0xd9, 0xed, 0x56, 0x68, 0x33, 0xe2, 0xee, 0x40, 0xe5, + 0x90, 0xd1, 0x3e, 0x11, 0x35, 0x7b, 0xb3, 0xb4, 0xb5, 0xb4, 0xfd, 0x56, 0xf0, 0xfc, 0x5c, 0x83, + 0x5d, 0x65, 0xdd, 0x74, 0x9e, 0xe4, 0xde, 0xff, 0x42, 0x03, 0xf5, 0x13, 0x28, 0xe3, 0xb5, 0x7b, + 0x6f, 0x14, 0x65, 0xa5, 0x79, 0x47, 0x47, 0x39, 0xcb, 0xbd, 0x8f, 0x7b, 0x4c, 0x1e, 0x65, 0x9d, + 0xa0, 0xcb, 0x07, 0x0d, 0x43, 0xfc, 0xb9, 0xfc, 0x75, 0x0c, 0x43, 0x70, 0x1d, 0xca, 0x8f, 0xa3, + 0x7e, 0x46, 0x6b, 0xb6, 0xe2, 0x1e, 0xea, 0x83, 0xff, 0x73, 0x09, 0x56, 0xf6, 0x15, 0xa7, 0xaf, + 0x92, 0xcf, 0x69, 0x44, 0x68, 0xea, 0x46, 0xb0, 0x28, 0x92, 0xa8, 0x4b, 0x0f, 0x0c, 0x01, 0xa7, + 0xb9, 0x3b, 0xcc, 0xbd, 0x85, 0x7d, 0x75, 0x77, 0x39, 0x16, 0x06, 0x1a, 0x2e, 0xa0, 0xdf, 0x36, + 0x71, 0xbf, 0x85, 0x05, 0xc2, 0xc4, 0xb1, 0x8a, 0x60, 0x63, 0x8a, 0xad, 0x61, 0xee, 0x55, 0x5a, + 0x4c, 0x1c, 0x63, 0x80, 0x8f, 0x66, 0x0d, 0xa0, 0x91, 0x61, 0x45, 0x39, 0x6d, 0x13, 0xf7, 0x3e, + 0x38, 0x22, 0x63, 0xa4, 0x56, 0x42, 0xdf, 0xb7, 0x87, 0xb9, 0xe7, 0xec, 0x67, 0x8c, 0x9c, 0xe5, + 0xde, 0x87, 0x33, 0x53, 0xcf, 0x18, 0x09, 0xd1, 0x9b, 0xeb, 0xc3, 0x32, 0xf2, 0x7f, 0x48, 0x53, + 0xc1, 0x78, 0x5c, 0x73, 0x54, 0x6d, 0xc2, 0xa9, 0x3b, 0xf7, 0x06, 0xac, 0xa4, 0x3c, 0x93, 0xf4, + 0xe0, 0xb1, 0x31, 0x2a, 0x6b, 0x23, 0xbc, 0x2c, 0x8c, 0xde, 0x00, 0xc0, 0x29, 0x38, 0x38, 0xa6, + 0xa7, 0xa2, 0x56, 0xd9, 0x2c, 0x6d, 0x2d, 0x87, 0xff, 0xc7, 0x9b, 0x3d, 0x7a, 0x2a, 0xfc, 0x5f, + 0x2d, 0x58, 0x6b, 0xc7, 0x82, 0xa6, 0x52, 0xcd, 0x5b, 0x48, 0x1f, 0x65, 0x54, 0x48, 0x77, 0x0f, + 0x2a, 0x47, 0xd8, 0x1f, 0xec, 0xc9, 0xd2, 0xf6, 0xfb, 0x17, 0x8d, 0xd7, 0x54, 0x53, 0x8b, 0x31, + 0xd3, 0x2e, 0xdc, 0x4f, 0xc1, 0x61, 0x92, 0x0e, 0xb0, 0xf8, 0x4b, 0xdb, 0x6f, 0x5e, 0xe4, 0x4a, + 0xf1, 0x30, 0x1e, 0x10, 0xe7, 0xaf, 0x83, 0x3b, 0xc9, 0x50, 0x24, 0x3c, 0x16, 0x14, 0x89, 0x3f, + 0x48, 0x48, 0x24, 0xe9, 0x7f, 0x99, 0xf8, 0x24, 0x43, 0x43, 0xfc, 0x04, 0xd6, 0x5a, 0xb4, 0x4f, + 0xe7, 0xc8, 0x5b, 0x8b, 0x86, 0x7d, 0x5e, 0x34, 0x14, 0x9f, 0xc9, 0xc8, 0x86, 0x4f, 0x06, 0xaf, + 0x7c, 0x46, 0xe5, 0xb5, 0x93, 0xb9, 0x07, 0xd5, 0x51, 0x58, 0xcd, 0x64, 0x54, 0x6f, 0xeb, 0x92, + 0xf5, 0xfe, 0xcd, 0x82, 0xea, 0x97, 0x4c, 0xcc, 0x33, 0x97, 0x4a, 0x92, 0xd2, 0x43, 0x76, 0x62, + 0x54, 0xcd, 0x9c, 0xd4, 0xfd, 0x20, 0x4a, 0x8f, 0x69, 0x8a, 0x22, 0xb0, 0x1c, 0x9a, 0x93, 0x12, + 0xc1, 0x2e, 0xcf, 0x62, 0x69, 0x5e, 0xaf, 0x3e, 0xf8, 0x12, 0x56, 0xc7, 0x2c, 0x4d, 0xea, 0xb7, + 0xa1, 0xac, 0x52, 0x10, 0x35, 0x0b, 0xe5, 0x7c, 0x96, 0xdc, 0x35, 0xd0, 0xad, 0x03, 0xc4, 0xf4, + 0x44, 0xde, 0xd5, 0x3c, 0x34, 0xbf, 0x89, 0x1b, 0xff, 0x2f, 0x1b, 0xaa, 0x77, 0x08, 0xc1, 0xf4, + 0x8a, 0xe2, 0x4c, 0x28, 0xa3, 0x35, 0x47, 0x65, 0xb4, 0x51, 0xd7, 0xaf, 0x4a, 0x19, 0x77, 0xa0, + 0x8c, 0x0b, 0x18, 0x6b, 0xbd, 0xb4, 0xfd, 0xf6, 0xd3, 0xa5, 0xd2, 0xc8, 0xa0, 0xd8, 0xd7, 0x41, + 0xa8, 0xcc, 0x8b, 0x6a, 0x21, 0xd6, 0xdd, 0x85, 0x72, 0x16, 0x33, 0x29, 0x6a, 0x0e, 0xd6, 0xfb, + 0x9d, 0x7f, 0xaf, 0xf7, 0x78, 0x8d, 0xeb, 0xb1, 0x78, 0x10, 0x33, 0x59, 0xf8, 0x41, 0xb8, 0xea, + 0x30, 0x4d, 0x78, 0xf7, 0xc8, 0x48, 0xaf, 0x3e, 0xf8, 0x2e, 0xac, 0x8e, 0x4b, 0x6d, 0x9e, 0xd9, + 0x2f, 0xa5, 0x42, 0x0d, 0x5e, 0xfe, 0x16, 0xfc, 0x60, 0x41, 0x55, 0x17, 0x04, 0x13, 0xba, 0x7f, + 0x9a, 0x50, 0xb3, 0xfe, 0x1e, 0x0e, 0x73, 0xef, 0xfc, 0x4f, 0x67, 0xb9, 0x77, 0x6b, 0xe6, 0x60, + 0xd3, 0x2e, 0xc2, 0xf3, 0x3e, 0xdd, 0x16, 0x38, 0xaa, 0x03, 0xf8, 0xb2, 0x2e, 0xd3, 0x3f, 0x44, + 0xfb, 0xaf, 0xc1, 0xab, 0x53, 0x3d, 0x31, 0xbd, 0xfa, 0xdd, 0x42, 0x71, 0x7a, 0xe9, 0x1b, 0xe5, + 0x7f, 0x07, 0xab, 0xe3, 0x3c, 0x8c, 0xd4, 0x7c, 0x01, 0x65, 0x7c, 0x19, 0x46, 0x10, 0x83, 0x17, + 0x2f, 0x5d, 0x3b, 0x3e, 0xe4, 0xc5, 0xf8, 0xa3, 0x0b, 0xff, 0x6f, 0x0b, 0xd6, 0x76, 0x52, 0x1a, + 0x49, 0xda, 0xec, 0xf3, 0xce, 0x5c, 0x34, 0xd7, 0x05, 0x27, 0x8e, 0x06, 0xc5, 0xff, 0x48, 0xfc, + 0x76, 0x7b, 0xb0, 0xd8, 0xe5, 0x84, 0x0e, 0x38, 0x29, 0xe6, 0x6e, 0x6f, 0x98, 0x7b, 0x8b, 0x3b, + 0x9c, 0xd0, 0xbb, 0x9c, 0xa8, 0x81, 0xbb, 0xf9, 0xe2, 0x45, 0x2b, 0x3c, 0x05, 0x05, 0x3c, 0x1c, + 0x39, 0x57, 0xc1, 0x05, 0xfb, 0x9e, 0x1a, 0xfd, 0xc6, 0x6f, 0x7f, 0x1f, 0xdc, 0xc9, 0x94, 0x4d, + 0x55, 0x3f, 0x01, 0x47, 0x79, 0x36, 0x19, 0xdf, 0x78, 0xa6, 0x28, 0xe9, 0x16, 0x29, 0x68, 0x31, + 0x88, 0xca, 0xc4, 0x7f, 0x84, 0x4b, 0xf8, 0x3a, 0x8b, 0xe8, 0x7f, 0x8d, 0x33, 0x7e, 0x95, 0x49, + 0x14, 0xfb, 0x77, 0x6e, 0x69, 0x5c, 0xcd, 0xfe, 0x15, 0x7a, 0xff, 0x4e, 0x65, 0x7e, 0x0b, 0xca, + 0x48, 0xc8, 0xec, 0xdf, 0x19, 0x52, 0xd7, 0xb8, 0x0b, 0xd7, 0xaf, 0x2c, 0xfe, 0xf5, 0x5d, 0x6b, + 0x8f, 0x7f, 0xb2, 0x60, 0x2d, 0xa4, 0x32, 0x62, 0xf1, 0xb5, 0xbe, 0xcf, 0x75, 0x28, 0xf7, 0x69, + 0x24, 0xf4, 0xe3, 0x74, 0x42, 0x7d, 0xf0, 0x7f, 0x54, 0x5b, 0x83, 0x46, 0xfd, 0x79, 0x52, 0xc1, + 0xd7, 0x6a, 0x8f, 0x5f, 0xeb, 0x88, 0x5e, 0x69, 0x4c, 0xaf, 0xb9, 0xf1, 0x64, 0x58, 0xb7, 0xfe, + 0x18, 0xd6, 0xad, 0x3f, 0x87, 0x75, 0xeb, 0x9b, 0xe5, 0xa0, 0x71, 0x73, 0xe4, 0xbf, 0x53, 0xc1, + 0x06, 0x7f, 0xf0, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xb9, 0x91, 0xfb, 0x72, 0x4d, 0x10, 0x00, + 0x00, } func (m *Item) Marshal() (dAtA []byte, err error) { @@ -1941,6 +2492,424 @@ func (m *GetShardResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *CreateBlobRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CreateBlobRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *CreateBlobRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.Size_ != 0 { + i = encodeVarintShardnode(dAtA, i, uint64(m.Size_)) + i-- + dAtA[i] = 0x20 + } + if m.CodeMode != 0 { + i = encodeVarintShardnode(dAtA, i, uint64(m.CodeMode)) + i-- + dAtA[i] = 0x18 + } + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintShardnode(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0x12 + } + { + size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShardnode(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + return len(dAtA) - i, nil +} + +func (m *CreateBlobResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CreateBlobResponse) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *CreateBlobResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + { + size, err := m.Blob.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShardnode(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + return len(dAtA) - i, nil +} + +func (m *GetBlobRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *GetBlobRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *GetBlobRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintShardnode(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0x12 + } + { + size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShardnode(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + return len(dAtA) - i, nil +} + +func (m *GetBlobResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *GetBlobResponse) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *GetBlobResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + { + size, err := m.Blob.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShardnode(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + return len(dAtA) - i, nil +} + +func (m *ListBlobRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ListBlobRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ListBlobRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.Count != 0 { + i = encodeVarintShardnode(dAtA, i, uint64(m.Count)) + i-- + dAtA[i] = 0x20 + } + if len(m.Marker) > 0 { + i -= len(m.Marker) + copy(dAtA[i:], m.Marker) + i = encodeVarintShardnode(dAtA, i, uint64(len(m.Marker))) + i-- + dAtA[i] = 0x1a + } + if len(m.Prefix) > 0 { + i -= len(m.Prefix) + copy(dAtA[i:], m.Prefix) + i = encodeVarintShardnode(dAtA, i, uint64(len(m.Prefix))) + i-- + dAtA[i] = 0x12 + } + { + size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShardnode(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + return len(dAtA) - i, nil +} + +func (m *ListBlobResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ListBlobResponse) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ListBlobResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.NextMarker) > 0 { + i -= len(m.NextMarker) + copy(dAtA[i:], m.NextMarker) + i = encodeVarintShardnode(dAtA, i, uint64(len(m.NextMarker))) + i-- + dAtA[i] = 0x12 + } + if len(m.Blobs) > 0 { + for iNdEx := len(m.Blobs) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Blobs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShardnode(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *DeleteBlobRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *DeleteBlobRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *DeleteBlobRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintShardnode(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0x12 + } + { + size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShardnode(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + return len(dAtA) - i, nil +} + +func (m *RetainBlobRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RetainBlobRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *RetainBlobRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.Lease != 0 { + i = encodeVarintShardnode(dAtA, i, uint64(m.Lease)) + i-- + dAtA[i] = 0x18 + } + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintShardnode(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0x12 + } + { + size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShardnode(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + return len(dAtA) - i, nil +} + +func (m *SealBlobRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SealBlobRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SealBlobRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintShardnode(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0x1a + } + if m.Size_ != 0 { + i = encodeVarintShardnode(dAtA, i, uint64(m.Size_)) + i-- + dAtA[i] = 0x10 + } + { + size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShardnode(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + return len(dAtA) - i, nil +} + func encodeVarintShardnode(dAtA []byte, offset int, v uint64) int { offset -= sovShardnode(v) base := offset @@ -2299,6 +3268,183 @@ func (m *GetShardResponse) Size() (n int) { return n } +func (m *CreateBlobRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = m.Header.Size() + n += 1 + l + sovShardnode(uint64(l)) + l = len(m.Name) + if l > 0 { + n += 1 + l + sovShardnode(uint64(l)) + } + if m.CodeMode != 0 { + n += 1 + sovShardnode(uint64(m.CodeMode)) + } + if m.Size_ != 0 { + n += 1 + sovShardnode(uint64(m.Size_)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *CreateBlobResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = m.Blob.Size() + n += 1 + l + sovShardnode(uint64(l)) + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *GetBlobRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = m.Header.Size() + n += 1 + l + sovShardnode(uint64(l)) + l = len(m.Name) + if l > 0 { + n += 1 + l + sovShardnode(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *GetBlobResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = m.Blob.Size() + n += 1 + l + sovShardnode(uint64(l)) + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ListBlobRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = m.Header.Size() + n += 1 + l + sovShardnode(uint64(l)) + l = len(m.Prefix) + if l > 0 { + n += 1 + l + sovShardnode(uint64(l)) + } + l = len(m.Marker) + if l > 0 { + n += 1 + l + sovShardnode(uint64(l)) + } + if m.Count != 0 { + n += 1 + sovShardnode(uint64(m.Count)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ListBlobResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Blobs) > 0 { + for _, e := range m.Blobs { + l = e.Size() + n += 1 + l + sovShardnode(uint64(l)) + } + } + l = len(m.NextMarker) + if l > 0 { + n += 1 + l + sovShardnode(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *DeleteBlobRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = m.Header.Size() + n += 1 + l + sovShardnode(uint64(l)) + l = len(m.Name) + if l > 0 { + n += 1 + l + sovShardnode(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *RetainBlobRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = m.Header.Size() + n += 1 + l + sovShardnode(uint64(l)) + l = len(m.Name) + if l > 0 { + n += 1 + l + sovShardnode(uint64(l)) + } + if m.Lease != 0 { + n += 1 + sovShardnode(uint64(m.Lease)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *SealBlobRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = m.Header.Size() + n += 1 + l + sovShardnode(uint64(l)) + if m.Size_ != 0 { + n += 1 + sovShardnode(uint64(m.Size_)) + } + l = len(m.Name) + if l > 0 { + n += 1 + l + sovShardnode(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func sovShardnode(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -4294,6 +5440,1130 @@ func (m *GetShardResponse) Unmarshal(dAtA []byte) error { } return nil } +func (m *CreateBlobRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CreateBlobRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CreateBlobRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShardnode + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShardnode + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthShardnode + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShardnode + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = append(m.Name[:0], dAtA[iNdEx:postIndex]...) + if m.Name == nil { + m.Name = []byte{} + } + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CodeMode", wireType) + } + m.CodeMode = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CodeMode |= github_com_cubefs_cubefs_blobstore_common_codemode.CodeMode(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType) + } + m.Size_ = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Size_ |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipShardnode(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShardnode + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *CreateBlobResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CreateBlobResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CreateBlobResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Blob", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShardnode + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShardnode + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Blob.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipShardnode(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShardnode + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *GetBlobRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: GetBlobRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: GetBlobRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShardnode + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShardnode + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthShardnode + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShardnode + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = append(m.Name[:0], dAtA[iNdEx:postIndex]...) + if m.Name == nil { + m.Name = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipShardnode(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShardnode + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *GetBlobResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: GetBlobResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: GetBlobResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Blob", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShardnode + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShardnode + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Blob.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipShardnode(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShardnode + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ListBlobRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ListBlobRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ListBlobRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShardnode + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShardnode + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Prefix", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthShardnode + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShardnode + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Prefix = append(m.Prefix[:0], dAtA[iNdEx:postIndex]...) + if m.Prefix == nil { + m.Prefix = []byte{} + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Marker", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthShardnode + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShardnode + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Marker = append(m.Marker[:0], dAtA[iNdEx:postIndex]...) + if m.Marker == nil { + m.Marker = []byte{} + } + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Count", wireType) + } + m.Count = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Count |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipShardnode(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShardnode + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ListBlobResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ListBlobResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ListBlobResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Blobs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShardnode + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShardnode + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Blobs = append(m.Blobs, proto1.Blob{}) + if err := m.Blobs[len(m.Blobs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field NextMarker", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthShardnode + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShardnode + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.NextMarker = append(m.NextMarker[:0], dAtA[iNdEx:postIndex]...) + if m.NextMarker == nil { + m.NextMarker = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipShardnode(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShardnode + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *DeleteBlobRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: DeleteBlobRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: DeleteBlobRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShardnode + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShardnode + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthShardnode + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShardnode + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = append(m.Name[:0], dAtA[iNdEx:postIndex]...) + if m.Name == nil { + m.Name = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipShardnode(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShardnode + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RetainBlobRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RetainBlobRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RetainBlobRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShardnode + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShardnode + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthShardnode + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShardnode + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = append(m.Name[:0], dAtA[iNdEx:postIndex]...) + if m.Name == nil { + m.Name = []byte{} + } + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Lease", wireType) + } + m.Lease = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Lease |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipShardnode(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShardnode + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SealBlobRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SealBlobRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SealBlobRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShardnode + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShardnode + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType) + } + m.Size_ = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Size_ |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthShardnode + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShardnode + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = append(m.Name[:0], dAtA[iNdEx:postIndex]...) + if m.Name == nil { + m.Name = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipShardnode(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShardnode + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipShardnode(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/blobstore/api/shardnode/shardnode.proto b/blobstore/api/shardnode/shardnode.proto index c92213ca1..fb0bf9348 100644 --- a/blobstore/api/shardnode/shardnode.proto +++ b/blobstore/api/shardnode/shardnode.proto @@ -1,3 +1,17 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + syntax = "proto3"; package cubefs.blobstore.api.shardnode; @@ -10,6 +24,7 @@ option (gogoproto.unmarshaler_all) = true; import "gogoproto/gogo.proto"; import "cubefs/blobstore/common/sharding/range.proto"; import "cubefs/blobstore/api/clustermgr/shard.proto"; +import "cubefs/blobstore/common/proto/blob.proto"; message Item { bytes id = 1 [(gogoproto.customname) = "ID"]; @@ -98,5 +113,54 @@ message GetShardRequest { } message GetShardResponse { - cubefs.blobstore.api.clustermgr.Shard shard = 1 [(gogoproto.nullable) = false]; + cubefs.blobstore.api.clustermgr.ShardUnitInfo shard = 1 [(gogoproto.nullable) = false]; +} + +message CreateBlobRequest { + ShardOpHeader header = 1 [(gogoproto.nullable) = false]; + bytes name = 2; + uint32 codemode = 3 [(gogoproto.customname) = "CodeMode", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/codemode.CodeMode"]; + uint64 size = 4; +} + +message CreateBlobResponse { + cubefs.blobstore.common.proto.Blob blob = 1 [(gogoproto.nullable) = false]; +} + +message GetBlobRequest { + ShardOpHeader header = 1 [(gogoproto.nullable) = false]; + bytes name = 2; +} + +message GetBlobResponse { + cubefs.blobstore.common.proto.Blob blob = 1 [(gogoproto.nullable) = false]; +} + +message ListBlobRequest { + ShardOpHeader header = 1 [(gogoproto.nullable) = false]; + bytes prefix = 2; + bytes marker = 3; + uint64 count = 4; +} + +message ListBlobResponse { + repeated cubefs.blobstore.common.proto.Blob blobs = 1 [(gogoproto.nullable) = false]; + bytes nextMarker = 2; +} + +message DeleteBlobRequest { + ShardOpHeader header = 1 [(gogoproto.nullable) = false]; + bytes name = 2; +} + +message RetainBlobRequest { + ShardOpHeader header = 1 [(gogoproto.nullable) = false]; + bytes name = 2; + uint64 lease = 3; +} + +message SealBlobRequest { + ShardOpHeader header = 1 [(gogoproto.nullable) = false]; + uint64 size = 2; + bytes name = 3; } \ No newline at end of file diff --git a/blobstore/blobstore.go b/blobstore/blobstore.go index 6d9be4e6f..68a84dbf4 100644 --- a/blobstore/blobstore.go +++ b/blobstore/blobstore.go @@ -35,10 +35,12 @@ package blobstore // protoc -I /usr/local/include/ -I ${current}/vendor/github.com/gogo/protobuf/ -I ${current}../ --proto_path=. --gogo_out=Mcubefs/blobstore/common/sharding/range.proto=github.com/cubefs/cubefs/blobstore/common/sharding,plugins=grpc:. *.proto // generate api/shardnode // cd blobstore/api/shardnode -// protoc -I ${current}/../ -I /usr/local/include/ -I ${current}/vendor/github.com/gogo/protobuf/ --proto_path=. --gogo_out=Mcubefs/blobstore/common/sharding/range.proto=github.com/cubefs/cubefs/blobstore/common/sharding,Mcubefs/blobstore/api/clustermgr/shard.proto=github.com/cubefs/cubefs/blobstore/api/clustermgr,plugins=grpc:. *.proto +// protoc -I ${current}/../ -I /usr/local/include/ -I ${current}/vendor/github.com/gogo/protobuf/ --proto_path=. --gogo_out=Mcubefs/blobstore/common/sharding/range.proto=github.com/cubefs/cubefs/blobstore/common/sharding,Mcubefs/blobstore/api/clustermgr/shard.proto=github.com/cubefs/cubefs/blobstore/api/clustermgr,Mcubefs/blobstore/common/proto/blob.proto=github.com/cubefs/cubefs/blobstore/common/proto,plugins=grpc:. *.proto // generate api/clustermgr // cd blobstore/api/clustermgr // protoc -I ${current}/../ -I /usr/local/include/ -I ${current}/vendor/github.com/gogo/protobuf/ --proto_path=. --gogo_out=Mcubefs/blobstore/common/sharding/range.proto=github.com/cubefs/cubefs/blobstore/common/sharding,plugins=grpc:. *.proto // generate raft package proto file // cd blobstore/common/raft // protoc -I /usr/local/include/ -I ${current}/vendor/go.etcd.io/etcd/raft/v3/ -I ${current}/vendor/github.com/gogo/protobuf/ --proto_path=. --gogo_out=plugins=grpc:. --gogo_opt=Mraftpb/raft.proto=go.etcd.io/etcd/raft/v3/raftpb *.proto +// cd blobstore/common/proto +// protoc -I ${current}/../ -I /usr/local/include/ -I ${current}/vendor/github.com/gogo/protobuf/ --proto_path=. --gogo_out=Mcubefs/blobstore/common/sharding/range.proto=github.com/cubefs/cubefs/blobstore/common/sharding,Mcubefs/blobstore/api/clustermgr/shard.proto=github.com/cubefs/cubefs/blobstore/api/clustermgr,plugins=grpc:. *.proto diff --git a/blobstore/cli/access/access.go b/blobstore/cli/access/access.go index 55e579698..d9ba49df4 100644 --- a/blobstore/cli/access/access.go +++ b/blobstore/cli/access/access.go @@ -26,6 +26,7 @@ import ( "github.com/cubefs/cubefs/blobstore/cli/common/flags" "github.com/cubefs/cubefs/blobstore/cli/common/fmt" "github.com/cubefs/cubefs/blobstore/cli/config" + "github.com/cubefs/cubefs/blobstore/common/proto" ) func newAccessClient() (access.API, error) { @@ -45,7 +46,7 @@ func newAccessClient() (access.API, error) { }) } -func readLocation(f grumble.FlagMap) (loc access.Location, err error) { +func readLocation(f grumble.FlagMap) (loc proto.Location, err error) { if f.String("location") != "" { loc, err = cfmt.ParseLocation(f.String("location")) return diff --git a/blobstore/cli/access/del.go b/blobstore/cli/access/del.go index 06460b601..913097faf 100644 --- a/blobstore/cli/access/del.go +++ b/blobstore/cli/access/del.go @@ -23,6 +23,7 @@ import ( "github.com/cubefs/cubefs/blobstore/cli/common/flags" "github.com/cubefs/cubefs/blobstore/cli/common/fmt" "github.com/cubefs/cubefs/blobstore/cli/config" + "github.com/cubefs/cubefs/blobstore/common/proto" ) func delFile(c *grumble.Context) error { @@ -48,7 +49,7 @@ func delFile(c *grumble.Context) error { return nil } deleteArgs := &access.DeleteArgs{ - Locations: []access.Location{location}, + Locations: []proto.Location{location}, } _, err = client.Delete(common.CmdContext(), deleteArgs) if err != nil { diff --git a/blobstore/cli/access/get.go b/blobstore/cli/access/get.go index 5faea1157..63914bfae 100644 --- a/blobstore/cli/access/get.go +++ b/blobstore/cli/access/get.go @@ -51,7 +51,7 @@ func getFile(c *grumble.Context) error { size := c.Flags.Uint64("readsize") if size == 0 { - size = location.Size - c.Flags.Uint64("offset") + size = location.Size_ - c.Flags.Uint64("offset") } r, err := client.Get(common.CmdContext(), &access.GetArgs{ diff --git a/blobstore/cli/common/cfmt/access.go b/blobstore/cli/common/cfmt/access.go index 4e34f7c99..f005a4cc3 100644 --- a/blobstore/cli/common/cfmt/access.go +++ b/blobstore/cli/common/cfmt/access.go @@ -21,31 +21,32 @@ import ( "github.com/cubefs/cubefs/blobstore/api/access" "github.com/cubefs/cubefs/blobstore/cli/common" + "github.com/cubefs/cubefs/blobstore/common/proto" ) // ParseLocation parst location from json or string -func ParseLocation(jsonORstr string) (access.Location, error) { - var loc access.Location +func ParseLocation(jsonORstr string) (proto.Location, error) { + var loc proto.Location var err error if err = common.Unmarshal([]byte(jsonORstr), &loc); err == nil { return loc, nil } - if loc, err = access.DecodeLocationFromHex(jsonORstr); err == nil { + if loc, err = proto.DecodeLocationFromHex(jsonORstr); err == nil { return loc, nil } - if loc, err = access.DecodeLocationFromBase64(jsonORstr); err == nil { + if loc, err = proto.DecodeLocationFromBase64(jsonORstr); err == nil { return loc, nil } return loc, fmt.Errorf("invalid (%s) %s", jsonORstr, err.Error()) } // LocationJoin join line into string -func LocationJoin(loc *access.Location, prefix string) string { +func LocationJoin(loc *proto.Location, prefix string) string { return joinWithPrefix(prefix, LocationF(loc)) } // LocationF fmt pointer of Location -func LocationF(loc *access.Location) (vals []string) { +func LocationF(loc *proto.Location) (vals []string) { if loc == nil { return nilStrings[:] } @@ -54,16 +55,16 @@ func LocationF(loc *access.Location) (vals []string) { fmt.Sprintf("Crc : %-12d (0x%x)", loc.Crc, loc.Crc), fmt.Sprintf("ClusterID : %d", loc.ClusterID), fmt.Sprintf("CodeMode : %-12d (%s)", loc.CodeMode, loc.CodeMode.String()), - fmt.Sprintf("Size : %-12d (%s)", loc.Size, humanize.IBytes(loc.Size)), - fmt.Sprintf("BlobSize : %-12d (%s)", loc.BlobSize, humanize.IBytes(uint64(loc.BlobSize))), - fmt.Sprintf("Blobs: (%d) [", len(loc.Blobs)), + fmt.Sprintf("Size_ : %-12d (%s)", loc.Size_, humanize.IBytes(loc.Size_)), + fmt.Sprintf("SliceSize : %-12d (%s)", loc.SliceSize, humanize.IBytes(uint64(loc.SliceSize))), + fmt.Sprintf("Slices: (%d) [", len(loc.Slices)), }...) - for idx, blob := range loc.Blobs { - vals = append(vals, fmt.Sprintf(" >:%3d| MinBid: %-20d Vid: %-10d Count: %-10d", - idx, blob.MinBid, blob.Vid, blob.Count)) + for idx, blob := range loc.Slices { + vals = append(vals, fmt.Sprintf(" >:%3d| MinSliceID: %-20d Vid: %-10d Count: %-10d", + idx, blob.MinSliceID, blob.Vid, blob.Count)) } vals = append(vals, "]") - vals = append(vals, fmt.Sprintf("--> Encode: %d of %d bytes", len(loc.Encode()), 21+16*len(loc.Blobs))) + vals = append(vals, fmt.Sprintf("--> Encode: %d of %d bytes", len(loc.Encode()), 21+16*len(loc.Slices))) vals = append(vals, fmt.Sprintf("--> Hex : %s", loc.HexString())) vals = append(vals, fmt.Sprintf("--> Base64: %s", loc.Base64String())) return diff --git a/blobstore/cli/common/cfmt/access_test.go b/blobstore/cli/common/cfmt/access_test.go index 1f92ac35d..534f3d310 100644 --- a/blobstore/cli/common/cfmt/access_test.go +++ b/blobstore/cli/common/cfmt/access_test.go @@ -28,6 +28,7 @@ import ( "github.com/cubefs/cubefs/blobstore/cli/common" "github.com/cubefs/cubefs/blobstore/cli/common/cfmt" "github.com/cubefs/cubefs/blobstore/cli/common/fmt" + "github.com/cubefs/cubefs/blobstore/common/proto" ) func init() { @@ -38,16 +39,16 @@ func init() { } func TestParseLocation(t *testing.T) { - loc := access.Location{ + loc := proto.Location{ ClusterID: 1, CodeMode: 3, - Size: 19213422425, - BlobSize: 1 << 22, + Size_: 19213422425, + SliceSize: 1 << 22, Crc: 1 << 31, - Blobs: []access.SliceInfo{ - {MinBid: 0x199, Vid: 100020, Count: 0}, - {MinBid: 10, Vid: 20, Count: 300}, - {MinBid: 14225224, Vid: 0xffffffff, Count: 1 << 31}, + Slices: []proto.Slice{ + {MinSliceID: 0x199, Vid: 100020, Count: 0}, + {MinSliceID: 10, Vid: 20, Count: 300}, + {MinSliceID: 14225224, Vid: 0xffffffff, Count: 1 << 31}, }, } @@ -59,7 +60,7 @@ func TestParseLocation(t *testing.T) { locx, err := cfmt.ParseLocation("{}") require.NoError(t, err) - require.Equal(t, access.Location{}, locx) + require.Equal(t, proto.Location{}, locx) locx, err = cfmt.ParseLocation(loc.ToString()) require.NoError(t, err) @@ -76,16 +77,16 @@ func TestParseLocation(t *testing.T) { } func TestLocation(t *testing.T) { - loc := access.Location{ + loc := proto.Location{ ClusterID: 1, CodeMode: 3, - Size: 19213422425, - BlobSize: 1 << 22, + Size_: 19213422425, + SliceSize: 1 << 22, Crc: 1 << 31, - Blobs: []access.SliceInfo{ - {MinBid: 0x199, Vid: 100020, Count: 0}, - {MinBid: 10, Vid: 20, Count: 300}, - {MinBid: 14225224, Vid: 0xffffffff, Count: 1 << 31}, + Slices: []proto.Slice{ + {MinSliceID: 0x199, Vid: 100020, Count: 0}, + {MinSliceID: 10, Vid: 20, Count: 300}, + {MinSliceID: 14225224, Vid: 0xffffffff, Count: 1 << 31}, }, } printLine() diff --git a/blobstore/cli/util.go b/blobstore/cli/util.go index 361bde2f5..72272bb96 100644 --- a/blobstore/cli/util.go +++ b/blobstore/cli/util.go @@ -25,11 +25,11 @@ import ( "github.com/dustin/go-humanize" "github.com/fatih/color" - "github.com/cubefs/cubefs/blobstore/api/access" "github.com/cubefs/cubefs/blobstore/cli/common" "github.com/cubefs/cubefs/blobstore/cli/common/args" "github.com/cubefs/cubefs/blobstore/cli/common/cfmt" "github.com/cubefs/cubefs/blobstore/cli/common/fmt" + "github.com/cubefs/cubefs/blobstore/common/proto" "github.com/cubefs/cubefs/blobstore/common/uptoken" ) @@ -164,7 +164,7 @@ func registerUtil(app *grumble.App) { if err != nil { return fmt.Errorf("invalid (%s) %s", jsonORstr, err.Error()) } - loc, n, err := access.DecodeLocation(src) + loc, n, err := proto.DecodeLocation(src) if err != nil { fmt.Printf("has read bytes %d / %d\n", n, len(src)) fmt.Println(cfmt.LocationJoin(&loc, "")) diff --git a/blobstore/common/proto/blob.pb.go b/blobstore/common/proto/blob.pb.go new file mode 100644 index 000000000..27c8634d3 --- /dev/null +++ b/blobstore/common/proto/blob.pb.go @@ -0,0 +1,999 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: blob.proto + +package proto + +import ( + fmt "fmt" + github_com_cubefs_cubefs_blobstore_common_codemode "github.com/cubefs/cubefs/blobstore/common/codemode" + _ "github.com/gogo/protobuf/gogoproto" + proto "github.com/gogo/protobuf/proto" + io "io" + math "math" + math_bits "math/bits" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +type Location struct { + ClusterID ClusterID `protobuf:"varint,1,opt,name=cluster_id,json=clusterId,proto3,casttype=ClusterID" json:"cluster_id,omitempty"` + CodeMode github_com_cubefs_cubefs_blobstore_common_codemode.CodeMode `protobuf:"varint,2,opt,name=codemode,proto3,casttype=github.com/cubefs/cubefs/blobstore/common/codemode.CodeMode" json:"codemode,omitempty"` + Size_ uint64 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"` + SliceSize uint32 `protobuf:"varint,4,opt,name=slice_size,json=sliceSize,proto3" json:"slice_size,omitempty"` + Crc uint32 `protobuf:"varint,5,opt,name=crc,proto3" json:"crc,omitempty"` + Slices []Slice `protobuf:"bytes,6,rep,name=slices,proto3" json:"slices"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Location) Reset() { *m = Location{} } +func (m *Location) String() string { return proto.CompactTextString(m) } +func (*Location) ProtoMessage() {} +func (*Location) Descriptor() ([]byte, []int) { + return fileDescriptor_6903d1e8a20272e8, []int{0} +} +func (m *Location) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Location) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Location.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Location) XXX_Merge(src proto.Message) { + xxx_messageInfo_Location.Merge(m, src) +} +func (m *Location) XXX_Size() int { + return m.Size() +} +func (m *Location) XXX_DiscardUnknown() { + xxx_messageInfo_Location.DiscardUnknown(m) +} + +var xxx_messageInfo_Location proto.InternalMessageInfo + +func (m *Location) GetClusterID() ClusterID { + if m != nil { + return m.ClusterID + } + return 0 +} + +func (m *Location) GetCodeMode() github_com_cubefs_cubefs_blobstore_common_codemode.CodeMode { + if m != nil { + return m.CodeMode + } + return 0 +} + +func (m *Location) GetSize_() uint64 { + if m != nil { + return m.Size_ + } + return 0 +} + +func (m *Location) GetSliceSize() uint32 { + if m != nil { + return m.SliceSize + } + return 0 +} + +func (m *Location) GetCrc() uint32 { + if m != nil { + return m.Crc + } + return 0 +} + +func (m *Location) GetSlices() []Slice { + if m != nil { + return m.Slices + } + return nil +} + +type Blob struct { + Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Location Location `protobuf:"bytes,2,opt,name=location,proto3" json:"location"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Blob) Reset() { *m = Blob{} } +func (m *Blob) String() string { return proto.CompactTextString(m) } +func (*Blob) ProtoMessage() {} +func (*Blob) Descriptor() ([]byte, []int) { + return fileDescriptor_6903d1e8a20272e8, []int{1} +} +func (m *Blob) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Blob) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Blob.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Blob) XXX_Merge(src proto.Message) { + xxx_messageInfo_Blob.Merge(m, src) +} +func (m *Blob) XXX_Size() int { + return m.Size() +} +func (m *Blob) XXX_DiscardUnknown() { + xxx_messageInfo_Blob.DiscardUnknown(m) +} + +var xxx_messageInfo_Blob proto.InternalMessageInfo + +func (m *Blob) GetName() []byte { + if m != nil { + return m.Name + } + return nil +} + +func (m *Blob) GetLocation() Location { + if m != nil { + return m.Location + } + return Location{} +} + +type Slice struct { + MinSliceID BlobID `protobuf:"varint,1,opt,name=min_slice_id,json=minSliceId,proto3,casttype=BlobID" json:"min_slice_id,omitempty"` + Vid Vid `protobuf:"varint,2,opt,name=vid,proto3,casttype=Vid" json:"vid,omitempty"` + Count uint32 `protobuf:"varint,3,opt,name=count,proto3" json:"count,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Slice) Reset() { *m = Slice{} } +func (m *Slice) String() string { return proto.CompactTextString(m) } +func (*Slice) ProtoMessage() {} +func (*Slice) Descriptor() ([]byte, []int) { + return fileDescriptor_6903d1e8a20272e8, []int{2} +} +func (m *Slice) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Slice) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Slice.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Slice) XXX_Merge(src proto.Message) { + xxx_messageInfo_Slice.Merge(m, src) +} +func (m *Slice) XXX_Size() int { + return m.Size() +} +func (m *Slice) XXX_DiscardUnknown() { + xxx_messageInfo_Slice.DiscardUnknown(m) +} + +var xxx_messageInfo_Slice proto.InternalMessageInfo + +func (m *Slice) GetMinSliceID() BlobID { + if m != nil { + return m.MinSliceID + } + return 0 +} + +func (m *Slice) GetVid() Vid { + if m != nil { + return m.Vid + } + return 0 +} + +func (m *Slice) GetCount() uint32 { + if m != nil { + return m.Count + } + return 0 +} + +func init() { + proto.RegisterType((*Location)(nil), "cubefs.blobstore.common.proto.Location") + proto.RegisterType((*Blob)(nil), "cubefs.blobstore.common.proto.Blob") + proto.RegisterType((*Slice)(nil), "cubefs.blobstore.common.proto.Slice") +} + +func init() { proto.RegisterFile("blob.proto", fileDescriptor_6903d1e8a20272e8) } + +var fileDescriptor_6903d1e8a20272e8 = []byte{ + // 396 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x51, 0x3d, 0x8f, 0xd3, 0x40, + 0x10, 0xc5, 0x67, 0xc7, 0x38, 0x73, 0x77, 0x12, 0x5a, 0x9d, 0x90, 0x39, 0xe9, 0xe2, 0x28, 0x42, + 0x22, 0x95, 0x2d, 0x1d, 0x15, 0x5c, 0xe7, 0xa4, 0xb1, 0x20, 0xcd, 0x46, 0xa2, 0xa0, 0x89, 0xe2, + 0xdd, 0xc5, 0xac, 0x64, 0x7b, 0x90, 0x3f, 0x28, 0xf8, 0x63, 0xfc, 0x85, 0x94, 0xfc, 0x02, 0x0b, + 0xf9, 0x67, 0xa4, 0x42, 0x3b, 0xb6, 0x23, 0x2a, 0xa8, 0xf6, 0xed, 0x9b, 0x99, 0x9d, 0xb7, 0xef, + 0x01, 0xa4, 0x39, 0xa6, 0xe1, 0xb7, 0x0a, 0x1b, 0x64, 0x0f, 0xa2, 0x4d, 0xd5, 0x97, 0x3a, 0x34, + 0x54, 0xdd, 0x60, 0xa5, 0x42, 0x81, 0x45, 0x81, 0xe5, 0x50, 0xbe, 0xbf, 0xcb, 0x30, 0x43, 0x82, + 0x91, 0x41, 0x03, 0xbb, 0xfa, 0x79, 0x05, 0xde, 0x47, 0x14, 0xc7, 0x46, 0x63, 0xc9, 0xde, 0x01, + 0x88, 0xbc, 0xad, 0x1b, 0x55, 0x1d, 0xb4, 0xf4, 0xad, 0xa5, 0xb5, 0xbe, 0x8d, 0xef, 0xfb, 0x2e, + 0x98, 0x6f, 0x06, 0x36, 0xd9, 0x9e, 0xff, 0xbe, 0xf0, 0xf9, 0xd8, 0x9d, 0x48, 0x96, 0x81, 0x27, + 0x50, 0xaa, 0x02, 0xa5, 0xf2, 0xaf, 0x68, 0xf0, 0x43, 0xdf, 0x05, 0xde, 0x06, 0xa5, 0xda, 0xa1, + 0x54, 0xe7, 0x2e, 0x78, 0xca, 0x74, 0xf3, 0xb5, 0x4d, 0x8d, 0xa8, 0x68, 0x50, 0x3a, 0x1d, 0x17, + 0xc1, 0xd1, 0x20, 0x38, 0x9a, 0x5e, 0x0a, 0xa7, 0x71, 0x7e, 0x79, 0x9c, 0x31, 0x70, 0x6a, 0xfd, + 0x43, 0xf9, 0xf6, 0xd2, 0x5a, 0x3b, 0x9c, 0x30, 0x7b, 0x00, 0xa8, 0x73, 0x2d, 0xd4, 0x81, 0x2a, + 0x8e, 0x59, 0xcf, 0xe7, 0xc4, 0xec, 0x4d, 0xf9, 0x05, 0xd8, 0xa2, 0x12, 0xfe, 0x8c, 0x78, 0x03, + 0x59, 0x0c, 0x2e, 0x95, 0x6b, 0xdf, 0x5d, 0xda, 0xeb, 0xeb, 0xc7, 0xd7, 0xe1, 0x3f, 0xbd, 0x0b, + 0xf7, 0xa6, 0x39, 0x76, 0x4e, 0x5d, 0xf0, 0x8c, 0x8f, 0x93, 0x2b, 0x05, 0x4e, 0x9c, 0x63, 0x6a, + 0x04, 0x95, 0xc7, 0x42, 0x91, 0x5d, 0x37, 0x9c, 0x30, 0x4b, 0xc0, 0xcb, 0x47, 0x53, 0xc9, 0x8d, + 0xeb, 0xc7, 0x37, 0xff, 0xd9, 0x30, 0x65, 0x30, 0x2e, 0xb9, 0x8c, 0xaf, 0x1a, 0x98, 0xd1, 0x76, + 0xf6, 0x1e, 0x6e, 0x0a, 0x5d, 0x1e, 0x86, 0x8f, 0x8e, 0xf1, 0x38, 0xb1, 0xdf, 0x77, 0x01, 0xec, + 0x74, 0x49, 0x3d, 0x94, 0x8f, 0x6b, 0x54, 0x25, 0x5b, 0x0e, 0xc5, 0xc4, 0x4a, 0xf6, 0x0a, 0xec, + 0xef, 0x5a, 0x8e, 0xc1, 0x3c, 0x3f, 0x77, 0x81, 0xfd, 0x49, 0x4b, 0x6e, 0x38, 0x76, 0x07, 0x33, + 0x81, 0x6d, 0xd9, 0x90, 0xa1, 0xb7, 0x7c, 0xb8, 0xc4, 0x2f, 0x4f, 0xfd, 0xc2, 0xfa, 0xd5, 0x2f, + 0xac, 0xdf, 0xfd, 0xc2, 0xfa, 0xec, 0x85, 0xd1, 0x13, 0xc9, 0x4c, 0x5d, 0x3a, 0xde, 0xfe, 0x09, + 0x00, 0x00, 0xff, 0xff, 0x1d, 0xa9, 0x5c, 0xb0, 0x78, 0x02, 0x00, 0x00, +} + +func (m *Location) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Location) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Location) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Slices) > 0 { + for iNdEx := len(m.Slices) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Slices[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintBlob(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x32 + } + } + if m.Crc != 0 { + i = encodeVarintBlob(dAtA, i, uint64(m.Crc)) + i-- + dAtA[i] = 0x28 + } + if m.SliceSize != 0 { + i = encodeVarintBlob(dAtA, i, uint64(m.SliceSize)) + i-- + dAtA[i] = 0x20 + } + if m.Size_ != 0 { + i = encodeVarintBlob(dAtA, i, uint64(m.Size_)) + i-- + dAtA[i] = 0x18 + } + if m.CodeMode != 0 { + i = encodeVarintBlob(dAtA, i, uint64(m.CodeMode)) + i-- + dAtA[i] = 0x10 + } + if m.ClusterID != 0 { + i = encodeVarintBlob(dAtA, i, uint64(m.ClusterID)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *Blob) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Blob) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Blob) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + { + size, err := m.Location.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintBlob(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintBlob(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *Slice) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Slice) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Slice) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.Count != 0 { + i = encodeVarintBlob(dAtA, i, uint64(m.Count)) + i-- + dAtA[i] = 0x18 + } + if m.Vid != 0 { + i = encodeVarintBlob(dAtA, i, uint64(m.Vid)) + i-- + dAtA[i] = 0x10 + } + if m.MinSliceID != 0 { + i = encodeVarintBlob(dAtA, i, uint64(m.MinSliceID)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func encodeVarintBlob(dAtA []byte, offset int, v uint64) int { + offset -= sovBlob(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *Location) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.ClusterID != 0 { + n += 1 + sovBlob(uint64(m.ClusterID)) + } + if m.CodeMode != 0 { + n += 1 + sovBlob(uint64(m.CodeMode)) + } + if m.Size_ != 0 { + n += 1 + sovBlob(uint64(m.Size_)) + } + if m.SliceSize != 0 { + n += 1 + sovBlob(uint64(m.SliceSize)) + } + if m.Crc != 0 { + n += 1 + sovBlob(uint64(m.Crc)) + } + if len(m.Slices) > 0 { + for _, e := range m.Slices { + l = e.Size() + n += 1 + l + sovBlob(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *Blob) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovBlob(uint64(l)) + } + l = m.Location.Size() + n += 1 + l + sovBlob(uint64(l)) + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *Slice) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.MinSliceID != 0 { + n += 1 + sovBlob(uint64(m.MinSliceID)) + } + if m.Vid != 0 { + n += 1 + sovBlob(uint64(m.Vid)) + } + if m.Count != 0 { + n += 1 + sovBlob(uint64(m.Count)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func sovBlob(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozBlob(x uint64) (n int) { + return sovBlob(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *Location) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Location: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Location: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ClusterID", wireType) + } + m.ClusterID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ClusterID |= ClusterID(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CodeMode", wireType) + } + m.CodeMode = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CodeMode |= github_com_cubefs_cubefs_blobstore_common_codemode.CodeMode(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType) + } + m.Size_ = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Size_ |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SliceSize", wireType) + } + m.SliceSize = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.SliceSize |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Crc", wireType) + } + m.Crc = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Crc |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Slices", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthBlob + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthBlob + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Slices = append(m.Slices, Slice{}) + if err := m.Slices[len(m.Slices)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipBlob(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthBlob + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Blob) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Blob: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Blob: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBlob + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBlob + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = append(m.Name[:0], dAtA[iNdEx:postIndex]...) + if m.Name == nil { + m.Name = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Location", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthBlob + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthBlob + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Location.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipBlob(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthBlob + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Slice) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Slice: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Slice: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field MinSliceID", wireType) + } + m.MinSliceID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.MinSliceID |= BlobID(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Vid", wireType) + } + m.Vid = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Vid |= Vid(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Count", wireType) + } + m.Count = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Count |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipBlob(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthBlob + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipBlob(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowBlob + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowBlob + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowBlob + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthBlob + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupBlob + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthBlob + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthBlob = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowBlob = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupBlob = fmt.Errorf("proto: unexpected end of group") +) diff --git a/blobstore/common/proto/blob.proto b/blobstore/common/proto/blob.proto new file mode 100644 index 000000000..21e9f7217 --- /dev/null +++ b/blobstore/common/proto/blob.proto @@ -0,0 +1,44 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +syntax = "proto3"; + +package cubefs.blobstore.common.proto; +option go_package = "./;proto"; + +option (gogoproto.sizer_all) = true; +option (gogoproto.marshaler_all) = true; +option (gogoproto.unmarshaler_all) = true; + +import "gogoproto/gogo.proto"; + +message Location { + uint32 cluster_id = 1 [(gogoproto.customname) = "ClusterID", (gogoproto.casttype) = "ClusterID"]; + uint32 codemode = 2 [(gogoproto.customname) = "CodeMode", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/codemode.CodeMode"]; + uint64 size = 3; + uint32 slice_size = 4; + uint32 crc = 5; + repeated Slice slices = 6 [(gogoproto.nullable) = false]; +} + +message Blob { + bytes name = 1; + Location location = 2 [(gogoproto.nullable) = false]; +} + +message Slice { + uint64 min_slice_id = 1 [(gogoproto.customname) = "MinSliceID", (gogoproto.casttype) = "BlobID"]; + uint32 vid = 2 [(gogoproto.casttype) = "Vid"]; + uint32 count = 3; +} \ No newline at end of file diff --git a/blobstore/common/proto/blob_ext.go b/blobstore/common/proto/blob_ext.go new file mode 100644 index 000000000..c589d18e9 --- /dev/null +++ b/blobstore/common/proto/blob_ext.go @@ -0,0 +1,272 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package proto + +import ( + "encoding/base64" + "encoding/binary" + "encoding/hex" + "fmt" + + "github.com/cubefs/cubefs/blobstore/common/codemode" +) + +// BlobUnit is one piece of data in a location +// +// Bid is the blob id +// Vid is which volume the blob in +// Size is real size of the blob +type BlobUnit struct { + Bid BlobID + Vid Vid + Size uint32 +} + +// Copy returns a new same Location +func (loc *Location) Copy() Location { + dst := Location{ + ClusterID: loc.ClusterID, + CodeMode: loc.CodeMode, + Size_: loc.Size_, + SliceSize: loc.SliceSize, + Crc: loc.Crc, + Slices: make([]Slice, len(loc.Slices)), + } + copy(dst.Slices, loc.Slices) + return dst +} + +// Encode transfer Location to slice byte +// Returns the buf created by me +// +// (n) means max-n bytes +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// | field | crc | clusterid | codemode | size | blobsize | +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// | n-bytes | 4 | uvarint(5) | 1 | uvarint(10) | uvarint(5) | +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// 25 + (5){len(blobs)} + len(Slices) * 20 +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// | blobs | minbid | vid | count | ... | +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// | n-bytes | (10) | (5) | (5) | (20) | (20) | ... | +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +func (loc *Location) Encode() []byte { + if loc == nil { + return nil + } + n := 25 + 5 + len(loc.Slices)*20 + buf := make([]byte, n) + n = loc.Encode2(buf) + return buf[:n] +} + +// Encode2 transfer Location to the buf, the buf reuse by yourself +// Returns the number of bytes read +// If the buffer is too small, Encode2 will panic +func (loc *Location) Encode2(buf []byte) int { + if loc == nil { + return 0 + } + + n := 0 + binary.BigEndian.PutUint32(buf[n:], loc.Crc) + n += 4 + n += binary.PutUvarint(buf[n:], uint64(loc.ClusterID)) + buf[n] = byte(loc.CodeMode) + n++ + n += binary.PutUvarint(buf[n:], uint64(loc.Size_)) + n += binary.PutUvarint(buf[n:], uint64(loc.SliceSize)) + + n += binary.PutUvarint(buf[n:], uint64(len(loc.Slices))) + for _, blob := range loc.Slices { + n += binary.PutUvarint(buf[n:], uint64(blob.MinSliceID)) + n += binary.PutUvarint(buf[n:], uint64(blob.Vid)) + n += binary.PutUvarint(buf[n:], uint64(blob.Count)) + } + + return n +} + +// Decode parse location from buf +// Returns the number of bytes read +// Error is not nil when parsing failed +func (loc *Location) Decode(buf []byte) (int, error) { + if loc == nil { + return 0, fmt.Errorf("location receiver is nil") + } + + location, n, err := DecodeLocation(buf) + if err != nil { + return n, err + } + + *loc = location + return n, nil +} + +// ToString transfer location to hex string +func (loc *Location) ToString() string { + return loc.HexString() +} + +// HexString transfer location to hex string +func (loc *Location) HexString() string { + return hex.EncodeToString(loc.Encode()) +} + +// Base64String transfer location to base64 string +func (loc *Location) Base64String() string { + return base64.StdEncoding.EncodeToString(loc.Encode()) +} + +// Spread location blobs to slice +func (loc *Location) Spread() []BlobUnit { + count := 0 + for _, blob := range loc.Slices { + count += int(blob.Count) + } + + blobs := make([]BlobUnit, 0, count) + for _, blob := range loc.Slices { + for offset := uint32(0); offset < blob.Count; offset++ { + blobs = append(blobs, BlobUnit{ + Bid: blob.MinSliceID + BlobID(offset), + Vid: blob.Vid, + Size: loc.SliceSize, + }) + } + } + if len(blobs) > 0 && loc.SliceSize > 0 { + if lastSize := loc.Size_ % uint64(loc.SliceSize); lastSize > 0 { + blobs[len(blobs)-1].Size = uint32(lastSize) + } + } + return blobs +} + +// DecodeLocation parse location from buf +// Returns Location and the number of bytes read +// Error is not nil when parsing failed +func DecodeLocation(buf []byte) (Location, int, error) { + var ( + loc Location + n int + + val uint64 + nn int + ) + next := func() (uint64, int) { + val, nn := binary.Uvarint(buf) + if nn <= 0 { + return 0, nn + } + n += nn + buf = buf[nn:] + return val, nn + } + + if len(buf) < 4 { + return loc, n, fmt.Errorf("bytes crc %d", len(buf)) + } + loc.Crc = binary.BigEndian.Uint32(buf) + n += 4 + buf = buf[4:] + + if val, nn = next(); nn <= 0 { + return loc, n, fmt.Errorf("bytes cluster_id %d", nn) + } + loc.ClusterID = ClusterID(val) + + if len(buf) < 1 { + return loc, n, fmt.Errorf("bytes codemode %d", len(buf)) + } + loc.CodeMode = codemode.CodeMode(buf[0]) + n++ + buf = buf[1:] + + if val, nn = next(); nn <= 0 { + return loc, n, fmt.Errorf("bytes size %d", nn) + } + loc.Size_ = val + + if val, nn = next(); nn <= 0 { + return loc, n, fmt.Errorf("bytes blob_size %d", nn) + } + loc.SliceSize = uint32(val) + + if val, nn = next(); nn <= 0 { + return loc, n, fmt.Errorf("bytes length blobs %d", nn) + } + length := int(val) + + if length > 0 { + loc.Slices = make([]Slice, 0, length) + } + for index := 0; index < length; index++ { + var blob Slice + if val, nn = next(); nn <= 0 { + return loc, n, fmt.Errorf("bytes %dth-blob min_bid %d", index, nn) + } + blob.MinSliceID = BlobID(val) + + if val, nn = next(); nn <= 0 { + return loc, n, fmt.Errorf("bytes %dth-blob vid %d", index, nn) + } + blob.Vid = Vid(val) + + if val, nn = next(); nn <= 0 { + return loc, n, fmt.Errorf("bytes %dth-blob count %d", index, nn) + } + blob.Count = uint32(val) + + loc.Slices = append(loc.Slices, blob) + } + + return loc, n, nil +} + +// DecodeLocationFrom decode location from hex string +func DecodeLocationFrom(s string) (Location, error) { + return DecodeLocationFromHex(s) +} + +// DecodeLocationFromHex decode location from hex string +func DecodeLocationFromHex(s string) (Location, error) { + var loc Location + src, err := hex.DecodeString(s) + if err != nil { + return loc, err + } + _, err = loc.Decode(src) + if err != nil { + return loc, err + } + return loc, nil +} + +// DecodeLocationFromBase64 decode location from base64 string +func DecodeLocationFromBase64(s string) (Location, error) { + var loc Location + src, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return loc, err + } + _, err = loc.Decode(src) + if err != nil { + return loc, err + } + return loc, nil +} diff --git a/blobstore/common/proto/blob_ext_test.go b/blobstore/common/proto/blob_ext_test.go new file mode 100644 index 000000000..7382790a9 --- /dev/null +++ b/blobstore/common/proto/blob_ext_test.go @@ -0,0 +1,210 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package proto + +import ( + "math" + mrand "math/rand" + "testing" + + "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/stretchr/testify/require" +) + +func TestLocationEncodeDecodeNil(t *testing.T) { + var loc *Location + require.Nil(t, loc.Encode()) + require.Equal(t, 0, loc.Encode2(nil)) + require.Equal(t, "", loc.ToString()) + require.Equal(t, "", loc.HexString()) + require.Equal(t, "", loc.Base64String()) + + n, err := loc.Decode(nil) + require.Error(t, err) + require.Equal(t, 0, n) + + locx, n, err := DecodeLocation(nil) + require.Error(t, err) + require.Equal(t, 0, n) + require.Equal(t, Location{}, locx) + + locx, err = DecodeLocationFrom("") + require.Error(t, err) + require.Equal(t, Location{}, locx) + + locx, err = DecodeLocationFrom("xxx") + require.Error(t, err) + require.Equal(t, Location{}, locx) + + locx, err = DecodeLocationFromHex("xxx") + require.Error(t, err) + require.Equal(t, Location{}, locx) + + locx, err = DecodeLocationFromBase64("xxx") + require.Error(t, err) + require.Equal(t, Location{}, locx) +} + +func TestLocationEncodeDecode(t *testing.T) { + for ii := 0; ii < 100; ii++ { + loc := &Location{ + ClusterID: ClusterID(mrand.Uint32()), + CodeMode: codemode.CodeMode(mrand.Intn(0xff)), + Size_: mrand.Uint64(), + SliceSize: mrand.Uint32(), + Crc: mrand.Uint32(), + } + + num := mrand.Intn(5) + for i := 0; i < num; i++ { + loc.Slices = append(loc.Slices, Slice{ + MinSliceID: BlobID(mrand.Uint64()), + Vid: Vid(mrand.Uint32()), + Count: mrand.Uint32(), + }) + } + + buf := loc.Encode() + bufx := make([]byte, len(buf)) + n := loc.Encode2(bufx) + require.Equal(t, len(buf), n) + require.Equal(t, buf, bufx) + + require.Panics(t, func() { loc.Encode2(nil) }) + require.Panics(t, func() { loc.Encode2(bufx[:3]) }) + require.Panics(t, func() { loc.Encode2(bufx[:n/2]) }) + require.Panics(t, func() { loc.Encode2(bufx[:n-1]) }) + + locx := Location{} + locx.Decode(bufx) + require.Equal(t, loc.ToString(), locx.ToString()) + require.Equal(t, loc.HexString(), locx.HexString()) + require.Equal(t, loc.Base64String(), locx.Base64String()) + + str := loc.ToString() + locx, err := DecodeLocationFrom(str) + require.NoError(t, err) + require.Equal(t, *loc, locx) + + str = loc.HexString() + locx, err = DecodeLocationFromHex(str) + require.NoError(t, err) + require.Equal(t, *loc, locx) + + str = loc.Base64String() + locx, err = DecodeLocationFromBase64(str) + require.NoError(t, err) + require.Equal(t, *loc, locx) + } +} + +func TestLocationDecodeError(t *testing.T) { + loc := &Location{ + ClusterID: ClusterID(math.MaxUint32), + CodeMode: codemode.CodeMode(math.MaxInt8), + Size_: math.MaxUint64, + SliceSize: math.MaxUint32, + Crc: math.MaxUint32, + } + + buf := loc.Encode() + require.Equal(t, 25+1, len(buf)) + for _, n := range []int{3, 8, 9, 19, 24} { + _, _, err := DecodeLocation(buf[:n]) + require.Error(t, err) + t.Log(err) + } + + loc.Slices = append(loc.Slices, Slice{ + MinSliceID: BlobID(math.MaxUint64), + Vid: Vid(math.MaxUint32), + Count: math.MaxUint32, + }) + + buf = loc.Encode() + require.Equal(t, 25+1+20, len(buf)) + for _, n := range []int{25, 35, 40, 45} { + _, _, err := DecodeLocation(buf[:n]) + require.Error(t, err) + t.Log(err) + } +} + +func TestLocationSpread(t *testing.T) { + { + var loc Location + blobs := loc.Spread() + require.NotNil(t, blobs) + require.Equal(t, 0, len(blobs)) + } + { + loc := &Location{ + Size_: 10, + SliceSize: 1 << 22, + Slices: []Slice{{ + MinSliceID: 100, + Vid: 4, + Count: 1, + }}, + } + blobs := loc.Spread() + require.Equal(t, 1, len(blobs)) + require.Equal(t, BlobID(100), blobs[0].Bid) + require.Equal(t, Vid(4), blobs[0].Vid) + require.Equal(t, uint32(10), blobs[0].Size) + } + { + loc := &Location{ + Size_: (1 << 22) + 10, + SliceSize: 1 << 22, + Slices: []Slice{{ + MinSliceID: 100, + Vid: 4, + Count: 2, + }}, + } + blobs := loc.Spread() + require.Equal(t, 2, len(blobs)) + require.Equal(t, BlobID(100), blobs[0].Bid) + require.Equal(t, Vid(4), blobs[0].Vid) + require.Equal(t, uint32(1<<22), blobs[0].Size) + require.Equal(t, BlobID(101), blobs[1].Bid) + require.Equal(t, Vid(4), blobs[1].Vid) + require.Equal(t, uint32(10), blobs[1].Size) + } + { + loc := &Location{ + Size_: 1 << 23, + SliceSize: 1 << 22, + Slices: []Slice{{ + MinSliceID: 100, + Vid: 4, + Count: 1, + }, { + MinSliceID: 200, + Vid: 4, + Count: 1, + }}, + } + blobs := loc.Spread() + require.Equal(t, 2, len(blobs)) + require.Equal(t, BlobID(100), blobs[0].Bid) + require.Equal(t, Vid(4), blobs[0].Vid) + require.Equal(t, uint32(1<<22), blobs[0].Size) + require.Equal(t, BlobID(200), blobs[1].Bid) + require.Equal(t, Vid(4), blobs[1].Vid) + require.Equal(t, uint32(1<<22), blobs[1].Size) + } +} diff --git a/blobstore/common/sharding/proto_test.go b/blobstore/common/sharding/proto_test.go index 936175b4a..d722bbd71 100644 --- a/blobstore/common/sharding/proto_test.go +++ b/blobstore/common/sharding/proto_test.go @@ -1,3 +1,17 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + package sharding import ( diff --git a/blobstore/common/sharding/range.pb.go b/blobstore/common/sharding/range.pb.go index 77e3cf57e..bd5ac52f6 100644 --- a/blobstore/common/sharding/range.pb.go +++ b/blobstore/common/sharding/range.pb.go @@ -1,3 +1,17 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + // Code generated by protoc-gen-gogo. DO NOT EDIT. // source: range.proto diff --git a/blobstore/common/sharding/range.proto b/blobstore/common/sharding/range.proto index 3858a2288..23f1afd75 100644 --- a/blobstore/common/sharding/range.proto +++ b/blobstore/common/sharding/range.proto @@ -1,3 +1,17 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + syntax = "proto3"; package cubefs.blobstore.common.sharding; diff --git a/blobstore/common/sharding/rangeext_test.go b/blobstore/common/sharding/rangeext_test.go index 48ef595e5..208cadbee 100644 --- a/blobstore/common/sharding/rangeext_test.go +++ b/blobstore/common/sharding/rangeext_test.go @@ -1,3 +1,17 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + package sharding import ( diff --git a/blobstore/sdk/sdk_client.go b/blobstore/sdk/sdk_client.go index cd2040f6e..3c52b6512 100644 --- a/blobstore/sdk/sdk_client.go +++ b/blobstore/sdk/sdk_client.go @@ -114,7 +114,7 @@ func (s *sdkHandler) Get(ctx context.Context, args *acapi.GetArgs) (io.ReadClose } ctx = acapi.ClientWithReqidContext(ctx) - if args.Location.Size == 0 || args.ReadSize == 0 { + if args.Location.Size_ == 0 || args.ReadSize == 0 { return noopBody{}, nil } @@ -129,15 +129,15 @@ func (s *sdkHandler) Get(ctx context.Context, args *acapi.GetArgs) (io.ReadClose return s.doGet(ctx, args) } -func (s *sdkHandler) Delete(ctx context.Context, args *acapi.DeleteArgs) (failedLocations []acapi.Location, err error) { +func (s *sdkHandler) Delete(ctx context.Context, args *acapi.DeleteArgs) (failedLocations []proto.Location, err error) { if !args.IsValid() { return nil, errcode.ErrIllegalArguments } ctx = acapi.ClientWithReqidContext(ctx) - locations := make([]acapi.Location, 0, len(args.Locations)) // check location size + locations := make([]proto.Location, 0, len(args.Locations)) // check location size for _, loc := range args.Locations { - if loc.Size > 0 { + if loc.Size_ > 0 { locations = append(locations, loc) } } @@ -170,9 +170,9 @@ func (s *sdkHandler) Delete(ctx context.Context, args *acapi.DeleteArgs) (failed return nil, nil } -func (s *sdkHandler) Put(ctx context.Context, args *acapi.PutArgs) (lc acapi.Location, hm acapi.HashSumMap, err error) { +func (s *sdkHandler) Put(ctx context.Context, args *acapi.PutArgs) (lc proto.Location, hm acapi.HashSumMap, err error) { if args == nil { - return acapi.Location{}, nil, errcode.ErrIllegalArguments + return proto.Location{}, nil, errcode.ErrIllegalArguments } if args.Size == 0 { @@ -180,7 +180,7 @@ func (s *sdkHandler) Put(ctx context.Context, args *acapi.PutArgs) (lc acapi.Loc for alg := range hashSumMap { hashSumMap[alg] = alg.ToHasher().Sum(nil) } - return acapi.Location{Blobs: make([]acapi.SliceInfo, 0)}, hashSumMap, nil + return proto.Location{Slices: make([]proto.Slice, 0)}, hashSumMap, nil } ctx = acapi.ClientWithReqidContext(ctx) @@ -189,7 +189,7 @@ func (s *sdkHandler) Put(ctx context.Context, args *acapi.PutArgs) (lc acapi.Loc if err := s.limiter.Acquire(name); err != nil { span := trace.SpanFromContextSafe(ctx) span.Debugf("access concurrent limited %s, err:%+v", name, err) - return acapi.Location{}, nil, errcode.ErrAccessLimited + return proto.Location{}, nil, errcode.ErrAccessLimited } defer s.limiter.Release(name) @@ -268,13 +268,13 @@ func (s *sdkHandler) deleteBlob(ctx context.Context, args *acapi.DeleteBlobArgs) return errcode.ErrIllegalArguments } - if err := s.handler.Delete(ctx, &acapi.Location{ + if err := s.handler.Delete(ctx, &proto.Location{ ClusterID: args.ClusterID, - BlobSize: 1, - Blobs: []acapi.SliceInfo{{ - MinBid: args.BlobID, - Vid: args.Vid, - Count: 1, + SliceSize: 1, + Slices: []proto.Slice{{ + MinSliceID: args.BlobID, + Vid: args.Vid, + Count: 1, }}, }); err != nil { span.Error("stream delete blob failed", errors.Detail(err)) @@ -373,14 +373,14 @@ func (s *sdkHandler) doDelete(ctx context.Context, args *acapi.DeleteArgs) (resp err = errcode.ErrIllegalArguments return } - clusterBlobsN[loc.ClusterID] += len(loc.Blobs) + clusterBlobsN[loc.ClusterID] += len(loc.Slices) } if len(args.Locations) == 1 { loc := args.Locations[0] if err := s.handler.Delete(ctx, &loc); err != nil { span.Error("stream delete failed", errors.Detail(err)) - resp.FailedLocations = []acapi.Location{loc} + resp.FailedLocations = []proto.Location{loc} } return } @@ -391,19 +391,19 @@ func (s *sdkHandler) doDelete(ctx context.Context, args *acapi.DeleteArgs) (resp // a min delete message about 10-20 bytes, // max delete locations is 1024, one location is max to 5G, // merged message max size about 40MB. - merged := make(map[proto.ClusterID][]acapi.SliceInfo, len(clusterBlobsN)) + merged := make(map[proto.ClusterID][]proto.Slice, len(clusterBlobsN)) for id, n := range clusterBlobsN { - merged[id] = make([]acapi.SliceInfo, 0, n) + merged[id] = make([]proto.Slice, 0, n) } for _, loc := range args.Locations { - merged[loc.ClusterID] = append(merged[loc.ClusterID], loc.Blobs...) + merged[loc.ClusterID] = append(merged[loc.ClusterID], loc.Slices...) } for cid := range merged { - if err := s.handler.Delete(ctx, &acapi.Location{ + if err := s.handler.Delete(ctx, &proto.Location{ ClusterID: cid, - BlobSize: 1, - Blobs: merged[cid], + SliceSize: 1, + Slices: merged[cid], }); err != nil { span.Error("stream delete failed", cid, errors.Detail(err)) for i := range args.Locations { @@ -417,7 +417,7 @@ func (s *sdkHandler) doDelete(ctx context.Context, args *acapi.DeleteArgs) (resp return } -func (s *sdkHandler) doPutObject(ctx context.Context, args *acapi.PutArgs) (acapi.Location, acapi.HashSumMap, error) { +func (s *sdkHandler) doPutObject(ctx context.Context, args *acapi.PutArgs) (proto.Location, acapi.HashSumMap, error) { span := trace.SpanFromContextSafe(ctx) var err error @@ -425,7 +425,7 @@ func (s *sdkHandler) doPutObject(ctx context.Context, args *acapi.PutArgs) (acap if !args.IsValid() { err = errcode.ErrIllegalArguments span.Error("stream get args is invalid ", errors.Detail(err)) - return acapi.Location{}, nil, err + return proto.Location{}, nil, err } hashSumMap := args.Hashes.ToHashSumMap() @@ -440,7 +440,7 @@ func (s *sdkHandler) doPutObject(ctx context.Context, args *acapi.PutArgs) (acap if err != nil { span.Error("stream put failed", errors.Detail(err)) err = httpError(err) - return acapi.Location{}, nil, err + return proto.Location{}, nil, err } // hasher sum @@ -451,7 +451,7 @@ func (s *sdkHandler) doPutObject(ctx context.Context, args *acapi.PutArgs) (acap if err = stream.LocationCrcFill(loc); err != nil { span.Error("stream put fill location crc", err) err = httpError(err) - return acapi.Location{}, nil, err + return proto.Location{}, nil, err } span.Infof("done /put request location:%+v hash:%+v", loc, hashSumMap.All()) @@ -604,7 +604,7 @@ func (s *sdkHandler) readerPipeline(span trace.Span, reqBody io.Reader, return ch } -func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.Location, acapi.HashSumMap, error) { +func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (proto.Location, acapi.HashSumMap, error) { span := trace.SpanFromContextSafe(ctx) hashSumMap := args.Hashes.ToHashSumMap() @@ -619,7 +619,7 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L } var ( - loc acapi.Location + loc proto.Location tokens []string ) @@ -637,7 +637,7 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L signArgs.Location = loc.Copy() signResp, err := s.sign(newCtx, &signArgs) if err == nil { - locations = []acapi.Location{signResp.Location.Copy()} + locations = []proto.Location{signResp.Location.Copy()} } } if len(locations) > 0 { @@ -650,7 +650,7 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L // alloc allocResp, err := s.Alloc(ctx, &acapi.AllocArgs{Size: uint64(args.Size)}) if err != nil { - return acapi.Location{}, nil, err + return proto.Location{}, nil, err } loc = allocResp.Location tokens = allocResp.Tokens @@ -658,7 +658,7 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L // buffer pipeline closeCh := make(chan struct{}) - bufferPipe := s.readerPipeline(span, reqBody, closeCh, int(loc.Size), int(loc.BlobSize)) + bufferPipe := s.readerPipeline(span, reqBody, closeCh, int(loc.Size_), int(loc.SliceSize)) defer func() { close(closeCh) // waiting pipeline close if has error @@ -677,17 +677,17 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L currBlobIdx := 0 currBlobCount := uint32(0) - remainSize := loc.Size + remainSize := loc.Size_ restPartsLoc := loc readSize := 0 - for readSize < int(loc.Size) { + for readSize < int(loc.Size_) { parts := make([]blobPart, 0, s.conf.PartConcurrence) // waiting at least one blob buf, ok := <-bufferPipe - if !ok && readSize < int(loc.Size) { - return acapi.Location{}, nil, errcode.ErrAccessReadRequestBody + if !ok && readSize < int(loc.Size_) { + return proto.Location{}, nil, errcode.ErrAccessReadRequestBody } readSize += len(buf) parts = append(parts, blobPart{size: len(buf), buf: buf}) @@ -697,9 +697,9 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L select { case buf, ok := <-bufferPipe: if !ok { - if readSize < int(loc.Size) { + if readSize < int(loc.Size_) { releaseBuffer(parts) - return acapi.Location{}, nil, errcode.ErrAccessReadRequestBody + return proto.Location{}, nil, errcode.ErrAccessReadRequestBody } more = false } else { @@ -713,9 +713,9 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L tryTimes := s.conf.MaxRetry for { - if len(loc.Blobs) > acapi.MaxLocationBlobs { + if len(loc.Slices) > acapi.MaxLocationBlobs { releaseBuffer(parts) - return acapi.Location{}, nil, errcode.ErrUnexpected + return proto.Location{}, nil, errcode.ErrUnexpected } // feed new params @@ -723,16 +723,16 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L currCount := currBlobCount for i := range parts { token := tokens[currIdx] - if restPartsLoc.Size > uint64(loc.BlobSize) && parts[i].size < int(loc.BlobSize) { + if restPartsLoc.Size_ > uint64(loc.SliceSize) && parts[i].size < int(loc.SliceSize) { token = tokens[currIdx+1] } parts[i].token = token parts[i].cid = loc.ClusterID - parts[i].vid = loc.Blobs[currIdx].Vid - parts[i].bid = loc.Blobs[currIdx].MinBid + proto.BlobID(currCount) + parts[i].vid = loc.Slices[currIdx].Vid + parts[i].bid = loc.Slices[currIdx].MinSliceID + proto.BlobID(currCount) currCount++ - if loc.Blobs[currIdx].Count == currCount { + if loc.Slices[currIdx].Count == currCount { currIdx++ currCount = 0 } @@ -744,7 +744,7 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L remainSize -= uint64(part.size) currBlobCount++ // next blobs - if loc.Blobs[currBlobIdx].Count == currBlobCount { + if loc.Slices[currBlobIdx].Count == currBlobCount { currBlobIdx++ currBlobCount = 0 } @@ -758,7 +758,7 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L if tryTimes == 1 { releaseBuffer(parts) span.Error("exceed the max retry limit", s.conf.MaxRetry) - return acapi.Location{}, nil, errcode.ErrUnexpected + return proto.Location{}, nil, errcode.ErrUnexpected } tryTimes-- } @@ -768,15 +768,15 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L err = retry.Timed(s.conf.MaxRetry, s.conf.RetryDelayMs).RuptOn(func() (bool, error) { resp, err1 := s.Alloc(ctx, &acapi.AllocArgs{ Size: remainSize, - BlobSize: loc.BlobSize, + BlobSize: loc.SliceSize, AssignClusterID: loc.ClusterID, CodeMode: loc.CodeMode, }) if err1 != nil { return true, err1 } - if len(resp.Location.Blobs) > 0 { - if newVid := resp.Location.Blobs[0].Vid; newVid == loc.Blobs[currBlobIdx].Vid { + if len(resp.Location.Slices) > 0 { + if newVid := resp.Location.Slices[0].Vid; newVid == loc.Slices[currBlobIdx].Vid { return false, fmt.Errorf("alloc the same vid %d", newVid) } } @@ -786,17 +786,17 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L if err != nil { releaseBuffer(parts) span.Error("alloc another parts to put", err) - return acapi.Location{}, nil, errcode.ErrUnexpected + return proto.Location{}, nil, errcode.ErrUnexpected } restPartsLoc = restPartsResp.Location signArgs.Locations = append(signArgs.Locations, restPartsLoc.Copy()) if currBlobCount > 0 { - loc.Blobs[currBlobIdx].Count = currBlobCount + loc.Slices[currBlobIdx].Count = currBlobCount currBlobIdx++ } - loc.Blobs = append(loc.Blobs[:currBlobIdx], restPartsLoc.Blobs...) + loc.Slices = append(loc.Slices[:currBlobIdx], restPartsLoc.Slices...) tokens = append(tokens[:currBlobIdx], restPartsResp.Tokens...) currBlobCount = 0 @@ -811,7 +811,7 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L signResp, err1 := s.sign(ctx, &signArgs) if err1 != nil { span.Error("sign location with crc", err1) - return acapi.Location{}, nil, errcode.ErrUnexpected + return proto.Location{}, nil, errcode.ErrUnexpected } loc = signResp.Location } diff --git a/blobstore/sdk/sdk_client_test.go b/blobstore/sdk/sdk_client_test.go index 8824c715d..3546d7556 100644 --- a/blobstore/sdk/sdk_client_test.go +++ b/blobstore/sdk/sdk_client_test.go @@ -14,6 +14,7 @@ import ( acapi "github.com/cubefs/cubefs/blobstore/api/access" "github.com/cubefs/cubefs/blobstore/common/codemode" errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" "github.com/cubefs/cubefs/blobstore/testing/mocks" "github.com/cubefs/cubefs/blobstore/util/closer" "github.com/cubefs/cubefs/blobstore/util/log" @@ -62,12 +63,12 @@ func TestSdkHandler_Delete(t *testing.T) { require.NotNil(t, err) require.ErrorIs(t, err, errcode.ErrIllegalArguments) - args := &acapi.DeleteArgs{Locations: []acapi.Location{{}}} + args := &acapi.DeleteArgs{Locations: []proto.Location{{}}} ret, err := hd.Delete(ctx, args) require.NoError(t, err) require.Nil(t, ret) - args.Locations[0].Size = 1 + args.Locations[0].Size_ = 1 _, err = hd.Delete(ctx, args) require.NotNil(t, err) require.ErrorIs(t, err, errcode.ErrIllegalArguments) @@ -76,7 +77,7 @@ func TestSdkHandler_Delete(t *testing.T) { hd.handler.(*mocks.MockStreamHandler).EXPECT().Delete(any, any).Times(3).Return(errMock) crc, _ := stream.LocationCrcCalculate(&args.Locations[0]) args.Locations[0].Crc = crc - args.Locations[0].Blobs = make([]acapi.SliceInfo, 0) + args.Locations[0].Slices = make([]proto.Slice, 0) ret, err = hd.Delete(ctx, args) require.NotNil(t, err) require.ErrorIs(t, err, errcode.ErrUnexpected) @@ -88,14 +89,14 @@ func TestSdkHandler_Delete(t *testing.T) { require.Nil(t, ret) // 3 location - loc := acapi.Location{ + loc := proto.Location{ ClusterID: 1, - Size: 1, - Blobs: []acapi.SliceInfo{{Vid: 9}}, + Size_: 1, + Slices: []proto.Slice{{Vid: 9}}, } crc, _ = stream.LocationCrcCalculate(&loc) loc.Crc = crc - args.Locations = make([]acapi.Location, 0) + args.Locations = make([]proto.Location, 0) for len(args.Locations) < 3 { args.Locations = append(args.Locations, loc) } @@ -131,8 +132,8 @@ func TestSdkHandler_Get(t *testing.T) { // args error args := &acapi.GetArgs{ ReadSize: 1, - Location: acapi.Location{ - Size: 2, + Location: proto.Location{ + Size_: 2, }, } _, err = hd.Get(ctx, args) @@ -167,12 +168,12 @@ func TestSdkHandler_Get(t *testing.T) { // ok data := "test read" args.ReadSize = uint64(len(data)) - args.Location.Size = args.ReadSize + args.Location.Size_ = args.ReadSize crc, _ = stream.LocationCrcCalculate(&args.Location) args.Location.Crc = crc rd, wr := io.Pipe() hd.handler.(*mocks.MockStreamHandler).EXPECT().Get(any, any, any, any, any).DoAndReturn( - func(ctx context.Context, w io.Writer, location acapi.Location, readSize, offset uint64) (func() error, error) { + func(ctx context.Context, w io.Writer, location proto.Location, readSize, offset uint64) (func() error, error) { w = wr if readSize > 0 && readSize < 1024 { return func() error { @@ -193,13 +194,13 @@ func TestSdkHandler_Get(t *testing.T) { // ok, zero copy data = "test read" args.ReadSize = uint64(len(data)) - args.Location.Size = args.ReadSize + args.Location.Size_ = args.ReadSize crc, _ = stream.LocationCrcCalculate(&args.Location) args.Location.Crc = crc buff := bytes.NewBuffer([]byte{}) args.Writer = buff hd.handler.(*mocks.MockStreamHandler).EXPECT().Get(any, args.Writer, any, any, any).DoAndReturn( - func(ctx context.Context, w io.Writer, location acapi.Location, readSize, offset uint64) (func() error, error) { + func(ctx context.Context, w io.Writer, location proto.Location, readSize, offset uint64) (func() error, error) { if readSize > 0 && readSize < 1024 { return func() error { _, err1 := w.Write([]byte(data)) @@ -230,7 +231,7 @@ func TestSdkHandler_Put(t *testing.T) { // size 0 loc, hash, err := hd.Put(ctx, &acapi.PutArgs{Hashes: 1}) require.NoError(t, err) - require.Equal(t, uint64(0), loc.Size) + require.Equal(t, uint64(0), loc.Size_) require.Equal(t, 1, len(hash)) args := &acapi.PutArgs{Size: 2} @@ -239,16 +240,16 @@ func TestSdkHandler_Put(t *testing.T) { hd.handler.(*mocks.MockStreamHandler).EXPECT().Put(any, any, any, any).Return(nil, errMock) loc, hash, err = hd.Put(ctx, args) require.NotNil(t, err) - require.Equal(t, uint64(0), loc.Size) + require.Equal(t, uint64(0), loc.Size_) require.Equal(t, 0, len(hash)) // ok args.Hashes = 1 - mockLoc := acapi.Location{Size: 2} + mockLoc := proto.Location{Size_: 2} hd.handler.(*mocks.MockStreamHandler).EXPECT().Put(any, any, any, any).Return(&mockLoc, nil) loc, hash, err = hd.Put(ctx, args) require.NoError(t, err) - require.Equal(t, mockLoc.Size, loc.Size) + require.Equal(t, mockLoc.Size_, loc.Size_) require.Equal(t, int(args.Hashes), len(hash)) // handler put error, retry 3 times @@ -258,7 +259,7 @@ func TestSdkHandler_Put(t *testing.T) { hd.handler.(*mocks.MockStreamHandler).EXPECT().Put(any, any, any, any).Return(nil, errMock).Times(3) loc, hash, err = hd.Put(ctx, args) require.NotNil(t, err) - require.Equal(t, uint64(0), loc.Size) + require.Equal(t, uint64(0), loc.Size_) require.Equal(t, 0, len(hash)) // retry ok @@ -271,7 +272,7 @@ func TestSdkHandler_Put(t *testing.T) { hd.handler.(*mocks.MockStreamHandler).EXPECT().Put(any, any, any, any).Return(&mockLoc, nil) loc, hash, err = hd.Put(ctx, args) require.NoError(t, err) - require.Equal(t, mockLoc.Size, loc.Size) + require.Equal(t, mockLoc.Size_, loc.Size_) require.Equal(t, int(args.Hashes), len(hash)) } @@ -296,12 +297,12 @@ func TestSdkHandler_Alloc(t *testing.T) { hd.handler.(*mocks.MockStreamHandler).EXPECT().Alloc(any, any, any, any, any).Return(nil, errMock) ret, err := hd.Alloc(ctx, args) require.NotNil(t, err) - require.Equal(t, acapi.Location{}, ret.Location) + require.Equal(t, proto.Location{}, ret.Location) - loca := &acapi.Location{ + loca := &proto.Location{ ClusterID: 1, - Size: 2, - BlobSize: 1, + Size_: 2, + SliceSize: 1, } crc, _ := stream.LocationCrcCalculate(loca) loca.Crc = crc @@ -326,21 +327,21 @@ func TestSdkHandler_putParts(t *testing.T) { loc, hash, err := hd.Put(ctx, args) require.NotNil(t, err) require.ErrorIs(t, err, errMock) - require.NotEqual(t, args.Size, int64(loc.Size)) + require.NotEqual(t, args.Size, int64(loc.Size_)) require.Nil(t, hash) // all ok data := "test alloc put" args.Body = bytes.NewBuffer([]byte(data)) args.Size = int64(len(data)) - loca := &acapi.Location{ + loca := &proto.Location{ ClusterID: 1, - Size: uint64(args.Size), - BlobSize: 4, - Blobs: []acapi.SliceInfo{{ - MinBid: 1001, - Vid: 10, - Count: 4, + Size_: uint64(args.Size), + SliceSize: 4, + Slices: []proto.Slice{{ + MinSliceID: 1001, + Vid: 10, + Count: 4, }}, } crc, _ := stream.LocationCrcCalculate(loca) @@ -374,14 +375,14 @@ func TestSdkHandler_putParts(t *testing.T) { loca.CodeMode = codemode.EC3P3 args.Body = bytes.NewBuffer([]byte(data)) hd.handler.(*mocks.MockStreamHandler).EXPECT().Alloc(any, any, any, any, any).Return(loca, nil) - locb := &acapi.Location{ + locb := &proto.Location{ ClusterID: 1, - Size: uint64(args.Size), - BlobSize: 4, - Blobs: []acapi.SliceInfo{{ - MinBid: 1001, - Vid: 11, // loca + 1 - Count: 4, + Size_: uint64(args.Size), + SliceSize: 4, + Slices: []proto.Slice{{ + MinSliceID: 1001, + Vid: 11, // loca + 1 + Count: 4, }}, } stream.LocationCrcFill(locb) diff --git a/blobstore/shardnode/svr.go b/blobstore/shardnode/svr.go index abbd50f61..5526e649d 100644 --- a/blobstore/shardnode/svr.go +++ b/blobstore/shardnode/svr.go @@ -17,22 +17,20 @@ package shardnode import ( "context" - "golang.org/x/sync/singleflight" "sync" - "github.com/cubefs/cubefs/blobstore/util/taskpool" - - "github.com/cubefs/cubefs/blobstore/shardnode/base" - - apierr "github.com/cubefs/cubefs/blobstore/common/errors" - "github.com/cubefs/cubefs/blobstore/shardnode/catalog" - "github.com/cubefs/cubefs/blobstore/util/closer" + "golang.org/x/sync/singleflight" "github.com/cubefs/cubefs/blobstore/api/clustermgr" + apierr "github.com/cubefs/cubefs/blobstore/common/errors" "github.com/cubefs/cubefs/blobstore/common/proto" "github.com/cubefs/cubefs/blobstore/common/raft" + "github.com/cubefs/cubefs/blobstore/shardnode/base" + "github.com/cubefs/cubefs/blobstore/shardnode/catalog" "github.com/cubefs/cubefs/blobstore/shardnode/storage" "github.com/cubefs/cubefs/blobstore/shardnode/storage/store" + "github.com/cubefs/cubefs/blobstore/util/closer" + "github.com/cubefs/cubefs/blobstore/util/taskpool" ) type Config struct { diff --git a/blobstore/testing/dial/connect.go b/blobstore/testing/dial/connect.go index 8469b30b6..61296b5b4 100644 --- a/blobstore/testing/dial/connect.go +++ b/blobstore/testing/dial/connect.go @@ -21,6 +21,7 @@ import ( "math/rand" "github.com/cubefs/cubefs/blobstore/api/access" + "github.com/cubefs/cubefs/blobstore/common/proto" "github.com/cubefs/cubefs/blobstore/common/trace" "github.com/cubefs/cubefs/blobstore/util/retry" "github.com/cubefs/cubefs/blobstore/util/task" @@ -80,7 +81,7 @@ func connect(conn Connection) { span, ctx = trace.StartSpanFromContextWithTraceID( context.Background(), "dial", "dial-"+hostname+"-"+trace.RandomID().String()) body io.ReadCloser - location access.Location + location proto.Location err error ) defer func() { @@ -89,7 +90,7 @@ func connect(conn Connection) { } if err = retry.Timed(10, 1).On(func() error { _, e := conn.api.Delete(ctx, &access.DeleteArgs{ - Locations: []access.Location{location}, + Locations: []proto.Location{location}, }) return e }); err != nil { @@ -145,7 +146,7 @@ func connect(conn Connection) { span.Infof("to delete conn:%v", conn) runTimer(conn, "del", 0, func() error { _, err = conn.api.Delete(ctx, &access.DeleteArgs{ - Locations: []access.Location{location}, + Locations: []proto.Location{location}, }) return err }) diff --git a/blobstore/testing/mocks/access_stream.go b/blobstore/testing/mocks/access_stream.go index b3e8ff754..6ccc791a0 100644 --- a/blobstore/testing/mocks/access_stream.go +++ b/blobstore/testing/mocks/access_stream.go @@ -53,10 +53,10 @@ func (mr *MockStreamHandlerMockRecorder) Admin() *gomock.Call { } // Alloc mocks base method. -func (m *MockStreamHandler) Alloc(arg0 context.Context, arg1 uint64, arg2 uint32, arg3 proto.ClusterID, arg4 codemode.CodeMode) (*access.Location, error) { +func (m *MockStreamHandler) Alloc(arg0 context.Context, arg1 uint64, arg2 uint32, arg3 proto.ClusterID, arg4 codemode.CodeMode) (*proto.Location, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Alloc", arg0, arg1, arg2, arg3, arg4) - ret0, _ := ret[0].(*access.Location) + ret0, _ := ret[0].(*proto.Location) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -68,7 +68,7 @@ func (mr *MockStreamHandlerMockRecorder) Alloc(arg0, arg1, arg2, arg3, arg4 inte } // Delete mocks base method. -func (m *MockStreamHandler) Delete(arg0 context.Context, arg1 *access.Location) error { +func (m *MockStreamHandler) Delete(arg0 context.Context, arg1 *proto.Location) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Delete", arg0, arg1) ret0, _ := ret[0].(error) @@ -82,7 +82,7 @@ func (mr *MockStreamHandlerMockRecorder) Delete(arg0, arg1 interface{}) *gomock. } // Get mocks base method. -func (m *MockStreamHandler) Get(arg0 context.Context, arg1 io.Writer, arg2 access.Location, arg3, arg4 uint64) (func() error, error) { +func (m *MockStreamHandler) Get(arg0 context.Context, arg1 io.Writer, arg2 proto.Location, arg3, arg4 uint64) (func() error, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2, arg3, arg4) ret0, _ := ret[0].(func() error) @@ -97,10 +97,10 @@ func (mr *MockStreamHandlerMockRecorder) Get(arg0, arg1, arg2, arg3, arg4 interf } // Put mocks base method. -func (m *MockStreamHandler) Put(arg0 context.Context, arg1 io.Reader, arg2 int64, arg3 access.HasherMap) (*access.Location, error) { +func (m *MockStreamHandler) Put(arg0 context.Context, arg1 io.Reader, arg2 int64, arg3 access.HasherMap) (*proto.Location, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Put", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(*access.Location) + ret0, _ := ret[0].(*proto.Location) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/blobstore/testing/mocks/api_access.go b/blobstore/testing/mocks/api_access.go index 81f9bb3ba..a13b8f90d 100644 --- a/blobstore/testing/mocks/api_access.go +++ b/blobstore/testing/mocks/api_access.go @@ -10,6 +10,7 @@ import ( reflect "reflect" access "github.com/cubefs/cubefs/blobstore/api/access" + proto "github.com/cubefs/cubefs/blobstore/common/proto" gomock "github.com/golang/mock/gomock" ) @@ -37,10 +38,10 @@ func (m *MockAccessAPI) EXPECT() *MockAccessAPIMockRecorder { } // Delete mocks base method. -func (m *MockAccessAPI) Delete(arg0 context.Context, arg1 *access.DeleteArgs) ([]access.Location, error) { +func (m *MockAccessAPI) Delete(arg0 context.Context, arg1 *access.DeleteArgs) ([]proto.Location, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Delete", arg0, arg1) - ret0, _ := ret[0].([]access.Location) + ret0, _ := ret[0].([]proto.Location) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -67,10 +68,10 @@ func (mr *MockAccessAPIMockRecorder) Get(arg0, arg1 interface{}) *gomock.Call { } // Put mocks base method. -func (m *MockAccessAPI) Put(arg0 context.Context, arg1 *access.PutArgs) (access.Location, access.HashSumMap, error) { +func (m *MockAccessAPI) Put(arg0 context.Context, arg1 *access.PutArgs) (proto.Location, access.HashSumMap, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Put", arg0, arg1) - ret0, _ := ret[0].(access.Location) + ret0, _ := ret[0].(proto.Location) ret1, _ := ret[1].(access.HashSumMap) ret2, _ := ret[2].(error) return ret0, ret1, ret2 diff --git a/sdk/data/blobstore/blobstore_client.go b/sdk/data/blobstore/blobstore_client.go index 1efa4df50..e1ca11096 100644 --- a/sdk/data/blobstore/blobstore_client.go +++ b/sdk/data/blobstore/blobstore_client.go @@ -64,22 +64,22 @@ func (ebs *BlobStoreClient) Read(ctx context.Context, volName string, buf []byte metric.SetWithLabels(err, map[string]string{exporter.Vol: volName}) }() blobs := oek.Blobs - sliceInfos := make([]access.SliceInfo, 0) + sliceInfos := make([]ebsproto.Slice, 0) for _, b := range blobs { - sliceInfo := access.SliceInfo{ - MinBid: ebsproto.BlobID(b.MinBid), - Vid: ebsproto.Vid(b.Vid), - Count: uint32(b.Count), + sliceInfo := ebsproto.Slice{ + MinSliceID: ebsproto.BlobID(b.MinBid), + Vid: ebsproto.Vid(b.Vid), + Count: uint32(b.Count), } sliceInfos = append(sliceInfos, sliceInfo) } - loc := access.Location{ + loc := ebsproto.Location{ ClusterID: ebsproto.ClusterID(oek.Cid), - Size: oek.Size, + Size_: oek.Size, Crc: oek.Crc, CodeMode: codemode.CodeMode(oek.CodeMode), - BlobSize: oek.BlobSize, - Blobs: sliceInfos, + SliceSize: oek.BlobSize, + Slices: sliceInfos, } // func get has retry log.LogDebugf("TRACE Ebs Read,oek(%v) loc(%v)", oek, loc) @@ -112,7 +112,7 @@ func (ebs *BlobStoreClient) Read(ctx context.Context, volName string, buf []byte return readN, nil } -func (ebs *BlobStoreClient) Write(ctx context.Context, volName string, data []byte, size uint32) (location access.Location, err error) { +func (ebs *BlobStoreClient) Write(ctx context.Context, volName string, data []byte, size uint32) (location ebsproto.Location, err error) { bgTime := stat.BeginStat() defer func() { stat.EndStat("ebs-write", err, bgTime, 1) @@ -156,26 +156,26 @@ func (ebs *BlobStoreClient) Delete(oeks []proto.ObjExtentKey) (err error) { ctx, cancel := context.WithTimeout(context.TODO(), time.Second*3) defer cancel() - locs := make([]access.Location, 0) + locs := make([]ebsproto.Location, 0) for _, oek := range oeks { - sliceInfos := make([]access.SliceInfo, 0) + sliceInfos := make([]ebsproto.Slice, 0) for _, b := range oek.Blobs { - sliceInfo := access.SliceInfo{ - MinBid: ebsproto.BlobID(b.MinBid), - Vid: ebsproto.Vid(b.Vid), - Count: uint32(b.Count), + sliceInfo := ebsproto.Slice{ + MinSliceID: ebsproto.BlobID(b.MinBid), + Vid: ebsproto.Vid(b.Vid), + Count: uint32(b.Count), } sliceInfos = append(sliceInfos, sliceInfo) } - loc := access.Location{ + loc := ebsproto.Location{ ClusterID: ebsproto.ClusterID(oek.Cid), - Size: oek.Size, + Size_: oek.Size, Crc: oek.Crc, CodeMode: codemode.CodeMode(oek.CodeMode), - BlobSize: oek.BlobSize, - Blobs: sliceInfos, + SliceSize: oek.BlobSize, + Slices: sliceInfos, } locs = append(locs, loc) } @@ -261,7 +261,7 @@ func (ebs *BlobStoreClient) Put(ctx context.Context, volName string, f io.Reader putSize = rest } rest -= putSize - var location access.Location + var location ebsproto.Location var hash access.HashSumMap location, hash, err = ebs.client.Put(ctx, &access.PutArgs{ Size: int64(putSize), @@ -290,11 +290,11 @@ func (ebs *BlobStoreClient) Put(ctx context.Context, volName string, f io.Reader return } -func locationToObjExtentKey(location access.Location, from uint64) (oek proto.ObjExtentKey) { +func locationToObjExtentKey(location ebsproto.Location, from uint64) (oek proto.ObjExtentKey) { blobs := make([]proto.Blob, 0) - for _, info := range location.Blobs { + for _, info := range location.Slices { blob := proto.Blob{ - MinBid: uint64(info.MinBid), + MinBid: uint64(info.MinSliceID), Count: uint64(info.Count), Vid: uint64(info.Vid), } @@ -303,8 +303,8 @@ func locationToObjExtentKey(location access.Location, from uint64) (oek proto.Ob oek = proto.ObjExtentKey{ Cid: uint64(location.ClusterID), CodeMode: uint8(location.CodeMode), - Size: location.Size, - BlobSize: location.BlobSize, + Size: location.Size_, + BlobSize: location.SliceSize, Blobs: blobs, BlobsLen: uint32(len(blobs)), FileOffset: from, @@ -329,22 +329,22 @@ func (ebs *BlobStoreClient) Get(ctx context.Context, volName string, offset uint metric.SetWithLabels(err, map[string]string{exporter.Vol: volName}) }() blobs := oek.Blobs - sliceInfos := make([]access.SliceInfo, 0) + sliceInfos := make([]ebsproto.Slice, 0) for _, b := range blobs { - sliceInfo := access.SliceInfo{ - MinBid: ebsproto.BlobID(b.MinBid), - Vid: ebsproto.Vid(b.Vid), - Count: uint32(b.Count), + sliceInfo := ebsproto.Slice{ + MinSliceID: ebsproto.BlobID(b.MinBid), + Vid: ebsproto.Vid(b.Vid), + Count: uint32(b.Count), } sliceInfos = append(sliceInfos, sliceInfo) } - loc := access.Location{ + loc := ebsproto.Location{ ClusterID: ebsproto.ClusterID(oek.Cid), - Size: oek.Size, + Size_: oek.Size, Crc: oek.Crc, CodeMode: codemode.CodeMode(oek.CodeMode), - BlobSize: oek.BlobSize, - Blobs: sliceInfos, + SliceSize: oek.BlobSize, + Slices: sliceInfos, } // func get has retry log.LogDebugf("TRACE Ebs Read, oek(%v) loc(%v)", oek, loc) diff --git a/sdk/data/blobstore/blobstore_client_test.go b/sdk/data/blobstore/blobstore_client_test.go index d3bd20624..6edfe20d5 100644 --- a/sdk/data/blobstore/blobstore_client_test.go +++ b/sdk/data/blobstore/blobstore_client_test.go @@ -27,6 +27,7 @@ import ( "github.com/cubefs/cubefs/blobstore/api/access" "github.com/cubefs/cubefs/blobstore/common/crc32block" + "github.com/cubefs/cubefs/blobstore/common/proto" "github.com/cubefs/cubefs/blobstore/util/bytespool" cproto "github.com/cubefs/cubefs/proto" "github.com/stretchr/testify/require" @@ -69,7 +70,7 @@ func NewMockEbsService() *MockEbsService { hashSumMap[alg] = hasher.Sum(nil) } - loc := access.Location{Size: uint64(dataSize)} + loc := proto.Location{Size_: uint64(dataSize)} fillCrc(&loc) resp := access.PutResp{ Location: loc, @@ -129,7 +130,7 @@ func requestBody(req *http.Request, val interface{}) { json.Unmarshal(data, val) } -func calcCrc(loc *access.Location) (uint32, error) { +func calcCrc(loc *proto.Location) (uint32, error) { crcWriter := crc32.New(crc32.IEEETable) buf := bytespool.Alloc(1024) @@ -147,7 +148,7 @@ func calcCrc(loc *access.Location) (uint32, error) { return crcWriter.Sum32(), nil } -func fillCrc(loc *access.Location) error { +func fillCrc(loc *proto.Location) error { crc, err := calcCrc(loc) if err != nil { return err @@ -156,7 +157,7 @@ func fillCrc(loc *access.Location) error { return nil } -func verifyCrc(loc *access.Location) bool { +func verifyCrc(loc *proto.Location) bool { crc, err := calcCrc(loc) if err != nil { return false @@ -192,9 +193,9 @@ func TestEbsClient_Write_Read(t *testing.T) { // read prepare blobs := make([]cproto.Blob, 0) - for _, info := range location.Blobs { + for _, info := range location.Slices { blob := cproto.Blob{ - MinBid: uint64(info.MinBid), + MinBid: uint64(info.MinSliceID), Count: uint64(info.Count), Vid: uint64(info.Vid), } @@ -203,8 +204,8 @@ func TestEbsClient_Write_Read(t *testing.T) { oek := cproto.ObjExtentKey{ Cid: uint64(location.ClusterID), CodeMode: uint8(location.CodeMode), - Size: location.Size, - BlobSize: location.BlobSize, + Size: location.Size_, + BlobSize: location.SliceSize, Blobs: blobs, BlobsLen: uint32(len(blobs)), Crc: location.Crc, diff --git a/sdk/data/blobstore/writer.go b/sdk/data/blobstore/writer.go index a1cfa78a7..a609864f6 100644 --- a/sdk/data/blobstore/writer.go +++ b/sdk/data/blobstore/writer.go @@ -474,9 +474,9 @@ func (writer *Writer) writeSlice(ctx context.Context, wSlice *rwSlice, wg bool) } log.LogDebugf("TRACE blobStore,location(%v)", location) blobs := make([]proto.Blob, 0) - for _, info := range location.Blobs { + for _, info := range location.Slices { blob := proto.Blob{ - MinBid: uint64(info.MinBid), + MinBid: uint64(info.MinSliceID), Count: uint64(info.Count), Vid: uint64(info.Vid), } @@ -485,8 +485,8 @@ func (writer *Writer) writeSlice(ctx context.Context, wSlice *rwSlice, wg bool) wSlice.objExtentKey = proto.ObjExtentKey{ Cid: uint64(location.ClusterID), CodeMode: uint8(location.CodeMode), - Size: location.Size, - BlobSize: location.BlobSize, + Size: location.Size_, + BlobSize: location.SliceSize, Blobs: blobs, BlobsLen: uint32(len(blobs)), FileOffset: wSlice.fileOffset, diff --git a/sdk/data/blobstore/writer_test.go b/sdk/data/blobstore/writer_test.go index 24a7e9bed..abe5a9c4a 100644 --- a/sdk/data/blobstore/writer_test.go +++ b/sdk/data/blobstore/writer_test.go @@ -21,6 +21,8 @@ import ( "syscall" "testing" + proto2 "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/brahma-adshonor/gohook" "github.com/stretchr/testify/assert" @@ -360,7 +362,7 @@ func TestBufferWrite(t *testing.T) { func TestWriteSlice(t *testing.T) { testCase := []struct { wg bool - ebsWriteFunc func(*BlobStoreClient, context.Context, string, []byte, uint32) (access.Location, error) + ebsWriteFunc func(*BlobStoreClient, context.Context, string, []byte, uint32) (proto2.Location, error) expectError error }{ {false, MockEbscWriteTrue, nil}, @@ -401,26 +403,26 @@ func TestWriteSlice(t *testing.T) { } } -func MockEbscWriteTrue(ebs *BlobStoreClient, ctx context.Context, volName string, data []byte, l uint32) (location access.Location, err error) { - loc := access.Location{ +func MockEbscWriteTrue(ebs *BlobStoreClient, ctx context.Context, volName string, data []byte, l uint32) (location proto2.Location, err error) { + loc := proto2.Location{ ClusterID: 1, CodeMode: 0, - Size: 100, - BlobSize: 100, + Size_: 100, + SliceSize: 100, Crc: 1024, - Blobs: []access.SliceInfo{ + Slices: []proto2.Slice{ { - MinBid: 1, - Vid: 1, - Count: 1, + MinSliceID: 1, + Vid: 1, + Count: 1, }, }, } return loc, nil } -func MockEbscWriteFalse(ebs *BlobStoreClient, ctx context.Context, volName string, data []byte, l uint32) (location access.Location, err error) { - return access.Location{}, syscall.EIO +func MockEbscWriteFalse(ebs *BlobStoreClient, ctx context.Context, volName string, data []byte, l uint32) (location proto2.Location, err error) { + return proto2.Location{}, syscall.EIO } func MockAppendObjExtentKeysTrue(mw *meta.MetaWrapper, inode uint64, eks []proto.ObjExtentKey) error {