gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 02:00:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(api): init blob api proto, add missing license

Browse Source 
with #22357426

Signed-off-by: xiejian <xiejian3@oppo.com>
This commit is contained in:
xiejian 2024-08-01 12:40:26 +08:00 committed by slasher
parent 6b84ed7f32
commit 1626e0f9fc
42 changed files with 4531 additions and 1083 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
blobstore/access/server.go
View File

@ -391,9 +391,9 @@ func (s *Service) Get(c *rpc.Context) {
w.Header().Set(rpc.HeaderContentType, rpc.MIMEStream)
w.Header().Set(rpc.HeaderContentLength, strconv.FormatInt(int64(args.ReadSize), 10))
if args.ReadSize > 0 && args.ReadSize != args.Location.Size {
if args.ReadSize > 0 && args.ReadSize != args.Location.Size_ {
w.Header().Set(rpc.HeaderContentRange, fmt.Sprintf("bytes %d-%d/%d",
args.Offset, args.Offset+args.ReadSize-1, args.Location.Size))
args.Offset, args.Offset+args.ReadSize-1, args.Location.Size_))
c.RespondStatus(http.StatusPartialContent)
} else {
c.RespondStatus(http.StatusOK)
@ -456,14 +456,14 @@ func (s *Service) Delete(c *rpc.Context) {
err = errcode.ErrIllegalArguments
return
}
clusterBlobsN[loc.ClusterID] += len(loc.Blobs)
clusterBlobsN[loc.ClusterID] += len(loc.Slices)
}
if len(args.Locations) == 1 {
loc := args.Locations[0]
if err := s.streamHandler.Delete(ctx, &loc); err != nil {
span.Error("stream delete failed", errors.Detail(err))
resp.FailedLocations = []access.Location{loc}
resp.FailedLocations = []proto.Location{loc}
}
return
}
@ -475,12 +475,12 @@ func (s *Service) Delete(c *rpc.Context) {
// max delete locations is 1024, one location is max to 5G,
// merged message max size about 40MB.
merged := make(map[proto.ClusterID][]access.SliceInfo, len(clusterBlobsN))
merged := make(map[proto.ClusterID][]proto.Slice, len(clusterBlobsN))
for id, n := range clusterBlobsN {
merged[id] = make([]access.SliceInfo, 0, n)
merged[id] = make([]proto.Slice, 0, n)
}
for _, loc := range args.Locations {
merged[loc.ClusterID] = append(merged[loc.ClusterID], loc.Blobs...)
merged[loc.ClusterID] = append(merged[loc.ClusterID], loc.Slices...)
}
var wg sync.WaitGroup
@ -489,7 +489,7 @@ func (s *Service) Delete(c *rpc.Context) {
go func() {
for id := range failedCh {
if resp.FailedLocations == nil {
resp.FailedLocations = make([]access.Location, 0, len(args.Locations))
resp.FailedLocations = make([]proto.Location, 0, len(args.Locations))
}
for _, loc := range args.Locations {
if loc.ClusterID == id {
@ -503,10 +503,10 @@ func (s *Service) Delete(c *rpc.Context) {
wg.Add(len(merged))
for id := range merged {
go func(id proto.ClusterID) {
if err := s.streamHandler.Delete(ctx, &access.Location{
if err := s.streamHandler.Delete(ctx, &proto.Location{
ClusterID: id,
BlobSize: 1,
Blobs: merged[id],
SliceSize: 1,
Slices: merged[id],
}); err != nil {
span.Error("stream delete failed", id, errors.Detail(err))
failedCh <- id
@ -551,13 +551,13 @@ func (s *Service) DeleteBlob(c *rpc.Context) {
return
}
if err := s.streamHandler.Delete(ctx, &access.Location{
if err := s.streamHandler.Delete(ctx, &proto.Location{
ClusterID: args.ClusterID,
BlobSize: 1,
Blobs: []access.SliceInfo{{
MinBid: args.BlobID,
Vid: args.Vid,
Count: 1,
SliceSize: 1,
Slices: []proto.Slice{{
MinSliceID: args.BlobID,
Vid: args.Vid,
Count: 1,
}},
}); err != nil {
span.Error("stream delete blob failed", errors.Detail(err))

210
blobstore/access/server_test.go
View File

@ -44,15 +44,15 @@ import (
var (
ctx = context.Background()
_blobSize uint32 = 1 << 20
location = &access.Location{
location = &proto.Location{
ClusterID: 1,
CodeMode: 1,
BlobSize: _blobSize,
SliceSize: _blobSize,
Crc: 0,
Blobs: []access.SliceInfo{{
MinBid: 111,
Vid: 1111,
Count: 11,
Slices: []proto.Slice{{
MinSliceID: 111,
Vid: 1111,
Count: 11,
}},
}
@ -74,12 +74,12 @@ func newService() *Service {
s.EXPECT().Alloc(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
func(ctx context.Context, size uint64, blobSize uint32,
assignClusterID proto.ClusterID, codeMode codemode.CodeMode,
) (*access.Location, error) {
) (*proto.Location, error) {
if size < 1024 {
return nil, errors.New("fake alloc location")
}
loc := location.Copy()
loc.Size = uint64(size)
loc.Size_ = uint64(size)
stream.LocationCrcFill(&loc)
return &loc, nil
})
@ -96,28 +96,28 @@ func newService() *Service {
})
s.EXPECT().Put(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
func(ctx context.Context, rc io.Reader, size int64, hasherMap access.HasherMap) (*access.Location, error) {
func(ctx context.Context, rc io.Reader, size int64, hasherMap access.HasherMap) (*proto.Location, error) {
if size < 1024 {
return nil, errors.New("fake put nil body")
}
loc := location.Copy()
loc.Size = uint64(size)
loc.Size_ = uint64(size)
stream.LocationCrcFill(&loc)
return &loc, nil
})
s.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
func(ctx context.Context, w io.Writer, location access.Location, readSize, offset uint64) (func() error, error) {
func(ctx context.Context, w io.Writer, location proto.Location, readSize, offset uint64) (func() error, error) {
if readSize < 1024 {
return nil, errors.New("fake get nil body")
}
return func() error { return nil }, nil
})
s.EXPECT().Delete(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
func(ctx context.Context, location *access.Location) error {
func(ctx context.Context, location *proto.Location) error {
if location.ClusterID >= 10 {
return errors.New("fake delete error with cluster")
} else if location.ClusterID == 1 && location.Crc > 0 && location.Size < 1024 {
} else if location.ClusterID == 1 && location.Crc > 0 && location.Size_ < 1024 {
return errors.New("fake delete error")
}
return nil
@ -172,21 +172,21 @@ func TestAccessServiceAlloc(t *testing.T) {
resp := &access.AllocResp{}
err := cli.PostWith(ctx, url(), resp, args)
require.NoError(t, err)
require.Equal(t, uint64(1024), resp.Location.Size)
require.Equal(t, uint64(1024), resp.Location.Size_)
}
{
args.Size = uint64(_blobSize)
resp := &access.AllocResp{}
err := cli.PostWith(ctx, url(), resp, args)
require.NoError(t, err)
require.Equal(t, uint64(_blobSize), resp.Location.Size)
require.Equal(t, uint64(_blobSize), resp.Location.Size_)
}
{
args.Size = uint64(_blobSize) + 1
resp := &access.AllocResp{}
err := cli.PostWith(ctx, url(), resp, args)
require.NoError(t, err)
require.Equal(t, uint64(_blobSize)+1, resp.Location.Size)
require.Equal(t, uint64(_blobSize)+1, resp.Location.Size_)
}
}
@ -280,7 +280,7 @@ func TestAccessServicePut(t *testing.T) {
resp := &access.PutResp{}
err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode())
require.NoError(t, err)
require.Equal(t, uint64(1024), resp.Location.Size)
require.Equal(t, uint64(1024), resp.Location.Size_)
}
}
}
@ -305,7 +305,7 @@ func TestAccessServiceGet(t *testing.T) {
require.Equal(t, 400, resp.StatusCode, resp.Status)
}
{
args.Location.Size = 1023
args.Location.Size_ = 1023
args.ReadSize = 1023
stream.LocationCrcFill(&args.Location)
resp, err := cli.Post(ctx, url(), args)
@ -314,7 +314,7 @@ func TestAccessServiceGet(t *testing.T) {
require.Equal(t, 500, resp.StatusCode, resp.Status)
}
{
args.Location.Size = 1024
args.Location.Size_ = 1024
args.ReadSize = 1024
stream.LocationCrcFill(&args.Location)
resp, err := cli.Post(ctx, url(), args)
@ -323,7 +323,7 @@ func TestAccessServiceGet(t *testing.T) {
require.Equal(t, 200, resp.StatusCode, resp.Status)
}
{
args.Location.Size = 10240
args.Location.Size_ = 10240
args.Offset = 1000
args.ReadSize = 1024
stream.LocationCrcFill(&args.Location)
@ -366,7 +366,7 @@ func TestAccessServiceDelete(t *testing.T) {
}
args := access.DeleteArgs{
Locations: []access.Location{location.Copy()},
Locations: []proto.Location{location.Copy()},
}
{
code, _, err := deleteRequest(access.DeleteArgs{})
@ -387,7 +387,7 @@ func TestAccessServiceDelete(t *testing.T) {
}
{
loc := &args.Locations[0]
loc.Size = 1024
loc.Size_ = 1024
stream.LocationCrcFill(loc)
code, _, err := deleteRequest(args)
require.NoError(t, err)
@ -395,9 +395,9 @@ func TestAccessServiceDelete(t *testing.T) {
}
{
loc := location.Copy()
loc.Size = 1024
loc.Size_ = 1024
stream.LocationCrcFill(&loc)
locs := make([]access.Location, access.MaxDeleteLocations)
locs := make([]proto.Location, access.MaxDeleteLocations)
for idx := range locs {
locs[idx] = loc
}
@ -408,9 +408,9 @@ func TestAccessServiceDelete(t *testing.T) {
}
{
loc := location.Copy()
loc.Size = 1024
loc.Size_ = 1024
stream.LocationCrcFill(&loc)
locs := make([]access.Location, access.MaxDeleteLocations+1)
locs := make([]proto.Location, access.MaxDeleteLocations+1)
for idx := range locs {
locs[idx] = loc
}
@ -420,20 +420,20 @@ func TestAccessServiceDelete(t *testing.T) {
}
{
loc := location.Copy()
loc.Size = 1024
loc.Size_ = 1024
loc.ClusterID = proto.ClusterID(11)
stream.LocationCrcFill(&loc)
code, resp, err := deleteRequest(access.DeleteArgs{Locations: []access.Location{loc}})
code, resp, err := deleteRequest(access.DeleteArgs{Locations: []proto.Location{loc}})
require.NoError(t, err)
require.Equal(t, 226, code)
require.Equal(t, 1, len(resp.FailedLocations))
require.Equal(t, proto.ClusterID(11), resp.FailedLocations[0].ClusterID)
}
{
locs := make([]access.Location, access.MaxDeleteLocations)
locs := make([]proto.Location, access.MaxDeleteLocations)
for idx := range locs {
loc := location.Copy()
loc.Size = 1024
loc.Size_ = 1024
loc.ClusterID = proto.ClusterID(idx % 11)
stream.LocationCrcFill(&loc)
locs[idx] = loc
@ -506,7 +506,7 @@ func TestAccessServiceSign(t *testing.T) {
return fmt.Sprintf("%s/sign", host)
}
args := access.SignArgs{
Locations: []access.Location{location.Copy()},
Locations: []proto.Location{location.Copy()},
Location: location.Copy(),
}
{
@ -535,149 +535,149 @@ func assertErrorCode(t *testing.T, code int, err error) {
func TestAccessServiceTokens(t *testing.T) {
skey := stream.TokenSecretKeys()[0][:]
checker := func(loc *access.Location, tokens []string) {
if loc.Size == 0 {
checker := func(loc *proto.Location, tokens []string) {
if loc.Size_ == 0 {
require.Equal(t, 0, len(tokens))
return
}
hasMultiBlobs := loc.Size >= uint64(loc.BlobSize)
lastSize := uint32(loc.Size % uint64(loc.BlobSize))
hasMultiBlobs := loc.Size_ >= uint64(loc.SliceSize)
lastSize := uint32(loc.Size_ % uint64(loc.SliceSize))
if !hasMultiBlobs {
require.Equal(t, 1, len(tokens))
token := uptoken.DecodeToken(tokens[0])
blob := loc.Blobs[0]
for bid := blob.MinBid - 100; bid < blob.MinBid+100; bid++ {
require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
blob := loc.Slices[0]
for bid := blob.MinSliceID - 100; bid < blob.MinSliceID+100; bid++ {
require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey))
}
require.True(t, token.IsValid(loc.ClusterID, blob.Vid, blob.MinBid, lastSize, skey))
require.True(t, token.IsValid(loc.ClusterID, blob.Vid, blob.MinSliceID, lastSize, skey))
return
}
if lastSize == 0 {
require.Equal(t, len(loc.Blobs), len(tokens))
for idx, blob := range loc.Blobs {
require.Equal(t, len(loc.Slices), len(tokens))
for idx, blob := range loc.Slices {
token := uptoken.DecodeToken(tokens[idx])
for ii := uint32(0); ii < 100; ii++ {
bid := blob.MinBid - proto.BlobID(ii) - 1
require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
bid = blob.MinBid + proto.BlobID(blob.Count+ii)
require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
bid := blob.MinSliceID - proto.BlobID(ii) - 1
require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey))
bid = blob.MinSliceID + proto.BlobID(blob.Count+ii)
require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey))
}
for ii := uint32(0); ii < blob.Count; ii++ {
bid := blob.MinBid + proto.BlobID(ii)
require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
bid := blob.MinSliceID + proto.BlobID(ii)
require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey))
}
}
return
}
require.Equal(t, len(loc.Blobs)+1, len(tokens))
for ii := 0; ii < len(loc.Blobs)-1; ii++ {
require.Equal(t, len(loc.Slices)+1, len(tokens))
for ii := 0; ii < len(loc.Slices)-1; ii++ {
token := uptoken.DecodeToken(tokens[ii])
blob := loc.Blobs[ii]
blob := loc.Slices[ii]
for ii := uint32(0); ii < blob.Count; ii++ {
bid := blob.MinBid + proto.BlobID(ii)
require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
bid := blob.MinSliceID + proto.BlobID(ii)
require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey))
}
}
token := uptoken.DecodeToken(tokens[len(loc.Blobs)-1])
blob := loc.Blobs[len(loc.Blobs)-1]
token := uptoken.DecodeToken(tokens[len(loc.Slices)-1])
blob := loc.Slices[len(loc.Slices)-1]
for ii := uint32(0); ii < 100; ii++ {
bid := blob.MinBid - proto.BlobID(ii) - 1
require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
bid = blob.MinBid + proto.BlobID(blob.Count+ii) - 1
require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
bid := blob.MinSliceID - proto.BlobID(ii) - 1
require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey))
bid = blob.MinSliceID + proto.BlobID(blob.Count+ii) - 1
require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey))
}
for ii := uint32(0); ii < blob.Count-1; ii++ {
bid := blob.MinBid + proto.BlobID(ii)
require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
bid := blob.MinSliceID + proto.BlobID(ii)
require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey))
}
token = uptoken.DecodeToken(tokens[len(loc.Blobs)])
lastbid := blob.MinBid + proto.BlobID(blob.Count) - 1
token = uptoken.DecodeToken(tokens[len(loc.Slices)])
lastbid := blob.MinSliceID + proto.BlobID(blob.Count) - 1
require.True(t, token.IsValid(loc.ClusterID, blob.Vid, lastbid, lastSize, skey))
}
{
loc := &access.Location{
Size: 0,
BlobSize: 333,
Blobs: []access.SliceInfo{},
loc := &proto.Location{
Size_: 0,
SliceSize: 333,
Slices: []proto.Slice{},
}
checker(loc, stream.StreamGenTokens(loc))
}
{
loc := &access.Location{
Size: 1,
BlobSize: 1024,
Blobs: []access.SliceInfo{
{MinBid: 100, Vid: 1000, Count: 1},
loc := &proto.Location{
Size_: 1,
SliceSize: 1024,
Slices: []proto.Slice{
{MinSliceID: 100, Vid: 1000, Count: 1},
},
}
checker(loc, stream.StreamGenTokens(loc))
}
{
loc := &access.Location{
Size: 1024,
BlobSize: 1024,
Blobs: []access.SliceInfo{
{MinBid: 100, Vid: 1000, Count: 1},
loc := &proto.Location{
Size_: 1024,
SliceSize: 1024,
Slices: []proto.Slice{
{MinSliceID: 100, Vid: 1000, Count: 1},
},
}
checker(loc, stream.StreamGenTokens(loc))
}
{
loc := &access.Location{
Size: 1025,
BlobSize: 1024,
Blobs: []access.SliceInfo{
{MinBid: 100, Vid: 1000, Count: 2},
loc := &proto.Location{
Size_: 1025,
SliceSize: 1024,
Slices: []proto.Slice{
{MinSliceID: 100, Vid: 1000, Count: 2},
},
}
checker(loc, stream.StreamGenTokens(loc))
}
{
loc := &access.Location{
Size: 2048,
BlobSize: 1024,
Blobs: []access.SliceInfo{
{MinBid: 100, Vid: 1000, Count: 2},
loc := &proto.Location{
Size_: 2048,
SliceSize: 1024,
Slices: []proto.Slice{
{MinSliceID: 100, Vid: 1000, Count: 2},
},
}
checker(loc, stream.StreamGenTokens(loc))
}
{
loc := &access.Location{
Size: 10240,
BlobSize: 1024,
Blobs: []access.SliceInfo{
{MinBid: 100, Vid: 1000, Count: 4},
{MinBid: 200, Vid: 1000, Count: 6},
loc := &proto.Location{
Size_: 10240,
SliceSize: 1024,
Slices: []proto.Slice{
{MinSliceID: 100, Vid: 1000, Count: 4},
{MinSliceID: 200, Vid: 1000, Count: 6},
},
}
checker(loc, stream.StreamGenTokens(loc))
}
{
loc := &access.Location{
Size: 1025,
BlobSize: 1024,
Blobs: []access.SliceInfo{
{MinBid: 100, Vid: 1000, Count: 1},
{MinBid: 200, Vid: 1000, Count: 1},
loc := &proto.Location{
Size_: 1025,
SliceSize: 1024,
Slices: []proto.Slice{
{MinSliceID: 100, Vid: 1000, Count: 1},
{MinSliceID: 200, Vid: 1000, Count: 1},
},
}
checker(loc, stream.StreamGenTokens(loc))
}
{
loc := &access.Location{
Size: 10242,
BlobSize: 1024,
Blobs: []access.SliceInfo{
{MinBid: 100, Vid: 1000, Count: 5},
{MinBid: 200, Vid: 1000, Count: 6},
loc := &proto.Location{
Size_: 10242,
SliceSize: 1024,
Slices: []proto.Slice{
{MinSliceID: 100, Vid: 1000, Count: 5},
{MinSliceID: 200, Vid: 1000, Count: 6},
},
}
checker(loc, stream.StreamGenTokens(loc))
@ -702,7 +702,7 @@ func TestAccessServiceLimited(t *testing.T) {
if err != nil {
assertErrorCode(t, errcode.CodeAccessLimited, err)
} else {
require.Equal(t, uint64(1024), resp.Location.Size)
require.Equal(t, uint64(1024), resp.Location.Size_)
}
}()
}

41
blobstore/access/stream/server_location.go
View File

@ -20,7 +20,6 @@ import (
"sync"
"time"
"github.com/cubefs/cubefs/blobstore/api/access"
"github.com/cubefs/cubefs/blobstore/common/proto"
"github.com/cubefs/cubefs/blobstore/common/uptoken"
"github.com/cubefs/cubefs/blobstore/util/bytespool"
@ -82,7 +81,7 @@ func TokenSecretKeys() [][20]byte {
return tokenSecretKeys[:]
}
func calcCrc(loc *access.Location) (uint32, error) {
func calcCrc(loc *proto.Location) (uint32, error) {
crcWriter := crc32.New(_crcTable)
buf := bytespool.Alloc(1024)
@ -103,7 +102,7 @@ func calcCrc(loc *access.Location) (uint32, error) {
return crcWriter.Sum32(), nil
}
func fillCrc(loc *access.Location) error {
func fillCrc(loc *proto.Location) error {
crc, err := calcCrc(loc)
if err != nil {
return err
@ -112,7 +111,7 @@ func fillCrc(loc *access.Location) error {
return nil
}
func verifyCrc(loc *access.Location) bool {
func verifyCrc(loc *proto.Location) bool {
crc, err := calcCrc(loc)
if err != nil {
return false
@ -120,13 +119,13 @@ func verifyCrc(loc *access.Location) bool {
return loc.Crc == crc
}
func signCrc(loc *access.Location, locs []access.Location) error {
func signCrc(loc *proto.Location, locs []proto.Location) error {
first := locs[0]
bids := make(map[proto.BlobID]struct{}, 64)
if loc.ClusterID != first.ClusterID ||
loc.CodeMode != first.CodeMode ||
loc.BlobSize != first.BlobSize {
loc.SliceSize != first.SliceSize {
return fmt.Errorf("not equal in constant field")
}
@ -138,20 +137,20 @@ func signCrc(loc *access.Location, locs []access.Location) error {
// assert
if l.ClusterID != first.ClusterID ||
l.CodeMode != first.CodeMode ||
l.BlobSize != first.BlobSize {
l.SliceSize != first.SliceSize {
return fmt.Errorf("not equal in constant field")
}
for _, blob := range l.Blobs {
for _, blob := range l.Slices {
for c := 0; c < int(blob.Count); c++ {
bids[blob.MinBid+proto.BlobID(c)] = struct{}{}
bids[blob.MinSliceID+proto.BlobID(c)] = struct{}{}
}
}
}
for _, blob := range loc.Blobs {
for _, blob := range loc.Slices {
for c := 0; c < int(blob.Count); c++ {
bid := blob.MinBid + proto.BlobID(c)
bid := blob.MinSliceID + proto.BlobID(c)
if _, ok := bids[bid]; !ok {
return fmt.Errorf("not equal in blob_id(%d)", bid)
}
@ -169,27 +168,27 @@ func signCrc(loc *access.Location, locs []access.Location) error {
// will be used by the last blob, even if the last slice blobs' size
// less than blobsize.
// 5. Each segment blob has its specified token include the last blob.
func genTokens(location *access.Location) []string {
tokens := make([]string, 0, len(location.Blobs)+1)
func genTokens(location *proto.Location) []string {
tokens := make([]string, 0, len(location.Slices)+1)
hasMultiBlobs := location.Size >= uint64(location.BlobSize)
lastSize := uint32(location.Size % uint64(location.BlobSize))
for idx, blob := range location.Blobs {
hasMultiBlobs := location.Size_ >= uint64(location.SliceSize)
lastSize := uint32(location.Size_ % uint64(location.SliceSize))
for idx, blob := range location.Slices {
// returns one token if size < blobsize
if hasMultiBlobs {
count := blob.Count
if idx == len(location.Blobs)-1 && lastSize > 0 {
if idx == len(location.Slices)-1 && lastSize > 0 {
count--
}
tokens = append(tokens, uptoken.EncodeToken(uptoken.NewUploadToken(location.ClusterID,
blob.Vid, blob.MinBid, count,
location.BlobSize, _tokenExpiration, tokenSecretKeys[0][:])))
blob.Vid, blob.MinSliceID, count,
location.SliceSize, _tokenExpiration, tokenSecretKeys[0][:])))
}
// token of the last blob
if idx == len(location.Blobs)-1 && lastSize > 0 {
if idx == len(location.Slices)-1 && lastSize > 0 {
tokens = append(tokens, uptoken.EncodeToken(uptoken.NewUploadToken(location.ClusterID,
blob.Vid, blob.MinBid+proto.BlobID(blob.Count)-1, 1,
blob.Vid, blob.MinSliceID+proto.BlobID(blob.Count)-1, 1,
lastSize, _tokenExpiration, tokenSecretKeys[0][:])))
}
}

81
blobstore/access/stream/server_location_test.go
View File

@ -22,7 +22,6 @@ import (
"github.com/stretchr/testify/require"
"github.com/cubefs/cubefs/blobstore/api/access"
"github.com/cubefs/cubefs/blobstore/common/codemode"
"github.com/cubefs/cubefs/blobstore/common/proto"
"github.com/cubefs/cubefs/blobstore/common/uptoken"
@ -30,21 +29,21 @@ import (
)
var (
testMaxBlob = access.SliceInfo{
MinBid: proto.BlobID(math.MaxUint64),
Vid: proto.Vid(math.MaxInt32),
Count: math.MaxUint32,
testMaxBlob = proto.Slice{
MinSliceID: proto.BlobID(math.MaxUint64),
Vid: proto.Vid(math.MaxInt32),
Count: math.MaxUint32,
}
testMaxLoc = access.Location{
testMaxLoc = proto.Location{
ClusterID: proto.ClusterID(math.MaxUint32),
CodeMode: codemode.CodeMode(math.MaxInt8),
Size: math.MaxUint64,
BlobSize: math.MaxUint32,
Size_: math.MaxUint64,
SliceSize: math.MaxUint32,
Crc: math.MaxUint32,
}
testMinBlob = access.SliceInfo{}
testMinLoc = access.Location{}
testMinBlob = proto.Slice{}
testMinLoc = proto.Location{}
)
func TestAccessServiceLocationCrc(t *testing.T) {
@ -64,7 +63,7 @@ func TestAccessServiceLocationCrc(t *testing.T) {
}
{
loc := testMinLoc.Copy()
loc.Size = 1 << 30
loc.Size_ = 1 << 30
err := fillCrc(&loc)
require.NoError(t, err)
@ -72,7 +71,7 @@ func TestAccessServiceLocationCrc(t *testing.T) {
}
{
loc := testMinLoc.Copy()
loc.Size = 1 << 30
loc.Size_ = 1 << 30
require.False(t, verifyCrc(&loc))
loc.Crc = 0x9e17bc9e
@ -108,15 +107,15 @@ func TestAccessServiceTokenSecret(t *testing.T) {
tokenSecretKeys = keys
}()
b := [...]byte{1: 2, 7: 8}
loc := &access.Location{
loc := &proto.Location{
ClusterID: 1,
CodeMode: 1,
Size: 1023,
BlobSize: 1024,
Blobs: []access.SliceInfo{{
MinBid: 11,
Vid: 199,
Count: 1,
Size_: 1023,
SliceSize: 1024,
Slices: []proto.Slice{{
MinSliceID: 11,
Vid: 199,
Count: 1,
}},
}
@ -131,16 +130,16 @@ func TestAccessServiceTokenSecret(t *testing.T) {
}
func TestAccessServiceLocationSignCrc(t *testing.T) {
loc := &access.Location{
loc := &proto.Location{
ClusterID: 1,
CodeMode: 1,
Size: 1023,
BlobSize: 6,
Size_: 1023,
SliceSize: 6,
Crc: 0,
Blobs: []access.SliceInfo{{
MinBid: 11,
Vid: 199,
Count: 10,
Slices: []proto.Slice{{
MinSliceID: 11,
Vid: 199,
Count: 10,
}},
}
fillCrc(loc)
@ -148,42 +147,42 @@ func TestAccessServiceLocationSignCrc(t *testing.T) {
{
loc1, loc2 := loc.Copy(), loc.Copy()
require.NoError(t, signCrc(loc, []access.Location{loc1, loc2}))
require.NoError(t, signCrc(loc, []proto.Location{loc1, loc2}))
}
{
loc1, loc2 := loc.Copy(), loc.Copy()
loc1.BlobSize = 100
loc1.SliceSize = 100
fillCrc(&loc1)
require.Error(t, signCrc(loc, []access.Location{loc1, loc2}))
require.Error(t, signCrc(loc, []proto.Location{loc1, loc2}))
}
{
loc1, loc2 := loc.Copy(), loc.Copy()
loc2.Crc = 0
require.Error(t, signCrc(loc, []access.Location{loc1, loc2}))
require.Error(t, signCrc(loc, []proto.Location{loc1, loc2}))
}
{
loc1, loc2 := loc.Copy(), loc.Copy()
loc2.ClusterID = 2
fillCrc(&loc2)
require.Error(t, signCrc(loc, []access.Location{loc1, loc2}))
require.Error(t, signCrc(loc, []proto.Location{loc1, loc2}))
}
{
loc1, loc2 := loc.Copy(), loc.Copy()
loc2.CodeMode = 100
fillCrc(&loc2)
require.Error(t, signCrc(loc, []access.Location{loc1, loc2}))
require.Error(t, signCrc(loc, []proto.Location{loc1, loc2}))
}
{
loc1, loc2 := loc.Copy(), loc.Copy()
loc1.Blobs = nil
loc2.Blobs[0].Count = 5
loc1.Slices = nil
loc2.Slices[0].Count = 5
fillCrc(&loc1)
fillCrc(&loc2)
require.Error(t, signCrc(loc, []access.Location{loc1, loc2}))
require.Error(t, signCrc(loc, []proto.Location{loc1, loc2}))
}
}
func calcCrcWithoutMagic(loc *access.Location) (uint32, error) {
func calcCrcWithoutMagic(loc *proto.Location) (uint32, error) {
crcWriter := crc32.New(_crcTable)
buf := bytespool.Alloc(1024)
@ -196,17 +195,17 @@ func calcCrcWithoutMagic(loc *access.Location) (uint32, error) {
}
func benchmarkCrc(b *testing.B, key string,
location access.Location, blob access.SliceInfo,
run func(loc *access.Location) (uint32, error),
location proto.Location, blob proto.Slice,
run func(loc *proto.Location) (uint32, error),
) {
cases := []int{0, 2, 4, 8, 16, 32}
for _, l := range cases {
b.ResetTimer()
b.Run(fmt.Sprintf(key+"-%d", l), func(b *testing.B) {
loc := location.Copy()
loc.Blobs = make([]access.SliceInfo, l)
for idx := range loc.Blobs {
loc.Blobs[idx] = blob
loc.Slices = make([]proto.Slice, l)
for idx := range loc.Slices {
loc.Slices[idx] = blob
}
b.ResetTimer()
for ii := 0; ii <= b.N; ii++ {

12
blobstore/access/stream/stream.go
View File

@ -56,7 +56,7 @@ type StreamHandler interface {
// codeMode > 0, alloc in this codemode
// return: a location of file
Alloc(ctx context.Context, size uint64, blobSize uint32,
assignClusterID proto.ClusterID, codeMode codemode.CodeMode) (*access.Location, error)
assignClusterID proto.ClusterID, codeMode codemode.CodeMode) (*proto.Location, error)
// PutAt access interface /putat, put one blob
// required: rc file reader
@ -69,7 +69,7 @@ type StreamHandler interface {
// Put put one object
// required: size, file size
// optional: hasher map to calculate hash.Hash
Put(ctx context.Context, rc io.Reader, size int64, hasherMap access.HasherMap) (*access.Location, error)
Put(ctx context.Context, rc io.Reader, size int64, hasherMap access.HasherMap) (*proto.Location, error)
// Get read file
// required: location, readSize
@ -93,10 +93,10 @@ type StreamHandler interface {
//...
//read-9 [d4 p5]
//failed
Get(ctx context.Context, w io.Writer, location access.Location, readSize, offset uint64) (func() error, error)
Get(ctx context.Context, w io.Writer, location proto.Location, readSize, offset uint64) (func() error, error)
// Delete delete all blobs in this location
Delete(ctx context.Context, location *access.Location) error
Delete(ctx context.Context, location *proto.Location) error
// Admin returns internal admin interface.
Admin() interface{}
@ -317,7 +317,7 @@ func NewStreamHandler(cfg *StreamConfig, stopCh <-chan struct{}) (h StreamHandle
}
// Delete delete all blobs in this location
func (h *Handler) Delete(ctx context.Context, location *access.Location) error {
func (h *Handler) Delete(ctx context.Context, location *proto.Location) error {
span := trace.SpanFromContextSafe(ctx)
span.Debugf("to delete %+v", location)
return h.clearGarbage(ctx, location)
@ -384,7 +384,7 @@ func (h *Handler) sendRepairMsg(ctx context.Context, blob blobIdent, badIdxes []
span.Infof("send repair message(%+v)", repairArgs)
}
func (h *Handler) clearGarbage(ctx context.Context, location *access.Location) error {
func (h *Handler) clearGarbage(ctx context.Context, location *proto.Location) error {
span := trace.SpanFromContextSafe(ctx)
serviceController, err := h.clusterController.GetServiceController(location.ClusterID)
if err != nil {

24
blobstore/access/stream/stream_alloc.go
View File

@ -41,7 +41,7 @@ var errAllocatePunishedVolume = errors.New("allocate punished volume")
// return: a location of file
func (h *Handler) Alloc(ctx context.Context, size uint64, blobSize uint32,
assignClusterID proto.ClusterID, codeMode codemode.CodeMode,
) (*access.Location, error) {
) (*proto.Location, error) {
span := trace.SpanFromContextSafe(ctx)
span.Debugf("alloc request with size:%d blobsize:%d cluster:%d codemode:%d",
size, blobSize, assignClusterID, codeMode)
@ -72,12 +72,12 @@ func (h *Handler) Alloc(ctx context.Context, size uint64, blobSize uint32,
}
span.Debugf("allocated from %d %+v", clusterID, blobs)
location := &access.Location{
location := &proto.Location{
ClusterID: clusterID,
CodeMode: codeMode,
Size: size,
BlobSize: blobSize,
Blobs: blobs,
Size_: size,
SliceSize: blobSize,
Slices: blobs,
}
span.Debugf("alloc ok %+v", location)
return location, nil
@ -85,7 +85,7 @@ func (h *Handler) Alloc(ctx context.Context, size uint64, blobSize uint32,
func (h *Handler) allocFromAllocatorWithHystrix(ctx context.Context,
codeMode codemode.CodeMode, size uint64, blobSize uint32, clusterID proto.ClusterID,
) (cid proto.ClusterID, bidRets []access.SliceInfo, err error) {
) (cid proto.ClusterID, bidRets []proto.Slice, err error) {
err = hystrix.Do(allocCommand, func() error {
cid, bidRets, err = h.allocFromAllocator(ctx, codeMode, size, blobSize, clusterID)
return err
@ -95,7 +95,7 @@ func (h *Handler) allocFromAllocatorWithHystrix(ctx context.Context,
func (h *Handler) allocFromAllocator(ctx context.Context,
codeMode codemode.CodeMode, size uint64, blobSize uint32, clusterID proto.ClusterID,
) (proto.ClusterID, []access.SliceInfo, error) {
) (proto.ClusterID, []proto.Slice, error) {
span := trace.SpanFromContextSafe(ctx)
if blobSize == 0 {
@ -183,7 +183,7 @@ func (h *Handler) allocFromAllocator(ctx context.Context,
}
blobN := blobCount(size, blobSize)
blobs := make([]access.SliceInfo, 0, blobN)
blobs := make([]proto.Slice, 0, blobN)
for _, bidRet := range allocRets {
if blobN <= 0 {
break
@ -192,10 +192,10 @@ func (h *Handler) allocFromAllocator(ctx context.Context,
count := minU64(blobN, uint64(bidRet.BidEnd)-uint64(bidRet.BidStart)+1)
blobN -= count
blobs = append(blobs, access.SliceInfo{
MinBid: bidRet.BidStart,
Vid: bidRet.Vid,
Count: uint32(count),
blobs = append(blobs, proto.Slice{
MinSliceID: bidRet.BidStart,
Vid: bidRet.Vid,
Count: uint32(count),
})
}
if blobN > 0 {

22
blobstore/access/stream/stream_alloc_test.go
View File

@ -32,26 +32,26 @@ func TestAccessStreamAllocBase(t *testing.T) {
require.NoError(t, err)
require.Equal(t, clusterID, loc.ClusterID)
require.Equal(t, codemode.EC6P6, loc.CodeMode)
require.Equal(t, uint64(1<<30), loc.Size)
require.Equal(t, uint32(1<<22), loc.BlobSize)
require.Equal(t, 2, len(loc.Blobs))
require.Equal(t, uint32(1), loc.Blobs[0].Count)
require.Equal(t, uint32((1<<8)-1), loc.Blobs[1].Count)
require.Equal(t, uint64(1<<30), loc.Size_)
require.Equal(t, uint32(1<<22), loc.SliceSize)
require.Equal(t, 2, len(loc.Slices))
require.Equal(t, uint32(1), loc.Slices[0].Count)
require.Equal(t, uint32((1<<8)-1), loc.Slices[1].Count)
}
{
loc, err := streamer.Alloc(ctx(), (1<<30)+1, 0, 0, 0)
require.NoError(t, err)
require.Equal(t, 2, len(loc.Blobs))
require.Equal(t, uint32(1), loc.Blobs[0].Count)
require.Equal(t, uint32(1<<8), loc.Blobs[1].Count)
require.Equal(t, 2, len(loc.Slices))
require.Equal(t, uint32(1), loc.Slices[0].Count)
require.Equal(t, uint32(1<<8), loc.Slices[1].Count)
}
// 1M blobsize
{
loc, err := streamer.Alloc(ctx(), 1<<30, 1<<20, 0, 0)
require.NoError(t, err)
require.Equal(t, 2, len(loc.Blobs))
require.Equal(t, uint32(1), loc.Blobs[0].Count)
require.Equal(t, uint32((1<<10)-1), loc.Blobs[1].Count)
require.Equal(t, 2, len(loc.Slices))
require.Equal(t, uint32(1), loc.Slices[0].Count)
require.Equal(t, uint32((1<<10)-1), loc.Slices[1].Count)
}
// max size + 1
{

19
blobstore/access/stream/stream_get.go
View File

@ -27,7 +27,6 @@ import (
"github.com/afex/hystrix-go/hystrix"
"github.com/cubefs/cubefs/blobstore/access/controller"
"github.com/cubefs/cubefs/blobstore/api/access"
"github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/common/codemode"
"github.com/cubefs/cubefs/blobstore/common/ec"
@ -112,7 +111,7 @@ type pipeBuffer struct {
// ...
// read-9 [d4 p5]
// failed
func (h *Handler) Get(ctx context.Context, w io.Writer, location access.Location, readSize, offset uint64) (func() error, error) {
func (h *Handler) Get(ctx context.Context, w io.Writer, location proto.Location, readSize, offset uint64) (func() error, error) {
span := trace.SpanFromContextSafe(ctx)
span.Debugf("get request cluster:%d size:%d offset:%d", location.ClusterID, readSize, offset)
@ -758,14 +757,14 @@ func (h *Handler) getOneShardFromHost(ctx context.Context, serviceController con
return
}
func genLocationBlobs(location *access.Location, readSize uint64, offset uint64) ([]blobGetArgs, error) {
if readSize > location.Size || offset > location.Size || offset+readSize > location.Size {
return nil, fmt.Errorf("FileSize:%d ReadSize:%d Offset:%d", location.Size, readSize, offset)
func genLocationBlobs(location *proto.Location, readSize uint64, offset uint64) ([]blobGetArgs, error) {
if readSize > location.Size_ || offset > location.Size_ || offset+readSize > location.Size_ {
return nil, fmt.Errorf("FileSize:%d ReadSize:%d Offset:%d", location.Size_, readSize, offset)
}
blobSize := uint64(location.BlobSize)
blobSize := uint64(location.SliceSize)
if blobSize <= 0 {
return nil, fmt.Errorf("BlobSize:%d", blobSize)
return nil, fmt.Errorf("SliceSize:%d", blobSize)
}
remainSize := readSize
@ -776,8 +775,8 @@ func genLocationBlobs(location *access.Location, readSize uint64, offset uint64)
idx := uint64(0)
blobs := make([]blobGetArgs, 0, 1+(readSize+blobOffset)/blobSize)
for _, blob := range location.Blobs {
currBlobID := blob.MinBid
for _, blob := range location.Slices {
currBlobID := blob.MinSliceID
for ii := uint32(0); ii < blob.Count; ii++ {
if remainSize <= 0 {
@ -788,7 +787,7 @@ func genLocationBlobs(location *access.Location, readSize uint64, offset uint64)
toReadSize := minU64(remainSize, blobSize-blobOffset)
if toReadSize > 0 {
// update the last blob size
fixedBlobSize := minU64(location.Size-idx*blobSize, blobSize)
fixedBlobSize := minU64(location.Size_-idx*blobSize, blobSize)
sizes, _ := ec.GetBufferSizes(int(fixedBlobSize), tactic)
shardSize := sizes.ShardSize

21
blobstore/access/stream/stream_get_test.go
View File

@ -24,7 +24,6 @@ import (
"github.com/stretchr/testify/require"
"github.com/cubefs/cubefs/blobstore/api/access"
"github.com/cubefs/cubefs/blobstore/common/codemode"
"github.com/cubefs/cubefs/blobstore/common/proto"
)
@ -530,21 +529,21 @@ func TestAccessStreamGenLocationBlobs(t *testing.T) {
firstSliceStart := proto.BlobID(100)
secondSliceStart := proto.BlobID(200)
loc := access.Location{
loc := proto.Location{
ClusterID: 0,
CodeMode: codemode.EC6P6,
Size: 1024*4 + 37 + 1024*2, // 5 fine blobs and 2 missing blobs
BlobSize: 1024,
Blobs: []access.SliceInfo{
Size_: 1024*4 + 37 + 1024*2, // 5 fine blobs and 2 missing blobs
SliceSize: 1024,
Slices: []proto.Slice{
{
MinBid: firstSliceStart,
Vid: proto.Vid(1001),
Count: 3,
MinSliceID: firstSliceStart,
Vid: proto.Vid(1001),
Count: 3,
},
{
MinBid: secondSliceStart,
Vid: proto.Vid(2001),
Count: 2,
MinSliceID: secondSliceStart,
Vid: proto.Vid(2001),
Count: 2,
},
},
}

11
blobstore/access/stream/stream_put.go
View File

@ -30,6 +30,7 @@ import (
"github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/common/ec"
errcode "github.com/cubefs/cubefs/blobstore/common/errors"
"github.com/cubefs/cubefs/blobstore/common/proto"
"github.com/cubefs/cubefs/blobstore/common/rpc"
"github.com/cubefs/cubefs/blobstore/common/trace"
"github.com/cubefs/cubefs/blobstore/util/errors"
@ -45,7 +46,7 @@ import (
// optional: hasher map to calculate hash.Hash
func (h *Handler) Put(ctx context.Context,
rc io.Reader, size int64, hasherMap access.HasherMap,
) (*access.Location, error) {
) (*proto.Location, error) {
span := trace.SpanFromContextSafe(ctx)
span.Debugf("put request size:%d hashes:b(%b)", size, hasherMap.ToHashAlgorithm())
@ -76,12 +77,12 @@ func (h *Handler) Put(ctx context.Context,
// 3.read body and split, alloc from mem pool;ec encode and put into data node
limitReader := io.LimitReader(rc, int64(size))
location := &access.Location{
location := &proto.Location{
ClusterID: clusterID,
CodeMode: selectedCodeMode,
Size: uint64(size),
BlobSize: blobSize,
Blobs: blobs,
Size_: uint64(size),
SliceSize: blobSize,
Slices: blobs,
}
uploadSucc := false

12
blobstore/access/stream/stream_put_test.go
View File

@ -50,8 +50,8 @@ func TestAccessStreamPutBase(t *testing.T) {
size := 1
loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil)
require.NoError(t, err)
require.Equal(t, 1, len(loc.Blobs))
require.Equal(t, uint32(1), loc.Blobs[0].Count)
require.Equal(t, 1, len(loc.Slices))
require.Equal(t, uint32(1), loc.Slices[0].Count)
// time wait the punished services
time.Sleep(time.Second * time.Duration(punishServiceS))
}
@ -60,8 +60,8 @@ func TestAccessStreamPutBase(t *testing.T) {
size := 1 << 18
loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil)
require.NoError(t, err)
require.Equal(t, 1, len(loc.Blobs))
require.Equal(t, uint32(1), loc.Blobs[0].Count)
require.Equal(t, 1, len(loc.Slices))
require.Equal(t, uint32(1), loc.Slices[0].Count)
time.Sleep(time.Second * time.Duration(punishServiceS))
}
// 8M + 1k
@ -69,8 +69,8 @@ func TestAccessStreamPutBase(t *testing.T) {
size := (1 << 23) + 1024
loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil)
require.NoError(t, err)
require.Equal(t, 2, len(loc.Blobs))
require.Equal(t, uint32(2), loc.Blobs[1].Count)
require.Equal(t, 2, len(loc.Slices))
require.Equal(t, uint32(2), loc.Slices[1].Count)
time.Sleep(time.Second * time.Duration(punishServiceS))
}
// max size + 1

68
blobstore/api/access/client.go
View File

@ -212,7 +212,7 @@ type API interface {
//
// If PutArgs' body is of type *bytes.Buffer, *bytes.Reader, or *strings.Reader,
// GetBody is populated, then the Put once request has retry ability.
Put(ctx context.Context, args *PutArgs) (location Location, hashSumMap HashSumMap, err error)
Put(ctx context.Context, args *PutArgs) (location proto.Location, hashSumMap HashSumMap, err error)
// Get object, range is supported.
Get(ctx context.Context, args *GetArgs) (body io.ReadCloser, err error)
// Delete all blobs in these locations.
@ -221,7 +221,7 @@ type API interface {
// - (nil, nil): all blobs deleted successfully.
// - (nil, ErrIllegalArguments): when args is invalid.
// - (failedLocations, err): returns the list of locations that have not yet been deleted.
Delete(ctx context.Context, args *DeleteArgs) (failedLocations []Location, err error)
Delete(ctx context.Context, args *DeleteArgs) (failedLocations []proto.Location, err error)
}
var _ API = (*client)(nil)
@ -391,13 +391,13 @@ func getClient(cfg *Config, hosts []string) rpc.Client {
return rpc.NewLbClient(lbConfig, nil)
}
func (c *client) Put(ctx context.Context, args *PutArgs) (location Location, hashSumMap HashSumMap, err error) {
func (c *client) Put(ctx context.Context, args *PutArgs) (location proto.Location, hashSumMap HashSumMap, err error) {
if args.Size == 0 {
hashSumMap := args.Hashes.ToHashSumMap()
for alg := range hashSumMap {
hashSumMap[alg] = alg.ToHasher().Sum(nil)
}
return Location{Blobs: make([]SliceInfo, 0)}, hashSumMap, nil
return proto.Location{Slices: make([]proto.Slice, 0)}, hashSumMap, nil
}
ctx = withReqidContext(ctx)
@ -407,7 +407,7 @@ func (c *client) Put(ctx context.Context, args *PutArgs) (location Location, has
return c.putParts(ctx, args)
}
func (c *client) putObject(ctx context.Context, args *PutArgs) (location Location, hashSumMap HashSumMap, err error) {
func (c *client) putObject(ctx context.Context, args *PutArgs) (location proto.Location, hashSumMap HashSumMap, err error) {
rpcClient := c.rpcClient.Load().(rpc.Client)
urlStr := fmt.Sprintf("/put?size=%d&hashes=%d", args.Size, args.Hashes)
@ -510,7 +510,7 @@ func (c *client) readerPipeline(span trace.Span, reqBody io.Reader,
return ch
}
func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSumMap, error) {
func (c *client) putParts(ctx context.Context, args *PutArgs) (proto.Location, HashSumMap, error) {
span := trace.SpanFromContextSafe(ctx)
rpcClient := c.rpcClient.Load().(rpc.Client)
@ -526,7 +526,7 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
}
var (
loc Location
loc proto.Location
tokens []string
)
@ -544,7 +544,7 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
signArgs.Location = loc.Copy()
signResp := &SignResp{}
if err := rpcClient.PostWith(newCtx, "/sign", signResp, signArgs); err == nil {
locations = []Location{signResp.Location.Copy()}
locations = []proto.Location{signResp.Location.Copy()}
}
}
if len(locations) > 0 {
@ -565,7 +565,7 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
// buffer pipeline
closeCh := make(chan struct{})
bufferPipe := c.readerPipeline(span, reqBody, closeCh, int(loc.Size), int(loc.BlobSize))
bufferPipe := c.readerPipeline(span, reqBody, closeCh, int(loc.Size_), int(loc.SliceSize))
defer func() {
close(closeCh)
// waiting pipeline close if has error
@ -584,17 +584,17 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
currBlobIdx := 0
currBlobCount := uint32(0)
remainSize := loc.Size
remainSize := loc.Size_
restPartsLoc := loc
readSize := 0
for readSize < int(loc.Size) {
for readSize < int(loc.Size_) {
parts := make([]blobPart, 0, c.config.PartConcurrence)
// waiting at least one blob
buf, ok := <-bufferPipe
if !ok && readSize < int(loc.Size) {
return Location{}, nil, errcode.ErrAccessReadRequestBody
if !ok && readSize < int(loc.Size_) {
return proto.Location{}, nil, errcode.ErrAccessReadRequestBody
}
readSize += len(buf)
parts = append(parts, blobPart{size: len(buf), buf: buf})
@ -604,9 +604,9 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
select {
case buf, ok := <-bufferPipe:
if !ok {
if readSize < int(loc.Size) {
if readSize < int(loc.Size_) {
releaseBuffer(parts)
return Location{}, nil, errcode.ErrAccessReadRequestBody
return proto.Location{}, nil, errcode.ErrAccessReadRequestBody
}
more = false
} else {
@ -620,9 +620,9 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
tryTimes := c.config.MaxPartRetry
for {
if len(loc.Blobs) > MaxLocationBlobs {
if len(loc.Slices) > MaxLocationBlobs {
releaseBuffer(parts)
return Location{}, nil, errcode.ErrUnexpected
return proto.Location{}, nil, errcode.ErrUnexpected
}
// feed new params
@ -630,16 +630,16 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
currCount := currBlobCount
for i := range parts {
token := tokens[currIdx]
if restPartsLoc.Size > uint64(loc.BlobSize) && parts[i].size < int(loc.BlobSize) {
if restPartsLoc.Size_ > uint64(loc.SliceSize) && parts[i].size < int(loc.SliceSize) {
token = tokens[currIdx+1]
}
parts[i].token = token
parts[i].cid = loc.ClusterID
parts[i].vid = loc.Blobs[currIdx].Vid
parts[i].bid = loc.Blobs[currIdx].MinBid + proto.BlobID(currCount)
parts[i].vid = loc.Slices[currIdx].Vid
parts[i].bid = loc.Slices[currIdx].MinSliceID + proto.BlobID(currCount)
currCount++
if loc.Blobs[currIdx].Count == currCount {
if loc.Slices[currIdx].Count == currCount {
currIdx++
currCount = 0
}
@ -651,7 +651,7 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
remainSize -= uint64(part.size)
currBlobCount++
// next blobs
if loc.Blobs[currBlobIdx].Count == currBlobCount {
if loc.Slices[currBlobIdx].Count == currBlobCount {
currBlobIdx++
currBlobCount = 0
}
@ -665,7 +665,7 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
if tryTimes == 1 {
releaseBuffer(parts)
span.Error("exceed the max retry limit", c.config.MaxPartRetry)
return Location{}, nil, errcode.ErrUnexpected
return proto.Location{}, nil, errcode.ErrUnexpected
}
tryTimes--
}
@ -676,14 +676,14 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
resp := &AllocResp{}
if err := rpcClient.PostWith(ctx, "/alloc", resp, AllocArgs{
Size: remainSize,
BlobSize: loc.BlobSize,
BlobSize: loc.SliceSize,
CodeMode: loc.CodeMode,
AssignClusterID: loc.ClusterID,
}); err != nil {
return true, err
}
if len(resp.Location.Blobs) > 0 {
if newVid := resp.Location.Blobs[0].Vid; newVid == loc.Blobs[currBlobIdx].Vid {
if len(resp.Location.Slices) > 0 {
if newVid := resp.Location.Slices[0].Vid; newVid == loc.Slices[currBlobIdx].Vid {
return false, fmt.Errorf("alloc the same vid %d", newVid)
}
}
@ -693,17 +693,17 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
if err != nil {
releaseBuffer(parts)
span.Error("alloc another parts to put", err)
return Location{}, nil, errcode.ErrUnexpected
return proto.Location{}, nil, errcode.ErrUnexpected
}
restPartsLoc = restPartsResp.Location
signArgs.Locations = append(signArgs.Locations, restPartsLoc.Copy())
if currBlobCount > 0 {
loc.Blobs[currBlobIdx].Count = currBlobCount
loc.Slices[currBlobIdx].Count = currBlobCount
currBlobIdx++
}
loc.Blobs = append(loc.Blobs[:currBlobIdx], restPartsLoc.Blobs...)
loc.Slices = append(loc.Slices[:currBlobIdx], restPartsLoc.Slices...)
tokens = append(tokens[:currBlobIdx], restPartsResp.Tokens...)
currBlobCount = 0
@ -718,7 +718,7 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
signResp := &SignResp{}
if err := rpcClient.PostWith(ctx, "/sign", signResp, signArgs); err != nil {
span.Error("sign location with crc", err)
return Location{}, nil, errcode.ErrUnexpected
return proto.Location{}, nil, errcode.ErrUnexpected
}
loc = signResp.Location
}
@ -737,7 +737,7 @@ func (c *client) Get(ctx context.Context, args *GetArgs) (body io.ReadCloser, er
rpcClient := c.rpcClient.Load().(rpc.Client)
ctx = withReqidContext(ctx)
if args.Location.Size == 0 || args.ReadSize == 0 {
if args.Location.Size_ == 0 || args.ReadSize == 0 {
return noopBody{}, nil
}
@ -752,7 +752,7 @@ func (c *client) Get(ctx context.Context, args *GetArgs) (body io.ReadCloser, er
return resp.Body, nil
}
func (c *client) Delete(ctx context.Context, args *DeleteArgs) ([]Location, error) {
func (c *client) Delete(ctx context.Context, args *DeleteArgs) ([]proto.Location, error) {
if !args.IsValid() {
if args == nil {
return nil, errcode.ErrIllegalArguments
@ -762,9 +762,9 @@ func (c *client) Delete(ctx context.Context, args *DeleteArgs) ([]Location, erro
rpcClient := c.rpcClient.Load().(rpc.Client)
ctx = withReqidContext(ctx)
locations := make([]Location, 0, len(args.Locations))
locations := make([]proto.Location, 0, len(args.Locations))
for _, loc := range args.Locations {
if loc.Size > 0 {
if loc.Size_ > 0 {
locations = append(locations, loc.Copy())
}
}

92
blobstore/api/access/client_test.go
View File

@ -157,60 +157,60 @@ func handleAlloc(c *rpc.Context) {
return
}
loc := access.Location{
loc := proto.Location{
ClusterID: 1,
Size: args.Size,
BlobSize: blobSize,
Blobs: []access.SliceInfo{
Size_: args.Size,
SliceSize: blobSize,
Slices: []proto.Slice{
{
MinBid: proto.BlobID(mrand.Int()),
Vid: proto.Vid(mrand.Int()),
Count: uint32((args.Size + blobSize - 1) / blobSize),
MinSliceID: proto.BlobID(mrand.Int()),
Vid: proto.Vid(mrand.Int()),
Count: uint32((args.Size + blobSize - 1) / blobSize),
},
},
}
// split to two blobs if large enough
if loc.Blobs[0].Count > 2 {
loc.Blobs[0].Count = 2
loc.Blobs = append(loc.Blobs, []access.SliceInfo{
if loc.Slices[0].Count > 2 {
loc.Slices[0].Count = 2
loc.Slices = append(loc.Slices, []proto.Slice{
{
MinBid: proto.BlobID(mrand.Int()),
Vid: proto.Vid(mrand.Int()),
Count: uint32((args.Size - 2*blobSize + blobSize - 1) / blobSize),
MinSliceID: proto.BlobID(mrand.Int()),
Vid: proto.Vid(mrand.Int()),
Count: uint32((args.Size - 2*blobSize + blobSize - 1) / blobSize),
},
}...)
}
// alloc the rest parts
if args.AssignClusterID > 0 {
loc.Blobs = []access.SliceInfo{
loc.Slices = []proto.Slice{
{
MinBid: proto.BlobID(mrand.Int()),
Vid: proto.Vid(mrand.Int()),
Count: uint32((args.Size + blobSize - 1) / blobSize),
MinSliceID: proto.BlobID(mrand.Int()),
Vid: proto.Vid(mrand.Int()),
Count: uint32((args.Size + blobSize - 1) / blobSize),
},
}
}
tokens := make([]string, 0, len(loc.Blobs)+1)
tokens := make([]string, 0, len(loc.Slices)+1)
hasMultiBlobs := loc.Size >= uint64(loc.BlobSize)
lastSize := uint32(loc.Size % uint64(loc.BlobSize))
for idx, blob := range loc.Blobs {
hasMultiBlobs := loc.Size_ >= uint64(loc.SliceSize)
lastSize := uint32(loc.Size_ % uint64(loc.SliceSize))
for idx, blob := range loc.Slices {
// returns one token if size < blobsize
if hasMultiBlobs {
count := blob.Count
if idx == len(loc.Blobs)-1 && lastSize > 0 {
if idx == len(loc.Slices)-1 && lastSize > 0 {
count--
}
tokens = append(tokens, uptoken.EncodeToken(uptoken.NewUploadToken(loc.ClusterID,
blob.Vid, blob.MinBid, count,
loc.BlobSize, 0, tokenAlloc[:])))
blob.Vid, blob.MinSliceID, count,
loc.SliceSize, 0, tokenAlloc[:])))
}
// token of the last blob
if idx == len(loc.Blobs)-1 && lastSize > 0 {
if idx == len(loc.Slices)-1 && lastSize > 0 {
tokens = append(tokens, uptoken.EncodeToken(uptoken.NewUploadToken(loc.ClusterID,
blob.Vid, blob.MinBid+proto.BlobID(blob.Count)-1, 1,
blob.Vid, blob.MinSliceID+proto.BlobID(blob.Count)-1, 1,
lastSize, 0, tokenAlloc[:])))
}
}
@ -251,7 +251,7 @@ func handlePut(c *rpc.Context) {
hashSumMap[alg] = hasher.Sum(nil)
}
loc := access.Location{Size: uint64(args.Size)}
loc := proto.Location{Size_: uint64(args.Size)}
fillCrc(&loc)
c.RespondJSON(access.PutResp{
Location: loc,
@ -298,7 +298,7 @@ func handleGet(c *rpc.Context) {
return
}
if args.Location.Size == 100 {
if args.Location.Size_ == 100 {
c.RespondStatus(http.StatusBadRequest)
return
}
@ -359,7 +359,7 @@ func handleSign(c *rpc.Context) {
c.RespondJSON(access.SignResp{Location: args.Location})
}
func calcCrc(loc *access.Location) (uint32, error) {
func calcCrc(loc *proto.Location) (uint32, error) {
crcWriter := crc32.New(crc32.IEEETable)
buf := bytespool.Alloc(1024)
@ -377,7 +377,7 @@ func calcCrc(loc *access.Location) (uint32, error) {
return crcWriter.Sum32(), nil
}
func fillCrc(loc *access.Location) error {
func fillCrc(loc *proto.Location) error {
crc, err := calcCrc(loc)
if err != nil {
return err
@ -386,7 +386,7 @@ func fillCrc(loc *access.Location) error {
return nil
}
func verifyCrc(loc *access.Location) bool {
func verifyCrc(loc *proto.Location) bool {
crc, err := calcCrc(loc)
if err != nil {
return false
@ -394,13 +394,13 @@ func verifyCrc(loc *access.Location) bool {
return loc.Crc == crc
}
func signCrc(loc *access.Location, locs []access.Location) error {
func signCrc(loc *proto.Location, locs []proto.Location) error {
first := locs[0]
bids := make(map[proto.BlobID]struct{}, 64)
if loc.ClusterID != first.ClusterID ||
loc.CodeMode != first.CodeMode ||
loc.BlobSize != first.BlobSize {
loc.SliceSize != first.SliceSize {
return fmt.Errorf("not equal in constant field")
}
@ -412,20 +412,20 @@ func signCrc(loc *access.Location, locs []access.Location) error {
// assert
if l.ClusterID != first.ClusterID ||
l.CodeMode != first.CodeMode ||
l.BlobSize != first.BlobSize {
l.SliceSize != first.SliceSize {
return fmt.Errorf("not equal in constant field")
}
for _, blob := range l.Blobs {
for _, blob := range l.Slices {
for c := 0; c < int(blob.Count); c++ {
bids[blob.MinBid+proto.BlobID(c)] = struct{}{}
bids[blob.MinSliceID+proto.BlobID(c)] = struct{}{}
}
}
}
for _, blob := range loc.Blobs {
for _, blob := range loc.Slices {
for c := 0; c < int(blob.Count); c++ {
bid := blob.MinBid + proto.BlobID(c)
bid := blob.MinSliceID + proto.BlobID(c)
if _, ok := bids[bid]; !ok {
return fmt.Errorf("not equal in blob_id(%d)", bid)
}
@ -500,10 +500,10 @@ func TestAccessClientConnectionMode(t *testing.T) {
if cs.size <= 0 {
continue
}
loc := access.Location{Size: uint64(mrand.Int63n(cs.size))}
loc := proto.Location{Size_: uint64(mrand.Int63n(cs.size))}
fillCrc(&loc)
_, err = cli.Delete(randCtx(), &access.DeleteArgs{
Locations: []access.Location{loc},
Locations: []proto.Location{loc},
})
require.NoError(t, err)
}
@ -545,7 +545,7 @@ func TestAccessClientPutGet(t *testing.T) {
}
// test code 400
_, err := client.Get(randCtx(), &access.GetArgs{Location: access.Location{Size: 100}, ReadSize: uint64(100)})
_, err := client.Get(randCtx(), &access.GetArgs{Location: proto.Location{Size_: 100}, ReadSize: uint64(100)})
require.Error(t, err)
}
@ -786,31 +786,31 @@ func TestAccessClientDelete(t *testing.T) {
}
{
locs, err := client.Delete(randCtx(), &access.DeleteArgs{
Locations: make([]access.Location, 1),
Locations: make([]proto.Location, 1),
})
require.Nil(t, locs)
require.NoError(t, err)
}
{
args := &access.DeleteArgs{
Locations: make([]access.Location, 1000),
Locations: make([]proto.Location, 1000),
}
_, err := client.Delete(randCtx(), args)
require.NoError(t, err)
}
{
args := &access.DeleteArgs{
Locations: make([]access.Location, access.MaxDeleteLocations+1),
Locations: make([]proto.Location, access.MaxDeleteLocations+1),
}
locs, err := client.Delete(randCtx(), args)
require.Equal(t, args.Locations, locs)
require.ErrorIs(t, errcode.ErrIllegalArguments, err)
}
{
loc := access.Location{Size: 100, Blobs: make([]access.SliceInfo, 0)}
loc := proto.Location{Size_: 100, Slices: make([]proto.Slice, 0)}
fillCrc(&loc)
args := &access.DeleteArgs{
Locations: make([]access.Location, 0, access.MaxDeleteLocations),
Locations: make([]proto.Location, 0, access.MaxDeleteLocations),
}
for i := 1; i < access.MaxDeleteLocations/10; i++ {
args.Locations = append(args.Locations, loc)

320
blobstore/api/access/proto.go
View File

@ -18,10 +18,7 @@ import (
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"encoding/base64"
"encoding/binary"
"encoding/hex"
"fmt"
"hash"
"hash/crc32"
"io"
@ -196,291 +193,6 @@ func (h HashSumMap) All() map[string]interface{} {
return m
}
// Location file location, 4 + 1 + 8 + 4 + 4 + len*16 bytes
// | |
// | ClusterID(4) | CodeMode(1) |
// | Size(8) |
// | BlobSize(4) | Crc(4) |
// | len*SliceInfo(16) |
//
// ClusterID which cluster file is in
// CodeMode is ec encode mode, see defined in "common/lib/codemode"
// Size is file size
// BlobSize is every blob's size but the last one which's size=(Size mod BlobSize)
// Crc is the checksum, change anything of the location, crc will mismatch
// Blobs all blob information
type Location struct {
_ [0]byte
ClusterID proto.ClusterID `json:"cluster_id"`
CodeMode codemode.CodeMode `json:"code_mode"`
Size uint64 `json:"size"`
BlobSize uint32 `json:"blob_size"`
Crc uint32 `json:"crc"`
Blobs []SliceInfo `json:"blobs"`
}
// SliceInfo blobs info, 8 + 4 + 4 bytes
//
// MinBid is the first blob id
// Vid is which volume all blobs in
// Count is num of consecutive blob ids, count=1 just has one blob
//
// blob ids = [MinBid, MinBid+count)
type SliceInfo struct {
_ [0]byte
MinBid proto.BlobID `json:"min_bid"`
Vid proto.Vid `json:"vid"`
Count uint32 `json:"count"`
}
// Blob is one piece of data in a location
//
// Bid is the blob id
// Vid is which volume the blob in
// Size is real size of the blob
type Blob struct {
Bid proto.BlobID
Vid proto.Vid
Size uint32
}
// Copy returns a new same Location
func (loc *Location) Copy() Location {
dst := Location{
ClusterID: loc.ClusterID,
CodeMode: loc.CodeMode,
Size: loc.Size,
BlobSize: loc.BlobSize,
Crc: loc.Crc,
Blobs: make([]SliceInfo, len(loc.Blobs)),
}
copy(dst.Blobs, loc.Blobs)
return dst
}
// Encode transfer Location to slice byte
// Returns the buf created by me
//
// (n) means max-n bytes
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | field | crc | clusterid | codemode | size | blobsize |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | n-bytes | 4 | uvarint(5) | 1 | uvarint(10) | uvarint(5) |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// 25 + (5){len(blobs)} + len(Blobs) * 20
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | blobs | minbid | vid | count | ... |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | n-bytes | (10) | (5) | (5) | (20) | (20) | ... |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
func (loc *Location) Encode() []byte {
if loc == nil {
return nil
}
n := 25 + 5 + len(loc.Blobs)*20
buf := make([]byte, n)
n = loc.Encode2(buf)
return buf[:n]
}
// Encode2 transfer Location to the buf, the buf reuse by yourself
// Returns the number of bytes read
// If the buffer is too small, Encode2 will panic
func (loc *Location) Encode2(buf []byte) int {
if loc == nil {
return 0
}
n := 0
binary.BigEndian.PutUint32(buf[n:], loc.Crc)
n += 4
n += binary.PutUvarint(buf[n:], uint64(loc.ClusterID))
buf[n] = byte(loc.CodeMode)
n++
n += binary.PutUvarint(buf[n:], uint64(loc.Size))
n += binary.PutUvarint(buf[n:], uint64(loc.BlobSize))
n += binary.PutUvarint(buf[n:], uint64(len(loc.Blobs)))
for _, blob := range loc.Blobs {
n += binary.PutUvarint(buf[n:], uint64(blob.MinBid))
n += binary.PutUvarint(buf[n:], uint64(blob.Vid))
n += binary.PutUvarint(buf[n:], uint64(blob.Count))
}
return n
}
// Decode parse location from buf
// Returns the number of bytes read
// Error is not nil when parsing failed
func (loc *Location) Decode(buf []byte) (int, error) {
if loc == nil {
return 0, fmt.Errorf("location receiver is nil")
}
location, n, err := DecodeLocation(buf)
if err != nil {
return n, err
}
*loc = location
return n, nil
}
// ToString transfer location to hex string
func (loc *Location) ToString() string {
return loc.HexString()
}
// HexString transfer location to hex string
func (loc *Location) HexString() string {
return hex.EncodeToString(loc.Encode())
}
// Base64String transfer location to base64 string
func (loc *Location) Base64String() string {
return base64.StdEncoding.EncodeToString(loc.Encode())
}
// Spread location blobs to slice
func (loc *Location) Spread() []Blob {
count := 0
for _, blob := range loc.Blobs {
count += int(blob.Count)
}
blobs := make([]Blob, 0, count)
for _, blob := range loc.Blobs {
for offset := uint32(0); offset < blob.Count; offset++ {
blobs = append(blobs, Blob{
Bid: blob.MinBid + proto.BlobID(offset),
Vid: blob.Vid,
Size: loc.BlobSize,
})
}
}
if len(blobs) > 0 && loc.BlobSize > 0 {
if lastSize := loc.Size % uint64(loc.BlobSize); lastSize > 0 {
blobs[len(blobs)-1].Size = uint32(lastSize)
}
}
return blobs
}
// DecodeLocation parse location from buf
// Returns Location and the number of bytes read
// Error is not nil when parsing failed
func DecodeLocation(buf []byte) (Location, int, error) {
var (
loc Location
n int
val uint64
nn int
)
next := func() (uint64, int) {
val, nn := binary.Uvarint(buf)
if nn <= 0 {
return 0, nn
}
n += nn
buf = buf[nn:]
return val, nn
}
if len(buf) < 4 {
return loc, n, fmt.Errorf("bytes crc %d", len(buf))
}
loc.Crc = binary.BigEndian.Uint32(buf)
n += 4
buf = buf[4:]
if val, nn = next(); nn <= 0 {
return loc, n, fmt.Errorf("bytes cluster_id %d", nn)
}
loc.ClusterID = proto.ClusterID(val)
if len(buf) < 1 {
return loc, n, fmt.Errorf("bytes codemode %d", len(buf))
}
loc.CodeMode = codemode.CodeMode(buf[0])
n++
buf = buf[1:]
if val, nn = next(); nn <= 0 {
return loc, n, fmt.Errorf("bytes size %d", nn)
}
loc.Size = val
if val, nn = next(); nn <= 0 {
return loc, n, fmt.Errorf("bytes blob_size %d", nn)
}
loc.BlobSize = uint32(val)
if val, nn = next(); nn <= 0 {
return loc, n, fmt.Errorf("bytes length blobs %d", nn)
}
length := int(val)
if length > 0 {
loc.Blobs = make([]SliceInfo, 0, length)
}
for index := 0; index < length; index++ {
var blob SliceInfo
if val, nn = next(); nn <= 0 {
return loc, n, fmt.Errorf("bytes %dth-blob min_bid %d", index, nn)
}
blob.MinBid = proto.BlobID(val)
if val, nn = next(); nn <= 0 {
return loc, n, fmt.Errorf("bytes %dth-blob vid %d", index, nn)
}
blob.Vid = proto.Vid(val)
if val, nn = next(); nn <= 0 {
return loc, n, fmt.Errorf("bytes %dth-blob count %d", index, nn)
}
blob.Count = uint32(val)
loc.Blobs = append(loc.Blobs, blob)
}
return loc, n, nil
}
// DecodeLocationFrom decode location from hex string
func DecodeLocationFrom(s string) (Location, error) {
return DecodeLocationFromHex(s)
}
// DecodeLocationFromHex decode location from hex string
func DecodeLocationFromHex(s string) (Location, error) {
var loc Location
src, err := hex.DecodeString(s)
if err != nil {
return loc, err
}
_, err = loc.Decode(src)
if err != nil {
return loc, err
}
return loc, nil
}
// DecodeLocationFromBase64 decode location from base64 string
func DecodeLocationFromBase64(s string) (Location, error) {
var loc Location
src, err := base64.StdEncoding.DecodeString(s)
if err != nil {
return loc, err
}
_, err = loc.Decode(src)
if err != nil {
return loc, err
}
return loc, nil
}
// PutArgs for service /put
// Hashes means how to calculate check sum,
// HashAlgCRC32 | HashAlgMD5 equal 2 + 4 = 6
@ -507,8 +219,8 @@ func (args *PutArgs) IsValid() bool {
// PutResp put response result
type PutResp struct {
Location Location `json:"location"`
HashSumMap HashSumMap `json:"hashsum"`
Location proto.Location `json:"location"`
HashSumMap HashSumMap `json:"hashsum"`
}
// PutAtArgs for service /putat
@ -562,16 +274,16 @@ func (args *AllocArgs) IsValid() bool {
// if size mod blobsize == 0, length of tokens equal length of location blobs
// otherwise additional token for the last blob uploading
type AllocResp struct {
Location Location `json:"location"`
Tokens []string `json:"tokens"`
Location proto.Location `json:"location"`
Tokens []string `json:"tokens"`
}
// GetArgs for service /get
type GetArgs struct {
Location Location `json:"location"`
Offset uint64 `json:"offset"`
ReadSize uint64 `json:"read_size"`
Writer io.Writer `json:"-"`
Location proto.Location `json:"location"`
Offset uint64 `json:"offset"`
ReadSize uint64 `json:"read_size"`
Writer io.Writer `json:"-"`
}
// IsValid is valid get args
@ -579,14 +291,14 @@ func (args *GetArgs) IsValid() bool {
if args == nil {
return false
}
return args.Offset <= args.Location.Size &&
args.ReadSize <= args.Location.Size &&
args.Offset+args.ReadSize <= args.Location.Size
return args.Offset <= args.Location.Size_ &&
args.ReadSize <= args.Location.Size_ &&
args.Offset+args.ReadSize <= args.Location.Size_
}
// DeleteArgs for service /delete
type DeleteArgs struct {
Locations []Location `json:"locations"`
Locations []proto.Location `json:"locations"`
}
// IsValid is valid delete args
@ -599,7 +311,7 @@ func (args *DeleteArgs) IsValid() bool {
// DeleteResp delete response with failed locations
type DeleteResp struct {
FailedLocations []Location `json:"failed_locations,omitempty"`
FailedLocations []proto.Location `json:"failed_locations,omitempty"`
}
// DeleteBlobArgs for service /deleteblob
@ -626,8 +338,8 @@ func (args *DeleteBlobArgs) IsValid() bool {
// Locations are signed location getting from /alloc
// Location is to be signed location which merged by yourself
type SignArgs struct {
Locations []Location `json:"locations"`
Location Location `json:"location"`
Locations []proto.Location `json:"locations"`
Location proto.Location `json:"location"`
}
// IsValid is valid sign args
@ -640,5 +352,5 @@ func (args *SignArgs) IsValid() bool {
// SignResp sign response location with crc
type SignResp struct {
Location Location `json:"location"`
Location proto.Location `json:"location"`
}

196
blobstore/api/access/proto_test.go
View File

@ -259,192 +259,6 @@ func TestHashAlgorithm2HashSumMap(t *testing.T) {
}
}
func TestLocationEncodeDecodeNil(t *testing.T) {
var loc *access.Location
require.Nil(t, loc.Encode())
require.Equal(t, 0, loc.Encode2(nil))
require.Equal(t, "", loc.ToString())
require.Equal(t, "", loc.HexString())
require.Equal(t, "", loc.Base64String())
n, err := loc.Decode(nil)
require.Error(t, err)
require.Equal(t, 0, n)
locx, n, err := access.DecodeLocation(nil)
require.Error(t, err)
require.Equal(t, 0, n)
require.Equal(t, access.Location{}, locx)
locx, err = access.DecodeLocationFrom("")
require.Error(t, err)
require.Equal(t, access.Location{}, locx)
locx, err = access.DecodeLocationFrom("xxx")
require.Error(t, err)
require.Equal(t, access.Location{}, locx)
locx, err = access.DecodeLocationFromHex("xxx")
require.Error(t, err)
require.Equal(t, access.Location{}, locx)
locx, err = access.DecodeLocationFromBase64("xxx")
require.Error(t, err)
require.Equal(t, access.Location{}, locx)
}
func TestLocationEncodeDecode(t *testing.T) {
for ii := 0; ii < 100; ii++ {
loc := &access.Location{
ClusterID: proto.ClusterID(mrand.Uint32()),
CodeMode: codemode.CodeMode(mrand.Intn(0xff)),
Size: mrand.Uint64(),
BlobSize: mrand.Uint32(),
Crc: mrand.Uint32(),
}
num := mrand.Intn(5)
for i := 0; i < num; i++ {
loc.Blobs = append(loc.Blobs, access.SliceInfo{
MinBid: proto.BlobID(mrand.Uint64()),
Vid: proto.Vid(mrand.Uint32()),
Count: mrand.Uint32(),
})
}
buf := loc.Encode()
bufx := make([]byte, len(buf))
n := loc.Encode2(bufx)
require.Equal(t, len(buf), n)
require.Equal(t, buf, bufx)
require.Panics(t, func() { loc.Encode2(nil) })
require.Panics(t, func() { loc.Encode2(bufx[:3]) })
require.Panics(t, func() { loc.Encode2(bufx[:n/2]) })
require.Panics(t, func() { loc.Encode2(bufx[:n-1]) })
locx := access.Location{}
locx.Decode(bufx)
require.Equal(t, loc.ToString(), locx.ToString())
require.Equal(t, loc.HexString(), locx.HexString())
require.Equal(t, loc.Base64String(), locx.Base64String())
str := loc.ToString()
locx, err := access.DecodeLocationFrom(str)
require.NoError(t, err)
require.Equal(t, *loc, locx)
str = loc.HexString()
locx, err = access.DecodeLocationFromHex(str)
require.NoError(t, err)
require.Equal(t, *loc, locx)
str = loc.Base64String()
locx, err = access.DecodeLocationFromBase64(str)
require.NoError(t, err)
require.Equal(t, *loc, locx)
}
}
func TestLocationDecodeError(t *testing.T) {
loc := &access.Location{
ClusterID: proto.ClusterID(math.MaxUint32),
CodeMode: codemode.CodeMode(math.MaxInt8),
Size: math.MaxUint64,
BlobSize: math.MaxUint32,
Crc: math.MaxUint32,
}
buf := loc.Encode()
require.Equal(t, 25+1, len(buf))
for _, n := range []int{3, 8, 9, 19, 24} {
_, _, err := access.DecodeLocation(buf[:n])
require.Error(t, err)
t.Log(err)
}
loc.Blobs = append(loc.Blobs, access.SliceInfo{
MinBid: proto.BlobID(math.MaxUint64),
Vid: proto.Vid(math.MaxUint32),
Count: math.MaxUint32,
})
buf = loc.Encode()
require.Equal(t, 25+1+20, len(buf))
for _, n := range []int{25, 35, 40, 45} {
_, _, err := access.DecodeLocation(buf[:n])
require.Error(t, err)
t.Log(err)
}
}
func TestLocationSpread(t *testing.T) {
{
var loc access.Location
blobs := loc.Spread()
require.NotNil(t, blobs)
require.Equal(t, 0, len(blobs))
}
{
loc := &access.Location{
Size: 10,
BlobSize: 1 << 22,
Blobs: []access.SliceInfo{{
MinBid: 100,
Vid: 4,
Count: 1,
}},
}
blobs := loc.Spread()
require.Equal(t, 1, len(blobs))
require.Equal(t, proto.BlobID(100), blobs[0].Bid)
require.Equal(t, proto.Vid(4), blobs[0].Vid)
require.Equal(t, uint32(10), blobs[0].Size)
}
{
loc := &access.Location{
Size: (1 << 22) + 10,
BlobSize: 1 << 22,
Blobs: []access.SliceInfo{{
MinBid: 100,
Vid: 4,
Count: 2,
}},
}
blobs := loc.Spread()
require.Equal(t, 2, len(blobs))
require.Equal(t, proto.BlobID(100), blobs[0].Bid)
require.Equal(t, proto.Vid(4), blobs[0].Vid)
require.Equal(t, uint32(1<<22), blobs[0].Size)
require.Equal(t, proto.BlobID(101), blobs[1].Bid)
require.Equal(t, proto.Vid(4), blobs[1].Vid)
require.Equal(t, uint32(10), blobs[1].Size)
}
{
loc := &access.Location{
Size: 1 << 23,
BlobSize: 1 << 22,
Blobs: []access.SliceInfo{{
MinBid: 100,
Vid: 4,
Count: 1,
}, {
MinBid: 200,
Vid: 4,
Count: 1,
}},
}
blobs := loc.Spread()
require.Equal(t, 2, len(blobs))
require.Equal(t, proto.BlobID(100), blobs[0].Bid)
require.Equal(t, proto.Vid(4), blobs[0].Vid)
require.Equal(t, uint32(1<<22), blobs[0].Size)
require.Equal(t, proto.BlobID(200), blobs[1].Bid)
require.Equal(t, proto.Vid(4), blobs[1].Vid)
require.Equal(t, uint32(1<<22), blobs[1].Size)
}
}
func TestPutArgs(t *testing.T) {
cases := []struct {
size int64
@ -538,7 +352,7 @@ func TestGetArgs(t *testing.T) {
Offset: cs.offset,
ReadSize: cs.readSize,
}
args.Location.Size = cs.size
args.Location.Size_ = cs.size
require.Equal(t, cs.valid, args.IsValid())
}
}
@ -547,11 +361,11 @@ func TestDeleteArgs(t *testing.T) {
args := access.DeleteArgs{}
require.False(t, args.IsValid())
require.False(t, (*access.DeleteArgs)(nil).IsValid())
args.Locations = []access.Location{{}}
args.Locations = []proto.Location{{}}
require.True(t, args.IsValid())
args.Locations = make([]access.Location, access.MaxDeleteLocations)
args.Locations = make([]proto.Location, access.MaxDeleteLocations)
require.True(t, args.IsValid())
args.Locations = make([]access.Location, access.MaxDeleteLocations+1)
args.Locations = make([]proto.Location, access.MaxDeleteLocations+1)
require.False(t, args.IsValid())
}
@ -573,7 +387,7 @@ func TestSignArgs(t *testing.T) {
args := access.SignArgs{}
require.False(t, args.IsValid())
require.False(t, (*access.DeleteArgs)(nil).IsValid())
args.Locations = []access.Location{{}}
args.Locations = []proto.Location{{}}
require.True(t, args.IsValid())
}

2388
blobstore/api/shardnode/shardnode.pb.go
View File

File diff suppressed because it is too large Load Diff

66
blobstore/api/shardnode/shardnode.proto
View File

@ -1,3 +1,17 @@
// Copyright 2022 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
syntax = "proto3";
package cubefs.blobstore.api.shardnode;
@ -10,6 +24,7 @@ option (gogoproto.unmarshaler_all) = true;
import "gogoproto/gogo.proto";
import "cubefs/blobstore/common/sharding/range.proto";
import "cubefs/blobstore/api/clustermgr/shard.proto";
import "cubefs/blobstore/common/proto/blob.proto";
message Item {
bytes id = 1 [(gogoproto.customname) = "ID"];
@ -98,5 +113,54 @@ message GetShardRequest {
}
message GetShardResponse {
cubefs.blobstore.api.clustermgr.Shard shard = 1 [(gogoproto.nullable) = false];
cubefs.blobstore.api.clustermgr.ShardUnitInfo shard = 1 [(gogoproto.nullable) = false];
}
message CreateBlobRequest {
ShardOpHeader header = 1 [(gogoproto.nullable) = false];
bytes name = 2;
uint32 codemode = 3 [(gogoproto.customname) = "CodeMode", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/codemode.CodeMode"];
uint64 size = 4;
}
message CreateBlobResponse {
cubefs.blobstore.common.proto.Blob blob = 1 [(gogoproto.nullable) = false];
}
message GetBlobRequest {
ShardOpHeader header = 1 [(gogoproto.nullable) = false];
bytes name = 2;
}
message GetBlobResponse {
cubefs.blobstore.common.proto.Blob blob = 1 [(gogoproto.nullable) = false];
}
message ListBlobRequest {
ShardOpHeader header = 1 [(gogoproto.nullable) = false];
bytes prefix = 2;
bytes marker = 3;
uint64 count = 4;
}
message ListBlobResponse {
repeated cubefs.blobstore.common.proto.Blob blobs = 1 [(gogoproto.nullable) = false];
bytes nextMarker = 2;
}
message DeleteBlobRequest {
ShardOpHeader header = 1 [(gogoproto.nullable) = false];
bytes name = 2;
}
message RetainBlobRequest {
ShardOpHeader header = 1 [(gogoproto.nullable) = false];
bytes name = 2;
uint64 lease = 3;
}
message SealBlobRequest {
ShardOpHeader header = 1 [(gogoproto.nullable) = false];
uint64 size = 2;
bytes name = 3;
}

4
blobstore/blobstore.go
View File

@ -35,10 +35,12 @@ package blobstore
// protoc -I /usr/local/include/ -I ${current}/vendor/github.com/gogo/protobuf/ -I ${current}../ --proto_path=. --gogo_out=Mcubefs/blobstore/common/sharding/range.proto=github.com/cubefs/cubefs/blobstore/common/sharding,plugins=grpc:. *.proto
// generate api/shardnode
// cd blobstore/api/shardnode
// protoc -I ${current}/../ -I /usr/local/include/ -I ${current}/vendor/github.com/gogo/protobuf/ --proto_path=. --gogo_out=Mcubefs/blobstore/common/sharding/range.proto=github.com/cubefs/cubefs/blobstore/common/sharding,Mcubefs/blobstore/api/clustermgr/shard.proto=github.com/cubefs/cubefs/blobstore/api/clustermgr,plugins=grpc:. *.proto
// protoc -I ${current}/../ -I /usr/local/include/ -I ${current}/vendor/github.com/gogo/protobuf/ --proto_path=. --gogo_out=Mcubefs/blobstore/common/sharding/range.proto=github.com/cubefs/cubefs/blobstore/common/sharding,Mcubefs/blobstore/api/clustermgr/shard.proto=github.com/cubefs/cubefs/blobstore/api/clustermgr,Mcubefs/blobstore/common/proto/blob.proto=github.com/cubefs/cubefs/blobstore/common/proto,plugins=grpc:. *.proto
// generate api/clustermgr
// cd blobstore/api/clustermgr
// protoc -I ${current}/../ -I /usr/local/include/ -I ${current}/vendor/github.com/gogo/protobuf/ --proto_path=. --gogo_out=Mcubefs/blobstore/common/sharding/range.proto=github.com/cubefs/cubefs/blobstore/common/sharding,plugins=grpc:. *.proto
// generate raft package proto file
// cd blobstore/common/raft
// protoc -I /usr/local/include/ -I ${current}/vendor/go.etcd.io/etcd/raft/v3/ -I ${current}/vendor/github.com/gogo/protobuf/ --proto_path=. --gogo_out=plugins=grpc:. --gogo_opt=Mraftpb/raft.proto=go.etcd.io/etcd/raft/v3/raftpb *.proto
// cd blobstore/common/proto
// protoc -I ${current}/../ -I /usr/local/include/ -I ${current}/vendor/github.com/gogo/protobuf/ --proto_path=. --gogo_out=Mcubefs/blobstore/common/sharding/range.proto=github.com/cubefs/cubefs/blobstore/common/sharding,Mcubefs/blobstore/api/clustermgr/shard.proto=github.com/cubefs/cubefs/blobstore/api/clustermgr,plugins=grpc:. *.proto

3
blobstore/cli/access/access.go
View File

@ -26,6 +26,7 @@ import (
"github.com/cubefs/cubefs/blobstore/cli/common/flags"
"github.com/cubefs/cubefs/blobstore/cli/common/fmt"
"github.com/cubefs/cubefs/blobstore/cli/config"
"github.com/cubefs/cubefs/blobstore/common/proto"
)
func newAccessClient() (access.API, error) {
@ -45,7 +46,7 @@ func newAccessClient() (access.API, error) {
})
}
func readLocation(f grumble.FlagMap) (loc access.Location, err error) {
func readLocation(f grumble.FlagMap) (loc proto.Location, err error) {
if f.String("location") != "" {
loc, err = cfmt.ParseLocation(f.String("location"))
return

3
blobstore/cli/access/del.go
View File

@ -23,6 +23,7 @@ import (
"github.com/cubefs/cubefs/blobstore/cli/common/flags"
"github.com/cubefs/cubefs/blobstore/cli/common/fmt"
"github.com/cubefs/cubefs/blobstore/cli/config"
"github.com/cubefs/cubefs/blobstore/common/proto"
)
func delFile(c *grumble.Context) error {
@ -48,7 +49,7 @@ func delFile(c *grumble.Context) error {
return nil
}
deleteArgs := &access.DeleteArgs{
Locations: []access.Location{location},
Locations: []proto.Location{location},
}
_, err = client.Delete(common.CmdContext(), deleteArgs)
if err != nil {

2
blobstore/cli/access/get.go
View File

@ -51,7 +51,7 @@ func getFile(c *grumble.Context) error {
size := c.Flags.Uint64("readsize")
if size == 0 {
size = location.Size - c.Flags.Uint64("offset")
size = location.Size_ - c.Flags.Uint64("offset")
}
r, err := client.Get(common.CmdContext(), &access.GetArgs{

27
blobstore/cli/common/cfmt/access.go
View File

@ -21,31 +21,32 @@ import (
"github.com/cubefs/cubefs/blobstore/api/access"
"github.com/cubefs/cubefs/blobstore/cli/common"
"github.com/cubefs/cubefs/blobstore/common/proto"
)
// ParseLocation parst location from json or string
func ParseLocation(jsonORstr string) (access.Location, error) {
var loc access.Location
func ParseLocation(jsonORstr string) (proto.Location, error) {
var loc proto.Location
var err error
if err = common.Unmarshal([]byte(jsonORstr), &loc); err == nil {
return loc, nil
}
if loc, err = access.DecodeLocationFromHex(jsonORstr); err == nil {
if loc, err = proto.DecodeLocationFromHex(jsonORstr); err == nil {
return loc, nil
}
if loc, err = access.DecodeLocationFromBase64(jsonORstr); err == nil {
if loc, err = proto.DecodeLocationFromBase64(jsonORstr); err == nil {
return loc, nil
}
return loc, fmt.Errorf("invalid (%s) %s", jsonORstr, err.Error())
}
// LocationJoin join line into string
func LocationJoin(loc *access.Location, prefix string) string {
func LocationJoin(loc *proto.Location, prefix string) string {
return joinWithPrefix(prefix, LocationF(loc))
}
// LocationF fmt pointer of Location
func LocationF(loc *access.Location) (vals []string) {
func LocationF(loc *proto.Location) (vals []string) {
if loc == nil {
return nilStrings[:]
}
@ -54,16 +55,16 @@ func LocationF(loc *access.Location) (vals []string) {
fmt.Sprintf("Crc : %-12d (0x%x)", loc.Crc, loc.Crc),
fmt.Sprintf("ClusterID : %d", loc.ClusterID),
fmt.Sprintf("CodeMode : %-12d (%s)", loc.CodeMode, loc.CodeMode.String()),
fmt.Sprintf("Size : %-12d (%s)", loc.Size, humanize.IBytes(loc.Size)),
fmt.Sprintf("BlobSize : %-12d (%s)", loc.BlobSize, humanize.IBytes(uint64(loc.BlobSize))),
fmt.Sprintf("Blobs: (%d) [", len(loc.Blobs)),
fmt.Sprintf("Size_ : %-12d (%s)", loc.Size_, humanize.IBytes(loc.Size_)),
fmt.Sprintf("SliceSize : %-12d (%s)", loc.SliceSize, humanize.IBytes(uint64(loc.SliceSize))),
fmt.Sprintf("Slices: (%d) [", len(loc.Slices)),
}...)
for idx, blob := range loc.Blobs {
vals = append(vals, fmt.Sprintf(" >:%3d| MinBid: %-20d Vid: %-10d Count: %-10d",
idx, blob.MinBid, blob.Vid, blob.Count))
for idx, blob := range loc.Slices {
vals = append(vals, fmt.Sprintf(" >:%3d| MinSliceID: %-20d Vid: %-10d Count: %-10d",
idx, blob.MinSliceID, blob.Vid, blob.Count))
}
vals = append(vals, "]")
vals = append(vals, fmt.Sprintf("--> Encode: %d of %d bytes", len(loc.Encode()), 21+16*len(loc.Blobs)))
vals = append(vals, fmt.Sprintf("--> Encode: %d of %d bytes", len(loc.Encode()), 21+16*len(loc.Slices)))
vals = append(vals, fmt.Sprintf("--> Hex : %s", loc.HexString()))
vals = append(vals, fmt.Sprintf("--> Base64: %s", loc.Base64String()))
return

31
blobstore/cli/common/cfmt/access_test.go
View File

@ -28,6 +28,7 @@ import (
"github.com/cubefs/cubefs/blobstore/cli/common"
"github.com/cubefs/cubefs/blobstore/cli/common/cfmt"
"github.com/cubefs/cubefs/blobstore/cli/common/fmt"
"github.com/cubefs/cubefs/blobstore/common/proto"
)
func init() {
@ -38,16 +39,16 @@ func init() {
}
func TestParseLocation(t *testing.T) {
loc := access.Location{
loc := proto.Location{
ClusterID: 1,
CodeMode: 3,
Size: 19213422425,
BlobSize: 1 << 22,
Size_: 19213422425,
SliceSize: 1 << 22,
Crc: 1 << 31,
Blobs: []access.SliceInfo{
{MinBid: 0x199, Vid: 100020, Count: 0},
{MinBid: 10, Vid: 20, Count: 300},
{MinBid: 14225224, Vid: 0xffffffff, Count: 1 << 31},
Slices: []proto.Slice{
{MinSliceID: 0x199, Vid: 100020, Count: 0},
{MinSliceID: 10, Vid: 20, Count: 300},
{MinSliceID: 14225224, Vid: 0xffffffff, Count: 1 << 31},
},
}
@ -59,7 +60,7 @@ func TestParseLocation(t *testing.T) {
locx, err := cfmt.ParseLocation("{}")
require.NoError(t, err)
require.Equal(t, access.Location{}, locx)
require.Equal(t, proto.Location{}, locx)
locx, err = cfmt.ParseLocation(loc.ToString())
require.NoError(t, err)
@ -76,16 +77,16 @@ func TestParseLocation(t *testing.T) {
}
func TestLocation(t *testing.T) {
loc := access.Location{
loc := proto.Location{
ClusterID: 1,
CodeMode: 3,
Size: 19213422425,
BlobSize: 1 << 22,
Size_: 19213422425,
SliceSize: 1 << 22,
Crc: 1 << 31,
Blobs: []access.SliceInfo{
{MinBid: 0x199, Vid: 100020, Count: 0},
{MinBid: 10, Vid: 20, Count: 300},
{MinBid: 14225224, Vid: 0xffffffff, Count: 1 << 31},
Slices: []proto.Slice{
{MinSliceID: 0x199, Vid: 100020, Count: 0},
{MinSliceID: 10, Vid: 20, Count: 300},
{MinSliceID: 14225224, Vid: 0xffffffff, Count: 1 << 31},
},
}
printLine()

4
blobstore/cli/util.go
View File

@ -25,11 +25,11 @@ import (
"github.com/dustin/go-humanize"
"github.com/fatih/color"
"github.com/cubefs/cubefs/blobstore/api/access"
"github.com/cubefs/cubefs/blobstore/cli/common"
"github.com/cubefs/cubefs/blobstore/cli/common/args"
"github.com/cubefs/cubefs/blobstore/cli/common/cfmt"
"github.com/cubefs/cubefs/blobstore/cli/common/fmt"
"github.com/cubefs/cubefs/blobstore/common/proto"
"github.com/cubefs/cubefs/blobstore/common/uptoken"
)
@ -164,7 +164,7 @@ func registerUtil(app *grumble.App) {
if err != nil {
return fmt.Errorf("invalid (%s) %s", jsonORstr, err.Error())
}
loc, n, err := access.DecodeLocation(src)
loc, n, err := proto.DecodeLocation(src)
if err != nil {
fmt.Printf("has read bytes %d / %d\n", n, len(src))
fmt.Println(cfmt.LocationJoin(&loc, ""))

999
blobstore/common/proto/blob.pb.go Normal file
View File

@ -0,0 +1,999 @@
// Code generated by protoc-gen-gogo. DO NOT EDIT.
// source: blob.proto
package proto
import (
fmt "fmt"
github_com_cubefs_cubefs_blobstore_common_codemode "github.com/cubefs/cubefs/blobstore/common/codemode"
_ "github.com/gogo/protobuf/gogoproto"
proto "github.com/gogo/protobuf/proto"
io "io"
math "math"
math_bits "math/bits"
)
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
type Location struct {
ClusterID ClusterID `protobuf:"varint,1,opt,name=cluster_id,json=clusterId,proto3,casttype=ClusterID" json:"cluster_id,omitempty"`
CodeMode github_com_cubefs_cubefs_blobstore_common_codemode.CodeMode `protobuf:"varint,2,opt,name=codemode,proto3,casttype=github.com/cubefs/cubefs/blobstore/common/codemode.CodeMode" json:"codemode,omitempty"`
Size_ uint64 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"`
SliceSize uint32 `protobuf:"varint,4,opt,name=slice_size,json=sliceSize,proto3" json:"slice_size,omitempty"`
Crc uint32 `protobuf:"varint,5,opt,name=crc,proto3" json:"crc,omitempty"`
Slices []Slice `protobuf:"bytes,6,rep,name=slices,proto3" json:"slices"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *Location) Reset() { *m = Location{} }
func (m *Location) String() string { return proto.CompactTextString(m) }
func (*Location) ProtoMessage() {}
func (*Location) Descriptor() ([]byte, []int) {
return fileDescriptor_6903d1e8a20272e8, []int{0}
}
func (m *Location) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *Location) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_Location.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *Location) XXX_Merge(src proto.Message) {
xxx_messageInfo_Location.Merge(m, src)
}
func (m *Location) XXX_Size() int {
return m.Size()
}
func (m *Location) XXX_DiscardUnknown() {
xxx_messageInfo_Location.DiscardUnknown(m)
}
var xxx_messageInfo_Location proto.InternalMessageInfo
func (m *Location) GetClusterID() ClusterID {
if m != nil {
return m.ClusterID
}
return 0
}
func (m *Location) GetCodeMode() github_com_cubefs_cubefs_blobstore_common_codemode.CodeMode {
if m != nil {
return m.CodeMode
}
return 0
}
func (m *Location) GetSize_() uint64 {
if m != nil {
return m.Size_
}
return 0
}
func (m *Location) GetSliceSize() uint32 {
if m != nil {
return m.SliceSize
}
return 0
}
func (m *Location) GetCrc() uint32 {
if m != nil {
return m.Crc
}
return 0
}
func (m *Location) GetSlices() []Slice {
if m != nil {
return m.Slices
}
return nil
}
type Blob struct {
Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
Location Location `protobuf:"bytes,2,opt,name=location,proto3" json:"location"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *Blob) Reset() { *m = Blob{} }
func (m *Blob) String() string { return proto.CompactTextString(m) }
func (*Blob) ProtoMessage() {}
func (*Blob) Descriptor() ([]byte, []int) {
return fileDescriptor_6903d1e8a20272e8, []int{1}
}
func (m *Blob) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *Blob) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_Blob.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *Blob) XXX_Merge(src proto.Message) {
xxx_messageInfo_Blob.Merge(m, src)
}
func (m *Blob) XXX_Size() int {
return m.Size()
}
func (m *Blob) XXX_DiscardUnknown() {
xxx_messageInfo_Blob.DiscardUnknown(m)
}
var xxx_messageInfo_Blob proto.InternalMessageInfo
func (m *Blob) GetName() []byte {
if m != nil {
return m.Name
}
return nil
}
func (m *Blob) GetLocation() Location {
if m != nil {
return m.Location
}
return Location{}
}
type Slice struct {
MinSliceID BlobID `protobuf:"varint,1,opt,name=min_slice_id,json=minSliceId,proto3,casttype=BlobID" json:"min_slice_id,omitempty"`
Vid Vid `protobuf:"varint,2,opt,name=vid,proto3,casttype=Vid" json:"vid,omitempty"`
Count uint32 `protobuf:"varint,3,opt,name=count,proto3" json:"count,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *Slice) Reset() { *m = Slice{} }
func (m *Slice) String() string { return proto.CompactTextString(m) }
func (*Slice) ProtoMessage() {}
func (*Slice) Descriptor() ([]byte, []int) {
return fileDescriptor_6903d1e8a20272e8, []int{2}
}
func (m *Slice) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *Slice) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_Slice.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *Slice) XXX_Merge(src proto.Message) {
xxx_messageInfo_Slice.Merge(m, src)
}
func (m *Slice) XXX_Size() int {
return m.Size()
}
func (m *Slice) XXX_DiscardUnknown() {
xxx_messageInfo_Slice.DiscardUnknown(m)
}
var xxx_messageInfo_Slice proto.InternalMessageInfo
func (m *Slice) GetMinSliceID() BlobID {
if m != nil {
return m.MinSliceID
}
return 0
}
func (m *Slice) GetVid() Vid {
if m != nil {
return m.Vid
}
return 0
}
func (m *Slice) GetCount() uint32 {
if m != nil {
return m.Count
}
return 0
}
func init() {
proto.RegisterType((*Location)(nil), "cubefs.blobstore.common.proto.Location")
proto.RegisterType((*Blob)(nil), "cubefs.blobstore.common.proto.Blob")
proto.RegisterType((*Slice)(nil), "cubefs.blobstore.common.proto.Slice")
}
func init() { proto.RegisterFile("blob.proto", fileDescriptor_6903d1e8a20272e8) }
var fileDescriptor_6903d1e8a20272e8 = []byte{
// 396 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x51, 0x3d, 0x8f, 0xd3, 0x40,
0x10, 0xc5, 0x67, 0xc7, 0x38, 0x73, 0x77, 0x12, 0x5a, 0x9d, 0x90, 0x39, 0xe9, 0xe2, 0x28, 0x42,
0x22, 0x95, 0x2d, 0x1d, 0x15, 0x5c, 0xe7, 0xa4, 0xb1, 0x20, 0xcd, 0x46, 0xa2, 0xa0, 0x89, 0xe2,
0xdd, 0xc5, 0xac, 0x64, 0x7b, 0x90, 0x3f, 0x28, 0xf8, 0x63, 0xfc, 0x85, 0x94, 0xfc, 0x02, 0x0b,
0xf9, 0x67, 0xa4, 0x42, 0x3b, 0xb6, 0x23, 0x2a, 0xa8, 0xf6, 0xed, 0x9b, 0x99, 0x9d, 0xb7, 0xef,
0x01, 0xa4, 0x39, 0xa6, 0xe1, 0xb7, 0x0a, 0x1b, 0x64, 0x0f, 0xa2, 0x4d, 0xd5, 0x97, 0x3a, 0x34,
0x54, 0xdd, 0x60, 0xa5, 0x42, 0x81, 0x45, 0x81, 0xe5, 0x50, 0xbe, 0xbf, 0xcb, 0x30, 0x43, 0x82,
0x91, 0x41, 0x03, 0xbb, 0xfa, 0x79, 0x05, 0xde, 0x47, 0x14, 0xc7, 0x46, 0x63, 0xc9, 0xde, 0x01,
0x88, 0xbc, 0xad, 0x1b, 0x55, 0x1d, 0xb4, 0xf4, 0xad, 0xa5, 0xb5, 0xbe, 0x8d, 0xef, 0xfb, 0x2e,
0x98, 0x6f, 0x06, 0x36, 0xd9, 0x9e, 0xff, 0xbe, 0xf0, 0xf9, 0xd8, 0x9d, 0x48, 0x96, 0x81, 0x27,
0x50, 0xaa, 0x02, 0xa5, 0xf2, 0xaf, 0x68, 0xf0, 0x43, 0xdf, 0x05, 0xde, 0x06, 0xa5, 0xda, 0xa1,
0x54, 0xe7, 0x2e, 0x78, 0xca, 0x74, 0xf3, 0xb5, 0x4d, 0x8d, 0xa8, 0x68, 0x50, 0x3a, 0x1d, 0x17,
0xc1, 0xd1, 0x20, 0x38, 0x9a, 0x5e, 0x0a, 0xa7, 0x71, 0x7e, 0x79, 0x9c, 0x31, 0x70, 0x6a, 0xfd,
0x43, 0xf9, 0xf6, 0xd2, 0x5a, 0x3b, 0x9c, 0x30, 0x7b, 0x00, 0xa8, 0x73, 0x2d, 0xd4, 0x81, 0x2a,
0x8e, 0x59, 0xcf, 0xe7, 0xc4, 0xec, 0x4d, 0xf9, 0x05, 0xd8, 0xa2, 0x12, 0xfe, 0x8c, 0x78, 0x03,
0x59, 0x0c, 0x2e, 0x95, 0x6b, 0xdf, 0x5d, 0xda, 0xeb, 0xeb, 0xc7, 0xd7, 0xe1, 0x3f, 0xbd, 0x0b,
0xf7, 0xa6, 0x39, 0x76, 0x4e, 0x5d, 0xf0, 0x8c, 0x8f, 0x93, 0x2b, 0x05, 0x4e, 0x9c, 0x63, 0x6a,
0x04, 0x95, 0xc7, 0x42, 0x91, 0x5d, 0x37, 0x9c, 0x30, 0x4b, 0xc0, 0xcb, 0x47, 0x53, 0xc9, 0x8d,
0xeb, 0xc7, 0x37, 0xff, 0xd9, 0x30, 0x65, 0x30, 0x2e, 0xb9, 0x8c, 0xaf, 0x1a, 0x98, 0xd1, 0x76,
0xf6, 0x1e, 0x6e, 0x0a, 0x5d, 0x1e, 0x86, 0x8f, 0x8e, 0xf1, 0x38, 0xb1, 0xdf, 0x77, 0x01, 0xec,
0x74, 0x49, 0x3d, 0x94, 0x8f, 0x6b, 0x54, 0x25, 0x5b, 0x0e, 0xc5, 0xc4, 0x4a, 0xf6, 0x0a, 0xec,
0xef, 0x5a, 0x8e, 0xc1, 0x3c, 0x3f, 0x77, 0x81, 0xfd, 0x49, 0x4b, 0x6e, 0x38, 0x76, 0x07, 0x33,
0x81, 0x6d, 0xd9, 0x90, 0xa1, 0xb7, 0x7c, 0xb8, 0xc4, 0x2f, 0x4f, 0xfd, 0xc2, 0xfa, 0xd5, 0x2f,
0xac, 0xdf, 0xfd, 0xc2, 0xfa, 0xec, 0x85, 0xd1, 0x13, 0xc9, 0x4c, 0x5d, 0x3a, 0xde, 0xfe, 0x09,
0x00, 0x00, 0xff, 0xff, 0x1d, 0xa9, 0x5c, 0xb0, 0x78, 0x02, 0x00, 0x00,
}
func (m *Location) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *Location) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *Location) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if m.XXX_unrecognized != nil {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if len(m.Slices) > 0 {
for iNdEx := len(m.Slices) - 1; iNdEx >= 0; iNdEx-- {
{
size, err := m.Slices[iNdEx].MarshalToSizedBuffer(dAtA[:i])
if err != nil {
return 0, err
}
i -= size
i = encodeVarintBlob(dAtA, i, uint64(size))
}
i--
dAtA[i] = 0x32
}
}
if m.Crc != 0 {
i = encodeVarintBlob(dAtA, i, uint64(m.Crc))
i--
dAtA[i] = 0x28
}
if m.SliceSize != 0 {
i = encodeVarintBlob(dAtA, i, uint64(m.SliceSize))
i--
dAtA[i] = 0x20
}
if m.Size_ != 0 {
i = encodeVarintBlob(dAtA, i, uint64(m.Size_))
i--
dAtA[i] = 0x18
}
if m.CodeMode != 0 {
i = encodeVarintBlob(dAtA, i, uint64(m.CodeMode))
i--
dAtA[i] = 0x10
}
if m.ClusterID != 0 {
i = encodeVarintBlob(dAtA, i, uint64(m.ClusterID))
i--
dAtA[i] = 0x8
}
return len(dAtA) - i, nil
}
func (m *Blob) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *Blob) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *Blob) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if m.XXX_unrecognized != nil {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
{
size, err := m.Location.MarshalToSizedBuffer(dAtA[:i])
if err != nil {
return 0, err
}
i -= size
i = encodeVarintBlob(dAtA, i, uint64(size))
}
i--
dAtA[i] = 0x12
if len(m.Name) > 0 {
i -= len(m.Name)
copy(dAtA[i:], m.Name)
i = encodeVarintBlob(dAtA, i, uint64(len(m.Name)))
i--
dAtA[i] = 0xa
}
return len(dAtA) - i, nil
}
func (m *Slice) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *Slice) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *Slice) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if m.XXX_unrecognized != nil {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.Count != 0 {
i = encodeVarintBlob(dAtA, i, uint64(m.Count))
i--
dAtA[i] = 0x18
}
if m.Vid != 0 {
i = encodeVarintBlob(dAtA, i, uint64(m.Vid))
i--
dAtA[i] = 0x10
}
if m.MinSliceID != 0 {
i = encodeVarintBlob(dAtA, i, uint64(m.MinSliceID))
i--
dAtA[i] = 0x8
}
return len(dAtA) - i, nil
}
func encodeVarintBlob(dAtA []byte, offset int, v uint64) int {
offset -= sovBlob(v)
base := offset
for v >= 1<<7 {
dAtA[offset] = uint8(v&0x7f | 0x80)
v >>= 7
offset++
}
dAtA[offset] = uint8(v)
return base
}
func (m *Location) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
if m.ClusterID != 0 {
n += 1 + sovBlob(uint64(m.ClusterID))
}
if m.CodeMode != 0 {
n += 1 + sovBlob(uint64(m.CodeMode))
}
if m.Size_ != 0 {
n += 1 + sovBlob(uint64(m.Size_))
}
if m.SliceSize != 0 {
n += 1 + sovBlob(uint64(m.SliceSize))
}
if m.Crc != 0 {
n += 1 + sovBlob(uint64(m.Crc))
}
if len(m.Slices) > 0 {
for _, e := range m.Slices {
l = e.Size()
n += 1 + l + sovBlob(uint64(l))
}
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
return n
}
func (m *Blob) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.Name)
if l > 0 {
n += 1 + l + sovBlob(uint64(l))
}
l = m.Location.Size()
n += 1 + l + sovBlob(uint64(l))
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
return n
}
func (m *Slice) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
if m.MinSliceID != 0 {
n += 1 + sovBlob(uint64(m.MinSliceID))
}
if m.Vid != 0 {
n += 1 + sovBlob(uint64(m.Vid))
}
if m.Count != 0 {
n += 1 + sovBlob(uint64(m.Count))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
return n
}
func sovBlob(x uint64) (n int) {
return (math_bits.Len64(x|1) + 6) / 7
}
func sozBlob(x uint64) (n int) {
return sovBlob(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
func (m *Location) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowBlob
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: Location: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: Location: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field ClusterID", wireType)
}
m.ClusterID = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowBlob
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.ClusterID |= ClusterID(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field CodeMode", wireType)
}
m.CodeMode = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowBlob
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.CodeMode |= github_com_cubefs_cubefs_blobstore_common_codemode.CodeMode(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType)
}
m.Size_ = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowBlob
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Size_ |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 4:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field SliceSize", wireType)
}
m.SliceSize = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowBlob
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.SliceSize |= uint32(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 5:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Crc", wireType)
}
m.Crc = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowBlob
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Crc |= uint32(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 6:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Slices", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowBlob
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthBlob
}
postIndex := iNdEx + msglen
if postIndex < 0 {
return ErrInvalidLengthBlob
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Slices = append(m.Slices, Slice{})
if err := m.Slices[len(m.Slices)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipBlob(dAtA[iNdEx:])
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthBlob
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *Blob) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowBlob
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: Blob: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: Blob: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowBlob
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
byteLen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthBlob
}
postIndex := iNdEx + byteLen
if postIndex < 0 {
return ErrInvalidLengthBlob
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Name = append(m.Name[:0], dAtA[iNdEx:postIndex]...)
if m.Name == nil {
m.Name = []byte{}
}
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Location", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowBlob
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthBlob
}
postIndex := iNdEx + msglen
if postIndex < 0 {
return ErrInvalidLengthBlob
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
if err := m.Location.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipBlob(dAtA[iNdEx:])
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthBlob
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *Slice) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowBlob
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: Slice: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: Slice: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field MinSliceID", wireType)
}
m.MinSliceID = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowBlob
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.MinSliceID |= BlobID(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Vid", wireType)
}
m.Vid = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowBlob
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Vid |= Vid(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Count", wireType)
}
m.Count = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowBlob
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Count |= uint32(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipBlob(dAtA[iNdEx:])
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthBlob
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipBlob(dAtA []byte) (n int, err error) {
l := len(dAtA)
iNdEx := 0
depth := 0
for iNdEx < l {
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowBlob
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
wireType := int(wire & 0x7)
switch wireType {
case 0:
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowBlob
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
iNdEx++
if dAtA[iNdEx-1] < 0x80 {
break
}
}
case 1:
iNdEx += 8
case 2:
var length int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowBlob
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
length |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if length < 0 {
return 0, ErrInvalidLengthBlob
}
iNdEx += length
case 3:
depth++
case 4:
if depth == 0 {
return 0, ErrUnexpectedEndOfGroupBlob
}
depth--
case 5:
iNdEx += 4
default:
return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
}
if iNdEx < 0 {
return 0, ErrInvalidLengthBlob
}
if depth == 0 {
return iNdEx, nil
}
}
return 0, io.ErrUnexpectedEOF
}
var (
ErrInvalidLengthBlob = fmt.Errorf("proto: negative length found during unmarshaling")
ErrIntOverflowBlob = fmt.Errorf("proto: integer overflow")
ErrUnexpectedEndOfGroupBlob = fmt.Errorf("proto: unexpected end of group")
)

44
blobstore/common/proto/blob.proto Normal file
View File

@ -0,0 +1,44 @@
// Copyright 2022 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
syntax = "proto3";
package cubefs.blobstore.common.proto;
option go_package = "./;proto";
option (gogoproto.sizer_all) = true;
option (gogoproto.marshaler_all) = true;
option (gogoproto.unmarshaler_all) = true;
import "gogoproto/gogo.proto";
message Location {
uint32 cluster_id = 1 [(gogoproto.customname) = "ClusterID", (gogoproto.casttype) = "ClusterID"];
uint32 codemode = 2 [(gogoproto.customname) = "CodeMode", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/codemode.CodeMode"];
uint64 size = 3;
uint32 slice_size = 4;
uint32 crc = 5;
repeated Slice slices = 6 [(gogoproto.nullable) = false];
}
message Blob {
bytes name = 1;
Location location = 2 [(gogoproto.nullable) = false];
}
message Slice {
uint64 min_slice_id = 1 [(gogoproto.customname) = "MinSliceID", (gogoproto.casttype) = "BlobID"];
uint32 vid = 2 [(gogoproto.casttype) = "Vid"];
uint32 count = 3;
}

272
blobstore/common/proto/blob_ext.go Normal file
View File

@ -0,0 +1,272 @@
// Copyright 2022 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package proto
import (
"encoding/base64"
"encoding/binary"
"encoding/hex"
"fmt"
"github.com/cubefs/cubefs/blobstore/common/codemode"
)
// BlobUnit is one piece of data in a location
//
// Bid is the blob id
// Vid is which volume the blob in
// Size is real size of the blob
type BlobUnit struct {
Bid BlobID
Vid Vid
Size uint32
}
// Copy returns a new same Location
func (loc *Location) Copy() Location {
dst := Location{
ClusterID: loc.ClusterID,
CodeMode: loc.CodeMode,
Size_: loc.Size_,
SliceSize: loc.SliceSize,
Crc: loc.Crc,
Slices: make([]Slice, len(loc.Slices)),
}
copy(dst.Slices, loc.Slices)
return dst
}
// Encode transfer Location to slice byte
// Returns the buf created by me
//
// (n) means max-n bytes
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | field | crc | clusterid | codemode | size | blobsize |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | n-bytes | 4 | uvarint(5) | 1 | uvarint(10) | uvarint(5) |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// 25 + (5){len(blobs)} + len(Slices) * 20
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | blobs | minbid | vid | count | ... |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | n-bytes | (10) | (5) | (5) | (20) | (20) | ... |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
func (loc *Location) Encode() []byte {
if loc == nil {
return nil
}
n := 25 + 5 + len(loc.Slices)*20
buf := make([]byte, n)
n = loc.Encode2(buf)
return buf[:n]
}
// Encode2 transfer Location to the buf, the buf reuse by yourself
// Returns the number of bytes read
// If the buffer is too small, Encode2 will panic
func (loc *Location) Encode2(buf []byte) int {
if loc == nil {
return 0
}
n := 0
binary.BigEndian.PutUint32(buf[n:], loc.Crc)
n += 4
n += binary.PutUvarint(buf[n:], uint64(loc.ClusterID))
buf[n] = byte(loc.CodeMode)
n++
n += binary.PutUvarint(buf[n:], uint64(loc.Size_))
n += binary.PutUvarint(buf[n:], uint64(loc.SliceSize))
n += binary.PutUvarint(buf[n:], uint64(len(loc.Slices)))
for _, blob := range loc.Slices {
n += binary.PutUvarint(buf[n:], uint64(blob.MinSliceID))
n += binary.PutUvarint(buf[n:], uint64(blob.Vid))
n += binary.PutUvarint(buf[n:], uint64(blob.Count))
}
return n
}
// Decode parse location from buf
// Returns the number of bytes read
// Error is not nil when parsing failed
func (loc *Location) Decode(buf []byte) (int, error) {
if loc == nil {
return 0, fmt.Errorf("location receiver is nil")
}
location, n, err := DecodeLocation(buf)
if err != nil {
return n, err
}
*loc = location
return n, nil
}
// ToString transfer location to hex string
func (loc *Location) ToString() string {
return loc.HexString()
}
// HexString transfer location to hex string
func (loc *Location) HexString() string {
return hex.EncodeToString(loc.Encode())
}
// Base64String transfer location to base64 string
func (loc *Location) Base64String() string {
return base64.StdEncoding.EncodeToString(loc.Encode())
}
// Spread location blobs to slice
func (loc *Location) Spread() []BlobUnit {
count := 0
for _, blob := range loc.Slices {
count += int(blob.Count)
}
blobs := make([]BlobUnit, 0, count)
for _, blob := range loc.Slices {
for offset := uint32(0); offset < blob.Count; offset++ {
blobs = append(blobs, BlobUnit{
Bid: blob.MinSliceID + BlobID(offset),
Vid: blob.Vid,
Size: loc.SliceSize,
})
}
}
if len(blobs) > 0 && loc.SliceSize > 0 {
if lastSize := loc.Size_ % uint64(loc.SliceSize); lastSize > 0 {
blobs[len(blobs)-1].Size = uint32(lastSize)
}
}
return blobs
}
// DecodeLocation parse location from buf
// Returns Location and the number of bytes read
// Error is not nil when parsing failed
func DecodeLocation(buf []byte) (Location, int, error) {
var (
loc Location
n int
val uint64
nn int
)
next := func() (uint64, int) {
val, nn := binary.Uvarint(buf)
if nn <= 0 {
return 0, nn
}
n += nn
buf = buf[nn:]
return val, nn
}
if len(buf) < 4 {
return loc, n, fmt.Errorf("bytes crc %d", len(buf))
}
loc.Crc = binary.BigEndian.Uint32(buf)
n += 4
buf = buf[4:]
if val, nn = next(); nn <= 0 {
return loc, n, fmt.Errorf("bytes cluster_id %d", nn)
}
loc.ClusterID = ClusterID(val)
if len(buf) < 1 {
return loc, n, fmt.Errorf("bytes codemode %d", len(buf))
}
loc.CodeMode = codemode.CodeMode(buf[0])
n++
buf = buf[1:]
if val, nn = next(); nn <= 0 {
return loc, n, fmt.Errorf("bytes size %d", nn)
}
loc.Size_ = val
if val, nn = next(); nn <= 0 {
return loc, n, fmt.Errorf("bytes blob_size %d", nn)
}
loc.SliceSize = uint32(val)
if val, nn = next(); nn <= 0 {
return loc, n, fmt.Errorf("bytes length blobs %d", nn)
}
length := int(val)
if length > 0 {
loc.Slices = make([]Slice, 0, length)
}
for index := 0; index < length; index++ {
var blob Slice
if val, nn = next(); nn <= 0 {
return loc, n, fmt.Errorf("bytes %dth-blob min_bid %d", index, nn)
}
blob.MinSliceID = BlobID(val)
if val, nn = next(); nn <= 0 {
return loc, n, fmt.Errorf("bytes %dth-blob vid %d", index, nn)
}
blob.Vid = Vid(val)
if val, nn = next(); nn <= 0 {
return loc, n, fmt.Errorf("bytes %dth-blob count %d", index, nn)
}
blob.Count = uint32(val)
loc.Slices = append(loc.Slices, blob)
}
return loc, n, nil
}
// DecodeLocationFrom decode location from hex string
func DecodeLocationFrom(s string) (Location, error) {
return DecodeLocationFromHex(s)
}
// DecodeLocationFromHex decode location from hex string
func DecodeLocationFromHex(s string) (Location, error) {
var loc Location
src, err := hex.DecodeString(s)
if err != nil {
return loc, err
}
_, err = loc.Decode(src)
if err != nil {
return loc, err
}
return loc, nil
}
// DecodeLocationFromBase64 decode location from base64 string
func DecodeLocationFromBase64(s string) (Location, error) {
var loc Location
src, err := base64.StdEncoding.DecodeString(s)
if err != nil {
return loc, err
}
_, err = loc.Decode(src)
if err != nil {
return loc, err
}
return loc, nil
}

210
blobstore/common/proto/blob_ext_test.go Normal file
View File

@ -0,0 +1,210 @@
// Copyright 2022 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package proto
import (
"math"
mrand "math/rand"
"testing"
"github.com/cubefs/cubefs/blobstore/common/codemode"
"github.com/stretchr/testify/require"
)
func TestLocationEncodeDecodeNil(t *testing.T) {
var loc *Location
require.Nil(t, loc.Encode())
require.Equal(t, 0, loc.Encode2(nil))
require.Equal(t, "", loc.ToString())
require.Equal(t, "", loc.HexString())
require.Equal(t, "", loc.Base64String())
n, err := loc.Decode(nil)
require.Error(t, err)
require.Equal(t, 0, n)
locx, n, err := DecodeLocation(nil)
require.Error(t, err)
require.Equal(t, 0, n)
require.Equal(t, Location{}, locx)
locx, err = DecodeLocationFrom("")
require.Error(t, err)
require.Equal(t, Location{}, locx)
locx, err = DecodeLocationFrom("xxx")
require.Error(t, err)
require.Equal(t, Location{}, locx)
locx, err = DecodeLocationFromHex("xxx")
require.Error(t, err)
require.Equal(t, Location{}, locx)
locx, err = DecodeLocationFromBase64("xxx")
require.Error(t, err)
require.Equal(t, Location{}, locx)
}
func TestLocationEncodeDecode(t *testing.T) {
for ii := 0; ii < 100; ii++ {
loc := &Location{
ClusterID: ClusterID(mrand.Uint32()),
CodeMode: codemode.CodeMode(mrand.Intn(0xff)),
Size_: mrand.Uint64(),
SliceSize: mrand.Uint32(),
Crc: mrand.Uint32(),
}
num := mrand.Intn(5)
for i := 0; i < num; i++ {
loc.Slices = append(loc.Slices, Slice{
MinSliceID: BlobID(mrand.Uint64()),
Vid: Vid(mrand.Uint32()),
Count: mrand.Uint32(),
})
}
buf := loc.Encode()
bufx := make([]byte, len(buf))
n := loc.Encode2(bufx)
require.Equal(t, len(buf), n)
require.Equal(t, buf, bufx)
require.Panics(t, func() { loc.Encode2(nil) })
require.Panics(t, func() { loc.Encode2(bufx[:3]) })
require.Panics(t, func() { loc.Encode2(bufx[:n/2]) })
require.Panics(t, func() { loc.Encode2(bufx[:n-1]) })
locx := Location{}
locx.Decode(bufx)
require.Equal(t, loc.ToString(), locx.ToString())
require.Equal(t, loc.HexString(), locx.HexString())
require.Equal(t, loc.Base64String(), locx.Base64String())
str := loc.ToString()
locx, err := DecodeLocationFrom(str)
require.NoError(t, err)
require.Equal(t, *loc, locx)
str = loc.HexString()
locx, err = DecodeLocationFromHex(str)
require.NoError(t, err)
require.Equal(t, *loc, locx)
str = loc.Base64String()
locx, err = DecodeLocationFromBase64(str)
require.NoError(t, err)
require.Equal(t, *loc, locx)
}
}
func TestLocationDecodeError(t *testing.T) {
loc := &Location{
ClusterID: ClusterID(math.MaxUint32),
CodeMode: codemode.CodeMode(math.MaxInt8),
Size_: math.MaxUint64,
SliceSize: math.MaxUint32,
Crc: math.MaxUint32,
}
buf := loc.Encode()
require.Equal(t, 25+1, len(buf))
for _, n := range []int{3, 8, 9, 19, 24} {
_, _, err := DecodeLocation(buf[:n])
require.Error(t, err)
t.Log(err)
}
loc.Slices = append(loc.Slices, Slice{
MinSliceID: BlobID(math.MaxUint64),
Vid: Vid(math.MaxUint32),
Count: math.MaxUint32,
})
buf = loc.Encode()
require.Equal(t, 25+1+20, len(buf))
for _, n := range []int{25, 35, 40, 45} {
_, _, err := DecodeLocation(buf[:n])
require.Error(t, err)
t.Log(err)
}
}
func TestLocationSpread(t *testing.T) {
{
var loc Location
blobs := loc.Spread()
require.NotNil(t, blobs)
require.Equal(t, 0, len(blobs))
}
{
loc := &Location{
Size_: 10,
SliceSize: 1 << 22,
Slices: []Slice{{
MinSliceID: 100,
Vid: 4,
Count: 1,
}},
}
blobs := loc.Spread()
require.Equal(t, 1, len(blobs))
require.Equal(t, BlobID(100), blobs[0].Bid)
require.Equal(t, Vid(4), blobs[0].Vid)
require.Equal(t, uint32(10), blobs[0].Size)
}
{
loc := &Location{
Size_: (1 << 22) + 10,
SliceSize: 1 << 22,
Slices: []Slice{{
MinSliceID: 100,
Vid: 4,
Count: 2,
}},
}
blobs := loc.Spread()
require.Equal(t, 2, len(blobs))
require.Equal(t, BlobID(100), blobs[0].Bid)
require.Equal(t, Vid(4), blobs[0].Vid)
require.Equal(t, uint32(1<<22), blobs[0].Size)
require.Equal(t, BlobID(101), blobs[1].Bid)
require.Equal(t, Vid(4), blobs[1].Vid)
require.Equal(t, uint32(10), blobs[1].Size)
}
{
loc := &Location{
Size_: 1 << 23,
SliceSize: 1 << 22,
Slices: []Slice{{
MinSliceID: 100,
Vid: 4,
Count: 1,
}, {
MinSliceID: 200,
Vid: 4,
Count: 1,
}},
}
blobs := loc.Spread()
require.Equal(t, 2, len(blobs))
require.Equal(t, BlobID(100), blobs[0].Bid)
require.Equal(t, Vid(4), blobs[0].Vid)
require.Equal(t, uint32(1<<22), blobs[0].Size)
require.Equal(t, BlobID(200), blobs[1].Bid)
require.Equal(t, Vid(4), blobs[1].Vid)
require.Equal(t, uint32(1<<22), blobs[1].Size)
}
}

14
blobstore/common/sharding/proto_test.go
View File

@ -1,3 +1,17 @@
// Copyright 2022 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package sharding
import (

14
blobstore/common/sharding/range.pb.go
View File

@ -1,3 +1,17 @@
// Copyright 2022 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
// Code generated by protoc-gen-gogo. DO NOT EDIT.
// source: range.proto

14
blobstore/common/sharding/range.proto
View File

@ -1,3 +1,17 @@
// Copyright 2022 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
syntax = "proto3";
package cubefs.blobstore.common.sharding;

14
blobstore/common/sharding/rangeext_test.go
View File

@ -1,3 +1,17 @@
// Copyright 2022 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package sharding
import (

104
blobstore/sdk/sdk_client.go
View File

@ -114,7 +114,7 @@ func (s *sdkHandler) Get(ctx context.Context, args *acapi.GetArgs) (io.ReadClose
}
ctx = acapi.ClientWithReqidContext(ctx)
if args.Location.Size == 0 || args.ReadSize == 0 {
if args.Location.Size_ == 0 || args.ReadSize == 0 {
return noopBody{}, nil
}
@ -129,15 +129,15 @@ func (s *sdkHandler) Get(ctx context.Context, args *acapi.GetArgs) (io.ReadClose
return s.doGet(ctx, args)
}
func (s *sdkHandler) Delete(ctx context.Context, args *acapi.DeleteArgs) (failedLocations []acapi.Location, err error) {
func (s *sdkHandler) Delete(ctx context.Context, args *acapi.DeleteArgs) (failedLocations []proto.Location, err error) {
if !args.IsValid() {
return nil, errcode.ErrIllegalArguments
}
ctx = acapi.ClientWithReqidContext(ctx)
locations := make([]acapi.Location, 0, len(args.Locations)) // check location size
locations := make([]proto.Location, 0, len(args.Locations)) // check location size
for _, loc := range args.Locations {
if loc.Size > 0 {
if loc.Size_ > 0 {
locations = append(locations, loc)
}
}
@ -170,9 +170,9 @@ func (s *sdkHandler) Delete(ctx context.Context, args *acapi.DeleteArgs) (failed
return nil, nil
}
func (s *sdkHandler) Put(ctx context.Context, args *acapi.PutArgs) (lc acapi.Location, hm acapi.HashSumMap, err error) {
func (s *sdkHandler) Put(ctx context.Context, args *acapi.PutArgs) (lc proto.Location, hm acapi.HashSumMap, err error) {
if args == nil {
return acapi.Location{}, nil, errcode.ErrIllegalArguments
return proto.Location{}, nil, errcode.ErrIllegalArguments
}
if args.Size == 0 {
@ -180,7 +180,7 @@ func (s *sdkHandler) Put(ctx context.Context, args *acapi.PutArgs) (lc acapi.Loc
for alg := range hashSumMap {
hashSumMap[alg] = alg.ToHasher().Sum(nil)
}
return acapi.Location{Blobs: make([]acapi.SliceInfo, 0)}, hashSumMap, nil
return proto.Location{Slices: make([]proto.Slice, 0)}, hashSumMap, nil
}
ctx = acapi.ClientWithReqidContext(ctx)
@ -189,7 +189,7 @@ func (s *sdkHandler) Put(ctx context.Context, args *acapi.PutArgs) (lc acapi.Loc
if err := s.limiter.Acquire(name); err != nil {
span := trace.SpanFromContextSafe(ctx)
span.Debugf("access concurrent limited %s, err:%+v", name, err)
return acapi.Location{}, nil, errcode.ErrAccessLimited
return proto.Location{}, nil, errcode.ErrAccessLimited
}
defer s.limiter.Release(name)
@ -268,13 +268,13 @@ func (s *sdkHandler) deleteBlob(ctx context.Context, args *acapi.DeleteBlobArgs)
return errcode.ErrIllegalArguments
}
if err := s.handler.Delete(ctx, &acapi.Location{
if err := s.handler.Delete(ctx, &proto.Location{
ClusterID: args.ClusterID,
BlobSize: 1,
Blobs: []acapi.SliceInfo{{
MinBid: args.BlobID,
Vid: args.Vid,
Count: 1,
SliceSize: 1,
Slices: []proto.Slice{{
MinSliceID: args.BlobID,
Vid: args.Vid,
Count: 1,
}},
}); err != nil {
span.Error("stream delete blob failed", errors.Detail(err))
@ -373,14 +373,14 @@ func (s *sdkHandler) doDelete(ctx context.Context, args *acapi.DeleteArgs) (resp
err = errcode.ErrIllegalArguments
return
}
clusterBlobsN[loc.ClusterID] += len(loc.Blobs)
clusterBlobsN[loc.ClusterID] += len(loc.Slices)
}
if len(args.Locations) == 1 {
loc := args.Locations[0]
if err := s.handler.Delete(ctx, &loc); err != nil {
span.Error("stream delete failed", errors.Detail(err))
resp.FailedLocations = []acapi.Location{loc}
resp.FailedLocations = []proto.Location{loc}
}
return
}
@ -391,19 +391,19 @@ func (s *sdkHandler) doDelete(ctx context.Context, args *acapi.DeleteArgs) (resp
// a min delete message about 10-20 bytes,
// max delete locations is 1024, one location is max to 5G,
// merged message max size about 40MB.
merged := make(map[proto.ClusterID][]acapi.SliceInfo, len(clusterBlobsN))
merged := make(map[proto.ClusterID][]proto.Slice, len(clusterBlobsN))
for id, n := range clusterBlobsN {
merged[id] = make([]acapi.SliceInfo, 0, n)
merged[id] = make([]proto.Slice, 0, n)
}
for _, loc := range args.Locations {
merged[loc.ClusterID] = append(merged[loc.ClusterID], loc.Blobs...)
merged[loc.ClusterID] = append(merged[loc.ClusterID], loc.Slices...)
}
for cid := range merged {
if err := s.handler.Delete(ctx, &acapi.Location{
if err := s.handler.Delete(ctx, &proto.Location{
ClusterID: cid,
BlobSize: 1,
Blobs: merged[cid],
SliceSize: 1,
Slices: merged[cid],
}); err != nil {
span.Error("stream delete failed", cid, errors.Detail(err))
for i := range args.Locations {
@ -417,7 +417,7 @@ func (s *sdkHandler) doDelete(ctx context.Context, args *acapi.DeleteArgs) (resp
return
}
func (s *sdkHandler) doPutObject(ctx context.Context, args *acapi.PutArgs) (acapi.Location, acapi.HashSumMap, error) {
func (s *sdkHandler) doPutObject(ctx context.Context, args *acapi.PutArgs) (proto.Location, acapi.HashSumMap, error) {
span := trace.SpanFromContextSafe(ctx)
var err error
@ -425,7 +425,7 @@ func (s *sdkHandler) doPutObject(ctx context.Context, args *acapi.PutArgs) (acap
if !args.IsValid() {
err = errcode.ErrIllegalArguments
span.Error("stream get args is invalid ", errors.Detail(err))
return acapi.Location{}, nil, err
return proto.Location{}, nil, err
}
hashSumMap := args.Hashes.ToHashSumMap()
@ -440,7 +440,7 @@ func (s *sdkHandler) doPutObject(ctx context.Context, args *acapi.PutArgs) (acap
if err != nil {
span.Error("stream put failed", errors.Detail(err))
err = httpError(err)
return acapi.Location{}, nil, err
return proto.Location{}, nil, err
}
// hasher sum
@ -451,7 +451,7 @@ func (s *sdkHandler) doPutObject(ctx context.Context, args *acapi.PutArgs) (acap
if err = stream.LocationCrcFill(loc); err != nil {
span.Error("stream put fill location crc", err)
err = httpError(err)
return acapi.Location{}, nil, err
return proto.Location{}, nil, err
}
span.Infof("done /put request location:%+v hash:%+v", loc, hashSumMap.All())
@ -604,7 +604,7 @@ func (s *sdkHandler) readerPipeline(span trace.Span, reqBody io.Reader,
return ch
}
func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.Location, acapi.HashSumMap, error) {
func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (proto.Location, acapi.HashSumMap, error) {
span := trace.SpanFromContextSafe(ctx)
hashSumMap := args.Hashes.ToHashSumMap()
@ -619,7 +619,7 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L
}
var (
loc acapi.Location
loc proto.Location
tokens []string
)
@ -637,7 +637,7 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L
signArgs.Location = loc.Copy()
signResp, err := s.sign(newCtx, &signArgs)
if err == nil {
locations = []acapi.Location{signResp.Location.Copy()}
locations = []proto.Location{signResp.Location.Copy()}
}
}
if len(locations) > 0 {
@ -650,7 +650,7 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L
// alloc
allocResp, err := s.Alloc(ctx, &acapi.AllocArgs{Size: uint64(args.Size)})
if err != nil {
return acapi.Location{}, nil, err
return proto.Location{}, nil, err
}
loc = allocResp.Location
tokens = allocResp.Tokens
@ -658,7 +658,7 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L
// buffer pipeline
closeCh := make(chan struct{})
bufferPipe := s.readerPipeline(span, reqBody, closeCh, int(loc.Size), int(loc.BlobSize))
bufferPipe := s.readerPipeline(span, reqBody, closeCh, int(loc.Size_), int(loc.SliceSize))
defer func() {
close(closeCh)
// waiting pipeline close if has error
@ -677,17 +677,17 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L
currBlobIdx := 0
currBlobCount := uint32(0)
remainSize := loc.Size
remainSize := loc.Size_
restPartsLoc := loc
readSize := 0
for readSize < int(loc.Size) {
for readSize < int(loc.Size_) {
parts := make([]blobPart, 0, s.conf.PartConcurrence)
// waiting at least one blob
buf, ok := <-bufferPipe
if !ok && readSize < int(loc.Size) {
return acapi.Location{}, nil, errcode.ErrAccessReadRequestBody
if !ok && readSize < int(loc.Size_) {
return proto.Location{}, nil, errcode.ErrAccessReadRequestBody
}
readSize += len(buf)
parts = append(parts, blobPart{size: len(buf), buf: buf})
@ -697,9 +697,9 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L
select {
case buf, ok := <-bufferPipe:
if !ok {
if readSize < int(loc.Size) {
if readSize < int(loc.Size_) {
releaseBuffer(parts)
return acapi.Location{}, nil, errcode.ErrAccessReadRequestBody
return proto.Location{}, nil, errcode.ErrAccessReadRequestBody
}
more = false
} else {
@ -713,9 +713,9 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L
tryTimes := s.conf.MaxRetry
for {
if len(loc.Blobs) > acapi.MaxLocationBlobs {
if len(loc.Slices) > acapi.MaxLocationBlobs {
releaseBuffer(parts)
return acapi.Location{}, nil, errcode.ErrUnexpected
return proto.Location{}, nil, errcode.ErrUnexpected
}
// feed new params
@ -723,16 +723,16 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L
currCount := currBlobCount
for i := range parts {
token := tokens[currIdx]
if restPartsLoc.Size > uint64(loc.BlobSize) && parts[i].size < int(loc.BlobSize) {
if restPartsLoc.Size_ > uint64(loc.SliceSize) && parts[i].size < int(loc.SliceSize) {
token = tokens[currIdx+1]
}
parts[i].token = token
parts[i].cid = loc.ClusterID
parts[i].vid = loc.Blobs[currIdx].Vid
parts[i].bid = loc.Blobs[currIdx].MinBid + proto.BlobID(currCount)
parts[i].vid = loc.Slices[currIdx].Vid
parts[i].bid = loc.Slices[currIdx].MinSliceID + proto.BlobID(currCount)
currCount++
if loc.Blobs[currIdx].Count == currCount {
if loc.Slices[currIdx].Count == currCount {
currIdx++
currCount = 0
}
@ -744,7 +744,7 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L
remainSize -= uint64(part.size)
currBlobCount++
// next blobs
if loc.Blobs[currBlobIdx].Count == currBlobCount {
if loc.Slices[currBlobIdx].Count == currBlobCount {
currBlobIdx++
currBlobCount = 0
}
@ -758,7 +758,7 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L
if tryTimes == 1 {
releaseBuffer(parts)
span.Error("exceed the max retry limit", s.conf.MaxRetry)
return acapi.Location{}, nil, errcode.ErrUnexpected
return proto.Location{}, nil, errcode.ErrUnexpected
}
tryTimes--
}
@ -768,15 +768,15 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L
err = retry.Timed(s.conf.MaxRetry, s.conf.RetryDelayMs).RuptOn(func() (bool, error) {
resp, err1 := s.Alloc(ctx, &acapi.AllocArgs{
Size: remainSize,
BlobSize: loc.BlobSize,
BlobSize: loc.SliceSize,
AssignClusterID: loc.ClusterID,
CodeMode: loc.CodeMode,
})
if err1 != nil {
return true, err1
}
if len(resp.Location.Blobs) > 0 {
if newVid := resp.Location.Blobs[0].Vid; newVid == loc.Blobs[currBlobIdx].Vid {
if len(resp.Location.Slices) > 0 {
if newVid := resp.Location.Slices[0].Vid; newVid == loc.Slices[currBlobIdx].Vid {
return false, fmt.Errorf("alloc the same vid %d", newVid)
}
}
@ -786,17 +786,17 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L
if err != nil {
releaseBuffer(parts)
span.Error("alloc another parts to put", err)
return acapi.Location{}, nil, errcode.ErrUnexpected
return proto.Location{}, nil, errcode.ErrUnexpected
}
restPartsLoc = restPartsResp.Location
signArgs.Locations = append(signArgs.Locations, restPartsLoc.Copy())
if currBlobCount > 0 {
loc.Blobs[currBlobIdx].Count = currBlobCount
loc.Slices[currBlobIdx].Count = currBlobCount
currBlobIdx++
}
loc.Blobs = append(loc.Blobs[:currBlobIdx], restPartsLoc.Blobs...)
loc.Slices = append(loc.Slices[:currBlobIdx], restPartsLoc.Slices...)
tokens = append(tokens[:currBlobIdx], restPartsResp.Tokens...)
currBlobCount = 0
@ -811,7 +811,7 @@ func (s *sdkHandler) putParts(ctx context.Context, args *acapi.PutArgs) (acapi.L
signResp, err1 := s.sign(ctx, &signArgs)
if err1 != nil {
span.Error("sign location with crc", err1)
return acapi.Location{}, nil, errcode.ErrUnexpected
return proto.Location{}, nil, errcode.ErrUnexpected
}
loc = signResp.Location
}

77
blobstore/sdk/sdk_client_test.go
View File

@ -14,6 +14,7 @@ import (
acapi "github.com/cubefs/cubefs/blobstore/api/access"
"github.com/cubefs/cubefs/blobstore/common/codemode"
errcode "github.com/cubefs/cubefs/blobstore/common/errors"
"github.com/cubefs/cubefs/blobstore/common/proto"
"github.com/cubefs/cubefs/blobstore/testing/mocks"
"github.com/cubefs/cubefs/blobstore/util/closer"
"github.com/cubefs/cubefs/blobstore/util/log"
@ -62,12 +63,12 @@ func TestSdkHandler_Delete(t *testing.T) {
require.NotNil(t, err)
require.ErrorIs(t, err, errcode.ErrIllegalArguments)
args := &acapi.DeleteArgs{Locations: []acapi.Location{{}}}
args := &acapi.DeleteArgs{Locations: []proto.Location{{}}}
ret, err := hd.Delete(ctx, args)
require.NoError(t, err)
require.Nil(t, ret)
args.Locations[0].Size = 1
args.Locations[0].Size_ = 1
_, err = hd.Delete(ctx, args)
require.NotNil(t, err)
require.ErrorIs(t, err, errcode.ErrIllegalArguments)
@ -76,7 +77,7 @@ func TestSdkHandler_Delete(t *testing.T) {
hd.handler.(*mocks.MockStreamHandler).EXPECT().Delete(any, any).Times(3).Return(errMock)
crc, _ := stream.LocationCrcCalculate(&args.Locations[0])
args.Locations[0].Crc = crc
args.Locations[0].Blobs = make([]acapi.SliceInfo, 0)
args.Locations[0].Slices = make([]proto.Slice, 0)
ret, err = hd.Delete(ctx, args)
require.NotNil(t, err)
require.ErrorIs(t, err, errcode.ErrUnexpected)
@ -88,14 +89,14 @@ func TestSdkHandler_Delete(t *testing.T) {
require.Nil(t, ret)
// 3 location
loc := acapi.Location{
loc := proto.Location{
ClusterID: 1,
Size: 1,
Blobs: []acapi.SliceInfo{{Vid: 9}},
Size_: 1,
Slices: []proto.Slice{{Vid: 9}},
}
crc, _ = stream.LocationCrcCalculate(&loc)
loc.Crc = crc
args.Locations = make([]acapi.Location, 0)
args.Locations = make([]proto.Location, 0)
for len(args.Locations) < 3 {
args.Locations = append(args.Locations, loc)
}
@ -131,8 +132,8 @@ func TestSdkHandler_Get(t *testing.T) {
// args error
args := &acapi.GetArgs{
ReadSize: 1,
Location: acapi.Location{
Size: 2,
Location: proto.Location{
Size_: 2,
},
}
_, err = hd.Get(ctx, args)
@ -167,12 +168,12 @@ func TestSdkHandler_Get(t *testing.T) {
// ok
data := "test read"
args.ReadSize = uint64(len(data))
args.Location.Size = args.ReadSize
args.Location.Size_ = args.ReadSize
crc, _ = stream.LocationCrcCalculate(&args.Location)
args.Location.Crc = crc
rd, wr := io.Pipe()
hd.handler.(*mocks.MockStreamHandler).EXPECT().Get(any, any, any, any, any).DoAndReturn(
func(ctx context.Context, w io.Writer, location acapi.Location, readSize, offset uint64) (func() error, error) {
func(ctx context.Context, w io.Writer, location proto.Location, readSize, offset uint64) (func() error, error) {
w = wr
if readSize > 0 && readSize < 1024 {
return func() error {
@ -193,13 +194,13 @@ func TestSdkHandler_Get(t *testing.T) {
// ok, zero copy
data = "test read"
args.ReadSize = uint64(len(data))
args.Location.Size = args.ReadSize
args.Location.Size_ = args.ReadSize
crc, _ = stream.LocationCrcCalculate(&args.Location)
args.Location.Crc = crc
buff := bytes.NewBuffer([]byte{})
args.Writer = buff
hd.handler.(*mocks.MockStreamHandler).EXPECT().Get(any, args.Writer, any, any, any).DoAndReturn(
func(ctx context.Context, w io.Writer, location acapi.Location, readSize, offset uint64) (func() error, error) {
func(ctx context.Context, w io.Writer, location proto.Location, readSize, offset uint64) (func() error, error) {
if readSize > 0 && readSize < 1024 {
return func() error {
_, err1 := w.Write([]byte(data))
@ -230,7 +231,7 @@ func TestSdkHandler_Put(t *testing.T) {
// size 0
loc, hash, err := hd.Put(ctx, &acapi.PutArgs{Hashes: 1})
require.NoError(t, err)
require.Equal(t, uint64(0), loc.Size)
require.Equal(t, uint64(0), loc.Size_)
require.Equal(t, 1, len(hash))
args := &acapi.PutArgs{Size: 2}
@ -239,16 +240,16 @@ func TestSdkHandler_Put(t *testing.T) {
hd.handler.(*mocks.MockStreamHandler).EXPECT().Put(any, any, any, any).Return(nil, errMock)
loc, hash, err = hd.Put(ctx, args)
require.NotNil(t, err)
require.Equal(t, uint64(0), loc.Size)
require.Equal(t, uint64(0), loc.Size_)
require.Equal(t, 0, len(hash))
// ok
args.Hashes = 1
mockLoc := acapi.Location{Size: 2}
mockLoc := proto.Location{Size_: 2}
hd.handler.(*mocks.MockStreamHandler).EXPECT().Put(any, any, any, any).Return(&mockLoc, nil)
loc, hash, err = hd.Put(ctx, args)
require.NoError(t, err)
require.Equal(t, mockLoc.Size, loc.Size)
require.Equal(t, mockLoc.Size_, loc.Size_)
require.Equal(t, int(args.Hashes), len(hash))
// handler put error, retry 3 times
@ -258,7 +259,7 @@ func TestSdkHandler_Put(t *testing.T) {
hd.handler.(*mocks.MockStreamHandler).EXPECT().Put(any, any, any, any).Return(nil, errMock).Times(3)
loc, hash, err = hd.Put(ctx, args)
require.NotNil(t, err)
require.Equal(t, uint64(0), loc.Size)
require.Equal(t, uint64(0), loc.Size_)
require.Equal(t, 0, len(hash))
// retry ok
@ -271,7 +272,7 @@ func TestSdkHandler_Put(t *testing.T) {
hd.handler.(*mocks.MockStreamHandler).EXPECT().Put(any, any, any, any).Return(&mockLoc, nil)
loc, hash, err = hd.Put(ctx, args)
require.NoError(t, err)
require.Equal(t, mockLoc.Size, loc.Size)
require.Equal(t, mockLoc.Size_, loc.Size_)
require.Equal(t, int(args.Hashes), len(hash))
}
@ -296,12 +297,12 @@ func TestSdkHandler_Alloc(t *testing.T) {
hd.handler.(*mocks.MockStreamHandler).EXPECT().Alloc(any, any, any, any, any).Return(nil, errMock)
ret, err := hd.Alloc(ctx, args)
require.NotNil(t, err)
require.Equal(t, acapi.Location{}, ret.Location)
require.Equal(t, proto.Location{}, ret.Location)
loca := &acapi.Location{
loca := &proto.Location{
ClusterID: 1,
Size: 2,
BlobSize: 1,
Size_: 2,
SliceSize: 1,
}
crc, _ := stream.LocationCrcCalculate(loca)
loca.Crc = crc
@ -326,21 +327,21 @@ func TestSdkHandler_putParts(t *testing.T) {
loc, hash, err := hd.Put(ctx, args)
require.NotNil(t, err)
require.ErrorIs(t, err, errMock)
require.NotEqual(t, args.Size, int64(loc.Size))
require.NotEqual(t, args.Size, int64(loc.Size_))
require.Nil(t, hash)
// all ok
data := "test alloc put"
args.Body = bytes.NewBuffer([]byte(data))
args.Size = int64(len(data))
loca := &acapi.Location{
loca := &proto.Location{
ClusterID: 1,
Size: uint64(args.Size),
BlobSize: 4,
Blobs: []acapi.SliceInfo{{
MinBid: 1001,
Vid: 10,
Count: 4,
Size_: uint64(args.Size),
SliceSize: 4,
Slices: []proto.Slice{{
MinSliceID: 1001,
Vid: 10,
Count: 4,
}},
}
crc, _ := stream.LocationCrcCalculate(loca)
@ -374,14 +375,14 @@ func TestSdkHandler_putParts(t *testing.T) {
loca.CodeMode = codemode.EC3P3
args.Body = bytes.NewBuffer([]byte(data))
hd.handler.(*mocks.MockStreamHandler).EXPECT().Alloc(any, any, any, any, any).Return(loca, nil)
locb := &acapi.Location{
locb := &proto.Location{
ClusterID: 1,
Size: uint64(args.Size),
BlobSize: 4,
Blobs: []acapi.SliceInfo{{
MinBid: 1001,
Vid: 11, // loca + 1
Count: 4,
Size_: uint64(args.Size),
SliceSize: 4,
Slices: []proto.Slice{{
MinSliceID: 1001,
Vid: 11, // loca + 1
Count: 4,
}},
}
stream.LocationCrcFill(locb)

14
blobstore/shardnode/svr.go
View File

@ -17,22 +17,20 @@ package shardnode
import (
"context"
"golang.org/x/sync/singleflight"
"sync"
"github.com/cubefs/cubefs/blobstore/util/taskpool"
"github.com/cubefs/cubefs/blobstore/shardnode/base"
apierr "github.com/cubefs/cubefs/blobstore/common/errors"
"github.com/cubefs/cubefs/blobstore/shardnode/catalog"
"github.com/cubefs/cubefs/blobstore/util/closer"
"golang.org/x/sync/singleflight"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
apierr "github.com/cubefs/cubefs/blobstore/common/errors"
"github.com/cubefs/cubefs/blobstore/common/proto"
"github.com/cubefs/cubefs/blobstore/common/raft"
"github.com/cubefs/cubefs/blobstore/shardnode/base"
"github.com/cubefs/cubefs/blobstore/shardnode/catalog"
"github.com/cubefs/cubefs/blobstore/shardnode/storage"
"github.com/cubefs/cubefs/blobstore/shardnode/storage/store"
"github.com/cubefs/cubefs/blobstore/util/closer"
"github.com/cubefs/cubefs/blobstore/util/taskpool"
)
type Config struct {

7
blobstore/testing/dial/connect.go
View File

@ -21,6 +21,7 @@ import (
"math/rand"
"github.com/cubefs/cubefs/blobstore/api/access"
"github.com/cubefs/cubefs/blobstore/common/proto"
"github.com/cubefs/cubefs/blobstore/common/trace"
"github.com/cubefs/cubefs/blobstore/util/retry"
"github.com/cubefs/cubefs/blobstore/util/task"
@ -80,7 +81,7 @@ func connect(conn Connection) {
span, ctx = trace.StartSpanFromContextWithTraceID(
context.Background(), "dial", "dial-"+hostname+"-"+trace.RandomID().String())
body io.ReadCloser
location access.Location
location proto.Location
err error
)
defer func() {
@ -89,7 +90,7 @@ func connect(conn Connection) {
}
if err = retry.Timed(10, 1).On(func() error {
_, e := conn.api.Delete(ctx, &access.DeleteArgs{
Locations: []access.Location{location},
Locations: []proto.Location{location},
})
return e
}); err != nil {
@ -145,7 +146,7 @@ func connect(conn Connection) {
span.Infof("to delete conn:%v", conn)
runTimer(conn, "del", 0, func() error {
_, err = conn.api.Delete(ctx, &access.DeleteArgs{
Locations: []access.Location{location},
Locations: []proto.Location{location},
})
return err
})

12
blobstore/testing/mocks/access_stream.go
View File

@ -53,10 +53,10 @@ func (mr *MockStreamHandlerMockRecorder) Admin() *gomock.Call {
}
// Alloc mocks base method.
func (m *MockStreamHandler) Alloc(arg0 context.Context, arg1 uint64, arg2 uint32, arg3 proto.ClusterID, arg4 codemode.CodeMode) (*access.Location, error) {
func (m *MockStreamHandler) Alloc(arg0 context.Context, arg1 uint64, arg2 uint32, arg3 proto.ClusterID, arg4 codemode.CodeMode) (*proto.Location, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Alloc", arg0, arg1, arg2, arg3, arg4)
ret0, _ := ret[0].(*access.Location)
ret0, _ := ret[0].(*proto.Location)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@ -68,7 +68,7 @@ func (mr *MockStreamHandlerMockRecorder) Alloc(arg0, arg1, arg2, arg3, arg4 inte
}
// Delete mocks base method.
func (m *MockStreamHandler) Delete(arg0 context.Context, arg1 *access.Location) error {
func (m *MockStreamHandler) Delete(arg0 context.Context, arg1 *proto.Location) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Delete", arg0, arg1)
ret0, _ := ret[0].(error)
@ -82,7 +82,7 @@ func (mr *MockStreamHandlerMockRecorder) Delete(arg0, arg1 interface{}) *gomock.
}
// Get mocks base method.
func (m *MockStreamHandler) Get(arg0 context.Context, arg1 io.Writer, arg2 access.Location, arg3, arg4 uint64) (func() error, error) {
func (m *MockStreamHandler) Get(arg0 context.Context, arg1 io.Writer, arg2 proto.Location, arg3, arg4 uint64) (func() error, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2, arg3, arg4)
ret0, _ := ret[0].(func() error)
@ -97,10 +97,10 @@ func (mr *MockStreamHandlerMockRecorder) Get(arg0, arg1, arg2, arg3, arg4 interf
}
// Put mocks base method.
func (m *MockStreamHandler) Put(arg0 context.Context, arg1 io.Reader, arg2 int64, arg3 access.HasherMap) (*access.Location, error) {
func (m *MockStreamHandler) Put(arg0 context.Context, arg1 io.Reader, arg2 int64, arg3 access.HasherMap) (*proto.Location, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Put", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(*access.Location)
ret0, _ := ret[0].(*proto.Location)
ret1, _ := ret[1].(error)
return ret0, ret1
}

9
blobstore/testing/mocks/api_access.go
View File

@ -10,6 +10,7 @@ import (
reflect "reflect"
access "github.com/cubefs/cubefs/blobstore/api/access"
proto "github.com/cubefs/cubefs/blobstore/common/proto"
gomock "github.com/golang/mock/gomock"
)
@ -37,10 +38,10 @@ func (m *MockAccessAPI) EXPECT() *MockAccessAPIMockRecorder {
}
// Delete mocks base method.
func (m *MockAccessAPI) Delete(arg0 context.Context, arg1 *access.DeleteArgs) ([]access.Location, error) {
func (m *MockAccessAPI) Delete(arg0 context.Context, arg1 *access.DeleteArgs) ([]proto.Location, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Delete", arg0, arg1)
ret0, _ := ret[0].([]access.Location)
ret0, _ := ret[0].([]proto.Location)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@ -67,10 +68,10 @@ func (mr *MockAccessAPIMockRecorder) Get(arg0, arg1 interface{}) *gomock.Call {
}
// Put mocks base method.
func (m *MockAccessAPI) Put(arg0 context.Context, arg1 *access.PutArgs) (access.Location, access.HashSumMap, error) {
func (m *MockAccessAPI) Put(arg0 context.Context, arg1 *access.PutArgs) (proto.Location, access.HashSumMap, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Put", arg0, arg1)
ret0, _ := ret[0].(access.Location)
ret0, _ := ret[0].(proto.Location)
ret1, _ := ret[1].(access.HashSumMap)
ret2, _ := ret[2].(error)
return ret0, ret1, ret2

70
sdk/data/blobstore/blobstore_client.go
View File

@ -64,22 +64,22 @@ func (ebs *BlobStoreClient) Read(ctx context.Context, volName string, buf []byte
metric.SetWithLabels(err, map[string]string{exporter.Vol: volName})
}()
blobs := oek.Blobs
sliceInfos := make([]access.SliceInfo, 0)
sliceInfos := make([]ebsproto.Slice, 0)
for _, b := range blobs {
sliceInfo := access.SliceInfo{
MinBid: ebsproto.BlobID(b.MinBid),
Vid: ebsproto.Vid(b.Vid),
Count: uint32(b.Count),
sliceInfo := ebsproto.Slice{
MinSliceID: ebsproto.BlobID(b.MinBid),
Vid: ebsproto.Vid(b.Vid),
Count: uint32(b.Count),
}
sliceInfos = append(sliceInfos, sliceInfo)
}
loc := access.Location{
loc := ebsproto.Location{
ClusterID: ebsproto.ClusterID(oek.Cid),
Size: oek.Size,
Size_: oek.Size,
Crc: oek.Crc,
CodeMode: codemode.CodeMode(oek.CodeMode),
BlobSize: oek.BlobSize,
Blobs: sliceInfos,
SliceSize: oek.BlobSize,
Slices: sliceInfos,
}
// func get has retry
log.LogDebugf("TRACE Ebs Read,oek(%v) loc(%v)", oek, loc)
@ -112,7 +112,7 @@ func (ebs *BlobStoreClient) Read(ctx context.Context, volName string, buf []byte
return readN, nil
}
func (ebs *BlobStoreClient) Write(ctx context.Context, volName string, data []byte, size uint32) (location access.Location, err error) {
func (ebs *BlobStoreClient) Write(ctx context.Context, volName string, data []byte, size uint32) (location ebsproto.Location, err error) {
bgTime := stat.BeginStat()
defer func() {
stat.EndStat("ebs-write", err, bgTime, 1)
@ -156,26 +156,26 @@ func (ebs *BlobStoreClient) Delete(oeks []proto.ObjExtentKey) (err error) {
ctx, cancel := context.WithTimeout(context.TODO(), time.Second*3)
defer cancel()
locs := make([]access.Location, 0)
locs := make([]ebsproto.Location, 0)
for _, oek := range oeks {
sliceInfos := make([]access.SliceInfo, 0)
sliceInfos := make([]ebsproto.Slice, 0)
for _, b := range oek.Blobs {
sliceInfo := access.SliceInfo{
MinBid: ebsproto.BlobID(b.MinBid),
Vid: ebsproto.Vid(b.Vid),
Count: uint32(b.Count),
sliceInfo := ebsproto.Slice{
MinSliceID: ebsproto.BlobID(b.MinBid),
Vid: ebsproto.Vid(b.Vid),
Count: uint32(b.Count),
}
sliceInfos = append(sliceInfos, sliceInfo)
}
loc := access.Location{
loc := ebsproto.Location{
ClusterID: ebsproto.ClusterID(oek.Cid),
Size: oek.Size,
Size_: oek.Size,
Crc: oek.Crc,
CodeMode: codemode.CodeMode(oek.CodeMode),
BlobSize: oek.BlobSize,
Blobs: sliceInfos,
SliceSize: oek.BlobSize,
Slices: sliceInfos,
}
locs = append(locs, loc)
}
@ -261,7 +261,7 @@ func (ebs *BlobStoreClient) Put(ctx context.Context, volName string, f io.Reader
putSize = rest
}
rest -= putSize
var location access.Location
var location ebsproto.Location
var hash access.HashSumMap
location, hash, err = ebs.client.Put(ctx, &access.PutArgs{
Size: int64(putSize),
@ -290,11 +290,11 @@ func (ebs *BlobStoreClient) Put(ctx context.Context, volName string, f io.Reader
return
}
func locationToObjExtentKey(location access.Location, from uint64) (oek proto.ObjExtentKey) {
func locationToObjExtentKey(location ebsproto.Location, from uint64) (oek proto.ObjExtentKey) {
blobs := make([]proto.Blob, 0)
for _, info := range location.Blobs {
for _, info := range location.Slices {
blob := proto.Blob{
MinBid: uint64(info.MinBid),
MinBid: uint64(info.MinSliceID),
Count: uint64(info.Count),
Vid: uint64(info.Vid),
}
@ -303,8 +303,8 @@ func locationToObjExtentKey(location access.Location, from uint64) (oek proto.Ob
oek = proto.ObjExtentKey{
Cid: uint64(location.ClusterID),
CodeMode: uint8(location.CodeMode),
Size: location.Size,
BlobSize: location.BlobSize,
Size: location.Size_,
BlobSize: location.SliceSize,
Blobs: blobs,
BlobsLen: uint32(len(blobs)),
FileOffset: from,
@ -329,22 +329,22 @@ func (ebs *BlobStoreClient) Get(ctx context.Context, volName string, offset uint
metric.SetWithLabels(err, map[string]string{exporter.Vol: volName})
}()
blobs := oek.Blobs
sliceInfos := make([]access.SliceInfo, 0)
sliceInfos := make([]ebsproto.Slice, 0)
for _, b := range blobs {
sliceInfo := access.SliceInfo{
MinBid: ebsproto.BlobID(b.MinBid),
Vid: ebsproto.Vid(b.Vid),
Count: uint32(b.Count),
sliceInfo := ebsproto.Slice{
MinSliceID: ebsproto.BlobID(b.MinBid),
Vid: ebsproto.Vid(b.Vid),
Count: uint32(b.Count),
}
sliceInfos = append(sliceInfos, sliceInfo)
}
loc := access.Location{
loc := ebsproto.Location{
ClusterID: ebsproto.ClusterID(oek.Cid),
Size: oek.Size,
Size_: oek.Size,
Crc: oek.Crc,
CodeMode: codemode.CodeMode(oek.CodeMode),
BlobSize: oek.BlobSize,
Blobs: sliceInfos,
SliceSize: oek.BlobSize,
Slices: sliceInfos,
}
// func get has retry
log.LogDebugf("TRACE Ebs Read, oek(%v) loc(%v)", oek, loc)

17
sdk/data/blobstore/blobstore_client_test.go
View File

@ -27,6 +27,7 @@ import (
"github.com/cubefs/cubefs/blobstore/api/access"
"github.com/cubefs/cubefs/blobstore/common/crc32block"
"github.com/cubefs/cubefs/blobstore/common/proto"
"github.com/cubefs/cubefs/blobstore/util/bytespool"
cproto "github.com/cubefs/cubefs/proto"
"github.com/stretchr/testify/require"
@ -69,7 +70,7 @@ func NewMockEbsService() *MockEbsService {
hashSumMap[alg] = hasher.Sum(nil)
}
loc := access.Location{Size: uint64(dataSize)}
loc := proto.Location{Size_: uint64(dataSize)}
fillCrc(&loc)
resp := access.PutResp{
Location: loc,
@ -129,7 +130,7 @@ func requestBody(req *http.Request, val interface{}) {
json.Unmarshal(data, val)
}
func calcCrc(loc *access.Location) (uint32, error) {
func calcCrc(loc *proto.Location) (uint32, error) {
crcWriter := crc32.New(crc32.IEEETable)
buf := bytespool.Alloc(1024)
@ -147,7 +148,7 @@ func calcCrc(loc *access.Location) (uint32, error) {
return crcWriter.Sum32(), nil
}
func fillCrc(loc *access.Location) error {
func fillCrc(loc *proto.Location) error {
crc, err := calcCrc(loc)
if err != nil {
return err
@ -156,7 +157,7 @@ func fillCrc(loc *access.Location) error {
return nil
}
func verifyCrc(loc *access.Location) bool {
func verifyCrc(loc *proto.Location) bool {
crc, err := calcCrc(loc)
if err != nil {
return false
@ -192,9 +193,9 @@ func TestEbsClient_Write_Read(t *testing.T) {
// read prepare
blobs := make([]cproto.Blob, 0)
for _, info := range location.Blobs {
for _, info := range location.Slices {
blob := cproto.Blob{
MinBid: uint64(info.MinBid),
MinBid: uint64(info.MinSliceID),
Count: uint64(info.Count),
Vid: uint64(info.Vid),
}
@ -203,8 +204,8 @@ func TestEbsClient_Write_Read(t *testing.T) {
oek := cproto.ObjExtentKey{
Cid: uint64(location.ClusterID),
CodeMode: uint8(location.CodeMode),
Size: location.Size,
BlobSize: location.BlobSize,
Size: location.Size_,
BlobSize: location.SliceSize,
Blobs: blobs,
BlobsLen: uint32(len(blobs)),
Crc: location.Crc,

8
sdk/data/blobstore/writer.go
View File

@ -474,9 +474,9 @@ func (writer *Writer) writeSlice(ctx context.Context, wSlice *rwSlice, wg bool)
}
log.LogDebugf("TRACE blobStore,location(%v)", location)
blobs := make([]proto.Blob, 0)
for _, info := range location.Blobs {
for _, info := range location.Slices {
blob := proto.Blob{
MinBid: uint64(info.MinBid),
MinBid: uint64(info.MinSliceID),
Count: uint64(info.Count),
Vid: uint64(info.Vid),
}
@ -485,8 +485,8 @@ func (writer *Writer) writeSlice(ctx context.Context, wSlice *rwSlice, wg bool)
wSlice.objExtentKey = proto.ObjExtentKey{
Cid: uint64(location.ClusterID),
CodeMode: uint8(location.CodeMode),
Size: location.Size,
BlobSize: location.BlobSize,
Size: location.Size_,
BlobSize: location.SliceSize,
Blobs: blobs,
BlobsLen: uint32(len(blobs)),
FileOffset: wSlice.fileOffset,

24
sdk/data/blobstore/writer_test.go
View File

@ -21,6 +21,8 @@ import (
"syscall"
"testing"
proto2 "github.com/cubefs/cubefs/blobstore/common/proto"
"github.com/brahma-adshonor/gohook"
"github.com/stretchr/testify/assert"
@ -360,7 +362,7 @@ func TestBufferWrite(t *testing.T) {
func TestWriteSlice(t *testing.T) {
testCase := []struct {
wg bool
ebsWriteFunc func(*BlobStoreClient, context.Context, string, []byte, uint32) (access.Location, error)
ebsWriteFunc func(*BlobStoreClient, context.Context, string, []byte, uint32) (proto2.Location, error)
expectError error
}{
{false, MockEbscWriteTrue, nil},
@ -401,26 +403,26 @@ func TestWriteSlice(t *testing.T) {
}
}
func MockEbscWriteTrue(ebs *BlobStoreClient, ctx context.Context, volName string, data []byte, l uint32) (location access.Location, err error) {
loc := access.Location{
func MockEbscWriteTrue(ebs *BlobStoreClient, ctx context.Context, volName string, data []byte, l uint32) (location proto2.Location, err error) {
loc := proto2.Location{
ClusterID: 1,
CodeMode: 0,
Size: 100,
BlobSize: 100,
Size_: 100,
SliceSize: 100,
Crc: 1024,
Blobs: []access.SliceInfo{
Slices: []proto2.Slice{
{
MinBid: 1,
Vid: 1,
Count: 1,
MinSliceID: 1,
Vid: 1,
Count: 1,
},
},
}
return loc, nil
}
func MockEbscWriteFalse(ebs *BlobStoreClient, ctx context.Context, volName string, data []byte, l uint32) (location access.Location, err error) {
return access.Location{}, syscall.EIO
func MockEbscWriteFalse(ebs *BlobStoreClient, ctx context.Context, volName string, data []byte, l uint32) (location proto2.Location, err error) {
return proto2.Location{}, syscall.EIO
}
func MockAppendObjExtentKeysTrue(mw *meta.MetaWrapper, inode uint64, eks []proto.ObjExtentKey) error {