gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 02:00:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix(clustermgr): basic clustermgr dev-branch for shardnode management

Browse Source 
1. fix refactor code and test problem
2. add clustermgr api about shardnode and space

. #22360410

Signed-off-by: tangdeyi <tangdeyi@oppo.com>
This commit is contained in:
tangdeyi 2024-07-10 20:03:53 +08:00 committed by slasher
parent 792f3654eb
commit b7cade30e6
120 changed files with 2375 additions and 1221 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
blobstore/access/controller/mock_test.go
View File

@ -22,7 +22,6 @@ import (
"github.com/golang/mock/gomock"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/api/proxy"
"github.com/cubefs/cubefs/blobstore/common/codemode"
@ -51,7 +50,7 @@ var (
dataCalled map[proto.Vid]int
dataNodes map[string]cmapi.ServiceInfo
dataVolumes map[proto.Vid]cmapi.VolumeInfo
dataDisks map[proto.DiskID]bnapi.DiskInfo
dataDisks map[proto.DiskID]cmapi.BlobNodeDiskInfo
)
func init() {
@ -84,20 +83,24 @@ func init() {
},
}
dataDisks = make(map[proto.DiskID]bnapi.DiskInfo)
dataDisks[10001] = bnapi.DiskInfo{
ClusterID: 1,
Idc: idc,
Host: "blobnode-1",
DiskHeartBeatInfo: bnapi.DiskHeartBeatInfo{
dataDisks = make(map[proto.DiskID]cmapi.BlobNodeDiskInfo)
dataDisks[10001] = cmapi.BlobNodeDiskInfo{
DiskInfo: cmapi.DiskInfo{
ClusterID: 1,
Idc: idc,
Host: "blobnode-1",
},
DiskHeartBeatInfo: cmapi.DiskHeartBeatInfo{
DiskID: 10001,
},
}
dataDisks[10002] = bnapi.DiskInfo{
ClusterID: 1,
Idc: idc,
Host: "blobnode-2",
DiskHeartBeatInfo: bnapi.DiskHeartBeatInfo{
dataDisks[10002] = cmapi.BlobNodeDiskInfo{
DiskInfo: cmapi.DiskInfo{
ClusterID: 1,
Idc: idc,
Host: "blobnode-2",
},
DiskHeartBeatInfo: cmapi.DiskHeartBeatInfo{
DiskID: 10002,
},
}
@ -131,7 +134,7 @@ func init() {
return nil, errNotFound
})
pcli.EXPECT().GetCacheDisk(A, A, A).AnyTimes().DoAndReturn(
func(_ context.Context, _ string, args *proxy.CacheDiskArgs) (*bnapi.DiskInfo, error) {
func(_ context.Context, _ string, args *proxy.CacheDiskArgs) (*cmapi.BlobNodeDiskInfo, error) {
if val, ok := dataDisks[args.DiskID]; ok {
return &val, nil
}

9
blobstore/access/controller/service_test.go
View File

@ -23,7 +23,6 @@ import (
"github.com/stretchr/testify/require"
"github.com/cubefs/cubefs/blobstore/access/controller"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/api/proxy"
"github.com/cubefs/cubefs/blobstore/common/proto"
@ -206,9 +205,9 @@ func TestAccessServiceGetDiskHost(t *testing.T) {
func TestAccessServiceGetBrokenDiskHost(t *testing.T) {
brokenRet := cmapi.ListDiskRet{}
brokenRet.Disks = make([]*bnapi.DiskInfo, 2)
brokenRet.Disks[0] = &bnapi.DiskInfo{}
brokenRet.Disks[1] = &bnapi.DiskInfo{}
brokenRet.Disks = make([]*cmapi.BlobNodeDiskInfo, 2)
brokenRet.Disks[0] = &cmapi.BlobNodeDiskInfo{}
brokenRet.Disks[1] = &cmapi.BlobNodeDiskInfo{}
cli := mocks.NewMockClientAPI(C(t))
cli.EXPECT().GetService(A, A).Times(5).DoAndReturn(
@ -222,7 +221,7 @@ func TestAccessServiceGetBrokenDiskHost(t *testing.T) {
pcli := mocks.NewMockProxyClient(C(t))
pcli.EXPECT().GetCacheDisk(A, A, A).AnyTimes().DoAndReturn(
func(_ context.Context, _ string, args *proxy.CacheDiskArgs) (*bnapi.DiskInfo, error) {
func(_ context.Context, _ string, args *proxy.CacheDiskArgs) (*cmapi.BlobNodeDiskInfo, error) {
if val, ok := dataDisks[args.DiskID]; ok {
return &val, nil
}

14
blobstore/access/stream/stream_mock_test.go
View File

@ -83,7 +83,7 @@ var (
dataVolume *proxy.VersionVolume
dataAllocs []proxy.AllocRet
dataNodes map[string]clustermgr.ServiceInfo
dataDisks map[proto.DiskID]clustermgr.DiskInfo
dataDisks map[proto.DiskID]clustermgr.BlobNodeDiskInfo
dataShards *shardsData
vuidController *vuidControl
@ -359,16 +359,16 @@ func initMockData() {
Nodes: proxyNodes,
}
dataDisks = make(map[proto.DiskID]clustermgr.DiskInfo)
dataDisks = make(map[proto.DiskID]clustermgr.BlobNodeDiskInfo)
for _, id := range idcID {
dataDisks[proto.DiskID(id)] = clustermgr.DiskInfo{
ClusterID: clusterID, Idc: idc, Host: strconv.Itoa(id),
dataDisks[proto.DiskID(id)] = clustermgr.BlobNodeDiskInfo{
DiskInfo: clustermgr.DiskInfo{ClusterID: clusterID, Idc: idc, Host: strconv.Itoa(id)},
DiskHeartBeatInfo: clustermgr.DiskHeartBeatInfo{DiskID: proto.DiskID(id)},
}
}
for _, id := range idcOtherID {
dataDisks[proto.DiskID(id)] = clustermgr.DiskInfo{
ClusterID: clusterID, Idc: idcOther, Host: strconv.Itoa(id),
dataDisks[proto.DiskID(id)] = clustermgr.BlobNodeDiskInfo{
DiskInfo: clustermgr.DiskInfo{ClusterID: clusterID, Idc: idcOther, Host: strconv.Itoa(id)},
DiskHeartBeatInfo: clustermgr.DiskHeartBeatInfo{DiskID: proto.DiskID(id)},
}
}
@ -400,7 +400,7 @@ func initMockData() {
proxycli.EXPECT().GetCacheVolume(gomock.Any(), gomock.Any(), gomock.Any()).
AnyTimes().Return(dataVolume, nil)
proxycli.EXPECT().GetCacheDisk(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
func(_ context.Context, _ string, args *proxy.CacheDiskArgs) (*clustermgr.DiskInfo, error) {
func(_ context.Context, _ string, args *proxy.CacheDiskArgs) (*clustermgr.BlobNodeDiskInfo, error) {
if val, ok := dataDisks[args.DiskID]; ok {
return &val, nil
}

166
blobstore/api/blobnode/chunk.go
View File

@ -16,172 +16,24 @@ package blobnode
import (
"context"
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"time"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
bloberr "github.com/cubefs/cubefs/blobstore/common/errors"
"github.com/cubefs/cubefs/blobstore/common/proto"
"github.com/cubefs/cubefs/blobstore/common/rpc"
)
const (
ChunkStatusDefault ChunkStatus = iota // 0
ChunkStatusNormal // 1
ChunkStatusReadOnly // 2
ChunkStatusRelease // 3
ChunkNumStatus // 4
)
const (
ReleaseForUser = "release for user"
ReleaseForCompact = "release for compact"
)
// Chunk ID
// vuid + timestamp
const (
chunkVuidLen = 8
chunkTimestmapLen = 8
ChunkIdLength = chunkVuidLen + chunkTimestmapLen
)
var InvalidChunkId ChunkId = [ChunkIdLength]byte{}
var (
_vuidHexLen = hex.EncodedLen(chunkVuidLen)
_timestampHexLen = hex.EncodedLen(chunkTimestmapLen)
// ${vuid_hex}-${tiemstamp_hex}
// |-- 8 Bytes --|-- 1 Bytes --|-- 8 Bytes --|
delimiter = []byte("-")
ChunkIdEncodeLen = _vuidHexLen + _timestampHexLen + len(delimiter)
)
type (
ChunkId [ChunkIdLength]byte
ChunkStatus uint8
)
func (c ChunkId) UnixTime() uint64 {
return binary.BigEndian.Uint64(c[chunkVuidLen:ChunkIdLength])
}
func (c ChunkId) VolumeUnitId() proto.Vuid {
return proto.Vuid(binary.BigEndian.Uint64(c[:chunkVuidLen]))
}
func (c *ChunkId) Marshal() ([]byte, error) {
buf := make([]byte, ChunkIdEncodeLen)
var i int
hex.Encode(buf[i:_vuidHexLen], c[:chunkVuidLen])
i += _vuidHexLen
copy(buf[i:i+len(delimiter)], delimiter)
i += len(delimiter)
hex.Encode(buf[i:], c[chunkVuidLen:ChunkIdLength])
return buf, nil
}
func (c *ChunkId) Unmarshal(data []byte) error {
if len(data) != ChunkIdEncodeLen {
panic(errors.New("chunk buf size not match"))
}
var i int
_, err := hex.Decode(c[:chunkVuidLen], data[i:_vuidHexLen])
if err != nil {
return err
}
i += _vuidHexLen
i += len(delimiter)
_, err = hex.Decode(c[chunkVuidLen:], data[i:])
if err != nil {
return err
}
return nil
}
func (c ChunkId) String() string {
buf, _ := c.Marshal()
return string(buf[:])
}
func (c ChunkId) MarshalJSON() ([]byte, error) {
b := make([]byte, ChunkIdEncodeLen+2)
b[0], b[ChunkIdEncodeLen+1] = '"', '"'
buf, _ := c.Marshal()
copy(b[1:], buf)
return b, nil
}
func (c *ChunkId) UnmarshalJSON(data []byte) (err error) {
if len(data) != ChunkIdEncodeLen+2 {
return errors.New("failed unmarshal json")
}
return c.Unmarshal(data[1 : ChunkIdEncodeLen+1])
}
func EncodeChunk(id ChunkId) string {
return id.String()
}
func NewChunkId(vuid proto.Vuid) (chunkId ChunkId) {
binary.BigEndian.PutUint64(chunkId[:chunkVuidLen], uint64(vuid))
binary.BigEndian.PutUint64(chunkId[chunkVuidLen:ChunkIdLength], uint64(time.Now().UnixNano()))
return
}
func IsValidDiskID(id proto.DiskID) bool {
return id != proto.InvalidDiskID
}
func IsValidChunkId(id ChunkId) bool {
return id != InvalidChunkId
func IsValidChunkID(id clustermgr.ChunkID) bool {
return id != clustermgr.InvalidChunkID
}
func IsValidChunkStatus(status ChunkStatus) bool {
return status < ChunkNumStatus
}
func DecodeChunk(name string) (id ChunkId, err error) {
buf := []byte(name)
if len(buf) != ChunkIdEncodeLen {
return InvalidChunkId, errors.New("invalid chunk name")
}
if err = id.Unmarshal(buf); err != nil {
return InvalidChunkId, errors.New("chunk unmarshal failed")
}
return
}
func (s *ChunkStatus) String() string {
switch *s {
case ChunkStatusDefault:
return "default"
case ChunkStatusNormal:
return "normal"
case ChunkStatusReadOnly:
return "readOnly"
case ChunkStatusRelease:
return "release"
default:
return "unkown"
}
func IsValidChunkStatus(status clustermgr.ChunkStatus) bool {
return status < clustermgr.ChunkNumStatus
}
type CreateChunkArgs struct {
@ -208,14 +60,14 @@ type StatChunkArgs struct {
Vuid proto.Vuid `json:"vuid"`
}
func (c *client) StatChunk(ctx context.Context, host string, args *StatChunkArgs) (ci *ChunkInfo, err error) {
func (c *client) StatChunk(ctx context.Context, host string, args *StatChunkArgs) (ci *clustermgr.ChunkInfo, err error) {
if !IsValidDiskID(args.DiskID) {
err = bloberr.ErrInvalidDiskId
return
}
urlStr := fmt.Sprintf("%v/chunk/stat/diskid/%v/vuid/%v", host, args.DiskID, args.Vuid)
ci = new(ChunkInfo)
ci = new(clustermgr.ChunkInfo)
err = c.GetWith(ctx, urlStr, ci)
return
}
@ -270,10 +122,10 @@ type ListChunkArgs struct {
}
type ListChunkRet struct {
ChunkInfos []*ChunkInfo `json:"chunk_infos"`
ChunkInfos []*clustermgr.ChunkInfo `json:"chunk_infos"`
}
func (c *client) ListChunks(ctx context.Context, host string, args *ListChunkArgs) (ret []*ChunkInfo, err error) {
func (c *client) ListChunks(ctx context.Context, host string, args *ListChunkArgs) (ret []*clustermgr.ChunkInfo, err error) {
if !IsValidDiskID(args.DiskID) {
err = bloberr.ErrInvalidDiskId
return

31
blobstore/api/blobnode/chunk_test.go
View File

@ -21,58 +21,59 @@ import (
"github.com/stretchr/testify/require"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/common/proto"
"github.com/cubefs/cubefs/blobstore/util/log"
)
func TestIsValidChunkId(t *testing.T) {
id := InvalidChunkId
require.Equal(t, false, IsValidChunkId(id))
id := clustermgr.InvalidChunkID
require.Equal(t, false, IsValidChunkID(id))
id = ChunkId{0x1}
require.Equal(t, true, IsValidChunkId(id))
id = clustermgr.ChunkID{0x1}
require.Equal(t, true, IsValidChunkID(id))
}
func TestChunkIdNew(t *testing.T) {
chunkid := NewChunkId(101)
require.Equal(t, ChunkIdLength, len(chunkid))
require.NotEqual(t, InvalidChunkId, chunkid)
chunkid := clustermgr.NewChunkID(101)
require.Equal(t, clustermgr.ChunkIDLength, len(chunkid))
require.NotEqual(t, clustermgr.InvalidChunkID, chunkid)
expectedVuid := chunkid.VolumeUnitId()
require.Equal(t, expectedVuid, proto.Vuid(101))
chunkname := chunkid.String()
require.Equal(t, ChunkIdEncodeLen, len(chunkname))
require.Equal(t, clustermgr.ChunkIDEncodeLen, len(chunkname))
arrs := strings.Split(chunkname, string(delimiter))
arrs := strings.Split(chunkname, "-")
require.Equal(t, 2, len(arrs))
require.Equal(t, "0000000000000065", arrs[0])
}
func TestChunkId_Marshal(t *testing.T) {
chunkid := NewChunkId(101)
chunkid := clustermgr.NewChunkID(101)
data, err := chunkid.Marshal()
require.NoError(t, err)
require.Equal(t, ChunkIdEncodeLen, len(data))
require.Equal(t, clustermgr.ChunkIDEncodeLen, len(data))
log.Infof("data:%s", data)
var newchunk ChunkId
var newchunk clustermgr.ChunkID
err = newchunk.Unmarshal(data)
require.NoError(t, err)
require.Equal(t, chunkid, newchunk)
}
func TestChunkId_MarshalJSON(t *testing.T) {
chunkid := NewChunkId(101)
chunkid := clustermgr.NewChunkID(101)
data, err := json.Marshal(chunkid)
require.NoError(t, err)
require.Equal(t, ChunkIdEncodeLen+2, len(data))
require.Equal(t, clustermgr.ChunkIDEncodeLen+2, len(data))
log.Infof("data:%s", data)
var newchunk ChunkId
var newchunk clustermgr.ChunkID
err = json.Unmarshal(data, &newchunk)
require.NoError(t, err)
require.Equal(t, chunkid, newchunk)

17
blobstore/api/blobnode/client.go
View File

@ -19,6 +19,7 @@ import (
"fmt"
"io"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/common/errors"
"github.com/cubefs/cubefs/blobstore/common/proto"
"github.com/cubefs/cubefs/blobstore/common/rpc"
@ -48,9 +49,9 @@ func (c *client) Close(ctx context.Context, host string) (err error) {
return nil
}
func (c *client) Stat(ctx context.Context, host string) (dis []*DiskInfo, err error) {
func (c *client) Stat(ctx context.Context, host string) (dis []*clustermgr.BlobNodeDiskInfo, err error) {
urlStr := fmt.Sprintf("%v/stat", host)
dis = make([]*DiskInfo, 0)
dis = make([]*clustermgr.BlobNodeDiskInfo, 0)
err = c.GetWith(ctx, urlStr, &dis)
return
}
@ -68,13 +69,13 @@ type DiskStatArgs struct {
DiskID proto.DiskID `json:"diskid"`
}
func (c *client) DiskInfo(ctx context.Context, host string, args *DiskStatArgs) (di *DiskInfo, err error) {
func (c *client) DiskInfo(ctx context.Context, host string, args *DiskStatArgs) (di *clustermgr.BlobNodeDiskInfo, err error) {
if !IsValidDiskID(args.DiskID) {
return nil, errors.ErrInvalidDiskId
}
urlStr := fmt.Sprintf("%v/disk/stat/diskid/%v", host, args.DiskID)
di = new(DiskInfo)
di = new(clustermgr.BlobNodeDiskInfo)
err = c.GetWith(ctx, urlStr, di)
return
}
@ -83,16 +84,16 @@ type StorageAPI interface {
String(ctx context.Context, host string) string
IsOnline(ctx context.Context, host string) bool
Close(ctx context.Context, host string) error
Stat(ctx context.Context, host string) (infos []*DiskInfo, err error)
DiskInfo(ctx context.Context, host string, args *DiskStatArgs) (di *DiskInfo, err error)
Stat(ctx context.Context, host string) (infos []*clustermgr.BlobNodeDiskInfo, err error)
DiskInfo(ctx context.Context, host string, args *DiskStatArgs) (di *clustermgr.BlobNodeDiskInfo, err error)
// chunks
CreateChunk(ctx context.Context, host string, args *CreateChunkArgs) (err error)
StatChunk(ctx context.Context, host string, args *StatChunkArgs) (ci *ChunkInfo, err error)
StatChunk(ctx context.Context, host string, args *StatChunkArgs) (ci *clustermgr.ChunkInfo, err error)
ReleaseChunk(ctx context.Context, host string, args *ChangeChunkStatusArgs) (err error)
SetChunkReadonly(ctx context.Context, host string, args *ChangeChunkStatusArgs) (err error)
SetChunkReadwrite(ctx context.Context, host string, args *ChangeChunkStatusArgs) (err error)
ListChunks(ctx context.Context, host string, args *ListChunkArgs) (cis []*ChunkInfo, err error)
ListChunks(ctx context.Context, host string, args *ListChunkArgs) (cis []*clustermgr.ChunkInfo, err error)
// shard
GetShard(ctx context.Context, host string, args *GetShardArgs) (body io.ReadCloser, shardCrc uint32, err error)

40
blobstore/api/blobnode/datainfo.go
View File

@ -1,40 +0,0 @@
// Copyright 2022 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package blobnode
import (
"github.com/cubefs/cubefs/blobstore/common/proto"
)
type ChunkInfo struct {
Id ChunkId `json:"id"`
Vuid proto.Vuid `json:"vuid"`
DiskID proto.DiskID `json:"diskid"`
Total uint64 `json:"total"` // ChunkSize
Used uint64 `json:"used"` // user data size
Free uint64 `json:"free"` // ChunkSize - Used
Size uint64 `json:"size"` // Chunk File Size (logic size)
Status ChunkStatus `json:"status"` // normal、readOnly
Compacting bool `json:"compacting"`
}
type ShardInfo struct {
Vuid proto.Vuid `json:"vuid"`
Bid proto.BlobID `json:"bid"`
Size int64 `json:"size"`
Crc uint32 `json:"crc"`
Flag ShardStatus `json:"flag"` // 1:normal,2:markDelete
Inline bool `json:"inline"`
}

9
blobstore/api/blobnode/shard.go
View File

@ -33,6 +33,15 @@ const (
MaxShardSize = math.MaxUint32
)
type ShardInfo struct {
Vuid proto.Vuid `json:"vuid"`
Bid proto.BlobID `json:"bid"`
Size int64 `json:"size"`
Crc uint32 `json:"crc"`
Flag ShardStatus `json:"flag"` // 1:normal,2:markDelete
Inline bool `json:"inline"`
}
type ShardStatus uint8
const (

181
blobstore/api/clustermgr/chunk.go Normal file
View File

@ -0,0 +1,181 @@
// Copyright 2024 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package clustermgr
import (
"encoding/binary"
"encoding/hex"
"errors"
"time"
"github.com/cubefs/cubefs/blobstore/common/proto"
)
type ChunkInfo struct {
Id ChunkID `json:"id"`
Vuid proto.Vuid `json:"vuid"`
DiskID proto.DiskID `json:"diskid"`
Total uint64 `json:"total"` // ChunkSize
Used uint64 `json:"used"` // user data size
Free uint64 `json:"free"` // ChunkSize - Used
Size uint64 `json:"size"` // Chunk File Size (logic size)
Status ChunkStatus `json:"status"` // normal、readOnly
Compacting bool `json:"compacting"`
}
const (
ChunkStatusDefault ChunkStatus = iota // 0
ChunkStatusNormal // 1
ChunkStatusReadOnly // 2
ChunkStatusRelease // 3
ChunkNumStatus // 4
)
const (
ReleaseForUser = "release for user"
ReleaseForCompact = "release for compact"
)
// Chunk ID
// vuid + timestamp
const (
chunkVuidLen = 8
chunkTimestmapLen = 8
ChunkIDLength = chunkVuidLen + chunkTimestmapLen
)
var InvalidChunkID ChunkID = [ChunkIDLength]byte{}
var (
_vuidHexLen = hex.EncodedLen(chunkVuidLen)
_timestampHexLen = hex.EncodedLen(chunkTimestmapLen)
// ${vuid_hex}-${tiemstamp_hex}
// |-- 8 Bytes --|-- 1 Bytes --|-- 8 Bytes --|
delimiter = []byte("-")
ChunkIDEncodeLen = _vuidHexLen + _timestampHexLen + len(delimiter)
)
type (
ChunkID [ChunkIDLength]byte
ChunkStatus uint8
)
func (c ChunkID) UnixTime() uint64 {
return binary.BigEndian.Uint64(c[chunkVuidLen:ChunkIDLength])
}
func (c ChunkID) VolumeUnitId() proto.Vuid {
return proto.Vuid(binary.BigEndian.Uint64(c[:chunkVuidLen]))
}
func (c *ChunkID) Marshal() ([]byte, error) {
buf := make([]byte, ChunkIDEncodeLen)
var i int
hex.Encode(buf[i:_vuidHexLen], c[:chunkVuidLen])
i += _vuidHexLen
copy(buf[i:i+len(delimiter)], delimiter)
i += len(delimiter)
hex.Encode(buf[i:], c[chunkVuidLen:ChunkIDLength])
return buf, nil
}
func (c *ChunkID) Unmarshal(data []byte) error {
if len(data) != ChunkIDEncodeLen {
panic(errors.New("chunk buf size not match"))
}
var i int
_, err := hex.Decode(c[:chunkVuidLen], data[i:_vuidHexLen])
if err != nil {
return err
}
i += _vuidHexLen
i += len(delimiter)
_, err = hex.Decode(c[chunkVuidLen:], data[i:])
if err != nil {
return err
}
return nil
}
func (c ChunkID) String() string {
buf, _ := c.Marshal()
return string(buf[:])
}
func (c ChunkID) MarshalJSON() ([]byte, error) {
b := make([]byte, ChunkIDEncodeLen+2)
b[0], b[ChunkIDEncodeLen+1] = '"', '"'
buf, _ := c.Marshal()
copy(b[1:], buf)
return b, nil
}
func (c *ChunkID) UnmarshalJSON(data []byte) (err error) {
if len(data) != ChunkIDEncodeLen+2 {
return errors.New("failed unmarshal json")
}
return c.Unmarshal(data[1 : ChunkIDEncodeLen+1])
}
func EncodeChunk(id ChunkID) string {
return id.String()
}
func NewChunkID(vuid proto.Vuid) (chunkId ChunkID) {
binary.BigEndian.PutUint64(chunkId[:chunkVuidLen], uint64(vuid))
binary.BigEndian.PutUint64(chunkId[chunkVuidLen:ChunkIDLength], uint64(time.Now().UnixNano()))
return
}
func DecodeChunk(name string) (id ChunkID, err error) {
buf := []byte(name)
if len(buf) != ChunkIDEncodeLen {
return InvalidChunkID, errors.New("invalid chunk name")
}
if err = id.Unmarshal(buf); err != nil {
return InvalidChunkID, errors.New("chunk unmarshal failed")
}
return
}
func (s *ChunkStatus) String() string {
switch *s {
case ChunkStatusDefault:
return "default"
case ChunkStatusNormal:
return "normal"
case ChunkStatusReadOnly:
return "readOnly"
case ChunkStatusRelease:
return "release"
default:
return "unkown"
}
}

24
blobstore/api/clustermgr/disk.go
View File

@ -40,10 +40,12 @@ func (s *ShardNodeDiskInfo) Unmarshal(raw []byte) error {
type ShardNodeDiskHeartbeatInfo struct {
DiskID proto.DiskID `json:"disk_id"`
Used int64 `json:"used"` // disk used space
Free int64 `json:"free"` // remaining free space on the disk
Size int64 `json:"size"` // total physical disk space
UsedShardCnt int32 `json:"used_shard_cnt"`
Used int64 `json:"used"` // disk used space
Free int64 `json:"free"` // remaining free space on the disk
Size int64 `json:"size"` // total physical disk space
MaxShardCnt int32 `json:"max_shard_cnt"` // note: maintained by clustermgr
FreeShardCnt int32 `json:"free_shard_cnt"` // note: maintained by clustermgr
UsedShardCnt int32 `json:"used_shard_cnt"` // current number of shards on the disk
}
type BlobNodeDiskInfo struct {
@ -64,7 +66,6 @@ type DiskHeartBeatInfo struct {
}
type DiskInfo struct {
// DiskID proto.DiskID `json:"disk_id"`
ClusterID proto.ClusterID `json:"cluster_id"`
Idc string `json:"idc,omitempty"`
Rack string `json:"rack,omitempty"`
@ -247,3 +248,16 @@ func (c *Client) SetReadonlyDisk(ctx context.Context, id proto.DiskID, readonly
err = c.PostWith(ctx, "/disk/access", nil, &DiskAccessArgs{DiskID: id, Readonly: readonly})
return
}
// AddShardNodeDisk add/register a new disk of shardnode into cluster manager
func (c *Client) AddShardNodeDisk(ctx context.Context, info *ShardNodeDiskInfo) (err error) {
err = c.PostWith(ctx, "shardnode/disk/add", nil, info)
return
}
// HeartbeatShardNodeDisk report shardnode disk latest capacity info to cluster manager
func (c *Client) HeartbeatShardNodeDisk(ctx context.Context, infos []*ShardNodeDiskHeartbeatInfo) (err error) {
args := &ShardNodeDisksHeartbeatArgs{Disks: infos}
err = c.PostWith(ctx, "shardnode/disk/heartbeat", nil, args)
return
}

10
blobstore/api/clustermgr/node.go
View File

@ -95,3 +95,13 @@ func (c *Client) TopoInfo(ctx context.Context) (ret *TopoInfo, err error) {
err = c.GetWith(ctx, "/topo/info", ret)
return
}
// AddShardNode add a new shardnode into cluster manager and return allocated nodeID
func (c *Client) AddShardNode(ctx context.Context, info *ShardNodeInfo) (proto.NodeID, error) {
ret := &NodeIDAllocRet{}
err := c.PostWith(ctx, "/shardnode/add", ret, info)
if err != nil {
return 0, err
}
return ret.NodeID, nil
}

105
blobstore/api/clustermgr/shard.go Normal file
View File

@ -0,0 +1,105 @@
// Copyright 2024 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package clustermgr
import (
"context"
"fmt"
"github.com/cubefs/cubefs/blobstore/common/proto"
)
type AllocShardUnitArgs struct {
Suid proto.Suid `json:"suid"`
}
type AllocShardUnitRet struct {
Suid proto.Suid `json:"suid"`
DiskID proto.DiskID `json:"disk_id"`
Host string `json:"host"`
}
func (c *Client) AllocShardUnit(ctx context.Context, args *AllocShardUnitArgs) (ret *AllocShardUnitRet, err error) {
err = c.PostWith(ctx, "/shard/unit/alloc", &ret, args)
return
}
type UpdateShardArgs struct {
NewSuid proto.Suid `json:"new_suid"`
NewIsLeaner bool `json:"new_is_leaner"`
NewDiskID proto.DiskID `json:"new_disk_id"`
OldSuid proto.Suid `json:"old_suid"`
OldIsLeaner bool `json:"old_is_leaner"`
}
func (c *Client) UpdateShard(ctx context.Context, args *UpdateShardArgs) (err error) {
err = c.PostWith(ctx, "/shard/update", nil, args)
return
}
type ReportShardArgs struct {
ShardReports []ShardReport `json:"shard_reports"`
}
type ReportShardRet struct {
ShardTasks []ShardTask `json:"shard_tasks"`
}
func (c *Client) ReportShard(ctx context.Context, args *ReportShardArgs) (ret []ShardTask, err error) {
result := &ReportShardRet{}
err = c.PostWith(ctx, "/shard/report", result, args)
return result.ShardTasks, err
}
type GetShardArgs struct {
ShardID proto.ShardID `json:"shard_id"`
}
func (c *Client) GetShardInfo(ctx context.Context, args *GetShardArgs) (ret *Shard, err error) {
ret = &Shard{}
err = c.GetWith(ctx, "/shard/get?shard_id="+args.ShardID.ToString(), ret)
return
}
type ListShardUnitArgs struct {
DiskID proto.DiskID `json:"disk_id"`
}
type ListShardUnitInfos struct {
ShardUnitInfos []*ShardUnitInfo `json:"shard_unit_infos"`
}
func (c *Client) ListShardUnit(ctx context.Context, args *ListShardUnitArgs) ([]*ShardUnitInfo, error) {
ret := &ListShardUnitInfos{}
err := c.GetWith(ctx, "/shard/unit/list?disk_id="+args.DiskID.ToString(), &ret)
return ret.ShardUnitInfos, err
}
type ListShardArgs struct {
// list marker
Marker proto.ShardID `json:"marker,omitempty"`
// one-page count
Count int `json:"count"`
}
type ListShards struct {
Shards []*Shard `json:"shards"`
Marker proto.ShardID `json:"marker"`
}
func (c *Client) ListShard(ctx context.Context, args *ListShardArgs) (ret ListShards, err error) {
err = c.GetWith(ctx, fmt.Sprintf("/shard/list?marker=%d&count=%d", args.Marker, args.Count), &ret)
return
}

40
blobstore/api/clustermgr/space.go
View File

@ -1,6 +1,24 @@
// Copyright 2024 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package clustermgr
import "github.com/cubefs/cubefs/blobstore/common/proto"
import (
"context"
"github.com/cubefs/cubefs/blobstore/common/proto"
)
type Space struct {
SpaceID proto.SpaceID `json:"sid"`
@ -15,3 +33,23 @@ type FieldMeta struct {
FieldType proto.FieldType `json:"field_type"`
IndexOption proto.IndexOption `json:"index_option"`
}
type CreateSpaceArgs struct {
Name string `json:"name"`
FieldMetas []FieldMeta `json:"field_metas"`
}
func (c *Client) CreateSpace(ctx context.Context, args *CreateSpaceArgs) (err error) {
err = c.PostWith(ctx, "/space/create", nil, args)
return
}
type GetSpaceArgs struct {
Name string `json:"name"`
}
func (c *Client) GetSpace(ctx context.Context, args *GetSpaceArgs) (ret *Space, err error) {
ret = &Space{}
err = c.GetWith(ctx, "/space/get?name="+args.Name, ret)
return
}

9
blobstore/api/clustermgr/volume.go
View File

@ -24,7 +24,6 @@ import (
"net/http"
"unsafe"
"github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/common/codemode"
"github.com/cubefs/cubefs/blobstore/common/proto"
"github.com/cubefs/cubefs/blobstore/common/rpc"
@ -224,7 +223,7 @@ func (c *Client) ListVolumeUnit(ctx context.Context, args *ListVolumeUnitArgs) (
}
type ReportChunkArgs struct {
ChunkInfos []blobnode.ChunkInfo `json:"chunk_infos"`
ChunkInfos []ChunkInfo `json:"chunk_infos"`
}
func (r *ReportChunkArgs) Encode() ([]byte, error) {
@ -278,11 +277,11 @@ func (r *ReportChunkArgs) Decode(reader io.Reader) error {
if count > 1<<20 {
return fmt.Errorf("chunks is too much %d, limit: %d", count, 1<<20)
}
r.ChunkInfos = make([]blobnode.ChunkInfo, count)
r.ChunkInfos = make([]ChunkInfo, count)
for i := range r.ChunkInfos {
raw := make([]byte, blobnode.ChunkIdEncodeLen)
raw := make([]byte, ChunkIDEncodeLen)
n, err := io.ReadFull(reader, raw)
if n != blobnode.ChunkIdEncodeLen || err != nil {
if n != ChunkIDEncodeLen || err != nil {
return fmt.Errorf("invalid source reader, err: %v", err)
}
if err != nil {

7
blobstore/api/clustermgr/volume_test.go
View File

@ -20,7 +20,6 @@ import (
"github.com/stretchr/testify/require"
"github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/common/proto"
)
@ -28,9 +27,9 @@ func TestChunkReportArgs(t *testing.T) {
vuid1, _ := proto.NewVuid(1, 0, 1)
vuid2, _ := proto.NewVuid(1, 1, 1)
args := ReportChunkArgs{
ChunkInfos: []blobnode.ChunkInfo{
ChunkInfos: []ChunkInfo{
{
Id: blobnode.NewChunkId(vuid1),
Id: NewChunkID(vuid1),
Vuid: vuid1,
DiskID: 1,
Free: 1,
@ -38,7 +37,7 @@ func TestChunkReportArgs(t *testing.T) {
Size: 1,
},
{
Id: blobnode.NewChunkId(vuid2),
Id: NewChunkID(vuid2),
Vuid: vuid2,
DiskID: 2,
Free: 2,

2
blobstore/api/proxy/cacher.go
View File

@ -57,7 +57,7 @@ type CacheDiskArgs struct {
// Cacher interface of proxy cache.
type Cacher interface {
GetCacheVolume(ctx context.Context, host string, args *CacheVolumeArgs) (*VersionVolume, error)
GetCacheDisk(ctx context.Context, host string, args *CacheDiskArgs) (*clustermgr.DiskInfo, error)
GetCacheDisk(ctx context.Context, host string, args *CacheDiskArgs) (*clustermgr.BlobNodeDiskInfo, error)
// Erase cache in proxy memory and diskv.
// Volume key is "volume-{vid}", and disk key is "disk-{disk_id}".
// Notice: Erase all if key is "ALL"!

4
blobstore/api/proxy/client.go
View File

@ -66,8 +66,8 @@ func (c *client) GetCacheVolume(ctx context.Context, host string, args *CacheVol
return
}
func (c *client) GetCacheDisk(ctx context.Context, host string, args *CacheDiskArgs) (disk *clustermgr.DiskInfo, err error) {
disk = new(clustermgr.DiskInfo)
func (c *client) GetCacheDisk(ctx context.Context, host string, args *CacheDiskArgs) (disk *clustermgr.BlobNodeDiskInfo, err error) {
disk = new(clustermgr.BlobNodeDiskInfo)
url := fmt.Sprintf("%s/cache/disk/%d?flush=%v", host, args.DiskID, args.Flush)
err = c.GetWith(ctx, url, &disk)
return

17
blobstore/blobnode/chunk.go
View File

@ -16,6 +16,7 @@ package blobnode
import (
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/blobnode/core"
"github.com/cubefs/cubefs/blobstore/blobnode/core/disk"
bloberr "github.com/cubefs/cubefs/blobstore/common/errors"
@ -178,7 +179,7 @@ func (s *Service) ChunkRelease(c *rpc.Context) {
}
// only readonly chunk can be release
if !args.Force && cs.Status() != bnapi.ChunkStatusReadOnly {
if !args.Force && cs.Status() != clustermgr.ChunkStatusReadOnly {
span.Errorf("vuid:%v/chunk:%s (status:%v) not readonly", args.Vuid, cs.ID(), cs.Status())
c.RespondError(bloberr.ErrChunkNotReadonly)
return
@ -241,19 +242,19 @@ func (s *Service) ChunkReadonly(c *rpc.Context) {
return
}
if cs.Status() == bnapi.ChunkStatusReadOnly {
if cs.Status() == clustermgr.ChunkStatusReadOnly {
span.Warnf("chunk(%s) already in readonly", cs.ID())
return
}
if cs.Status() != bnapi.ChunkStatusNormal {
if cs.Status() != clustermgr.ChunkStatusNormal {
span.Warnf("chunk(%s) status no normal", cs.ID())
c.RespondError(bloberr.ErrChunkNotNormal)
return
}
// change persistence status
err = ds.UpdateChunkStatus(ctx, args.Vuid, bnapi.ChunkStatusReadOnly)
err = ds.UpdateChunkStatus(ctx, args.Vuid, clustermgr.ChunkStatusReadOnly)
if err != nil {
span.Errorf("set args:(%s) readOnly failed: %v", args, err)
c.RespondError(err)
@ -310,20 +311,20 @@ func (s *Service) ChunkReadwrite(c *rpc.Context) {
return
}
if cs.Status() == bnapi.ChunkStatusNormal {
if cs.Status() == clustermgr.ChunkStatusNormal {
span.Warnf("chunk(%s) already normal", cs.ID())
return
}
// only readonly -> normal
if cs.Status() != bnapi.ChunkStatusReadOnly {
if cs.Status() != clustermgr.ChunkStatusReadOnly {
span.Warnf("chunk(%s) status no readonly", cs.ID())
c.RespondError(bloberr.ErrChunkNotReadonly)
return
}
// change persistence status
err = ds.UpdateChunkStatus(ctx, args.Vuid, bnapi.ChunkStatusNormal)
err = ds.UpdateChunkStatus(ctx, args.Vuid, clustermgr.ChunkStatusNormal)
if err != nil {
span.Errorf("set args:(%s) readWrite failed: %v", args, err)
c.RespondError(err)
@ -369,7 +370,7 @@ func (s *Service) ChunkList(c *rpc.Context) {
return nil
})
infos := make([]*bnapi.ChunkInfo, 0)
infos := make([]*clustermgr.ChunkInfo, 0)
for _, cs := range chunks {
info := cs.ChunkInfo(ctx)
infos = append(infos, &info)

7
blobstore/blobnode/chunk_test.go
View File

@ -22,6 +22,7 @@ import (
"github.com/stretchr/testify/require"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/common/proto"
"github.com/cubefs/cubefs/blobstore/common/trace"
)
@ -132,7 +133,7 @@ func TestSetChunkStatus(t *testing.T) {
chunkStat, err := client.StatChunk(ctx, host, statChunkArg)
require.NoError(t, err)
require.Equal(t, bnapi.ChunkStatusReadOnly, chunkStat.Status)
require.Equal(t, clustermgr.ChunkStatusReadOnly, chunkStat.Status)
err = client.SetChunkReadonly(ctx, host, changeChunkArg)
require.NoError(t, err)
@ -142,7 +143,7 @@ func TestSetChunkStatus(t *testing.T) {
chunkStat, err = client.StatChunk(ctx, host, statChunkArg)
require.NoError(t, err)
require.Equal(t, bnapi.ChunkStatusNormal, chunkStat.Status)
require.Equal(t, clustermgr.ChunkStatusNormal, chunkStat.Status)
err = client.SetChunkReadwrite(ctx, host, changeChunkArg)
require.NoError(t, err)
@ -152,7 +153,7 @@ func TestSetChunkStatus(t *testing.T) {
cs, exist := ds.GetChunkStorage(vuid)
require.True(t, exist)
cs.SetStatus(bnapi.ChunkStatusRelease)
cs.SetStatus(clustermgr.ChunkStatusRelease)
err = client.SetChunkReadwrite(ctx, host, changeChunkArg)
require.Error(t, err)
}

6
blobstore/blobnode/chunkcheck.go
View File

@ -140,11 +140,11 @@ func (s *Service) rangeChunks(ctx context.Context, ds core.DiskAPI, cmVuidMaps m
return
}
for _, cs := range localChunks {
createTime := time.Unix(0, int64(cs.ChunkId.UnixTime()))
createTime := time.Unix(0, int64(cs.ChunkID.UnixTime()))
protectionPeriod := time.Duration(s.Conf.ChunkProtectionPeriodSec) * time.Second
if time.Since(createTime) < protectionPeriod {
span.Debugf("%s still in ctime protection", cs.ChunkId)
span.Debugf("%s still in ctime protection", cs.ChunkID)
continue
}
vuid := cs.Vuid
@ -182,7 +182,7 @@ func (s *Service) releaseEpochChunk(ctx context.Context, cs core.VuidMeta, ds co
// Important note: in the future, the logical consideration will be transferred to cm to judge
err := ds.ReleaseChunk(ctx, cs.Vuid, true)
if err != nil {
span.Errorf("release ChunkStorage(%s) form disk(%v) failed: %v", cs.ChunkId, ds.ID(), err)
span.Errorf("release ChunkStorage(%s) form disk(%v) failed: %v", cs.ChunkID, ds.ID(), err)
return
}
span.Infof("vuid(%v) have been release", cs.Vuid)

4
blobstore/blobnode/chunkcheck_test.go
View File

@ -102,8 +102,8 @@ func TestCheckRegisterChunk(t *testing.T) {
diskChunks, err := service.Disks[diskID].ListChunks(ctx)
require.NoError(t, err)
require.Equal(t, 2, len(diskChunks))
require.Equal(t, bnapi.ChunkStatusNormal, diskChunks[0].Status)
require.Equal(t, bnapi.ChunkStatusNormal, diskChunks[1].Status)
require.Equal(t, cmapi.ChunkStatusNormal, diskChunks[0].Status)
require.Equal(t, cmapi.ChunkStatusNormal, diskChunks[1].Status)
_, err = service.ClusterMgrClient.ListVolumeUnit(ctx, &cmapi.ListVolumeUnitArgs{DiskID: diskID})
require.NoError(t, err)

3
blobstore/blobnode/chunkreport.go
View File

@ -18,7 +18,6 @@ import (
"context"
"time"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/blobnode/base"
"github.com/cubefs/cubefs/blobstore/blobnode/core"
@ -49,7 +48,7 @@ func (s *Service) reportChunkInfoToClusterMgr() {
chunks := s.copyChunkStorages(ctx)
dirtychunks := make(map[proto.Vuid]core.ChunkAPI)
cis := make([]bnapi.ChunkInfo, 0)
cis := make([]cmapi.ChunkInfo, 0)
for _, cs := range chunks {
if !cs.Disk().IsWritable() || !cs.IsDirty() { // not writable, dont need report
continue

5
blobstore/blobnode/client/blobnode_cli.go
View File

@ -20,6 +20,7 @@ import (
"strings"
api "github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
errcode "github.com/cubefs/cubefs/blobstore/common/errors"
"github.com/cubefs/cubefs/blobstore/common/proto"
"github.com/cubefs/cubefs/blobstore/common/rpc"
@ -49,12 +50,12 @@ const (
// ChunkInfo chunk info
type ChunkInfo struct {
api.ChunkInfo
clustermgr.ChunkInfo
}
// Locked return true if chunk is locked
func (c *ChunkInfo) Locked() bool {
return c.ChunkInfo.Status == api.ChunkStatusReadOnly
return c.ChunkInfo.Status == clustermgr.ChunkStatusReadOnly
}
// ShardInfo shard info

21
blobstore/blobnode/core/chunk/chunk.go
View File

@ -25,6 +25,7 @@ import (
"time"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/blobnode/base"
"github.com/cubefs/cubefs/blobstore/blobnode/base/qos"
"github.com/cubefs/cubefs/blobstore/blobnode/core"
@ -79,7 +80,7 @@ type chunk struct {
// status
dirty uint32
status bnapi.ChunkStatus
status clustermgr.ChunkStatus
closed bool
lastModifyTime int64
@ -100,7 +101,7 @@ func newChunkStorage(ctx context.Context, dataPath string, vm core.VuidMeta, ioP
span := trace.SpanFromContextSafe(ctx)
// chunk data
chunkFile := filepath.Join(dataPath, vm.ChunkId.String())
chunkFile := filepath.Join(dataPath, vm.ChunkID.String())
opt := core.Option{}
for _, fn := range opts {
@ -299,7 +300,7 @@ func (cs *chunk) Disk() core.DiskAPI {
return cs.disk
}
func (cs *chunk) ID() bnapi.ChunkId {
func (cs *chunk) ID() clustermgr.ChunkID {
stg := cs.getStg()
return stg.ID()
}
@ -308,13 +309,13 @@ func (cs *chunk) Vuid() proto.Vuid {
return cs.vuid
}
func (cs *chunk) ChunkInfo(ctx context.Context) (info bnapi.ChunkInfo) {
func (cs *chunk) ChunkInfo(ctx context.Context) (info clustermgr.ChunkInfo) {
span := trace.SpanFromContextSafe(ctx)
err := cs.refreshFstat(ctx)
if err != nil {
span.Errorf("Failed refresh fstat, err:%v", err)
return bnapi.ChunkInfo{}
return clustermgr.ChunkInfo{}
}
cs.lock.RLock()
@ -346,7 +347,7 @@ func (cs *chunk) vuidMeta() (vm *core.VuidMeta) {
Version: cs.version,
Vuid: cs.vuid,
DiskID: cs.diskID,
ChunkId: stg.ID(),
ChunkID: stg.ID(),
ParentChunk: stat.ParentID,
ChunkSize: int64(cs.fileInfo.Total),
Ctime: stat.CreateTime,
@ -740,10 +741,10 @@ func (cs *chunk) AllowModify() (err error) {
if compacting && config.DisableModifyInCompacting {
return bloberr.ErrChunkInCompact
}
if status == bnapi.ChunkStatusReadOnly {
if status == clustermgr.ChunkStatusReadOnly {
return bloberr.ErrReadonlyVUID
}
if status == bnapi.ChunkStatusRelease {
if status == clustermgr.ChunkStatusRelease {
return bloberr.ErrReleaseVUID
}
@ -770,7 +771,7 @@ func (cs *chunk) HasEnoughSpace(needSize int64) bool {
return needSize+reserved-compactReserved < diskStats.Free
}
func (cs *chunk) SetStatus(status bnapi.ChunkStatus) (err error) {
func (cs *chunk) SetStatus(status clustermgr.ChunkStatus) (err error) {
cs.lock.Lock()
defer cs.lock.Unlock()
@ -778,7 +779,7 @@ func (cs *chunk) SetStatus(status bnapi.ChunkStatus) (err error) {
return nil
}
func (cs *chunk) Status() (status bnapi.ChunkStatus) {
func (cs *chunk) Status() (status clustermgr.ChunkStatus) {
cs.lock.RLock()
defer cs.lock.RUnlock()

31
blobstore/blobnode/core/chunk/chunk_test.go
View File

@ -26,6 +26,7 @@ import (
"testing"
"time"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/require"
@ -68,7 +69,7 @@ func TestNewChunkStorage(t *testing.T) {
}
vuid := proto.Vuid(1)
chunkid := bnapi.NewChunkId(vuid)
chunkid := clustermgr.NewChunkID(vuid)
err = core.EnsureDiskArea(testDir, "")
require.NoError(t, err)
@ -85,9 +86,9 @@ func TestNewChunkStorage(t *testing.T) {
vm := core.VuidMeta{
Vuid: vuid,
DiskID: 12,
ChunkId: chunkid,
ChunkID: chunkid,
Mtime: time.Now().UnixNano(),
Status: bnapi.ChunkStatusNormal,
Status: clustermgr.ChunkStatusNormal,
}
ioPools := newIoPoolMock(t)
ioQos, _ := qos.NewIoQueueQos(qos.Config{ReadQueueDepth: 2, WriteQueueDepth: 2, WriteChanQueCnt: 2})
@ -127,7 +128,7 @@ func TestChunkStorage_ReadWrite(t *testing.T) {
}
vuid := proto.Vuid(1)
chunkid := bnapi.NewChunkId(vuid)
chunkid := clustermgr.NewChunkID(vuid)
err = core.EnsureDiskArea(testDir, "")
require.NoError(t, err)
@ -144,9 +145,9 @@ func TestChunkStorage_ReadWrite(t *testing.T) {
vm := core.VuidMeta{
Vuid: vuid,
DiskID: 12,
ChunkId: chunkid,
ChunkID: chunkid,
Mtime: time.Now().UnixNano(),
Status: bnapi.ChunkStatusNormal,
Status: clustermgr.ChunkStatusNormal,
}
ioPools := newIoPoolMock(t)
@ -323,7 +324,7 @@ func TestChunkStorage_ReadWriteInline(t *testing.T) {
}
vuid := proto.Vuid(1)
chunkid := bnapi.NewChunkId(vuid)
chunkid := clustermgr.NewChunkID(vuid)
err = core.EnsureDiskArea(testDir, "")
require.NoError(t, err)
@ -340,9 +341,9 @@ func TestChunkStorage_ReadWriteInline(t *testing.T) {
vm := core.VuidMeta{
Vuid: vuid,
DiskID: 12,
ChunkId: chunkid,
ChunkID: chunkid,
Mtime: time.Now().UnixNano(),
Status: bnapi.ChunkStatusNormal,
Status: clustermgr.ChunkStatusNormal,
}
ioPools := newIoPoolMock(t)
@ -441,7 +442,7 @@ func TestChunkStorage_DeleteOp(t *testing.T) {
}
vuid := proto.Vuid(1)
chunkid := bnapi.NewChunkId(vuid)
chunkid := clustermgr.NewChunkID(vuid)
err = core.EnsureDiskArea(testDir, "")
require.NoError(t, err)
@ -458,9 +459,9 @@ func TestChunkStorage_DeleteOp(t *testing.T) {
vm := core.VuidMeta{
Vuid: vuid,
DiskID: 12,
ChunkId: chunkid,
ChunkID: chunkid,
Mtime: time.Now().UnixNano(),
Status: bnapi.ChunkStatusNormal,
Status: clustermgr.ChunkStatusNormal,
}
ioPools := newIoPoolMock(t)
@ -552,7 +553,7 @@ func TestChunkStorage_Finalizer(t *testing.T) {
}
vuid := proto.Vuid(1)
chunkid := bnapi.NewChunkId(vuid)
chunkid := clustermgr.NewChunkID(vuid)
err = core.EnsureDiskArea(testDir, "")
require.NoError(t, err)
@ -569,9 +570,9 @@ func TestChunkStorage_Finalizer(t *testing.T) {
vm := core.VuidMeta{
Vuid: vuid,
DiskID: 12,
ChunkId: chunkid,
ChunkID: chunkid,
Mtime: time.Now().UnixNano(),
Status: bnapi.ChunkStatusNormal,
Status: clustermgr.ChunkStatusNormal,
}
ioPools := newIoPoolMock(t)
ioQos, _ := qos.NewIoQueueQos(qos.Config{ReadQueueDepth: 2, WriteQueueDepth: 2, WriteChanQueCnt: 2})

5
blobstore/blobnode/core/chunk/compact.go
View File

@ -21,6 +21,7 @@ import (
"time"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/blobnode/core"
"github.com/cubefs/cubefs/blobstore/blobnode/core/storage"
"github.com/cubefs/cubefs/blobstore/common/proto"
@ -53,12 +54,12 @@ func (cs *chunk) StartCompact(ctx context.Context) (newcs core.ChunkAPI, err err
Version: cs.version,
Vuid: cs.vuid,
DiskID: cs.diskID,
ChunkId: bnapi.NewChunkId(cs.vuid),
ChunkID: clustermgr.NewChunkID(cs.vuid),
ParentChunk: cs.ID(),
ChunkSize: int64(cs.fileInfo.Total),
Ctime: now,
Mtime: now,
Status: bnapi.ChunkStatusDefault,
Status: clustermgr.ChunkStatusDefault,
}
stg := cs.getStg()

13
blobstore/blobnode/core/chunk/compact_test.go
View File

@ -31,6 +31,7 @@ import (
"github.com/stretchr/testify/require"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/blobnode/base/qos"
"github.com/cubefs/cubefs/blobstore/blobnode/core"
"github.com/cubefs/cubefs/blobstore/blobnode/db"
@ -56,7 +57,7 @@ func (mock *diskMock) Status() (status proto.DiskStatus) {
return mock.status
}
func (mock *diskMock) DiskInfo() (info bnapi.DiskInfo) {
func (mock *diskMock) DiskInfo() (info cmapi.BlobNodeDiskInfo) {
return
}
@ -104,7 +105,7 @@ func (mock *diskMock) ReleaseChunk(ctx context.Context, vuid proto.Vuid, force b
return
}
func (mock *diskMock) UpdateChunkStatus(ctx context.Context, vuid proto.Vuid, status bnapi.ChunkStatus) (err error) {
func (mock *diskMock) UpdateChunkStatus(ctx context.Context, vuid proto.Vuid, status cmapi.ChunkStatus) (err error) {
return
}
@ -120,7 +121,7 @@ func (mock *diskMock) EnqueueCompact(ctx context.Context, vuid proto.Vuid) {
// do nothing
}
func (mock *diskMock) GcRubbishChunk(ctx context.Context) (mayBeLost []bnapi.ChunkId, err error) {
func (mock *diskMock) GcRubbishChunk(ctx context.Context) (mayBeLost []cmapi.ChunkID, err error) {
return
}
@ -161,13 +162,13 @@ func createTestChunk(t *testing.T, ctx context.Context, diskRoot string, vuid pr
require.NoError(t, err)
require.NotNil(t, dbHandler)
chunkId := bnapi.NewChunkId(vuid)
chunkId := cmapi.NewChunkID(vuid)
vm := core.VuidMeta{
Vuid: vuid,
DiskID: 12,
ChunkId: chunkId,
ChunkID: chunkId,
Mtime: time.Now().UnixNano(),
Status: bnapi.ChunkStatusNormal,
Status: cmapi.ChunkStatusNormal,
}
conf := &core.Config{

19
blobstore/blobnode/core/disk/chunkinspect.go
View File

@ -22,11 +22,12 @@ import (
"time"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/blobnode/core"
"github.com/cubefs/cubefs/blobstore/common/trace"
)
func listPhyDiskChunkFile(ctx context.Context, dataPath string) (cis map[bnapi.ChunkId]struct{}, err error) {
func listPhyDiskChunkFile(ctx context.Context, dataPath string) (cis map[clustermgr.ChunkID]struct{}, err error) {
span := trace.SpanFromContextSafe(ctx)
dir, err := os.ReadDir(dataPath)
@ -35,14 +36,14 @@ func listPhyDiskChunkFile(ctx context.Context, dataPath string) (cis map[bnapi.C
return nil, err
}
cis = make(map[bnapi.ChunkId]struct{})
cis = make(map[clustermgr.ChunkID]struct{})
for i := range dir {
if dir[i].IsDir() {
span.Warnf("%s/%s is dir.", dataPath, dir[i].Name())
continue
}
id, err := bnapi.DecodeChunk(dir[i].Name())
id, err := clustermgr.DecodeChunk(dir[i].Name())
if err != nil {
span.Errorf("decode %v to chunkID failed: %v", dir[i].Name(), err)
continue
@ -54,7 +55,7 @@ func listPhyDiskChunkFile(ctx context.Context, dataPath string) (cis map[bnapi.C
}
func (ds *DiskStorage) GcRubbishChunk(ctx context.Context) (
mayBeLost []bnapi.ChunkId, err error,
mayBeLost []clustermgr.ChunkID, err error,
) {
span := trace.SpanFromContextSafe(ctx)
@ -70,11 +71,11 @@ func (ds *DiskStorage) GcRubbishChunk(ctx context.Context) (
return
}
chunkIdMetaMap := make(map[bnapi.ChunkId]struct{})
chunkIdMetaMap := make(map[clustermgr.ChunkID]struct{})
// Important: Confirm whether there is suspicious data loss
for _, vm := range chunkMetas {
id := vm.ChunkId
id := vm.ChunkID
chunkIdMetaMap[id] = struct{}{}
if _, exist := chunkIdDataMap[id]; exist {
continue
@ -100,12 +101,12 @@ func (ds *DiskStorage) GcRubbishChunk(ctx context.Context) (
return
}
func (ds *DiskStorage) maybeChunkLost(ctx context.Context, id bnapi.ChunkId, meta core.VuidMeta) (lost bool, err error) {
func (ds *DiskStorage) maybeChunkLost(ctx context.Context, id clustermgr.ChunkID, meta core.VuidMeta) (lost bool, err error) {
span := trace.SpanFromContextSafe(ctx)
span.Debugf("chunk:%s (meta:%v) maybe lost, will confirm.", id, meta)
if meta.Status != bnapi.ChunkStatusNormal {
if meta.Status != clustermgr.ChunkStatusNormal {
span.Debugf("chunk:%s, meta:%v", id, meta)
return false, nil
}
@ -127,7 +128,7 @@ func (ds *DiskStorage) maybeChunkLost(ctx context.Context, id bnapi.ChunkId, met
return true, nil
}
func (ds *DiskStorage) maybeCleanRubbishChunk(ctx context.Context, id bnapi.ChunkId) (err error) {
func (ds *DiskStorage) maybeCleanRubbishChunk(ctx context.Context, id clustermgr.ChunkID) (err error) {
span := trace.SpanFromContextSafe(ctx)
// set io type

18
blobstore/blobnode/core/disk/chunkinspect_test.go
View File

@ -21,7 +21,7 @@ import (
"testing"
"time"
dnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/blobnode/core"
"github.com/cubefs/cubefs/blobstore/common/proto"
@ -63,9 +63,9 @@ func TestMayChunkLost(t *testing.T) {
vm := core.VuidMeta{
Vuid: vuid,
DiskID: ds.DiskID,
ChunkId: cs.ID(),
ChunkID: cs.ID(),
Mtime: time.Now().UnixNano(),
Status: dnapi.ChunkStatusReadOnly,
Status: clustermgr.ChunkStatusReadOnly,
}
lost, err := ds.maybeChunkLost(ctx, cs.ID(), vm)
@ -75,16 +75,16 @@ func TestMayChunkLost(t *testing.T) {
vm1 := core.VuidMeta{
Vuid: vuid,
DiskID: ds.DiskID,
ChunkId: cs.ID(),
ChunkID: cs.ID(),
Mtime: time.Now().UnixNano(),
Status: dnapi.ChunkStatusNormal,
Status: clustermgr.ChunkStatusNormal,
}
lost, err = ds.maybeChunkLost(ctx, dnapi.ChunkId{12}, vm1)
lost, err = ds.maybeChunkLost(ctx, clustermgr.ChunkID{12}, vm1)
require.NoError(t, err)
require.Equal(t, false, lost)
var InvalidChunkId dnapi.ChunkId = [16]byte{}
var InvalidChunkId clustermgr.ChunkID = [16]byte{}
lost, err = ds.maybeChunkLost(ctx, InvalidChunkId, vm1)
require.Error(t, err)
@ -126,11 +126,11 @@ func TestMaybeCleanRubbishChunk(t *testing.T) {
err = ds.maybeCleanRubbishChunk(ctx, cs.ID())
require.Nil(t, err)
var InvalidChunkId dnapi.ChunkId = [16]byte{}
var InvalidChunkId clustermgr.ChunkID = [16]byte{}
err = ds.maybeCleanRubbishChunk(ctx, InvalidChunkId)
require.Error(t, err)
err = ds.maybeCleanRubbishChunk(ctx, dnapi.ChunkId{12})
err = ds.maybeCleanRubbishChunk(ctx, clustermgr.ChunkID{12})
require.Nil(t, err)
err = ds.SuperBlock.DeleteChunk(ctx, cs.ID())

15
blobstore/blobnode/core/disk/compact.go
View File

@ -18,7 +18,6 @@ import (
"context"
"time"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/blobnode/base"
"github.com/cubefs/cubefs/blobstore/blobnode/core"
@ -153,14 +152,14 @@ STOPCOMPACT:
// compact done and enable modify
err = cs.StopCompact(ctx, ncs)
if err != nil {
span.Errorf("Failed StopCompact vuid[%d] newchunkfile[%s]", vuid, ncsMeta.ChunkId)
span.Errorf("Failed StopCompact vuid[%d] newchunkfile[%s]", vuid, ncsMeta.ChunkID)
return err
}
// mark destroy ncs
err = ds.destroyRedundant(ctx, ncs)
if err != nil {
span.Errorf("Failed update chunk[%s] status. err:%v", ncsMeta.ChunkId, err)
span.Errorf("Failed update chunk[%s] status. err:%v", ncsMeta.ChunkID, err)
}
span.Warnf("compact success. vuid[%d] chunkfile[%s]", vuid, cs.ID())
@ -180,20 +179,20 @@ func (ds *DiskStorage) destroyRedundant(ctx context.Context, ncs core.ChunkAPI)
if !ncs.HasPendingRequest() {
break
}
span.Debugf("=== wait chunk(%s) all request done ===", ncsMeta.ChunkId)
span.Debugf("=== wait chunk(%s) all request done ===", ncsMeta.ChunkID)
}
span.Infof("safe here. id:%s request all done.", ncsMeta.ChunkId)
span.Infof("safe here. id:%s request all done.", ncsMeta.ChunkID)
// isolated. safe
ncs.Close(ctx)
// update chunk status, mark destroy. destroy async
ncsMeta.Status = bnapi.ChunkStatusRelease
ncsMeta.Reason = bnapi.ReleaseForCompact
ncsMeta.Status = cmapi.ChunkStatusRelease
ncsMeta.Reason = cmapi.ReleaseForCompact
ncsMeta.Mtime = time.Now().UnixNano()
return ds.SuperBlock.UpsertChunk(ctx, ncsMeta.ChunkId, *ncsMeta)
return ds.SuperBlock.UpsertChunk(ctx, ncsMeta.ChunkID, *ncsMeta)
}
func (ds *DiskStorage) ExecCompactChunk(vuid proto.Vuid) (err error) {

7
blobstore/blobnode/core/disk/compact_test.go
View File

@ -22,6 +22,7 @@ import (
"testing"
"time"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/require"
@ -52,7 +53,7 @@ func TestDiskStorage_StartCompact(t *testing.T) {
conf.CompactBatchSize = core.DefaultCompactBatchSize
vuid := proto.Vuid(2001)
chunkID := bnapi.NewChunkId(vuid)
chunkID := clustermgr.NewChunkID(vuid)
err = core.EnsureDiskArea(testDir, "")
require.NoError(t, err)
@ -76,9 +77,9 @@ func TestDiskStorage_StartCompact(t *testing.T) {
vm := core.VuidMeta{
Vuid: vuid,
DiskID: 12,
ChunkId: chunkID,
ChunkID: chunkID,
Mtime: time.Now().UnixNano(),
Status: bnapi.ChunkStatusNormal,
Status: clustermgr.ChunkStatusNormal,
}
ctr := gomock.NewController(t)

65
blobstore/blobnode/core/disk/disk.go
View File

@ -24,9 +24,8 @@ import (
"sync/atomic"
"time"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
bncom "github.com/cubefs/cubefs/blobstore/blobnode/base"
"github.com/cubefs/cubefs/blobstore/blobnode/base/flow"
"github.com/cubefs/cubefs/blobstore/blobnode/base/qos"
@ -47,10 +46,10 @@ const (
RandomIntervalS = 30
)
var StateTransitionRules = map[bnapi.ChunkStatus][]bnapi.ChunkStatus{
bnapi.ChunkStatusDefault: {bnapi.ChunkStatusNormal},
bnapi.ChunkStatusNormal: {bnapi.ChunkStatusNormal, bnapi.ChunkStatusReadOnly},
bnapi.ChunkStatusReadOnly: {bnapi.ChunkStatusNormal, bnapi.ChunkStatusReadOnly, bnapi.ChunkStatusRelease},
var StateTransitionRules = map[clustermgr.ChunkStatus][]clustermgr.ChunkStatus{
clustermgr.ChunkStatusDefault: {clustermgr.ChunkStatusNormal},
clustermgr.ChunkStatusNormal: {clustermgr.ChunkStatusNormal, clustermgr.ChunkStatusReadOnly},
clustermgr.ChunkStatusReadOnly: {clustermgr.ChunkStatusNormal, clustermgr.ChunkStatusReadOnly, clustermgr.ChunkStatusRelease},
}
var (
@ -322,18 +321,18 @@ func (dsw *DiskStorageWrapper) CreateChunk(ctx context.Context, vuid proto.Vuid,
}
super := ds.SuperBlock
chunkId := bnapi.NewChunkId(vuid)
chunkId := clustermgr.NewChunkID(vuid)
nowtime := time.Now().UnixNano()
vm := core.VuidMeta{
Version: _chunkVer[0],
Vuid: vuid,
DiskID: ds.DiskID,
ChunkId: chunkId,
ChunkID: chunkId,
ChunkSize: chunksize,
Ctime: nowtime,
Mtime: nowtime,
Status: bnapi.ChunkStatusNormal,
Status: clustermgr.ChunkStatusNormal,
}
// create chunk storage
@ -350,9 +349,9 @@ func (dsw *DiskStorageWrapper) CreateChunk(ctx context.Context, vuid proto.Vuid,
}
// save to superBlock
err = super.UpsertChunk(ctx, vm.ChunkId, vm)
err = super.UpsertChunk(ctx, vm.ChunkID, vm)
if err != nil {
span.Errorf("Failed upsert chunk<%s>, err:%v", vm.ChunkId, err)
span.Errorf("Failed upsert chunk<%s>, err:%v", vm.ChunkID, err)
return nil, err
}
@ -648,8 +647,8 @@ func (dsw *DiskStorageWrapper) RestoreChunkStorage(ctx context.Context) (err err
span.Debugf("vuid:%d, chunkid: %s", vuid, chunkid)
vm := vuidMetas[chunkid]
if vm.Status == bnapi.ChunkStatusRelease {
span.Warnf("vuid:%d(chunk:%s) status is release", vm.Vuid, vm.ChunkId)
if vm.Status == clustermgr.ChunkStatusRelease {
span.Warnf("vuid:%d(chunk:%s) status is release", vm.Vuid, vm.ChunkID)
continue
}
if vm.Compacting {
@ -719,7 +718,7 @@ func (ds *DiskStorage) ReleaseChunk(ctx context.Context, vuid proto.Vuid, force
}
// can not convert status
if !force && !isValidStateTransition(cs.Status(), bnapi.ChunkStatusRelease) {
if !force && !isValidStateTransition(cs.Status(), clustermgr.ChunkStatusRelease) {
span.Errorf("can not release chunk(%s) status:%v", cs.ID(), cs.Status())
return bloberr.ErrUnexpected
}
@ -745,22 +744,22 @@ func (ds *DiskStorage) ReleaseChunk(ctx context.Context, vuid proto.Vuid, force
// update chunk meta
vm := cs.VuidMeta()
vm.Status = bnapi.ChunkStatusRelease
vm.Reason = bnapi.ReleaseForUser
vm.Status = clustermgr.ChunkStatusRelease
vm.Reason = clustermgr.ReleaseForUser
vm.Mtime = time.Now().UnixNano()
err = ds.SuperBlock.UpsertChunk(ctx, cs.ID(), *vm)
if err != nil {
span.Errorf("update chunk(%s) status to release failed: %v", vm.ChunkId, err)
span.Errorf("update chunk(%s) status to release failed: %v", vm.ChunkID, err)
return err
}
// update ChunkStorage status in memory
cs.SetStatus(bnapi.ChunkStatusRelease)
cs.SetStatus(clustermgr.ChunkStatusRelease)
cs = nil
span.Infof("release chunk<%s> success", vm.ChunkId)
span.Infof("release chunk<%s> success", vm.ChunkID)
return nil
}
@ -771,7 +770,7 @@ func (ds *DiskStorage) ReleaseChunk(ctx context.Context, vuid proto.Vuid, force
* second: change status in memory
* concurrency safety: only allows serial execution for the same vuid
*/
func (ds *DiskStorage) UpdateChunkStatus(ctx context.Context, vuid proto.Vuid, status bnapi.ChunkStatus) (err error) {
func (ds *DiskStorage) UpdateChunkStatus(ctx context.Context, vuid proto.Vuid, status clustermgr.ChunkStatus) (err error) {
span := trace.SpanFromContextSafe(ctx)
if !bnapi.IsValidChunkStatus(status) {
@ -813,7 +812,7 @@ func (ds *DiskStorage) UpdateChunkStatus(ctx context.Context, vuid proto.Vuid, s
err = ds.SuperBlock.UpsertChunk(ctx, cs.ID(), *vm)
if err != nil {
span.Errorf("update chunk(%s) status to %v failed: %v", vm.ChunkId, status, err)
span.Errorf("update chunk(%s) status to %v failed: %v", vm.ChunkID, status, err)
return err
}
@ -853,7 +852,7 @@ func (ds *DiskStorage) UpdateChunkCompactState(ctx context.Context, vuid proto.V
err = ds.SuperBlock.UpsertChunk(ctx, cs.ID(), *vm)
if err != nil {
span.Errorf("update chunk(%s) status to %v failed: %v",
vm.ChunkId, compacting, err)
vm.ChunkID, compacting, err)
return err
}
@ -932,10 +931,10 @@ func (ds *DiskStorage) loopCleanChunk() {
func (ds *DiskStorage) sceneWithoutProtection(ctx context.Context, meta core.VuidMeta) bool {
span := trace.SpanFromContextSafe(ctx)
if meta.Reason != bnapi.ReleaseForCompact {
if meta.Reason != clustermgr.ReleaseForCompact {
return false
}
span.Debugf("id:%s meta:%v without protection", meta.ChunkId, meta)
span.Debugf("id:%s meta:%v without protection", meta.ChunkID, meta)
return true
}
@ -957,23 +956,23 @@ func (ds *DiskStorage) cleanReleasedChunks() (err error) {
}
for _, ck := range chunks {
if ck.Status != bnapi.ChunkStatusRelease {
if ck.Status != clustermgr.ChunkStatusRelease {
continue
}
if !ds.sceneWithoutProtection(ctx, ck) && now-ck.Mtime < int64(time.Minute*protectionPeriod) {
span.Debugf("%s still in protection period", ck.ChunkId)
span.Debugf("%s still in protection period", ck.ChunkID)
continue
}
chunkid, err := ds.SuperBlock.ReadVuidBind(ctx, ck.Vuid)
if err == nil && chunkid == ck.ChunkId {
span.Warnf("can not happen. vuid:%d bind %s. skip", ck.Vuid, ck.ChunkId)
if err == nil && chunkid == ck.ChunkID {
span.Warnf("can not happen. vuid:%d bind %s. skip", ck.Vuid, ck.ChunkID)
continue
}
if err = ds.realCleanChunk(ctx, ck.ChunkId); err != nil {
span.Errorf("failed clean chunk:%s, err:%v", ck.ChunkId, err)
if err = ds.realCleanChunk(ctx, ck.ChunkID); err != nil {
span.Errorf("failed clean chunk:%s, err:%v", ck.ChunkID, err)
continue
}
}
@ -981,7 +980,7 @@ func (ds *DiskStorage) cleanReleasedChunks() (err error) {
return
}
func (ds *DiskStorage) realCleanChunk(ctx context.Context, id bnapi.ChunkId) (err error) {
func (ds *DiskStorage) realCleanChunk(ctx context.Context, id clustermgr.ChunkID) (err error) {
span := trace.SpanFromContextSafe(ctx)
span.Warnf("will clean chunk:(%s)", id)
@ -1005,7 +1004,7 @@ func (ds *DiskStorage) realCleanChunk(ctx context.Context, id bnapi.ChunkId) (er
// Clean up the space of a chunk, including metadata and data
// NOTE: Maybe a long time
func (ds *DiskStorage) cleanChunk(ctx context.Context, id bnapi.ChunkId, toTrash bool) (err error) {
func (ds *DiskStorage) cleanChunk(ctx context.Context, id clustermgr.ChunkID, toTrash bool) (err error) {
span := trace.SpanFromContextSafe(ctx)
// clean meta
@ -1125,7 +1124,7 @@ func (ds *DiskStorage) IsWritable() bool {
return ds.Status() == proto.DiskStatusNormal
}
func isValidStateTransition(src, dest bnapi.ChunkStatus) bool {
func isValidStateTransition(src, dest clustermgr.ChunkStatus) bool {
validStates, exist := StateTransitionRules[src]
if !exist {
return false

18
blobstore/blobnode/core/disk/disk_test.go
View File

@ -227,14 +227,14 @@ func TestDiskStorage_UpdateChunkStatus(t *testing.T) {
require.NoError(t, err)
require.NotNil(t, cs)
err = ds.UpdateChunkStatus(context.TODO(), cs.Vuid(), bnapi.ChunkStatusReadOnly)
err = ds.UpdateChunkStatus(context.TODO(), cs.Vuid(), cmapi.ChunkStatusReadOnly)
require.NoError(t, err)
chunks, err := ds.ListChunks(context.TODO())
require.NoError(t, err)
require.Equal(t, 1, len(chunks))
require.Equal(t, bnapi.ChunkStatusReadOnly, chunks[0].Status)
require.Equal(t, cmapi.ChunkStatusReadOnly, chunks[0].Status)
ds = nil
cs = nil
@ -479,7 +479,7 @@ func TestCleanChunk(t *testing.T) {
require.Error(t, err)
require.Contains(t, err.Error(), "Unexpected")
err = ds.UpdateChunkStatus(ctx, vuid1, bnapi.ChunkStatusReadOnly)
err = ds.UpdateChunkStatus(ctx, vuid1, cmapi.ChunkStatusReadOnly)
require.NoError(t, err)
err = ds.ReleaseChunk(ctx, vuid1, false)
@ -536,7 +536,7 @@ func TestCheckChunkFile(t *testing.T) {
path, err := filepath.Abs(diskpath)
require.NoError(t, err)
mockChunkFileName := filepath.Join(core.GetDataPath(path), bnapi.NewChunkId(proto.Vuid(2003)).String())
mockChunkFileName := filepath.Join(core.GetDataPath(path), cmapi.NewChunkID(proto.Vuid(2003)).String())
mockChunkData, err := os.Create(mockChunkFileName)
require.NoError(t, err)
defer mockChunkData.Close()
@ -710,7 +710,7 @@ func TestDiskStorage_ReleaseChunk(t *testing.T) {
require.NoError(t, err)
require.NotNil(t, cs)
err = cs.SetStatus(bnapi.ChunkStatusRelease)
err = cs.SetStatus(cmapi.ChunkStatusRelease)
require.NoError(t, err)
err = ds.ReleaseChunk(ctx, vuid, false)
require.Error(t, err)
@ -755,15 +755,15 @@ func TestDiskStorage_UpdateChunkStatus2(t *testing.T) {
require.NoError(t, err)
require.NotNil(t, cs)
err = cs.SetStatus(bnapi.ChunkStatusRelease)
err = cs.SetStatus(cmapi.ChunkStatusRelease)
require.NoError(t, err)
err = ds.UpdateChunkStatus(ctx, vuid, bnapi.ChunkStatusRelease)
err = ds.UpdateChunkStatus(ctx, vuid, cmapi.ChunkStatusRelease)
require.Nil(t, err)
err = ds.UpdateChunkStatus(ctx, vuid, bnapi.ChunkStatusNormal)
err = ds.UpdateChunkStatus(ctx, vuid, cmapi.ChunkStatusNormal)
require.Error(t, err)
err = ds.UpdateChunkStatus(ctx, proto.Vuid(1003), bnapi.ChunkStatusReadOnly)
err = ds.UpdateChunkStatus(ctx, proto.Vuid(1003), cmapi.ChunkStatusReadOnly)
require.Error(t, err)
err = ds.UpdateChunkStatus(ctx, vuid, 6)

43
blobstore/blobnode/core/disk/superblock.go
View File

@ -23,6 +23,7 @@ import (
"strings"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/blobnode/core"
"github.com/cubefs/cubefs/blobstore/blobnode/core/storage"
"github.com/cubefs/cubefs/blobstore/blobnode/db"
@ -79,7 +80,7 @@ type SuperBlock struct {
db db.MetaHandler
}
func GenChunkKey(id bnapi.ChunkId) string {
func GenChunkKey(id clustermgr.ChunkID) string {
return fmt.Sprintf("%s/%s", _chunkSpacePrefix, id)
}
@ -137,8 +138,8 @@ func (s *SuperBlock) readData(ctx context.Context, Key []byte) (data []byte, err
return data, nil
}
func (s *SuperBlock) UpsertChunk(ctx context.Context, id bnapi.ChunkId, vm core.VuidMeta) (err error) {
if !bnapi.IsValidChunkId(id) {
func (s *SuperBlock) UpsertChunk(ctx context.Context, id clustermgr.ChunkID, vm core.VuidMeta) (err error) {
if !bnapi.IsValidChunkID(id) {
return bloberr.ErrInvalidChunkId
}
@ -151,8 +152,8 @@ func (s *SuperBlock) UpsertChunk(ctx context.Context, id bnapi.ChunkId, vm core.
return s.writeData(ctx, key, data)
}
func (s *SuperBlock) BindVuidChunk(ctx context.Context, vuid proto.Vuid, id bnapi.ChunkId) (err error) {
if !bnapi.IsValidChunkId(id) {
func (s *SuperBlock) BindVuidChunk(ctx context.Context, vuid proto.Vuid, id clustermgr.ChunkID) (err error) {
if !bnapi.IsValidChunkID(id) {
return bloberr.ErrInvalidChunkId
}
@ -162,7 +163,7 @@ func (s *SuperBlock) BindVuidChunk(ctx context.Context, vuid proto.Vuid, id bnap
return s.writeData(ctx, key, value)
}
func (s *SuperBlock) UnbindVuidChunk(ctx context.Context, vuid proto.Vuid, id bnapi.ChunkId) (err error) {
func (s *SuperBlock) UnbindVuidChunk(ctx context.Context, vuid proto.Vuid, id clustermgr.ChunkID) (err error) {
key := []byte(GenVuidSpaceKey(vuid))
err = s.db.Delete(ctx, key)
@ -173,24 +174,24 @@ func (s *SuperBlock) UnbindVuidChunk(ctx context.Context, vuid proto.Vuid, id bn
return nil
}
func (s *SuperBlock) ReadVuidBind(ctx context.Context, vuid proto.Vuid) (id bnapi.ChunkId, err error) {
func (s *SuperBlock) ReadVuidBind(ctx context.Context, vuid proto.Vuid) (id clustermgr.ChunkID, err error) {
key := []byte(GenVuidSpaceKey(vuid))
data, err := s.readData(ctx, key)
if err != nil {
return bnapi.InvalidChunkId, err
return clustermgr.InvalidChunkID, err
}
id, err = bnapi.DecodeChunk(string(data))
id, err = clustermgr.DecodeChunk(string(data))
if err != nil {
return bnapi.InvalidChunkId, err
return clustermgr.InvalidChunkID, err
}
return id, nil
}
func (s *SuperBlock) ReadChunk(ctx context.Context, id bnapi.ChunkId) (vm core.VuidMeta, err error) {
func (s *SuperBlock) ReadChunk(ctx context.Context, id clustermgr.ChunkID) (vm core.VuidMeta, err error) {
span := trace.SpanFromContextSafe(ctx)
if !bnapi.IsValidChunkId(id) {
if !bnapi.IsValidChunkID(id) {
span.Errorf("Invalid chunkid:%v", id)
return vm, bloberr.ErrInvalidChunkId
}
@ -252,13 +253,13 @@ func (s *SuperBlock) LoadDiskInfo(ctx context.Context) (dm core.DiskMeta, err er
return dm, err
}
func (s *SuperBlock) ListChunks(ctx context.Context) (chunks map[bnapi.ChunkId]core.VuidMeta, err error) {
func (s *SuperBlock) ListChunks(ctx context.Context) (chunks map[clustermgr.ChunkID]core.VuidMeta, err error) {
iter := s.db.NewIterator(ctx)
defer iter.Close()
prefix := []byte(_chunkSpacePrefix)
chunks = make(map[bnapi.ChunkId]core.VuidMeta)
chunks = make(map[clustermgr.ChunkID]core.VuidMeta)
for iter.Seek(prefix); iter.ValidForPrefix(prefix); iter.Next() {
v := iter.Value()
value := v.Data()
@ -271,7 +272,7 @@ func (s *SuperBlock) ListChunks(ctx context.Context) (chunks map[bnapi.ChunkId]c
}
v.Free()
chunks[vm.ChunkId] = vm
chunks[vm.ChunkID] = vm
}
if err = iter.Err(); err != nil {
@ -281,7 +282,7 @@ func (s *SuperBlock) ListChunks(ctx context.Context) (chunks map[bnapi.ChunkId]c
return chunks, nil
}
func (s *SuperBlock) ListVuids(ctx context.Context) (vuids map[proto.Vuid]bnapi.ChunkId, err error) {
func (s *SuperBlock) ListVuids(ctx context.Context) (vuids map[proto.Vuid]clustermgr.ChunkID, err error) {
span := trace.SpanFromContextSafe(ctx)
iter := s.db.NewIterator(ctx)
@ -289,7 +290,7 @@ func (s *SuperBlock) ListVuids(ctx context.Context) (vuids map[proto.Vuid]bnapi.
prefix := []byte(_vuidSpacePrefix)
vuids = make(map[proto.Vuid]bnapi.ChunkId)
vuids = make(map[proto.Vuid]clustermgr.ChunkID)
for iter.Seek(prefix); iter.ValidForPrefix(prefix); iter.Next() {
k, v := iter.Key(), iter.Value()
key, value := k.Data(), v.Data()
@ -302,7 +303,7 @@ func (s *SuperBlock) ListVuids(ctx context.Context) (vuids map[proto.Vuid]bnapi.
return nil, err
}
chunkid, err := bnapi.DecodeChunk(string(value))
chunkid, err := clustermgr.DecodeChunk(string(value))
if err != nil {
k.Free()
v.Free()
@ -322,7 +323,7 @@ func (s *SuperBlock) ListVuids(ctx context.Context) (vuids map[proto.Vuid]bnapi.
return vuids, nil
}
func (s *SuperBlock) CleanChunkSpace(ctx context.Context, id bnapi.ChunkId) (err error) {
func (s *SuperBlock) CleanChunkSpace(ctx context.Context, id clustermgr.ChunkID) (err error) {
span := trace.SpanFromContextSafe(ctx)
minKeyID := &core.ShardKey{
@ -348,10 +349,10 @@ func (s *SuperBlock) CleanChunkSpace(ctx context.Context, id bnapi.ChunkId) (err
return nil
}
func (s *SuperBlock) DeleteChunk(ctx context.Context, id bnapi.ChunkId) (err error) {
func (s *SuperBlock) DeleteChunk(ctx context.Context, id clustermgr.ChunkID) (err error) {
span := trace.SpanFromContextSafe(ctx)
if !bnapi.IsValidChunkId(id) {
if !bnapi.IsValidChunkID(id) {
return bloberr.ErrInvalidChunkId
}

31
blobstore/blobnode/core/disk/superblock_test.go
View File

@ -21,6 +21,7 @@ import (
"testing"
"time"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/stretchr/testify/require"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
@ -54,10 +55,10 @@ func TestNewSuperBlock(t *testing.T) {
// add chunk
vuid := proto.Vuid(1024)
diskid := proto.DiskID(10)
chunkid := bnapi.NewChunkId(vuid)
chunkid := clustermgr.NewChunkID(vuid)
vm := core.VuidMeta{
Vuid: vuid,
ChunkId: chunkid,
ChunkID: chunkid,
DiskID: diskid,
}
@ -72,13 +73,13 @@ func TestNewSuperBlock(t *testing.T) {
require.Equal(t, vm, vm_read)
_, err = s.ReadChunk(ctx, bnapi.InvalidChunkId)
_, err = s.ReadChunk(ctx, clustermgr.InvalidChunkID)
require.Error(t, err)
err = s.UpsertDisk(ctx, proto.InvalidDiskID, core.DiskMeta{})
require.Error(t, err)
err = s.DeleteChunk(ctx, bnapi.InvalidChunkId)
err = s.DeleteChunk(ctx, clustermgr.InvalidChunkID)
require.Error(t, err)
_, _ = s.ReadVuidBind(ctx, vuid)
@ -158,10 +159,10 @@ func TestSuperBlock_ListChunks(t *testing.T) {
for i := 0; i < 10; i++ {
vuid := 1024 + i
diskid := proto.DiskID(10)
chunkid := bnapi.NewChunkId(proto.Vuid(vuid))
chunkid := clustermgr.NewChunkID(proto.Vuid(vuid))
vm := core.VuidMeta{
Vuid: proto.Vuid(vuid),
ChunkId: chunkid,
ChunkID: chunkid,
DiskID: diskid,
}
err = s.UpsertChunk(ctx, chunkid, vm)
@ -172,10 +173,10 @@ func TestSuperBlock_ListChunks(t *testing.T) {
require.NoError(t, err)
require.Equal(t, 10, len(chunks))
err = s.CleanChunkSpace(ctx, bnapi.NewChunkId(proto.Vuid(1024)))
err = s.CleanChunkSpace(ctx, clustermgr.NewChunkID(proto.Vuid(1024)))
require.NoError(t, err)
err = s.DeleteChunk(ctx, bnapi.NewChunkId(proto.Vuid(1025)))
err = s.DeleteChunk(ctx, clustermgr.NewChunkID(proto.Vuid(1025)))
require.NoError(t, err)
}
@ -200,7 +201,7 @@ func TestSuperBlock_ListVuids(t *testing.T) {
// create chunk 0
for i := 0; i < 10; i++ {
vuid := 1024 + i
chunkid := bnapi.NewChunkId(proto.Vuid(vuid))
chunkid := clustermgr.NewChunkID(proto.Vuid(vuid))
err = s.BindVuidChunk(ctx, proto.Vuid(vuid), chunkid)
require.NoError(t, err)
}
@ -255,15 +256,15 @@ func TestSuperblockErrorCondition(t *testing.T) {
vuid := proto.Vuid(1023)
diskid := proto.DiskID(1)
chunkid := bnapi.NewChunkId(vuid)
chunkid := clustermgr.NewChunkID(vuid)
vm := core.VuidMeta{
Vuid: vuid,
ChunkId: chunkid,
ChunkID: chunkid,
DiskID: diskid,
}
var InvalidChunkID bnapi.ChunkId = [16]byte{}
// upsert invalid ChunkId
var InvalidChunkID clustermgr.ChunkID = [16]byte{}
// upsert invalid ChunkID
err = s.UpsertChunk(ctx, InvalidChunkID, vm)
require.Error(t, err)
@ -291,10 +292,10 @@ func TestCleanChunkSpace(t *testing.T) {
// create chunk meta
vuid := 1024
diskid := proto.DiskID(10)
chunkid := bnapi.NewChunkId(proto.Vuid(vuid))
chunkid := clustermgr.NewChunkID(proto.Vuid(vuid))
vm := core.VuidMeta{
Vuid: proto.Vuid(vuid),
ChunkId: chunkid,
ChunkID: chunkid,
DiskID: diskid,
}

49
blobstore/blobnode/core/proto.go
View File

@ -19,6 +19,7 @@ import (
"io"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/blobnode/base/qos"
"github.com/cubefs/cubefs/blobstore/blobnode/db"
"github.com/cubefs/cubefs/blobstore/common/proto"
@ -30,17 +31,17 @@ const (
// chunk meta data for kv db
type VuidMeta struct {
Version uint8 `json:"version"`
Vuid proto.Vuid `json:"vuid"`
DiskID proto.DiskID `json:"diskid"`
ChunkId bnapi.ChunkId `json:"chunkname"`
ParentChunk bnapi.ChunkId `json:"parentchunk"`
ChunkSize int64 `json:"chunksize"`
Ctime int64 `json:"ctime"` // nsec
Mtime int64 `json:"mtime"` // nsec
Compacting bool `json:"compacting"`
Status bnapi.ChunkStatus `json:"status"` // normal、release
Reason string `json:"reason"`
Version uint8 `json:"version"`
Vuid proto.Vuid `json:"vuid"`
DiskID proto.DiskID `json:"diskid"`
ChunkID clustermgr.ChunkID `json:"chunkname"`
ParentChunk clustermgr.ChunkID `json:"parentchunk"`
ChunkSize int64 `json:"chunksize"`
Ctime int64 `json:"ctime"` // nsec
Mtime int64 `json:"mtime"` // nsec
Compacting bool `json:"compacting"`
Status clustermgr.ChunkStatus `json:"status"` // normal、release
Reason string `json:"reason"`
}
// disk meta data for rocksdb
@ -61,14 +62,14 @@ type DiskStats struct {
}
type StorageStat struct {
FileSize int64 `json:"file_size"`
PhySize int64 `json:"phy_size"`
ParentID bnapi.ChunkId `json:"parent_id"`
CreateTime int64 `json:"create_time"`
FileSize int64 `json:"file_size"`
PhySize int64 `json:"phy_size"`
ParentID clustermgr.ChunkID `json:"parent_id"`
CreateTime int64 `json:"create_time"`
}
type MetaHandler interface {
ID() bnapi.ChunkId
ID() clustermgr.ChunkID
InnerDB() db.MetaHandler
SupportInline() bool
Write(ctx context.Context, bid proto.BlobID, value ShardMeta) (err error)
@ -91,7 +92,7 @@ type DataHandler interface {
}
type Storage interface {
ID() bnapi.ChunkId
ID() clustermgr.ChunkID
MetaHandler() MetaHandler
DataHandler() DataHandler
RawStorage() Storage
@ -116,12 +117,12 @@ type Storage interface {
// chunk storage api
type ChunkAPI interface {
// infos
ID() bnapi.ChunkId
ID() clustermgr.ChunkID
Vuid() proto.Vuid
Disk() (disk DiskAPI)
Status() bnapi.ChunkStatus
Status() clustermgr.ChunkStatus
VuidMeta() (vm *VuidMeta)
ChunkInfo(ctx context.Context) (info bnapi.ChunkInfo)
ChunkInfo(ctx context.Context) (info clustermgr.ChunkInfo)
// method
Write(ctx context.Context, b *Shard) (err error)
@ -145,14 +146,14 @@ type ChunkAPI interface {
AllowModify() (err error)
HasEnoughSpace(needSize int64) bool
HasPendingRequest() bool
SetStatus(status bnapi.ChunkStatus) (err error)
SetStatus(status clustermgr.ChunkStatus) (err error)
SetDirty(dirty bool)
}
type DiskAPI interface {
ID() proto.DiskID
Status() (status proto.DiskStatus)
DiskInfo() (info bnapi.DiskInfo)
DiskInfo() (info clustermgr.BlobNodeDiskInfo)
Stats() (stat DiskStats)
GetChunkStorage(vuid proto.Vuid) (cs ChunkAPI, found bool)
GetConfig() (config *Config)
@ -164,11 +165,11 @@ type DiskAPI interface {
UpdateDiskStatus(ctx context.Context, status proto.DiskStatus) (err error)
CreateChunk(ctx context.Context, vuid proto.Vuid, chunksize int64) (cs ChunkAPI, err error)
ReleaseChunk(ctx context.Context, vuid proto.Vuid, force bool) (err error)
UpdateChunkStatus(ctx context.Context, vuid proto.Vuid, status bnapi.ChunkStatus) (err error)
UpdateChunkStatus(ctx context.Context, vuid proto.Vuid, status clustermgr.ChunkStatus) (err error)
UpdateChunkCompactState(ctx context.Context, vuid proto.Vuid, compacting bool) (err error)
ListChunks(ctx context.Context) (chunks []VuidMeta, err error)
EnqueueCompact(ctx context.Context, vuid proto.Vuid)
GcRubbishChunk(ctx context.Context) (mayBeLost []bnapi.ChunkId, err error)
GcRubbishChunk(ctx context.Context) (mayBeLost []clustermgr.ChunkID, err error)
WalkChunksWithLock(ctx context.Context, fn func(cs ChunkAPI) error) (err error)
ResetChunks(ctx context.Context)
IsCleanUp(ctx context.Context) bool

5
blobstore/blobnode/core/shard.go
View File

@ -23,6 +23,7 @@ import (
"io"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/common/crc32block"
"github.com/cubefs/cubefs/blobstore/common/proto"
)
@ -124,8 +125,8 @@ const (
// meta db key
type ShardKey struct {
Chunk bnapi.ChunkId `json:"chunk"`
Bid proto.BlobID `json:"bid"`
Chunk clustermgr.ChunkID `json:"chunk"`
Bid proto.BlobID `json:"bid"`
}
// meta db value

11
blobstore/blobnode/core/storage/datafile.go
View File

@ -28,6 +28,7 @@ import (
"time"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
bncomm "github.com/cubefs/cubefs/blobstore/blobnode/base"
"github.com/cubefs/cubefs/blobstore/blobnode/base/qos"
"github.com/cubefs/cubefs/blobstore/blobnode/core"
@ -60,7 +61,7 @@ const (
_chunkHeaderSize = 4 * 1024
_chunkMagicSize = 4
_chunkVerSize = 1
_chunkParentChunkSize = bnapi.ChunkIdLength
_chunkParentChunkSize = clustermgr.ChunkIDLength
_chunkCreateTimeSize = 8
//_chunkPaddingSize = _chunkHeaderSize - _chunkMagicSize - _chunkVerSize - _chunkParentChunkSize - _chunkCreateTimeSize
@ -94,7 +95,7 @@ var (
type ChunkHeader struct {
magic [_chunkMagicSize]byte
version byte
parentChunk bnapi.ChunkId
parentChunk clustermgr.ChunkID
createTime int64
}
@ -104,7 +105,7 @@ type datafile struct {
wLock sync.RWMutex
File string
chunk bnapi.ChunkId
chunk clustermgr.ChunkID
header ChunkHeader
conf *core.Config
@ -171,11 +172,11 @@ func NewChunkData(ctx context.Context, vm core.VuidMeta, file string, conf *core
conf.HandleIOError(context.Background(), vm.DiskID, err)
}
ef := core.NewBlobFile(fd, handleIOError, uint64(vm.ChunkId.VolumeUnitId()), ioPools)
ef := core.NewBlobFile(fd, handleIOError, uint64(vm.ChunkID.VolumeUnitId()), ioPools)
cd = &datafile{
File: file,
chunk: vm.ChunkId,
chunk: vm.ChunkID,
conf: conf,
closed: false,
ef: ef,

25
blobstore/blobnode/core/storage/datafile_test.go
View File

@ -30,6 +30,7 @@ import (
"github.com/stretchr/testify/require"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/blobnode/base/qos"
"github.com/cubefs/cubefs/blobstore/blobnode/core"
"github.com/cubefs/cubefs/blobstore/common/crc32block"
@ -64,7 +65,7 @@ func TestNewChunkData(t *testing.T) {
defer os.RemoveAll(testDir)
conf := &core.Config{}
chunkid := bnapi.NewChunkId(0)
chunkid := clustermgr.NewChunkID(0)
chunkname := chunkid.String()
chunkname = filepath.Join(testDir, chunkname)
@ -80,7 +81,7 @@ func TestNewChunkData(t *testing.T) {
ioPools := newIoPoolMock(t)
// case: format data when first creating chunkdata
cd, err := NewChunkData(ctx, core.VuidMeta{ChunkId: chunkid, DiskID: 1, Version: 2, Ctime: 3}, chunkname, conf, true, nil, ioPools)
cd, err := NewChunkData(ctx, core.VuidMeta{ChunkID: chunkid, DiskID: 1, Version: 2, Ctime: 3}, chunkname, conf, true, nil, ioPools)
require.NoError(t, err)
require.NotNil(t, cd)
defer cd.Close()
@ -117,7 +118,7 @@ func TestChunkData_Write(t *testing.T) {
ctx := context.Background()
chunkname := bnapi.NewChunkId(0).String()
chunkname := clustermgr.NewChunkID(0).String()
chunkname = filepath.Join(testDir, chunkname)
log.Info(chunkname)
@ -402,7 +403,7 @@ func TestChunkData_ConcurrencyWrite(t *testing.T) {
ctx := context.Background()
chunkname := bnapi.NewChunkId(0).String()
chunkname := clustermgr.NewChunkID(0).String()
chunkname = filepath.Join(testDir, chunkname)
log.Info(chunkname)
@ -492,7 +493,7 @@ func TestChunkData_ConcurrencyWriteRead(t *testing.T) {
ctx := context.Background()
chunkname := bnapi.NewChunkId(0).String()
chunkname := clustermgr.NewChunkID(0).String()
chunkname = filepath.Join(testDir, chunkname)
log.Info(chunkname)
@ -607,7 +608,7 @@ func TestChunkData_ReadWrite(t *testing.T) {
ctx := context.Background()
chunkname := bnapi.NewChunkId(0).String()
chunkname := clustermgr.NewChunkID(0).String()
chunkname = filepath.Join(testDir, chunkname)
log.Info(chunkname)
@ -683,7 +684,7 @@ func TestChunkData_Delete(t *testing.T) {
ctx := context.Background()
chunkname := bnapi.NewChunkId(0).String()
chunkname := clustermgr.NewChunkID(0).String()
chunkname = filepath.Join(testDir, chunkname)
log.Info(chunkname)
@ -817,7 +818,7 @@ func TestChunkData_Destroy(t *testing.T) {
ctx := context.Background()
chunkname := bnapi.NewChunkId(0).String()
chunkname := clustermgr.NewChunkID(0).String()
chunkname = filepath.Join(testDir, chunkname)
log.Info(chunkname)
@ -863,7 +864,7 @@ func TestParseMeta(t *testing.T) {
ctx := context.Background()
chunkname := bnapi.NewChunkId(0).String()
chunkname := clustermgr.NewChunkID(0).String()
chunkname = filepath.Join(testDir, chunkname)
log.Info(chunkname)
@ -876,7 +877,7 @@ func TestParseMeta(t *testing.T) {
ctime := time.Now().UnixNano()
meta := core.VuidMeta{
Version: 0x1,
ParentChunk: bnapi.ChunkId{0x8},
ParentChunk: clustermgr.ChunkID{0x8},
Ctime: ctime,
}
@ -895,7 +896,7 @@ func TestParseMeta(t *testing.T) {
require.Equal(t, cd.header, cd1.header)
require.Equal(t, cd1.header.magic, chunkHeaderMagic)
require.Equal(t, cd1.header.version, uint8(0x1))
require.Equal(t, cd1.header.parentChunk, bnapi.ChunkId{0x8})
require.Equal(t, cd1.header.parentChunk, clustermgr.ChunkID{0x8})
require.Equal(t, cd1.header.createTime, ctime)
// scene 2
@ -923,7 +924,7 @@ func TestParseMeta(t *testing.T) {
func TestChunkHeader(t *testing.T) {
magic := chunkHeaderMagic
version := byte(0x2)
parent := bnapi.ChunkId{0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0x0}
parent := clustermgr.ChunkID{0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0x0}
createTime := time.Now().UnixNano()
hdr := ChunkHeader{

17
blobstore/blobnode/core/storage/metadb.go
View File

@ -23,6 +23,7 @@ import (
"time"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/blobnode/core"
"github.com/cubefs/cubefs/blobstore/blobnode/db"
bloberr "github.com/cubefs/cubefs/blobstore/common/errors"
@ -45,16 +46,16 @@ var (
var (
shardSpacePrefixLen = len(_chunkShardSpacePrefix)
shardChunkCommonLen = shardSpacePrefixLen + bnapi.ChunkIdLength
shardChunkCommonLen = shardSpacePrefixLen + clustermgr.ChunkIDLength
shardKeyLen = shardChunkCommonLen + 8
)
type metafile struct {
lock sync.RWMutex
db db.MetaHandler // meta kv db
id bnapi.ChunkId // chunk id
shardkeyPool sync.Pool // shard key pool
supportInline bool //
db db.MetaHandler // meta kv db
id clustermgr.ChunkID // chunk id
shardkeyPool sync.Pool // shard key pool
supportInline bool //
closed bool
}
@ -80,7 +81,7 @@ func GenShardKey(id *core.ShardKey) []byte {
return buf
}
func GenChunkCommonKey(id bnapi.ChunkId) []byte {
func GenChunkCommonKey(id clustermgr.ChunkID) []byte {
buf := make([]byte, shardChunkCommonLen)
copy(buf[0:shardSpacePrefixLen], []byte(_chunkShardSpacePrefix))
@ -177,7 +178,7 @@ func (cm *metafile) batchDeleteKey(ctx context.Context, keyPrefix []byte) (err e
return
}
func (cm *metafile) ID() bnapi.ChunkId {
func (cm *metafile) ID() clustermgr.ChunkID {
return cm.id
}
@ -353,7 +354,7 @@ func NewChunkMeta(ctx context.Context, config *core.Config, meta core.VuidMeta,
}
cm = &metafile{
id: meta.ChunkId,
id: meta.ChunkID,
db: db,
supportInline: config.SupportInline,
shardkeyPool: sync.Pool{

19
blobstore/blobnode/core/storage/metadb_test.go
View File

@ -20,6 +20,7 @@ import (
"path/filepath"
"testing"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/stretchr/testify/require"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
@ -47,10 +48,10 @@ func TestNewChunkMeta(t *testing.T) {
// create chunk meta
vuid := 1024
diskid := proto.DiskID(10)
chunkid := bnapi.NewChunkId(proto.Vuid(vuid))
chunkid := clustermgr.NewChunkID(proto.Vuid(vuid))
vm := core.VuidMeta{
Vuid: proto.Vuid(vuid),
ChunkId: chunkid,
ChunkID: chunkid,
DiskID: diskid,
}
@ -80,10 +81,10 @@ func TestChunkMeta_Write(t *testing.T) {
// create chunk meta
vuid := 1024
diskid := proto.DiskID(10)
chunkid := bnapi.NewChunkId(proto.Vuid(vuid))
chunkid := clustermgr.NewChunkID(proto.Vuid(vuid))
vm := core.VuidMeta{
Vuid: proto.Vuid(vuid),
ChunkId: chunkid,
ChunkID: chunkid,
DiskID: diskid,
}
@ -137,10 +138,10 @@ func TestChunkMeta_Scan(t *testing.T) {
// create chunk meta
vuid := 1024
diskid := proto.DiskID(10)
chunkid := bnapi.NewChunkId(proto.Vuid(vuid))
chunkid := clustermgr.NewChunkID(proto.Vuid(vuid))
vm := core.VuidMeta{
Vuid: proto.Vuid(vuid),
ChunkId: chunkid,
ChunkID: chunkid,
DiskID: diskid,
}
@ -212,7 +213,7 @@ func TestChunkMeta_Scan(t *testing.T) {
}
func TestGenShardKey(t *testing.T) {
chunkid := bnapi.ChunkId{12}
chunkid := clustermgr.ChunkID{12}
bid := proto.BlobID(12)
id := core.ShardKey{
@ -249,10 +250,10 @@ func TestChunkMeta_Destroy(t *testing.T) {
// create chunk meta
vuid := 1024
diskid := proto.DiskID(10)
chunkid := bnapi.NewChunkId(proto.Vuid(vuid))
chunkid := clustermgr.NewChunkID(proto.Vuid(vuid))
vm := core.VuidMeta{
Vuid: proto.Vuid(vuid),
ChunkId: chunkid,
ChunkID: chunkid,
DiskID: diskid,
}

3
blobstore/blobnode/core/storage/storage.go
View File

@ -22,6 +22,7 @@ import (
"sync/atomic"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/blobnode/core"
bloberr "github.com/cubefs/cubefs/blobstore/common/errors"
"github.com/cubefs/cubefs/blobstore/common/proto"
@ -58,7 +59,7 @@ func (stg *storage) PendingRequest() int64 {
return atomic.LoadInt64(&stg.pendingCnt)
}
func (stg *storage) ID() bnapi.ChunkId {
func (stg *storage) ID() clustermgr.ChunkID {
return stg.meta.ID()
}

4
blobstore/blobnode/core/storage/storage_cp.go
View File

@ -20,7 +20,7 @@ import (
"sync"
"time"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/blobnode/core"
bloberr "github.com/cubefs/cubefs/blobstore/common/errors"
"github.com/cubefs/cubefs/blobstore/common/proto"
@ -61,7 +61,7 @@ func (stg *replicateStorage) PendingRequest() int64 {
return stg.masterStg.PendingRequest() + stg.slaveStg.PendingRequest()
}
func (stg *replicateStorage) ID() bnapi.ChunkId {
func (stg *replicateStorage) ID() clustermgr.ChunkID {
return stg.masterStg.ID()
}

10
blobstore/blobnode/core/storage/storage_cp_test.go
View File

@ -19,9 +19,9 @@ import (
"io"
"testing"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/stretchr/testify/require"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/blobnode/core"
"github.com/cubefs/cubefs/blobstore/blobnode/db"
bloberr "github.com/cubefs/cubefs/blobstore/common/errors"
@ -29,12 +29,12 @@ import (
)
type mockBrokenMeta struct {
id bnapi.ChunkId
id clustermgr.ChunkID
}
type mockBrokenData struct{}
func (mm *mockBrokenMeta) ID() bnapi.ChunkId {
func (mm *mockBrokenMeta) ID() clustermgr.ChunkID {
return mm.id
}
@ -115,13 +115,13 @@ func (mm *mockBrokenData) Close() {
func TestReplStorage_Operations(t *testing.T) {
stg := NewStorage(&mockmeta{
id: bnapi.ChunkId{0x1},
id: clustermgr.ChunkID{0x1},
bids: map[proto.BlobID]core.ShardMeta{},
}, &mockdata{})
require.NotNil(t, stg)
stg1 := NewStorage(&mockmeta{
id: bnapi.ChunkId{0x1},
id: clustermgr.ChunkID{0x1},
bids: map[proto.BlobID]core.ShardMeta{},
}, &mockdata{})
require.NotNil(t, stg)

13
blobstore/blobnode/core/storage/storage_test.go
View File

@ -20,6 +20,7 @@ import (
"io"
"testing"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/stretchr/testify/require"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
@ -30,7 +31,7 @@ import (
)
type mockmeta struct {
id bnapi.ChunkId
id clustermgr.ChunkID
bids map[proto.BlobID]core.ShardMeta
supportInline bool
}
@ -70,7 +71,7 @@ var (
}
)
func (mm *mockmeta) ID() bnapi.ChunkId {
func (mm *mockmeta) ID() clustermgr.ChunkID {
return mm.id
}
@ -156,7 +157,7 @@ func TestNewStorage(t *testing.T) {
func TestStorage_Operations(t *testing.T) {
stg := NewStorage(&mockmeta{
id: bnapi.ChunkId{0x1},
id: clustermgr.ChunkID{0x1},
bids: map[proto.BlobID]core.ShardMeta{},
}, &mockdata{})
require.NotNil(t, stg)
@ -167,7 +168,7 @@ func TestStorage_Operations(t *testing.T) {
stg.DecrPendingCnt()
require.Equal(t, 0, int(stg.PendingRequest()))
require.Equal(t, bnapi.ChunkId{0x1}, stg.ID())
require.Equal(t, clustermgr.ChunkID{0x1}, stg.ID())
require.NotNil(t, stg.MetaHandler())
require.NotNil(t, stg.DataHandler())
@ -233,7 +234,7 @@ func TestStorage_Operations(t *testing.T) {
func TestStorage_ShardInline(t *testing.T) {
stg := NewStorage(&mockmeta{
id: bnapi.ChunkId{0x1},
id: clustermgr.ChunkID{0x1},
bids: map[proto.BlobID]core.ShardMeta{},
supportInline: true,
}, &mockdata{})
@ -242,7 +243,7 @@ func TestStorage_ShardInline(t *testing.T) {
stg = NewTinyFileStg(stg, 8)
require.NotNil(t, stg)
require.Equal(t, bnapi.ChunkId{0x1}, stg.ID())
require.Equal(t, clustermgr.ChunkID{0x1}, stg.ID())
require.NotNil(t, stg.MetaHandler())
require.NotNil(t, stg.DataHandler())

3
blobstore/blobnode/datainspect.go
View File

@ -8,6 +8,7 @@ import (
"sync"
"time"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/prometheus/client_golang/prometheus"
"golang.org/x/time/rate"
@ -126,7 +127,7 @@ func (mgr *DataInspectMgr) inspectDisk(ctx context.Context, ds core.DiskAPI, wg
}
for _, chunk := range chunks {
if chunk.Status == bnapi.ChunkStatusRelease {
if chunk.Status == clustermgr.ChunkStatusRelease {
continue
}
cs, found := ds.GetChunkStorage(chunk.Vuid)

7
blobstore/blobnode/datainspect_test.go
View File

@ -7,6 +7,7 @@ import (
"sync"
"testing"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/require"
@ -60,7 +61,7 @@ func TestDataInspect(t *testing.T) {
cs := NewMockChunkAPI(ctr)
cs.EXPECT().Vuid().Times(2).Return(proto.Vuid(1001))
cs.EXPECT().ID().Times(2).Return(bnapi.ChunkId{})
cs.EXPECT().ID().Times(2).Return(clustermgr.ChunkID{})
cs.EXPECT().Disk().Return(ds1)
cs.EXPECT().Read(any, any).Return(int64(0), nil)
cs.EXPECT().ListShards(any, any, any, any).Return([]*bnapi.ShardInfo{{Bid: 123456, Size: 1}}, proto.BlobID(123456), nil)
@ -74,7 +75,7 @@ func TestDataInspect(t *testing.T) {
{
cs := NewMockChunkAPI(ctr)
cs.EXPECT().Vuid().Times(2).Return(proto.Vuid(1001))
cs.EXPECT().ID().Times(2).Return(bnapi.ChunkId{})
cs.EXPECT().ID().Times(2).Return(clustermgr.ChunkID{})
cs.EXPECT().Disk().Return(ds1)
ds1.EXPECT().ID().Times(1).Return(proto.DiskID(11))
@ -95,7 +96,7 @@ func TestDataInspect(t *testing.T) {
{
cs := NewMockChunkAPI(ctr)
cs.EXPECT().Vuid().Return(proto.Vuid(1001)).AnyTimes()
cs.EXPECT().ID().Times(2).Return(bnapi.ChunkId{})
cs.EXPECT().ID().Times(2).Return(clustermgr.ChunkID{})
cs.EXPECT().Disk().Return(ds1)
cs.EXPECT().Read(any, any).Return(int64(0), nil)
cs.EXPECT().ListShards(any, any, any, any).Return([]*bnapi.ShardInfo{{Bid: 123456, Size: 8}}, proto.BlobID(123456+1), nil)

30
blobstore/blobnode/disk_mock_test.go
View File

@ -10,7 +10,7 @@ import (
reflect "reflect"
blobnode "github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
clustermgr "github.com/cubefs/cubefs/blobstore/api/clustermgr"
qos "github.com/cubefs/cubefs/blobstore/blobnode/base/qos"
core "github.com/cubefs/cubefs/blobstore/blobnode/core"
proto "github.com/cubefs/cubefs/blobstore/common/proto"
@ -68,10 +68,10 @@ func (mr *MockDiskAPIMockRecorder) CreateChunk(arg0, arg1, arg2 interface{}) *go
}
// DiskInfo mocks base method.
func (m *MockDiskAPI) DiskInfo() clustermgr.DiskInfo {
func (m *MockDiskAPI) DiskInfo() clustermgr.BlobNodeDiskInfo {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "DiskInfo")
ret0, _ := ret[0].(clustermgr.DiskInfo)
ret0, _ := ret[0].(clustermgr.BlobNodeDiskInfo)
return ret0
}
@ -94,10 +94,10 @@ func (mr *MockDiskAPIMockRecorder) EnqueueCompact(arg0, arg1 interface{}) *gomoc
}
// GcRubbishChunk mocks base method.
func (m *MockDiskAPI) GcRubbishChunk(arg0 context.Context) ([]blobnode.ChunkId, error) {
func (m *MockDiskAPI) GcRubbishChunk(arg0 context.Context) ([]clustermgr.ChunkID, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GcRubbishChunk", arg0)
ret0, _ := ret[0].([]blobnode.ChunkId)
ret0, _ := ret[0].([]clustermgr.ChunkID)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@ -332,7 +332,7 @@ func (mr *MockDiskAPIMockRecorder) UpdateChunkCompactState(arg0, arg1, arg2 inte
}
// UpdateChunkStatus mocks base method.
func (m *MockDiskAPI) UpdateChunkStatus(arg0 context.Context, arg1 proto.Vuid, arg2 blobnode.ChunkStatus) error {
func (m *MockDiskAPI) UpdateChunkStatus(arg0 context.Context, arg1 proto.Vuid, arg2 clustermgr.ChunkStatus) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "UpdateChunkStatus", arg0, arg1, arg2)
ret0, _ := ret[0].(error)
@ -462,10 +462,10 @@ func (mr *MockStorageMockRecorder) Destroy(arg0 interface{}) *gomock.Call {
}
// ID mocks base method.
func (m *MockStorage) ID() blobnode.ChunkId {
func (m *MockStorage) ID() clustermgr.ChunkID {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ID")
ret0, _ := ret[0].(blobnode.ChunkId)
ret0, _ := ret[0].(clustermgr.ChunkID)
return ret0
}
@ -696,10 +696,10 @@ func (mr *MockChunkAPIMockRecorder) AllowModify() *gomock.Call {
}
// ChunkInfo mocks base method.
func (m *MockChunkAPI) ChunkInfo(arg0 context.Context) blobnode.ChunkInfo {
func (m *MockChunkAPI) ChunkInfo(arg0 context.Context) clustermgr.ChunkInfo {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ChunkInfo", arg0)
ret0, _ := ret[0].(blobnode.ChunkInfo)
ret0, _ := ret[0].(clustermgr.ChunkInfo)
return ret0
}
@ -792,10 +792,10 @@ func (mr *MockChunkAPIMockRecorder) HasPendingRequest() *gomock.Call {
}
// ID mocks base method.
func (m *MockChunkAPI) ID() blobnode.ChunkId {
func (m *MockChunkAPI) ID() clustermgr.ChunkID {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ID")
ret0, _ := ret[0].(blobnode.ChunkId)
ret0, _ := ret[0].(clustermgr.ChunkID)
return ret0
}
@ -935,7 +935,7 @@ func (mr *MockChunkAPIMockRecorder) SetDirty(arg0 interface{}) *gomock.Call {
}
// SetStatus mocks base method.
func (m *MockChunkAPI) SetStatus(arg0 blobnode.ChunkStatus) error {
func (m *MockChunkAPI) SetStatus(arg0 clustermgr.ChunkStatus) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "SetStatus", arg0)
ret0, _ := ret[0].(error)
@ -964,10 +964,10 @@ func (mr *MockChunkAPIMockRecorder) StartCompact(arg0 interface{}) *gomock.Call
}
// Status mocks base method.
func (m *MockChunkAPI) Status() blobnode.ChunkStatus {
func (m *MockChunkAPI) Status() clustermgr.ChunkStatus {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Status")
ret0, _ := ret[0].(blobnode.ChunkStatus)
ret0, _ := ret[0].(clustermgr.ChunkStatus)
return ret0
}

3
blobstore/blobnode/heartbeat.go
View File

@ -18,7 +18,6 @@ import (
"context"
"time"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/blobnode/base"
bloberr "github.com/cubefs/cubefs/blobstore/common/errors"
@ -49,7 +48,7 @@ func (s *Service) heartbeatToClusterMgr() {
disks := s.copyDiskStorages(ctx)
dis := make([]*bnapi.DiskHeartBeatInfo, 0)
dis := make([]*cmapi.DiskHeartBeatInfo, 0)
for _, ds := range disks {
if ds.Status() != proto.DiskStatusNormal {
continue

5
blobstore/blobnode/shard_test.go
View File

@ -26,6 +26,7 @@ import (
"sync"
"testing"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/stretchr/testify/require"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
@ -218,12 +219,12 @@ func TestShardPutAndGet(t *testing.T) {
putShardArg.Body = bytes.NewReader(shardData)
putShardArg.Body = bytes.NewReader(shardData)
cs.SetStatus(bnapi.ChunkStatusRelease)
cs.SetStatus(clustermgr.ChunkStatusRelease)
_, err = client.PutShard(ctx, host, putShardArg)
require.Error(t, err)
putShardArg.Body = bytes.NewReader(shardData)
cs.SetStatus(bnapi.ChunkStatusNormal)
cs.SetStatus(clustermgr.ChunkStatusNormal)
_, _ = client.PutShard(ctx, host, putShardArg)
}

23
blobstore/blobnode/startup.go
View File

@ -22,7 +22,6 @@ import (
"sync/atomic"
"time"
bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/blobnode/base/flow"
"github.com/cubefs/cubefs/blobstore/blobnode/core"
@ -68,8 +67,8 @@ func readFormatInfo(ctx context.Context, diskRootPath string) (
return formatInfo, err
}
func findDisk(disks []*bnapi.DiskInfo, clusterID proto.ClusterID, diskID proto.DiskID) (
*bnapi.DiskInfo, bool,
func findDisk(disks []*cmapi.BlobNodeDiskInfo, clusterID proto.ClusterID, diskID proto.DiskID) (
*cmapi.BlobNodeDiskInfo, bool,
) {
for _, d := range disks {
if d.ClusterID == clusterID && d.DiskID == diskID {
@ -79,7 +78,7 @@ func findDisk(disks []*bnapi.DiskInfo, clusterID proto.ClusterID, diskID proto.D
return nil, false
}
func isAllInConfig(ctx context.Context, registeredDisks []*bnapi.DiskInfo, conf *Config) bool {
func isAllInConfig(ctx context.Context, registeredDisks []*cmapi.BlobNodeDiskInfo, conf *Config) bool {
span := trace.SpanFromContextSafe(ctx)
configDiskMap := make(map[string]struct{})
for i := range conf.Disks {
@ -470,13 +469,15 @@ func registerNode(ctx context.Context, clusterMgrCli *cmapi.Client, conf *Config
return err
}
nodeToCm := bnapi.NodeInfo{
ClusterID: conf.ClusterID,
DiskType: conf.DiskType,
Idc: conf.IDC,
Rack: conf.Rack,
Host: conf.Host,
Role: proto.NodeRoleBlobNode,
nodeToCm := cmapi.BlobNodeInfo{
NodeInfo: cmapi.NodeInfo{
ClusterID: conf.ClusterID,
DiskType: conf.DiskType,
Idc: conf.IDC,
Rack: conf.Rack,
Host: conf.Host,
Role: proto.NodeRoleBlobNode,
},
}
nodeID, err := clusterMgrCli.AddNode(ctx, &nodeToCm)

2
blobstore/blobnode/svr.go
View File

@ -153,7 +153,7 @@ func (s *Service) Stat(c *rpc.Context) {
span.Debugf("stat")
s.lock.RLock()
diskinfos := make([]*bnapi.DiskInfo, 0)
diskinfos := make([]*cmapi.BlobNodeDiskInfo, 0)
for i := range s.Disks {
diskInfo := s.Disks[i].DiskInfo()
diskinfos = append(diskinfos, &(diskInfo))

18
blobstore/blobnode/svr_test.go
View File

@ -463,7 +463,7 @@ func TestService_CmdpChunk(t *testing.T) {
defer resp.Body.Close()
}
err = cs.SetStatus(bnapi.ChunkStatusRelease)
err = cs.SetStatus(cmapi.ChunkStatusRelease)
require.NoError(t, err)
{
totalUrl := testServer.URL + "/chunk/readonly/diskid/101/vuid/2001"
@ -662,9 +662,11 @@ func (mcm *mockClusterMgr) DiskList(c *rpc.Context) {
}
ret := &cmapi.ListDiskRet{}
for _, d := range mcm.disks {
info := &bnapi.DiskInfo{
Path: d.path,
Status: d.status,
info := &cmapi.BlobNodeDiskInfo{
DiskInfo: cmapi.DiskInfo{
Path: d.path,
Status: d.status,
},
}
info.DiskID = d.diskId
ret.Disks = append(ret.Disks, info)
@ -724,7 +726,7 @@ func (mcm *mockClusterMgr) DiskInfo(c *rpc.Context) {
c.RespondError(bloberr.ErrIllegalArguments)
return
}
ret := &bnapi.DiskInfo{}
ret := &cmapi.BlobNodeDiskInfo{}
ret.DiskID = args.DiskID
c.RespondJSON(ret)
}
@ -750,7 +752,7 @@ func (mcm *mockClusterMgr) DiskHeartbeat(c *rpc.Context) {
}
func (mcm *mockClusterMgr) DiskAdd(c *rpc.Context) {
args := new(bnapi.DiskInfo)
args := new(cmapi.BlobNodeDiskInfo)
if err := c.ParseArgs(args); err != nil {
c.RespondError(bloberr.ErrIllegalArguments)
return
@ -820,7 +822,7 @@ func (mcm *mockClusterMgr) VolumeGet(c *rpc.Context) {
}
func (mcm *mockClusterMgr) NodeAdd(c *rpc.Context) {
args := new(bnapi.NodeInfo)
args := new(cmapi.BlobNodeInfo)
if err := c.ParseArgs(args); err != nil {
c.RespondError(bloberr.ErrIllegalArguments)
return
@ -848,7 +850,7 @@ func (mcm *mockClusterMgr) NodeInfo(c *rpc.Context) {
c.RespondError(bloberr.ErrIllegalArguments)
return
}
ret := &bnapi.NodeInfo{}
ret := &cmapi.BlobNodeInfo{}
ret.NodeID = args.NodeID
ret.DiskType = proto.DiskTypeHDD

3
blobstore/blobnode/task_runner_migrate_test.go
View File

@ -18,6 +18,7 @@ import (
"context"
"testing"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/stretchr/testify/require"
api "github.com/cubefs/cubefs/blobstore/api/blobnode"
@ -151,7 +152,7 @@ func TestMigrateGenTasklets(t *testing.T) {
{
for index, replica := range replicas {
if index < balanceTask.CodeMode.Tactic().PutQuorum {
getter.setVunitStatus(replica.Vuid, api.ChunkStatusNormal)
getter.setVunitStatus(replica.Vuid, clustermgr.ChunkStatusNormal)
}
}
_, err = w.GenTasklets(context.Background())

11
blobstore/blobnode/worker_for_test.go
View File

@ -22,6 +22,7 @@ import (
"sync"
"testing"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/golang/mock/gomock"
"github.com/klauspost/reedsolomon"
@ -89,7 +90,7 @@ func NewMockGetterWithBids(replicas []proto.VunitLocation, mode codemode.CodeMod
failVuid: make(map[proto.Vuid]error),
}
for _, replica := range replicas {
vunit := newMockVunit(replica.Vuid, api.ChunkStatusReadOnly)
vunit := newMockVunit(replica.Vuid, clustermgr.ChunkStatusReadOnly)
getter.vunits[replica.Vuid] = vunit
}
@ -173,7 +174,7 @@ func (getter *MockGetter) setWell(vuid proto.Vuid) {
delete(getter.failVuid, vuid)
}
func (getter *MockGetter) setVunitStatus(vuid proto.Vuid, status api.ChunkStatus) {
func (getter *MockGetter) setVunitStatus(vuid proto.Vuid, status clustermgr.ChunkStatus) {
getter.mu.Lock()
defer getter.mu.Unlock()
@ -240,7 +241,7 @@ func (getter *MockGetter) ListShards(ctx context.Context, location proto.VunitLo
func (getter *MockGetter) StatChunk(ctx context.Context, location proto.VunitLocation) (ci *client.ChunkInfo, err error) {
vuid := location.Vuid
if _, ok := getter.vunits[vuid]; ok {
vunitInfo := api.ChunkInfo{
vunitInfo := clustermgr.ChunkInfo{
Vuid: vuid,
Status: getter.vunits[vuid].status,
}
@ -293,13 +294,13 @@ func abstractShards(idxs []int, shards [][]byte) [][]byte {
type mockVunit struct {
mu sync.Mutex
vuid proto.Vuid
status api.ChunkStatus
status clustermgr.ChunkStatus
shards map[proto.BlobID][]byte
crc32 map[proto.BlobID]uint32
bidInfos map[proto.BlobID]*client.ShardInfo
}
func newMockVunit(vuid proto.Vuid, status api.ChunkStatus) *mockVunit {
func newMockVunit(vuid proto.Vuid, status clustermgr.ChunkStatus) *mockVunit {
m := mockVunit{
vuid: vuid,
status: status,

3
blobstore/cli/blobnode/chunk.go
View File

@ -16,6 +16,7 @@ package blobnode
import (
"github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/cli/common"
"github.com/cubefs/cubefs/blobstore/cli/common/cfmt"
"github.com/cubefs/cubefs/blobstore/cli/common/fmt"
@ -39,7 +40,7 @@ func addCmdChunk(cmd *grumble.Command) {
a.String("id", "chunk id")
},
Run: func(c *grumble.Context) error {
chunkID, err := blobnode.DecodeChunk(c.Args.String("id"))
chunkID, err := clustermgr.DecodeChunk(c.Args.String("id"))
if err != nil {
return err
}

6
blobstore/cli/blobnode/chunk_meta.go
View File

@ -19,10 +19,10 @@ import (
"encoding/binary"
"errors"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/desertbit/grumble"
"github.com/tecbot/gorocksdb"
"github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/blobnode/core"
"github.com/cubefs/cubefs/blobstore/blobnode/core/disk"
"github.com/cubefs/cubefs/blobstore/blobnode/core/storage"
@ -59,7 +59,7 @@ var (
_vuidSpacePrefix = []byte("vuids")
shardSpacePrefixLen = len(_shardSpacePrefix)
shardChunkCommonLen = shardSpacePrefixLen + blobnode.ChunkIdLength
shardChunkCommonLen = shardSpacePrefixLen + clustermgr.ChunkIDLength
shardKeyLen = shardChunkCommonLen + 8
)
@ -97,7 +97,7 @@ func chunkDumpMeta(c *grumble.Context) error {
return nil
}
id, err := blobnode.DecodeChunk(chunk)
id, err := clustermgr.DecodeChunk(chunk)
if err != nil {
return err
}

4
blobstore/cli/blobnode/disk.go
View File

@ -236,7 +236,7 @@ func walkSingleDisk(ctx context.Context, cmCli *clustermgr.Client, dh *clustermg
return vuidCnt, nil
}
func getChunkMeta(db kvstore.KVStore, chunkId blobnode.ChunkId) (vm core.VuidMeta) {
func getChunkMeta(db kvstore.KVStore, chunkId clustermgr.ChunkID) (vm core.VuidMeta) {
if db == nil {
return core.VuidMeta{}
}
@ -257,7 +257,7 @@ func getChunkMeta(db kvstore.KVStore, chunkId blobnode.ChunkId) (vm core.VuidMet
return vm
}
func parseChunkNameStr(name string) (chunkId blobnode.ChunkId, err error) {
func parseChunkNameStr(name string) (chunkId clustermgr.ChunkID, err error) {
const chunkFileLen = 33 // 16+1+16
if len(name) != chunkFileLen {

15
blobstore/cli/common/cfmt/blobnode.go
View File

@ -21,18 +21,17 @@ import (
"github.com/dustin/go-humanize"
"github.com/fatih/color"
"github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
)
var chunkStatus2Str = map[blobnode.ChunkStatus]string{
blobnode.ChunkStatusNormal: "normal",
blobnode.ChunkStatusReadOnly: "readonly",
blobnode.ChunkStatusRelease: "release",
var chunkStatus2Str = map[clustermgr.ChunkStatus]string{
clustermgr.ChunkStatusNormal: "normal",
clustermgr.ChunkStatusReadOnly: "readonly",
clustermgr.ChunkStatusRelease: "release",
}
// ChunkidF chunk id
func ChunkidF(id blobnode.ChunkId) string {
func ChunkidF(id clustermgr.ChunkID) string {
t := id.UnixTime()
c := color.New(color.Faint, color.Italic)
return fmt.Sprintf("%s (V:%20d T:%20d %s)", id.String(), id.VolumeUnitId(),
@ -41,12 +40,12 @@ func ChunkidF(id blobnode.ChunkId) string {
}
// ChunkInfoJoin chunk info
func ChunkInfoJoin(info *blobnode.ChunkInfo, prefix string) string {
func ChunkInfoJoin(info *clustermgr.ChunkInfo, prefix string) string {
return joinWithPrefix(prefix, ChunkInfoF(info))
}
// ChunkInfoF chunk info
func ChunkInfoF(info *blobnode.ChunkInfo) []string {
func ChunkInfoF(info *clustermgr.ChunkInfo) []string {
if info == nil {
return nilStrings[:]
}

7
blobstore/cli/common/cfmt/blobnode_test.go
View File

@ -18,7 +18,6 @@ import (
"testing"
"time"
"github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/cli/common/cfmt"
"github.com/cubefs/cubefs/blobstore/cli/common/fmt"
@ -26,15 +25,15 @@ import (
)
func TestChunkInfo(t *testing.T) {
val := blobnode.ChunkInfo{
Id: blobnode.NewChunkId(4191518780817409),
val := clustermgr.ChunkInfo{
Id: clustermgr.NewChunkID(4191518780817409),
Vuid: 4191518780817409,
DiskID: 0xff,
Total: 1 << 35,
Used: 1 << 34,
Free: 100,
Size: 1 << 32,
Status: blobnode.ChunkStatusReadOnly,
Status: clustermgr.ChunkStatusReadOnly,
Compacting: false,
}
printLine()

9
blobstore/clustermgr/cluster/alloc.go
View File

@ -314,18 +314,21 @@ func (d *nodeAllocator) allocDisk(ctx context.Context, excludes map[proto.DiskID
randTotal--
}()
disk := disks[randNum]
disk.withRLocked(func() error {
err := disk.withRLocked(func() error {
weight := disk.weight()
if weight <= 0 {
return nil
return ErrNoEnoughSpace
}
// ignore not writable disk
if !disk.isWritable() {
span.Debugf("disk %d is not writable, is it expired: %v", disk.diskID, disk.isExpire())
return nil
return ErrNoEnoughSpace
}
return nil
})
if err != nil {
return nil
}
if _, ok := excludes[disk.diskID]; !ok {
span.Debugf("chosen disk: %#v", disk.info)

134
blobstore/clustermgr/cluster/alloc_test.go
View File

@ -58,7 +58,7 @@ var (
hostPrefix = "test-host-"
)
func initTestDiskMgr(t *testing.T) (d *manager, closeFunc func()) {
func initTestDiskMgr(t *testing.T) (d *BlobNodeManager, closeFunc func()) {
var err error
testTmpDBPath := "/tmp/tmpdiskmgrnormaldb" + strconv.Itoa(rand.Intn(10000000000))
testDB, err := normaldb.OpenNormalDB(testTmpDBPath)
@ -76,7 +76,7 @@ func initTestDiskMgr(t *testing.T) (d *manager, closeFunc func()) {
DiskCountPerNodeInDiskSet: 20,
}
testDiskMgr, err := New(testMockScopeMgr, testDB, testDiskMgrConfig)
testDiskMgr, err := NewBlobNodeMgr(testMockScopeMgr, testDB, testDiskMgrConfig)
if err != nil {
t.Log(errors.Detail(err))
}
@ -90,9 +90,9 @@ func initTestDiskMgr(t *testing.T) (d *manager, closeFunc func()) {
}
}
func initTestDiskMgrDisks(t *testing.T, testDiskMgr *manager, start, end int, specifyNodeID bool, idcs ...string) {
func initTestDiskMgrDisks(t *testing.T, testDiskMgr *BlobNodeManager, start, end int, specifyNodeID bool, idcs ...string) {
_, ctx := trace.StartSpanFromContext(context.Background(), "")
diskInfo := clustermgr.DiskInfo{
diskInfo := clustermgr.BlobNodeDiskInfo{
DiskHeartBeatInfo: clustermgr.DiskHeartBeatInfo{
Used: 0,
Size: 14.5 * 1024 * 1024 * 1024 * 1024,
@ -100,10 +100,12 @@ func initTestDiskMgrDisks(t *testing.T, testDiskMgr *manager, start, end int, sp
MaxChunkCnt: 14.5 * 1024 / 16,
FreeChunkCnt: 14.5 * 1024 / 16,
},
ClusterID: proto.ClusterID(1),
Idc: "z0",
Status: proto.DiskStatusNormal,
Readonly: false,
DiskInfo: clustermgr.DiskInfo{
ClusterID: proto.ClusterID(1),
Idc: "z0",
Status: proto.DiskStatusNormal,
Readonly: false,
},
}
for idx, idc := range idcs {
for i := start; i <= end; i++ {
@ -118,15 +120,15 @@ func initTestDiskMgrDisks(t *testing.T, testDiskMgr *manager, start, end int, sp
diskInfo.Idc = idc
newDiskInfo := diskInfo
err := testDiskMgr.addDisk(ctx, &newDiskInfo)
err := testDiskMgr.applyAddDisk(ctx, &newDiskInfo)
require.NoError(t, err)
}
}
}
func initTestDiskMgrDisksWithReadonly(t *testing.T, testDiskMgr *manager, start, end int, idcs ...string) {
func initTestDiskMgrDisksWithReadonly(t *testing.T, testDiskMgr *BlobNodeManager, start, end int, idcs ...string) {
_, ctx := trace.StartSpanFromContext(context.Background(), "")
diskInfo := &clustermgr.DiskInfo{
diskInfo := &clustermgr.BlobNodeDiskInfo{
DiskHeartBeatInfo: clustermgr.DiskHeartBeatInfo{
Used: 0,
Size: 1024,
@ -134,10 +136,12 @@ func initTestDiskMgrDisksWithReadonly(t *testing.T, testDiskMgr *manager, start,
MaxChunkCnt: 1024 / 16,
FreeChunkCnt: 1024 / 16,
},
ClusterID: proto.ClusterID(1),
Idc: "z0",
Status: proto.DiskStatusNormal,
Readonly: false,
DiskInfo: clustermgr.DiskInfo{
ClusterID: proto.ClusterID(1),
Idc: "z0",
Status: proto.DiskStatusNormal,
Readonly: false,
},
}
for idx, idc := range idcs {
for i := start; i <= end; i++ {
@ -152,13 +156,13 @@ func initTestDiskMgrDisksWithReadonly(t *testing.T, testDiskMgr *manager, start,
} else {
diskInfo.Readonly = false
}
err := testDiskMgr.addDisk(ctx, diskInfo)
err := testDiskMgr.applyAddDisk(ctx, diskInfo)
require.NoError(t, err)
}
}
}
func initTestDiskMgrNodes(t *testing.T, testDiskMgr *manager, start, end int, idcs ...string) {
func initTestDiskMgrNodes(t *testing.T, testDiskMgr *BlobNodeManager, start, end int, idcs ...string) {
_, ctx := trace.StartSpanFromContext(context.Background(), "")
nodeInfo := clustermgr.NodeInfo{
ClusterID: proto.ClusterID(1),
@ -172,9 +176,10 @@ func initTestDiskMgrNodes(t *testing.T, testDiskMgr *manager, start, end int, id
nodeInfo.Rack = strconv.Itoa(i)
nodeInfo.Host = idc + hostPrefix + strconv.Itoa(i)
nodeInfo.Idc = idc
newNodeInfo := nodeInfo
err := testDiskMgr.addNode(ctx, &newNodeInfo)
newNodeInfo := clustermgr.BlobNodeInfo{
NodeInfo: nodeInfo,
}
err := testDiskMgr.applyAddNode(ctx, &newNodeInfo)
require.NoError(t, err)
}
}
@ -184,7 +189,7 @@ func TestAlloc(t *testing.T) {
testDiskMgr, closeTestDiskMgr := initTestDiskMgr(t)
defer closeTestDiskMgr()
// disk never expire
testDiskMgr.HeartbeatExpireIntervalS = 6000
testDiskMgr.cfg.HeartbeatExpireIntervalS = 6000
_, ctx := trace.StartSpanFromContext(context.Background(), "")
// disable same host, insert not enough disk
@ -199,7 +204,7 @@ func TestAlloc(t *testing.T) {
t.Logf("all disk length: %d", len(testDiskMgr.allDisks))
// alloc from not enough space, alloc should return ErrNoEnoughSpace
allocators := testDiskMgr.allocators[proto.NodeRoleBlobNode].Load().(*allocator)
allocators := testDiskMgr.manager.allocator.Load().(*allocator)
idcAllocators := allocators.nodeSets[proto.DiskTypeHDD][ecNodeSetID].diskSets[ecDiskSetID].idcAllocators
for _, idc := range testIdcs {
idcAllocator := idcAllocators[idc]
@ -208,9 +213,9 @@ func TestAlloc(t *testing.T) {
}
// alloc with diff rack
testDiskMgr.RackAware = true
testDiskMgr.cfg.RackAware = true
testDiskMgr.refresh(ctx)
allocators = testDiskMgr.allocators[proto.NodeRoleBlobNode].Load().(*allocator)
allocators = testDiskMgr.manager.allocator.Load().(*allocator)
idcAllocators = allocators.nodeSets[proto.DiskTypeHDD][ecNodeSetID].diskSets[ecDiskSetID].idcAllocators
allocator := idcAllocators[testIdcs[0]]
_, err := allocator.alloc(ctx, 9, nil)
@ -221,10 +226,10 @@ func TestAlloc(t *testing.T) {
// refresh cluster's disk space allocator when change HostAware
{
_, ctx = trace.StartSpanFromContext(context.Background(), "alloc-same-host")
testDiskMgr.HostAware = false
testDiskMgr.RackAware = false
testDiskMgr.cfg.HostAware = false
testDiskMgr.cfg.RackAware = false
testDiskMgr.refresh(ctx)
allocators := testDiskMgr.allocators[proto.NodeRoleBlobNode].Load().(*allocator)
allocators := testDiskMgr.manager.allocator.Load().(*allocator)
idcAllocators := allocators.nodeSets[proto.DiskTypeHDD][ecNodeSetID].diskSets[ecDiskSetID].idcAllocators
allocator := idcAllocators[testIdcs[0]]
ret, err := allocator.alloc(ctx, 9, nil)
@ -239,10 +244,10 @@ func TestAlloc(t *testing.T) {
initTestDiskMgrDisks(t, testDiskMgr, 301, 539, false, testIdcs[0])
// refresh cluster's disk space allocator
_, ctx = trace.StartSpanFromContext(context.Background(), "alloc-enough-space")
testDiskMgr.HostAware = true
testDiskMgr.RackAware = false
testDiskMgr.cfg.HostAware = true
testDiskMgr.cfg.RackAware = false
testDiskMgr.refresh(ctx)
allocators := testDiskMgr.allocators[proto.NodeRoleBlobNode].Load().(*allocator)
allocators := testDiskMgr.manager.allocator.Load().(*allocator)
idcAllocators := allocators.nodeSets[proto.DiskTypeHDD][ecNodeSetID].diskSets[ecDiskSetID].idcAllocators
// alloc from enough space
idcAllocator := idcAllocators[testIdcs[0]]
@ -252,9 +257,9 @@ func TestAlloc(t *testing.T) {
// alloc with diff rack
_, ctx = trace.StartSpanFromContext(context.Background(), "alloc-diff-race")
testDiskMgr.RackAware = true
testDiskMgr.cfg.RackAware = true
testDiskMgr.refresh(ctx)
allocators = testDiskMgr.allocators[proto.NodeRoleBlobNode].Load().(*allocator)
allocators = testDiskMgr.manager.allocator.Load().(*allocator)
idcAllocators = allocators.nodeSets[proto.DiskTypeHDD][ecNodeSetID].diskSets[ecDiskSetID].idcAllocators
idcAllocator = idcAllocators[testIdcs[0]]
ret, err = idcAllocator.alloc(ctx, 9, nil)
@ -267,8 +272,8 @@ func TestAlloc(t *testing.T) {
{
_, ctx = trace.StartSpanFromContext(context.Background(), "alloc-chunk")
testDiskMgr.HostAware = true
testDiskMgr.RackAware = false
testDiskMgr.cfg.HostAware = true
testDiskMgr.cfg.RackAware = false
testDiskMgr.refresh(ctx)
// testMockBlobNode.EXPECT().CreateChunk(gomock.Any(), gomock.Any(), gomock.Any()).MaxTimes(100000).Return(nil)
@ -356,7 +361,7 @@ func TestAllocWithSameHost(t *testing.T) {
testDiskMgr, closeTestDiskMgr := initTestDiskMgr(t)
defer closeTestDiskMgr()
// disk never expire
testDiskMgr.HeartbeatExpireIntervalS = 6000
testDiskMgr.cfg.HeartbeatExpireIntervalS = 6000
defaultRetrySleepIntervalS = 0
_, ctx := trace.StartSpanFromContext(context.Background(), "alloc-same-host-not-enough")
@ -366,14 +371,14 @@ func TestAllocWithSameHost(t *testing.T) {
{
initTestDiskMgrNodes(t, testDiskMgr, 1, 1, testIdcs...)
initTestDiskMgrDisks(t, testDiskMgr, 1, 10, false, testIdcs...)
testDiskMgr.HostAware = false
testDiskMgr.RackAware = false
testDiskMgr.cfg.HostAware = false
testDiskMgr.cfg.RackAware = false
testDiskMgr.refresh(ctx)
t.Logf("all disk length: %d", len(testDiskMgr.allDisks))
// alloc from not enough space, alloc should return ErrNoEnoughSpace
allocators := testDiskMgr.allocators[proto.NodeRoleBlobNode].Load().(*allocator)
allocators := testDiskMgr.manager.allocator.Load().(*allocator)
idcAllocators := allocators.nodeSets[proto.DiskTypeHDD][ecNodeSetID].diskSets[ecDiskSetID].idcAllocators
for _, idc := range testIdcs {
allocator := idcAllocators[idc]
@ -388,7 +393,7 @@ func TestAllocWithSameHost(t *testing.T) {
initTestDiskMgrDisks(t, testDiskMgr, 11, 12, false, testIdcs...)
_, ctx = trace.StartSpanFromContext(context.Background(), "alloc-same-host-not-enough")
testDiskMgr.refresh(ctx)
allocators := testDiskMgr.allocators[proto.NodeRoleBlobNode].Load().(*allocator)
allocators := testDiskMgr.manager.allocator.Load().(*allocator)
idcAllocators := allocators.nodeSets[proto.DiskTypeHDD][ecNodeSetID].diskSets[ecDiskSetID].idcAllocators
allocator := idcAllocators[testIdcs[0]]
ret, err := allocator.alloc(ctx, 12, nil)
@ -403,13 +408,14 @@ func TestAllocWithSameHost(t *testing.T) {
for i := 1; i <= 12; i++ {
diskItem := testDiskMgr.allDisks[proto.DiskID(i)]
diskItem.lock.Lock()
diskItem.info.FreeChunkCnt = 10
heartbeatInfo := diskItem.info.extraInfo.(*clustermgr.DiskHeartBeatInfo)
heartbeatInfo.FreeChunkCnt = 10
diskItem.lock.Unlock()
}
testDiskMgr.metaLock.RUnlock()
testDiskMgr.refresh(ctx)
defaultAllocTolerateBuff = 0
allocators := testDiskMgr.allocators[proto.NodeRoleBlobNode].Load().(*allocator)
allocators := testDiskMgr.manager.allocator.Load().(*allocator)
idcAllocators := allocators.nodeSets[proto.DiskTypeHDD][ecNodeSetID].diskSets[ecDiskSetID].idcAllocators
allocator := idcAllocators[testIdcs[0]]
for i := 1; i <= 10; i++ {
@ -431,13 +437,14 @@ func TestAllocWithSameHost(t *testing.T) {
for i := 1; i <= 6; i++ {
diskItem := testDiskMgr.allDisks[proto.DiskID(i)]
diskItem.lock.Lock()
diskItem.info.FreeChunkCnt = 10
heartbeatInfo := diskItem.info.extraInfo.(*clustermgr.DiskHeartBeatInfo)
heartbeatInfo.FreeChunkCnt = 10
diskItem.lock.Unlock()
}
testDiskMgr.metaLock.RUnlock()
testDiskMgr.refresh(ctx)
defaultAllocTolerateBuff = 0
allocators := testDiskMgr.allocators[proto.NodeRoleBlobNode].Load().(*allocator)
allocators := testDiskMgr.manager.allocator.Load().(*allocator)
idcAllocators := allocators.nodeSets[proto.DiskTypeHDD][ecNodeSetID].diskSets[ecDiskSetID].idcAllocators
allocator := idcAllocators[testIdcs[0]]
for i := 1; i <= 10; i++ {
@ -467,7 +474,7 @@ func TestAllocWithDiffRack(t *testing.T) {
testDiskMgr, closeTestDiskMgr := initTestDiskMgr(t)
defer closeTestDiskMgr()
// disk never expire
testDiskMgr.HeartbeatExpireIntervalS = 6000
testDiskMgr.cfg.HeartbeatExpireIntervalS = 6000
defaultRetrySleepIntervalS = 0
_, ctx := trace.StartSpanFromContext(context.Background(), "alloc-diff-rack-enough-host")
@ -496,11 +503,11 @@ func TestAllocWithDiffRack(t *testing.T) {
diskItem.lock.Unlock()
}
testDiskMgr.metaLock.RUnlock()
testDiskMgr.HostAware = true
testDiskMgr.RackAware = true
testDiskMgr.cfg.HostAware = true
testDiskMgr.cfg.RackAware = true
testDiskMgr.refresh(ctx)
// alloc from not enough rack, but enough data node, it should be successful
allocators := testDiskMgr.allocators[proto.NodeRoleBlobNode].Load().(*allocator)
allocators := testDiskMgr.manager.allocator.Load().(*allocator)
idcAllocators := allocators.nodeSets[proto.DiskTypeHDD][ecNodeSetID].diskSets[ecDiskSetID].idcAllocators
idcAllocator := idcAllocators[testIdcs[0]]
diskIDs, err := idcAllocator.alloc(ctx, 10, nil)
@ -512,13 +519,14 @@ func TestAllocWithDiffRack(t *testing.T) {
for i := 1; i <= 10; i++ {
diskItem := testDiskMgr.allDisks[proto.DiskID(i)]
diskItem.lock.Lock()
diskItem.info.FreeChunkCnt = 10
heartbeatInfo := diskItem.info.extraInfo.(*clustermgr.DiskHeartBeatInfo)
heartbeatInfo.FreeChunkCnt = 10
diskItem.lock.Unlock()
}
testDiskMgr.metaLock.RUnlock()
testDiskMgr.refresh(ctx)
defaultAllocTolerateBuff = 0
allocators = testDiskMgr.allocators[proto.NodeRoleBlobNode].Load().(*allocator)
allocators = testDiskMgr.manager.allocator.Load().(*allocator)
idcAllocators = allocators.nodeSets[proto.DiskTypeHDD][ecNodeSetID].diskSets[ecDiskSetID].idcAllocators
idcAllocator = idcAllocators[testIdcs[0]]
for i := 1; i <= 10; i++ {
@ -537,7 +545,7 @@ func TestAllocWithDiffHost(t *testing.T) {
testDiskMgr, closeTestDiskMgr := initTestDiskMgr(t)
defer closeTestDiskMgr()
// disk never expire
testDiskMgr.HeartbeatExpireIntervalS = 6000
testDiskMgr.cfg.HeartbeatExpireIntervalS = 6000
defaultRetrySleepIntervalS = 0
_, ctx := trace.StartSpanFromContext(context.Background(), "alloc-diff-host")
@ -558,10 +566,10 @@ func TestAllocWithDiffHost(t *testing.T) {
diskItem.lock.Unlock()
}
testDiskMgr.metaLock.RUnlock()
testDiskMgr.HostAware = true
testDiskMgr.RackAware = false
testDiskMgr.cfg.HostAware = true
testDiskMgr.cfg.RackAware = false
testDiskMgr.refresh(ctx)
allocators := testDiskMgr.allocators[proto.NodeRoleBlobNode].Load().(*allocator)
allocators := testDiskMgr.manager.allocator.Load().(*allocator)
idcAllocators := allocators.nodeSets[proto.DiskTypeHDD][ecNodeSetID].diskSets[ecDiskSetID].idcAllocators
idcAllocator := idcAllocators[testIdcs[0]]
diskIDs, err := idcAllocator.alloc(ctx, 10, nil)
@ -573,13 +581,14 @@ func TestAllocWithDiffHost(t *testing.T) {
for i := 1; i <= 10; i++ {
diskItem := testDiskMgr.allDisks[proto.DiskID(i)]
diskItem.lock.Lock()
diskItem.info.FreeChunkCnt = 10
heartbeatInfo := diskItem.info.extraInfo.(*clustermgr.DiskHeartBeatInfo)
heartbeatInfo.FreeChunkCnt = 10
diskItem.lock.Unlock()
}
testDiskMgr.metaLock.RUnlock()
testDiskMgr.refresh(ctx)
defaultAllocTolerateBuff = 0
allocators = testDiskMgr.allocators[proto.NodeRoleBlobNode].Load().(*allocator)
allocators = testDiskMgr.manager.allocator.Load().(*allocator)
idcAllocators = allocators.nodeSets[proto.DiskTypeHDD][ecNodeSetID].diskSets[ecDiskSetID].idcAllocators
idcAllocator = idcAllocators[testIdcs[0]]
for i := 1; i <= 10; i++ {
@ -598,7 +607,7 @@ func TestAllocWithDiffRackAndSameHost(t *testing.T) {
testDiskMgr, closeTestDiskMgr := initTestDiskMgr(t)
defer closeTestDiskMgr()
// disk never expire
testDiskMgr.HeartbeatExpireIntervalS = 6000
testDiskMgr.cfg.HeartbeatExpireIntervalS = 6000
defaultRetrySleepIntervalS = 0
_, ctx := trace.StartSpanFromContext(context.Background(), "alloc-diff-host")
@ -627,10 +636,10 @@ func TestAllocWithDiffRackAndSameHost(t *testing.T) {
}
testDiskMgr.metaLock.RUnlock()
testDiskMgr.HostAware = false
testDiskMgr.RackAware = true
testDiskMgr.cfg.HostAware = false
testDiskMgr.cfg.RackAware = true
testDiskMgr.refresh(ctx)
allocators := testDiskMgr.allocators[proto.NodeRoleBlobNode].Load().(*allocator)
allocators := testDiskMgr.manager.allocator.Load().(*allocator)
idcAllocators := allocators.nodeSets[proto.DiskTypeHDD][ecNodeSetID].diskSets[ecDiskSetID].idcAllocators
idcAllocator := idcAllocators[testIdcs[0]]
diskIDs, err := idcAllocator.alloc(ctx, 10, nil)
@ -642,13 +651,14 @@ func TestAllocWithDiffRackAndSameHost(t *testing.T) {
for i := 1; i <= 10; i++ {
diskItem := testDiskMgr.allDisks[proto.DiskID(i)]
diskItem.lock.Lock()
diskItem.info.FreeChunkCnt = 10
heartbeatInfo := diskItem.info.extraInfo.(*clustermgr.DiskHeartBeatInfo)
heartbeatInfo.FreeChunkCnt = 10
diskItem.lock.Unlock()
}
testDiskMgr.metaLock.RUnlock()
testDiskMgr.refresh(ctx)
defaultAllocTolerateBuff = 0
allocators = testDiskMgr.allocators[proto.NodeRoleBlobNode].Load().(*allocator)
allocators = testDiskMgr.manager.allocator.Load().(*allocator)
idcAllocators = allocators.nodeSets[proto.DiskTypeHDD][ecNodeSetID].diskSets[ecDiskSetID].idcAllocators
idcAllocator = idcAllocators[testIdcs[0]]
for i := 1; i <= 10; i++ {
@ -677,7 +687,7 @@ func TestAllocCost(t *testing.T) {
initTestDiskMgrDisks(t, testDiskMgr, 1, 1800, false, testIdcs[0])
// refresh cluster's disk space allocator
testDiskMgr.refresh(ctx)
allocators := testDiskMgr.allocators[proto.NodeRoleBlobNode].Load().(*allocator)
allocators := testDiskMgr.manager.allocator.Load().(*allocator)
idcAllocators := allocators.nodeSets[proto.DiskTypeHDD][ecNodeSetID].diskSets[ecDiskSetID].idcAllocators
allocator := idcAllocators["z0"]

18
blobstore/clustermgr/cluster/applier_test.go
View File

@ -55,7 +55,7 @@ func TestApplier_Others(t *testing.T) {
diskInfo.DiskHeartBeatInfo.FreeChunkCnt = 0
heartbeatInfos = append(heartbeatInfos, &diskInfo.DiskHeartBeatInfo)
}
err := testDiskMgr.heartBeatDiskInfo(ctx, heartbeatInfos)
err := testDiskMgr.applyHeartBeatDiskInfo(ctx, heartbeatInfos)
require.NoError(t, err)
err = testDiskMgr.Flush(ctx)
@ -63,7 +63,7 @@ func TestApplier_Others(t *testing.T) {
}
}
var testDiskInfo = clustermgr.DiskInfo{
var testDiskInfo = clustermgr.BlobNodeDiskInfo{
DiskHeartBeatInfo: clustermgr.DiskHeartBeatInfo{
Used: 0,
Size: 14.5 * 1024 * 1024 * 1024 * 1024,
@ -71,12 +71,14 @@ var testDiskInfo = clustermgr.DiskInfo{
MaxChunkCnt: 14.5 * 1024 / 16,
FreeChunkCnt: 14.5 * 1024 / 16,
},
ClusterID: proto.ClusterID(1),
Idc: "z0",
Rack: "testrack",
Status: proto.DiskStatusNormal,
Readonly: false,
NodeID: proto.NodeID(1),
DiskInfo: clustermgr.DiskInfo{
ClusterID: proto.ClusterID(1),
Idc: "z0",
Rack: "testrack",
Status: proto.DiskStatusNormal,
Readonly: false,
NodeID: proto.NodeID(1),
},
}
var testNodeInfo = clustermgr.NodeInfo{

35
blobstore/clustermgr/cluster/blobnode_mgr.go
View File

@ -1,3 +1,17 @@
// Copyright 2024 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package cluster
import (
@ -22,8 +36,8 @@ import (
)
const (
diskIDScopeName = "diskid"
nodeIDScopeName = "nodeid"
DiskIDScopeName = "diskid"
NodeIDScopeName = "nodeid"
)
type BlobNodeManagerAPI interface {
@ -44,10 +58,10 @@ type BlobNodeManagerAPI interface {
}
func NewBlobNodeMgr(scopeMgr scopemgr.ScopeMgrAPI, db *normaldb.NormalDB, cfg DiskMgrConfig) (*BlobNodeManager, error) {
_, ctx := trace.StartSpanFromContext(context.Background(), "NewDiskMgr")
_, ctx := trace.StartSpanFromContext(context.Background(), "NewBlobNodeMgr")
cfg.NodeIDScopeName = nodeIDScopeName
cfg.DiskIDScopeName = diskIDScopeName
cfg.NodeIDScopeName = NodeIDScopeName
cfg.DiskIDScopeName = DiskIDScopeName
defaulter.LessOrEqual(&cfg.RefreshIntervalS, defaultRefreshIntervalS)
defaulter.LessOrEqual(&cfg.HeartbeatExpireIntervalS, defaultHeartbeatExpireIntervalS)
defaulter.LessOrEqual(&cfg.FlushIntervalS, defaultFlushIntervalS)
@ -233,14 +247,16 @@ func (b *BlobNodeManager) AddDisk(ctx context.Context, args *clustermgr.BlobNode
span.Errorf("json marshal failed, disk info: %v, error: %v", args, err)
return errors.Info(apierrors.ErrUnexpected).Detail(err)
}
pendingKey := fmtApplyContextKey("disk-add", args.DiskID.ToString())
b.pendingEntries.Store(pendingKey, nil)
defer b.pendingEntries.Delete(pendingKey)
proposeInfo := base.EncodeProposeInfo(b.GetModuleName(), OperTypeAddDisk, data, base.ProposeContext{ReqID: span.TraceID()})
err = b.raftServer.Propose(ctx, proposeInfo)
if err != nil {
span.Error(err)
return apierrors.ErrRaftPropose
}
if v, ok := b.pendingEntries.Load(fmtApplyContextKey("disk-add", args.DiskID.ToString())); ok {
b.pendingEntries.Delete(args.DiskID)
if v, _ := b.manager.pendingEntries.Load(pendingKey); v != nil {
return v.(error)
}
return nil
@ -683,7 +699,10 @@ func (b *BlobNodeManager) applyAddDisk(ctx context.Context, info *clustermgr.Blo
if node, ok := b.allNodes[info.NodeID]; ok {
if node.info.Status == proto.NodeStatusDropped {
span.Warnf("node is dropped, disk info: %v", info)
b.pendingEntries.Store(fmtApplyContextKey("disk-add", info.DiskID.ToString()), apierrors.ErrCMNodeNotFound)
pendingKey := fmtApplyContextKey("disk-add", info.DiskID.ToString())
if _, ok := b.pendingEntries.Load(pendingKey); ok {
b.pendingEntries.Store(pendingKey, apierrors.ErrCMNodeNotFound)
}
return nil
}
info.DiskSetID = b.topoMgr.AllocDiskSetID(ctx, &info.DiskInfo, &node.info.NodeInfo, b.cfg.CopySetConfigs[node.info.Role][node.info.DiskType])

399
blobstore/clustermgr/cluster/blobnodemgr_mock.go Normal file
View File

@ -0,0 +1,399 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: github.com/cubefs/cubefs/blobstore/clustermgr/cluster (interfaces: BlobNodeManagerAPI)
// Package cluster is a generated GoMock package.
package cluster
import (
context "context"
reflect "reflect"
clustermgr "github.com/cubefs/cubefs/blobstore/api/clustermgr"
proto "github.com/cubefs/cubefs/blobstore/common/proto"
gomock "github.com/golang/mock/gomock"
)
// MockBlobNodeManagerAPI is a mock of BlobNodeManagerAPI interface.
type MockBlobNodeManagerAPI struct {
ctrl *gomock.Controller
recorder *MockBlobNodeManagerAPIMockRecorder
}
// MockBlobNodeManagerAPIMockRecorder is the mock recorder for MockBlobNodeManagerAPI.
type MockBlobNodeManagerAPIMockRecorder struct {
mock *MockBlobNodeManagerAPI
}
// NewMockBlobNodeManagerAPI creates a new mock instance.
func NewMockBlobNodeManagerAPI(ctrl *gomock.Controller) *MockBlobNodeManagerAPI {
mock := &MockBlobNodeManagerAPI{ctrl: ctrl}
mock.recorder = &MockBlobNodeManagerAPIMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use.
func (m *MockBlobNodeManagerAPI) EXPECT() *MockBlobNodeManagerAPIMockRecorder {
return m.recorder
}
// AddDisk mocks base method.
func (m *MockBlobNodeManagerAPI) AddDisk(arg0 context.Context, arg1 *clustermgr.BlobNodeDiskInfo) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "AddDisk", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// AddDisk indicates an expected call of AddDisk.
func (mr *MockBlobNodeManagerAPIMockRecorder) AddDisk(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddDisk", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).AddDisk), arg0, arg1)
}
// AllocChunks mocks base method.
func (m *MockBlobNodeManagerAPI) AllocChunks(arg0 context.Context, arg1 AllocPolicy) ([]proto.DiskID, []proto.Vuid, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "AllocChunks", arg0, arg1)
ret0, _ := ret[0].([]proto.DiskID)
ret1, _ := ret[1].([]proto.Vuid)
ret2, _ := ret[2].(error)
return ret0, ret1, ret2
}
// AllocChunks indicates an expected call of AllocChunks.
func (mr *MockBlobNodeManagerAPIMockRecorder) AllocChunks(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocChunks", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).AllocChunks), arg0, arg1)
}
// AllocDiskID mocks base method.
func (m *MockBlobNodeManagerAPI) AllocDiskID(arg0 context.Context) (proto.DiskID, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "AllocDiskID", arg0)
ret0, _ := ret[0].(proto.DiskID)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// AllocDiskID indicates an expected call of AllocDiskID.
func (mr *MockBlobNodeManagerAPIMockRecorder) AllocDiskID(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocDiskID", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).AllocDiskID), arg0)
}
// AllocNodeID mocks base method.
func (m *MockBlobNodeManagerAPI) AllocNodeID(arg0 context.Context) (proto.NodeID, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "AllocNodeID", arg0)
ret0, _ := ret[0].(proto.NodeID)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// AllocNodeID indicates an expected call of AllocNodeID.
func (mr *MockBlobNodeManagerAPIMockRecorder) AllocNodeID(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocNodeID", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).AllocNodeID), arg0)
}
// CheckDiskInfoDuplicated mocks base method.
func (m *MockBlobNodeManagerAPI) CheckDiskInfoDuplicated(arg0 context.Context, arg1 proto.DiskID, arg2 *clustermgr.DiskInfo, arg3 *clustermgr.NodeInfo) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "CheckDiskInfoDuplicated", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(error)
return ret0
}
// CheckDiskInfoDuplicated indicates an expected call of CheckDiskInfoDuplicated.
func (mr *MockBlobNodeManagerAPIMockRecorder) CheckDiskInfoDuplicated(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckDiskInfoDuplicated", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).CheckDiskInfoDuplicated), arg0, arg1, arg2, arg3)
}
// CheckNodeInfoDuplicated mocks base method.
func (m *MockBlobNodeManagerAPI) CheckNodeInfoDuplicated(arg0 context.Context, arg1 *clustermgr.NodeInfo) (proto.NodeID, bool) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "CheckNodeInfoDuplicated", arg0, arg1)
ret0, _ := ret[0].(proto.NodeID)
ret1, _ := ret[1].(bool)
return ret0, ret1
}
// CheckNodeInfoDuplicated indicates an expected call of CheckNodeInfoDuplicated.
func (mr *MockBlobNodeManagerAPIMockRecorder) CheckNodeInfoDuplicated(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckNodeInfoDuplicated", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).CheckNodeInfoDuplicated), arg0, arg1)
}
// GetDiskInfo mocks base method.
func (m *MockBlobNodeManagerAPI) GetDiskInfo(arg0 context.Context, arg1 proto.DiskID) (*clustermgr.BlobNodeDiskInfo, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetDiskInfo", arg0, arg1)
ret0, _ := ret[0].(*clustermgr.BlobNodeDiskInfo)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetDiskInfo indicates an expected call of GetDiskInfo.
func (mr *MockBlobNodeManagerAPIMockRecorder) GetDiskInfo(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDiskInfo", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).GetDiskInfo), arg0, arg1)
}
// GetHeartbeatChangeDisks mocks base method.
func (m *MockBlobNodeManagerAPI) GetHeartbeatChangeDisks() []HeartbeatEvent {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetHeartbeatChangeDisks")
ret0, _ := ret[0].([]HeartbeatEvent)
return ret0
}
// GetHeartbeatChangeDisks indicates an expected call of GetHeartbeatChangeDisks.
func (mr *MockBlobNodeManagerAPIMockRecorder) GetHeartbeatChangeDisks() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHeartbeatChangeDisks", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).GetHeartbeatChangeDisks))
}
// GetNodeInfo mocks base method.
func (m *MockBlobNodeManagerAPI) GetNodeInfo(arg0 context.Context, arg1 proto.NodeID) (*clustermgr.BlobNodeInfo, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetNodeInfo", arg0, arg1)
ret0, _ := ret[0].(*clustermgr.BlobNodeInfo)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetNodeInfo indicates an expected call of GetNodeInfo.
func (mr *MockBlobNodeManagerAPIMockRecorder) GetNodeInfo(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNodeInfo", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).GetNodeInfo), arg0, arg1)
}
// IsDiskWritable mocks base method.
func (m *MockBlobNodeManagerAPI) IsDiskWritable(arg0 context.Context, arg1 proto.DiskID) (bool, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "IsDiskWritable", arg0, arg1)
ret0, _ := ret[0].(bool)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// IsDiskWritable indicates an expected call of IsDiskWritable.
func (mr *MockBlobNodeManagerAPIMockRecorder) IsDiskWritable(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsDiskWritable", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).IsDiskWritable), arg0, arg1)
}
// IsDroppedNode mocks base method.
func (m *MockBlobNodeManagerAPI) IsDroppedNode(arg0 context.Context, arg1 proto.NodeID) (bool, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "IsDroppedNode", arg0, arg1)
ret0, _ := ret[0].(bool)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// IsDroppedNode indicates an expected call of IsDroppedNode.
func (mr *MockBlobNodeManagerAPIMockRecorder) IsDroppedNode(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsDroppedNode", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).IsDroppedNode), arg0, arg1)
}
// IsDroppingDisk mocks base method.
func (m *MockBlobNodeManagerAPI) IsDroppingDisk(arg0 context.Context, arg1 proto.DiskID) (bool, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "IsDroppingDisk", arg0, arg1)
ret0, _ := ret[0].(bool)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// IsDroppingDisk indicates an expected call of IsDroppingDisk.
func (mr *MockBlobNodeManagerAPIMockRecorder) IsDroppingDisk(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsDroppingDisk", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).IsDroppingDisk), arg0, arg1)
}
// ListDiskInfo mocks base method.
func (m *MockBlobNodeManagerAPI) ListDiskInfo(arg0 context.Context, arg1 *clustermgr.ListOptionArgs) ([]*clustermgr.BlobNodeDiskInfo, proto.DiskID, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ListDiskInfo", arg0, arg1)
ret0, _ := ret[0].([]*clustermgr.BlobNodeDiskInfo)
ret1, _ := ret[1].(proto.DiskID)
ret2, _ := ret[2].(error)
return ret0, ret1, ret2
}
// ListDiskInfo indicates an expected call of ListDiskInfo.
func (mr *MockBlobNodeManagerAPIMockRecorder) ListDiskInfo(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDiskInfo", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).ListDiskInfo), arg0, arg1)
}
// ListDroppingDisk mocks base method.
func (m *MockBlobNodeManagerAPI) ListDroppingDisk(arg0 context.Context) ([]*clustermgr.BlobNodeDiskInfo, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ListDroppingDisk", arg0)
ret0, _ := ret[0].([]*clustermgr.BlobNodeDiskInfo)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// ListDroppingDisk indicates an expected call of ListDroppingDisk.
func (mr *MockBlobNodeManagerAPIMockRecorder) ListDroppingDisk(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDroppingDisk", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).ListDroppingDisk), arg0)
}
// RefreshExpireTime mocks base method.
func (m *MockBlobNodeManagerAPI) RefreshExpireTime() {
m.ctrl.T.Helper()
m.ctrl.Call(m, "RefreshExpireTime")
}
// RefreshExpireTime indicates an expected call of RefreshExpireTime.
func (mr *MockBlobNodeManagerAPIMockRecorder) RefreshExpireTime() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshExpireTime", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).RefreshExpireTime))
}
// SetStatus mocks base method.
func (m *MockBlobNodeManagerAPI) SetStatus(arg0 context.Context, arg1 proto.DiskID, arg2 proto.DiskStatus, arg3 bool) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "SetStatus", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(error)
return ret0
}
// SetStatus indicates an expected call of SetStatus.
func (mr *MockBlobNodeManagerAPIMockRecorder) SetStatus(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetStatus", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).SetStatus), arg0, arg1, arg2, arg3)
}
// Stat mocks base method.
func (m *MockBlobNodeManagerAPI) Stat(arg0 context.Context) *clustermgr.SpaceStatInfo {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Stat", arg0)
ret0, _ := ret[0].(*clustermgr.SpaceStatInfo)
return ret0
}
// Stat indicates an expected call of Stat.
func (mr *MockBlobNodeManagerAPIMockRecorder) Stat(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stat", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).Stat), arg0)
}
// ValidateNodeInfo mocks base method.
func (m *MockBlobNodeManagerAPI) ValidateNodeInfo(arg0 context.Context, arg1 *clustermgr.NodeInfo) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ValidateNodeInfo", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// ValidateNodeInfo indicates an expected call of ValidateNodeInfo.
func (mr *MockBlobNodeManagerAPIMockRecorder) ValidateNodeInfo(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateNodeInfo", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).ValidateNodeInfo), arg0, arg1)
}
// addDiskNoLocked mocks base method.
func (m *MockBlobNodeManagerAPI) addDiskNoLocked(arg0 *diskItem) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "addDiskNoLocked", arg0)
ret0, _ := ret[0].(error)
return ret0
}
// addDiskNoLocked indicates an expected call of addDiskNoLocked.
func (mr *MockBlobNodeManagerAPIMockRecorder) addDiskNoLocked(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "addDiskNoLocked", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).addDiskNoLocked), arg0)
}
// addDroppingDisk mocks base method.
func (m *MockBlobNodeManagerAPI) addDroppingDisk(arg0 proto.DiskID) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "addDroppingDisk", arg0)
ret0, _ := ret[0].(error)
return ret0
}
// addDroppingDisk indicates an expected call of addDroppingDisk.
func (mr *MockBlobNodeManagerAPIMockRecorder) addDroppingDisk(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "addDroppingDisk", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).addDroppingDisk), arg0)
}
// droppedDisk mocks base method.
func (m *MockBlobNodeManagerAPI) droppedDisk(arg0 proto.DiskID) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "droppedDisk", arg0)
ret0, _ := ret[0].(error)
return ret0
}
// droppedDisk indicates an expected call of droppedDisk.
func (mr *MockBlobNodeManagerAPIMockRecorder) droppedDisk(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "droppedDisk", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).droppedDisk), arg0)
}
// isDroppingDisk mocks base method.
func (m *MockBlobNodeManagerAPI) isDroppingDisk(arg0 proto.DiskID) (bool, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "isDroppingDisk", arg0)
ret0, _ := ret[0].(bool)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// isDroppingDisk indicates an expected call of isDroppingDisk.
func (mr *MockBlobNodeManagerAPIMockRecorder) isDroppingDisk(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "isDroppingDisk", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).isDroppingDisk), arg0)
}
// updateDiskNoLocked mocks base method.
func (m *MockBlobNodeManagerAPI) updateDiskNoLocked(arg0 *diskItem) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "updateDiskNoLocked", arg0)
ret0, _ := ret[0].(error)
return ret0
}
// updateDiskNoLocked indicates an expected call of updateDiskNoLocked.
func (mr *MockBlobNodeManagerAPIMockRecorder) updateDiskNoLocked(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "updateDiskNoLocked", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).updateDiskNoLocked), arg0)
}
// updateDiskStatusNoLocked mocks base method.
func (m *MockBlobNodeManagerAPI) updateDiskStatusNoLocked(arg0 proto.DiskID, arg1 proto.DiskStatus) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "updateDiskStatusNoLocked", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// updateDiskStatusNoLocked indicates an expected call of updateDiskStatusNoLocked.
func (mr *MockBlobNodeManagerAPIMockRecorder) updateDiskStatusNoLocked(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "updateDiskStatusNoLocked", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).updateDiskStatusNoLocked), arg0, arg1)
}
// updateNodeNoLocked mocks base method.
func (m *MockBlobNodeManagerAPI) updateNodeNoLocked(arg0 *nodeItem) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "updateNodeNoLocked", arg0)
ret0, _ := ret[0].(error)
return ret0
}
// updateNodeNoLocked indicates an expected call of updateNodeNoLocked.
func (mr *MockBlobNodeManagerAPIMockRecorder) updateNodeNoLocked(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "updateNodeNoLocked", reflect.TypeOf((*MockBlobNodeManagerAPI)(nil).updateNodeNoLocked), arg0)
}

24
blobstore/clustermgr/cluster/cluster.go
View File

@ -39,11 +39,11 @@ var (
)
const (
defaultRefreshIntervalS = 300
defaultHeartbeatExpireIntervalS = 60
defaultFlushIntervalS = 600
defaultApplyConcurrency = 10
defaultListDiskMaxCount = 200
defaultRefreshIntervalS = 300
defaultHeartbeatExpireIntervalS = 60
defaultFlushIntervalS = 600
defaultListDiskMaxCount = 200
defaultApplyConcurrency uint32 = 10
)
// CopySet Config
@ -107,10 +107,10 @@ type persistentHandler interface {
droppedDisk(id proto.DiskID) error
}
type Module struct {
blobNodeMgr *BlobNodeManager
shardNodeMgr *ShardNodeManager
}
//type Module struct {
// blobNodeMgr *BlobNodeManager
// shardNodeMgr *ShardNodeManager
//}
type HeartbeatEvent struct {
DiskID proto.DiskID
@ -130,6 +130,7 @@ type DiskMgrConfig struct {
IDC []string `json:"-"`
CodeModes []codemode.CodeMode `json:"-"`
ChunkSize int64 `json:"-"`
ChunkOversoldRatio float64 `json:"-"`
DiskIDScopeName string `json:"-"`
NodeIDScopeName string `json:"-"`
@ -338,9 +339,8 @@ func (d *manager) IsDroppingDisk(ctx context.Context, id proto.DiskID) (bool, er
// Stat return disk statistic info of a cluster
func (d *manager) Stat(ctx context.Context) *clustermgr.SpaceStatInfo {
spaceStatInfo := d.spaceStatInfo.Load().(map[proto.NodeRole]map[proto.DiskType]*clustermgr.SpaceStatInfo)
nodeSpaceInfo := spaceStatInfo[proto.NodeRoleBlobNode]
diskTypeInfo, ok := nodeSpaceInfo[proto.DiskTypeHDD]
spaceStatInfo := d.spaceStatInfo.Load().(map[proto.DiskType]*clustermgr.SpaceStatInfo)
diskTypeInfo, ok := spaceStatInfo[proto.DiskTypeHDD]
if !ok {
return &clustermgr.SpaceStatInfo{}
}

159
blobstore/clustermgr/cluster/cluster_test.go
View File

@ -20,7 +20,6 @@ import (
"math/rand"
"os"
"path"
"sync/atomic"
"testing"
"time"
@ -67,7 +66,7 @@ func TestDiskMgr_Normal(t *testing.T) {
diskInfo, err := testDiskMgr.GetDiskInfo(ctx, proto.DiskID(i))
require.NoError(t, err)
require.Equal(t, proto.DiskID(i), diskInfo.DiskID)
diskExist := testDiskMgr.CheckDiskInfoDuplicated(ctx, diskInfo, nodeInfo)
diskExist := testDiskMgr.CheckDiskInfoDuplicated(ctx, diskInfo.DiskID, &diskInfo.DiskInfo, &nodeInfo.NodeInfo)
require.Equal(t, apierrors.ErrExist, diskExist)
}
@ -77,7 +76,7 @@ func TestDiskMgr_Normal(t *testing.T) {
diskInfo.DiskID = proto.DiskID(11)
nodeInfo, err = testDiskMgr.GetNodeInfo(ctx, proto.NodeID(1))
require.NoError(t, err)
duplicated := testDiskMgr.CheckDiskInfoDuplicated(ctx, diskInfo, nodeInfo)
duplicated := testDiskMgr.CheckDiskInfoDuplicated(ctx, diskInfo.DiskID, &diskInfo.DiskInfo, &nodeInfo.NodeInfo)
require.Equal(t, apierrors.ErrIllegalArguments, duplicated)
// test normal case
@ -85,7 +84,7 @@ func TestDiskMgr_Normal(t *testing.T) {
diskInfo.Path += "notDuplicated"
nodeInfo, err = testDiskMgr.GetNodeInfo(ctx, proto.NodeID(1))
require.NoError(t, err)
duplicated = testDiskMgr.CheckDiskInfoDuplicated(ctx, diskInfo, nodeInfo)
duplicated = testDiskMgr.CheckDiskInfoDuplicated(ctx, diskInfo.DiskID, &diskInfo.DiskInfo, &nodeInfo.NodeInfo)
require.Equal(t, nil, duplicated)
}
@ -100,7 +99,7 @@ func TestDiskMgr_Normal(t *testing.T) {
err := testDiskMgr.SetStatus(ctx, 1, proto.DiskStatusBroken, true)
require.NoError(t, err)
err = testDiskMgr.SwitchReadonly(1, true)
err = testDiskMgr.applySwitchReadonly(1, true)
require.NoError(t, err)
for i := 1; i < 2; i++ {
@ -128,11 +127,11 @@ func TestDiskMgr_Dropping(t *testing.T) {
require.NoError(t, err)
require.Equal(t, 0, len(droppingList))
err = testDiskMgr.droppingDisk(ctx, 1)
err = testDiskMgr.applyDroppingDisk(ctx, 1)
require.NoError(t, err)
// add dropping disk repeatedly
err = testDiskMgr.droppingDisk(ctx, 1)
err = testDiskMgr.applyDroppingDisk(ctx, 1)
require.NoError(t, err)
// set status when disk is dropping, return ErrChangeDiskStatusNotAllow
@ -154,17 +153,17 @@ func TestDiskMgr_Dropping(t *testing.T) {
// dropped
{
err := testDiskMgr.droppingDisk(ctx, 2)
err := testDiskMgr.applyDroppingDisk(ctx, 2)
require.NoError(t, err)
droppingList, err := testDiskMgr.ListDroppingDisk(ctx)
require.NoError(t, err)
require.Equal(t, 2, len(droppingList))
err = testDiskMgr.droppedDisk(ctx, 1)
err = testDiskMgr.applyDroppedDisk(ctx, 1)
require.NoError(t, err)
// add dropping disk 1 repeatedly
err = testDiskMgr.droppingDisk(ctx, 1)
err = testDiskMgr.applyDroppingDisk(ctx, 1)
require.NoError(t, err)
droppingList, err = testDiskMgr.ListDroppingDisk(ctx)
@ -197,27 +196,28 @@ func TestDiskMgr_Heartbeat(t *testing.T) {
diskInfo.DiskHeartBeatInfo.FreeChunkCnt = 0
heartbeatInfos = append(heartbeatInfos, &diskInfo.DiskHeartBeatInfo)
}
err := testDiskMgr.heartBeatDiskInfo(ctx, heartbeatInfos)
err := testDiskMgr.applyHeartBeatDiskInfo(ctx, heartbeatInfos)
require.NoError(t, err)
// heartbeat check
for i := 1; i <= 10; i++ {
diskInfo, err := testDiskMgr.GetDiskInfo(ctx, proto.DiskID(i))
require.NoError(t, err)
require.Equal(t, diskInfo.Free/testDiskMgr.ChunkSize, diskInfo.FreeChunkCnt)
require.Equal(t, diskInfo.Free/testDiskMgr.cfg.ChunkSize, diskInfo.FreeChunkCnt)
require.Greater(t, diskInfo.OversoldFreeChunkCnt, diskInfo.FreeChunkCnt)
}
// reset oversold_chunk_ratio into 0
testDiskMgr.ChunkOversoldRatio = 0
err = testDiskMgr.heartBeatDiskInfo(ctx, heartbeatInfos)
testDiskMgr.cfg.ChunkOversoldRatio = 0
err = testDiskMgr.applyHeartBeatDiskInfo(ctx, heartbeatInfos)
require.NoError(t, err)
// validate OversoldFreeChunkCnt and FreeChunkCnt
for i := 1; i <= 10; i++ {
diskInfo, err := testDiskMgr.GetDiskInfo(ctx, proto.DiskID(i))
require.NoError(t, err)
require.Equal(t, diskInfo.Free/testDiskMgr.ChunkSize, diskInfo.FreeChunkCnt)
require.Equal(t, diskInfo.Free/testDiskMgr.cfg.ChunkSize, diskInfo.FreeChunkCnt)
require.Equal(t, int64(0), diskInfo.OversoldFreeChunkCnt)
require.Equal(t, int64(0), diskInfo.Free)
}
// get heartbeat change disk
@ -234,7 +234,7 @@ func TestDiskMgr_Heartbeat(t *testing.T) {
disk, _ = testDiskMgr.getDisk(proto.DiskID(2))
disk.lock.Lock()
disk.lastExpireTime = time.Now().Add(time.Duration(testDiskMgr.HeartbeatExpireIntervalS) * time.Second * -3)
disk.lastExpireTime = time.Now().Add(time.Duration(testDiskMgr.cfg.HeartbeatExpireIntervalS) * time.Second * -3)
disk.lock.Unlock()
disks = testDiskMgr.GetHeartbeatChangeDisks()
require.Equal(t, 2, len(disks))
@ -251,48 +251,48 @@ func TestDiskMgr_ListDisks(t *testing.T) {
require.NoError(t, err)
{
ret, err := testDiskMgr.ListDiskInfo(ctx, &clustermgr.ListOptionArgs{Host: diskInfo.Host, Count: 1000})
ret, marker, err := testDiskMgr.ListDiskInfo(ctx, &clustermgr.ListOptionArgs{Host: diskInfo.Host, Count: 1000})
require.NoError(t, err)
require.Equal(t, 10, len(ret.Disks))
ret, err = testDiskMgr.ListDiskInfo(ctx, &clustermgr.ListOptionArgs{Host: diskInfo.Host, Count: 1000, Marker: ret.Marker})
require.Equal(t, 10, len(ret))
ret, _, err = testDiskMgr.ListDiskInfo(ctx, &clustermgr.ListOptionArgs{Host: diskInfo.Host, Count: 1000, Marker: marker})
require.NoError(t, err)
require.Equal(t, 0, len(ret.Disks))
require.Equal(t, 0, len(ret))
ret, err = testDiskMgr.ListDiskInfo(ctx, &clustermgr.ListOptionArgs{Status: proto.DiskStatusNormal, Count: 1000})
ret, _, err = testDiskMgr.ListDiskInfo(ctx, &clustermgr.ListOptionArgs{Status: proto.DiskStatusNormal, Count: 1000})
require.NoError(t, err)
require.Equal(t, 10, len(ret.Disks))
require.Equal(t, 10, len(ret))
ret, err = testDiskMgr.ListDiskInfo(ctx, &clustermgr.ListOptionArgs{Idc: diskInfo.Idc, Rack: diskInfo.Rack, Count: 1000})
ret, _, err = testDiskMgr.ListDiskInfo(ctx, &clustermgr.ListOptionArgs{Idc: diskInfo.Idc, Rack: diskInfo.Rack, Count: 1000})
require.NoError(t, err)
require.Equal(t, 10, len(ret.Disks))
require.Equal(t, 10, len(ret))
ret, err = testDiskMgr.ListDiskInfo(ctx, &clustermgr.ListOptionArgs{Idc: diskInfo.Idc, Count: 2})
ret, marker, err = testDiskMgr.ListDiskInfo(ctx, &clustermgr.ListOptionArgs{Idc: diskInfo.Idc, Count: 2})
require.NoError(t, err)
require.Equal(t, 2, len(ret.Disks))
ret, err = testDiskMgr.ListDiskInfo(ctx, &clustermgr.ListOptionArgs{Idc: diskInfo.Idc, Count: 1000, Marker: ret.Marker})
require.Equal(t, 2, len(ret))
ret, _, err = testDiskMgr.ListDiskInfo(ctx, &clustermgr.ListOptionArgs{Idc: diskInfo.Idc, Count: 1000, Marker: marker})
require.NoError(t, err)
require.Equal(t, 8, len(ret.Disks))
require.Equal(t, 8, len(ret))
ret, err = testDiskMgr.ListDiskInfo(ctx, &clustermgr.ListOptionArgs{Idc: diskInfo.Idc, Rack: diskInfo.Rack, Status: proto.DiskStatusNormal, Count: 1000})
ret, _, err = testDiskMgr.ListDiskInfo(ctx, &clustermgr.ListOptionArgs{Idc: diskInfo.Idc, Rack: diskInfo.Rack, Status: proto.DiskStatusNormal, Count: 1000})
require.NoError(t, err)
require.Equal(t, 10, len(ret.Disks))
require.Equal(t, 10, len(ret))
ret, err = testDiskMgr.ListDiskInfo(ctx, &clustermgr.ListOptionArgs{Idc: diskInfo.Idc, Rack: diskInfo.Rack, Status: proto.DiskStatusDropped, Count: 1000})
ret, _, err = testDiskMgr.ListDiskInfo(ctx, &clustermgr.ListOptionArgs{Idc: diskInfo.Idc, Rack: diskInfo.Rack, Status: proto.DiskStatusDropped, Count: 1000})
require.NoError(t, err)
require.Equal(t, 0, len(ret.Disks))
require.Equal(t, 0, len(ret))
ret, err = testDiskMgr.ListDiskInfo(ctx, &clustermgr.ListOptionArgs{Idc: diskInfo.Idc, Rack: diskInfo.Rack, Host: diskInfo.Host, Status: proto.DiskStatusDropped, Count: 1000})
ret, _, err = testDiskMgr.ListDiskInfo(ctx, &clustermgr.ListOptionArgs{Idc: diskInfo.Idc, Rack: diskInfo.Rack, Host: diskInfo.Host, Status: proto.DiskStatusDropped, Count: 1000})
require.NoError(t, err)
require.Equal(t, 0, len(ret.Disks))
require.Equal(t, 0, len(ret))
}
{
err := testDiskMgr.SetStatus(ctx, proto.DiskID(1), proto.DiskStatusBroken, true)
require.NoError(t, err)
ret, err := testDiskMgr.ListDiskInfo(ctx, &clustermgr.ListOptionArgs{Status: proto.DiskStatusBroken, Count: 1000})
ret, _, err := testDiskMgr.ListDiskInfo(ctx, &clustermgr.ListOptionArgs{Status: proto.DiskStatusBroken, Count: 1000})
require.NoError(t, err)
require.Equal(t, 1, len(ret.Disks))
require.Equal(t, 1, len(ret))
}
}
@ -303,21 +303,24 @@ func TestDiskMgr_AdminUpdateDisk(t *testing.T) {
initTestDiskMgrDisks(t, testDiskMgr, 1, 10, false, testIdcs[0])
_, ctx := trace.StartSpanFromContext(context.Background(), "")
diskInfo := &clustermgr.DiskInfo{
diskInfo := &clustermgr.BlobNodeDiskInfo{
DiskHeartBeatInfo: clustermgr.DiskHeartBeatInfo{
DiskID: 1,
MaxChunkCnt: 99,
FreeChunkCnt: 9,
OversoldFreeChunkCnt: 19,
},
Status: 1,
DiskInfo: clustermgr.DiskInfo{
Status: 1,
},
}
err := testDiskMgr.adminUpdateDisk(ctx, diskInfo)
err := testDiskMgr.applyAdminUpdateDisk(ctx, diskInfo)
require.NoError(t, err)
diskItem := testDiskMgr.allDisks[diskInfo.DiskID]
require.Equal(t, diskItem.info.MaxChunkCnt, diskInfo.MaxChunkCnt)
require.Equal(t, diskItem.info.FreeChunkCnt, diskInfo.FreeChunkCnt)
heartbeatInfo := diskItem.info.extraInfo.(*clustermgr.DiskHeartBeatInfo)
require.Equal(t, heartbeatInfo.MaxChunkCnt, diskInfo.MaxChunkCnt)
require.Equal(t, heartbeatInfo.FreeChunkCnt, diskInfo.FreeChunkCnt)
require.Equal(t, diskItem.info.Status, diskInfo.Status)
diskRecord, err := testDiskMgr.diskTbl.GetDisk(diskInfo.DiskID)
@ -328,15 +331,17 @@ func TestDiskMgr_AdminUpdateDisk(t *testing.T) {
require.Equal(t, diskRecord.OversoldFreeChunkCnt, diskInfo.OversoldFreeChunkCnt)
// failed case, diskid not exisr
diskInfo1 := &clustermgr.DiskInfo{
diskInfo1 := &clustermgr.BlobNodeDiskInfo{
DiskHeartBeatInfo: clustermgr.DiskHeartBeatInfo{
DiskID: 199,
MaxChunkCnt: 99,
FreeChunkCnt: 9,
},
Status: 1,
DiskInfo: clustermgr.DiskInfo{
Status: 1,
},
}
err = testDiskMgr.adminUpdateDisk(ctx, diskInfo1)
err = testDiskMgr.applyAdminUpdateDisk(ctx, diskInfo1)
require.Error(t, err)
}
@ -348,54 +353,56 @@ func TestLoadData(t *testing.T) {
defer testDB.Close()
nr := normaldb.BlobNodeInfoRecord{
Version: normaldb.NodeInfoVersionNormal,
NodeID: proto.NodeID(1),
ClusterID: proto.ClusterID(1),
NodeSetID: proto.NodeSetID(2),
Status: proto.NodeStatusDropped,
Role: proto.NodeRoleBlobNode,
DiskType: proto.DiskTypeHDD,
NodeInfoRecord: normaldb.NodeInfoRecord{
Version: normaldb.NodeInfoVersionNormal,
NodeID: proto.NodeID(1),
ClusterID: proto.ClusterID(1),
NodeSetID: proto.NodeSetID(2),
Status: proto.NodeStatusDropped,
Role: proto.NodeRoleBlobNode,
DiskType: proto.DiskTypeHDD,
},
}
nodeTbl, err := normaldb.OpenBlobNodeTable(testDB)
require.NoError(t, err)
err = nodeTbl.UpdateNode(&nr)
require.NoError(t, err)
dr := normaldb.DiskInfoRecord{
Version: normaldb.DiskInfoVersionNormal,
DiskID: proto.DiskID(1),
NodeID: proto.NodeID(1),
ClusterID: proto.ClusterID(1),
DiskSetID: proto.DiskSetID(2),
Status: proto.DiskStatusRepaired,
blobNodeInfoRecord := normaldb.BlobNodeDiskInfoRecord{
DiskInfoRecord: normaldb.DiskInfoRecord{
Version: normaldb.DiskInfoVersionNormal,
DiskID: proto.DiskID(1),
NodeID: proto.NodeID(1),
ClusterID: proto.ClusterID(1),
DiskSetID: proto.DiskSetID(2),
Status: proto.DiskStatusRepaired,
},
}
diskTbl, err := normaldb.OpenBlobNodeDiskTable(testDB, true)
require.NoError(t, err)
err = diskTbl.AddDisk(&dr)
err = diskTbl.AddDisk(&blobNodeInfoRecord)
require.NoError(t, err)
droppedDiskTbl, err := normaldb.OpenDroppedDiskTable(testDB)
blobNodeMgr, err := NewBlobNodeMgr(testMockScopeMgr, testDB, testDiskMgrConfig)
require.NoError(t, err)
dm := &DiskMgr{
allocators: map[proto.NodeRole]*atomic.Value{},
topoMgrs: map[proto.NodeRole]*topoMgr{proto.NodeRoleBlobNode: newTopoMgr()},
taskPool: base.NewTaskDistribution(int(testDiskMgrConfig.ApplyConcurrency), 1),
scopeMgr: testMockScopeMgr,
// mock snapshot load data
_, ctx := trace.StartSpanFromContext(context.Background(), "")
bm := &BlobNodeManager{
manager: &manager{
topoMgr: newTopoMgr(),
scopeMgr: testMockScopeMgr,
taskPool: base.NewTaskDistribution(int(testDiskMgrConfig.ApplyConcurrency), 1),
cfg: testDiskMgrConfig,
},
diskTbl: diskTbl,
nodeTbl: nodeTbl,
droppedDiskTbl: droppedDiskTbl,
blobNodeClient: blobnode.New(&testDiskMgrConfig.BlobNodeConfig),
closeCh: make(chan interface{}),
DiskMgrConfig: testDiskMgrConfig,
}
_, ctx := trace.StartSpanFromContext(context.Background(), "")
// mock snapshot load data
err = dm.LoadData(ctx)
err = bm.LoadData(ctx)
require.NoError(t, err)
require.NotEqual(t, 0, len(dm.allocators))
require.NotNil(t, bm.allocator)
diskMgr, err := New(testMockScopeMgr, testDB, testDiskMgrConfig)
require.NoError(t, err)
topoInfo := diskMgr.GetTopoInfo(ctx)
blobNodeHDDNodeSets := topoInfo.AllNodeSets[proto.NodeRoleBlobNode.String()][proto.DiskTypeHDD.String()]
topoInfo := blobNodeMgr.GetTopoInfo(ctx)
blobNodeHDDNodeSets := topoInfo.AllNodeSets[proto.DiskTypeHDD.String()]
nodeSet, nodeSetExist := blobNodeHDDNodeSets[proto.NodeSetID(2)]
_, diskSetExist := nodeSet.DiskSets[proto.DiskSetID(2)]
require.Equal(t, nodeSetExist, true)

19
blobstore/clustermgr/cluster/proto.go
View File

@ -10,13 +10,12 @@ import (
type nodeItemInfo struct {
clustermgr.NodeInfo
extraInfo interface{}
// extraInfo interface{}
}
type nodeItem struct {
nodeID proto.NodeID
info nodeItemInfo
extraInfo interface{}
nodeID proto.NodeID
info nodeItemInfo
// disks map[proto.DiskID]*clustermgr.DiskInfo
disks map[proto.DiskID]*diskItem
@ -38,12 +37,12 @@ func (n *nodeItem) withRLocked(f func() error) error {
return err
}
func (n *nodeItem) withLocked(f func() error) error {
n.lock.Lock()
err := f()
n.lock.Unlock()
return err
}
//func (n *nodeItem) withLocked(f func() error) error {
// n.lock.Lock()
// err := f()
// n.lock.Unlock()
// return err
//}
type diskItemInfo struct {
clustermgr.DiskInfo

16
blobstore/clustermgr/cluster/refresh.go
View File

@ -168,8 +168,8 @@ func (b *BlobNodeManager) generateDiskSetStorage(ctx context.Context, disks []*d
rackFreeChunks := make(map[string]int64)
var (
free int64
idc, rack, host string
free, diskFreeChunk int64
idc, rack, host string
)
for _, disk := range disks {
// read one disk info
@ -183,7 +183,7 @@ func (b *BlobNodeManager) generateDiskSetStorage(ctx context.Context, disks []*d
host = node.info.Host
}
heartbeatInfo := disk.info.extraInfo.(*clustermgr.DiskHeartBeatInfo)
freeChunk := heartbeatInfo.FreeChunkCnt
diskFreeChunk = heartbeatInfo.FreeChunkCnt
maxChunk := heartbeatInfo.MaxChunkCnt
readonly := disk.info.Readonly
size := heartbeatInfo.Size
@ -198,7 +198,7 @@ func (b *BlobNodeManager) generateDiskSetStorage(ctx context.Context, disks []*d
}
diskStatInfosM[idc].Total += 1
diskStatInfosM[idc].TotalChunk += maxChunk
diskStatInfosM[idc].TotalFreeChunk += freeChunk
diskStatInfosM[idc].TotalFreeChunk += diskFreeChunk
if readonly {
diskStatInfosM[idc].Readonly += 1
}
@ -223,7 +223,6 @@ func (b *BlobNodeManager) generateDiskSetStorage(ctx context.Context, disks []*d
spaceStatInfo.TotalSpace += size
if readonly { // include dropping disk
spaceStatInfo.ReadOnlySpace += free
disk.lock.RUnlock()
return nil
}
spaceStatInfo.FreeSpace += free
@ -231,7 +230,6 @@ func (b *BlobNodeManager) generateDiskSetStorage(ctx context.Context, disks []*d
// filter expired disk
if disk.isExpire() {
disk.lock.RUnlock()
diskStatInfosM[idc].Expired += 1
return nil
}
@ -251,13 +249,13 @@ func (b *BlobNodeManager) generateDiskSetStorage(ctx context.Context, disks []*d
idcBlobNodeStgs[idc] = make([]*nodeAllocator, 0)
idcFreeChunks[idc] = 0
}
idcFreeChunks[idc] += freeChunk
idcFreeChunks[idc] += diskFreeChunk
// build for rackAllocator
if _, ok := rackBlobNodeStgs[rack]; !ok {
rackBlobNodeStgs[rack] = make([]*nodeAllocator, 0)
rackFreeChunks[rack] = 0
}
rackFreeChunks[rack] += freeChunk
rackFreeChunks[rack] += diskFreeChunk
// build for nodeAllocator
if _, ok := blobNodeStgs[host]; !ok {
blobNodeStgs[host] = &nodeAllocator{host: host, disks: make([]*diskItem, 0)}
@ -267,7 +265,7 @@ func (b *BlobNodeManager) generateDiskSetStorage(ctx context.Context, disks []*d
rackBlobNodeStgs[rack] = append(rackBlobNodeStgs[rack], blobNodeStgs[host])
}
blobNodeStgs[host].disks = append(blobNodeStgs[host].disks, disk)
blobNodeStgs[host].weight += freeChunk
blobNodeStgs[host].weight += diskFreeChunk
blobNodeStgs[host].free += free
}

4
blobstore/clustermgr/cluster/refresh_test.go
View File

@ -14,9 +14,9 @@ func TestWritableSpace(t *testing.T) {
spaceInfo := &clustermgr.SpaceStatInfo{}
idcBlobNodeStgs := make(map[string][]*nodeAllocator)
for i := range testDiskMgr.IDC {
for i := range testDiskMgr.cfg.IDC {
for j := 0; j < 16; j++ {
idcBlobNodeStgs[testDiskMgr.IDC[i]] = append(idcBlobNodeStgs[testDiskMgr.IDC[i]], &nodeAllocator{free: 100 * testDiskMgr.ChunkSize})
idcBlobNodeStgs[testDiskMgr.cfg.IDC[i]] = append(idcBlobNodeStgs[testDiskMgr.cfg.IDC[i]], &nodeAllocator{free: 100 * testDiskMgr.cfg.ChunkSize})
}
}
testDiskMgr.calculateWritable(idcBlobNodeStgs)

9
blobstore/clustermgr/cluster/shardnode_mgr.go
View File

@ -422,6 +422,8 @@ func (s *ShardNodeManager) diskInfoToDiskInfoRecord(info *clustermgr.ShardNodeDi
Used: info.Used,
Size: info.Size,
Free: info.Free,
MaxShardCnt: info.MaxShardCnt,
FreeShardCnt: info.FreeShardCnt,
UsedShardCnt: info.UsedShardCnt,
}
}
@ -446,6 +448,8 @@ func (s *ShardNodeManager) diskInfoRecordToDiskInfo(infoDB *normaldb.ShardNodeDi
Used: infoDB.Used,
Size: infoDB.Size,
Free: infoDB.Free,
MaxShardCnt: infoDB.MaxShardCnt,
FreeShardCnt: infoDB.FreeShardCnt,
UsedShardCnt: infoDB.UsedShardCnt,
},
}
@ -527,11 +531,10 @@ func (s *shardNodePersistentHandler) droppedDisk(id proto.DiskID) error {
}
func shardNodeDiskWeightGetter(extraInfo interface{}) int64 {
info := extraInfo.(*clustermgr.ShardNodeDiskHeartbeatInfo)
return info.Free / (info.Used / int64(info.UsedShardCnt))
return int64(extraInfo.(*clustermgr.ShardNodeDiskHeartbeatInfo).FreeShardCnt)
}
func shardNodeDiskWeightDecrease(extraInfo interface{}, num int64) {
info := extraInfo.(*clustermgr.ShardNodeDiskHeartbeatInfo)
info.UsedShardCnt += int32(num)
info.FreeShardCnt -= int32(num)
}

37
blobstore/clustermgr/cluster/topo_test.go
View File

@ -18,24 +18,28 @@ func TestTopoMgr_AllocSetID(t *testing.T) {
for i := startID; i < endID+1; i++ {
ni = &nodeItem{
nodeID: proto.NodeID(i),
info: &clustermgr.NodeInfo{
NodeID: proto.NodeID(i),
Role: proto.NodeRoleBlobNode,
DiskType: proto.DiskTypeHDD,
NodeSetID: proto.NodeSetID(i),
Status: proto.NodeStatusNormal,
Idc: "z0",
Rack: "rack0",
info: nodeItemInfo{
NodeInfo: clustermgr.NodeInfo{
NodeID: proto.NodeID(i),
Role: proto.NodeRoleBlobNode,
DiskType: proto.DiskTypeHDD,
NodeSetID: proto.NodeSetID(i),
Status: proto.NodeStatusNormal,
Idc: "z0",
Rack: "rack0",
},
},
}
testTopoMgr.AddNodeToNodeSet(ni)
di = &diskItem{
diskID: proto.DiskID(i),
info: &clustermgr.DiskInfo{
NodeID: proto.NodeID(startID),
DiskSetID: proto.DiskSetID(i),
Status: proto.DiskStatusNormal,
DiskHeartBeatInfo: clustermgr.DiskHeartBeatInfo{
info: diskItemInfo{
DiskInfo: clustermgr.DiskInfo{
NodeID: proto.NodeID(startID),
DiskSetID: proto.DiskSetID(i),
Status: proto.DiskStatusNormal,
},
extraInfo: &clustermgr.DiskHeartBeatInfo{
DiskID: proto.DiskID(i),
},
},
@ -45,7 +49,8 @@ func TestTopoMgr_AllocSetID(t *testing.T) {
_, ctx := trace.StartSpanFromContext(context.Background(), "")
ni.info.NodeID = proto.NodeID(startID)
ni.info.NodeSetID = proto.NodeSetID(startID)
di.info.DiskID = proto.DiskID(100)
heartbeatInfo := di.info.extraInfo.(*clustermgr.DiskHeartBeatInfo)
heartbeatInfo.DiskID = proto.DiskID(100)
conf := CopySetConfig{
NodeSetCap: 6,
NodeSetIdcCap: 2,
@ -54,8 +59,8 @@ func TestTopoMgr_AllocSetID(t *testing.T) {
DiskCountPerNodeInDiskSet: 3,
}
for i := 0; i < 99; i++ {
nodeSetID := testTopoMgr.AllocNodeSetID(ctx, ni.info, conf, false)
diskSetID := testTopoMgr.AllocDiskSetID(ctx, di.info, ni.info, conf)
nodeSetID := testTopoMgr.AllocNodeSetID(ctx, &ni.info.NodeInfo, conf, false)
diskSetID := testTopoMgr.AllocDiskSetID(ctx, &di.info.DiskInfo, &ni.info.NodeInfo, conf)
require.Equal(t, proto.NodeSetID(startID), nodeSetID)
require.Equal(t, proto.DiskSetID(startID), diskSetID)
}

58
blobstore/clustermgr/disk.go
View File

@ -32,7 +32,7 @@ func (s *Service) DiskIdAlloc(c *rpc.Context) {
span := trace.SpanFromContextSafe(ctx)
span.Info("accept DiskIdAlloc request")
diskID, err := s.DiskMgr.AllocDiskID(ctx)
diskID, err := s.BlobNodeMgr.AllocDiskID(ctx)
if err != nil {
span.Error("alloc disk id failed =>", errors.Detail(err))
c.RespondError(err)
@ -44,20 +44,20 @@ func (s *Service) DiskIdAlloc(c *rpc.Context) {
func (s *Service) DiskAdd(c *rpc.Context) {
ctx := c.Request.Context()
span := trace.SpanFromContextSafe(ctx)
args := new(clustermgr.DiskInfo)
args := new(clustermgr.BlobNodeDiskInfo)
if err := c.ParseArgs(args); err != nil {
c.RespondError(err)
return
}
span.Infof("accept DiskAdd request, args: %v", args)
nodeInfo, err := s.DiskMgr.GetNodeInfo(ctx, args.NodeID)
nodeInfo, err := s.BlobNodeMgr.GetNodeInfo(ctx, args.NodeID)
if err != nil || nodeInfo.Status == proto.NodeStatusDropped {
span.Warnf("nodeID not exist or node is dropped, disk info: %v", args)
c.RespondError(apierrors.ErrCMNodeNotFound)
return
}
if err = s.DiskMgr.CheckDiskInfoDuplicated(ctx, args, nodeInfo); err != nil {
if err = s.BlobNodeMgr.CheckDiskInfoDuplicated(ctx, args.DiskID, &args.DiskInfo, &nodeInfo.NodeInfo); err != nil {
c.RespondError(err)
return
}
@ -78,7 +78,7 @@ func (s *Service) DiskAdd(c *rpc.Context) {
c.RespondError(apierrors.ErrIllegalArguments)
return
}
err = s.DiskMgr.AddDisk(ctx, args)
err = s.BlobNodeMgr.AddDisk(ctx, args)
if err != nil {
c.RespondError(err)
return
@ -102,7 +102,7 @@ func (s *Service) DiskInfo(c *rpc.Context) {
return
}
ret, err := s.DiskMgr.GetDiskInfo(ctx, args.DiskID)
ret, err := s.BlobNodeMgr.GetDiskInfo(ctx, args.DiskID)
if err != nil || ret == nil {
span.Warnf("disk not found: %d", args.DiskID)
c.RespondError(err)
@ -134,7 +134,7 @@ func (s *Service) DiskList(c *rpc.Context) {
return
}
if args.Marker != proto.InvalidDiskID {
if _, err := s.DiskMgr.GetDiskInfo(ctx, args.Marker); err != nil {
if _, err := s.BlobNodeMgr.GetDiskInfo(ctx, args.Marker); err != nil {
span.Warnf("invalid marker, marker disk not exist")
err = apierrors.ErrIllegalArguments
c.RespondError(err)
@ -145,13 +145,17 @@ func (s *Service) DiskList(c *rpc.Context) {
args.Count = 10
}
ret, err := s.DiskMgr.ListDiskInfo(ctx, args)
disks, marker, err := s.BlobNodeMgr.ListDiskInfo(ctx, args)
if err != nil {
span.Errorf("list disk info failed =>", errors.Detail(err))
err = errors.Info(apierrors.ErrUnexpected).Detail(err)
c.RespondError(err)
return
}
ret := &clustermgr.ListDiskRet{
Disks: disks,
Marker: marker,
}
c.RespondJSON(ret)
}
@ -171,7 +175,7 @@ func (s *Service) DiskSet(c *rpc.Context) {
return
}
diskInfo, err := s.DiskMgr.GetDiskInfo(ctx, args.DiskID)
diskInfo, err := s.BlobNodeMgr.GetDiskInfo(ctx, args.DiskID)
if err != nil {
c.RespondError(err)
return
@ -180,7 +184,7 @@ func (s *Service) DiskSet(c *rpc.Context) {
return
}
err = s.DiskMgr.SetStatus(ctx, args.DiskID, args.Status, false)
err = s.BlobNodeMgr.SetStatus(ctx, args.DiskID, args.Status, false)
if err != nil {
span.Errorf("disk set failed =>", errors.Detail(err))
c.RespondError(err)
@ -193,7 +197,7 @@ func (s *Service) DiskSet(c *rpc.Context) {
c.RespondError(errors.Info(apierrors.ErrUnexpected).Detail(err))
return
}
proposeInfo := base.EncodeProposeInfo(s.DiskMgr.GetModuleName(), cluster.OperTypeSetDiskStatus, data, base.ProposeContext{ReqID: span.TraceID()})
proposeInfo := base.EncodeProposeInfo(s.BlobNodeMgr.GetModuleName(), cluster.OperTypeSetDiskStatus, data, base.ProposeContext{ReqID: span.TraceID()})
err = s.raftNode.Propose(ctx, proposeInfo)
if err != nil {
span.Error(err)
@ -218,7 +222,7 @@ func (s *Service) DiskDrop(c *rpc.Context) {
}
span.Infof("accept DiskDrop request, args: %v", args)
isDropping, err := s.DiskMgr.IsDroppingDisk(ctx, args.DiskID)
isDropping, err := s.BlobNodeMgr.IsDroppingDisk(ctx, args.DiskID)
if err != nil {
c.RespondError(err)
return
@ -227,7 +231,7 @@ func (s *Service) DiskDrop(c *rpc.Context) {
if isDropping {
return
}
diskInfo, err := s.DiskMgr.GetDiskInfo(ctx, args.DiskID)
diskInfo, err := s.BlobNodeMgr.GetDiskInfo(ctx, args.DiskID)
if err != nil {
c.RespondError(err)
return
@ -244,7 +248,7 @@ func (s *Service) DiskDrop(c *rpc.Context) {
c.RespondError(errors.Info(apierrors.ErrUnexpected).Detail(err))
return
}
proposeInfo := base.EncodeProposeInfo(s.DiskMgr.GetModuleName(), cluster.OperTypeDroppingDisk, data, base.ProposeContext{ReqID: span.TraceID()})
proposeInfo := base.EncodeProposeInfo(s.BlobNodeMgr.GetModuleName(), cluster.OperTypeDroppingDisk, data, base.ProposeContext{ReqID: span.TraceID()})
err = s.raftNode.Propose(ctx, proposeInfo)
if err != nil {
span.Error(err)
@ -262,7 +266,7 @@ func (s *Service) DiskDropped(c *rpc.Context) {
}
span.Infof("accept DiskDropped request, args: %v", args)
diskInfo, err := s.DiskMgr.GetDiskInfo(ctx, args.DiskID)
diskInfo, err := s.BlobNodeMgr.GetDiskInfo(ctx, args.DiskID)
if err != nil {
c.RespondError(err)
return
@ -272,7 +276,7 @@ func (s *Service) DiskDropped(c *rpc.Context) {
}
// 1. check disk if dropping
isDropping, err := s.DiskMgr.IsDroppingDisk(ctx, args.DiskID)
isDropping, err := s.BlobNodeMgr.IsDroppingDisk(ctx, args.DiskID)
if err != nil {
c.RespondError(err)
return
@ -303,7 +307,7 @@ func (s *Service) DiskDropped(c *rpc.Context) {
c.RespondError(errors.Info(apierrors.ErrUnexpected).Detail(err))
return
}
proposeInfo := base.EncodeProposeInfo(s.DiskMgr.GetModuleName(), cluster.OperTypeDroppedDisk, data, base.ProposeContext{ReqID: span.TraceID()})
proposeInfo := base.EncodeProposeInfo(s.BlobNodeMgr.GetModuleName(), cluster.OperTypeDroppedDisk, data, base.ProposeContext{ReqID: span.TraceID()})
err = s.raftNode.Propose(ctx, proposeInfo)
if err != nil {
span.Error(err)
@ -324,7 +328,7 @@ func (s *Service) DiskDroppingList(c *rpc.Context) {
ret := &clustermgr.ListDiskRet{}
var err error
ret.Disks, err = s.DiskMgr.ListDroppingDisk(ctx)
ret.Disks, err = s.BlobNodeMgr.ListDroppingDisk(ctx)
if err != nil {
span.Errorf("list dropping disk failed => ", errors.Detail(err))
err = errors.Info(apierrors.ErrUnexpected).Detail(err)
@ -346,7 +350,7 @@ func (s *Service) DiskHeartbeat(c *rpc.Context) {
heartbeatDisks := make([]*clustermgr.DiskHeartBeatInfo, 0)
disks := make([]*clustermgr.DiskHeartbeatRet, len(args.Disks))
for i := range args.Disks {
info, err := s.DiskMgr.GetDiskInfo(ctx, args.Disks[i].DiskID)
info, err := s.BlobNodeMgr.GetDiskInfo(ctx, args.Disks[i].DiskID)
if err != nil {
span.Errorf("get disk info %d failed, err: %v", args.Disks[i].DiskID, err)
c.RespondError(err)
@ -359,7 +363,7 @@ func (s *Service) DiskHeartbeat(c *rpc.Context) {
}
// filter frequentHeatBeat disk
frequentHeatBeat, err := s.DiskMgr.IsFrequentHeatBeat(args.Disks[i].DiskID, s.HeartbeatNotifyIntervalS)
frequentHeatBeat, err := s.BlobNodeMgr.IsFrequentHeatBeat(args.Disks[i].DiskID, s.HeartbeatNotifyIntervalS)
if err != nil {
span.Errorf("get disk info %d failed, err: %v", args.Disks[i].DiskID, err)
c.RespondError(err)
@ -386,7 +390,7 @@ func (s *Service) DiskHeartbeat(c *rpc.Context) {
c.RespondError(err)
return
}
proposeInfo := base.EncodeProposeInfo(s.DiskMgr.GetModuleName(), cluster.OperTypeHeartbeatDiskInfo, data, base.ProposeContext{ReqID: span.TraceID()})
proposeInfo := base.EncodeProposeInfo(s.BlobNodeMgr.GetModuleName(), cluster.OperTypeHeartbeatDiskInfo, data, base.ProposeContext{ReqID: span.TraceID()})
err = s.raftNode.Propose(ctx, proposeInfo)
if err != nil {
span.Error(err)
@ -406,7 +410,7 @@ func (s *Service) DiskAccess(c *rpc.Context) {
}
span.Infof("accept DiskAccess request, args: %v", args)
diskInfo, err := s.DiskMgr.GetDiskInfo(ctx, args.DiskID)
diskInfo, err := s.BlobNodeMgr.GetDiskInfo(ctx, args.DiskID)
if err != nil {
c.RespondError(err)
return
@ -415,7 +419,7 @@ func (s *Service) DiskAccess(c *rpc.Context) {
return
}
isDropping, err := s.DiskMgr.IsDroppingDisk(ctx, args.DiskID)
isDropping, err := s.BlobNodeMgr.IsDroppingDisk(ctx, args.DiskID)
if err != nil {
c.RespondError(err)
return
@ -431,7 +435,7 @@ func (s *Service) DiskAccess(c *rpc.Context) {
c.RespondError(errors.Info(apierrors.ErrUnexpected).Detail(err))
return
}
proposeInfo := base.EncodeProposeInfo(s.DiskMgr.GetModuleName(), cluster.OperTypeSwitchReadonly, data, base.ProposeContext{ReqID: span.TraceID()})
proposeInfo := base.EncodeProposeInfo(s.BlobNodeMgr.GetModuleName(), cluster.OperTypeSwitchReadonly, data, base.ProposeContext{ReqID: span.TraceID()})
err = s.raftNode.Propose(ctx, proposeInfo)
if err != nil {
span.Error(err)
@ -451,14 +455,14 @@ func (s *Service) DiskAccess(c *rpc.Context) {
func (s *Service) AdminDiskUpdate(c *rpc.Context) {
ctx := c.Request.Context()
span := trace.SpanFromContextSafe(ctx)
args := new(clustermgr.DiskInfo)
args := new(clustermgr.BlobNodeDiskInfo)
if err := c.ParseArgs(args); err != nil {
c.RespondError(err)
return
}
span.Infof("accept DiskAccess request, args: %v", args)
_, err := s.DiskMgr.GetDiskInfo(ctx, args.DiskID)
_, err := s.BlobNodeMgr.GetDiskInfo(ctx, args.DiskID)
if err != nil {
span.Errorf("admin update disk:%d not exist", args.DiskID)
c.RespondError(err)
@ -471,7 +475,7 @@ func (s *Service) AdminDiskUpdate(c *rpc.Context) {
c.RespondError(errors.Info(apierrors.ErrUnexpected).Detail(err))
return
}
proposeInfo := base.EncodeProposeInfo(s.DiskMgr.GetModuleName(), cluster.OperTypeAdminUpdateDisk, data, base.ProposeContext{ReqID: span.TraceID()})
proposeInfo := base.EncodeProposeInfo(s.BlobNodeMgr.GetModuleName(), cluster.OperTypeAdminUpdateDisk, data, base.ProposeContext{ReqID: span.TraceID()})
err = s.raftNode.Propose(ctx, proposeInfo)
if err != nil {
span.Error(err)

32
blobstore/clustermgr/disk_test.go
View File

@ -25,7 +25,7 @@ import (
"github.com/stretchr/testify/require"
)
var testDiskInfo = clustermgr.DiskInfo{
var testDiskInfo = clustermgr.BlobNodeDiskInfo{
DiskHeartBeatInfo: clustermgr.DiskHeartBeatInfo{
Used: 0,
Size: 14.5 * 1024 * 1024 * 1024 * 1024,
@ -33,18 +33,22 @@ var testDiskInfo = clustermgr.DiskInfo{
MaxChunkCnt: 14.5 * 1024 / 16,
FreeChunkCnt: 14.5 * 1024 / 16,
},
ClusterID: proto.ClusterID(1),
Idc: "z0",
Status: proto.DiskStatusNormal,
Readonly: false,
DiskInfo: clustermgr.DiskInfo{
ClusterID: proto.ClusterID(1),
Idc: "z0",
Status: proto.DiskStatusNormal,
Readonly: false,
},
}
var testNodeInfo = clustermgr.NodeInfo{
ClusterID: proto.ClusterID(1),
Idc: "z0",
Status: proto.NodeStatusNormal,
DiskType: proto.DiskTypeHDD,
Role: proto.NodeRoleBlobNode,
var testNodeInfo = clustermgr.BlobNodeInfo{
NodeInfo: clustermgr.NodeInfo{
ClusterID: proto.ClusterID(1),
Idc: "z0",
Status: proto.NodeStatusNormal,
DiskType: proto.DiskTypeHDD,
Role: proto.NodeRoleBlobNode,
},
}
func insertDiskInfos(t *testing.T, client *clustermgr.Client, start, end int, idcs ...string) {
@ -285,13 +289,15 @@ func TestDisk(t *testing.T) {
}
{
args := &clustermgr.DiskInfo{
args := &clustermgr.BlobNodeDiskInfo{
DiskHeartBeatInfo: clustermgr.DiskHeartBeatInfo{
DiskID: 1,
MaxChunkCnt: 99,
FreeChunkCnt: 9,
},
Status: 1,
DiskInfo: clustermgr.DiskInfo{
Status: proto.DiskStatusNormal,
},
}
err := testClusterClient.PostWith(ctx, "/admin/disk/update", nil, args)
require.NoError(t, err)

2
blobstore/clustermgr/manage.go
View File

@ -121,7 +121,7 @@ func (s *Service) Stat(c *rpc.Context) {
ret := new(clustermgr.StatInfo)
ret.RaftStatus = s.raftNode.Status()
ret.LeaderHost = s.raftNode.GetLeaderHost()
ret.SpaceStat = *(s.DiskMgr.Stat(ctx))
ret.SpaceStat = *(s.BlobNodeMgr.Stat(ctx))
ret.VolumeStat = s.VolumeMgr.Stat(ctx)
ret.ReadOnly = s.Readonly
c.RespondJSON(ret)

2
blobstore/clustermgr/mock/mock.go
View File

@ -19,4 +19,4 @@ package mock
//go:generate mockgen -destination=./kvmgr.go -package=mock -mock_names KvMgrAPI=MockKvMgrAPI github.com/cubefs/cubefs/blobstore/clustermgr/kvmgr KvMgrAPI
//go:generate mockgen -destination=./configmgr.go -package=mock -mock_names ConfigMgrAPI=MockConfigMgrAPI github.com/cubefs/cubefs/blobstore/clustermgr/configmgr ConfigMgrAPI
//go:generate mockgen -destination=./scopemgr.go -package=mock -mock_names ScopeMgrAPI=MockScopeMgrAPI github.com/cubefs/cubefs/blobstore/clustermgr/scopemgr ScopeMgrAPI
//go:generate mockgen -destination=../volumemgr/diskmgr_mock_test.go -package=volumemgr -mock_names DiskMgrAPI=MockDiskMgrAPI github.com/cubefs/cubefs/blobstore/clustermgr/diskmgr DiskMgrAPI
//go:generate mockgen -destination=../cluster/blobnodemgr_mock.go -package=cluster -mock_names DiskMgrAPI=MockDiskMgrAPI github.com/cubefs/cubefs/blobstore/clustermgr/cluster BlobNodeManagerAPI

18
blobstore/clustermgr/node.go
View File

@ -29,14 +29,14 @@ import (
func (s *Service) NodeAdd(c *rpc.Context) {
ctx := c.Request.Context()
span := trace.SpanFromContextSafe(ctx)
args := new(clustermgr.NodeInfo)
args := new(clustermgr.BlobNodeInfo)
if err := c.ParseArgs(args); err != nil {
c.RespondError(err)
return
}
span.Infof("accept NodeAdd request, args: %v", args)
if nodeID, ok := s.DiskMgr.CheckNodeInfoDuplicated(ctx, args); ok {
if nodeID, ok := s.BlobNodeMgr.CheckNodeInfoDuplicated(ctx, &args.NodeInfo); ok {
span.Warnf("node already exist, no need to create again, node info: %v", args)
c.RespondJSON(&clustermgr.NodeIDAllocRet{NodeID: nodeID})
return
@ -56,13 +56,13 @@ func (s *Service) NodeAdd(c *rpc.Context) {
return
}
}
if err := s.DiskMgr.ValidateNodeInfo(ctx, args); err != nil {
if err := s.BlobNodeMgr.ValidateNodeInfo(ctx, &args.NodeInfo); err != nil {
span.Warn("invalid nodeinfo")
c.RespondError(err)
return
}
nodeID, err := s.DiskMgr.AllocNodeID(ctx)
nodeID, err := s.BlobNodeMgr.AllocNodeID(ctx)
if err != nil {
span.Errorf("alloc node id failed =>", errors.Detail(err))
c.RespondError(err)
@ -76,7 +76,7 @@ func (s *Service) NodeAdd(c *rpc.Context) {
c.RespondError(errors.Info(apierrors.ErrUnexpected).Detail(err))
return
}
proposeInfo := base.EncodeProposeInfo(s.DiskMgr.GetModuleName(), cluster.OperTypeAddNode, data, base.ProposeContext{ReqID: span.TraceID()})
proposeInfo := base.EncodeProposeInfo(s.BlobNodeMgr.GetModuleName(), cluster.OperTypeAddNode, data, base.ProposeContext{ReqID: span.TraceID()})
err = s.raftNode.Propose(ctx, proposeInfo)
if err != nil {
span.Error(err)
@ -96,7 +96,7 @@ func (s *Service) NodeDrop(c *rpc.Context) {
}
span.Infof("accept NodeDrop request, args: %v", args)
isDropped, err := s.DiskMgr.IsDroppedNode(ctx, args.NodeID)
isDropped, err := s.BlobNodeMgr.IsDroppedNode(ctx, args.NodeID)
if err != nil {
span.Warnf("NodeDrop isDroppedNode err: %v", err)
c.RespondError(err)
@ -112,7 +112,7 @@ func (s *Service) NodeDrop(c *rpc.Context) {
c.RespondError(errors.Info(apierrors.ErrUnexpected).Detail(err))
return
}
proposeInfo := base.EncodeProposeInfo(s.DiskMgr.GetModuleName(), cluster.OperTypeDropNode, data, base.ProposeContext{ReqID: span.TraceID()})
proposeInfo := base.EncodeProposeInfo(s.BlobNodeMgr.GetModuleName(), cluster.OperTypeDropNode, data, base.ProposeContext{ReqID: span.TraceID()})
err = s.raftNode.Propose(ctx, proposeInfo)
if err != nil {
span.Error(err)
@ -138,7 +138,7 @@ func (s *Service) NodeInfo(c *rpc.Context) {
return
}
ret, err := s.DiskMgr.GetNodeInfo(ctx, args.NodeID)
ret, err := s.BlobNodeMgr.GetNodeInfo(ctx, args.NodeID)
if err != nil {
span.Warnf("node not found: %d", args.NodeID)
c.RespondError(err)
@ -158,5 +158,5 @@ func (s *Service) TopoInfo(c *rpc.Context) {
c.RespondError(apierrors.ErrRaftReadIndex)
return
}
c.RespondJSON(s.DiskMgr.GetTopoInfo(ctx))
c.RespondJSON(s.BlobNodeMgr.GetTopoInfo(ctx))
}

4
blobstore/clustermgr/node_test.go
View File

@ -102,8 +102,8 @@ func TestTopoInfo(t *testing.T) {
require.NoError(t, err)
var diskSetMaxLen, nodeSetMaxLen int
blobNodeHDDNodeSets := ret.AllNodeSets[proto.NodeRoleBlobNode.String()][proto.DiskTypeHDD.String()]
copySetConf := testService.DiskMgr.DiskMgrConfig.CopySetConfigs[proto.NodeRoleBlobNode][proto.DiskTypeHDD]
blobNodeHDDNodeSets := ret.AllNodeSets[proto.DiskTypeHDD.String()]
copySetConf := testService.Config.DiskMgrConfig.CopySetConfigs[proto.NodeRoleBlobNode][proto.DiskTypeHDD]
diskSetCap, nodeSetCap, diskSetIdcCap := copySetConf.DiskSetCap, copySetConf.NodeSetCap, copySetConf.NodeSetIdcCap
for _, nodeSet := range blobNodeHDDNodeSets {
if nodeSet.Number > nodeSetMaxLen {

19
blobstore/clustermgr/persistence/normaldb/blobnodedisktbl.go
View File

@ -1,3 +1,17 @@
// Copyright 2024 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package normaldb
import (
@ -37,6 +51,7 @@ func OpenBlobNodeDiskTable(db kvstore.KVStore, ensureIndex bool) (*BlobNodeDiskT
},
},
}
table.diskTable.rd = table
// ensure index
if ensureIndex {
@ -119,6 +134,6 @@ func (b *BlobNodeDiskTable) diskID(i interface{}) proto.DiskID {
return i.(*BlobNodeDiskInfoRecord).DiskID
}
func (b *BlobNodeDiskTable) diskInfo(i interface{}) DiskInfoRecord {
return i.(*BlobNodeDiskInfoRecord).DiskInfoRecord
func (b *BlobNodeDiskTable) diskInfo(i interface{}) *DiskInfoRecord {
return &i.(*BlobNodeDiskInfoRecord).DiskInfoRecord
}

40
blobstore/clustermgr/persistence/normaldb/blobnodetbl_test.go
View File

@ -12,27 +12,31 @@ import (
)
var nr1 = BlobNodeInfoRecord{
Version: NodeInfoVersionNormal,
NodeID: proto.NodeID(1),
ClusterID: proto.ClusterID(1),
Idc: "z0",
Rack: "rack1",
Host: "127.0.0.1",
Status: proto.NodeStatusNormal,
Role: proto.NodeRoleBlobNode,
DiskType: proto.DiskTypeHDD,
NodeInfoRecord: NodeInfoRecord{
Version: NodeInfoVersionNormal,
NodeID: proto.NodeID(1),
ClusterID: proto.ClusterID(1),
Idc: "z0",
Rack: "rack1",
Host: "127.0.0.1",
Status: proto.NodeStatusNormal,
Role: proto.NodeRoleBlobNode,
DiskType: proto.DiskTypeHDD,
},
}
var nr2 = BlobNodeInfoRecord{
Version: NodeInfoVersionNormal,
NodeID: proto.NodeID(2),
ClusterID: proto.ClusterID(1),
Idc: "z0",
Rack: "rack2",
Host: "127.0.0.2",
Status: proto.NodeStatusNormal,
Role: proto.NodeRoleBlobNode,
DiskType: proto.DiskTypeHDD,
NodeInfoRecord: NodeInfoRecord{
Version: NodeInfoVersionNormal,
NodeID: proto.NodeID(2),
ClusterID: proto.ClusterID(1),
Idc: "z0",
Rack: "rack2",
Host: "127.0.0.2",
Status: proto.NodeStatusNormal,
Role: proto.NodeRoleBlobNode,
DiskType: proto.DiskTypeHDD,
},
}
func TestNodeTbl(t *testing.T) {

4
blobstore/clustermgr/persistence/normaldb/disktbl.go
View File

@ -61,7 +61,7 @@ type diskRecordDescriptor interface {
unmarshalRecord(v []byte) (interface{}, error)
marshalRecord(i interface{}) ([]byte, error)
diskID(i interface{}) proto.DiskID
diskInfo(i interface{}) DiskInfoRecord
diskInfo(i interface{}) *DiskInfoRecord
}
type diskTable struct {
@ -82,7 +82,7 @@ func (d *diskTable) GetDisk(diskID proto.DiskID) (info interface{}, err error) {
if err != nil {
return nil, err
}
info, err = d.rd.marshalRecord(v)
info, err = d.rd.unmarshalRecord(v)
if err != nil {
return
}

58
blobstore/clustermgr/persistence/normaldb/disktbl_test.go
View File

@ -26,19 +26,21 @@ import (
"github.com/stretchr/testify/require"
)
var dr1 = DiskInfoRecord{
Version: DiskInfoVersionNormal,
DiskID: proto.DiskID(1),
ClusterID: proto.ClusterID(1),
Idc: "z0",
Rack: "rack1",
Host: "127.0.0.1",
Path: "",
Status: proto.DiskStatusNormal,
Readonly: false,
var dr1 = BlobNodeDiskInfoRecord{
DiskInfoRecord: DiskInfoRecord{
Version: DiskInfoVersionNormal,
DiskID: proto.DiskID(1),
ClusterID: proto.ClusterID(1),
Idc: "z0",
Rack: "rack1",
Host: "127.0.0.1",
Path: "",
Status: proto.DiskStatusNormal,
Readonly: false,
CreateAt: time.Now(),
LastUpdateAt: time.Now(),
},
UsedChunkCnt: 0,
CreateAt: time.Now(),
LastUpdateAt: time.Now(),
Used: 0,
Size: 100000,
Free: 100000,
@ -47,19 +49,21 @@ var dr1 = DiskInfoRecord{
OversoldFreeChunkCnt: 20,
}
var dr2 = DiskInfoRecord{
Version: DiskInfoVersionNormal,
DiskID: proto.DiskID(2),
ClusterID: proto.ClusterID(1),
Idc: "z0",
Rack: "rack2",
Host: "127.0.0.2",
Path: "",
Status: proto.DiskStatusBroken,
Readonly: false,
var dr2 = BlobNodeDiskInfoRecord{
DiskInfoRecord: DiskInfoRecord{
Version: DiskInfoVersionNormal,
DiskID: proto.DiskID(2),
ClusterID: proto.ClusterID(1),
Idc: "z0",
Rack: "rack2",
Host: "127.0.0.2",
Path: "",
Status: proto.DiskStatusBroken,
Readonly: false,
CreateAt: time.Now(),
LastUpdateAt: time.Now(),
},
UsedChunkCnt: 0,
CreateAt: time.Now(),
LastUpdateAt: time.Now(),
Used: 0,
Size: 100000,
Free: 100000,
@ -94,12 +98,6 @@ func TestDiskTbl(t *testing.T) {
diskList, err = diskTbl.GetAllDisks()
require.NoError(t, err)
require.Equal(t, 2, len(diskList))
err = diskTbl.DeleteDisk(dr2.DiskID)
require.NoError(t, err)
diskList, err = diskTbl.GetAllDisks()
require.NoError(t, err)
require.Equal(t, 1, len(diskList))
}
// get disk and update disk

6
blobstore/clustermgr/persistence/normaldb/shardnodedisktbl.go
View File

@ -13,6 +13,8 @@ import (
type ShardNodeDiskInfoRecord struct {
DiskInfoRecord
MaxShardCnt int32 `json:"max_shard_cnt"`
FreeShardCnt int32 `json:"free_shard_cnt"`
UsedShardCnt int32 `json:"used_shard_cnt"`
Size int64 `json:"size"`
Used int64 `json:"used"`
@ -116,6 +118,6 @@ func (b *ShardNodeDiskTable) diskID(i interface{}) proto.DiskID {
return i.(*ShardNodeDiskInfoRecord).DiskID
}
func (b *ShardNodeDiskTable) diskInfo(i interface{}) DiskInfoRecord {
return i.(*ShardNodeDiskInfoRecord).DiskInfoRecord
func (b *ShardNodeDiskTable) diskInfo(i interface{}) *DiskInfoRecord {
return &i.(*ShardNodeDiskInfoRecord).DiskInfoRecord
}

30
blobstore/clustermgr/svr.go
View File

@ -131,11 +131,11 @@ type Service struct {
ConfigMgr *configmgr.ConfigMgr
ScopeMgr *scopemgr.ScopeMgr
ServiceMgr *servicemgr.ServiceMgr
// Note: DiskMgr should always list before volumeMgr
// cause DiskMgr applier LoadData should be call first, or VolumeMgr LoadData may return error with disk not found
DiskMgr *cluster.manager
VolumeMgr *volumemgr.VolumeMgr
KvMgr *kvmgr.KvMgr
// Note: BlobNodeMgr should always list before volumeMgr
// cause BlobNodeMgr applier LoadData should be call first, or VolumeMgr LoadData may return error with disk not found
BlobNodeMgr *cluster.BlobNodeManager
VolumeMgr *volumemgr.VolumeMgr
KvMgr *kvmgr.KvMgr
dbs map[string]base.SnapshotDB
// status indicate service's current state, like normal/snapshot
@ -234,9 +234,9 @@ func New(cfg *Config) (*Service, error) {
if err != nil {
log.Fatalf("new scopeMgr failed, err: %v", err)
}
diskMgr, err := cluster.New(scopeMgr, normalDB, cfg.DiskMgrConfig)
blobNodeMgr, err := cluster.NewBlobNodeMgr(scopeMgr, normalDB, cfg.DiskMgrConfig)
if err != nil {
log.Fatalf("new diskMgr failed, err: %v", err)
log.Fatalf("new blobNodeMgr failed, err: %v", err)
}
kvMgr, err := kvmgr.NewKvMgr(kvDB)
@ -251,7 +251,7 @@ func New(cfg *Config) (*Service, error) {
serviceMgr := servicemgr.NewServiceMgr(normaldb.OpenServiceTable(normalDB))
volumeMgr, err := volumemgr.NewVolumeMgr(cfg.VolumeMgrConfig, diskMgr, scopeMgr, configMgr, volumeDB)
volumeMgr, err := volumemgr.NewVolumeMgr(cfg.VolumeMgrConfig, blobNodeMgr, scopeMgr, configMgr, volumeDB)
if err != nil {
log.Fatalf("new volumeMgr failed, error: %v", errors.Detail(err))
}
@ -259,7 +259,7 @@ func New(cfg *Config) (*Service, error) {
service.KvMgr = kvMgr
service.VolumeMgr = volumeMgr
service.ConfigMgr = configMgr
service.DiskMgr = diskMgr
service.BlobNodeMgr = blobNodeMgr
service.ServiceMgr = serviceMgr
service.ScopeMgr = scopeMgr
@ -303,7 +303,7 @@ func New(cfg *Config) (*Service, error) {
// set raftServer
service.raftNode.SetRaftServer(raftServer)
diskMgr.SetRaftServer(raftServer)
blobNodeMgr.SetRaftServer(raftServer)
scopeMgr.SetRaftServer(raftServer)
volumeMgr.SetRaftServer(raftServer)
configMgr.SetRaftServer(raftServer)
@ -313,7 +313,7 @@ func New(cfg *Config) (*Service, error) {
volumeMgr.Start()
// refresh disk expire time after all ready
diskMgr.RefreshExpireTime()
blobNodeMgr.RefreshExpireTime()
// start raft node background progress
go raftNode.Start()
@ -353,7 +353,7 @@ func (s *Service) Close() {
// 3. close module manager
s.VolumeMgr.Close()
s.DiskMgr.Close()
s.BlobNodeMgr.Close()
time.Sleep(1 * time.Second)
// 4. close all database
@ -597,7 +597,7 @@ func (s *Service) loop() {
Readonly: s.Readonly,
Nodes: make([]string, 0),
}
spaceStatInfo := s.DiskMgr.Stat(ctx)
spaceStatInfo := s.BlobNodeMgr.Stat(ctx)
clusterInfo.Capacity = spaceStatInfo.TotalSpace
clusterInfo.Available = spaceStatInfo.WritableSpace
// filter learner node
@ -628,7 +628,7 @@ func (s *Service) loop() {
if !s.raftNode.IsLeader() {
continue
}
changes := s.DiskMgr.GetHeartbeatChangeDisks()
changes := s.BlobNodeMgr.GetHeartbeatChangeDisks()
// report heartbeat change metric
s.reportHeartbeatChange(float64(len(changes)))
// in some case, like cm's network problem, it may trigger a mounts of disk heartbeat change
@ -699,7 +699,7 @@ func (s *Service) metricReport(ctx context.Context) {
isLeader := strconv.FormatBool(s.raftNode.IsLeader())
s.report(ctx)
s.VolumeMgr.Report(ctx, s.Region, s.ClusterID)
s.DiskMgr.Report(ctx, s.Region, s.ClusterID, isLeader)
s.BlobNodeMgr.Report(ctx, s.Region, s.ClusterID, isLeader)
}
func (s *Service) checkVolInfos(ctx context.Context, clis []*clustermgr.Client) ([]proto.Vid, error) {

2
blobstore/clustermgr/volume.go
View File

@ -449,7 +449,7 @@ func (s *Service) AdminUpdateVolumeUnit(c *rpc.Context) {
return
}
if args.DiskID > 0 {
_, err := s.DiskMgr.GetDiskInfo(ctx, args.DiskID)
_, err := s.BlobNodeMgr.GetDiskInfo(ctx, args.DiskID)
if err != nil {
c.RespondError(err)
return

53
blobstore/clustermgr/volume_test.go
View File

@ -24,7 +24,6 @@ import (
"github.com/stretchr/testify/require"
"github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/clustermgr/persistence/normaldb"
"github.com/cubefs/cubefs/blobstore/clustermgr/persistence/volumedb"
@ -370,10 +369,10 @@ func TestService_ChunkReport(t *testing.T) {
// chunk report
{
var chunks []blobnode.ChunkInfo
var chunks []clustermgr.ChunkInfo
for i := 1; i < 11; i++ {
vuid := proto.EncodeVuid(proto.EncodeVuidPrefix(proto.Vid(i), 2), 1)
chunk := blobnode.ChunkInfo{
chunk := clustermgr.ChunkInfo{
Vuid: vuid,
Total: uint64(1024 * 2),
Free: uint64(1025),
@ -540,33 +539,37 @@ func generateVolume(volumeDBPath, NormalDBPath string) error {
return err
}
for i := 1; i <= unitCount+3; i++ {
dr := &normaldb.DiskInfoRecord{
Version: normaldb.DiskInfoVersionNormal,
DiskID: proto.DiskID(i),
ClusterID: proto.ClusterID(1),
Path: "",
Status: proto.DiskStatusNormal,
Readonly: false,
UsedChunkCnt: 0,
CreateAt: time.Now(),
LastUpdateAt: time.Now(),
dr := &normaldb.BlobNodeDiskInfoRecord{
DiskInfoRecord: normaldb.DiskInfoRecord{
Version: normaldb.DiskInfoVersionNormal,
DiskID: proto.DiskID(i),
ClusterID: proto.ClusterID(1),
Path: "",
Status: proto.DiskStatusNormal,
Readonly: false,
CreateAt: time.Now(),
LastUpdateAt: time.Now(),
NodeID: proto.NodeID(i),
},
Used: 0,
Size: 100000,
Free: 100000,
MaxChunkCnt: 10,
FreeChunkCnt: 10,
NodeID: proto.NodeID(i),
UsedChunkCnt: 0,
}
nr := &normaldb.BlobNodeInfoRecord{
Version: normaldb.NodeInfoVersionNormal,
ClusterID: proto.ClusterID(1),
NodeID: proto.NodeID(i),
Idc: "z0",
Rack: "rack1",
Host: "http://127.0.0." + strconv.Itoa(i) + ":80800",
Role: proto.NodeRoleBlobNode,
Status: proto.NodeStatusNormal,
DiskType: proto.DiskTypeHDD,
NodeInfoRecord: normaldb.NodeInfoRecord{
Version: normaldb.NodeInfoVersionNormal,
ClusterID: proto.ClusterID(1),
NodeID: proto.NodeID(i),
Idc: "z0",
Rack: "rack1",
Host: "http://127.0.0." + strconv.Itoa(i) + ":80800",
Role: proto.NodeRoleBlobNode,
Status: proto.NodeStatusNormal,
DiskType: proto.DiskTypeHDD,
},
}
if i >= 9 && i < 18 {
dr.Idc = "z1"
@ -654,10 +657,10 @@ func BenchmarkService_ChunkReport(b *testing.B) {
cmClient := initTestClusterClient(testService)
ctx := newCtx()
var chunks []blobnode.ChunkInfo
var chunks []clustermgr.ChunkInfo
for i := 1; i < 11; i++ {
vuid := proto.EncodeVuid(proto.EncodeVuidPrefix(proto.Vid(i), 2), 1)
chunk := blobnode.ChunkInfo{
chunk := clustermgr.ChunkInfo{
Vuid: vuid,
Total: uint64(1024 * 2),
Free: uint64(1025),

3
blobstore/clustermgr/volumemgr/applier_test.go
View File

@ -20,7 +20,6 @@ import (
"testing"
"time"
"github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/clustermgr/base"
"github.com/cubefs/cubefs/blobstore/clustermgr/persistence/volumedb"
@ -148,7 +147,7 @@ func TestVolumeMgr_Apply(t *testing.T) {
// OperTypeChunkReport
{
args := &clustermgr.ReportChunkArgs{
ChunkInfos: []blobnode.ChunkInfo{
ChunkInfos: []clustermgr.ChunkInfo{
{
Vuid: proto.EncodeVuid(4294967296, 1),
DiskID: 1,

4
blobstore/clustermgr/volumemgr/createvolume_test.go
View File

@ -42,7 +42,7 @@ func TestVolumeMgr_CreateVolume(t *testing.T) {
mockRaftServer := mocks.NewMockRaftServer(ctr)
mockRaftServer.EXPECT().Status().AnyTimes().Return(raftserver.Status{Id: 1})
mockScopeMgr := mock.NewMockScopeMgrAPI(ctr)
mockDiskMgr := NewMockDiskMgrAPI(ctr)
mockDiskMgr := cluster.NewMockBlobNodeManagerAPI(ctr)
mockDiskMgr.EXPECT().AllocChunks(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(func(ctx context.Context, policy cluster.AllocPolicy) ([]proto.DiskID, []proto.Vuid, error) {
diskids := make([]proto.DiskID, len(policy.Vuids))
for i := range diskids {
@ -145,7 +145,7 @@ func TestVolumeMgr_finishLastCreateJob(t *testing.T) {
mockRaftServer := mocks.NewMockRaftServer(ctr)
mockScopeMgr := mock.NewMockScopeMgrAPI(ctr)
mockVolumeMgr.raftServer = mockRaftServer
mockDiskMgr := NewMockDiskMgrAPI(ctr)
mockDiskMgr := cluster.NewMockBlobNodeManagerAPI(ctr)
mockRaftServer.EXPECT().Status().AnyTimes().Return(raftserver.Status{Id: 1})
allocSuccess := func(n int) {
mockDiskMgr.EXPECT().AllocChunks(gomock.Any(), gomock.Any()).MaxTimes(n).DoAndReturn(func(ctx context.Context, policy cluster.AllocPolicy) ([]proto.DiskID, []proto.Vuid, error) {

214
blobstore/clustermgr/volumemgr/diskmgr_mock_test.go
View File

@ -1,214 +0,0 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: github.com/cubefs/cubefs/blobstore/clustermgr/diskmgr (interfaces: NodeManagerAPI)
// Package volumemgr is a generated GoMock package.
package volumemgr
import (
context "context"
reflect "reflect"
clustermgr "github.com/cubefs/cubefs/blobstore/api/clustermgr"
diskmgr "github.com/cubefs/cubefs/blobstore/clustermgr/cluster"
proto "github.com/cubefs/cubefs/blobstore/common/proto"
gomock "github.com/golang/mock/gomock"
)
// MockDiskMgrAPI is a mock of NodeManagerAPI interface.
type MockDiskMgrAPI struct {
ctrl *gomock.Controller
recorder *MockDiskMgrAPIMockRecorder
}
// MockDiskMgrAPIMockRecorder is the mock recorder for MockDiskMgrAPI.
type MockDiskMgrAPIMockRecorder struct {
mock *MockDiskMgrAPI
}
// NewMockDiskMgrAPI creates a new mock instance.
func NewMockDiskMgrAPI(ctrl *gomock.Controller) *MockDiskMgrAPI {
mock := &MockDiskMgrAPI{ctrl: ctrl}
mock.recorder = &MockDiskMgrAPIMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use.
func (m *MockDiskMgrAPI) EXPECT() *MockDiskMgrAPIMockRecorder {
return m.recorder
}
// AllocChunks mocks base method.
func (m *MockDiskMgrAPI) AllocChunks(arg0 context.Context, arg1 diskmgr.AllocPolicy) ([]proto.DiskID, []proto.Vuid, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "AllocChunks", arg0, arg1)
ret0, _ := ret[0].([]proto.DiskID)
ret1, _ := ret[1].([]proto.Vuid)
ret2, _ := ret[2].(error)
return ret0, ret1, ret2
}
// AllocChunks indicates an expected call of AllocChunks.
func (mr *MockDiskMgrAPIMockRecorder) AllocChunks(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocChunks", reflect.TypeOf((*MockDiskMgrAPI)(nil).AllocChunks), arg0, arg1)
}
// AllocDiskID mocks base method.
func (m *MockDiskMgrAPI) AllocDiskID(arg0 context.Context) (proto.DiskID, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "AllocDiskID", arg0)
ret0, _ := ret[0].(proto.DiskID)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// AllocDiskID indicates an expected call of AllocDiskID.
func (mr *MockDiskMgrAPIMockRecorder) AllocDiskID(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocDiskID", reflect.TypeOf((*MockDiskMgrAPI)(nil).AllocDiskID), arg0)
}
// CheckDiskInfoDuplicated mocks base method.
func (m *MockDiskMgrAPI) CheckDiskInfoDuplicated(arg0 context.Context, arg1 *clustermgr.DiskInfo, arg2 *clustermgr.NodeInfo) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "CheckDiskInfoDuplicated", arg0, arg1, arg2)
ret0, _ := ret[0].(error)
return ret0
}
// CheckDiskInfoDuplicated indicates an expected call of CheckDiskInfoDuplicated.
func (mr *MockDiskMgrAPIMockRecorder) CheckDiskInfoDuplicated(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckDiskInfoDuplicated", reflect.TypeOf((*MockDiskMgrAPI)(nil).CheckDiskInfoDuplicated), arg0, arg1, arg2)
}
// GetDiskInfo mocks base method.
func (m *MockDiskMgrAPI) GetDiskInfo(arg0 context.Context, arg1 proto.DiskID) (*clustermgr.DiskInfo, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetDiskInfo", arg0, arg1)
ret0, _ := ret[0].(*clustermgr.DiskInfo)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetDiskInfo indicates an expected call of GetDiskInfo.
func (mr *MockDiskMgrAPIMockRecorder) GetDiskInfo(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDiskInfo", reflect.TypeOf((*MockDiskMgrAPI)(nil).GetDiskInfo), arg0, arg1)
}
// GetHeartbeatChangeDisks mocks base method.
func (m *MockDiskMgrAPI) GetHeartbeatChangeDisks() []diskmgr.HeartbeatEvent {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetHeartbeatChangeDisks")
ret0, _ := ret[0].([]diskmgr.HeartbeatEvent)
return ret0
}
// GetHeartbeatChangeDisks indicates an expected call of GetHeartbeatChangeDisks.
func (mr *MockDiskMgrAPIMockRecorder) GetHeartbeatChangeDisks() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHeartbeatChangeDisks", reflect.TypeOf((*MockDiskMgrAPI)(nil).GetHeartbeatChangeDisks))
}
// IsDiskWritable mocks base method.
func (m *MockDiskMgrAPI) IsDiskWritable(arg0 context.Context, arg1 proto.DiskID) (bool, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "IsDiskWritable", arg0, arg1)
ret0, _ := ret[0].(bool)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// IsDiskWritable indicates an expected call of IsDiskWritable.
func (mr *MockDiskMgrAPIMockRecorder) IsDiskWritable(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsDiskWritable", reflect.TypeOf((*MockDiskMgrAPI)(nil).IsDiskWritable), arg0, arg1)
}
// IsDroppingDisk mocks base method.
func (m *MockDiskMgrAPI) IsDroppingDisk(arg0 context.Context, arg1 proto.DiskID) (bool, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "IsDroppingDisk", arg0, arg1)
ret0, _ := ret[0].(bool)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// IsDroppingDisk indicates an expected call of IsDroppingDisk.
func (mr *MockDiskMgrAPIMockRecorder) IsDroppingDisk(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsDroppingDisk", reflect.TypeOf((*MockDiskMgrAPI)(nil).IsDroppingDisk), arg0, arg1)
}
// ListDiskInfo mocks base method.
func (m *MockDiskMgrAPI) ListDiskInfo(arg0 context.Context, arg1 *clustermgr.ListOptionArgs) (*clustermgr.ListDiskRet, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ListDiskInfo", arg0, arg1)
ret0, _ := ret[0].(*clustermgr.ListDiskRet)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// ListDiskInfo indicates an expected call of ListDiskInfo.
func (mr *MockDiskMgrAPIMockRecorder) ListDiskInfo(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDiskInfo", reflect.TypeOf((*MockDiskMgrAPI)(nil).ListDiskInfo), arg0, arg1)
}
// ListDroppingDisk mocks base method.
func (m *MockDiskMgrAPI) ListDroppingDisk(arg0 context.Context) ([]*clustermgr.DiskInfo, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ListDroppingDisk", arg0)
ret0, _ := ret[0].([]*clustermgr.DiskInfo)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// ListDroppingDisk indicates an expected call of ListDroppingDisk.
func (mr *MockDiskMgrAPIMockRecorder) ListDroppingDisk(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDroppingDisk", reflect.TypeOf((*MockDiskMgrAPI)(nil).ListDroppingDisk), arg0)
}
// SetStatus mocks base method.
func (m *MockDiskMgrAPI) SetStatus(arg0 context.Context, arg1 proto.DiskID, arg2 proto.DiskStatus, arg3 bool) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "SetStatus", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(error)
return ret0
}
// SetStatus indicates an expected call of SetStatus.
func (mr *MockDiskMgrAPIMockRecorder) SetStatus(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetStatus", reflect.TypeOf((*MockDiskMgrAPI)(nil).SetStatus), arg0, arg1, arg2, arg3)
}
// Stat mocks base method.
func (m *MockDiskMgrAPI) Stat(arg0 context.Context) *clustermgr.SpaceStatInfo {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Stat", arg0)
ret0, _ := ret[0].(*clustermgr.SpaceStatInfo)
return ret0
}
// Stat indicates an expected call of Stat.
func (mr *MockDiskMgrAPIMockRecorder) Stat(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stat", reflect.TypeOf((*MockDiskMgrAPI)(nil).Stat), arg0)
}
// SwitchReadonly mocks base method.
func (m *MockDiskMgrAPI) SwitchReadonly(arg0 proto.DiskID, arg1 bool) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "SwitchReadonly", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// SwitchReadonly indicates an expected call of SwitchReadonly.
func (mr *MockDiskMgrAPIMockRecorder) SwitchReadonly(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SwitchReadonly", reflect.TypeOf((*MockDiskMgrAPI)(nil).SwitchReadonly), arg0, arg1)
}

6
blobstore/clustermgr/volumemgr/volume_task_test.go
View File

@ -23,11 +23,11 @@ import (
"testing"
"time"
"github.com/cubefs/cubefs/blobstore/clustermgr/cluster"
"github.com/golang/mock/gomock"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"github.com/cubefs/cubefs/blobstore/api/blobnode"
cm "github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/clustermgr/base"
"github.com/cubefs/cubefs/blobstore/clustermgr/persistence/volumedb"
@ -56,8 +56,8 @@ func TestTaskProc(t *testing.T) {
dnClient.EXPECT().SetChunkReadonly(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
dnClient.EXPECT().SetChunkReadwrite(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
diskmgr := NewMockDiskMgrAPI(ctrl)
diskmgr.EXPECT().GetDiskInfo(gomock.Any(), gomock.Any()).AnyTimes().Return(&clustermgr.DiskInfo{Host: "127.0.0.1:8080"}, nil)
diskmgr := cluster.NewMockBlobNodeManagerAPI(ctrl)
diskmgr.EXPECT().GetDiskInfo(gomock.Any(), gomock.Any()).AnyTimes().Return(&cm.BlobNodeDiskInfo{DiskInfo: cm.DiskInfo{Host: "127.0.0.1:8080"}}, nil)
volMgr := &VolumeMgr{
volumeTbl: volumeTbl,

14
blobstore/clustermgr/volumemgr/volumemgr_test.go
View File

@ -93,7 +93,7 @@ func initMockVolumeMgr(t testing.TB) (*VolumeMgr, func()) {
mockRaftServer := mocks.NewMockRaftServer(ctr)
mockScopeMgr := mock.NewMockScopeMgrAPI(ctr)
mockConfigMgr := mock.NewMockConfigMgrAPI(ctr)
mockDiskMgr := NewMockDiskMgrAPI(ctr)
mockDiskMgr := cluster.NewMockBlobNodeManagerAPI(ctr)
// mockRaftServer.EXPECT().IsLeader().AnyTimes().Return(true)
mockConfigMgr.EXPECT().Delete(gomock.Any(), "mockKey").AnyTimes().Return(nil)
@ -122,11 +122,13 @@ func mockIsDiskWritable(_ context.Context, id proto.DiskID) (bool, error) {
return id != proto.DiskID(29), nil
}
func mockGetDiskInfo(_ context.Context, id proto.DiskID) (*clustermgr.DiskInfo, error) {
return &clustermgr.DiskInfo{
func mockGetDiskInfo(_ context.Context, id proto.DiskID) (*clustermgr.BlobNodeDiskInfo, error) {
return &clustermgr.BlobNodeDiskInfo{
DiskHeartBeatInfo: clustermgr.DiskHeartBeatInfo{DiskID: id},
Idc: "z0",
Host: "127.0.0.1",
DiskInfo: clustermgr.DiskInfo{
Idc: "z0",
Host: "127.0.0.1",
},
}, nil
}
@ -268,7 +270,7 @@ func Test_NewVolumeMgr(t *testing.T) {
mockRaftServer := mocks.NewMockRaftServer(ctr)
mockScopeMgr := mock.NewMockScopeMgrAPI(ctr)
mockConfigMgr := mock.NewMockConfigMgrAPI(ctr)
mockDiskMgr := NewMockDiskMgrAPI(ctr)
mockDiskMgr := cluster.NewMockBlobNodeManagerAPI(ctr)
codeModeConfg := []codemode.Policy{
{

43
blobstore/clustermgr/volumemgr/volumeunit_test.go
View File

@ -21,7 +21,6 @@ import (
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/require"
"github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/clustermgr/cluster"
"github.com/cubefs/cubefs/blobstore/common/proto"
@ -50,7 +49,7 @@ func TestVolumeMgr_AllocVolumeUnit(t *testing.T) {
ctr := gomock.NewController(t)
mockRaftServer := mocks.NewMockRaftServer(ctr)
mockRaftServer.EXPECT().IsLeader().AnyTimes().Return(false)
mockDiskMgr := NewMockDiskMgrAPI(ctr)
mockDiskMgr := cluster.NewMockBlobNodeManagerAPI(ctr)
mockDiskMgr.EXPECT().AllocChunks(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(func(ctx context.Context, policy cluster.AllocPolicy) ([]proto.DiskID, []proto.Vuid, error) {
var diskids []proto.DiskID
for i := range policy.Vuids {
@ -156,7 +155,7 @@ func TestVolumeMgr_updateVolumeUnit(t *testing.T) {
NewDiskID: 30,
}
dnClient.EXPECT().StatChunk(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(
&blobnode.ChunkInfo{Vuid: args.NewVuid, DiskID: 30}, nil)
&clustermgr.ChunkInfo{Vuid: args.NewVuid, DiskID: 30}, nil)
// test preUpdateVolumeUnit()
{
@ -241,15 +240,17 @@ func TestVolumeMgr_updateVolumeUnit(t *testing.T) {
// test applyUpdateVolumeUnit, refresh health return err
{
_, ctx := trace.StartSpanFromContext(context.Background(), "applyVolumeUnit")
mockDiskMgr := NewMockDiskMgrAPI(ctr)
mockDiskMgr.EXPECT().GetDiskInfo(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(func(ctx context.Context, id proto.DiskID) (*clustermgr.DiskInfo, error) {
mockDiskMgr := cluster.NewMockBlobNodeManagerAPI(ctr)
mockDiskMgr.EXPECT().GetDiskInfo(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(func(ctx context.Context, id proto.DiskID) (*clustermgr.BlobNodeDiskInfo, error) {
heatInfo := clustermgr.DiskHeartBeatInfo{
DiskID: id,
}
diskInfo := &clustermgr.DiskInfo{
diskInfo := &clustermgr.BlobNodeDiskInfo{
DiskHeartBeatInfo: heatInfo,
Idc: "z0",
Host: "127.0.0.1",
DiskInfo: clustermgr.DiskInfo{
Idc: "z0",
Host: "127.0.0.1",
},
}
return diskInfo, nil
})
@ -269,7 +270,7 @@ func TestVolumeMgr_updateVolumeUnit(t *testing.T) {
// test applyUpdateVolumeUnit, get diskInfo return error
{
_, ctx := trace.StartSpanFromContext(context.Background(), "applyVolumeUnit")
mockDiskMgr := NewMockDiskMgrAPI(ctr)
mockDiskMgr := cluster.NewMockBlobNodeManagerAPI(ctr)
mockDiskMgr.EXPECT().GetDiskInfo(gomock.Any(), gomock.Any()).AnyTimes().Return(nil, errors.New("err"))
mockDiskMgr.EXPECT().IsDiskWritable(gomock.Any(), gomock.Any()).AnyTimes().Return(true, errors.New("err"))
mockVolumeMgr.diskMgr = mockDiskMgr
@ -375,10 +376,10 @@ func TestVolumeMgr_applyChunkReport(t *testing.T) {
mockVolumeMgr, clean := initMockVolumeMgr(t)
defer clean()
var args []blobnode.ChunkInfo
var args []clustermgr.ChunkInfo
for i := 0; i < volumeCount; i++ {
vuid := proto.EncodeVuid(proto.EncodeVuidPrefix(proto.Vid(i), 2), 1)
chunk := blobnode.ChunkInfo{
chunk := clustermgr.ChunkInfo{
Vuid: vuid,
Total: defaultChunkSize + 1,
Free: uint64(1024 * 1024),
@ -418,10 +419,10 @@ func TestVolumeMgr_applyChunkReportWithVolumeOverbought(t *testing.T) {
mockVolumeMgr, clean := initMockVolumeMgr(t)
defer clean()
var args []blobnode.ChunkInfo
var args []clustermgr.ChunkInfo
for i := 0; i < volumeCount; i++ {
vuid := proto.EncodeVuid(proto.EncodeVuidPrefix(proto.Vid(i), 2), 1)
chunk := blobnode.ChunkInfo{
chunk := clustermgr.ChunkInfo{
Vuid: vuid,
Total: defaultChunkSize,
Free: uint64(0.6 * float64(defaultChunkSize)),
@ -468,20 +469,22 @@ func TestVolumeMgr_ReleaseVolumeUnit(t *testing.T) {
// avoid background loopCreate volume
mockRaftServer.EXPECT().IsLeader().AnyTimes().Return(false)
mockVolumeMgr.raftServer = mockRaftServer
mockDiskMgr := NewMockDiskMgrAPI(ctr)
mockDiskMgr := cluster.NewMockBlobNodeManagerAPI(ctr)
mockVolumeMgr.diskMgr = mockDiskMgr
mockBlobNode := mocks.NewMockStorageAPI(ctr)
mockVolumeMgr.blobNodeClient = mockBlobNode
// mockDiskMgr.EXPECT().Stat(gomock.Any()).AnyTimes().Return(&clustermgr.SpaceStatInfo{TotalDisk: 60})
mockDiskMgr.EXPECT().GetDiskInfo(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, id proto.DiskID) (*clustermgr.DiskInfo, error) {
mockDiskMgr.EXPECT().GetDiskInfo(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, id proto.DiskID) (*clustermgr.BlobNodeDiskInfo, error) {
heatInfo := clustermgr.DiskHeartBeatInfo{
DiskID: 1,
}
diskInfo := &clustermgr.DiskInfo{
diskInfo := &clustermgr.BlobNodeDiskInfo{
DiskHeartBeatInfo: heatInfo,
Idc: "z0",
Host: "127.0.0.1",
DiskInfo: clustermgr.DiskInfo{
Idc: "z0",
Host: "127.0.0.1",
},
}
return diskInfo, nil
})
@ -500,10 +503,10 @@ func BenchmarkVolumeMgr_ChunkReport(b *testing.B) {
mockVolumeMgr, clean := initMockVolumeMgr(b)
defer clean()
var args []blobnode.ChunkInfo
var args []clustermgr.ChunkInfo
for i := 0; i < volumeCount; i++ {
vuid := proto.EncodeVuid(proto.EncodeVuidPrefix(proto.Vid(i), 2), 1)
chunk := blobnode.ChunkInfo{
chunk := clustermgr.ChunkInfo{
Vuid: vuid,
Total: defaultChunkSize + 1,
Free: uint64(1024 * 1024),

4
blobstore/common/proto/basic.go
View File

@ -97,3 +97,7 @@ func (id *NodeID) Decode(b []byte) NodeID {
func (id NodeID) ToString() string {
return strconv.FormatUint(uint64(id), 10)
}
func (id ShardID) ToString() string {
return strconv.FormatUint(uint64(id), 10)
}

3
blobstore/common/proto/const.go
View File

@ -253,3 +253,6 @@ func (t TaskSwitch) Valid() bool {
func (t TaskSwitch) String() string {
return string(t)
}
// MaxShardSize max size of a single shard
const MaxShardSize = 512 << 20

14
blobstore/common/proto/shard.go
View File

@ -1,3 +1,17 @@
// Copyright 2024 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package proto
type SpaceStatus uint8

14
blobstore/common/sharding/hashrange.go
View File

@ -1,3 +1,17 @@
// Copyright 2024 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package sharding
import (

14
blobstore/common/sharding/proto.go
View File

@ -1,3 +1,17 @@
// Copyright 2024 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package sharding
import (

14
blobstore/common/sharding/rangeext.go
View File

@ -1,3 +1,17 @@
// Copyright 2024 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package sharding
import "fmt"

3
blobstore/proxy/cacher/cacher.go
View File

@ -27,7 +27,6 @@ import (
"github.com/peterbourgon/diskv/v3"
"golang.org/x/sync/singleflight"
"github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/api/proxy"
errcode "github.com/cubefs/cubefs/blobstore/common/errors"
@ -113,7 +112,7 @@ func parseID(key string) (uint32, error) {
// Cacher memory cache handlers.
type Cacher interface {
GetVolume(ctx context.Context, args *proxy.CacheVolumeArgs) (*proxy.VersionVolume, error)
GetDisk(ctx context.Context, args *proxy.CacheDiskArgs) (*clustermgr.DiskInfo, error)
GetDisk(ctx context.Context, args *proxy.CacheDiskArgs) (*clustermgr.BlobNodeDiskInfo, error)
// Erase remove all if key is "ALL".
Erase(ctx context.Context, key string) error
}

3
blobstore/proxy/cacher/cacher_test.go
View File

@ -27,7 +27,6 @@ import (
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/require"
"github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/api/proxy"
errcode "github.com/cubefs/cubefs/blobstore/common/errors"
@ -122,7 +121,7 @@ func TestProxyCacherErase(t *testing.T) {
{
cmCli.EXPECT().GetVolumeInfo(A, A).Return(&clustermgr.VolumeInfo{}, nil)
cmCli.EXPECT().DiskInfo(A, A).Return(&clustermgr.DiskInfo{}, nil)
cmCli.EXPECT().DiskInfo(A, A).Return(&clustermgr.BlobNodeDiskInfo{}, nil)
_, err := c.GetVolume(context.Background(), &proxy.CacheVolumeArgs{Vid: 1})
require.NoError(t, err)
<-cc.syncChan

Some files were not shown because too many files have changed in this diff Show More