gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 02:00:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(clustermgr): add ChunkOversoldRatio and VolumeOverBoughtRatio support

Browse Source 
with #1000088261

Signed-off-by: Cloudstriff <chenjiongwendao@qq.com>
This commit is contained in:
Cloudstriff 2025-04-28 17:36:52 +08:00 committed by slasher
parent 92360e00a7
commit 319953c846
12 changed files with 112 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
blobstore/api/blobnode/datainfo.go
View File

@ -28,6 +28,8 @@ type DiskHeartBeatInfo struct {
MaxChunkCnt int64 `json:"max_chunk_cnt"` // note: maintained by clustermgr
FreeChunkCnt int64 `json:"free_chunk_cnt"` // note: maintained by clustermgr
UsedChunkCnt int64 `json:"used_chunk_cnt"` // current number of chunks on the disk
OversoldFreeChunkCnt int64 `json:"oversold_free_chunk_cnt"` // note: maintained by clustermgr
}
type DiskInfo struct {

25
blobstore/api/clustermgr/disk.go
View File

@ -68,18 +68,19 @@ type DiskHeartbeatRet struct {
}
type DiskStatInfo struct {
IDC string `json:"idc"`
Total int `json:"total"`
TotalChunk int64 `json:"total_chunk"`
TotalFreeChunk int64 `json:"total_free_chunk"`
Available int `json:"available"`
Readonly int `json:"readonly"`
Expired int `json:"expired"`
Broken int `json:"broken"`
Repairing int `json:"repairing"`
Repaired int `json:"repaired"`
Dropping int `json:"dropping"`
Dropped int `json:"dropped"`
IDC string `json:"idc"`
Total int `json:"total"`
TotalChunk int64 `json:"total_chunk"`
TotalFreeChunk int64 `json:"total_free_chunk"`
TotalOversoldFreeChunk int64 `json:"total_oversold_free_chunk"`
Available int `json:"available"`
Readonly int `json:"readonly"`
Expired int `json:"expired"`
Broken int `json:"broken"`
Repairing int `json:"repairing"`
Repaired int `json:"repaired"`
Dropping int `json:"dropping"`
Dropped int `json:"dropped"`
}
type SpaceStatInfo struct {

11
blobstore/clustermgr/diskmgr/alloc.go
View File

@ -316,7 +316,11 @@ func (d *blobNodeAllocator) allocDisk(ctx context.Context, excludes map[proto.Di
disk := disks[randNum]
disk.lock.RLock()
defer disk.lock.RUnlock()
freeChunk := disk.info.FreeChunkCnt
if disk.info.OversoldFreeChunkCnt > disk.info.FreeChunkCnt {
freeChunk = disk.info.OversoldFreeChunkCnt
}
if freeChunk <= 0 {
return nil
}
@ -372,7 +376,12 @@ func (s *idcAllocator) alloc(ctx context.Context, count int, excludes map[proto.
}
for id := range chosenDisks {
chosenDisks[id].lock.Lock()
chosenDisks[id].info.FreeChunkCnt -= 1
if chosenDisks[id].info.FreeChunkCnt > 0 {
chosenDisks[id].info.FreeChunkCnt -= 1
}
if chosenDisks[id].info.OversoldFreeChunkCnt > 0 {
chosenDisks[id].info.OversoldFreeChunkCnt -= 1
}
chosenDisks[id].lock.Unlock()
ret = append(ret, id)
}

1
blobstore/clustermgr/diskmgr/alloc_test.go
View File

@ -45,6 +45,7 @@ var testDiskMgrConfig = DiskMgrConfig{
FlushIntervalS: 300,
ChunkSize: 17179869184, // 16G
CodeModes: []codemode.CodeMode{codemode.EC15P12, codemode.EC6P6},
ChunkOversoldRatio: 0.5,
CopySetConfigs: make(map[proto.NodeRole]map[proto.DiskType]CopySetConfig),
}

4
blobstore/clustermgr/diskmgr/diskmgr.go
View File

@ -130,6 +130,7 @@ type DiskMgrConfig struct {
IDC []string `json:"-"`
CodeModes []codemode.CodeMode `json:"-"`
ChunkSize int64 `json:"-"`
ChunkOversoldRatio float64 `json:"-"`
CopySetConfigs map[proto.NodeRole]map[proto.DiskType]CopySetConfig `json:"copy_set_configs"`
}
@ -974,6 +975,9 @@ func (d *DiskMgr) heartBeatDiskInfo(ctx context.Context, infos []*blobnode.DiskH
diskInfo.info.MaxChunkCnt = info.Size / d.ChunkSize
// use the minimum value as free chunk count
diskInfo.info.FreeChunkCnt = diskInfo.info.MaxChunkCnt - diskInfo.info.UsedChunkCnt
if d.ChunkOversoldRatio > 0 {
diskInfo.info.OversoldFreeChunkCnt = int64(float64(diskInfo.info.MaxChunkCnt)*(1+d.ChunkOversoldRatio)) - diskInfo.info.UsedChunkCnt
}
freeChunkCnt := info.Free / d.ChunkSize
if freeChunkCnt < diskInfo.info.FreeChunkCnt {
span.Debugf("use minimum free chunk count, disk id[%d], free chunk[%d]", diskInfo.diskID, freeChunkCnt)

4
blobstore/clustermgr/diskmgr/diskmgr_test.go
View File

@ -193,7 +193,7 @@ func TestDiskMgr_Heartbeat(t *testing.T) {
for i := 1; i <= 10; i++ {
diskInfo, err := testDiskMgr.GetDiskInfo(ctx, proto.DiskID(i))
require.NoError(t, err)
diskInfo.DiskHeartBeatInfo.Free = 0
// diskInfo.DiskHeartBeatInfo.Free = 0
diskInfo.DiskHeartBeatInfo.FreeChunkCnt = 0
heartbeatInfos = append(heartbeatInfos, &diskInfo.DiskHeartBeatInfo)
}
@ -205,7 +205,7 @@ func TestDiskMgr_Heartbeat(t *testing.T) {
diskInfo, err := testDiskMgr.GetDiskInfo(ctx, proto.DiskID(i))
require.NoError(t, err)
require.Equal(t, diskInfo.Free/testDiskMgr.ChunkSize, diskInfo.FreeChunkCnt)
require.Equal(t, int64(0), diskInfo.Free)
require.Greater(t, diskInfo.OversoldFreeChunkCnt, diskInfo.FreeChunkCnt)
}
// get heartbeat change disk

8
blobstore/clustermgr/diskmgr/refresh.go
View File

@ -192,7 +192,10 @@ func (d *DiskMgr) generateDiskSetStorage(ctx context.Context, disks []*diskItem,
rack = node.info.Rack
host = node.info.Host
}
freeChunk := disk.info.FreeChunkCnt
originalFreeChunk, freeChunk := disk.info.FreeChunkCnt, disk.info.FreeChunkCnt
if disk.info.OversoldFreeChunkCnt > freeChunk {
freeChunk = disk.info.OversoldFreeChunkCnt
}
maxChunk := disk.info.MaxChunkCnt
readonly := disk.info.Readonly
size := disk.info.Size
@ -207,7 +210,8 @@ func (d *DiskMgr) generateDiskSetStorage(ctx context.Context, disks []*diskItem,
}
diskStatInfosM[idc].Total += 1
diskStatInfosM[idc].TotalChunk += maxChunk
diskStatInfosM[idc].TotalFreeChunk += freeChunk
diskStatInfosM[idc].TotalFreeChunk += originalFreeChunk
diskStatInfosM[idc].TotalOversoldFreeChunk += freeChunk
if readonly {
diskStatInfosM[idc].Readonly += 1
}

11
blobstore/clustermgr/svr.go
View File

@ -402,6 +402,17 @@ func (c *Config) checkAndFix() (err error) {
if c.ClusterCfg[proto.VolumeReserveSizeKey] == nil {
c.ClusterCfg[proto.VolumeReserveSizeKey] = DefaultVolumeReserveSize
}
if c.ClusterCfg[proto.VolumeOverboughtRatioKey] != nil || c.ClusterCfg[proto.ChunkOversoldRatioKey] != nil {
volumeOverboughtRatio, _ := strconv.ParseFloat(c.ClusterCfg[proto.VolumeOverboughtRatioKey].(string), 64)
chunkOversoldRatio, _ := strconv.ParseFloat(c.ClusterCfg[proto.ChunkOversoldRatioKey].(string), 64)
if chunkOversoldRatio < 0.5*volumeOverboughtRatio || chunkOversoldRatio > volumeOverboughtRatio {
return errors.New("ChunkOversoldRatio must be between 0.5*VolumeOverboughtRatioKey and VolumeOverboughtRatioKey")
}
c.VolumeMgrConfig.VolumeOverboughtRatio = volumeOverboughtRatio
c.DiskMgrConfig.ChunkOversoldRatio = chunkOversoldRatio
}
c.VolumeMgrConfig.ChunkSize = c.ChunkSize
c.DiskMgrConfig.ChunkSize = int64(c.ChunkSize)
c.ClusterCfg[proto.VolumeChunkSizeKey] = c.ChunkSize

2
blobstore/clustermgr/volumemgr/startup.go
View File

@ -65,6 +65,8 @@ type VolumeMgrConfig struct {
CodeModePolicies []codemode.Policy `json:"-"`
Region string `json:"-"`
ClusterID proto.ClusterID `json:"-"`
VolumeOverboughtRatio float64 `json:"-"`
}
func (c *VolumeMgrConfig) checkAndFix() {

12
blobstore/clustermgr/volumemgr/volumeunit.go
View File

@ -309,12 +309,20 @@ func (v *VolumeMgr) applyChunkReport(ctx context.Context, chunks *cmapi.ReportCh
vol.vUnits[idx].vuInfo.Total = chunk.Total
dataChunkNum := uint64(v.codeMode[vol.volInfoBase.CodeMode].tactic.N)
volFree := vol.vUnits[idx].vuInfo.Free * dataChunkNum
volUsed := vol.vUnits[idx].vuInfo.Used * dataChunkNum
volTotal := vol.vUnits[idx].vuInfo.Total * dataChunkNum
volFree := vol.vUnits[idx].vuInfo.Free * dataChunkNum
if v.VolumeOverboughtRatio > 0 {
volFree = 0
_volTotal := uint64(float64(volTotal) * v.VolumeOverboughtRatio)
if _volTotal > volUsed {
volFree = _volTotal - volUsed
}
}
// use the minimum free size as volume free
if vol.volInfoBase.Free > volFree {
// or use calculated free size when overboughtRatio is specified as overboughtRatio may set into larger value
if vol.volInfoBase.Free > volFree || v.VolumeOverboughtRatio > 0 {
vol.volInfoBase.Used = volUsed
vol.volInfoBase.Total = volTotal
vol.smallestVUIdx = idx

47
blobstore/clustermgr/volumemgr/volumeunit_test.go
View File

@ -382,14 +382,20 @@ func TestVolumeMgr_applyChunkReport(t *testing.T) {
Vuid: vuid,
Total: defaultChunkSize + 1,
Free: uint64(1024 * 1024),
Used: defaultChunkSize - uint64(1024*i),
Used: defaultChunkSize - uint64(1024*1024),
}
args = append(args, chunk)
}
// set report chunkInfo(vid:1) epoch=2,
// set report chunkInfo(vid:1) epoch=2,
args[1].Vuid = proto.EncodeVuid(proto.EncodeVuidPrefix(proto.Vid(1), 2), 0)
err := mockVolumeMgr.applyChunkReport(context.Background(), &clustermgr.ReportChunkArgs{ChunkInfos: args})
require.NoError(t, err)
// validate volume info
vol := mockVolumeMgr.all.getVol(args[0].Vuid.Vid())
dataChunkNum := uint64(mockVolumeMgr.codeMode[vol.volInfoBase.CodeMode].tactic.N)
require.Equal(t, args[0].Free*dataChunkNum, vol.volInfoBase.Free)
for i := 0; i < len(args); i++ {
vol := mockVolumeMgr.all.getVol(args[i].Vuid.Vid())
unit := vol.vUnits[args[i].Vuid.Index()]
@ -408,6 +414,43 @@ func TestVolumeMgr_applyChunkReport(t *testing.T) {
require.NoError(t, err)
}
func TestVolumeMgr_applyChunkReportWithVolumeOverbought(t *testing.T) {
mockVolumeMgr, clean := initMockVolumeMgr(t)
defer clean()
var args []blobnode.ChunkInfo
for i := 0; i < volumeCount; i++ {
vuid := proto.EncodeVuid(proto.EncodeVuidPrefix(proto.Vid(i), 2), 1)
chunk := blobnode.ChunkInfo{
Vuid: vuid,
Total: defaultChunkSize,
Free: uint64(0.6 * float64(defaultChunkSize)),
Used: uint64(0.4 * float64(defaultChunkSize)),
}
args = append(args, chunk)
}
// with VolumeOverboughtRatio, validate volume free size after chunk report
// as volume total * VolumeOverboughtRatio is larger than volume used, volume free should between 0 and calculated volume free
mockVolumeMgr.VolumeOverboughtRatio = 0.5
vol := mockVolumeMgr.all.getVol(args[0].Vuid.Vid())
err := mockVolumeMgr.applyChunkReport(context.Background(), &clustermgr.ReportChunkArgs{ChunkInfos: args})
require.NoError(t, err)
require.Greater(t, vol.volInfoBase.Free, uint64(0))
dataChunkNum := uint64(mockVolumeMgr.codeMode[vol.volInfoBase.CodeMode].tactic.N)
require.Less(t, vol.volInfoBase.Free, args[0].Free*dataChunkNum)
// with VolumeOverboughtRatio, validate volume free size after chunk report
// as volume total * VolumeOverboughtRatio is less than volume used, volume free should be set into 0
for i := range args {
args[i].Used = uint64(0.5 * float64(args[i].Total))
args[i].Free = args[i].Total - args[i].Used
}
err = mockVolumeMgr.applyChunkReport(context.Background(), &clustermgr.ReportChunkArgs{ChunkInfos: args})
require.NoError(t, err)
require.Equal(t, uint64(0), vol.volInfoBase.Free)
}
func TestVolumeMgr_ReleaseVolumeUnit(t *testing.T) {
mockVolumeMgr, clean := initMockVolumeMgr(t)
defer clean()

10
blobstore/common/proto/const.go
View File

@ -216,11 +216,13 @@ const (
volumeStatusMax
)
// system config key,not allow delete
// system config key, not allow to delete
const (
CodeModeConfigKey = "code_mode"
VolumeReserveSizeKey = "volume_reserve_size"
VolumeChunkSizeKey = "volume_chunk_size"
CodeModeConfigKey = "code_mode"
VolumeReserveSizeKey = "volume_reserve_size"
VolumeChunkSizeKey = "volume_chunk_size"
VolumeOverboughtRatioKey = "volume_overbought_ratio"
ChunkOversoldRatioKey = "chunk_oversold_ratio"
)
func IsSysConfigKey(key string) bool {