diff --git a/blobstore/api/blobnode/datainfo.go b/blobstore/api/blobnode/datainfo.go index 62f998395..a2c0a5b4a 100644 --- a/blobstore/api/blobnode/datainfo.go +++ b/blobstore/api/blobnode/datainfo.go @@ -28,6 +28,8 @@ type DiskHeartBeatInfo struct { MaxChunkCnt int64 `json:"max_chunk_cnt"` // note: maintained by clustermgr FreeChunkCnt int64 `json:"free_chunk_cnt"` // note: maintained by clustermgr UsedChunkCnt int64 `json:"used_chunk_cnt"` // current number of chunks on the disk + + OversoldFreeChunkCnt int64 `json:"oversold_free_chunk_cnt"` // note: maintained by clustermgr } type DiskInfo struct { diff --git a/blobstore/api/clustermgr/disk.go b/blobstore/api/clustermgr/disk.go index 30266ef01..763dd90a6 100644 --- a/blobstore/api/clustermgr/disk.go +++ b/blobstore/api/clustermgr/disk.go @@ -68,18 +68,19 @@ type DiskHeartbeatRet struct { } type DiskStatInfo struct { - IDC string `json:"idc"` - Total int `json:"total"` - TotalChunk int64 `json:"total_chunk"` - TotalFreeChunk int64 `json:"total_free_chunk"` - Available int `json:"available"` - Readonly int `json:"readonly"` - Expired int `json:"expired"` - Broken int `json:"broken"` - Repairing int `json:"repairing"` - Repaired int `json:"repaired"` - Dropping int `json:"dropping"` - Dropped int `json:"dropped"` + IDC string `json:"idc"` + Total int `json:"total"` + TotalChunk int64 `json:"total_chunk"` + TotalFreeChunk int64 `json:"total_free_chunk"` + TotalOversoldFreeChunk int64 `json:"total_oversold_free_chunk"` + Available int `json:"available"` + Readonly int `json:"readonly"` + Expired int `json:"expired"` + Broken int `json:"broken"` + Repairing int `json:"repairing"` + Repaired int `json:"repaired"` + Dropping int `json:"dropping"` + Dropped int `json:"dropped"` } type SpaceStatInfo struct { diff --git a/blobstore/clustermgr/diskmgr/alloc.go b/blobstore/clustermgr/diskmgr/alloc.go index dd4462748..622a35314 100644 --- a/blobstore/clustermgr/diskmgr/alloc.go +++ b/blobstore/clustermgr/diskmgr/alloc.go @@ -316,7 +316,11 @@ func (d *blobNodeAllocator) allocDisk(ctx context.Context, excludes map[proto.Di disk := disks[randNum] disk.lock.RLock() defer disk.lock.RUnlock() + freeChunk := disk.info.FreeChunkCnt + if disk.info.OversoldFreeChunkCnt > disk.info.FreeChunkCnt { + freeChunk = disk.info.OversoldFreeChunkCnt + } if freeChunk <= 0 { return nil } @@ -372,7 +376,12 @@ func (s *idcAllocator) alloc(ctx context.Context, count int, excludes map[proto. } for id := range chosenDisks { chosenDisks[id].lock.Lock() - chosenDisks[id].info.FreeChunkCnt -= 1 + if chosenDisks[id].info.FreeChunkCnt > 0 { + chosenDisks[id].info.FreeChunkCnt -= 1 + } + if chosenDisks[id].info.OversoldFreeChunkCnt > 0 { + chosenDisks[id].info.OversoldFreeChunkCnt -= 1 + } chosenDisks[id].lock.Unlock() ret = append(ret, id) } diff --git a/blobstore/clustermgr/diskmgr/alloc_test.go b/blobstore/clustermgr/diskmgr/alloc_test.go index 8e2bdf42e..7a8256d74 100644 --- a/blobstore/clustermgr/diskmgr/alloc_test.go +++ b/blobstore/clustermgr/diskmgr/alloc_test.go @@ -45,6 +45,7 @@ var testDiskMgrConfig = DiskMgrConfig{ FlushIntervalS: 300, ChunkSize: 17179869184, // 16G CodeModes: []codemode.CodeMode{codemode.EC15P12, codemode.EC6P6}, + ChunkOversoldRatio: 0.5, CopySetConfigs: make(map[proto.NodeRole]map[proto.DiskType]CopySetConfig), } diff --git a/blobstore/clustermgr/diskmgr/diskmgr.go b/blobstore/clustermgr/diskmgr/diskmgr.go index 2583c8a58..084c10581 100644 --- a/blobstore/clustermgr/diskmgr/diskmgr.go +++ b/blobstore/clustermgr/diskmgr/diskmgr.go @@ -130,6 +130,7 @@ type DiskMgrConfig struct { IDC []string `json:"-"` CodeModes []codemode.CodeMode `json:"-"` ChunkSize int64 `json:"-"` + ChunkOversoldRatio float64 `json:"-"` CopySetConfigs map[proto.NodeRole]map[proto.DiskType]CopySetConfig `json:"copy_set_configs"` } @@ -974,6 +975,9 @@ func (d *DiskMgr) heartBeatDiskInfo(ctx context.Context, infos []*blobnode.DiskH diskInfo.info.MaxChunkCnt = info.Size / d.ChunkSize // use the minimum value as free chunk count diskInfo.info.FreeChunkCnt = diskInfo.info.MaxChunkCnt - diskInfo.info.UsedChunkCnt + if d.ChunkOversoldRatio > 0 { + diskInfo.info.OversoldFreeChunkCnt = int64(float64(diskInfo.info.MaxChunkCnt)*(1+d.ChunkOversoldRatio)) - diskInfo.info.UsedChunkCnt + } freeChunkCnt := info.Free / d.ChunkSize if freeChunkCnt < diskInfo.info.FreeChunkCnt { span.Debugf("use minimum free chunk count, disk id[%d], free chunk[%d]", diskInfo.diskID, freeChunkCnt) diff --git a/blobstore/clustermgr/diskmgr/diskmgr_test.go b/blobstore/clustermgr/diskmgr/diskmgr_test.go index 7b7950161..7dc3603c0 100644 --- a/blobstore/clustermgr/diskmgr/diskmgr_test.go +++ b/blobstore/clustermgr/diskmgr/diskmgr_test.go @@ -193,7 +193,7 @@ func TestDiskMgr_Heartbeat(t *testing.T) { for i := 1; i <= 10; i++ { diskInfo, err := testDiskMgr.GetDiskInfo(ctx, proto.DiskID(i)) require.NoError(t, err) - diskInfo.DiskHeartBeatInfo.Free = 0 + // diskInfo.DiskHeartBeatInfo.Free = 0 diskInfo.DiskHeartBeatInfo.FreeChunkCnt = 0 heartbeatInfos = append(heartbeatInfos, &diskInfo.DiskHeartBeatInfo) } @@ -205,7 +205,7 @@ func TestDiskMgr_Heartbeat(t *testing.T) { diskInfo, err := testDiskMgr.GetDiskInfo(ctx, proto.DiskID(i)) require.NoError(t, err) require.Equal(t, diskInfo.Free/testDiskMgr.ChunkSize, diskInfo.FreeChunkCnt) - require.Equal(t, int64(0), diskInfo.Free) + require.Greater(t, diskInfo.OversoldFreeChunkCnt, diskInfo.FreeChunkCnt) } // get heartbeat change disk diff --git a/blobstore/clustermgr/diskmgr/refresh.go b/blobstore/clustermgr/diskmgr/refresh.go index e60cd627d..e0301a81e 100644 --- a/blobstore/clustermgr/diskmgr/refresh.go +++ b/blobstore/clustermgr/diskmgr/refresh.go @@ -192,7 +192,10 @@ func (d *DiskMgr) generateDiskSetStorage(ctx context.Context, disks []*diskItem, rack = node.info.Rack host = node.info.Host } - freeChunk := disk.info.FreeChunkCnt + originalFreeChunk, freeChunk := disk.info.FreeChunkCnt, disk.info.FreeChunkCnt + if disk.info.OversoldFreeChunkCnt > freeChunk { + freeChunk = disk.info.OversoldFreeChunkCnt + } maxChunk := disk.info.MaxChunkCnt readonly := disk.info.Readonly size := disk.info.Size @@ -207,7 +210,8 @@ func (d *DiskMgr) generateDiskSetStorage(ctx context.Context, disks []*diskItem, } diskStatInfosM[idc].Total += 1 diskStatInfosM[idc].TotalChunk += maxChunk - diskStatInfosM[idc].TotalFreeChunk += freeChunk + diskStatInfosM[idc].TotalFreeChunk += originalFreeChunk + diskStatInfosM[idc].TotalOversoldFreeChunk += freeChunk if readonly { diskStatInfosM[idc].Readonly += 1 } diff --git a/blobstore/clustermgr/svr.go b/blobstore/clustermgr/svr.go index 968073cf8..dd8cecd22 100644 --- a/blobstore/clustermgr/svr.go +++ b/blobstore/clustermgr/svr.go @@ -402,6 +402,17 @@ func (c *Config) checkAndFix() (err error) { if c.ClusterCfg[proto.VolumeReserveSizeKey] == nil { c.ClusterCfg[proto.VolumeReserveSizeKey] = DefaultVolumeReserveSize } + if c.ClusterCfg[proto.VolumeOverboughtRatioKey] != nil || c.ClusterCfg[proto.ChunkOversoldRatioKey] != nil { + volumeOverboughtRatio, _ := strconv.ParseFloat(c.ClusterCfg[proto.VolumeOverboughtRatioKey].(string), 64) + chunkOversoldRatio, _ := strconv.ParseFloat(c.ClusterCfg[proto.ChunkOversoldRatioKey].(string), 64) + + if chunkOversoldRatio < 0.5*volumeOverboughtRatio || chunkOversoldRatio > volumeOverboughtRatio { + return errors.New("ChunkOversoldRatio must be between 0.5*VolumeOverboughtRatioKey and VolumeOverboughtRatioKey") + } + c.VolumeMgrConfig.VolumeOverboughtRatio = volumeOverboughtRatio + c.DiskMgrConfig.ChunkOversoldRatio = chunkOversoldRatio + } + c.VolumeMgrConfig.ChunkSize = c.ChunkSize c.DiskMgrConfig.ChunkSize = int64(c.ChunkSize) c.ClusterCfg[proto.VolumeChunkSizeKey] = c.ChunkSize diff --git a/blobstore/clustermgr/volumemgr/startup.go b/blobstore/clustermgr/volumemgr/startup.go index b62cd9545..663282221 100644 --- a/blobstore/clustermgr/volumemgr/startup.go +++ b/blobstore/clustermgr/volumemgr/startup.go @@ -65,6 +65,8 @@ type VolumeMgrConfig struct { CodeModePolicies []codemode.Policy `json:"-"` Region string `json:"-"` ClusterID proto.ClusterID `json:"-"` + + VolumeOverboughtRatio float64 `json:"-"` } func (c *VolumeMgrConfig) checkAndFix() { diff --git a/blobstore/clustermgr/volumemgr/volumeunit.go b/blobstore/clustermgr/volumemgr/volumeunit.go index 76389c7b8..d87cec98a 100644 --- a/blobstore/clustermgr/volumemgr/volumeunit.go +++ b/blobstore/clustermgr/volumemgr/volumeunit.go @@ -309,12 +309,20 @@ func (v *VolumeMgr) applyChunkReport(ctx context.Context, chunks *cmapi.ReportCh vol.vUnits[idx].vuInfo.Total = chunk.Total dataChunkNum := uint64(v.codeMode[vol.volInfoBase.CodeMode].tactic.N) - volFree := vol.vUnits[idx].vuInfo.Free * dataChunkNum volUsed := vol.vUnits[idx].vuInfo.Used * dataChunkNum volTotal := vol.vUnits[idx].vuInfo.Total * dataChunkNum + volFree := vol.vUnits[idx].vuInfo.Free * dataChunkNum + if v.VolumeOverboughtRatio > 0 { + volFree = 0 + _volTotal := uint64(float64(volTotal) * v.VolumeOverboughtRatio) + if _volTotal > volUsed { + volFree = _volTotal - volUsed + } + } // use the minimum free size as volume free - if vol.volInfoBase.Free > volFree { + // or use calculated free size when overboughtRatio is specified as overboughtRatio may set into larger value + if vol.volInfoBase.Free > volFree || v.VolumeOverboughtRatio > 0 { vol.volInfoBase.Used = volUsed vol.volInfoBase.Total = volTotal vol.smallestVUIdx = idx diff --git a/blobstore/clustermgr/volumemgr/volumeunit_test.go b/blobstore/clustermgr/volumemgr/volumeunit_test.go index 7bd20e5ed..84b8ec745 100644 --- a/blobstore/clustermgr/volumemgr/volumeunit_test.go +++ b/blobstore/clustermgr/volumemgr/volumeunit_test.go @@ -382,14 +382,20 @@ func TestVolumeMgr_applyChunkReport(t *testing.T) { Vuid: vuid, Total: defaultChunkSize + 1, Free: uint64(1024 * 1024), - Used: defaultChunkSize - uint64(1024*i), + Used: defaultChunkSize - uint64(1024*1024), } args = append(args, chunk) } - // set report chunkInfo(vid:1) epoch=2, + // set report chunkInfo(vid:1) epoch=2, args[1].Vuid = proto.EncodeVuid(proto.EncodeVuidPrefix(proto.Vid(1), 2), 0) err := mockVolumeMgr.applyChunkReport(context.Background(), &clustermgr.ReportChunkArgs{ChunkInfos: args}) require.NoError(t, err) + + // validate volume info + vol := mockVolumeMgr.all.getVol(args[0].Vuid.Vid()) + dataChunkNum := uint64(mockVolumeMgr.codeMode[vol.volInfoBase.CodeMode].tactic.N) + require.Equal(t, args[0].Free*dataChunkNum, vol.volInfoBase.Free) + for i := 0; i < len(args); i++ { vol := mockVolumeMgr.all.getVol(args[i].Vuid.Vid()) unit := vol.vUnits[args[i].Vuid.Index()] @@ -408,6 +414,43 @@ func TestVolumeMgr_applyChunkReport(t *testing.T) { require.NoError(t, err) } +func TestVolumeMgr_applyChunkReportWithVolumeOverbought(t *testing.T) { + mockVolumeMgr, clean := initMockVolumeMgr(t) + defer clean() + + var args []blobnode.ChunkInfo + for i := 0; i < volumeCount; i++ { + vuid := proto.EncodeVuid(proto.EncodeVuidPrefix(proto.Vid(i), 2), 1) + chunk := blobnode.ChunkInfo{ + Vuid: vuid, + Total: defaultChunkSize, + Free: uint64(0.6 * float64(defaultChunkSize)), + Used: uint64(0.4 * float64(defaultChunkSize)), + } + args = append(args, chunk) + } + + // with VolumeOverboughtRatio, validate volume free size after chunk report + // as volume total * VolumeOverboughtRatio is larger than volume used, volume free should between 0 and calculated volume free + mockVolumeMgr.VolumeOverboughtRatio = 0.5 + vol := mockVolumeMgr.all.getVol(args[0].Vuid.Vid()) + err := mockVolumeMgr.applyChunkReport(context.Background(), &clustermgr.ReportChunkArgs{ChunkInfos: args}) + require.NoError(t, err) + require.Greater(t, vol.volInfoBase.Free, uint64(0)) + dataChunkNum := uint64(mockVolumeMgr.codeMode[vol.volInfoBase.CodeMode].tactic.N) + require.Less(t, vol.volInfoBase.Free, args[0].Free*dataChunkNum) + + // with VolumeOverboughtRatio, validate volume free size after chunk report + // as volume total * VolumeOverboughtRatio is less than volume used, volume free should be set into 0 + for i := range args { + args[i].Used = uint64(0.5 * float64(args[i].Total)) + args[i].Free = args[i].Total - args[i].Used + } + err = mockVolumeMgr.applyChunkReport(context.Background(), &clustermgr.ReportChunkArgs{ChunkInfos: args}) + require.NoError(t, err) + require.Equal(t, uint64(0), vol.volInfoBase.Free) +} + func TestVolumeMgr_ReleaseVolumeUnit(t *testing.T) { mockVolumeMgr, clean := initMockVolumeMgr(t) defer clean() diff --git a/blobstore/common/proto/const.go b/blobstore/common/proto/const.go index 9f6b4bcf6..8f581f574 100644 --- a/blobstore/common/proto/const.go +++ b/blobstore/common/proto/const.go @@ -216,11 +216,13 @@ const ( volumeStatusMax ) -// system config key,not allow delete +// system config key, not allow to delete const ( - CodeModeConfigKey = "code_mode" - VolumeReserveSizeKey = "volume_reserve_size" - VolumeChunkSizeKey = "volume_chunk_size" + CodeModeConfigKey = "code_mode" + VolumeReserveSizeKey = "volume_reserve_size" + VolumeChunkSizeKey = "volume_chunk_size" + VolumeOverboughtRatioKey = "volume_overbought_ratio" + ChunkOversoldRatioKey = "chunk_oversold_ratio" ) func IsSysConfigKey(key string) bool {