gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 02:00:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(master): Adaptation of the maximum number of large disk node dp. #1000562853

Browse Source 
Signed-off-by: zhumingze <zhumingze@oppo.com>
This commit is contained in:
zhumingze 2025-12-19 10:01:28 +08:00 committed by 曾雪伟
parent fa86ecde97
commit 4c7e409a73
15 changed files with 359 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
cli/cmd/cluster.go
View File

@ -69,6 +69,10 @@ const (
nodeAutoRepairRateKey = "autoRepairRate"
nodeMaxDpCntLimit = "maxDpCntLimit"
nodeMaxMpCntLimit = "maxMpCntLimit"
dpLimitSsdBaseCountKey = "dpLimitSsdBaseCount"
dpLimitSsdFactorKey = "dpLimitSsdFactor"
dpLimitHddBaseCountKey = "dpLimitHddBaseCount"
dpLimitHddFactorKey = "dpLimitHddFactor"
cmdForbidMpDecommission = "forbid meta partition decommission"
// cmdSetDecommissionLimitShort = "set cluster decommission limit"
cmdQueryDecommissionStatus = "query decommission status"
@ -113,6 +117,10 @@ func newClusterInfoCmd(client *master.MasterClient) *cobra.Command {
stdout(fmt.Sprintf(" AutoRepairRate : %v\n", clusterPara[nodeAutoRepairRateKey]))
stdout(fmt.Sprintf(" MaxDpCntLimit : %v\n", clusterPara[nodeMaxDpCntLimit]))
stdout(fmt.Sprintf(" MaxMpCntLimit : %v\n", clusterPara[nodeMaxMpCntLimit]))
stdout(fmt.Sprintf(" DpLimitSsdBaseCount: %v\n", clusterPara[dpLimitSsdBaseCountKey]))
stdout(fmt.Sprintf(" DpLimitSsdFactor : %v\n", clusterPara[dpLimitSsdFactorKey]))
stdout(fmt.Sprintf(" DpLimitHddBaseCount: %v\n", clusterPara[dpLimitHddBaseCountKey]))
stdout(fmt.Sprintf(" DpLimitHddFactor : %v\n", clusterPara[dpLimitHddFactorKey]))
stdout("\n")
@ -311,6 +319,10 @@ func newClusterSetParasCmd(client *master.MasterClient) *cobra.Command {
optFlashWriteFlowLimit := ""
optFlashKeyFlowLimit := ""
optRemoteClientFlowLimit := ""
optDpLimitSsdBaseCount := ""
optDpLimitSsdFactor := ""
optDpLimitHddBaseCount := ""
optDpLimitHddFactor := ""
cmd := &cobra.Command{
Use: CliOpSetCluster,
Short: cmdClusterSetClusterInfoShort,
@ -470,7 +482,6 @@ func newClusterSetParasCmd(client *master.MasterClient) *cobra.Command {
}
}
// Validate distribution optimization parameters
if distributionOptimizationConDpCnt != "" {
var count int64
count, err = strconv.ParseInt(distributionOptimizationConDpCnt, 10, 64)
@ -572,6 +583,31 @@ func newClusterSetParasCmd(client *master.MasterClient) *cobra.Command {
}
}
if optDpLimitSsdBaseCount != "" {
if _, err = strconv.ParseUint(optDpLimitSsdBaseCount, 10, 64); err != nil {
err = fmt.Errorf("param dpLimitSsdBaseCount(%v) should be uint", optDpLimitSsdBaseCount)
return
}
}
if optDpLimitSsdFactor != "" {
if _, err = strconv.ParseUint(optDpLimitSsdFactor, 10, 64); err != nil {
err = fmt.Errorf("param dpLimitSsdFactor(%v) should be uint", optDpLimitSsdFactor)
return
}
}
if optDpLimitHddBaseCount != "" {
if _, err = strconv.ParseUint(optDpLimitHddBaseCount, 10, 64); err != nil {
err = fmt.Errorf("param dpLimitHddBaseCount(%v) should be uint", optDpLimitHddBaseCount)
return
}
}
if optDpLimitHddFactor != "" {
if _, err = strconv.ParseUint(optDpLimitHddFactor, 10, 64); err != nil {
err = fmt.Errorf("param dpLimitHddFactor(%v) should be uint", optDpLimitHddFactor)
return
}
}
if optRemoteCacheMultiRead != "" {
if _, err = strconv.ParseBool(optRemoteCacheMultiRead); err != nil {
err = fmt.Errorf("param remoteCacheMultiRead(%v) should be true or false", optRemoteCacheMultiRead)
@ -618,7 +654,7 @@ func newClusterSetParasCmd(client *master.MasterClient) *cobra.Command {
distributionOptimizationConDpCnt, distributionOptimizationThreshold,
optRcTTL, optRcReadTimeout, optRemoteCacheMultiRead, optFlashNodeTimeoutCount,
optRemoteCacheSameZoneTimeout, optRemoteCacheSameRegionTimeout, optFlashHotKeyMissCount, optFlashReadFlowLimit, optFlashWriteFlowLimit, optFlashKeyFlowLimit, optRemoteClientFlowLimit,
enableMpDecommissionByLearner, learnerRecoverTimeoutSeconds); err != nil {
enableMpDecommissionByLearner, learnerRecoverTimeoutSeconds, optDpLimitSsdBaseCount, optDpLimitSsdFactor, optDpLimitHddBaseCount, optDpLimitHddFactor); err != nil {
return
}
stdout("Cluster parameters has been set successfully. \n")
@ -670,6 +706,10 @@ func newClusterSetParasCmd(client *master.MasterClient) *cobra.Command {
cmd.Flags().StringVar(&optFlashWriteFlowLimit, CliFlagFlashWriteFlowLimit, "", "Flash write flow limit(must >= 0)")
cmd.Flags().StringVar(&optFlashKeyFlowLimit, CliFlagFlashKeyFlowLimit, "", "Flash key flow limit(must >= 0)")
cmd.Flags().StringVar(&optRemoteClientFlowLimit, CliFlagRemoteClientFlowLimit, "", "Remote client flow limit(must >= 0)")
cmd.Flags().StringVar(&optDpLimitSsdBaseCount, CliFlagDpLimitSsdBaseCount, "", "DP limit SSD base count")
cmd.Flags().StringVar(&optDpLimitSsdFactor, CliFlagDpLimitSsdFactor, "", "DP limit SSD factor per 120GB")
cmd.Flags().StringVar(&optDpLimitHddBaseCount, CliFlagDpLimitHddBaseCount, "", "DP limit HDD base count")
cmd.Flags().StringVar(&optDpLimitHddFactor, CliFlagDpLimitHddFactor, "", "DP limit HDD factor per 120GB")
return cmd
}

4
cli/cmd/const.go
View File

@ -189,6 +189,10 @@ const (
CliFlagFlashWriteFlowLimit = "flashWriteFlowLimit"
CliFlagFlashKeyFlowLimit = "flashKeyFlowLimit"
CliFlagRemoteClientFlowLimit = "remoteClientFlowLimit"
CliFlagDpLimitSsdBaseCount = "dpLimitSsdBaseCount"
CliFlagDpLimitSsdFactor = "dpLimitSsdFactor"
CliFlagDpLimitHddBaseCount = "dpLimitHddBaseCount"
CliFlagDpLimitHddFactor = "dpLimitHddFactor"
// CliFlagSetDataPartitionCount = "count" use dp-count instead

30
cli/cmd/datanode.go
View File

@ -38,6 +38,7 @@ func newDataNodeCmd(client *master.MasterClient) *cobra.Command {
cmd.AddCommand(
newDataNodeListCmd(client),
newDataNodeInfoCmd(client),
newDataNodeSetCmd(client),
newDataNodeDecommissionCmd(client),
newDataNodeMigrateCmd(client),
newDataNodeQueryDecommissionProgress(client),
@ -53,6 +54,7 @@ func newDataNodeCmd(client *master.MasterClient) *cobra.Command {
const (
cmdDataNodeListShort = "List information of data nodes"
cmdDataNodeInfoShort = "Show information of a data node"
cmdDataNodeSetShort = "Set parameters for a data node"
cmdDataNodeDecommissionInfoShort = "decommission partitions in a data node to others"
cmdDataNodeQueryDecommissionedDisksShort = "query datanode decommissioned disks"
cmdDataNodeQueryDecommissionSuccessDisksShort = "query datanode decommissionSuccess disks"
@ -122,6 +124,34 @@ func newDataNodeInfoCmd(client *master.MasterClient) *cobra.Command {
return cmd
}
func newDataNodeSetCmd(client *master.MasterClient) *cobra.Command {
var maxDpCntLimit uint64
cmd := &cobra.Command{
Use: CliOpSet + " [{HOST}:{PORT}]",
Short: cmdDataNodeSetShort,
Args: cobra.MinimumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
nodeAddr := args[0]
if !cmd.Flags().Changed(CliFlagMaxDpCntLimit) {
return fmt.Errorf("no parameter specified, use --%s to set max DP count limit", CliFlagMaxDpCntLimit)
}
if err := client.NodeAPI().SetDataNodeMaxDpCntLimit(nodeAddr, maxDpCntLimit); err != nil {
return err
}
stdoutln(fmt.Sprintf("Set max DP count limit to %d for data node %s successfully", maxDpCntLimit, nodeAddr))
return nil
},
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if len(args) != 0 {
return nil, cobra.ShellCompDirectiveNoFileComp
}
return validDataNodes(client, toComplete), cobra.ShellCompDirectiveNoFileComp
},
}
cmd.Flags().Uint64Var(&maxDpCntLimit, CliFlagMaxDpCntLimit, 0, "Maximum DP count limit for this data node")
return cmd
}
func newDataNodeDecommissionCmd(client *master.MasterClient) *cobra.Command {
var (
optCount int

35
cli/cmd/metanode.go
View File

@ -18,6 +18,7 @@ import (
"sort"
"strings"
"github.com/cubefs/cubefs/blobstore/cli/common/fmt"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/sdk/master"
"github.com/cubefs/cubefs/util"
@ -37,6 +38,7 @@ func newMetaNodeCmd(client *master.MasterClient) *cobra.Command {
cmd.AddCommand(
newMetaNodeListCmd(client),
newMetaNodeInfoCmd(client),
newMetaNodeSetCmd(client),
newMetaNodeDecommissionCmd(client),
newMetaNodeMigrateCmd(client),
newMetaNodeOfflineCmd(client),
@ -48,6 +50,7 @@ func newMetaNodeCmd(client *master.MasterClient) *cobra.Command {
const (
cmdMetaNodeListShort = "List information of meta nodes"
cmdMetaNodeInfoShort = "Show information of meta nodes"
cmdMetaNodeSetShort = "Set parameters for a meta node"
cmdMetaNodeDecommissionInfoShort = "Decommission partitions in a meta node to other nodes"
cmdMetaNodeMigrateInfoShort = "Migrate partitions from a meta node to the other node"
cmdMetaNodeOfflineInfoShort = "Offline meta node in background"
@ -122,6 +125,38 @@ func newMetaNodeInfoCmd(client *master.MasterClient) *cobra.Command {
return cmd
}
func newMetaNodeSetCmd(client *master.MasterClient) *cobra.Command {
var maxMpCntLimit uint64
cmd := &cobra.Command{
Use: CliOpSet + " [{HOST}:{PORT}]",
Short: cmdMetaNodeSetShort,
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
var err error
defer func() {
errout(err)
}()
nodeAddr := args[0]
if !cmd.Flags().Changed(CliFlagMaxMpCntLimit) {
err = fmt.Errorf("no parameter specified, use --%s to set max MP count limit", CliFlagMaxMpCntLimit)
return
}
if err = client.NodeAPI().SetMetaNodeMaxMpCntLimit(nodeAddr, maxMpCntLimit); err != nil {
return
}
stdout("Set max MP count limit to %d for meta node %s successfully\n", maxMpCntLimit, nodeAddr)
},
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if len(args) != 0 {
return nil, cobra.ShellCompDirectiveNoFileComp
}
return validMetaNodes(client, toComplete), cobra.ShellCompDirectiveNoFileComp
},
}
cmd.Flags().Uint64Var(&maxMpCntLimit, CliFlagMaxMpCntLimit, 0, "Maximum MP count limit for this meta node")
return cmd
}
func newMetaNodeDecommissionCmd(client *master.MasterClient) *cobra.Command {
var (
optCount int

41
master/api_args_parse.go
View File

@ -1459,6 +1459,47 @@ func parseAndExtractSetNodeInfoParams(r *http.Request) (params map[string]interf
params[nodeDpMaxRepairErrCntKey] = val
}
if value = r.FormValue(dpLimitSsdBaseCountKey); value != "" {
noParams = false
val := uint64(0)
val, err = strconv.ParseUint(value, 10, 64)
if err != nil {
err = unmatchedKey(dpLimitSsdBaseCountKey)
return
}
params[dpLimitSsdBaseCountKey] = val
}
if value = r.FormValue(dpLimitSsdFactorKey); value != "" {
noParams = false
val := uint64(0)
val, err = strconv.ParseUint(value, 10, 64)
if err != nil {
err = unmatchedKey(dpLimitSsdFactorKey)
return
}
params[dpLimitSsdFactorKey] = val
}
if value = r.FormValue(dpLimitHddBaseCountKey); value != "" {
noParams = false
val := uint64(0)
val, err = strconv.ParseUint(value, 10, 64)
if err != nil {
err = unmatchedKey(dpLimitHddBaseCountKey)
return
}
params[dpLimitHddBaseCountKey] = val
}
if value = r.FormValue(dpLimitHddFactorKey); value != "" {
noParams = false
val := uint64(0)
val, err = strconv.ParseUint(value, 10, 64)
if err != nil {
err = unmatchedKey(dpLimitHddFactorKey)
return
}
params[dpLimitHddFactorKey] = val
}
if value = r.FormValue(clusterCreateTimeKey); value != "" {
noParams = false
params[clusterCreateTimeKey] = value

37
master/api_service.go
View File

@ -4042,6 +4042,39 @@ func (m *Server) setNodeInfoHandler(w http.ResponseWriter, r *http.Request) {
}
}
changedDpLimit := false
if val, ok := params[dpLimitSsdBaseCountKey]; ok {
if v, ok := val.(uint64); ok {
m.cluster.cfg.DpLimitSsdBaseCount = v
changedDpLimit = true
}
}
if val, ok := params[dpLimitSsdFactorKey]; ok {
if v, ok := val.(uint64); ok {
m.cluster.cfg.DpLimitSsdFactor = v
changedDpLimit = true
}
}
if val, ok := params[dpLimitHddBaseCountKey]; ok {
if v, ok := val.(uint64); ok {
m.cluster.cfg.DpLimitHddBaseCount = v
changedDpLimit = true
}
}
if val, ok := params[dpLimitHddFactorKey]; ok {
if v, ok := val.(uint64); ok {
m.cluster.cfg.DpLimitHddFactor = v
changedDpLimit = true
}
}
if changedDpLimit {
if err = m.cluster.syncPutCluster(); err != nil {
sendErrReply(w, r, newErrHTTPReply(err))
return
}
}
if val, ok := params[flashNodeReadDataNodeTimeout]; ok {
if v, ok := val.(int64); ok {
if err = m.setConfig(flashNodeReadDataNodeTimeout, strconv.FormatInt(v, 10)); err != nil {
@ -5082,6 +5115,10 @@ func (m *Server) getNodeInfoHandler(w http.ResponseWriter, r *http.Request) {
resp[clusterLoadFactorKey] = fmt.Sprintf("%v", m.cluster.cfg.ClusterLoadFactor)
resp[maxDpCntLimitKey] = fmt.Sprintf("%v", m.cluster.getMaxDpCntLimit())
resp[maxMpCntLimitKey] = fmt.Sprintf("%v", m.cluster.getMaxMpCntLimit())
resp[dpLimitSsdBaseCountKey] = fmt.Sprintf("%v", m.cluster.cfg.DpLimitSsdBaseCount)
resp[dpLimitSsdFactorKey] = fmt.Sprintf("%v", m.cluster.cfg.DpLimitSsdFactor)
resp[dpLimitHddBaseCountKey] = fmt.Sprintf("%v", m.cluster.cfg.DpLimitHddBaseCount)
resp[dpLimitHddFactorKey] = fmt.Sprintf("%v", m.cluster.cfg.DpLimitHddFactor)
sendOkReply(w, r, newSuccessHTTPReply(resp))
}

13
master/cluster.go
View File

@ -455,12 +455,6 @@ func newCluster(name string, leaderInfo *LeaderInfo, fsm *MetadataFsm, partition
c.delayDeleteVolsInfo = make([]*delayDeleteVolInfo, 0)
c.stopc = make(chan bool)
c.cfg = cfg
if clusterDpCntLimit == 0 {
atomic.StoreUint64(&clusterDpCntLimit, defaultMaxDpCntLimit)
}
if clusterMpCntLimit == 0 {
atomic.StoreUint64(&clusterMpCntLimit, defaultMaxMpCntLimit)
}
if distributionOptimizationThreshold.Load() == 0 {
distributionOptimizationThreshold.Store(defaultDistributionOptimizationThreshold)
}
@ -5177,8 +5171,8 @@ func (c *Cluster) setMetaNodeDeleteWorkerSleepMs(val uint64) (err error) {
return
}
func (c *Cluster) getMaxDpCntLimit() (dpCntInLimit uint64) {
dpCntInLimit = atomic.LoadUint64(&clusterDpCntLimit)
func (c *Cluster) getMaxDpCntLimit() (dpCntLimit uint64) {
dpCntLimit = atomic.LoadUint64(&clusterDpCntLimit)
return
}
@ -5187,10 +5181,9 @@ func (c *Cluster) setMaxDpCntLimit(val uint64) (err error) {
val = defaultMaxDpCntLimit
}
oldVal := c.getMaxDpCntLimit()
// atomic.StoreUint64(&c.cfg.MaxDpCntLimit, val)
atomic.StoreUint64(&clusterDpCntLimit, val)
if err = c.syncPutCluster(); err != nil {
log.LogErrorf("action[MaxDpCntLimit] err[%v]", err)
log.LogErrorf("[setMaxDpCntLimit] failed to set dp limit to value(%v), err(%v)", val, err)
atomic.StoreUint64(&clusterDpCntLimit, oldVal)
err = proto.ErrPersistenceByRaft
return

15
master/config.go
View File

@ -135,6 +135,13 @@ const (
defaultFlashKeyFlowLimit = 0
defaultRemoteClientFlowLimit = 0
defaultDpLimitUnitSizeGB uint64 = 120
defaultDpLimitSsdBaseCount uint64 = 200
// factor is in tenths, e.g. 50 -> 5.0
defaultDpLimitSsdFactor uint64 = 50
defaultDpLimitHddBaseCount uint64 = 100
defaultDpLimitHddFactor uint64 = 20
defaultMetaNodeGOGC = 100
defaultDataNodeGOGC = 100
)
@ -233,6 +240,10 @@ type clusterConfig struct {
MaxWritableDataPartitionCnt int
RackAwareLevel pt.RackAwareLevel
LearnerRecoverTimeoutSeconds int64 // Timeout in seconds for learner mode recovery
DpLimitSsdBaseCount uint64
DpLimitSsdFactor uint64
DpLimitHddBaseCount uint64
DpLimitHddFactor uint64
}
func newClusterConfig() (cfg *clusterConfig) {
@ -280,6 +291,10 @@ func newClusterConfig() (cfg *clusterConfig) {
cfg.mpMigrateThreads = defaultMpMigrateThreads
cfg.RackAwareLevel = pt.RackAwareNone
cfg.LearnerRecoverTimeoutSeconds = defaultLearnerRecoverTimeout
cfg.DpLimitSsdBaseCount = defaultDpLimitSsdBaseCount
cfg.DpLimitSsdFactor = defaultDpLimitSsdFactor
cfg.DpLimitHddBaseCount = defaultDpLimitHddBaseCount
cfg.DpLimitHddFactor = defaultDpLimitHddFactor
return
}

4
master/const.go
View File

@ -162,6 +162,10 @@ const (
quotaClass = "quotaClass"
quotaOfClass = "quotaOfStorageClass"
dataMediaTypeKey = "dataMediaType"
dpLimitSsdBaseCountKey = "dpLimitSsdBaseCount"
dpLimitSsdFactorKey = "dpLimitSsdFactor"
dpLimitHddBaseCountKey = "dpLimitHddBaseCount"
dpLimitHddFactorKey = "dpLimitHddFactor"
remoteCacheEnable = "remoteCacheEnable"
remoteCacheAutoPrepare = "remoteCacheAutoPrepare"

50
master/data_node.go
View File

@ -82,6 +82,7 @@ type DataNode struct {
DecommissionTime uint64
DecommissionCompleteTime int64
DpCntLimit uint64 `json:"-"` // max count of data partition in a data node
CalculatedDpCntLimit uint64 `json:"-"`
CpuUtil atomicutil.Float64 `json:"-"`
ioUtils atomic.Value `json:"-"`
DecommissionDiskList []string // NOTE: the disks that running decommission
@ -244,6 +245,8 @@ func (dataNode *DataNode) updateNodeMetric(c *Cluster, resp *proto.DataNodeHeart
dataNode.DiskOpLogs = resp.DiskOpLogs
dataNode.DpOpLogs = resp.DpOpLogs
dataNode.CalculatedDpCntLimit = dataNode.calculateDpLimitByDiskCapacity(c)
dataNode.StartTime = resp.StartTime
if dataNode.Total == 0 {
dataNode.UsageRatio = 0.0
@ -391,12 +394,55 @@ func (dataNode *DataNode) GetPartitionLimitCnt() uint64 {
if dataNode.DpCntLimit != 0 {
return dataNode.DpCntLimit
}
if clusterDpCntLimit != 0 {
return clusterDpCntLimit
// pick the larger one between calculated limit and cluster value
dpCntLimit := dataNode.CalculatedDpCntLimit
if clusterLimit := atomic.LoadUint64(&clusterDpCntLimit); clusterLimit > dpCntLimit {
dpCntLimit = clusterLimit
}
if dpCntLimit != 0 {
return dpCntLimit
}
return defaultMaxDpCntLimit
}
// calculateDpLimitByDiskCapacity computes DP limit using disk capacity with formula:
// node DP limit = baseCount * len(AllDisks) + (TotalGB * factor) / (120 * 10)
// where factor is in tenths (e.g., factor=15 means 1.5)
func (dataNode *DataNode) calculateDpLimitByDiskCapacity(c *Cluster) uint64 {
diskCount := len(dataNode.AllDisks)
if diskCount == 0 {
log.LogWarnf("[calculateDpLimitByDiskCapacity] node(%s) has no disks", dataNode.Addr)
return 0
}
if dataNode.Total == 0 {
log.LogWarnf("[calculateDpLimitByDiskCapacity] node(%s) total capacity is 0", dataNode.Addr)
return 0
}
// choose params by media type from cluster config
var baseCount uint64
var factor uint64
if dataNode.MediaType == proto.MediaType_SSD {
baseCount = c.cfg.DpLimitSsdBaseCount
factor = c.cfg.DpLimitSsdFactor
} else {
baseCount = c.cfg.DpLimitHddBaseCount
factor = c.cfg.DpLimitHddFactor
}
// Calculate: baseCount * diskCount + (totalCapacityGB * factor) / (120 * 10)
totalCapacityGB := dataNode.Total / util.GB
limit := baseCount*uint64(diskCount) + (totalCapacityGB*factor)/(defaultDpLimitUnitSizeGB*10)
log.LogInfof("[calculateDpLimitByDiskCapacity] node(%s) media(%s) limit(%d) = base(%d)*disks(%d) + totalGB(%d)*factor(%d)/(120*10)",
dataNode.Addr, proto.MediaTypeString(dataNode.MediaType), limit, baseCount, diskCount, totalCapacityGB, factor)
return limit
}
func (dataNode *DataNode) GetAvailableSpace() uint64 {
return dataNode.AvailableSpace
}

39
master/data_node_test.go
View File

@ -6,6 +6,7 @@ import (
"time"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/util"
"github.com/stretchr/testify/require"
)
@ -59,3 +60,41 @@ func updateDisks(addr string, t *testing.T) {
require.Equal(t, allDisk, dn.AllDisks)
require.Equal(t, badDisk, dn.BadDisks)
}
func TestCalculateDpLimitByDiskCapacity(t *testing.T) {
t.Run("SSD", func(t *testing.T) {
cfg := newClusterConfig()
cfg.DpLimitSsdBaseCount = 150
cfg.DpLimitSsdFactor = 50 // 5.0 in tenths
cluster := &Cluster{cfg: cfg}
dn := &DataNode{
AllDisks: []string{"/data1", "/data2"},
Total: 4096 * util.GB, // 4TB in bytes
MediaType: proto.MediaType_SSD,
}
got := dn.calculateDpLimitByDiskCapacity(cluster)
// expected = base*diskCount + (totalGB*factor)/(120*10) where factor is tenths
want := uint64(150*2 + (4096*50)/(120*10))
require.Equal(t, want, got)
})
t.Run("HDD", func(t *testing.T) {
cfg := newClusterConfig()
cfg.DpLimitHddBaseCount = 100
cfg.DpLimitHddFactor = 20 // 2.0 in tenths
cluster := &Cluster{cfg: cfg}
dn := &DataNode{
AllDisks: []string{"/data1"},
Total: 14336 * util.GB, // 14TB in bytes
MediaType: proto.MediaType_HDD,
}
got := dn.calculateDpLimitByDiskCapacity(cluster)
// expected = base*diskCount + (totalGB*factor)/(120*10)
want := uint64(100*1 + (14336*20)/(120*10))
require.Equal(t, want, got)
})
}

27
master/metadata_fsm_op.go
View File

@ -97,6 +97,10 @@ type clusterValue struct {
FlashKeyFlowLimit int64
RemoteClientFlowLimit int64
LearnerRecoverTimeoutSeconds int64
DpLimitSsdBaseCount uint64
DpLimitSsdFactor uint64
DpLimitHddBaseCount uint64
DpLimitHddFactor uint64
}
func newClusterValue(c *Cluster) (cv *clusterValue) {
@ -159,6 +163,10 @@ func newClusterValue(c *Cluster) (cv *clusterValue) {
FlashKeyFlowLimit: c.cfg.flashKeyFlowLimit,
RemoteClientFlowLimit: c.cfg.remoteClientFlowLimit,
LearnerRecoverTimeoutSeconds: c.cfg.LearnerRecoverTimeoutSeconds,
DpLimitSsdBaseCount: c.cfg.DpLimitSsdBaseCount,
DpLimitSsdFactor: c.cfg.DpLimitSsdFactor,
DpLimitHddBaseCount: c.cfg.DpLimitHddBaseCount,
DpLimitHddFactor: c.cfg.DpLimitHddFactor,
}
return cv
}
@ -1517,6 +1525,25 @@ func (c *Cluster) loadClusterValue() (err error) {
c.cfg.flashNodeReadDataNodeTimeout = cv.FlashNodeReadDataNodeTimeout
log.LogInfof("action[loadClusterValue] flashNodeHandleReadTimeout %v(ms), flashNodeReadDataNodeTimeout %v(ms), flashHotKeyMissCount %v, flashReadFlowLimit %v, flashWriteFlowLimit %v, flashKeyFlowLimit %v, remoteClientFlowLimit %v",
cv.FlashNodeHandleReadTimeout, cv.FlashNodeReadDataNodeTimeout, cv.FlashHotKeyMissCount, cv.FlashReadFlowLimit, cv.FlashWriteFlowLimit, cv.FlashKeyFlowLimit, cv.RemoteClientFlowLimit)
if cv.DpLimitSsdBaseCount == 0 {
cv.DpLimitSsdBaseCount = defaultDpLimitSsdBaseCount
}
c.cfg.DpLimitSsdBaseCount = cv.DpLimitSsdBaseCount
if cv.DpLimitSsdFactor == 0 {
cv.DpLimitSsdFactor = defaultDpLimitSsdFactor
}
c.cfg.DpLimitSsdFactor = cv.DpLimitSsdFactor
if cv.DpLimitHddBaseCount == 0 {
cv.DpLimitHddBaseCount = defaultDpLimitHddBaseCount
}
c.cfg.DpLimitHddBaseCount = cv.DpLimitHddBaseCount
if cv.DpLimitHddFactor == 0 {
cv.DpLimitHddFactor = defaultDpLimitHddFactor
}
c.cfg.DpLimitHddFactor = cv.DpLimitHddFactor
log.LogInfof("action[loadClusterValue] dp limit params SSD(base=%d,factor=%d) HDD(base=%d,factor=%d)",
c.cfg.DpLimitSsdBaseCount, c.cfg.DpLimitSsdFactor, c.cfg.DpLimitHddBaseCount, c.cfg.DpLimitHddFactor)
}
return

13
sdk/master/api_admin.go
View File

@ -652,6 +652,7 @@ func (api *AdminAPI) SetClusterParas(batchCount, markDeleteRate, deleteWorkerSle
remoteCacheSameZoneTimeout string, remoteCacheSameRegionTimeout string, flashHotKeyMissCount string,
flashReadFlowLimit string, flashWriteFlowLimit string, flashKeyFlowLimit string, remoteClientFlowLimit string,
enableMpDecommissionByLearner string, learnerRecoverTimeoutSeconds string,
dpLimitSsdBaseCount string, dpLimitSsdFactor string, dpLimitHddBaseCount string, dpLimitHddFactor string,
) (err error) {
request := newRequest(get, proto.AdminSetNodeInfo).Header(api.h)
request.addParam("batchCount", batchCount)
@ -765,6 +766,18 @@ func (api *AdminAPI) SetClusterParas(batchCount, markDeleteRate, deleteWorkerSle
if learnerRecoverTimeoutSeconds != "" {
request.addParam("learnerRecoverTimeoutSeconds", learnerRecoverTimeoutSeconds)
}
if dpLimitSsdBaseCount != "" {
request.addParamAny("dpLimitSsdBaseCount", dpLimitSsdBaseCount)
}
if dpLimitSsdFactor != "" {
request.addParamAny("dpLimitSsdFactor", dpLimitSsdFactor)
}
if dpLimitHddBaseCount != "" {
request.addParamAny("dpLimitHddBaseCount", dpLimitHddBaseCount)
}
if dpLimitHddFactor != "" {
request.addParamAny("dpLimitHddFactor", dpLimitHddFactor)
}
_, err = api.mc.serveRequest(request)
return
}

20
sdk/master/api_node.go
View File

@ -101,12 +101,32 @@ func (api *NodeAPI) GetDataNode(serverHost string) (node *proto.DataNodeInfo, er
return
}
func (api *NodeAPI) SetDataNodeMaxDpCntLimit(nodeAddr string, limit uint64) (err error) {
request := newRequest(get, proto.SetDpCntLimit).Header(api.h)
request.addParam("addr", nodeAddr)
request.addParam("maxDpCntLimit", strconv.FormatUint(limit, 10))
if _, err = api.mc.serveRequest(request); err != nil {
return
}
return
}
func (api *NodeAPI) GetMetaNode(serverHost string) (node *proto.MetaNodeInfo, err error) {
node = &proto.MetaNodeInfo{}
err = api.mc.requestWith(node, newRequest(get, proto.GetMetaNode).Header(api.h).addParam("addr", serverHost))
return
}
func (api *NodeAPI) SetMetaNodeMaxMpCntLimit(nodeAddr string, limit uint64) (err error) {
request := newRequest(get, proto.SetMpCntLimit).Header(api.h)
request.addParam("addr", nodeAddr)
request.addParam("maxMpCntLimit", strconv.FormatUint(limit, 10))
if _, err = api.mc.serveRequest(request); err != nil {
return
}
return
}
func (api *NodeAPI) ResponseMetaNodeTask(task *proto.AdminTask) (err error) {
return api.mc.request(newRequest(post, proto.GetMetaNodeTaskResponse).Header(api.h).Body(task))
}

2
tool/remotecache-config/cmd/cluster.go
View File

@ -215,7 +215,7 @@ func newClusterSetParasCmd(client *master.MasterClient) *cobra.Command {
"", "",
optRcTTL, optRcReadTimeout, optRemoteCacheMultiRead, optFlashNodeTimeoutCount,
optRemoteCacheSameZoneTimeout, optRemoteCacheSameRegionTimeout, optFlashHotKeyMissCount,
optFlashReadFlowLimit, optFlashWriteFlowLimit, optFlashKeyFlowLimit, optRemoteClientFlowLimit, "", ""); err != nil {
optFlashReadFlowLimit, optFlashWriteFlowLimit, optFlashKeyFlowLimit, optRemoteClientFlowLimit, "", "", "", "", "", ""); err != nil {
return
}
stdout("Cluster parameters has been set successfully. \n")