refactor(master): support report monitor stats for cluster different media datanode. #22929430

Signed-off-by: Victor1319 <zengxuewei@oppo.com>

master/cluster.go

@@ -66,6 +66,7 @@ type ClusterTopoSubItem struct {
 	idAlloc            *IDAllocator
 	t                  *topology
 	dataNodeStatInfo   *nodeStatInfo
+	dataStatsByMedia   map[string]*nodeStatInfo
 	metaNodeStatInfo   *nodeStatInfo
 	zoneStatInfos      map[string]*proto.ZoneStat
 	volStatInfo        sync.Map
@@ -426,6 +427,7 @@ func newCluster(name string, leaderInfo *LeaderInfo, fsm *MetadataFsm, partition
 	c.BadDataPartitionIds = new(sync.Map)
 	c.BadMetaPartitionIds = new(sync.Map)
 	c.dataNodeStatInfo = new(nodeStatInfo)
+	c.dataStatsByMedia = make(map[string]*proto.NodeStatInfo)
 	c.metaNodeStatInfo = new(nodeStatInfo)
 	c.FaultDomain = cfg.faultDomain
 	c.zoneStatInfos = make(map[string]*proto.ZoneStat)

master/cluster_stat.go

@@ -118,6 +118,8 @@ func (c *Cluster) updateDataNodeStatInfo() {
 		used  uint64
 		avail uint64
 	)
+
+	statsByMedia := make(map[string]*nodeStatInfo)
 	c.dataNodes.Range(func(addr, node interface{}) bool {
 		dataNode := node.(*DataNode)
 		total = total + dataNode.Total
@@ -126,6 +128,17 @@ func (c *Cluster) updateDataNodeStatInfo() {
 		if dataNode.isActive {
 			avail = avail + dataNode.AvailableSpace
 		}
+
+		media := proto.MediaTypeString(dataNode.MediaType)
+		stat, OK := statsByMedia[media]
+		if !OK {
+			stat = new(nodeStatInfo)
+			statsByMedia[media] = stat
+		}
+
+		stat.TotalGB += dataNode.Total / util.GB
+		stat.UsedGB += dataNode.Used / util.GB
+
 		return true
 	})
 	if total <= 0 {
@@ -142,6 +155,8 @@ func (c *Cluster) updateDataNodeStatInfo() {
 	c.dataNodeStatInfo.IncreasedGB = int64(usedGB) - int64(c.dataNodeStatInfo.UsedGB)
 	c.dataNodeStatInfo.UsedGB = usedGB
 	c.dataNodeStatInfo.UsedRatio = strconv.FormatFloat(usedRate, 'f', 3, 32)
+
+	c.dataStatsByMedia = statsByMedia
 }
 
 func (c *Cluster) updateMetaNodeStatInfo() {

master/monitor_metrics.go

@@ -31,6 +31,7 @@ const (
 	StatPeriod                 = time.Minute * time.Duration(1)
 	MetricDataNodesUsedGB      = "dataNodes_used_GB"
 	MetricDataNodesTotalGB     = "dataNodes_total_GB"
+	MetricDataNodesStat        = "dataNodes_stats"
 	MetricDataNodesIncreasedGB = "dataNodes_increased_GB"
 	MetricMetaNodesUsedGB      = "metaNodes_used_GB"
 	MetricMetaNodesTotalGB     = "metaNodes_total_GB"
@@ -100,6 +101,7 @@ type monitorMetrics struct {
 	volCount                 *exporter.Gauge
 	dataNodesTotal           *exporter.Gauge
 	dataNodesUsed            *exporter.Gauge
+	dataNodeStat             *exporter.GaugeVec
 	dataNodeIncreased        *exporter.Gauge
 	metaNodesTotal           *exporter.Gauge
 	metaNodesUsed            *exporter.Gauge
@@ -473,6 +475,7 @@ func (m *warningMetrics) WarnMpNoLeader(clusterName string, partitionID uint64,
 func (mm *monitorMetrics) start() {
 	mm.dataNodesTotal = exporter.NewGauge(MetricDataNodesTotalGB)
 	mm.dataNodesUsed = exporter.NewGauge(MetricDataNodesUsedGB)
+	mm.dataNodeStat = exporter.NewGaugeVec(MetricDataNodesStat, "", []string{"media", "type"})
 	mm.dataNodeIncreased = exporter.NewGauge(MetricDataNodesIncreasedGB)
 	mm.metaNodesTotal = exporter.NewGauge(MetricMetaNodesTotalGB)
 	mm.metaNodesUsed = exporter.NewGauge(MetricMetaNodesUsedGB)
@@ -571,6 +574,10 @@ func (mm *monitorMetrics) doStat() {
 	mm.lcNodesCount.Set(float64(lcNodeCount))
 	volCount := len(mm.cluster.vols)
 	mm.volCount.Set(float64(volCount))
+	for m, s := range mm.cluster.dataStatsByMedia {
+		mm.dataNodeStat.SetWithLabelValues(float64(s.UsedGB), m, "used")
+		mm.dataNodeStat.SetWithLabelValues(float64(s.TotalGB), m, "total")
+	}
 	mm.dataNodesTotal.Set(float64(mm.cluster.dataNodeStatInfo.TotalGB))
 	mm.dataNodesUsed.Set(float64(mm.cluster.dataNodeStatInfo.UsedGB))
 	mm.dataNodeIncreased.Set(float64(mm.cluster.dataNodeStatInfo.IncreasedGB))
@@ -1163,6 +1170,7 @@ func (mm *monitorMetrics) resetAllLeaderMetrics() {
 	mm.metaNodesCount.Set(0)
 	mm.lcNodesCount.Set(0)
 	mm.volCount.Set(0)
+	mm.dataNodeStat.Reset()
 	mm.dataNodesTotal.Set(0)
 	mm.dataNodesUsed.Set(0)
 	mm.dataNodeIncreased.Set(0)

util/exporter/gauge.go

@@ -148,3 +148,7 @@ func (v *GaugeVec) SetBoolWithLabelValues(val bool, lvs ...string) {
 		v.SetWithLabelValues(0, lvs...)
 	}
 }
+
+func (v *GaugeVec) Reset() {
+	v.GaugeVec.Reset()
+}