gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 02:00:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor(master): support report zone, set info for node metrics. #1000138242

Browse Source 
Signed-off-by: Victor1319 <zengxuewei@oppo.com>
This commit is contained in:
Victor1319 2025-06-19 16:33:56 +08:00 committed by zhumingze1108
parent 76ea9434ea
commit d6f98b4fe6
2 changed files with 83 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

177
master/monitor_metrics.go
View File

@ -142,30 +142,28 @@ type monitorMetrics struct {
DpMissingLeaderCount *exporter.GaugeVec DpMissingLeaderCount *exporter.GaugeVec
MpMissingLeaderCount *exporter.Gauge MpMissingLeaderCount *exporter.Gauge
MpMissingReplicaCount *exporter.Gauge MpMissingReplicaCount *exporter.Gauge
dataNodesetInactiveCount *exporter.GaugeVec
metaNodesetInactiveCount *exporter.GaugeVec
metaEqualCheckFail *exporter.GaugeVec metaEqualCheckFail *exporter.GaugeVec
masterNoLeader *exporter.Gauge masterNoLeader *exporter.Gauge
masterNoCache *exporter.GaugeVec masterNoCache *exporter.GaugeVec
masterSnapshot *exporter.Gauge masterSnapshot *exporter.Gauge
nodesetMetaTotal *exporter.GaugeVec // TODO remove in next version
nodesetMetaUsed *exporter.GaugeVec nodesetMetaTotal *exporter.GaugeVec
nodesetMetaUsageRatio *exporter.GaugeVec nodesetMetaUsed *exporter.GaugeVec
nodesetDataTotal *exporter.GaugeVec nodesetMetaUsageRatio *exporter.GaugeVec
nodesetDataUsed *exporter.GaugeVec nodesetDataTotal *exporter.GaugeVec
nodesetDataUsageRatio *exporter.GaugeVec nodesetDataUsed *exporter.GaugeVec
nodesetMpReplicaCount *exporter.GaugeVec nodesetDataUsageRatio *exporter.GaugeVec
nodesetDpReplicaCount *exporter.GaugeVec nodesetMpReplicaCount *exporter.GaugeVec
nodeStat *exporter.GaugeVec nodesetDpReplicaCount *exporter.GaugeVec
volNames map[string]struct{} nodeStat *exporter.GaugeVec
badDisks map[string]string
flashNodesBadDisks map[string]string volNames map[string]struct{}
nodesetInactiveDataNodesCount map[uint64]int64 badDisks map[string]string
nodesetInactiveMetaNodesCount map[uint64]int64 flashNodesBadDisks map[string]string
inconsistentMps map[string]string inconsistentMps map[string]string
replicaCntMap map[uint64]struct{} replicaCntMap map[uint64]struct{}
nodesetIds map[uint64]string nodesetIds map[uint64]string
lcNodesCount *exporter.Gauge lcNodesCount *exporter.Gauge
lcId map[string]struct{} lcId map[string]struct{}
@ -180,15 +178,13 @@ type monitorMetrics struct {
func newMonitorMetrics(c *Cluster) *monitorMetrics { func newMonitorMetrics(c *Cluster) *monitorMetrics {
return &monitorMetrics{ return &monitorMetrics{
cluster: c, cluster: c,
volNames: make(map[string]struct{}), volNames: make(map[string]struct{}),
badDisks: make(map[string]string), badDisks: make(map[string]string),
flashNodesBadDisks: make(map[string]string), flashNodesBadDisks: make(map[string]string),
nodesetInactiveDataNodesCount: make(map[uint64]int64), inconsistentMps: make(map[string]string),
nodesetInactiveMetaNodesCount: make(map[uint64]int64), replicaCntMap: make(map[uint64]struct{}),
inconsistentMps: make(map[string]string), lcId: make(map[string]struct{}),
replicaCntMap: make(map[uint64]struct{}),
lcId: make(map[string]struct{}),
} }
} }
@ -520,7 +516,7 @@ func (mm *monitorMetrics) start() {
mm.diskLost = exporter.NewGaugeVec(MetricDiskLost, "", []string{"addr", "path"}) mm.diskLost = exporter.NewGaugeVec(MetricDiskLost, "", []string{"addr", "path"})
mm.dpUnableDecommissionCount = exporter.NewGauge(MetricDpUnableDecommissionCount) mm.dpUnableDecommissionCount = exporter.NewGauge(MetricDpUnableDecommissionCount)
mm.dpNoSamePeer = exporter.NewGaugeVec(MetricDpNoSamePeer, "", []string{"dpId"}) mm.dpNoSamePeer = exporter.NewGaugeVec(MetricDpNoSamePeer, "", []string{"dpId"})
mm.nodeStat = exporter.NewGaugeVec(MetricNodeStat, "", []string{"type", "addr", "stat"}) mm.nodeStat = exporter.NewGaugeVec(MetricNodeStat, "", []string{"type", "addr", "stat", "zone", "set", "media", "writable", "alloc"})
mm.dataNodesInactive = exporter.NewGauge(MetricDataNodesInactive) mm.dataNodesInactive = exporter.NewGauge(MetricDataNodesInactive)
mm.InactiveDataNodeInfo = exporter.NewGaugeVec(MetricInactiveDataNodeInfo, "", []string{"clusterName", "addr"}) mm.InactiveDataNodeInfo = exporter.NewGaugeVec(MetricInactiveDataNodeInfo, "", []string{"clusterName", "addr"})
mm.metaNodesInactive = exporter.NewGauge(MetricMetaNodesInactive) mm.metaNodesInactive = exporter.NewGauge(MetricMetaNodesInactive)
@ -532,8 +528,6 @@ func (mm *monitorMetrics) start() {
mm.DpMissingLeaderCount = exporter.NewGaugeVec(MetricDpMissingLeaderCount, "", []string{"replicaNum"}) mm.DpMissingLeaderCount = exporter.NewGaugeVec(MetricDpMissingLeaderCount, "", []string{"replicaNum"})
mm.MpMissingLeaderCount = exporter.NewGauge(MetricMpMissingLeaderCount) mm.MpMissingLeaderCount = exporter.NewGauge(MetricMpMissingLeaderCount)
mm.MpMissingReplicaCount = exporter.NewGauge(MetricMpMissingReplicaCount) mm.MpMissingReplicaCount = exporter.NewGauge(MetricMpMissingReplicaCount)
mm.dataNodesetInactiveCount = exporter.NewGaugeVec(MetricDataNodesetInactiveCount, "", []string{"nodeset"})
mm.metaNodesetInactiveCount = exporter.NewGaugeVec(MetricMetaNodesetInactiveCount, "", []string{"nodeset"})
mm.metaEqualCheckFail = exporter.NewGaugeVec(MetricMetaInconsistent, "", []string{"volume", "mpId"}) mm.metaEqualCheckFail = exporter.NewGaugeVec(MetricMetaInconsistent, "", []string{"volume", "mpId"})
mm.masterSnapshot = exporter.NewGauge(MetricMasterSnapshot) mm.masterSnapshot = exporter.NewGauge(MetricMasterSnapshot)
@ -976,12 +970,6 @@ func (mm *monitorMetrics) setDiskDecommissionedMetric() {
func (mm *monitorMetrics) updateMetaNodesStat() { func (mm *monitorMetrics) updateMetaNodesStat() {
var inactiveMetaNodesCount int64 var inactiveMetaNodesCount int64
deleteNodesetCount := make(map[uint64]int64)
for k, v := range mm.nodesetInactiveMetaNodesCount {
deleteNodesetCount[k] = v
delete(mm.nodesetInactiveMetaNodesCount, k)
}
mm.cluster.metaNodes.Range(func(addr, node interface{}) bool { mm.cluster.metaNodes.Range(func(addr, node interface{}) bool {
metaNode, ok := node.(*MetaNode) metaNode, ok := node.(*MetaNode)
if !ok { if !ok {
@ -990,46 +978,43 @@ func (mm *monitorMetrics) updateMetaNodesStat() {
if !metaNode.IsActive { if !metaNode.IsActive {
inactiveMetaNodesCount++ inactiveMetaNodesCount++
mm.InactiveMetaNodeInfo.SetWithLabelValues(1, mm.cluster.Name, metaNode.Addr) mm.InactiveMetaNodeInfo.SetWithLabelValues(1, mm.cluster.Name, metaNode.Addr)
mm.nodesetInactiveMetaNodesCount[metaNode.NodeSetID] = mm.nodesetInactiveMetaNodesCount[metaNode.NodeSetID] + 1
delete(deleteNodesetCount, metaNode.NodeSetID)
} else { } else {
mm.InactiveMetaNodeInfo.DeleteLabelValues(mm.cluster.Name, metaNode.Addr) mm.InactiveMetaNodeInfo.DeleteLabelValues(mm.cluster.Name, metaNode.Addr)
} }
mm.nodeStat.SetWithLabelValues(metaNode.Ratio, MetricRoleMetaNode, metaNode.Addr, "usageRatio")
mm.nodeStat.SetWithLabelValues(float64(metaNode.Total), MetricRoleMetaNode, metaNode.Addr, "memTotal") zone := metaNode.ZoneName
mm.nodeStat.SetWithLabelValues(float64(metaNode.Used), MetricRoleMetaNode, metaNode.Addr, "memUsed") setId := strconv.Itoa(int(metaNode.NodeSetID))
mm.nodeStat.SetWithLabelValues(float64(metaNode.MetaPartitionCount), MetricRoleMetaNode, metaNode.Addr, "mpCount") media := "default"
mm.nodeStat.SetWithLabelValues(float64(metaNode.Threshold), MetricRoleMetaNode, metaNode.Addr, "threshold") mAddr := metaNode.Addr
mm.nodeStat.SetBoolWithLabelValues(metaNode.IsWriteAble(), MetricRoleMetaNode, metaNode.Addr, "writable")
mm.nodeStat.SetBoolWithLabelValues(metaNode.IsActive, MetricRoleMetaNode, metaNode.Addr, "active") writable := "false"
if metaNode.IsWriteAble() {
writable = "true"
}
alloc := "false"
if metaNode.PartitionCntLimited() && metaNode.IsWriteAble() {
alloc = "true"
}
mm.nodeStat.Delete(map[string]string{"addr": mAddr})
mm.nodeStat.SetWithLabelValues(metaNode.Ratio, MetricRoleMetaNode, mAddr, "usageRatio", zone, setId, media, writable, alloc)
mm.nodeStat.SetWithLabelValues(float64(metaNode.Total), MetricRoleMetaNode, mAddr, "memTotal", zone, setId, media, writable, alloc)
mm.nodeStat.SetWithLabelValues(float64(metaNode.Used), MetricRoleMetaNode, mAddr, "memUsed", zone, setId, media, writable, alloc)
mm.nodeStat.SetWithLabelValues(float64(metaNode.MetaPartitionCount), MetricRoleMetaNode, mAddr, "mpCount", zone, setId, media, writable, alloc)
mm.nodeStat.SetWithLabelValues(float64(metaNode.Threshold), MetricRoleMetaNode, mAddr, "threshold", zone, setId, media, writable, alloc)
mm.nodeStat.SetBoolWithLabelValues(metaNode.IsWriteAble(), MetricRoleMetaNode, mAddr, "writable", zone, setId, media, writable, alloc)
mm.nodeStat.SetBoolWithLabelValues(metaNode.IsActive, MetricRoleMetaNode, mAddr, "active", zone, setId, media, writable, alloc)
mm.nodeStat.SetBoolWithLabelValues(metaNode.PartitionCntLimited() && metaNode.IsWriteAble(), MetricRoleMetaNode, mAddr, "alloc", zone, setId, media, writable, alloc)
return true return true
}) })
mm.metaNodesInactive.Set(float64(inactiveMetaNodesCount)) mm.metaNodesInactive.Set(float64(inactiveMetaNodesCount))
for id, count := range mm.nodesetInactiveMetaNodesCount {
mm.metaNodesetInactiveCount.SetWithLabelValues(float64(count), strconv.FormatUint(id, 10))
}
for k := range deleteNodesetCount {
mm.metaNodesetInactiveCount.DeleteLabelValues(strconv.FormatUint(k, 10))
}
}
func (mm *monitorMetrics) clearInactiveMetaNodesCountMetric() {
for k := range mm.nodesetInactiveMetaNodesCount {
mm.metaNodesetInactiveCount.DeleteLabelValues(strconv.FormatUint(k, 10))
}
} }
func (mm *monitorMetrics) updateDataNodesStat() { func (mm *monitorMetrics) updateDataNodesStat() {
var inactiveDataNodesCount uint64 var inactiveDataNodesCount uint64
deleteNodesetCount := make(map[uint64]int64)
for k, v := range mm.nodesetInactiveDataNodesCount {
log.LogErrorf("setInactiveDataNodesCountMetric, init deleteNodesetCount")
deleteNodesetCount[k] = v
delete(mm.nodesetInactiveDataNodesCount, k)
}
mm.cluster.dataNodes.Range(func(addr, node interface{}) bool { mm.cluster.dataNodes.Range(func(addr, node interface{}) bool {
dataNode, ok := node.(*DataNode) dataNode, ok := node.(*DataNode)
@ -1039,36 +1024,40 @@ func (mm *monitorMetrics) updateDataNodesStat() {
if !dataNode.isActive { if !dataNode.isActive {
inactiveDataNodesCount++ inactiveDataNodesCount++
mm.InactiveDataNodeInfo.SetWithLabelValues(1, mm.cluster.Name, dataNode.Addr) mm.InactiveDataNodeInfo.SetWithLabelValues(1, mm.cluster.Name, dataNode.Addr)
mm.nodesetInactiveDataNodesCount[dataNode.NodeSetID] = mm.nodesetInactiveDataNodesCount[dataNode.NodeSetID] + 1
delete(deleteNodesetCount, dataNode.NodeSetID)
} else { } else {
mm.InactiveDataNodeInfo.DeleteLabelValues(mm.cluster.Name, dataNode.Addr) mm.InactiveDataNodeInfo.DeleteLabelValues(mm.cluster.Name, dataNode.Addr)
} }
mm.nodeStat.SetWithLabelValues(float64(dataNode.DataPartitionCount), MetricRoleDataNode, dataNode.Addr, "dpCount")
mm.nodeStat.SetWithLabelValues(float64(dataNode.Total), MetricRoleDataNode, dataNode.Addr, "diskTotal") zone := dataNode.ZoneName
mm.nodeStat.SetWithLabelValues(float64(dataNode.Used), MetricRoleDataNode, dataNode.Addr, "diskUsed") setId := strconv.Itoa(int(dataNode.NodeSetID))
mm.nodeStat.SetWithLabelValues(float64(dataNode.AvailableSpace), MetricRoleDataNode, dataNode.Addr, "diskAvail") media := proto.MediaTypeString(dataNode.MediaType)
mm.nodeStat.SetWithLabelValues(dataNode.UsageRatio, MetricRoleDataNode, dataNode.Addr, "usageRatio") dAddr := dataNode.Addr
mm.nodeStat.SetWithLabelValues(float64(len(dataNode.BadDisks)), MetricRoleDataNode, dataNode.Addr, "badDiskCount")
mm.nodeStat.SetWithLabelValues(float64(len(dataNode.LostDisks)), MetricRoleDataNode, dataNode.Addr, "lostDiskCount") writable := "false"
mm.nodeStat.SetBoolWithLabelValues(dataNode.isActive, MetricRoleDataNode, dataNode.Addr, "active") if dataNode.IsWriteAble() {
mm.nodeStat.SetBoolWithLabelValues(dataNode.IsWriteAble(), MetricRoleDataNode, dataNode.Addr, "writable") writable = "true"
}
alloc := "false"
if dataNode.canAllocDp() {
alloc = "true"
}
mm.nodeStat.Delete(map[string]string{"addr": dAddr})
mm.nodeStat.SetWithLabelValues(float64(dataNode.DataPartitionCount), MetricRoleDataNode, dAddr, "dpCount", zone, setId, media, writable, alloc)
mm.nodeStat.SetWithLabelValues(float64(dataNode.Total), MetricRoleDataNode, dAddr, "diskTotal", zone, setId, media, writable, alloc)
mm.nodeStat.SetWithLabelValues(float64(dataNode.Used), MetricRoleDataNode, dAddr, "diskUsed", zone, setId, media, writable, alloc)
mm.nodeStat.SetWithLabelValues(float64(dataNode.AvailableSpace), MetricRoleDataNode, dAddr, "diskAvail", zone, setId, media, writable, alloc)
mm.nodeStat.SetWithLabelValues(dataNode.UsageRatio, MetricRoleDataNode, dAddr, "usageRatio", zone, setId, media, writable, alloc)
mm.nodeStat.SetWithLabelValues(float64(len(dataNode.BadDisks)), MetricRoleDataNode, dAddr, "badDiskCount", zone, setId, media, writable, alloc)
mm.nodeStat.SetWithLabelValues(float64(len(dataNode.LostDisks)), MetricRoleDataNode, dAddr, "lostDiskCount", zone, setId, media, writable, alloc)
mm.nodeStat.SetBoolWithLabelValues(dataNode.isActive, MetricRoleDataNode, dAddr, "active", zone, setId, media, writable, alloc)
mm.nodeStat.SetBoolWithLabelValues(dataNode.IsWriteAble(), MetricRoleDataNode, dAddr, "writable", zone, setId, media, writable, alloc)
mm.nodeStat.SetBoolWithLabelValues(dataNode.canAllocDp(), MetricRoleDataNode, dAddr, "canAlloc", zone, setId, media, writable, alloc)
return true return true
}) })
mm.dataNodesInactive.Set(float64(inactiveDataNodesCount)) mm.dataNodesInactive.Set(float64(inactiveDataNodesCount))
for id, count := range mm.nodesetInactiveDataNodesCount {
mm.dataNodesetInactiveCount.SetWithLabelValues(float64(count), strconv.FormatUint(id, 10))
}
for k := range deleteNodesetCount {
mm.dataNodesetInactiveCount.DeleteLabelValues(strconv.FormatUint(k, 10))
}
}
func (mm *monitorMetrics) clearInactiveDataNodesCountMetric() {
for k := range mm.nodesetInactiveDataNodesCount {
mm.dataNodesetInactiveCount.DeleteLabelValues(strconv.FormatUint(k, 10))
}
} }
func (mm *monitorMetrics) updateMastersStat() { func (mm *monitorMetrics) updateMastersStat() {
@ -1148,12 +1137,6 @@ func (mm *monitorMetrics) setNotWritableDataNodesCount() {
} }
} }
func (mm *monitorMetrics) clearInconsistentMps() {
for k := range mm.inconsistentMps {
mm.dataNodesetInactiveCount.DeleteLabelValues(k)
}
}
func (mm *monitorMetrics) deleteS3LcVolMetric(id string) { func (mm *monitorMetrics) deleteS3LcVolMetric(id string) {
mm.lcVolStatus.DeleteLabelValues(id) mm.lcVolStatus.DeleteLabelValues(id)
mm.lcVolScanned.DeleteLabelValues(id, "file") mm.lcVolScanned.DeleteLabelValues(id, "file")
@ -1319,9 +1302,7 @@ func (mm *monitorMetrics) resetAllLeaderMetrics() {
mm.clearVolMetrics() mm.clearVolMetrics()
mm.clearDiskErrMetrics() mm.clearDiskErrMetrics()
mm.clearFlashNodesDiskErrMetrics() mm.clearFlashNodesDiskErrMetrics()
mm.clearInactiveMetaNodesCountMetric() mm.metaEqualCheckFail.Reset()
mm.clearInactiveDataNodesCountMetric()
mm.clearInconsistentMps()
mm.clearNodesetMetrics() mm.clearNodesetMetrics()
mm.clearLcMetrics() mm.clearLcMetrics()

4
util/exporter/gauge.go
View File

@ -170,6 +170,10 @@ func (v *GaugeVec) SetBoolWithLabelValues(val bool, lvs ...string) {
} }
} }
func (v *GaugeVec) Delete(kvs map[string]string) {
v.GaugeVec.DeletePartialMatch(kvs)
}
func (v *GaugeVec) Reset() { func (v *GaugeVec) Reset() {
v.GaugeVec.Reset() v.GaugeVec.Reset()
} }