From ac3a91e105122fa8b0e6981e36626cebf0cdc1e4 Mon Sep 17 00:00:00 2001 From: wangxiaodong1 Date: Thu, 9 May 2024 20:05:07 +0800 Subject: [PATCH] feat(master): add replicas laber for dp and mp noleader metrics Signed-off-by: wangxiaodong1 --- master/data_partition_check.go | 2 +- master/meta_partition.go | 2 +- master/monitor_metrics.go | 53 +++++++++++++++++++--------------- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/master/data_partition_check.go b/master/data_partition_check.go index 1fff2c527..960e57972 100644 --- a/master/data_partition_check.go +++ b/master/data_partition_check.go @@ -149,7 +149,7 @@ func (partition *DataPartition) checkLeader(c *Cluster, clusterID string, timeOu report = true } if WarnMetrics != nil { - WarnMetrics.WarnDpNoLeader(clusterID, partition.PartitionID, report) + WarnMetrics.WarnDpNoLeader(clusterID, partition.PartitionID, partition.ReplicaNum, report) } partition.Unlock() if report && partition.ReplicaNum == 1 && partition.GetDecommissionStatus() == DecommissionInitial && diff --git a/master/meta_partition.go b/master/meta_partition.go index 63807406a..6c65a2e77 100644 --- a/master/meta_partition.go +++ b/master/meta_partition.go @@ -285,7 +285,7 @@ func (mp *MetaPartition) checkLeader(clusterID string) { report = true } if WarnMetrics != nil { - WarnMetrics.WarnMpNoLeader(clusterID, mp.PartitionID, report) + WarnMetrics.WarnMpNoLeader(clusterID, mp.PartitionID, mp.ReplicaNum, report) } } diff --git a/master/monitor_metrics.go b/master/monitor_metrics.go index f2edec92c..44e6c8aca 100644 --- a/master/monitor_metrics.go +++ b/master/monitor_metrics.go @@ -173,6 +173,11 @@ type addrSet struct { replicaAlive string } +type NoLeaderPartInfo struct { + ReportTime int64 + Replicas uint8 +} + type warningMetrics struct { cluster *Cluster missingDp *exporter.GaugeVec @@ -181,8 +186,8 @@ type warningMetrics struct { mpNoLeader *exporter.GaugeVec dpMutex sync.Mutex mpMutex sync.Mutex - dpNoLeaderInfo map[uint64]int64 - mpNoLeaderInfo map[uint64]int64 + dpNoLeaderInfo map[uint64]NoLeaderPartInfo + mpNoLeaderInfo map[uint64]NoLeaderPartInfo dpMissingReplicaMutex sync.Mutex mpMissingReplicaMutex sync.Mutex dpMissingReplicaInfo map[string]addrSet @@ -193,11 +198,11 @@ func newWarningMetrics(c *Cluster) *warningMetrics { return &warningMetrics{ cluster: c, missingDp: exporter.NewGaugeVec(MetricMissingDp, "", []string{"clusterName", "partitionID", "addr", "ReplicaAlive", "ReplicaNum"}), - dpNoLeader: exporter.NewGaugeVec(MetricDpNoLeader, "", []string{"clusterName", "partitionID"}), + dpNoLeader: exporter.NewGaugeVec(MetricDpNoLeader, "", []string{"clusterName", "partitionID", "ReplicaNum"}), missingMp: exporter.NewGaugeVec(MetricMissingMp, "", []string{"clusterName", "partitionID", "addr"}), - mpNoLeader: exporter.NewGaugeVec(MetricMpNoLeader, "", []string{"clusterName", "partitionID"}), - dpNoLeaderInfo: make(map[uint64]int64), - mpNoLeaderInfo: make(map[uint64]int64), + mpNoLeader: exporter.NewGaugeVec(MetricMpNoLeader, "", []string{"clusterName", "partitionID", "ReplicaNum"}), + dpNoLeaderInfo: make(map[uint64]NoLeaderPartInfo), + mpNoLeaderInfo: make(map[uint64]NoLeaderPartInfo), dpMissingReplicaInfo: make(map[string]addrSet), mpMissingReplicaInfo: make(map[string]addrSet), } @@ -206,18 +211,18 @@ func newWarningMetrics(c *Cluster) *warningMetrics { func (m *warningMetrics) reset() { log.LogInfo("action[warningMetrics] reset all") m.dpMutex.Lock() - for dp := range m.dpNoLeaderInfo { + for dp, noLeaderInfo := range m.dpNoLeaderInfo { if m.dpNoLeader != nil { - m.dpNoLeader.DeleteLabelValues(m.cluster.Name, strconv.FormatUint(dp, 10)) + m.dpNoLeader.DeleteLabelValues(m.cluster.Name, strconv.FormatUint(dp, 10), strconv.FormatUint(uint64(noLeaderInfo.Replicas), 10)) } delete(m.dpNoLeaderInfo, dp) } m.dpMutex.Unlock() m.mpMutex.Lock() - for mp := range m.mpNoLeaderInfo { + for mp, noLeaderInfo := range m.mpNoLeaderInfo { if m.mpNoLeader != nil { - m.mpNoLeader.DeleteLabelValues(m.cluster.Name, strconv.FormatUint(mp, 10)) + m.mpNoLeader.DeleteLabelValues(m.cluster.Name, strconv.FormatUint(mp, 10), strconv.FormatUint(uint64(noLeaderInfo.Replicas), 10)) } delete(m.mpNoLeaderInfo, mp) } @@ -318,19 +323,19 @@ func (m *warningMetrics) CleanObsoleteDpMissing(clusterName string, dp *DataPart } // leader only -func (m *warningMetrics) WarnDpNoLeader(clusterName string, partitionID uint64, report bool) { +func (m *warningMetrics) WarnDpNoLeader(clusterName string, partitionID uint64, replicas uint8, report bool) { if clusterName != m.cluster.Name { return } m.dpMutex.Lock() defer m.dpMutex.Unlock() - t, ok := m.dpNoLeaderInfo[partitionID] + info, ok := m.dpNoLeaderInfo[partitionID] if !report { if ok { delete(m.dpNoLeaderInfo, partitionID) if m.dpNoLeader != nil { - m.dpNoLeader.DeleteLabelValues(clusterName, strconv.FormatUint(partitionID, 10)) + m.dpNoLeader.DeleteLabelValues(clusterName, strconv.FormatUint(partitionID, 10), strconv.FormatUint(uint64(replicas), 10)) } } return @@ -338,14 +343,14 @@ func (m *warningMetrics) WarnDpNoLeader(clusterName string, partitionID uint64, now := time.Now().Unix() if !ok { - m.dpNoLeaderInfo[partitionID] = now + m.dpNoLeaderInfo[partitionID] = NoLeaderPartInfo{ReportTime: now, Replicas: replicas} return } - if now-t > m.cluster.cfg.DpNoLeaderReportIntervalSec { + if now-info.ReportTime > m.cluster.cfg.DpNoLeaderReportIntervalSec { if m.dpNoLeader != nil { - m.dpNoLeader.SetWithLabelValues(1, clusterName, strconv.FormatUint(partitionID, 10)) + m.dpNoLeader.SetWithLabelValues(1, clusterName, strconv.FormatUint(partitionID, 10), strconv.FormatUint(uint64(replicas), 10)) } - m.dpNoLeaderInfo[partitionID] = now + m.dpNoLeaderInfo[partitionID] = NoLeaderPartInfo{ReportTime: now, Replicas: replicas} } } @@ -417,18 +422,18 @@ func (m *warningMetrics) CleanObsoleteMpMissing(clusterName string, mp *MetaPart } // leader only -func (m *warningMetrics) WarnMpNoLeader(clusterName string, partitionID uint64, report bool) { +func (m *warningMetrics) WarnMpNoLeader(clusterName string, partitionID uint64, replicas uint8, report bool) { if clusterName != m.cluster.Name { return } m.mpMutex.Lock() defer m.mpMutex.Unlock() - t, ok := m.mpNoLeaderInfo[partitionID] + info, ok := m.mpNoLeaderInfo[partitionID] if !report { if ok { delete(m.mpNoLeaderInfo, partitionID) if m.mpNoLeader != nil { - m.mpNoLeader.DeleteLabelValues(clusterName, strconv.FormatUint(partitionID, 10)) + m.mpNoLeader.DeleteLabelValues(clusterName, strconv.FormatUint(partitionID, 10), strconv.FormatUint(uint64(replicas), 10)) } } return @@ -437,15 +442,15 @@ func (m *warningMetrics) WarnMpNoLeader(clusterName string, partitionID uint64, now := time.Now().Unix() if !ok { - m.mpNoLeaderInfo[partitionID] = now + m.mpNoLeaderInfo[partitionID] = NoLeaderPartInfo{ReportTime: now, Replicas: replicas} return } - if now-t > m.cluster.cfg.MpNoLeaderReportIntervalSec { + if now-info.ReportTime > m.cluster.cfg.MpNoLeaderReportIntervalSec { if m.mpNoLeader != nil { - m.mpNoLeader.SetWithLabelValues(1, clusterName, strconv.FormatUint(partitionID, 10)) + m.mpNoLeader.SetWithLabelValues(1, clusterName, strconv.FormatUint(partitionID, 10), strconv.FormatUint(uint64(replicas), 10)) } - m.mpNoLeaderInfo[partitionID] = now + m.mpNoLeaderInfo[partitionID] = NoLeaderPartInfo{ReportTime: now, Replicas: replicas} } }