gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 02:00:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(master): add replicas laber for dp and mp noleader metrics

Browse Source 
Signed-off-by: wangxiaodong1 <wangxiaodong1@oppo.com>
This commit is contained in:
wangxiaodong1 2024-05-09 20:05:07 +08:00 committed by longerfly
parent 2848a2c6db
commit ac3a91e105
3 changed files with 31 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
master/data_partition_check.go
View File

@ -149,7 +149,7 @@ func (partition *DataPartition) checkLeader(c *Cluster, clusterID string, timeOu
report = true
}
if WarnMetrics != nil {
WarnMetrics.WarnDpNoLeader(clusterID, partition.PartitionID, report)
WarnMetrics.WarnDpNoLeader(clusterID, partition.PartitionID, partition.ReplicaNum, report)
}
partition.Unlock()
if report && partition.ReplicaNum == 1 && partition.GetDecommissionStatus() == DecommissionInitial &&

2
master/meta_partition.go
View File

@ -285,7 +285,7 @@ func (mp *MetaPartition) checkLeader(clusterID string) {
report = true
}
if WarnMetrics != nil {
WarnMetrics.WarnMpNoLeader(clusterID, mp.PartitionID, report)
WarnMetrics.WarnMpNoLeader(clusterID, mp.PartitionID, mp.ReplicaNum, report)
}
}

53
master/monitor_metrics.go
View File

@ -173,6 +173,11 @@ type addrSet struct {
replicaAlive string
}
type NoLeaderPartInfo struct {
ReportTime int64
Replicas uint8
}
type warningMetrics struct {
cluster *Cluster
missingDp *exporter.GaugeVec
@ -181,8 +186,8 @@ type warningMetrics struct {
mpNoLeader *exporter.GaugeVec
dpMutex sync.Mutex
mpMutex sync.Mutex
dpNoLeaderInfo map[uint64]int64
mpNoLeaderInfo map[uint64]int64
dpNoLeaderInfo map[uint64]NoLeaderPartInfo
mpNoLeaderInfo map[uint64]NoLeaderPartInfo
dpMissingReplicaMutex sync.Mutex
mpMissingReplicaMutex sync.Mutex
dpMissingReplicaInfo map[string]addrSet
@ -193,11 +198,11 @@ func newWarningMetrics(c *Cluster) *warningMetrics {
return &warningMetrics{
cluster: c,
missingDp: exporter.NewGaugeVec(MetricMissingDp, "", []string{"clusterName", "partitionID", "addr", "ReplicaAlive", "ReplicaNum"}),
dpNoLeader: exporter.NewGaugeVec(MetricDpNoLeader, "", []string{"clusterName", "partitionID"}),
dpNoLeader: exporter.NewGaugeVec(MetricDpNoLeader, "", []string{"clusterName", "partitionID", "ReplicaNum"}),
missingMp: exporter.NewGaugeVec(MetricMissingMp, "", []string{"clusterName", "partitionID", "addr"}),
mpNoLeader: exporter.NewGaugeVec(MetricMpNoLeader, "", []string{"clusterName", "partitionID"}),
dpNoLeaderInfo: make(map[uint64]int64),
mpNoLeaderInfo: make(map[uint64]int64),
mpNoLeader: exporter.NewGaugeVec(MetricMpNoLeader, "", []string{"clusterName", "partitionID", "ReplicaNum"}),
dpNoLeaderInfo: make(map[uint64]NoLeaderPartInfo),
mpNoLeaderInfo: make(map[uint64]NoLeaderPartInfo),
dpMissingReplicaInfo: make(map[string]addrSet),
mpMissingReplicaInfo: make(map[string]addrSet),
}
@ -206,18 +211,18 @@ func newWarningMetrics(c *Cluster) *warningMetrics {
func (m *warningMetrics) reset() {
log.LogInfo("action[warningMetrics] reset all")
m.dpMutex.Lock()
for dp := range m.dpNoLeaderInfo {
for dp, noLeaderInfo := range m.dpNoLeaderInfo {
if m.dpNoLeader != nil {
m.dpNoLeader.DeleteLabelValues(m.cluster.Name, strconv.FormatUint(dp, 10))
m.dpNoLeader.DeleteLabelValues(m.cluster.Name, strconv.FormatUint(dp, 10), strconv.FormatUint(uint64(noLeaderInfo.Replicas), 10))
}
delete(m.dpNoLeaderInfo, dp)
}
m.dpMutex.Unlock()
m.mpMutex.Lock()
for mp := range m.mpNoLeaderInfo {
for mp, noLeaderInfo := range m.mpNoLeaderInfo {
if m.mpNoLeader != nil {
m.mpNoLeader.DeleteLabelValues(m.cluster.Name, strconv.FormatUint(mp, 10))
m.mpNoLeader.DeleteLabelValues(m.cluster.Name, strconv.FormatUint(mp, 10), strconv.FormatUint(uint64(noLeaderInfo.Replicas), 10))
}
delete(m.mpNoLeaderInfo, mp)
}
@ -318,19 +323,19 @@ func (m *warningMetrics) CleanObsoleteDpMissing(clusterName string, dp *DataPart
}
// leader only
func (m *warningMetrics) WarnDpNoLeader(clusterName string, partitionID uint64, report bool) {
func (m *warningMetrics) WarnDpNoLeader(clusterName string, partitionID uint64, replicas uint8, report bool) {
if clusterName != m.cluster.Name {
return
}
m.dpMutex.Lock()
defer m.dpMutex.Unlock()
t, ok := m.dpNoLeaderInfo[partitionID]
info, ok := m.dpNoLeaderInfo[partitionID]
if !report {
if ok {
delete(m.dpNoLeaderInfo, partitionID)
if m.dpNoLeader != nil {
m.dpNoLeader.DeleteLabelValues(clusterName, strconv.FormatUint(partitionID, 10))
m.dpNoLeader.DeleteLabelValues(clusterName, strconv.FormatUint(partitionID, 10), strconv.FormatUint(uint64(replicas), 10))
}
}
return
@ -338,14 +343,14 @@ func (m *warningMetrics) WarnDpNoLeader(clusterName string, partitionID uint64,
now := time.Now().Unix()
if !ok {
m.dpNoLeaderInfo[partitionID] = now
m.dpNoLeaderInfo[partitionID] = NoLeaderPartInfo{ReportTime: now, Replicas: replicas}
return
}
if now-t > m.cluster.cfg.DpNoLeaderReportIntervalSec {
if now-info.ReportTime > m.cluster.cfg.DpNoLeaderReportIntervalSec {
if m.dpNoLeader != nil {
m.dpNoLeader.SetWithLabelValues(1, clusterName, strconv.FormatUint(partitionID, 10))
m.dpNoLeader.SetWithLabelValues(1, clusterName, strconv.FormatUint(partitionID, 10), strconv.FormatUint(uint64(replicas), 10))
}
m.dpNoLeaderInfo[partitionID] = now
m.dpNoLeaderInfo[partitionID] = NoLeaderPartInfo{ReportTime: now, Replicas: replicas}
}
}
@ -417,18 +422,18 @@ func (m *warningMetrics) CleanObsoleteMpMissing(clusterName string, mp *MetaPart
}
// leader only
func (m *warningMetrics) WarnMpNoLeader(clusterName string, partitionID uint64, report bool) {
func (m *warningMetrics) WarnMpNoLeader(clusterName string, partitionID uint64, replicas uint8, report bool) {
if clusterName != m.cluster.Name {
return
}
m.mpMutex.Lock()
defer m.mpMutex.Unlock()
t, ok := m.mpNoLeaderInfo[partitionID]
info, ok := m.mpNoLeaderInfo[partitionID]
if !report {
if ok {
delete(m.mpNoLeaderInfo, partitionID)
if m.mpNoLeader != nil {
m.mpNoLeader.DeleteLabelValues(clusterName, strconv.FormatUint(partitionID, 10))
m.mpNoLeader.DeleteLabelValues(clusterName, strconv.FormatUint(partitionID, 10), strconv.FormatUint(uint64(replicas), 10))
}
}
return
@ -437,15 +442,15 @@ func (m *warningMetrics) WarnMpNoLeader(clusterName string, partitionID uint64,
now := time.Now().Unix()
if !ok {
m.mpNoLeaderInfo[partitionID] = now
m.mpNoLeaderInfo[partitionID] = NoLeaderPartInfo{ReportTime: now, Replicas: replicas}
return
}
if now-t > m.cluster.cfg.MpNoLeaderReportIntervalSec {
if now-info.ReportTime > m.cluster.cfg.MpNoLeaderReportIntervalSec {
if m.mpNoLeader != nil {
m.mpNoLeader.SetWithLabelValues(1, clusterName, strconv.FormatUint(partitionID, 10))
m.mpNoLeader.SetWithLabelValues(1, clusterName, strconv.FormatUint(partitionID, 10), strconv.FormatUint(uint64(replicas), 10))
}
m.mpNoLeaderInfo[partitionID] = now
m.mpNoLeaderInfo[partitionID] = NoLeaderPartInfo{ReportTime: now, Replicas: replicas}
}
}