diff --git a/datanode/disk.go b/datanode/disk.go index c26b90752..145f313a7 100644 --- a/datanode/disk.go +++ b/datanode/disk.go @@ -104,6 +104,7 @@ type Disk struct { // diskPartition info diskPartition *disk.PartitionStat DiskErrPartitionSet sync.Map + BadDiskFirstReportTime time.Time decommission bool extentRepairReadLimit chan struct{} enableExtentRepairReadLimit bool diff --git a/datanode/space_manager.go b/datanode/space_manager.go index 83ef21c55..92cf37a4e 100644 --- a/datanode/space_manager.go +++ b/datanode/space_manager.go @@ -844,10 +844,14 @@ func (s *DataNode) buildHeartBeatResponse(response *proto.DataNodeHeartbeatRespo log.LogInfof("[buildHeartBeatResponse] disk(%v) status(%v) broken dp len(%v)", d.Path, d.Status, brokenDpsCnt) if d.Status == proto.Unavailable || brokenDpsCnt != 0 { response.BadDisks = append(response.BadDisks, d.Path) + if d.BadDiskFirstReportTime.IsZero() { + d.BadDiskFirstReportTime = time.Now() + } bds := proto.BadDiskStat{ DiskPath: d.Path, TotalPartitionCnt: d.PartitionCount(), DiskErrPartitionList: brokenDps, + FirstReportTime: d.BadDiskFirstReportTime, } response.BadDiskStats = append(response.BadDiskStats, bds) log.LogErrorf("[buildHeartBeatResponse] disk(%v) total(%v) broken dp len(%v) %v", diff --git a/master/monitor_metrics.go b/master/monitor_metrics.go index 688c376ca..894bc6708 100644 --- a/master/monitor_metrics.go +++ b/master/monitor_metrics.go @@ -31,43 +31,44 @@ import ( // metrics const ( - StatPeriod = time.Minute * time.Duration(1) - MetricDataNodesUsedGB = "dataNodes_used_GB" - MetricDataNodesTotalGB = "dataNodes_total_GB" - MetricDataNodesStat = "dataNodes_stats" - MetricDataNodesIncreasedGB = "dataNodes_increased_GB" - MetricMetaNodesUsedGB = "metaNodes_used_GB" - MetricMetaNodesTotalGB = "metaNodes_total_GB" - MetricMetaNodesIncreasedGB = "metaNodes_increased_GB" - MetricDataNodesCount = "dataNodes_count" - MetricMetaNodesCount = "metaNodes_count" - MetricNodeStat = "node_stat" - MetricVolCount = "vol_count" - MetricVolTotalGB = "vol_total_GB" - MetricVolUsedGB = "vol_used_GB" - MetricVolUsageGB = "vol_usage_ratio" - MetricVolStats = "vol_stats" - MetricVolMetaCount = "vol_meta_count" - MetricBadMpCount = "bad_mp_count" - MetricBadDpCount = "bad_dp_count" - MetricDiskError = "disk_error" - MetricFlashNodesDiskError = "flashNodes_disk_error" - MetricDiskLost = "disk_lost" - MetricDpUnableDecommissionCount = "dp_unable_decommission_count" - MetricDpNoSamePeer = "dp_no_same_peer" - MetricDataNodesInactive = "dataNodes_inactive" - MetricInactiveDataNodeInfo = "inactive_dataNodes_info" - MetricMetaNodesInactive = "metaNodes_inactive" - MetricDataNodesNotWritable = "dataNodes_not_writable" - MetricDataNodesAllocable = "dataNodes_allocable" - MetricMetaNodesNotWritable = "metaNodes_not_writable" - MetricInactiveMetaNodeInfo = "inactive_metaNodes_info" - MetricMetaInconsistent = "mp_inconsistent" - MetricMasterNoLeader = "master_no_leader" - MetricMasterNoCache = "master_no_cache" - MetricMasterSnapshot = "master_snapshot" - MetricMastersInactive = "masters_inactive" - MetricInactiveMasterInfo = "inactive_masters_info" + StatPeriod = time.Minute * time.Duration(1) + MetricDataNodesUsedGB = "dataNodes_used_GB" + MetricDataNodesTotalGB = "dataNodes_total_GB" + MetricDataNodesStat = "dataNodes_stats" + MetricDataNodesIncreasedGB = "dataNodes_increased_GB" + MetricMetaNodesUsedGB = "metaNodes_used_GB" + MetricMetaNodesTotalGB = "metaNodes_total_GB" + MetricMetaNodesIncreasedGB = "metaNodes_increased_GB" + MetricDataNodesCount = "dataNodes_count" + MetricMetaNodesCount = "metaNodes_count" + MetricNodeStat = "node_stat" + MetricVolCount = "vol_count" + MetricVolTotalGB = "vol_total_GB" + MetricVolUsedGB = "vol_used_GB" + MetricVolUsageGB = "vol_usage_ratio" + MetricVolStats = "vol_stats" + MetricVolMetaCount = "vol_meta_count" + MetricBadMpCount = "bad_mp_count" + MetricBadDpCount = "bad_dp_count" + MetricDiskError = "disk_error" + MetricFlashNodesDiskError = "flashNodes_disk_error" + MetricDiskLost = "disk_lost" + MetricDpUnableDecommissionCount = "dp_unable_decommission_count" + MetricDpNoSamePeer = "dp_no_same_peer" + MetricBadDiskDecommissionTimeOverLimit = "bad_disk_decommission_time_over_limit" + MetricDataNodesInactive = "dataNodes_inactive" + MetricInactiveDataNodeInfo = "inactive_dataNodes_info" + MetricMetaNodesInactive = "metaNodes_inactive" + MetricDataNodesNotWritable = "dataNodes_not_writable" + MetricDataNodesAllocable = "dataNodes_allocable" + MetricMetaNodesNotWritable = "metaNodes_not_writable" + MetricInactiveMetaNodeInfo = "inactive_metaNodes_info" + MetricMetaInconsistent = "mp_inconsistent" + MetricMasterNoLeader = "master_no_leader" + MetricMasterNoCache = "master_no_cache" + MetricMasterSnapshot = "master_snapshot" + MetricMastersInactive = "masters_inactive" + MetricInactiveMasterInfo = "inactive_masters_info" MetricMissingDp = "missing_dp" MetricDpNoLeader = "dp_no_leader" @@ -106,46 +107,47 @@ const ( var WarnMetrics *warningMetrics type monitorMetrics struct { - cluster *Cluster - dataNodesCount *exporter.Gauge - metaNodesCount *exporter.Gauge - volCount *exporter.Gauge - dataNodesTotal *exporter.Gauge - dataNodesUsed *exporter.Gauge - dataNodeStat *exporter.GaugeVec - dataNodeIncreased *exporter.Gauge - metaNodesTotal *exporter.Gauge - metaNodesUsed *exporter.Gauge - metaNodesIncreased *exporter.Gauge - volTotalSpace *exporter.GaugeVec - volUsedSpace *exporter.GaugeVec - volUsage *exporter.GaugeVec - volMetaCount *exporter.GaugeVec - volStats *exporter.GaugeVec - badMpCount *exporter.Gauge - badDpCount *exporter.Gauge - diskError *exporter.GaugeVec - flashNodesDiskError *exporter.GaugeVec - diskLost *exporter.GaugeVec - dpUnableDecommissionCount *exporter.Gauge - dpNoSamePeer *exporter.GaugeVec - dataNodesNotWritable *exporter.Gauge // TODO: remove in the future - dataNodesAllocable *exporter.Gauge // TODO: remove in the future - metaNodesNotWritable *exporter.Gauge // TODO: remove in the future - dataNodesInactive *exporter.Gauge // TODO: remove in the future - InactiveDataNodeInfo *exporter.GaugeVec // TODO: remove in the future - metaNodesInactive *exporter.Gauge // TODO: remove in the future - InactiveMetaNodeInfo *exporter.GaugeVec // TODO: remove in the future - mastersInactive *exporter.Gauge - InactiveMasterInfo *exporter.GaugeVec - ReplicaMissingDPCount *exporter.GaugeVec - DpMissingLeaderCount *exporter.GaugeVec - MpMissingLeaderCount *exporter.Gauge - MpMissingReplicaCount *exporter.Gauge - metaEqualCheckFail *exporter.GaugeVec - masterNoLeader *exporter.Gauge - masterNoCache *exporter.GaugeVec - masterSnapshot *exporter.Gauge + cluster *Cluster + dataNodesCount *exporter.Gauge + metaNodesCount *exporter.Gauge + volCount *exporter.Gauge + dataNodesTotal *exporter.Gauge + dataNodesUsed *exporter.Gauge + dataNodeStat *exporter.GaugeVec + dataNodeIncreased *exporter.Gauge + metaNodesTotal *exporter.Gauge + metaNodesUsed *exporter.Gauge + metaNodesIncreased *exporter.Gauge + volTotalSpace *exporter.GaugeVec + volUsedSpace *exporter.GaugeVec + volUsage *exporter.GaugeVec + volMetaCount *exporter.GaugeVec + volStats *exporter.GaugeVec + badMpCount *exporter.Gauge + badDpCount *exporter.Gauge + diskError *exporter.GaugeVec + flashNodesDiskError *exporter.GaugeVec + diskLost *exporter.GaugeVec + dpUnableDecommissionCount *exporter.Gauge + dpNoSamePeer *exporter.GaugeVec + badDiskDecommissionTimeOverLimit *exporter.GaugeVec + dataNodesNotWritable *exporter.Gauge // TODO: remove in the future + dataNodesAllocable *exporter.Gauge // TODO: remove in the future + metaNodesNotWritable *exporter.Gauge // TODO: remove in the future + dataNodesInactive *exporter.Gauge // TODO: remove in the future + InactiveDataNodeInfo *exporter.GaugeVec // TODO: remove in the future + metaNodesInactive *exporter.Gauge // TODO: remove in the future + InactiveMetaNodeInfo *exporter.GaugeVec // TODO: remove in the future + mastersInactive *exporter.Gauge + InactiveMasterInfo *exporter.GaugeVec + ReplicaMissingDPCount *exporter.GaugeVec + DpMissingLeaderCount *exporter.GaugeVec + MpMissingLeaderCount *exporter.Gauge + MpMissingReplicaCount *exporter.Gauge + metaEqualCheckFail *exporter.GaugeVec + masterNoLeader *exporter.Gauge + masterNoCache *exporter.GaugeVec + masterSnapshot *exporter.Gauge // TODO remove in next version nodesetMetaTotal *exporter.GaugeVec nodesetMetaUsed *exporter.GaugeVec @@ -516,6 +518,7 @@ func (mm *monitorMetrics) start() { mm.diskLost = exporter.NewGaugeVec(MetricDiskLost, "", []string{"addr", "path"}) mm.dpUnableDecommissionCount = exporter.NewGauge(MetricDpUnableDecommissionCount) mm.dpNoSamePeer = exporter.NewGaugeVec(MetricDpNoSamePeer, "", []string{"dpId"}) + mm.badDiskDecommissionTimeOverLimit = exporter.NewGaugeVec(MetricBadDiskDecommissionTimeOverLimit, "", []string{"addr", "path", "firstReportTime"}) mm.nodeStat = exporter.NewGaugeVec(MetricNodeStat, "", []string{"type", "addr", "stat", "zone", "set", "media", "writable", "alloc"}) mm.dataNodesInactive = exporter.NewGauge(MetricDataNodesInactive) mm.InactiveDataNodeInfo = exporter.NewGaugeVec(MetricInactiveDataNodeInfo, "", []string{"clusterName", "addr"}) @@ -618,6 +621,7 @@ func (mm *monitorMetrics) doStat() { mm.setFlashNodesDiskErrorMetric() mm.setDpUnableDecommissionMetric() mm.setDpNoSamePeerMetric() + mm.setBadDiskDecommissionTimeOverLimit() mm.setNotWritableDataNodesCount() mm.setNotWritableMetaNodesCount() mm.setMpInconsistentErrorMetric() @@ -972,6 +976,25 @@ func (mm *monitorMetrics) setDpNoSamePeerMetric() { }) } +func (mm *monitorMetrics) setBadDiskDecommissionTimeOverLimit() { + mm.badDiskDecommissionTimeOverLimit.Reset() + + mm.cluster.dataNodes.Range(func(addr, node interface{}) bool { + dataNode, ok := node.(*DataNode) + if !ok { + return true + } + for _, badDiskStat := range dataNode.BadDiskStats { + _, isSuccess := dataNode.DecommissionSuccessDisks.Load(badDiskStat.DiskPath) + if !badDiskStat.FirstReportTime.IsZero() && time.Since(badDiskStat.FirstReportTime) > 24*time.Hour && !isSuccess { + mm.badDiskDecommissionTimeOverLimit.SetWithLabelValues(1, dataNode.Addr, badDiskStat.DiskPath, + badDiskStat.FirstReportTime.Format("2006-01-02 15:04:05")) + } + } + return true + }) +} + func (mm *monitorMetrics) setDiskDecommissionedMetric() { mm.diskDecommissionSuccess.Reset() @@ -1343,6 +1366,7 @@ func (mm *monitorMetrics) resetAllLeaderMetrics() { mm.diskLost.Reset() mm.dpUnableDecommissionCount.Set(0) mm.dpNoSamePeer.Reset() + mm.badDiskDecommissionTimeOverLimit.Reset() mm.diskDecommissionSuccess.Reset() mm.dataNodesInactive.Set(0) mm.metaNodesInactive.Set(0) diff --git a/proto/admin_proto.go b/proto/admin_proto.go index ef1a199b1..e029dbb1e 100644 --- a/proto/admin_proto.go +++ b/proto/admin_proto.go @@ -897,6 +897,7 @@ type BadDiskStat struct { DiskPath string TotalPartitionCnt int DiskErrPartitionList []uint64 + FirstReportTime time.Time } type DiskStat struct {