diff --git a/cli/cmd/datapartition.go b/cli/cmd/datapartition.go index 55a940bbf..17264f26b 100644 --- a/cli/cmd/datapartition.go +++ b/cli/cmd/datapartition.go @@ -98,6 +98,7 @@ func newListCorruptDataPartitionCmd(client *master.MasterClient) *cobra.Command var showBadDp bool var showUnavailable bool var showExcess bool + var showMissingTinyExtent bool var showDiskError bool var showDiscard bool var showDiff bool @@ -123,7 +124,7 @@ The "reset" command will be released in next version`, }() showAll := !showInactiveNodes && !showNoLeader && !showLack && !showDiscard && !showBadDp && - !showDiff && !showUnavailable && !showExcess && !showDiskError && !showSimplified + !showDiff && !showUnavailable && !showExcess && !showMissingTinyExtent && !showDiskError && !showSimplified if diagnosis, err = client.AdminAPI().DiagnoseDataPartition(true); err != nil { return } @@ -353,6 +354,27 @@ The "reset" command will be released in next version`, stdoutlnf("[Partition with excessive replicas count]: %v", len(diagnosis.ExcessReplicaDpIDs)) } + stdoutln() + if !showSimplified && showMissingTinyExtent { + stdoutln("[Partition with missing tiny extent]:") + stdoutln(partitionInfoTableHeader) + sort.SliceStable(diagnosis.MissingTinyExtentDpIDs, func(i, j int) bool { + return diagnosis.MissingTinyExtentDpIDs[i] < diagnosis.MissingTinyExtentDpIDs[j] + }) + for _, pid := range diagnosis.MissingTinyExtentDpIDs { + var partition *proto.DataPartitionInfo + if partition, err = client.AdminAPI().GetDataPartition("", pid); err != nil { + err = fmt.Errorf("Partition not found[%v], err:[%v] ", pid, err) + return + } + if partition != nil { + stdoutln(formatDataPartitionInfoRow(partition)) + } + } + } else { + stdoutlnf("[Partition with missing tiny extent count]: %v", len(diagnosis.MissingTinyExtentDpIDs)) + } + stdoutln() if !showSimplified && (showAll || showDiskError) { stdoutln("[Partition with disk error replicas]:") @@ -404,6 +426,7 @@ The "reset" command will be released in next version`, cmd.Flags().BoolVarP(&showUnavailable, "showUnavailable", "u", false, "true for display dp with unavailable replicas") cmd.Flags().BoolVarP(&showBadDp, "showBadDp", "b", false, "true for display bad dp") cmd.Flags().BoolVarP(&showExcess, "showExcess", "e", false, "true for display dp with excess replicas") + cmd.Flags().BoolVarP(&showMissingTinyExtent, "showMissingTinyExtent", "m", false, "true for display dp with missing tiny extent") cmd.Flags().BoolVarP(&showDiskError, "showDiskError", "E", false, "true for display dp with disk error replicas") cmd.Flags().BoolVarP(&showDiscard, "showDiscard", "D", false, "true for display discard dp") cmd.Flags().BoolVarP(&showDiff, "showDiff", "d", false, "true for display dp those replica file count count or size differ significantly") diff --git a/datanode/data_partition_repair.go b/datanode/data_partition_repair.go index 9c97d1854..d27888806 100644 --- a/datanode/data_partition_repair.go +++ b/datanode/data_partition_repair.go @@ -151,6 +151,7 @@ func (dp *DataPartition) repair(extentType uint8) { // error check if dp.extentStore.AvailableTinyExtentCnt()+dp.extentStore.BrokenTinyExtentCnt() != storage.TinyExtentCount { + dp.isMissingTinyExtent = true log.LogWarnf("action[repair] partition(%v) GoodTinyExtents(%v) "+ "BadTinyExtents(%v) finish cost[%v] extentType %v.", dp.partitionID, dp.extentStore.AvailableTinyExtentCnt(), dp.extentStore.BrokenTinyExtentCnt(), time.Since(start).String(), extentType) diff --git a/datanode/partition.go b/datanode/partition.go index a6985f699..f520001ce 100644 --- a/datanode/partition.go +++ b/datanode/partition.go @@ -154,8 +154,9 @@ type DataPartition struct { responseStatus uint32 PersistApplyIdChan chan PersistApplyIdRequest - readOnlyReasons uint32 - isRepairing bool + readOnlyReasons uint32 + isMissingTinyExtent bool + isRepairing bool } type PersistApplyIdRequest struct { diff --git a/datanode/space_manager.go b/datanode/space_manager.go index 53cd829e4..2526bc8e8 100644 --- a/datanode/space_manager.go +++ b/datanode/space_manager.go @@ -24,8 +24,6 @@ import ( "sync/atomic" "time" - "github.com/cubefs/cubefs/datanode/storage" - syslog "log" "github.com/cubefs/cubefs/proto" @@ -766,7 +764,7 @@ func (s *DataNode) buildHeartBeatResponse(response *proto.DataNodeHeartbeatRespo TriggerDiskError: atomic.LoadUint64(&partition.diskErrCnt) > 0, ForbidWriteOpOfProtoVer0: dpForbid, ReadOnlyReasons: partition.ReadOnlyReasons(), - IsMissingTinyExtent: partition.extentStore.AvailableTinyExtentCnt()+partition.extentStore.BrokenTinyExtentCnt() < storage.TinyExtentCount, + IsMissingTinyExtent: partition.isMissingTinyExtent, IsRepairing: partition.isRepairing, } log.LogDebugf("action[Heartbeats] dpid(%v), status(%v) total(%v) used(%v) leader(%v) isLeader(%v) "+ diff --git a/master/api_service.go b/master/api_service.go index aaee7151f..bfb860ebf 100644 --- a/master/api_service.go +++ b/master/api_service.go @@ -2171,12 +2171,14 @@ func (m *Server) diagnoseDataPartition(w http.ResponseWriter, r *http.Request) { repFileCountDifferDps []*DataPartition repUsedSizeDifferDps []*DataPartition excessReplicaDPs []*DataPartition + missingTinyExtentDPs []*DataPartition corruptDpIDs []uint64 lackReplicaDpIDs []uint64 badReplicaDpIDs []uint64 repFileCountDifferDpIDs []uint64 repUsedSizeDifferDpIDs []uint64 excessReplicaDpIDs []uint64 + missingTinyExtentDpIDs []uint64 badDataPartitionInfos []proto.BadPartitionRepairView diskErrorDataPartitionInfos proto.DiskErrPartitionView start = time.Now() @@ -2198,6 +2200,7 @@ func (m *Server) diagnoseDataPartition(w http.ResponseWriter, r *http.Request) { repFileCountDifferDpIDs = make([]uint64, 0) repUsedSizeDifferDpIDs = make([]uint64, 0) excessReplicaDpIDs = make([]uint64, 0) + missingTinyExtentDpIDs = make([]uint64, 0) subStep := time.Now() if inactiveNodes, err = m.cluster.checkInactiveDataNodes(); err != nil { @@ -2206,7 +2209,7 @@ func (m *Server) diagnoseDataPartition(w http.ResponseWriter, r *http.Request) { } log.LogDebugf("diagnoseDataPartition checkInactiveDataNodes cost %v", time.Since(subStep).String()) if lackReplicaDps, badReplicaDps, repFileCountDifferDps, repUsedSizeDifferDps, excessReplicaDPs, - corruptDps, err = m.cluster.checkReplicaOfDataPartitions(ignoreDiscardDp); err != nil { + corruptDps, missingTinyExtentDPs, err = m.cluster.checkReplicaOfDataPartitions(ignoreDiscardDp); err != nil { sendErrReply(w, r, newErrHTTPReply(err)) return } @@ -2228,6 +2231,9 @@ func (m *Server) diagnoseDataPartition(w http.ResponseWriter, r *http.Request) { for _, dp := range excessReplicaDPs { excessReplicaDpIDs = append(excessReplicaDpIDs, dp.PartitionID) } + for _, dp := range missingTinyExtentDPs { + missingTinyExtentDpIDs = append(missingTinyExtentDpIDs, dp.PartitionID) + } // badDataPartitions = m.cluster.getBadDataPartitionsView() subStep = time.Now() @@ -2245,6 +2251,7 @@ func (m *Server) diagnoseDataPartition(w http.ResponseWriter, r *http.Request) { RepFileCountDifferDpIDs: repFileCountDifferDpIDs, RepUsedSizeDifferDpIDs: repUsedSizeDifferDpIDs, ExcessReplicaDpIDs: excessReplicaDpIDs, + MissingTinyExtentDpIDs: missingTinyExtentDpIDs, DiskErrorDataPartitionInfos: diskErrorDataPartitionInfos, } log.LogInfof("diagnose dataPartition[%v] inactiveNodes:[%v], corruptDpIDs:[%v], "+ diff --git a/master/cluster.go b/master/cluster.go index a2e4d450f..b51f332e9 100644 --- a/master/cluster.go +++ b/master/cluster.go @@ -1538,12 +1538,13 @@ func (c *Cluster) checkInactiveDataNodes() (inactiveDataNodes []string, err erro func (c *Cluster) checkReplicaOfDataPartitions(ignoreDiscardDp bool) ( lackReplicaDPs []*DataPartition, unavailableReplicaDPs []*DataPartition, repFileCountDifferDps []*DataPartition, - repUsedSizeDifferDps []*DataPartition, excessReplicaDPs []*DataPartition, noLeaderDPs []*DataPartition, err error, + repUsedSizeDifferDps []*DataPartition, excessReplicaDPs []*DataPartition, noLeaderDPs []*DataPartition, missingTinyExtentDPs []*DataPartition, err error, ) { noLeaderDPs = make([]*DataPartition, 0) lackReplicaDPs = make([]*DataPartition, 0) unavailableReplicaDPs = make([]*DataPartition, 0) excessReplicaDPs = make([]*DataPartition, 0) + missingTinyExtentDPs = make([]*DataPartition, 0) vols := c.copyVols() for _, vol := range vols { @@ -1583,7 +1584,13 @@ func (c *Cluster) checkReplicaOfDataPartitions(ignoreDiscardDp bool) ( } recordReplicaUnavailable := false + recordReplicaMissingTinyExtent := false for _, replica := range dp.Replicas { + if !recordReplicaMissingTinyExtent && dp.IsDecommissionRunning() && replica.IsMissingTinyExtent { + missingTinyExtentDPs = append(missingTinyExtentDPs, dp) + recordReplicaMissingTinyExtent = true + } + if !recordReplicaUnavailable && replica.Status == proto.Unavailable { unavailableReplicaDPs = append(unavailableReplicaDPs, dp) recordReplicaUnavailable = true diff --git a/master/monitor_metrics.go b/master/monitor_metrics.go index 675dddd4a..a10b7b6f2 100644 --- a/master/monitor_metrics.go +++ b/master/monitor_metrics.go @@ -54,7 +54,6 @@ const ( MetricFlashNodesDiskError = "flashNodes_disk_error" MetricDiskLost = "disk_lost" MetricDpUnableDecommissionCount = "dp_unable_decommission_count" - MetricDpMissingTinyExtent = "dp_missing_tinyExtent" MetricDpNoSamePeer = "dp_no_same_peer" MetricDataNodesInactive = "dataNodes_inactive" MetricInactiveDataNodeInfo = "inactive_dataNodes_info" @@ -129,7 +128,6 @@ type monitorMetrics struct { flashNodesDiskError *exporter.GaugeVec diskLost *exporter.GaugeVec dpUnableDecommissionCount *exporter.Gauge - dpMissingTinyExtent *exporter.GaugeVec dpNoSamePeer *exporter.GaugeVec dataNodesNotWritable *exporter.Gauge dataNodesAllocable *exporter.Gauge @@ -521,7 +519,6 @@ func (mm *monitorMetrics) start() { mm.flashNodesDiskError = exporter.NewGaugeVec(MetricFlashNodesDiskError, "", []string{"addr", "path"}) mm.diskLost = exporter.NewGaugeVec(MetricDiskLost, "", []string{"addr", "path"}) mm.dpUnableDecommissionCount = exporter.NewGauge(MetricDpUnableDecommissionCount) - mm.dpMissingTinyExtent = exporter.NewGaugeVec(MetricDpMissingTinyExtent, "", []string{"dpId", "addr"}) mm.dpNoSamePeer = exporter.NewGaugeVec(MetricDpNoSamePeer, "", []string{"dpId"}) mm.nodeStat = exporter.NewGaugeVec(MetricNodeStat, "", []string{"type", "addr", "stat"}) mm.dataNodesInactive = exporter.NewGauge(MetricDataNodesInactive) @@ -626,7 +623,6 @@ func (mm *monitorMetrics) doStat() { mm.setDiskLostMetric() mm.setFlashNodesDiskErrorMetric() mm.setDpUnableDecommissionMetric() - mm.setDpMissingTinyExtentMetric() mm.setDpNoSamePeerMetric() mm.setNotWritableDataNodesCount() mm.setNotWritableMetaNodesCount() @@ -949,23 +945,6 @@ func (mm *monitorMetrics) setDpUnableDecommissionMetric() { mm.dpUnableDecommissionCount.Set(float64(dpUnableDecommissionCount)) } -func (mm *monitorMetrics) setDpMissingTinyExtentMetric() { - mm.dpMissingTinyExtent.Reset() - - vols := mm.cluster.allVols() - for _, vol := range vols { - partitions := vol.dataPartitions.clonePartitions() - for _, dp := range partitions { - for _, replica := range dp.Replicas { - if replica.IsMissingTinyExtent { - idStr := strconv.FormatUint(dp.PartitionID, 10) - mm.dpMissingTinyExtent.SetWithLabelValues(1, idStr, replica.Addr) - } - } - } - } -} - func (mm *monitorMetrics) setDpNoSamePeerMetric() { mm.dpNoSamePeer.Reset() @@ -1360,7 +1339,6 @@ func (mm *monitorMetrics) resetAllLeaderMetrics() { // mm.diskError.Set(0) mm.diskLost.Reset() mm.dpUnableDecommissionCount.Set(0) - mm.dpMissingTinyExtent.Reset() mm.dpNoSamePeer.Reset() mm.diskDecommissionSuccess.Reset() mm.dataNodesInactive.Set(0) diff --git a/proto/model.go b/proto/model.go index 178b0e237..49b9d1ee9 100644 --- a/proto/model.go +++ b/proto/model.go @@ -394,6 +394,7 @@ type DataPartitionDiagnosis struct { RepFileCountDifferDpIDs []uint64 RepUsedSizeDifferDpIDs []uint64 ExcessReplicaDpIDs []uint64 + MissingTinyExtentDpIDs []uint64 // BadDataPartitionIDs []BadPartitionView BadDataPartitionInfos []BadPartitionRepairView BadReplicaDataPartitionIDs []uint64