refactor(master): limit the interval between deleteing mp replica to 5 minutes. #1000230481

Signed-off-by: Victor1319 <zengxuewei@oppo.com>

master/api_service.go

@@ -6052,6 +6052,7 @@ func getMetaPartitionView(mp *MetaPartition) (mpView *proto.MetaPartitionView) {
 	mpView.TxRbDenCnt = mp.TxRbDenCnt
 	mpView.IsRecover = mp.IsRecover
 	mpView.Freeze = mp.Freeze
+	mpView.LastDelReplicaTime = mp.LastDelReplicaTime
 	return
 }
 

master/cluster_balance.go

@@ -1142,6 +1142,11 @@ func (c *Cluster) DoMetaPartitionBalanceTask(plan *proto.ClusterPlan) {
 				return
 			}
 
+			if !mp.CheckLastDelReplicaTime() {
+				log.LogWarnf("DoMetaPartitionBalanceTask: mp try wait, last %d, mp %d", mp.LastDelReplicaTime, mp.PartitionID)
+				time.Sleep(time.Second * (mpReplicaDelInterval + 10))
+			}
+
 			log.LogDebugf("Start to migrate meta partition(%d) from %s to %s", mpPlan.ID, mrPlan.Source, mrPlan.Destination)
 			err = c.migrateMetaPartition(mrPlan.Source, mrPlan.Destination, mp)
 			if err != nil {

master/cluster_task.go

@@ -413,6 +413,11 @@ func (c *Cluster) deleteMetaReplica(partition *MetaPartition, addr string, valid
 		}
 	}()
 
+	if !partition.CheckLastDelReplicaTime() {
+		err = fmt.Errorf("deleteMetaReplica: the interval between deleting or decommission mp replica should over 5 minute. last %d", partition.LastDelReplicaTime)
+		return
+	}
+
 	if validate {
 		if err = c.validateDecommissionMetaPartition(partition, addr, forceDel); err != nil {
 			return
@@ -424,6 +429,7 @@ func (c *Cluster) deleteMetaReplica(partition *MetaPartition, addr string, valid
 		return
 	}
 
+	partition.LastDelReplicaTime = time.Now().Unix()
 	removePeer := proto.Peer{ID: metaNode.ID, Addr: addr, HeartbeatPort: metaNode.HeartbeatPort, ReplicaPort: metaNode.ReplicaPort}
 	if err = c.removeMetaPartitionRaftMember(partition, removePeer); err != nil {
 		return

master/const.go

@@ -255,7 +255,8 @@ const (
 	diskDecommissionInfoStatType                  = 1
 	dataNodeDecommissionInfoStatType              = 2
 
-	maxTrashInterval = 365 * 24 * 60
+	maxTrashInterval     = 365 * 24 * 60
+	mpReplicaDelInterval = 300 // 5 minutes
 )
 
 const (

master/meta_partition.go

@@ -84,6 +84,8 @@ type MetaPartition struct {
 	StatByStorageClass        []*proto.StatOfStorageClass
 	StatByMigrateStorageClass []*proto.StatOfStorageClass
 	sync.RWMutex
+
+	LastDelReplicaTime int64
 }
 
 func newMetaReplica(start, end uint64, metaNode *MetaNode) (mr *MetaReplica) {
@@ -111,6 +113,14 @@ func newMetaPartition(partitionID, start, end uint64, replicaNum uint8, volName
 	return
 }
 
+func (mp *MetaPartition) CheckLastDelReplicaTime() bool {
+	return mp.GetLastDelTime()+mpReplicaDelInterval < time.Now().Unix()
+}
+
+func (mp *MetaPartition) GetLastDelTime() int64 {
+	return mp.LastDelReplicaTime
+}
+
 func (mp *MetaPartition) setPeers(peers []proto.Peer) {
 	mp.Peers = peers
 }

master/metadata_fsm_op.go

@@ -138,34 +138,36 @@ func newClusterValue(c *Cluster) (cv *clusterValue) {
 }
 
 type metaPartitionValue struct {
-	PartitionID   uint64
-	Start         uint64
-	End           uint64
-	VolID         uint64
-	ReplicaNum    uint8
-	Status        int8
-	VolName       string
-	Hosts         string
-	OfflinePeerID uint64
-	Peers         []proto.Peer
-	IsRecover     bool
-	Freeze        int8
+	PartitionID        uint64
+	Start              uint64
+	End                uint64
+	VolID              uint64
+	ReplicaNum         uint8
+	Status             int8
+	VolName            string
+	Hosts              string
+	OfflinePeerID      uint64
+	Peers              []proto.Peer
+	IsRecover          bool
+	Freeze             int8
+	LastDelReplicaTime int64
 }
 
 func newMetaPartitionValue(mp *MetaPartition) (mpv *metaPartitionValue) {
 	mpv = &metaPartitionValue{
-		PartitionID:   mp.PartitionID,
-		Start:         mp.Start,
-		End:           mp.End,
-		VolID:         mp.volID,
-		ReplicaNum:    mp.ReplicaNum,
-		Status:        mp.Status,
-		VolName:       mp.volName,
-		Hosts:         mp.hostsToString(),
-		Peers:         mp.Peers,
-		OfflinePeerID: mp.OfflinePeerID,
-		IsRecover:     mp.IsRecover,
-		Freeze:        mp.Freeze,
+		PartitionID:        mp.PartitionID,
+		Start:              mp.Start,
+		End:                mp.End,
+		VolID:              mp.volID,
+		ReplicaNum:         mp.ReplicaNum,
+		Status:             mp.Status,
+		VolName:            mp.volName,
+		Hosts:              mp.hostsToString(),
+		Peers:              mp.Peers,
+		OfflinePeerID:      mp.OfflinePeerID,
+		IsRecover:          mp.IsRecover,
+		Freeze:             mp.Freeze,
+		LastDelReplicaTime: mp.LastDelReplicaTime,
 	}
 	return
 }
@@ -1846,6 +1848,7 @@ func (c *Cluster) loadMetaPartitions() (err error) {
 		mp.OfflinePeerID = mpv.OfflinePeerID
 		mp.IsRecover = mpv.IsRecover
 		mp.Freeze = mpv.Freeze
+		mp.LastDelReplicaTime = mpv.LastDelReplicaTime
 		vol.addMetaPartition(mp)
 		c.addBadMetaParitionIdMap(mp)
 		log.LogInfof("action[loadMetaPartitions],vol[%v],mp[%v]", vol.Name, mp.PartitionID)

proto/admin_proto.go

@@ -1150,21 +1150,22 @@ func NewDataPartitionsView() (dataPartitionsView *DataPartitionsView) {
 
 // MetaPartitionView defines the view of a meta partition
 type MetaPartitionView struct {
-	PartitionID uint64
-	Start       uint64
-	End         uint64
-	MaxInodeID  uint64
-	InodeCount  uint64
-	DentryCount uint64
-	FreeListLen uint64
-	TxCnt       uint64
-	TxRbInoCnt  uint64
-	TxRbDenCnt  uint64
-	IsRecover   bool
-	Members     []string
-	LeaderAddr  string
-	Status      int8
-	Freeze      int8
+	PartitionID        uint64
+	Start              uint64
+	End                uint64
+	MaxInodeID         uint64
+	InodeCount         uint64
+	DentryCount        uint64
+	FreeListLen        uint64
+	TxCnt              uint64
+	TxRbInoCnt         uint64
+	TxRbDenCnt         uint64
+	IsRecover          bool
+	Members            []string
+	LeaderAddr         string
+	Status             int8
+	Freeze             int8
+	LastDelReplicaTime int64
 }
 
 type DataNodeDisksRequest struct{}