fix(master): limit the last dp replica can't be deleted. #1000182796

Signed-off-by: Victor1319 <zengxuewei@oppo.com>

master/cluster.go

@@ -3359,6 +3359,10 @@ func (c *Cluster) removeDataReplica(dp *DataPartition, addr string, validate boo
 		}
 	}
 
+	if err = dp.isLastReplicas(addr); err != nil {
+		return err
+	}
+
 	dataNode, err := c.dataNode(addr)
 	if err != nil {
 		return

master/data_partition.go

@@ -262,8 +262,14 @@ func (partition *DataPartition) createTaskToRemoveRaftMember(c *Cluster, removeP
 				if replica.Addr != removePeer.Addr {
 					leaderAddr = replica.Addr
 				}
-				doWork(leaderAddr, autoRemove)
+				err = doWork(leaderAddr, autoRemove)
+				if err != nil {
+					continue
+				}
+				return nil
 			}
+
+			return fmt.Errorf("createTaskToRemoveRaftMember: force delte raft memeber failed, dp %d, err %v", partition.PartitionID, err)
 		} else {
 			err = proto.ErrNoLeader
 			return
@@ -271,7 +277,6 @@ func (partition *DataPartition) createTaskToRemoveRaftMember(c *Cluster, removeP
 	} else {
 		return doWork(leaderAddr, autoRemove)
 	}
-	return
 }
 
 func (partition *DataPartition) createTaskToCreateDataPartition(addr string, dataPartitionSize uint64,
@@ -309,6 +314,28 @@ func (partition *DataPartition) resetTaskID(t *proto.AdminTask) {
 	t.PartitionID = partition.PartitionID
 }
 
+func (partition *DataPartition) isLastReplicas(host string) error {
+	if len(partition.Replicas) == 1 && partition.Replicas[0].Addr == host {
+		return fmt.Errorf("partition %v has only one replica %v", partition.PartitionID, host)
+	}
+	if len(partition.Hosts) == 1 && partition.Hosts[0] == host {
+		return fmt.Errorf("partition %v has only one host %v", partition.PartitionID, host)
+	}
+
+	normalHosts := make([]string, 0, len(partition.Replicas))
+	for _, r := range partition.Replicas {
+		if r.isNormal(partition.PartitionID, defaultDataPartitionTimeOutSec) {
+			normalHosts = append(normalHosts, r.Addr)
+		}
+	}
+
+	if len(normalHosts) == 1 && normalHosts[0] == host {
+		return fmt.Errorf("partition %v only has one normal host, %v", partition.PartitionID, host)
+	}
+
+	return nil
+}
+
 // Check if there is a replica missing or not.
 func (partition *DataPartition) hasMissingOneReplica(addr string, replicaNum int) (err error) {
 	hostNum := len(partition.Replicas)

master/data_replica.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/cubefs/cubefs/proto"
 	"github.com/cubefs/cubefs/util/log"
+	"github.com/cubefs/cubefs/util/timeutil"
 )
 
 // DataReplica represents the replica of a data partition
@@ -47,6 +48,17 @@ func (replica *DataReplica) isMissing(interval int64) (isMissing bool) {
 	return
 }
 
+func (replica *DataReplica) isNormal(id uint64, ReportTime int64) (isNormal bool) {
+	if replica.dataNode.isActive &&
+		(replica.Status == proto.ReadWrite || replica.Status == proto.ReadOnly) &&
+		timeutil.GetCurrentTimeUnix()-replica.ReportTime <= ReportTime {
+		return true
+	}
+	log.LogDebugf("action[isLive] partition %v, replica addr %v, datanode active %v replica status %v and is active %v",
+		id, replica.Addr, replica.dataNode.isActive, replica.Status, replica.ReportTime)
+	return false
+}
+
 func (replica *DataReplica) isLive(id uint64, timeOutSec int64) (isAvailable bool) {
 	if replica.dataNode.isActive && replica.Status != proto.Unavailable &&
 		replica.isActive(timeOutSec) {