diff --git a/master/cluster.go b/master/cluster.go index faf16936a..0ca5d7672 100644 --- a/master/cluster.go +++ b/master/cluster.go @@ -3359,6 +3359,10 @@ func (c *Cluster) removeDataReplica(dp *DataPartition, addr string, validate boo } } + if err = dp.isLastReplicas(addr); err != nil { + return err + } + dataNode, err := c.dataNode(addr) if err != nil { return diff --git a/master/data_partition.go b/master/data_partition.go index 870c57e64..1aef3680d 100644 --- a/master/data_partition.go +++ b/master/data_partition.go @@ -262,8 +262,14 @@ func (partition *DataPartition) createTaskToRemoveRaftMember(c *Cluster, removeP if replica.Addr != removePeer.Addr { leaderAddr = replica.Addr } - doWork(leaderAddr, autoRemove) + err = doWork(leaderAddr, autoRemove) + if err != nil { + continue + } + return nil } + + return fmt.Errorf("createTaskToRemoveRaftMember: force delte raft memeber failed, dp %d, err %v", partition.PartitionID, err) } else { err = proto.ErrNoLeader return @@ -271,7 +277,6 @@ func (partition *DataPartition) createTaskToRemoveRaftMember(c *Cluster, removeP } else { return doWork(leaderAddr, autoRemove) } - return } func (partition *DataPartition) createTaskToCreateDataPartition(addr string, dataPartitionSize uint64, @@ -309,6 +314,28 @@ func (partition *DataPartition) resetTaskID(t *proto.AdminTask) { t.PartitionID = partition.PartitionID } +func (partition *DataPartition) isLastReplicas(host string) error { + if len(partition.Replicas) == 1 && partition.Replicas[0].Addr == host { + return fmt.Errorf("partition %v has only one replica %v", partition.PartitionID, host) + } + if len(partition.Hosts) == 1 && partition.Hosts[0] == host { + return fmt.Errorf("partition %v has only one host %v", partition.PartitionID, host) + } + + normalHosts := make([]string, 0, len(partition.Replicas)) + for _, r := range partition.Replicas { + if r.isNormal(partition.PartitionID, defaultDataPartitionTimeOutSec) { + normalHosts = append(normalHosts, r.Addr) + } + } + + if len(normalHosts) == 1 && normalHosts[0] == host { + return fmt.Errorf("partition %v only has one normal host, %v", partition.PartitionID, host) + } + + return nil +} + // Check if there is a replica missing or not. func (partition *DataPartition) hasMissingOneReplica(addr string, replicaNum int) (err error) { hostNum := len(partition.Replicas) diff --git a/master/data_replica.go b/master/data_replica.go index f1195a49a..0b239295c 100644 --- a/master/data_replica.go +++ b/master/data_replica.go @@ -19,6 +19,7 @@ import ( "github.com/cubefs/cubefs/proto" "github.com/cubefs/cubefs/util/log" + "github.com/cubefs/cubefs/util/timeutil" ) // DataReplica represents the replica of a data partition @@ -47,6 +48,17 @@ func (replica *DataReplica) isMissing(interval int64) (isMissing bool) { return } +func (replica *DataReplica) isNormal(id uint64, ReportTime int64) (isNormal bool) { + if replica.dataNode.isActive && + (replica.Status == proto.ReadWrite || replica.Status == proto.ReadOnly) && + timeutil.GetCurrentTimeUnix()-replica.ReportTime <= ReportTime { + return true + } + log.LogDebugf("action[isLive] partition %v, replica addr %v, datanode active %v replica status %v and is active %v", + id, replica.Addr, replica.dataNode.isActive, replica.Status, replica.ReportTime) + return false +} + func (replica *DataReplica) isLive(id uint64, timeOutSec int64) (isAvailable bool) { if replica.dataNode.isActive && replica.Status != proto.Unavailable && replica.isActive(timeOutSec) {