gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 02:00:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix(master): return timeout error for setting restore replica status

Browse Source 
Signed-off-by: chihe <chihe@oppo.com>
This commit is contained in:
chihe 2024-07-22 19:14:15 +08:00 committed by AmazingChi
parent a348b6acb3
commit 78c6f40829
2 changed files with 24 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
master/data_partition.go
View File

@ -968,9 +968,11 @@ const (
const InvalidDecommissionDpCnt = -1
const (
defaultDecommissionParallelLimit = 10
defaultDecommissionRetryLimit = 5
defaultDecommissionRollbackLimit = 3
defaultDecommissionParallelLimit = 10
defaultDecommissionRetryLimit = 5
defaultDecommissionRollbackLimit = 3
defaultDecommissionDiskParallelFactor = 0
defaultSetRestoreReplicaStatusLimit = 300
)
func GetDecommissionStatusMessage(status uint32) string {
@ -1138,6 +1140,7 @@ func (partition *DataPartition) MarkDecommissionStatus(srcAddr, dstAddr, srcDisk
}
}
directly:
waitTimes := 0
if partition.IsDecommissionPaused() {
if !partition.pauseReplicaRepair(partition.DecommissionDstAddr, false, c) {
log.LogWarnf("action[MarkDecommissionStatus] dp [%d] recover from stop failed", partition.PartitionID)
@ -1146,12 +1149,13 @@ directly:
// forbidden dp to restore meta for replica
// if migrateType is AutoAddReplica, RestoreReplica is already RestoreReplicaMetaRunning, so do not need to check
// this flag again
for {
if !partition.setRestoreReplicaForbidden() && migrateType != AutoAddReplica {
// partition.DecommissionRetry++
// if partition.DecommissionRetry >= defaultDecommissionRetryLimit {
// return errors.NewErrorf("set RestoreReplicaMetaForbidden failed")
// }
waitTimes++
if waitTimes >= defaultSetRestoreReplicaStatusLimit {
return errors.NewErrorf("set RestoreReplicaMetaForbidden timeout")
}
// wait for checkReplicaMeta ended
log.LogWarnf("action[MarkDecommissionStatus] dp [%d]wait for setting restore replica forbidden",
partition.PartitionID)
@ -1171,6 +1175,10 @@ directly:
// forbidden dp to restore meta for replica
for {
if !partition.setRestoreReplicaForbidden() && migrateType != AutoAddReplica {
waitTimes++
if waitTimes >= defaultSetRestoreReplicaStatusLimit {
return errors.NewErrorf("set RestoreReplicaMetaForbidden timeout")
}
log.LogWarnf("action[MarkDecommissionStatus] dp [%d]wait for setting restore replica forbidden",
partition.PartitionID)
time.Sleep(1 * time.Second)
@ -2317,11 +2325,15 @@ func (partition *DataPartition) tryRecoverReplicaMeta(c *Cluster, migrateType ui
}
return nil
}
waitTimes := 0
for {
err := partition.checkReplicaMeta(c)
if err != nil {
if err == proto.ErrPerformingRestoreReplica {
waitTimes++
if waitTimes > defaultSetRestoreReplicaStatusLimit {
return errors.NewErrorf("set restore replica status timeout:5min")
}
log.LogDebugf("action[tryRecoverReplicaMeta]dp(%v) wait for checking replica",
partition.PartitionID)
time.Sleep(time.Second)

8
master/topology.go
View File

@ -1145,18 +1145,18 @@ func (ns *nodeSet) removeAutoDecommissionDisk(dd *DecommissionDisk) {
func (ns *nodeSet) traverseDecommissionDisk(c *Cluster) {
t := time.NewTicker(DecommissionInterval)
// wait for loading all decommissionDisk when reload metadata
log.LogInfof("action[traverseDecommissionDisk]wait %v", ns.ID)
log.LogInfof("action[traverseDecommissionDisk]wait ns %v", ns.ID)
<-ns.startDecommissionDiskListTraverse
log.LogInfof("action[traverseDecommissionDisk] traverseDecommissionDisk start %v", ns.ID)
log.LogInfof("action[traverseDecommissionDisk] traverseDecommissionDisk start ns %v", ns.ID)
defer t.Stop()
for {
select {
case <-ns.doneDecommissionDiskListTraverse:
log.LogWarnf("traverse stopped")
log.LogWarnf("ns %v traverse stopped", ns.ID)
return
case <-t.C:
if c.partition != nil && !c.partition.IsRaftLeader() {
log.LogWarnf("Leader changed, stop traverse!")
log.LogWarnf("ns %v Leader changed, stop traverse!", ns.ID)
ns.DecommissionDisks.Range(func(key, value interface{}) bool {
ns.RemoveDecommissionDisk(value.(*DecommissionDisk))
return true