gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 02:00:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix(master): deadlock for datanode and nodeset

Browse Source 
Signed-off-by: chihe <chihe@oppo.com>
This commit is contained in:
chihe 2024-09-15 21:24:41 +08:00 committed by AmazingChi
parent 8a76004126
commit 65ce9f8b7c
3 changed files with 16 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
master/cluster.go
View File

@ -2916,10 +2916,14 @@ func (c *Cluster) putBadDataPartitionIDs(replica *DataReplica, addr string, part
func (c *Cluster) clearBadDataPartitionIDS() {
c.badPartitionMutex.Lock()
defer c.badPartitionMutex.Unlock()
keysToDelete := make([]string, 0)
c.BadDataPartitionIds.Range(func(key, value interface{}) bool {
c.BadDataPartitionIds.Delete(key)
keysToDelete = append(keysToDelete, key.(string))
return true
})
for _, key := range keysToDelete {
c.BadDataPartitionIds.Delete(key)
}
}
func (c *Cluster) putBadDataPartitionIDsByDiskPath(disk, addr string, partitionID uint64) {
@ -4391,10 +4395,14 @@ func (c *Cluster) canAutoDecommissionDisk(addr string, diskPath string) (yes boo
}
func (c *Cluster) handleDataNodeBadDisk(dataNode *DataNode) {
badDisks := make([]proto.BadDiskStat, 0)
dataNode.RLock()
defer dataNode.RUnlock()
for _, disk := range dataNode.BadDiskStats {
badDisks = append(badDisks, disk)
}
dataNode.RUnlock()
for _, disk := range badDisks {
// TODO:no dp left on bad disk, notify sre to remove this disk
// decommission failed, but lack replica for disk err dp is already removed
retry := c.RetryDecommissionDisk(dataNode.Addr, disk.DiskPath)

2
master/cluster_test.go
View File

@ -2,6 +2,7 @@ package master
import (
"fmt"
"sync"
"testing"
"time"
@ -364,6 +365,7 @@ func TestMasterClientLeaderChange(t *testing.T) {
masterClient: masterSDK.NewMasterClient(nil, false),
}
cluster.t = newTopology()
cluster.BadDataPartitionIds = new(sync.Map)
server := &Server{
cluster: cluster,
leaderInfo: &LeaderInfo{

4
master/topology.go
View File

@ -1174,7 +1174,7 @@ func (ns *nodeSet) traverseDecommissionDisk(c *Cluster) {
for {
select {
case <-ns.doneDecommissionDiskListTraverse:
log.LogWarnf("ns %v(%p) traverse stopped", ns.ID)
log.LogWarnf("ns %v(%p) traverse stopped", ns.ID, ns)
ns.ClearDecommissionDisks()
return
case <-t.C:
@ -1247,9 +1247,11 @@ func (ns *nodeSet) traverseDecommissionDisk(c *Cluster) {
}
}
}
func (ns *nodeSet) stopDecommissionDiskSchedule() {
ns.doneDecommissionDiskListTraverse <- struct{}{}
}
func (t *topology) isSingleZone() bool {
t.zoneLock.RLock()
defer t.zoneLock.RUnlock()