gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 02:00:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix(shardnode): asynchronous execute handleRaftError to avoid deadlock

Browse Source 
with #23025534 #23031244

Signed-off-by: xiejian <xiejian3@oppo.com>
This commit is contained in:
xiejian 2025-02-11 15:07:54 +08:00 committed by slasher
parent 717792ff45
commit ad2ace90d8
2 changed files with 34 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
blobstore/shardnode/shard.go
View File

@ -20,10 +20,9 @@ import (
"math/rand"
"time"
"github.com/cubefs/cubefs/blobstore/common/codemode"
"github.com/cubefs/cubefs/blobstore/api/clustermgr"
"github.com/cubefs/cubefs/blobstore/api/shardnode"
"github.com/cubefs/cubefs/blobstore/common/codemode"
"github.com/cubefs/cubefs/blobstore/common/proto"
"github.com/cubefs/cubefs/blobstore/common/trace"
"github.com/cubefs/cubefs/blobstore/shardnode/storage"

64
blobstore/shardnode/storage/disk.go
View File

@ -626,40 +626,42 @@ func (d *Disk) handleRaftError(groupID uint64, err error) {
span.Fatalf("unexpect raftgroup error: %s", err.Error())
}
shards := make([]*shard, 0)
shardList := list.New()
go func() {
shards := make([]*shard, 0)
shardList := list.New()
d.shardsMu.RLock()
for i := range d.shardsMu.shards {
shards = append(shards, d.shardsMu.shards[i])
shardList.PushBack(d.shardsMu.shards[i])
}
d.shardsMu.RUnlock()
start := time.Now()
for shardList.Len() > 0 {
e := shardList.Front()
s := e.Value.(*shard)
shardList.Remove(e)
if _err := s.Stop(); _err != nil {
span.Errorf("stop shard:%d failed: %s", s.suid.ShardID(), _err.Error())
shardList.PushBack(s)
continue
d.shardsMu.RLock()
for i := range d.shardsMu.shards {
shards = append(shards, d.shardsMu.shards[i])
shardList.PushBack(d.shardsMu.shards[i])
}
}
span.Infof("all shards stop success")
d.shardsMu.RUnlock()
for i := range shards {
shards[i].WaitStop()
}
span.Infof("all shards wait stop success, num:%d, cost: %d us", len(shards), time.Since(start).Microseconds())
for i := range shards {
gid := uint64(shards[i].GetSuid().ShardID())
if _err := d.raftManager.RemoveRaftGroup(ctx, gid, false); _err != nil {
span.Fatalf("remove raft group:%d failed: %s", gid, _err.Error())
start := time.Now()
for shardList.Len() > 0 {
e := shardList.Front()
s := e.Value.(*shard)
shardList.Remove(e)
if _err := s.Stop(); _err != nil {
span.Errorf("stop shard:%d failed: %s", s.suid.ShardID(), _err.Error())
shardList.PushBack(s)
continue
}
}
}
span.Infof("all shards stop success")
span.Infof("remove all shards raft group success, num:%d", len(shards))
for i := range shards {
shards[i].WaitStop()
}
span.Infof("all shards wait stop success, num:%d, cost: %d us", len(shards), time.Since(start).Microseconds())
for i := range shards {
gid := uint64(shards[i].GetSuid().ShardID())
if _err := d.raftManager.RemoveRaftGroup(ctx, gid, false); _err != nil {
span.Fatalf("remove raft group:%d failed: %s", gid, _err.Error())
}
}
span.Infof("remove all shards raft group success, num:%d", len(shards))
}()
}