build(scheduler): add blobstore module scheduler

Signed-off-by: pengtianyue025 <pengtianyue025@gmail.com>

blobstore/api/proxy/allocator.go

@@ -0,0 +1,50 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package proxy
+
+import (
+	"context"
+
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+type Allocator interface {
+	VolumeAlloc(ctx context.Context, host string, args *AllocVolsArgs) (ret []AllocRet, err error)
+}
+
+type ListVolsArgs struct {
+	CodeMode codemode.CodeMode `json:"code_mode"`
+}
+
+type VolumeList struct {
+	Vids    []proto.Vid                  `json:"vids"`
+	Volumes []clustermgr.AllocVolumeInfo `json:"volumes"`
+}
+
+type AllocRet struct {
+	BidStart proto.BlobID `json:"bid_start"`
+	BidEnd   proto.BlobID `json:"bid_end"`
+	Vid      proto.Vid    `json:"vid"`
+}
+
+type AllocVolsArgs struct {
+	Fsize    uint64            `json:"fsize"`
+	CodeMode codemode.CodeMode `json:"code_mode"`
+	BidCount uint64            `json:"bid_count"`
+	Excludes []proto.Vid       `json:"excludes"`
+	Discards []proto.Vid       `json:"discards"`
+}

blobstore/api/proxy/client.go

@@ -0,0 +1,52 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package proxy
+
+import (
+	"context"
+
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+)
+
+type Config struct {
+	rpc.Config
+}
+
+type client struct {
+	rpc.Client
+}
+
+type Client interface {
+	MsgSender
+	Allocator
+}
+
+func New(cfg *Config) Client {
+	return &client{rpc.NewClient(&cfg.Config)}
+}
+
+func (c *client) VolumeAlloc(ctx context.Context, host string, args *AllocVolsArgs) (ret []AllocRet, err error) {
+	ret = make([]AllocRet, 0)
+	err = c.PostWith(ctx, host+"/volume/alloc", &ret, args)
+	return
+}
+
+func (c *client) SendShardRepairMsg(ctx context.Context, host string, args *ShardRepairArgs) error {
+	return c.PostWith(ctx, host+"/repairmsg", nil, args)
+}
+
+func (c *client) SendDeleteMsg(ctx context.Context, host string, args *DeleteArgs) error {
+	return c.PostWith(ctx, host+"/deletemsg", nil, args)
+}

blobstore/api/proxy/client_test.go

@@ -0,0 +1,154 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package proxy
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+)
+
+func TestClient_VolumeAlloc(t *testing.T) {
+	cli := New(&Config{})
+	mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer mockServer.Close()
+	args := &AllocVolsArgs{Fsize: 8, CodeMode: 2, BidCount: 1}
+	ret, err := cli.VolumeAlloc(context.Background(), mockServer.URL, args)
+	require.NoError(t, err)
+	require.Equal(t, make([]AllocRet, 0), ret)
+}
+
+func TestLbClient_SendShardRepairMsg(t *testing.T) {
+	mqproxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		w.WriteHeader(200)
+	}))
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		w.WriteHeader(200)
+		w.Write([]byte(fmt.Sprintf(`{"nodes":[{"cluster_id":1,"name":"PROXY","host":"%s","idc":"z0"}]}`, mqproxyServer.URL)))
+	}))
+	defer func() {
+		s.Close()
+		mqproxyServer.Close()
+	}()
+
+	cmCfg := clustermgr.Config{LbConfig: rpc.LbConfig{
+		Hosts: []string{s.URL},
+	}}
+	cm := clustermgr.New(&cmCfg)
+	cli := NewMQLbClient(&LbConfig{
+		Config:             Config{},
+		RetryHostsCnt:      0,
+		HostSyncIntervalMs: 0,
+	}, cm, 1)
+
+	err := cli.SendShardRepairMsg(context.Background(), &ShardRepairArgs{
+		ClusterID: 0,
+		Bid:       0,
+		Vid:       0,
+		BadIdxes:  nil,
+		Reason:    "test",
+	})
+	require.NoError(t, err)
+}
+
+func TestLbClient_SendShardRepairMsg_failed(t *testing.T) {
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		w.WriteHeader(200)
+		w.Write([]byte(fmt.Sprintf(`{"nodes":[{"cluster_id":1,"name":"PROXY","host":"%s","idc":"z0"}]}`, "abc.com")))
+	}))
+	defer s.Close()
+
+	cmCfg := clustermgr.Config{LbConfig: rpc.LbConfig{
+		Hosts: []string{s.URL},
+	}}
+	cm := clustermgr.New(&cmCfg)
+	cli := NewMQLbClient(&LbConfig{
+		Config:             Config{},
+		RetryHostsCnt:      0,
+		HostSyncIntervalMs: 0,
+	}, cm, 1)
+
+	err := cli.SendShardRepairMsg(context.Background(), &ShardRepairArgs{
+		ClusterID: 0,
+		Bid:       0,
+		Vid:       0,
+		BadIdxes:  nil,
+		Reason:    "test",
+	})
+	require.Error(t, err)
+}
+
+func TestLbClient_BlobDelete_failed(t *testing.T) {
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		w.WriteHeader(200)
+		w.Write([]byte(fmt.Sprintf(`{"nodes":[{"cluster_id":1,"name":"PROXY","host":"%s","idc":"z0"}]}`, "abc.com")))
+	}))
+	defer s.Close()
+
+	cmCfg := clustermgr.Config{LbConfig: rpc.LbConfig{
+		Hosts: []string{s.URL},
+	}}
+	cm := clustermgr.New(&cmCfg)
+	cli := NewMQLbClient(&LbConfig{
+		Config:             Config{},
+		RetryHostsCnt:      0,
+		HostSyncIntervalMs: 0,
+	}, cm, 1)
+
+	err := cli.SendDeleteMsg(context.Background(), &DeleteArgs{
+		ClusterID: 0,
+		Blobs:     nil,
+	})
+	require.Error(t, err)
+}
+
+func TestLbClient_BlobDelete(t *testing.T) {
+	mqproxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		w.WriteHeader(200)
+	}))
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		w.WriteHeader(200)
+		w.Write([]byte(fmt.Sprintf(`{"nodes":[{"cluster_id":1,"name":"PROXY","host":"%s","idc":"z0"}]}`, mqproxyServer.URL)))
+	}))
+	defer func() {
+		s.Close()
+		mqproxyServer.Close()
+	}()
+
+	cmCfg := clustermgr.Config{LbConfig: rpc.LbConfig{
+		Hosts: []string{s.URL},
+	}}
+	cm := clustermgr.New(&cmCfg)
+	cli := NewMQLbClient(&LbConfig{
+		Config:             Config{},
+		RetryHostsCnt:      0,
+		HostSyncIntervalMs: 0,
+	}, cm, 1)
+
+	err := cli.SendDeleteMsg(context.Background(), &DeleteArgs{
+		ClusterID: 0,
+		Blobs:     nil,
+	})
+	require.NoError(t, err)
+}

blobstore/api/proxy/lb_client.go

@@ -0,0 +1,120 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package proxy
+
+import (
+	"context"
+
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/util/errors"
+	"github.com/cubefs/cubefs/blobstore/util/selector"
+)
+
+var errNoServiceAvailable = errors.New("no service available")
+
+type LbConfig struct {
+	Config
+
+	RetryHostsCnt      int   `json:"retry_hosts_cnt"`
+	HostSyncIntervalMs int64 `json:"host_sync_interval_ms"`
+}
+
+type lbClient struct {
+	Client
+	selector      selector.Selector
+	retryHostsCnt int
+}
+
+func NewMQLbClient(cfg *LbConfig, service clustermgr.APIService, clusterID proto.ClusterID) LbMsgSender {
+	hostGetter := func() ([]string, error) {
+		svrInfos, err := service.GetService(context.Background(), clustermgr.GetServiceArgs{Name: proto.ServiceNameProxy})
+		if err != nil {
+			return nil, err
+		}
+
+		var hosts []string
+		for _, s := range svrInfos.Nodes {
+			if clusterID == proto.ClusterID(s.ClusterID) {
+				hosts = append(hosts, s.Host)
+			}
+		}
+		if len(hosts) == 0 {
+			return nil, errNoServiceAvailable
+		}
+
+		return hosts, nil
+	}
+
+	if cfg.HostSyncIntervalMs == 0 {
+		cfg.HostSyncIntervalMs = 1000
+	}
+	if cfg.RetryHostsCnt == 0 {
+		cfg.RetryHostsCnt = 1
+	}
+
+	return &lbClient{
+		retryHostsCnt: cfg.RetryHostsCnt,
+		Client:        New(&cfg.Config),
+		selector:      selector.NewSelectorWithGetter(cfg.HostSyncIntervalMs, hostGetter),
+	}
+}
+
+func (c *lbClient) SendDeleteMsg(ctx context.Context, args *DeleteArgs) (err error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	hosts := c.selector.GetRandomN(c.retryHostsCnt)
+	if len(hosts) == 0 {
+		return errNoServiceAvailable
+	}
+	for _, h := range hosts {
+		err = c.Client.SendDeleteMsg(ctx, h, args)
+		if err == nil || !shouldRetry(err) {
+			return err
+		}
+		span.Errorf("send delete message failed, host: %s, args: %+v, err:%+v", h, args, err)
+	}
+
+	return err
+}
+
+func (c *lbClient) SendShardRepairMsg(ctx context.Context, args *ShardRepairArgs) (err error) {
+	span := trace.SpanFromContextSafe(ctx)
+	ctx = trace.ContextWithSpan(ctx, span)
+
+	hosts := c.selector.GetRandomN(c.retryHostsCnt)
+	if len(hosts) == 0 {
+		return errNoServiceAvailable
+	}
+	for _, h := range hosts {
+		err = c.Client.SendShardRepairMsg(ctx, h, args)
+		if err == nil || !shouldRetry(err) {
+			return err
+		}
+		span.Errorf("seed shard repair failed, host: %s, args: %+v, err:%+v", h, args, err)
+	}
+
+	return err
+}
+
+func shouldRetry(err error) bool {
+	if err == nil {
+		return false // success
+	}
+	_, ok := err.(rpc.HTTPError)
+	return ok
+}

blobstore/api/proxy/mq.go

@@ -0,0 +1,49 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package proxy
+
+import (
+	"context"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+type MsgSender interface {
+	SendDeleteMsg(ctx context.Context, host string, args *DeleteArgs) error
+	SendShardRepairMsg(ctx context.Context, host string, args *ShardRepairArgs) error
+}
+
+type LbMsgSender interface {
+	SendDeleteMsg(ctx context.Context, args *DeleteArgs) error
+	SendShardRepairMsg(ctx context.Context, args *ShardRepairArgs) error
+}
+
+type DeleteArgs struct {
+	ClusterID proto.ClusterID `json:"cluster_id"`
+	Blobs     []BlobDelete    `json:"blobs"`
+}
+
+type BlobDelete struct {
+	Bid proto.BlobID `json:"bid"`
+	Vid proto.Vid    `json:"vid"`
+}
+
+type ShardRepairArgs struct {
+	ClusterID proto.ClusterID `json:"cluster_id"`
+	Bid       proto.BlobID    `json:"bid"`
+	Vid       proto.Vid       `json:"vid"`
+	BadIdxes  []uint8         `json:"bad_idxes"`
+	Reason    string          `json:"reason"`
+}

blobstore/api/scheduler/client.go

@@ -0,0 +1,131 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package scheduler
+
+import (
+	"context"
+
+	cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+	"github.com/cubefs/cubefs/blobstore/util/errors"
+	"github.com/cubefs/cubefs/blobstore/util/selector"
+)
+
+// defined http server path.
+const (
+	PathStats       = "/stats"
+	PathLeaderStats = "/leader/stats"
+
+	PathTaskAcquire          = "/task/acquire"
+	PathTaskReclaim          = "/task/reclaim"
+	PathTaskCancel           = "/task/cancel"
+	PathTaskComplete         = "/task/complete"
+	PathTaskReport           = "/task/report"
+	PathTaskRenewal          = "/task/renewal"
+	PathInspectComplete      = "/inspect/complete"
+	PathInspectAcquire       = "/inspect/acquire"
+	PathManualMigrateTaskAdd = "/manual/migrate/task/add"
+
+	PathBalanceTaskDetail       = "/balance/task/detail"
+	PathRepairTaskDetail        = "/repair/task/detail"
+	PathDropTaskDetail          = "/drop/task/detail"
+	PathManualMigrateTaskDetail = "/manual/migrate/task/detail"
+
+	PathUpdateVolume = "/update/vol"
+)
+
+var errNoServiceAvailable = errors.New("no service available")
+
+type IScheduler interface {
+	AcquireTask(ctx context.Context, args *AcquireArgs) (ret *WorkerTask, err error)
+	AcquireInspectTask(ctx context.Context) (ret *WorkerInspectTask, err error)
+	// report alive tasks
+	RenewalTask(ctx context.Context, args *TaskRenewalArgs) (ret *TaskRenewalRet, err error)
+	ReportTask(ctx context.Context, args *TaskReportArgs) (err error)
+	ReclaimTask(ctx context.Context, args *ReclaimTaskArgs) (err error)
+	CancelTask(ctx context.Context, args *CancelTaskArgs) (err error)
+	CompleteTask(ctx context.Context, args *CompleteTaskArgs) (err error)
+	CompleteInspect(ctx context.Context, args *CompleteInspectArgs) (err error)
+
+	// stats
+	DiskRepairTaskDetail(ctx context.Context, args *TaskStatArgs) (ret RepairTaskDetail, err error)
+	BalanceTaskDetail(ctx context.Context, args *TaskStatArgs) (ret MigrateTaskDetail, err error)
+	DiskDropTaskDetail(ctx context.Context, args *TaskStatArgs) (ret MigrateTaskDetail, err error)
+	ManualMigrateTaskDetail(ctx context.Context, args *TaskStatArgs) (ret MigrateTaskDetail, err error)
+	Stats(ctx context.Context, host string) (ret TasksStat, err error)
+	LeaderStats(ctx context.Context) (ret TasksStat, err error)
+
+	// add manual migrate task
+	AddManualMigrateTask(ctx context.Context, args *AddManualMigrateArgs) (err error)
+	IVolumeUpdater
+}
+
+type IVolumeUpdater interface {
+	UpdateVol(ctx context.Context, host string, vid proto.Vid) (err error)
+}
+
+// UpdateVolumeArgs argument of volume to update.
+type UpdateVolumeArgs struct {
+	Vid proto.Vid `json:"vid"`
+}
+
+type Config struct {
+	HostSyncIntervalMs int64 `json:"host_sync_interval_ms"`
+	rpc.Config
+}
+
+type client struct {
+	selector selector.Selector
+	rpc.Client
+}
+
+func NewVolumeUpdater(cfg *Config) IVolumeUpdater {
+	return &client{
+		Client: rpc.NewClient(&cfg.Config),
+	}
+}
+
+func New(cfg *Config, service cmapi.APIService, clusterID proto.ClusterID) IScheduler {
+	hostGetter := func() ([]string, error) {
+		svrInfos, err := service.GetService(context.Background(), cmapi.GetServiceArgs{Name: proto.ServiceNameScheduler})
+		if err != nil {
+			return nil, err
+		}
+
+		var hosts []string
+		for _, s := range svrInfos.Nodes {
+			if clusterID == proto.ClusterID(s.ClusterID) {
+				hosts = append(hosts, s.Host)
+			}
+		}
+		if len(hosts) == 0 {
+			return nil, errNoServiceAvailable
+		}
+
+		return hosts, nil
+	}
+	if cfg.HostSyncIntervalMs == 0 {
+		cfg.HostSyncIntervalMs = 1000
+	}
+	return &client{
+		selector: selector.NewSelectorWithGetter(cfg.HostSyncIntervalMs, hostGetter),
+		Client:   rpc.NewClient(&cfg.Config),
+	}
+}
+
+func (c *client) UpdateVol(ctx context.Context, host string, vid proto.Vid) (err error) {
+	return c.PostWith(ctx, host+PathUpdateVolume, nil, UpdateVolumeArgs{Vid: vid})
+}

blobstore/api/scheduler/task.go

@@ -0,0 +1,349 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package scheduler
+
+import (
+	"context"
+
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+type AcquireArgs struct {
+	IDC string `json:"idc"`
+}
+
+type WorkerTask struct {
+	TaskType      string               `json:"task_type"`      // task type
+	Repair        *proto.VolRepairTask `json:"repair"`         // repair task
+	Balance       *proto.MigrateTask   `json:"balance"`        // balance task
+	DiskDrop      *proto.MigrateTask   `json:"disk_drop"`      // disk drop task
+	ManualMigrate *proto.MigrateTask   `json:"manual_migrate"` // manual migrate task
+}
+
+func (task *WorkerTask) IsValid() bool {
+	var (
+		mode        codemode.CodeMode
+		destination proto.VunitLocation
+		srcs        []proto.VunitLocation
+	)
+	switch task.TaskType {
+	case proto.RepairTaskType:
+		mode = task.Repair.CodeMode
+		destination = task.Repair.Destination
+		srcs = task.Repair.Sources
+	case proto.BalanceTaskType:
+		mode = task.Balance.CodeMode
+		destination = task.Balance.Destination
+		srcs = task.Balance.Sources
+	case proto.DiskDropTaskType:
+		mode = task.DiskDrop.CodeMode
+		destination = task.DiskDrop.Destination
+		srcs = task.DiskDrop.Sources
+	case proto.ManualMigrateType:
+		mode = task.ManualMigrate.CodeMode
+		destination = task.ManualMigrate.Destination
+		srcs = task.ManualMigrate.Sources
+	default:
+		return false
+	}
+
+	if !mode.IsValid() {
+		return false
+	}
+	// check destination
+	if !proto.CheckVunitLocations([]proto.VunitLocation{destination}) {
+		return false
+	}
+	// check sources
+	if !proto.CheckVunitLocations(srcs) {
+		return false
+	}
+
+	return true
+}
+
+func (c *client) AcquireTask(ctx context.Context, args *AcquireArgs) (ret *WorkerTask, err error) {
+	err = c.request(func(host string) error {
+		return c.GetWith(ctx, host+PathTaskAcquire+"?idc="+args.IDC, &ret)
+	})
+	return
+}
+
+type WorkerInspectTask struct {
+	Task *proto.InspectTask `json:"task"`
+}
+
+func (task *WorkerInspectTask) IsValid() bool {
+	if !task.Task.Mode.IsValid() {
+		return false
+	}
+
+	if !proto.CheckVunitLocations(task.Task.Replicas) {
+		return false
+	}
+
+	return true
+}
+
+func (c *client) AcquireInspectTask(ctx context.Context) (*WorkerInspectTask, error) {
+	ret := WorkerInspectTask{
+		Task: &proto.InspectTask{},
+	}
+	err := c.request(func(host string) error {
+		return c.GetWith(ctx, host+PathInspectAcquire, &ret)
+	})
+	return &ret, err
+}
+
+type TaskRenewalArgs struct {
+	IDC           string              `json:"idc"`
+	Repair        map[string]struct{} `json:"repair"`
+	Balance       map[string]struct{} `json:"balance"`
+	DiskDrop      map[string]struct{} `json:"disk_drop"`
+	ManualMigrate map[string]struct{} `json:"manual_migrate"`
+}
+
+type TaskRenewalRet struct {
+	Repair        map[string]string `json:"repair"`
+	Balance       map[string]string `json:"balance"`
+	DiskDrop      map[string]string `json:"disk_drop"`
+	ManualMigrate map[string]string `json:"manual_migrate"`
+}
+
+func (c *client) RenewalTask(ctx context.Context, args *TaskRenewalArgs) (ret *TaskRenewalRet, err error) {
+	err = c.request(func(host string) error {
+		return c.PostWith(ctx, host+PathTaskRenewal, &ret, args)
+	})
+	return
+}
+
+type TaskReportArgs struct {
+	TaskType string `json:"task_type"`
+	TaskId   string `json:"task_id"`
+
+	TaskStats            proto.TaskStatistics `json:"task_stats"`
+	IncreaseDataSizeByte int                  `json:"increase_data_size_byte"`
+	IncreaseShardCnt     int                  `json:"increase_shard_cnt"`
+}
+
+func (c *client) ReportTask(ctx context.Context, args *TaskReportArgs) (err error) {
+	return c.request(func(host string) error {
+		return c.PostWith(ctx, host+PathTaskReport, nil, args)
+	})
+}
+
+type ReclaimTaskArgs struct {
+	TaskId   string                `json:"task_id"`
+	IDC      string                `json:"idc"`
+	TaskType string                `json:"task_type"`
+	Src      []proto.VunitLocation `json:"src"`
+	Dest     proto.VunitLocation   `json:"dest"`
+	Reason   string                `json:"reason"`
+}
+
+func (c *client) ReclaimTask(ctx context.Context, args *ReclaimTaskArgs) (err error) {
+	return c.request(func(host string) error {
+		return c.PostWith(ctx, host+PathTaskReclaim, nil, args)
+	})
+}
+
+type CancelTaskArgs struct {
+	TaskId   string                `json:"task_id"`
+	IDC      string                `json:"idc"`
+	TaskType string                `json:"task_type"`
+	Src      []proto.VunitLocation `json:"src"`
+	Dest     proto.VunitLocation   `json:"dest"`
+	Reason   string                `json:"reason"`
+}
+
+func (c *client) CancelTask(ctx context.Context, args *CancelTaskArgs) (err error) {
+	return c.request(func(host string) error {
+		return c.PostWith(ctx, host+PathTaskCancel, nil, args)
+	})
+}
+
+type CompleteTaskArgs struct {
+	TaskId   string                `json:"task_id"`
+	IDC      string                `json:"idc"`
+	TaskType string                `json:"task_type"`
+	Src      []proto.VunitLocation `json:"src"`
+	Dest     proto.VunitLocation   `json:"dest"`
+}
+
+func (c *client) CompleteTask(ctx context.Context, args *CompleteTaskArgs) (err error) {
+	return c.request(func(host string) error {
+		return c.PostWith(ctx, host+PathTaskComplete, nil, args)
+	})
+}
+
+type CompleteInspectArgs struct {
+	*proto.InspectRet
+}
+
+func (c *client) CompleteInspect(ctx context.Context, args *CompleteInspectArgs) (err error) {
+	return c.request(func(host string) error {
+		return c.PostWith(ctx, host+PathInspectComplete, nil, args)
+	})
+}
+
+type AddManualMigrateArgs struct {
+	Vuid           proto.Vuid `json:"vuid"`
+	DirectDownload bool       `json:"direct_download"`
+}
+
+func (args *AddManualMigrateArgs) Valid() bool {
+	return args.Vuid.IsValid()
+}
+
+func (c *client) AddManualMigrateTask(ctx context.Context, args *AddManualMigrateArgs) (err error) {
+	return c.request(func(host string) error {
+		return c.PostWith(ctx, host+PathManualMigrateTaskAdd, nil, args)
+	})
+}
+
+// for task stat
+type TaskStatArgs struct {
+	TaskId string `json:"task_id"`
+}
+
+type RepairTaskDetail struct {
+	TaskInfo proto.VolRepairTask  `json:"task_info"`
+	RunStats proto.TaskStatistics `json:"run_stats"`
+}
+
+type MigrateTaskDetail struct {
+	TaskInfo proto.MigrateTask    `json:"task_info"`
+	RunStats proto.TaskStatistics `json:"run_stats"`
+}
+
+type PerMinStats struct {
+	FinishedCnt    string `json:"finished_cnt"`
+	ShardCnt       string `json:"shard_cnt"`
+	DataAmountByte string `json:"data_amount_byte"`
+}
+
+type DiskRepairTasksStat struct {
+	Enable           bool         `json:"enable"`
+	RepairingDiskID  proto.DiskID `json:"repairing_disk_id"`
+	TotalTasksCnt    int          `json:"total_tasks_cnt"`
+	RepairedTasksCnt int          `json:"repaired_tasks_cnt"`
+	MigrateTasksStat
+}
+
+type MigrateTasksStat struct {
+	PreparingCnt   int         `json:"preparing_cnt"`
+	WorkerDoingCnt int         `json:"worker_doing_cnt"`
+	FinishingCnt   int         `json:"finishing_cnt"`
+	StatsPerMin    PerMinStats `json:"stats_per_min"`
+}
+
+type DiskDropTasksStat struct {
+	Enable          bool         `json:"enable"`
+	DroppingDiskID  proto.DiskID `json:"dropping_disk_id"`
+	TotalTasksCnt   int          `json:"total_tasks_cnt"`
+	DroppedTasksCnt int          `json:"dropped_tasks_cnt"`
+	MigrateTasksStat
+}
+
+type BalanceTasksStat struct {
+	Enable bool `json:"enable"`
+	MigrateTasksStat
+}
+
+type ManualMigrateTasksStat struct {
+	MigrateTasksStat
+}
+
+type VolumeInspectTasksStat struct {
+	Enable         bool   `json:"enable"`
+	FinishedPerMin string `json:"finished_per_min"`
+	TimeOutPerMin  string `json:"time_out_per_min"`
+}
+
+// RunnerStat shard repair and blob delete stat
+type RunnerStat struct {
+	Enable        bool     `json:"enable"`
+	SuccessPerMin string   `json:"success_per_min"`
+	FailedPerMin  string   `json:"failed_per_min"`
+	TotalErrCnt   uint64   `json:"total_err_cnt"`
+	ErrStats      []string `json:"err_stats"`
+}
+
+type TasksStat struct {
+	DiskRepair    *DiskRepairTasksStat    `json:"disk_repair,omitempty"`
+	DiskDrop      *DiskDropTasksStat      `json:"disk_drop,omitempty"`
+	Balance       *BalanceTasksStat       `json:"balance,omitempty"`
+	ManualMigrate *ManualMigrateTasksStat `json:"manual_migrate,omitempty"`
+	VolumeInspect *VolumeInspectTasksStat `json:"volume_inspect,omitempty"`
+	ShardRepair   *RunnerStat             `json:"shard_repair"`
+	BlobDelete    *RunnerStat             `json:"blob_delete"`
+}
+
+func (c *client) DiskRepairTaskDetail(ctx context.Context, args *TaskStatArgs) (ret RepairTaskDetail, err error) {
+	err = c.request(func(host string) error {
+		return c.PostWith(ctx, host+PathRepairTaskDetail, &ret, args)
+	})
+	return
+}
+
+func (c *client) BalanceTaskDetail(ctx context.Context, args *TaskStatArgs) (ret MigrateTaskDetail, err error) {
+	err = c.request(func(host string) error {
+		return c.PostWith(ctx, host+PathBalanceTaskDetail, &ret, args)
+	})
+	return
+}
+
+func (c *client) DiskDropTaskDetail(ctx context.Context, args *TaskStatArgs) (ret MigrateTaskDetail, err error) {
+	err = c.request(func(host string) error {
+		return c.PostWith(ctx, host+PathDropTaskDetail, &ret, args)
+	})
+	return
+}
+
+func (c *client) ManualMigrateTaskDetail(ctx context.Context, args *TaskStatArgs) (ret MigrateTaskDetail, err error) {
+	err = c.request(func(host string) error {
+		return c.PostWith(ctx, host+PathManualMigrateTaskDetail, &ret, args)
+	})
+	return
+}
+
+func (c *client) Stats(ctx context.Context, host string) (ret TasksStat, err error) {
+	err = c.GetWith(ctx, host+PathStats, &ret)
+	return
+}
+
+func (c *client) LeaderStats(ctx context.Context) (ret TasksStat, err error) {
+	err = c.request(func(host string) error {
+		return c.GetWith(ctx, host+PathLeaderStats, &ret)
+	})
+	return
+}
+
+func (c *client) selectHost() (string, error) {
+	hosts := c.selector.GetRandomN(1)
+	if len(hosts) == 0 {
+		return "", errNoServiceAvailable
+	}
+	return hosts[0], nil
+}
+
+func (c *client) request(req func(host string) error) error {
+	host, err := c.selectHost()
+	if err != nil {
+		return err
+	}
+	return req(host)
+}

blobstore/cmd/cmd.go

@@ -0,0 +1,223 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package cmd
+
+import (
+	"context"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"os/signal"
+	"runtime"
+	"syscall"
+	"time"
+
+	"gopkg.in/natefinch/lumberjack.v2"
+
+	"github.com/cubefs/cubefs/blobstore/common/config"
+	"github.com/cubefs/cubefs/blobstore/common/profile"
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+	"github.com/cubefs/cubefs/blobstore/common/rpc/auditlog"
+	"github.com/cubefs/cubefs/blobstore/common/rpc/auth"
+	"github.com/cubefs/cubefs/blobstore/util/graceful"
+	"github.com/cubefs/cubefs/blobstore/util/log"
+
+	// module release init functions
+	_ "github.com/cubefs/cubefs/blobstore/util/version"
+)
+
+const (
+	defaultShutdownTimeoutS = 30
+)
+
+type LogConfig struct {
+	Level      log.Level `json:"level"`
+	Filename   string    `json:"filename"`
+	MaxSize    int       `json:"maxsize"`
+	MaxAge     int       `json:"maxage"`
+	MaxBackups int       `json:"maxbackups"`
+}
+
+type Config struct {
+	MaxProcs         int       `json:"max_procs"`
+	LogConf          LogConfig `json:"log"`
+	BindAddr         string    `json:"bind_addr"`
+	ShutdownTimeoutS int       `json:"shutdown_timeout_s"`
+
+	AuditLog auditlog.Config `json:"auditlog"`
+	Auth     auth.Config     `json:"auth"`
+}
+
+type Module struct {
+	Name       string
+	InitConfig func(args []string) (*Config, error)
+	SetUp      func() (*rpc.Router, []rpc.ProgressHandler)
+	TearDown   func()
+	graceful   bool
+}
+
+var mod *Module
+
+func RegisterModule(m *Module) {
+	mod = m
+	mod.graceful = false
+}
+
+func RegisterGracefulModule(m *Module) {
+	mod = m
+	mod.graceful = true
+}
+
+func newLogWriter(cfg *LogConfig) io.Writer {
+	maxsize := cfg.MaxSize
+	if maxsize == 0 {
+		maxsize = 1024
+	}
+	maxage := cfg.MaxAge
+	if maxage == 0 {
+		maxage = 7
+	}
+	maxbackups := cfg.MaxBackups
+	if maxbackups == 0 {
+		maxbackups = 7
+	}
+	return &lumberjack.Logger{
+		Filename:   cfg.Filename,
+		MaxSize:    maxsize,
+		MaxAge:     maxage,
+		MaxBackups: maxbackups,
+		LocalTime:  true,
+	}
+}
+
+func Main(args []string) {
+	cfg, err := mod.InitConfig(args)
+	if err != nil {
+		log.Fatalf("init config error: %v", err)
+	}
+	if cfg.MaxProcs > 0 {
+		runtime.GOMAXPROCS(cfg.MaxProcs)
+	}
+	log.SetOutputLevel(cfg.LogConf.Level)
+	registerLogLevel()
+	if cfg.LogConf.Filename != "" {
+		log.SetOutput(newLogWriter(&cfg.LogConf))
+	}
+	if cfg.ShutdownTimeoutS <= 0 {
+		cfg.ShutdownTimeoutS = defaultShutdownTimeoutS
+	}
+
+	lh, logf, err := auditlog.Open(mod.Name, &cfg.AuditLog)
+	if err != nil {
+		log.Fatal("failed to open auditlog:", err)
+	}
+	defer logf.Close()
+
+	ctx, cancel1 := context.WithCancel(context.Background())
+	defer cancel1()
+	config.HotReload(ctx, config.ConfName())
+
+	if mod.graceful {
+		programEntry := func(state *graceful.State) {
+			router, handlers := mod.SetUp()
+
+			httpServer := &http.Server{
+				Addr:    cfg.BindAddr,
+				Handler: reorderMiddleWareHandlers(router, lh, cfg.BindAddr, cfg.Auth, handlers),
+			}
+
+			log.Info("server is running at:", cfg.BindAddr)
+			go func() {
+				if err := httpServer.Serve(state.ListenerFds[0].(*net.TCPListener)); err != nil && err != http.ErrServerClosed {
+					log.Fatal("server exits:", err)
+				}
+			}()
+
+			// wait for signal
+			<-state.CloseCh
+			log.Info("graceful shutdown...")
+			ctx, cancel := context.WithTimeout(context.Background(), time.Duration(cfg.ShutdownTimeoutS)*time.Second)
+			defer cancel()
+			httpServer.Shutdown(ctx)
+
+			if mod.TearDown != nil {
+				mod.TearDown()
+			}
+		}
+		graceful.Run(&graceful.Config{
+			Entry:           programEntry,
+			ListenAddresses: []string{cfg.BindAddr},
+		})
+		return
+	}
+
+	router, handlers := mod.SetUp()
+
+	httpServer := &http.Server{
+		Addr:    cfg.BindAddr,
+		Handler: reorderMiddleWareHandlers(router, lh, cfg.BindAddr, cfg.Auth, handlers),
+	}
+
+	log.Info("Server is running at", cfg.BindAddr)
+	go func() {
+		if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			log.Fatalf("Server exits, err: %v", err)
+		}
+	}()
+
+	// wait for signal
+	ch := make(chan os.Signal, 1)
+	signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT)
+	sig := <-ch
+	log.Infof("receive signal: %s, stop service...", sig.String())
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(cfg.ShutdownTimeoutS)*time.Second)
+	defer cancel()
+	httpServer.Shutdown(ctx)
+
+	if mod.TearDown != nil {
+		mod.TearDown()
+	}
+}
+
+// reorderMiddleWareHandlers
+//	the order of handlers in MiddlewareHandler has some constraints:
+//	1. the first is router,
+//	2. the second is AuditLog handler,
+//	3. the third is profile handler if config,
+//	4. the fourth is Auth handler if config,
+//	5. others self define handlers by modules.
+func reorderMiddleWareHandlers(r *rpc.Router, lh rpc.ProgressHandler, profileAddr string, authCfg auth.Config, handlers []rpc.ProgressHandler) (mux http.Handler) {
+	hs := append([]rpc.ProgressHandler{}, lh)
+	if profileHandler := profile.NewProfileHandler(profileAddr); profileHandler != nil {
+		hs = append(hs, profileHandler)
+	}
+	if authCfg.EnableAuth && authCfg.Secret != "" {
+		hs = append(hs, auth.NewAuthHandler(&authCfg))
+	}
+	hs = append(hs, handlers...)
+
+	return rpc.MiddlewareHandlerWith(r, hs...)
+}
+
+func registerLogLevel() {
+	logLevelPath, logLevelHandler := log.ChangeDefaultLevelHandler()
+	profile.HandleFunc(http.MethodPost, logLevelPath, func(c *rpc.Context) {
+		logLevelHandler.ServeHTTP(c.Writer, c.Request)
+	})
+	profile.HandleFunc(http.MethodGet, logLevelPath, func(c *rpc.Context) {
+		logLevelHandler.ServeHTTP(c.Writer, c.Request)
+	})
+}

blobstore/cmd/scheduler/main.go

@@ -0,0 +1,27 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package main
+
+import (
+	"os"
+
+	"github.com/cubefs/cubefs/blobstore/cmd"
+
+	_ "github.com/cubefs/cubefs/blobstore/scheduler"
+)
+
+func main() {
+	cmd.Main(os.Args)
+}

blobstore/cmd/scheduler/scheduler.conf

@@ -0,0 +1,35 @@
+{
+  "bind_addr": ":9800",
+  "cluster_id": 1,
+  "services": {
+    "leader": 1,
+    "node_id": 1,
+    "members": {"1": "127.0.0.1:9800"}
+  },
+  "service_register": {
+    "host": "http://127.0.0.1:9800",
+    "idc": "z0"
+  },
+  "clustermgr": {
+    "hosts": ["http://127.0.0.1:10110", "http://127.0.0.1:10111", "http://127.0.0.1:10112"]
+  },
+  "database": {
+    "mongo": {
+      "uri": "mongodb://127.0.0.1:27017"
+    }
+  },
+  "kafka": {
+    "broker_list": ["127.0.0.1:9092"]
+  },
+  "blob_delete": {
+    "delete_log": {
+      "dir": "./delete_log"
+    }
+  },
+  "log": {
+    "level": 0
+  },
+  "auditlog": {
+    "logdir": "/tmp/scheduler/"
+  }
+}

blobstore/cmd/scheduler/scheduler.follower.conf

@@ -0,0 +1,35 @@
+{
+  "bind_addr": ":9880",
+  "cluster_id": 1,
+  "services": {
+    "leader": 1,
+    "node_id": 2,
+    "members": {"1": "127.0.0.1:9800", "2": "127.0.0.1:9880"}
+  },
+  "service_register": {
+    "host": "http://127.0.0.1:9880",
+    "idc": "z0"
+  },
+  "clustermgr": {
+    "hosts": ["http://127.0.0.1:10110", "http://127.0.0.1:10111", "http://127.0.0.1:10112"]
+  },
+  "database": {
+    "mongo": {
+      "uri": "mongodb://127.0.0.1:27017"
+    }
+  },
+  "kafka": {
+    "broker_list": ["127.0.0.1:9092"]
+  },
+  "blob_delete": {
+    "delete_log": {
+      "dir": "./delete_log"
+    }
+  },
+  "log": {
+    "level": 0
+  },
+  "auditlog": {
+    "logdir": "./auditlog/scheduler"
+  }
+}

blobstore/cmd/scheduler/scheduler.leader.conf

@@ -0,0 +1,35 @@
+{
+  "bind_addr": ":9800",
+  "cluster_id": 1,
+  "services": {
+    "leader": 1,
+    "node_id": 1,
+    "members": {"1": "127.0.0.1:9800", "2": "127.0.0.1:9880"}
+  },
+  "service_register": {
+    "host": "http://127.0.0.1:9800",
+    "idc": "z0"
+  },
+  "clustermgr": {
+    "hosts": ["http://127.0.0.1:10110", "http://127.0.0.1:10111", "http://127.0.0.1:10112"]
+  },
+  "database": {
+    "mongo": {
+      "uri": "mongodb://127.0.0.1:27017"
+    }
+  },
+  "kafka": {
+    "broker_list": ["127.0.0.1:9092"]
+  },
+  "blob_delete": {
+    "delete_log": {
+      "dir": "./delete_log"
+    }
+  },
+  "log": {
+    "level": 0
+  },
+  "auditlog": {
+    "logdir": "./auditlog/scheduler"
+  }
+}

blobstore/common/errors/access.go

@@ -0,0 +1,35 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package errors
+
+// code for access
+const (
+	CodeAccessReadRequestBody  = 466 // read request body error
+	CodeAccessReadConflictBody = 499 // read conflict body error
+	CodeAccessUnexpect         = 550 // unexpect
+	CodeAccessServiceDiscovery = 551 // service discovery for access api client
+	CodeAccessLimited          = 552 // read write limited for access api client
+	CodeAccessExceedSize       = 553 // exceed max size
+)
+
+// errro of access
+var (
+	ErrAccessReadRequestBody  = Error(CodeAccessReadRequestBody)
+	ErrAccessReadConflictBody = Error(CodeAccessReadConflictBody)
+	ErrAccessUnexpect         = Error(CodeAccessUnexpect)
+	ErrAccessServiceDiscovery = Error(CodeAccessServiceDiscovery)
+	ErrAccessLimited          = Error(CodeAccessLimited)
+	ErrAccessExceedSize       = Error(CodeAccessExceedSize)
+)

blobstore/common/errors/allocator.go

@@ -0,0 +1,25 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package errors
+
+const (
+	CodeNoAvaliableVolume = 801
+	CodeAllocBidFromCm    = 802
+)
+
+var (
+	ErrNoAvaliableVolume = Error(CodeNoAvaliableVolume)
+	ErrAllocBidFromCm    = Error(CodeAllocBidFromCm)
+)

blobstore/common/errors/blobnode.go

@@ -0,0 +1,80 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package errors
+
+const (
+	CodeInvalidParam = 600
+	CodeAlreadyExist = 601
+	CodeOutOfLimit   = 602
+	CodeInternal     = 603
+	CodeOverload     = 604
+
+	CodeDiskNotFound  = 611
+	CodeDiskBroken    = 613
+	CodeInvalidDiskId = 614
+	CodeDiskNoSpace   = 615
+
+	CodeVuidNotFound     = 621
+	CodeVUIDReadonly     = 622
+	CodeVUIDRelease      = 623
+	CodeVuidNotMatch     = 624
+	CodeChunkNotReadonly = 625
+	CodeChunkNotNormal   = 626
+	CodeChunkNoSpace     = 627
+	CodeChunkCompacting  = 628
+	CodeInvalidChunkId   = 630
+	CodeTooManyChunks    = 632
+	CodeChunkInuse       = 633
+
+	CodeBidNotFound          = 651
+	CodeShardSizeTooLarge    = 652
+	CodeShardNotMarkDelete   = 653
+	CodeShardMarkDeleted     = 654
+	CodeShardInvalidOffset   = 655
+	CodeShardListExceedLimit = 656
+	CodeShardInvalidBid      = 657
+)
+
+var (
+	ErrInvalidParam = Error(CodeInvalidParam)
+	ErrAlreadyExist = Error(CodeAlreadyExist)
+	ErrOutOfLimit   = Error(CodeOutOfLimit)
+	ErrOverload     = Error(CodeOverload)
+
+	ErrNoSuchDisk    = Error(CodeDiskNotFound)
+	ErrDiskBroken    = Error(CodeDiskBroken)
+	ErrDiskNoSpace   = Error(CodeDiskNoSpace)
+	ErrInvalidDiskId = Error(CodeInvalidDiskId)
+
+	ErrNoSuchVuid       = Error(CodeVuidNotFound)
+	ErrReadonlyVUID     = Error(CodeVUIDReadonly)
+	ErrReleaseVUID      = Error(CodeVUIDRelease)
+	ErrVuidNotMatch     = Error(CodeVuidNotMatch)
+	ErrChunkNotReadonly = Error(CodeChunkNotReadonly)
+	ErrChunkNotNormal   = Error(CodeChunkNotNormal)
+	ErrChunkNoSpace     = Error(CodeChunkNoSpace)
+	ErrChunkInCompact   = Error(CodeChunkCompacting)
+	ErrInvalidChunkId   = Error(CodeInvalidChunkId)
+	ErrTooManyChunks    = Error(CodeTooManyChunks)
+	ErrChunkInuse       = Error(CodeChunkInuse)
+
+	ErrNoSuchBid            = Error(CodeBidNotFound)
+	ErrShardSizeTooLarge    = Error(CodeShardSizeTooLarge)
+	ErrShardNotMarkDelete   = Error(CodeShardNotMarkDelete)
+	ErrShardMarkDeleted     = Error(CodeShardMarkDeleted)
+	ErrShardInvalidOffset   = Error(CodeShardInvalidOffset)
+	ErrShardListExceedLimit = Error(CodeShardListExceedLimit)
+	ErrShardInvalidBid      = Error(CodeShardInvalidBid)
+)

blobstore/common/errors/clustermgr.go

@@ -0,0 +1,87 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package errors
+
+const (
+	CodeCMUnexpect                   = 900
+	CodeActiveVolume                 = 901
+	CodeLockNotAllow                 = 902
+	CodeUnlockNotAllow               = 903
+	CodeVolumeNotExist               = 904
+	CodeVolumeStatusNotAcceptable    = 905
+	CodeRaftPropose                  = 906
+	CodeNoLeader                     = 907
+	CodeRaftReadIndex                = 908
+	CodeUpdateVolumeParamInvalid     = 909
+	CodeDuplicatedMemberInfo         = 910
+	CodeCMDiskNotFound               = 911
+	CodeInvalidDiskStatus            = 912
+	CodeChangeDiskStatusNotAllow     = 913
+	CodeConcurrentAllocVolumeUnit    = 914
+	CodeOverMaxVolumeThreshold       = 915
+	CodeAllocVolumeInvalidParams     = 916
+	CodeNoAvailableVolume            = 917
+	CodeOldVuidNotMatch              = 918
+	CodeNewVuidNotMatch              = 919
+	CodeNewDiskIDNotMatch            = 920
+	CodeConfigArgument               = 921
+	CodeInvalidClusterID             = 922
+	CodeInvalidIDC                   = 923
+	CodeVolumeUnitNotExist           = 924
+	CodeRegisterServiceInvalidParams = 925
+	CodeDiskAbnormalOrNotReadOnly    = 926
+	CodeStatChunkFailed              = 927
+	CodeInvalidCodeMode              = 928
+	CodeRetainVolumeNotAlloc         = 929
+	CodeDroppedDiskHasVolumeUnit     = 930
+	CodeNotSupportIdle               = 931
+	CodeDiskIsDropping               = 932
+)
+
+var (
+	ErrCMUnexpect                   = Error(CodeCMUnexpect)
+	ErrActiveVolume                 = Error(CodeActiveVolume)
+	ErrLockNotAllow                 = Error(CodeLockNotAllow)
+	ErrUnlockNotAllow               = Error(CodeUnlockNotAllow)
+	ErrVolumeNotExist               = Error(CodeVolumeNotExist)
+	ErrVolumeStatusNotAcceptable    = Error(CodeVolumeStatusNotAcceptable)
+	ErrRaftPropose                  = Error(CodeRaftPropose)
+	ErrNoLeader                     = Error(CodeNoLeader)
+	ErrRaftReadIndex                = Error(CodeRaftReadIndex)
+	ErrUpdateVolumeParamInvalid     = Error(CodeUpdateVolumeParamInvalid)
+	ErrDuplicatedMemberInfo         = Error(CodeDuplicatedMemberInfo)
+	ErrCMDiskNotFound               = Error(CodeCMDiskNotFound)
+	ErrInvalidStatus                = Error(CodeInvalidDiskStatus)
+	ErrChangeDiskStatusNotAllow     = Error(CodeChangeDiskStatusNotAllow)
+	ErrConcurrentAllocVolumeUnit    = Error(CodeConcurrentAllocVolumeUnit)
+	ErrOverMaxVolumeThreshold       = Error(CodeOverMaxVolumeThreshold)
+	ErrNoAvailableVolume            = Error(CodeNoAvailableVolume)
+	ErrAllocVolumeInvalidParams     = Error(CodeAllocVolumeInvalidParams)
+	ErrOldVuidNotMatch              = Error(CodeOldVuidNotMatch)
+	ErrNewVuidNotMatch              = Error(CodeNewVuidNotMatch)
+	ErrNewDiskIDNotMatch            = Error(CodeNewDiskIDNotMatch)
+	ErrConfigArgument               = Error(CodeConfigArgument)
+	ErrInvalidClusterID             = Error(CodeInvalidClusterID)
+	ErrInvalidIDC                   = Error(CodeInvalidIDC)
+	ErrVolumeUnitNotExist           = Error(CodeVolumeUnitNotExist)
+	ErrRegisterServiceInvalidParams = Error(CodeRegisterServiceInvalidParams)
+	ErrDiskAbnormalOrNotReadOnly    = Error(CodeDiskAbnormalOrNotReadOnly)
+	ErrStatChunkFailed              = Error(CodeStatChunkFailed)
+	ErrInvalidCodeMode              = Error(CodeInvalidCodeMode)
+	ErrRetainVolumeNotAlloc         = Error(CodeRetainVolumeNotAlloc)
+	ErrDroppedDiskHasVolumeUnit     = Error(CodeDroppedDiskHasVolumeUnit)
+	ErrNotSupportIdle               = Error(CodeNotSupportIdle)
+	ErrDiskIsDropping               = Error(CodeDiskIsDropping)
+)

blobstore/common/errors/common.go

@@ -0,0 +1,42 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package errors
+
+import (
+	"errors"
+	"net/http"
+
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+)
+
+var (
+	// 2xx
+	ErrExist = newError(http.StatusCreated, "Data Already Exist")
+
+	// 4xx
+	ErrIllegalArguments             = newError(http.StatusBadRequest, "Illegal Arguments")
+	ErrNotFound                     = newError(http.StatusNotFound, "Not Found")
+	ErrRequestTimeout               = newError(http.StatusRequestTimeout, "Request Timeout")
+	ErrRequestedRangeNotSatisfiable = newError(http.StatusRequestedRangeNotSatisfiable, "Request Range Not Satisfiable")
+	ErrRequestNotAllow              = newError(http.StatusBadRequest, "Request Not Allow")
+	ErrReaderError                  = newError(499, "Reader Error")
+
+	// 5xx errUnexpected - unexpected error, requires manual intervention.
+	ErrUnexpected = newError(http.StatusInternalServerError, "Unexpected Error")
+)
+
+func newError(status int, msg string) *rpc.Error {
+	return rpc.NewError(status, "", errors.New(msg))
+}

blobstore/common/errors/errors.go

@@ -0,0 +1,170 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package errors
+
+import (
+	"net/http"
+
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+)
+
+// access             550-599
+// blobnode           600-699
+// background service 700-799
+// allocator          800-899
+// clusterMgr         900-999
+
+// Error http status code for all application
+type Error int
+
+var _ rpc.HTTPError = Error(0)
+
+// Error implements error and rpc.HTTPError
+func (e Error) Error() string {
+	return errCodeMap[int(e)]
+}
+
+// StatusCode implements rpc.HTTPError
+func (e Error) StatusCode() int {
+	return int(e)
+}
+
+// ErrorCode implements rpc.HTTPError
+func (e Error) ErrorCode() string {
+	return ""
+}
+
+var errCodeMap = map[int]string{
+	// access
+	CodeAccessReadRequestBody:  "access read request body",
+	CodeAccessReadConflictBody: "access read conflict body",
+	CodeAccessUnexpect:         "access unexpected error",
+	CodeAccessServiceDiscovery: "access client service discovery disconnect",
+	CodeAccessLimited:          "access limited",
+	CodeAccessExceedSize:       "access exceed object size",
+
+	// clustermgr
+	CodeCMUnexpect:                "cm: unexpected error",
+	CodeActiveVolume:              "volume is activity status",
+	CodeLockNotAllow:              "lock volume not allow",
+	CodeUnlockNotAllow:            "unlock volume not allow",
+	CodeVolumeNotExist:            "volume not exist",
+	CodeVolumeStatusNotAcceptable: "volume status not acceptable",
+	CodeRaftPropose:               "raft propose error",
+	CodeNoLeader:                  "no leader",
+	CodeRaftReadIndex:             "raft read index error",
+	CodeUpdateVolumeParamInvalid:  "update volume params invalid",
+	CodeDuplicatedMemberInfo:      "duplicated member info",
+	CodeCMDiskNotFound:            "disk not found",
+	CodeInvalidDiskStatus:         "invalid status",
+	CodeChangeDiskStatusNotAllow:  "not allow to change status back",
+	CodeConcurrentAllocVolumeUnit: "alloc volume unit concurrently",
+	CodeOverMaxVolumeThreshold:    "allocator request alloc volume over max threshold",
+	CodeNoAvailableVolume:         "no available volume",
+	CodeAllocVolumeInvalidParams:  "alloc volume request params is invalid",
+	CodeOldVuidNotMatch:           "update volume unit, old vuid not match",
+	CodeNewVuidNotMatch:           "update volume unit, new vuid not match",
+	CodeNewDiskIDNotMatch:         "update volume unit, new diskID not match",
+	CodeConfigArgument:            "config argument marshal error",
+	CodeInvalidClusterID:          "request params error, invalid clusterID",
+	CodeInvalidIDC:                "request params error,invalid idc",
+	CodeVolumeUnitNotExist:        "volume unit not exist",
+	CodeDiskAbnormalOrNotReadOnly: "disk is abnormal or not readonly, can't add into dropping list",
+	CodeStatChunkFailed:           "stat blob node chunk failed",
+	CodeInvalidCodeMode:           "request alloc volume codeMode not invalid",
+	CodeRetainVolumeNotAlloc:      "retain volume is not alloc",
+	CodeDroppedDiskHasVolumeUnit:  "dropped disk still has volume unit remain, migrate them firstly",
+	CodeNotSupportIdle:            "list volume v2 not support idle status",
+	CodeDiskIsDropping:            "dropping disk not allow change state or set readonly",
+
+	// background
+	CodeNotingTodo:                   "nothing to do",
+	CodeDestReplicaBad:               "dest replica is bad can not repair",
+	CodeOrphanShard:                  "shard is an orphan",
+	CodeIllegalTask:                  "illegal task",
+	CodeNoInspect:                    "no inspect mgr instance",
+	CodeClusterIDNotMatch:            "clusterId not match",
+	CodeRegisterServiceInvalidParams: "register service params is invalid",
+	CodeRequestLimited:               "request limited",
+
+	// allocator
+	CodeNoAvaliableVolume: "this codemode has no avaliable volume",
+	CodeAllocBidFromCm:    "alloc bid from clustermgr error",
+
+	// blobnode
+	CodeInvalidParam: "blobnode: invalid params",
+	CodeAlreadyExist: "blobnode: entry already exist",
+	CodeOutOfLimit:   "blobnode: out of limit",
+	CodeInternal:     "blobnode: internal error",
+	CodeOverload:     "blobnode: service is overload",
+
+	CodeDiskNotFound:  "disk not found",
+	CodeDiskBroken:    "disk is broken",
+	CodeInvalidDiskId: "disk id is invalid",
+	CodeDiskNoSpace:   "disk no space",
+
+	CodeVuidNotFound:     "vuid not found",
+	CodeVUIDReadonly:     "vuid readonly",
+	CodeVUIDRelease:      "vuid released",
+	CodeVuidNotMatch:     "vuid not match",
+	CodeChunkNotReadonly: "chunk must readonly",
+	CodeChunkNotNormal:   "chunk must normal",
+	CodeChunkNoSpace:     "chunk no space",
+	CodeChunkCompacting:  "chunk is compacting",
+	CodeInvalidChunkId:   "chunk id is invalid",
+	CodeTooManyChunks:    "too many chunks",
+	CodeChunkInuse:       "chunk in use",
+
+	CodeBidNotFound:          "bid not found",
+	CodeShardSizeTooLarge:    "shard size too large",
+	CodeShardNotMarkDelete:   "shard must mark delete",
+	CodeShardMarkDeleted:     "shard already mark delete",
+	CodeShardInvalidOffset:   "shard offset is invalid",
+	CodeShardInvalidBid:      "shard key bid is invalid",
+	CodeShardListExceedLimit: "shard list exceed the limit",
+}
+
+// HTTPError make rpc.HTTPError
+func HTTPError(statusCode int, errCode string, err error) error {
+	return rpc.NewError(statusCode, errCode, err)
+}
+
+// Error2HTTPError transfer error to rpc.HTTPError
+func Error2HTTPError(err error) error {
+	if err == nil {
+		return nil
+	}
+	if e, ok := err.(rpc.HTTPError); ok {
+		return e
+	}
+	if code, ok := err.(Error); ok {
+		return code
+	}
+	return rpc.NewError(http.StatusInternalServerError, "ServerError", err)
+}
+
+// DetectCode detect code
+func DetectCode(err error) int {
+	if err == nil {
+		return http.StatusOK
+	}
+	if code, ok := err.(Error); ok {
+		return int(code)
+	}
+	if httpErr, ok := err.(rpc.HTTPError); ok {
+		return httpErr.StatusCode()
+	}
+	return http.StatusInternalServerError
+}

blobstore/common/errors/scheduler_sys.go

@@ -0,0 +1,59 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package errors
+
+import (
+	"errors"
+)
+
+const (
+	CodeNotingTodo        = 700
+	CodeDestReplicaBad    = 702
+	CodeOrphanShard       = 703
+	CodeIllegalTask       = 704
+	CodeNoInspect         = 705
+	CodeClusterIDNotMatch = 706
+	CodeRequestLimited    = 707
+)
+
+// common
+var (
+	ErrRequestLimited = Error(CodeRequestLimited)
+)
+
+// scheduler
+var (
+	ErrNoSuchService   = errors.New("no such service")
+	ErrIllegalTaskType = errors.New("illegal task type")
+	ErrCanNotDropped   = errors.New("disk can not dropped")
+
+	// error code
+	ErrNothingTodo = Error(CodeNotingTodo)
+	ErrNoInspect   = Error(CodeNoInspect)
+)
+
+// worker
+var (
+	ErrShardMayBeLost = errors.New("shard may be lost")
+	// error code
+	ErrOrphanShard    = Error(CodeOrphanShard)
+	ErrIllegalTask    = Error(CodeIllegalTask)
+	ErrDestReplicaBad = Error(CodeDestReplicaBad)
+)
+
+//
+var (
+	ErrClusterIDNotMatch = Error(CodeClusterIDNotMatch)
+)

blobstore/common/kafka/monitor.go

@@ -0,0 +1,221 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package kafka
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/Shopify/sarama"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/util/log"
+)
+
+var mockTestKafkaClient sarama.Client
+
+func newKafkaOffsetGauge() *prometheus.GaugeVec {
+	gaugeOpts := prometheus.GaugeOpts{
+		Namespace: "kafka",
+		Subsystem: "topic_partition",
+		Name:      "offset",
+		Help:      "monitor kafka newest oldest and consume offset",
+	}
+	labelNames := []string{"module_name", "cluster_id", "topic", "partition", "type"}
+	kafkaOffsetGaugeVec := prometheus.NewGaugeVec(gaugeOpts, labelNames)
+
+	err := prometheus.Register(kafkaOffsetGaugeVec)
+	if err == nil {
+		return kafkaOffsetGaugeVec
+	}
+	if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+		return are.ExistingCollector.(*prometheus.GaugeVec)
+	}
+	panic(err)
+}
+
+func newKafkaLatencyGauge() *prometheus.GaugeVec {
+	gaugeOpts := prometheus.GaugeOpts{
+		Namespace: "kafka",
+		Subsystem: "topic_partition",
+		Name:      "consume_lag",
+		Help:      "monitor kafka latency",
+	}
+	labelNames := []string{"module_name", "cluster_id", "topic", "partition"}
+	kafkaLatencyGaugeVec := prometheus.NewGaugeVec(gaugeOpts, labelNames)
+
+	err := prometheus.Register(kafkaLatencyGaugeVec)
+	if err == nil {
+		return kafkaLatencyGaugeVec
+	}
+	if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+		return are.ExistingCollector.(*prometheus.GaugeVec)
+	}
+	panic(err)
+}
+
+type offsetMap struct {
+	offsetMap  map[int32]int64
+	OffsetLock sync.RWMutex
+}
+
+func newOffsetMap() *offsetMap {
+	m := make(map[int32]int64)
+	retOffsetMap := offsetMap{offsetMap: m}
+	return &retOffsetMap
+}
+
+func (o *offsetMap) getOffset(pid int32) int64 {
+	o.OffsetLock.RLock()
+	defer o.OffsetLock.RUnlock()
+	if _, ok := o.offsetMap[pid]; ok {
+		return o.offsetMap[pid]
+	}
+	return 0
+}
+
+func (o *offsetMap) setOffset(offset int64, pid int32) {
+	o.OffsetLock.Lock()
+	defer o.OffsetLock.Unlock()
+	o.offsetMap[pid] = offset
+}
+
+type Monitor struct {
+	clusterID                   proto.ClusterID
+	kafkaClient                 sarama.Client
+	topic                       string
+	pids                        []int32
+	newestOffsetMap             *offsetMap
+	oldestOffsetMap             *offsetMap
+	consumeOffsetMap            *offsetMap
+	kafkaOffAcquireIntervalSecs int64
+	offsetGauge                 *prometheus.GaugeVec
+	latencyGauge                *prometheus.GaugeVec
+	moduleName                  string
+}
+
+const DefauleintervalSecs = 60
+
+func NewKafkaMonitor(
+	clusterID proto.ClusterID,
+	moduleName string,
+	brokerHosts []string,
+	topic string,
+	pids []int32,
+	intervalSecs int64) (*Monitor, error,
+) {
+	monitor := Monitor{
+		clusterID:                   clusterID,
+		topic:                       topic,
+		pids:                        pids,
+		newestOffsetMap:             newOffsetMap(),
+		oldestOffsetMap:             newOffsetMap(),
+		consumeOffsetMap:            newOffsetMap(),
+		kafkaOffAcquireIntervalSecs: DefauleintervalSecs,
+		moduleName:                  moduleName,
+	}
+
+	if intervalSecs == 0 {
+		intervalSecs = DefauleintervalSecs
+	}
+	monitor.kafkaOffAcquireIntervalSecs = intervalSecs
+
+	if mockTestKafkaClient != nil {
+		monitor.kafkaClient = mockTestKafkaClient
+	} else {
+		client, err := sarama.NewClient(brokerHosts, nil)
+		if err != nil {
+			return nil, err
+		}
+		monitor.kafkaClient = client
+	}
+
+	monitor.offsetGauge = newKafkaOffsetGauge()
+	monitor.latencyGauge = newKafkaLatencyGauge()
+
+	go monitor.loopAcquireKafkaOffset()
+
+	return &monitor, nil
+}
+
+func (monitor *Monitor) loopAcquireKafkaOffset() {
+	for {
+		for _, pid := range monitor.pids {
+			newestOffset, err := monitor.kafkaClient.GetOffset(monitor.topic, pid, sarama.OffsetNewest)
+			if err != nil {
+				log.Error(fmt.Sprintf("get newest offset fail topic %v pid %v ", monitor.topic, pid))
+				continue
+			}
+			log.Debug("loopAcquireKafkaOffset newestOffset:", newestOffset)
+			monitor.newestOffsetMap.setOffset(newestOffset, pid)
+
+			oldestOffset, err := monitor.kafkaClient.GetOffset(monitor.topic, pid, sarama.OffsetOldest)
+			if err != nil {
+				log.Error(fmt.Sprintf("get oldest offset fail topic %v pid %v ", monitor.topic, pid))
+				continue
+			}
+			log.Debug("loopAcquireKafkaOffset oldestOffset:", oldestOffset)
+			monitor.oldestOffsetMap.setOffset(oldestOffset, pid)
+		}
+
+		monitor.report()
+		time.Sleep(time.Duration(monitor.kafkaOffAcquireIntervalSecs) * time.Second)
+	}
+}
+
+func (monitor *Monitor) report() {
+	for _, pid := range monitor.pids {
+		oldestOffset := monitor.oldestOffsetMap.getOffset(pid)
+		newestOffset := monitor.newestOffsetMap.getOffset(pid)
+		consumeOffset := monitor.consumeOffsetMap.getOffset(pid)
+		latency := newestOffset - consumeOffset - 1 //-1，because the newestOffset is the next message offset
+		if latency < 0 {
+			latency = 0
+		}
+
+		monitor.reportOffsetMetric(pid, string("oldest"), float64(oldestOffset))
+		monitor.reportOffsetMetric(pid, string("newest"), float64(newestOffset))
+		monitor.reportOffsetMetric(pid, string("consume"), float64(consumeOffset))
+		monitor.reportLatencyMetric(pid, float64(latency))
+		log.Debug("Report...")
+	}
+}
+
+func (monitor *Monitor) reportOffsetMetric(pid int32, metricType string, val float64) {
+	labels := prometheus.Labels{
+		"module_name": monitor.moduleName,
+		"cluster_id":  monitor.clusterID.ToString(),
+		"topic":       monitor.topic,
+		"partition":   fmt.Sprintf("%d", pid),
+		"type":        metricType,
+	}
+	monitor.offsetGauge.With(labels).Set(val)
+}
+
+func (monitor *Monitor) reportLatencyMetric(pid int32, val float64) {
+	labels := prometheus.Labels{
+		"module_name": monitor.moduleName,
+		"cluster_id":  monitor.clusterID.ToString(),
+		"topic":       monitor.topic,
+		"partition":   fmt.Sprintf("%d", pid),
+	}
+	monitor.latencyGauge.With(labels).Set(val)
+}
+
+func (monitor *Monitor) SetConsumeOffset(consumerOff int64, pid int32) {
+	monitor.consumeOffsetMap.setOffset(consumerOff, pid)
+}

blobstore/common/kafka/monitor_test.go

@@ -0,0 +1,103 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package kafka
+
+import (
+	"testing"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+
+	"github.com/Shopify/sarama"
+)
+
+type MockKafkaClient struct{}
+
+func (c *MockKafkaClient) Config() *sarama.Config {
+	return nil
+}
+
+func (c *MockKafkaClient) Controller() (*sarama.Broker, error) {
+	return nil, nil
+}
+
+func (c *MockKafkaClient) Brokers() []*sarama.Broker {
+	return nil
+}
+
+func (c *MockKafkaClient) Topics() ([]string, error) {
+	return nil, nil
+}
+
+func (c *MockKafkaClient) Partitions(topic string) ([]int32, error) {
+	return nil, nil
+}
+
+func (c *MockKafkaClient) WritablePartitions(topic string) ([]int32, error) {
+	return nil, nil
+}
+
+func (c *MockKafkaClient) Leader(topic string, partitionID int32) (*sarama.Broker, error) {
+	return nil, nil
+}
+
+func (c *MockKafkaClient) Replicas(topic string, partitionID int32) ([]int32, error) {
+	return nil, nil
+}
+
+func (c *MockKafkaClient) InSyncReplicas(topic string, partitionID int32) ([]int32, error) {
+	return nil, nil
+}
+
+func (c *MockKafkaClient) OfflineReplicas(topic string, partitionID int32) ([]int32, error) {
+	return nil, nil
+}
+
+func (c *MockKafkaClient) RefreshMetadata(topics ...string) error {
+	return nil
+}
+
+func (c *MockKafkaClient) GetOffset(topic string, partitionID int32, time int64) (int64, error) {
+	if time == sarama.OffsetNewest {
+		return 100, nil
+	}
+	return 1, nil
+}
+
+func (c *MockKafkaClient) Coordinator(consumerGroup string) (*sarama.Broker, error) {
+	return nil, nil
+}
+
+func (c *MockKafkaClient) RefreshCoordinator(consumerGroup string) error {
+	return nil
+}
+
+func (c *MockKafkaClient) InitProducerID() (*sarama.InitProducerIDResponse, error) {
+	return nil, nil
+}
+
+func (c *MockKafkaClient) Close() error {
+	return nil
+}
+
+func (c *MockKafkaClient) Closed() bool {
+	return true
+}
+
+func TestSetConsumeOffset(t *testing.T) {
+	brokens := []string{"127.0.01:9092"}
+	mockTestKafkaClient = &MockKafkaClient{}
+	monitor, _ := NewKafkaMonitor(proto.ClusterID(1), "", brokens, "Test_monitor", []int32{0, 1, 2}, 10)
+	monitor.SetConsumeOffset(1, 1)
+}

blobstore/common/kafka/msg_sender.go

@@ -0,0 +1,85 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package kafka
+
+import (
+	"time"
+
+	"github.com/Shopify/sarama"
+
+	"github.com/cubefs/cubefs/blobstore/util/log"
+)
+
+var DefaultKafkaVersion = sarama.V0_10_0_0
+
+type MsgProducer interface {
+	SendMessage(topic string, msg []byte) (err error)
+	SendMessages(topic string, msgs [][]byte) (err error)
+}
+
+type ProducerCfg struct {
+	BrokerList []string `json:"broker_list"`
+	Topic      string   `json:"topic"`
+	TimeoutMs  int64    `json:"timeout_ms"`
+}
+
+type Producer struct {
+	sarama.SyncProducer
+}
+
+func defaultCfg() *sarama.Config {
+	config := sarama.NewConfig()
+	config.Producer.Return.Successes = true
+	config.Producer.RequiredAcks = sarama.WaitForAll
+	config.Version = DefaultKafkaVersion
+	return config
+}
+
+func NewProducer(cfg *ProducerCfg) (*Producer, error) {
+	config := defaultCfg()
+	if cfg.TimeoutMs <= 0 {
+		cfg.TimeoutMs = 1000
+	}
+	config.Producer.Timeout = time.Duration(cfg.TimeoutMs) * time.Millisecond
+
+	producer, err := sarama.NewSyncProducer(cfg.BrokerList, config)
+	if err != nil {
+		log.Error("sarama.NewClient:", err)
+		return nil, err
+	}
+	return &Producer{producer}, err
+}
+
+func (p *Producer) SendMessage(topic string, msg []byte) (err error) {
+	m := &sarama.ProducerMessage{
+		Topic:     topic,
+		Timestamp: time.Now(),
+		Value:     sarama.ByteEncoder(msg),
+	}
+	_, _, err = p.SyncProducer.SendMessage(m)
+	return err
+}
+
+func (p *Producer) SendMessages(topic string, msgs [][]byte) (err error) {
+	sendMsgs := make([]*sarama.ProducerMessage, len(msgs))
+	for idx, msg := range msgs {
+		sendMsgs[idx] = &sarama.ProducerMessage{
+			Topic:     topic,
+			Timestamp: time.Now(),
+			Value:     sarama.ByteEncoder(msg),
+		}
+	}
+	return p.SyncProducer.SendMessages(sendMsgs)
+}

blobstore/common/mongoutil/mongoutil.go

@@ -0,0 +1,110 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package mongoutil
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"time"
+
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"go.mongodb.org/mongo-driver/mongo/readconcern"
+	"go.mongodb.org/mongo-driver/mongo/writeconcern"
+)
+
+// Config for mongo client
+type Config struct {
+	URI          string              `json:"uri"`
+	TimeoutMs    int64               `json:"timeout_ms"`
+	WriteConcern *WriteConcernConfig `json:"write_concern"`
+	ReadConcern  string              `json:"read_concern"`
+}
+
+// for detail：https://docs.mongodb.com/manual/reference/write-concern/。
+type WriteConcernConfig struct {
+	Majority  bool  `json:"majority"`
+	TimeoutMs int64 `json:"timeout_ms"`
+}
+
+var DefaultWriteConfig = WriteConcernConfig{
+	Majority:  true,
+	TimeoutMs: 3000,
+}
+
+const (
+	// ReadConcernLocal mean https://docs.mongodb.com/manual/reference/read-concern-local/#readconcern.%22local%22
+	ReadConcernLocal = "local"
+	// ReadConcernAvailable mean https://docs.mongodb.com/manual/reference/read-concern-available/#readconcern.%22available%22
+	ReadConcernAvailable = "available"
+	// ReadConcernMajority mean https://docs.mongodb.com/manual/reference/read-concern-majority/#readconcern.%22majority%22
+	ReadConcernMajority = "majority"
+	// ReadConcernLinearizable mean https://docs.mongodb.com/manual/reference/read-concern-linearizable/#readconcern.%22linearizable%22
+	ReadConcernLinearizable = "linearizable"
+	// ReadConcernSnapshot mean https://docs.mongodb.com/manual/reference/read-concern-snapshot/#readconcern.%22snapshot%22
+	ReadConcernSnapshot = "snapshot"
+)
+
+func checkValidWriteConcern(s string) error {
+	switch s {
+	case ReadConcernLocal, ReadConcernAvailable, ReadConcernMajority, ReadConcernLinearizable, ReadConcernSnapshot:
+		return nil
+	default:
+		return errors.New("invalid write concern")
+	}
+}
+
+func GetClient(conf Config) (*mongo.Client, error) {
+	opt := options.Client().ApplyURI(conf.URI)
+	if conf.TimeoutMs > 0 {
+		timeoutDur := time.Duration(conf.TimeoutMs) * time.Millisecond
+		opt.SetConnectTimeout(timeoutDur)
+		opt.SetServerSelectionTimeout(timeoutDur)
+		opt.SetSocketTimeout(timeoutDur)
+	}
+	if wcConf := conf.WriteConcern; wcConf != nil {
+		var wcOpts []writeconcern.Option
+		if wcConf.Majority {
+			wcOpts = append(wcOpts, writeconcern.WMajority())
+		}
+		if wcConf.TimeoutMs > 0 {
+			wcOpts = append(wcOpts, writeconcern.WTimeout(time.Duration(wcConf.TimeoutMs)*time.Millisecond))
+		}
+		wc := writeconcern.New(wcOpts...)
+		opt.SetWriteConcern(wc)
+	}
+	if conf.ReadConcern != "" {
+		if err := checkValidWriteConcern(conf.ReadConcern); err != nil {
+			return nil, err
+		}
+		opt.SetReadConcern(readconcern.New(readconcern.Level(conf.ReadConcern)))
+	}
+
+	client, err := mongo.NewClient(opt)
+	if err != nil {
+		return nil, err
+	}
+	err = client.Connect(context.TODO())
+	if err != nil {
+		return nil, err
+	}
+	return client, nil
+}
+
+// IsDupError : err is mongo E11000?。
+func IsDupError(err error) bool {
+	return strings.Contains(err.Error(), "E11000")
+}

blobstore/common/proto/basic.go

@@ -0,0 +1,77 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package proto
+
+import (
+	"encoding/binary"
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+// basic type for all module
+type (
+	DiskID    uint32
+	BlobID    uint64
+	Vid       uint32
+	ClusterID uint32
+)
+
+func (id DiskID) Encode() []byte {
+	key := make([]byte, 4)
+	binary.BigEndian.PutUint32(key, uint32(id))
+	return key
+}
+
+func (id *DiskID) Decode(b []byte) DiskID {
+	key := binary.BigEndian.Uint32(b)
+	*id = DiskID(key)
+	return *id
+}
+
+func (id DiskID) ToString() string {
+	return strconv.FormatUint(uint64(id), 10)
+}
+
+func (vid Vid) ToString() string {
+	return strconv.FormatUint(uint64(vid), 10)
+}
+
+func (id ClusterID) ToString() string {
+	return strconv.FormatUint(uint64(id), 10)
+}
+
+const seqToken = ";"
+
+// EncodeToken encode host and vid to a string token.
+func EncodeToken(host string, vid Vid) (token string) {
+	return fmt.Sprintf("%s%s%s", host, seqToken, strconv.FormatUint(uint64(vid), 10))
+}
+
+// DecodeToken decode host and vid from the token.
+func DecodeToken(token string) (host string, vid Vid, err error) {
+	parts := strings.SplitN(token, seqToken, 2)
+	if len(parts) != 2 {
+		err = fmt.Errorf("invalid token %s", token)
+		return
+	}
+	host = parts[0]
+	vidU32, err := strconv.ParseUint(parts[1], 10, 32)
+	if err != nil {
+		return
+	}
+	vid = Vid(vidU32)
+	return
+}

blobstore/common/proto/basic_test.go

@@ -0,0 +1,66 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package proto_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+func TestProtoDiskID(t *testing.T) {
+	id := proto.DiskID(9)
+	bytes := []byte{0x00, 0x00, 0x00, 0x09}
+	require.Equal(t, uint32(9), uint32(id))
+	require.Equal(t, bytes, id.Encode())
+	require.Equal(t, "9", id.ToString())
+
+	var dec proto.DiskID
+	require.Equal(t, id, dec.Decode(bytes))
+	require.Equal(t, id, dec)
+}
+
+func TestProtoVID(t *testing.T) {
+	id := proto.Vid(123)
+	require.Equal(t, "123", id.ToString())
+}
+
+func TestProtoClusterID(t *testing.T) {
+	id := proto.ClusterID(10)
+	require.Equal(t, "10", id.ToString())
+}
+
+func TestProtoToken(t *testing.T) {
+	host := "127.0.0.1:80"
+	vid := proto.Vid(123)
+	token := "127.0.0.1:80;123"
+	require.Equal(t, token, proto.EncodeToken(host, vid))
+	{
+		newHost, newVid, err := proto.DecodeToken(token)
+		require.NoError(t, err)
+		require.Equal(t, host, newHost)
+		require.Equal(t, vid, newVid)
+	}
+	{
+		_, _, err := proto.DecodeToken(host)
+		require.Error(t, err)
+	}
+	{
+		_, _, err := proto.DecodeToken(token + ";")
+		require.Error(t, err)
+	}
+}

blobstore/common/proto/const.go

@@ -0,0 +1,108 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package proto
+
+import (
+	"math"
+)
+
+// service names
+const (
+	ServiceNameBlobNode  = "BLOBNODE"
+	ServiceNameProxy     = "PROXY"
+	ServiceNameScheduler = "SCHEDULER"
+)
+
+type DiskStatus uint8
+
+// disk status
+const (
+	DiskStatusNormal    = DiskStatus(iota + 1) // 1
+	DiskStatusBroken                           // 2
+	DiskStatusRepairing                        // 3
+	DiskStatusRepaired                         // 4
+	DiskStatusDropped                          // 5
+	DiskStatusMax                              // 6
+)
+
+func (status DiskStatus) IsValid() bool {
+	return status >= DiskStatusNormal && status < DiskStatusMax
+}
+
+func (status DiskStatus) String() string {
+	switch status {
+	case DiskStatusNormal:
+		return "normal"
+	case DiskStatusBroken:
+		return "broken"
+	case DiskStatusRepairing:
+		return "repairing"
+	case DiskStatusRepaired:
+		return "repaired"
+	case DiskStatusDropped:
+		return "dropped"
+	default:
+		return "unknown"
+	}
+}
+
+const (
+	InvalidDiskID = DiskID(0)
+	InValidBlobID = BlobID(0)
+	InvalidCrc32  = uint32(0)
+	InvalidVid    = Vid(0)
+	InvalidVuid   = Vuid(0)
+)
+
+const (
+	MaxBlobID = BlobID(math.MaxUint64)
+)
+
+// volume status
+type VolumeStatus uint8
+
+func (status VolumeStatus) IsValid() bool {
+	return status > volumeStatusMin && status < volumeStatusMax
+}
+
+func (status VolumeStatus) String() string {
+	switch status {
+	case VolumeStatusIdle:
+		return "idle"
+	case VolumeStatusActive:
+		return "active"
+	case VolumeStatusLock:
+		return "lock"
+	case VolumeStatusUnlocking:
+		return "unlocking"
+	}
+	return "unknown"
+}
+
+const (
+	volumeStatusMin = VolumeStatus(iota)
+	VolumeStatusIdle
+	VolumeStatusActive
+	VolumeStatusLock
+	VolumeStatusUnlocking
+	volumeStatusMax
+)
+
+// config key
+const (
+	CodeModeConfigKey    = "code_mode"
+	VolumeReserveSizeKey = "volume_reserve_size"
+	VolumeChunkSizeKey   = "volume_chunk_size"
+)

blobstore/common/proto/const_test.go

@@ -0,0 +1,43 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package proto_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+func TestProtoDiskStatus(t *testing.T) {
+	for st := proto.DiskStatusNormal; st < proto.DiskStatusMax; st++ {
+		require.True(t, st.IsValid())
+		t.Logf("disk st %d -> %s", st, st)
+	}
+	st := proto.DiskStatus(0xff)
+	require.False(t, st.IsValid())
+	t.Logf("disk st %d -> %s", st, st)
+}
+
+func TestProtoVolumeStatus(t *testing.T) {
+	for st := proto.VolumeStatusIdle; st <= proto.VolumeStatusUnlocking; st++ {
+		require.True(t, st.IsValid())
+		t.Logf("volume st %d -> %s", st, st)
+	}
+	st := proto.VolumeStatus(0xff)
+	require.False(t, st.IsValid())
+	t.Logf("volume st %d -> %s", st, st)
+}

blobstore/common/proto/mqproxy_types.go

@@ -0,0 +1,103 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package proto
+
+import (
+	"github.com/cubefs/cubefs/blobstore/util/errors"
+)
+
+var ErrInvalidMsg = errors.New("msg is invalid")
+
+type DeleteStage byte
+
+const (
+	InitStage DeleteStage = iota
+	MarkDelStage
+	DelStage
+)
+
+type BlobDeleteStage struct {
+	Stages map[uint8]DeleteStage `json:"stages"`
+}
+
+func (s *BlobDeleteStage) SetStage(vuidIdx uint8, stage DeleteStage) {
+	if s.Stages == nil {
+		s.Stages = make(map[uint8]DeleteStage)
+	}
+	s.Stages[vuidIdx] = stage
+}
+
+func (s *BlobDeleteStage) Stage(vuid Vuid) (DeleteStage, bool) {
+	stage, exist := s.Stages[vuid.Index()]
+	return stage, exist
+}
+
+func (s *BlobDeleteStage) Copy() BlobDeleteStage {
+	myCopy := BlobDeleteStage{}
+	myCopy.Stages = make(map[uint8]DeleteStage)
+	for k, v := range s.Stages {
+		myCopy.Stages[k] = v
+	}
+	return myCopy
+}
+
+type DeleteMsg struct {
+	ClusterID     ClusterID       `json:"cluster_id"`
+	Bid           BlobID          `json:"bid"`
+	Vid           Vid             `json:"vid"`
+	Retry         int             `json:"retry"`
+	Time          int64           `json:"time"`
+	ReqId         string          `json:"req_id"`
+	BlobDelStages BlobDeleteStage `json:"blob_del_stages"`
+}
+
+func (msg *DeleteMsg) IsValid() bool {
+	if msg.Bid == InValidBlobID {
+		return false
+	}
+	if msg.Vid == InvalidVid {
+		return false
+	}
+	return true
+}
+
+func (msg *DeleteMsg) SetDeleteStage(stage BlobDeleteStage) {
+	for idx, s := range stage.Stages {
+		msg.BlobDelStages.SetStage(idx, s)
+	}
+}
+
+type ShardRepairMsg struct {
+	ClusterID ClusterID `json:"cluster_id"`
+	Bid       BlobID    `json:"bid"`
+	Vid       Vid       `json:"vid"`
+	BadIdx    []uint8   `json:"bad_idx"`
+	Retry     int       `json:"retry"`
+	Reason    string    `json:"reason"`
+	ReqId     string    `json:"req_id"`
+}
+
+func (msg *ShardRepairMsg) IsValid() bool {
+	if msg.Bid == InValidBlobID {
+		return false
+	}
+	if msg.Vid == InvalidVid {
+		return false
+	}
+	if len(msg.BadIdx) == 0 {
+		return false
+	}
+	return true
+}

blobstore/common/proto/mqproxy_types_test.go

@@ -0,0 +1,114 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package proto
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestShardRepairMsg_IsValid(t *testing.T) {
+	msg := ShardRepairMsg{
+		ClusterID: 1,
+		Bid:       1,
+		Vid:       1,
+		BadIdx:    []uint8{1},
+		Retry:     0,
+		Reason:    "access",
+	}
+	require.Equal(t, true, msg.IsValid())
+
+	msg = ShardRepairMsg{
+		ClusterID: 1,
+		Bid:       0,
+		Vid:       1,
+		BadIdx:    []uint8{1},
+		Retry:     0,
+		Reason:    "access",
+	}
+	require.Equal(t, false, msg.IsValid())
+
+	msg = ShardRepairMsg{
+		ClusterID: 1,
+		Bid:       1,
+		Vid:       0,
+		BadIdx:    []uint8{1},
+		Retry:     0,
+		Reason:    "access",
+	}
+	require.Equal(t, false, msg.IsValid())
+
+	msg = ShardRepairMsg{
+		ClusterID: 1,
+		Bid:       1,
+		Vid:       1,
+		BadIdx:    []uint8{},
+		Retry:     0,
+		Reason:    "access",
+	}
+	require.Equal(t, false, msg.IsValid())
+}
+
+func TestDeleteMsg_IsValid(t *testing.T) {
+	msg := DeleteMsg{
+		ClusterID: 1,
+		Bid:       1,
+		Vid:       1,
+		Retry:     0,
+		Time:      0,
+	}
+	require.Equal(t, true, msg.IsValid())
+
+	msg = DeleteMsg{
+		ClusterID: 1,
+		Bid:       0,
+		Vid:       1,
+		Retry:     0,
+		Time:      0,
+	}
+	require.Equal(t, false, msg.IsValid())
+
+	msg = DeleteMsg{
+		ClusterID: 1,
+		Bid:       1,
+		Vid:       0,
+		Retry:     0,
+		Time:      0,
+	}
+	require.Equal(t, false, msg.IsValid())
+}
+
+func TestMsgMarshal(t *testing.T) {
+	stags := BlobDeleteStage{}
+	stags.SetStage(1, 1)
+	stags.SetStage(2, 1)
+	msg := DeleteMsg{
+		ClusterID:     1,
+		Bid:           1,
+		Vid:           0,
+		Retry:         0,
+		Time:          0,
+		BlobDelStages: stags,
+	}
+	b, err := json.Marshal(msg)
+	require.NoError(t, err)
+
+	var delMsg DeleteMsg
+	err = json.Unmarshal(b, &delMsg)
+	require.NoError(t, err)
+	t.Logf("del msg %+v", delMsg)
+}

blobstore/common/proto/scheduler_types.go

@@ -0,0 +1,269 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package proto
+
+import (
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	"github.com/cubefs/cubefs/blobstore/util/errors"
+)
+
+var (
+	ErrTaskPaused = errors.New("task has paused")
+	ErrTaskEmpty  = errors.New("no task to run")
+)
+
+const (
+	// TaskRenewalPeriodS + RenewalTimeoutS < TaskLeaseExpiredS
+	TaskRenewalPeriodS = 5  // worker alive tasks  renewal period
+	RenewalTimeoutS    = 1  // timeout of worker task renewal
+	TaskLeaseExpiredS  = 10 // task lease duration in scheduler
+)
+
+type VunitLocation struct {
+	Vuid   Vuid   `json:"vuid" bson:"vuid"`
+	Host   string `json:"host" bson:"host"`
+	DiskID DiskID `json:"disk_id" bson:"disk_id"`
+}
+
+// for task check
+func CheckVunitLocations(locations []VunitLocation) bool {
+	if len(locations) == 0 {
+		return false
+	}
+
+	for _, l := range locations {
+		if l.Vuid == InvalidVuid || l.Host == "" || l.DiskID == InvalidDiskID {
+			return false
+		}
+	}
+	return true
+}
+
+const (
+	RepairTaskType    = "repair_task"
+	BalanceTaskType   = "balance_task"
+	DiskDropTaskType  = "disk_drop_task"
+	ManualMigrateType = "manual_migrate"
+)
+
+var _taskType = map[string]struct{}{
+	RepairTaskType:    {},
+	BalanceTaskType:   {},
+	DiskDropTaskType:  {},
+	ManualMigrateType: {},
+}
+
+func ValidTaskType(task string) bool {
+	_, ok := _taskType[task]
+	return ok
+}
+
+type RepairState uint8
+
+const (
+	RepairStateInited RepairState = iota + 1
+	RepairStatePrepared
+	RepairStateWorkCompleted
+	RepairStateFinished
+	RepairStateFinishedInAdvance
+)
+
+const (
+	BrokenDiskTrigger   = 0
+	BrokenStripeTrigger = 1
+)
+
+type VolRepairTask struct {
+	TaskID        string      `json:"task_id" bson:"_id"`
+	State         RepairState `json:"state" bson:"state"`
+	WorkerRedoCnt uint8       `json:"worker_redo_cnt" bson:"worker_redo_cnt"`
+	RepairDiskID  DiskID      `json:"repair_disk_id" bson:"repair_disk_id"`
+
+	CodeMode    codemode.CodeMode `json:"code_mode" bson:"code_mode"`
+	Sources     []VunitLocation   `json:"sources" bson:"sources"` // include all replicas of volumes
+	Destination VunitLocation     `json:"destination" bson:"destination"`
+
+	BadVuid Vuid  `json:"bad_vuid"`
+	BadIdx  uint8 `json:"bad_idx" bson:"bad_idx"` // index of repair replica in volume replicas
+
+	BrokenDiskIDC string `json:"broken_disk_idc"`
+
+	Ctime string `json:"ctime" bson:"ctime"` // task create time
+	MTime string `json:"mtime" bson:"mtime"` // task modify time
+
+	// BrokenDiskTrigger: trigger by broken disk,
+	// BrokenStripeTrigger: trigger by stripe which has broken replica
+	TriggerBy int `json:"trigger_by" bson:"trigger_by"`
+}
+
+func (t *VolRepairTask) GetSrc() []VunitLocation {
+	return t.Sources
+}
+
+func (t *VolRepairTask) GetDest() VunitLocation {
+	return t.Destination
+}
+
+func (t *VolRepairTask) SetDest(dst VunitLocation) {
+	t.Destination = dst
+}
+
+func (t *VolRepairTask) NewDiskId() DiskID {
+	return t.Destination.DiskID
+}
+
+func (t *VolRepairTask) Vid() Vid {
+	return t.BadVuid.Vid()
+}
+
+func (t *VolRepairTask) RepairVuid() Vuid {
+	return t.BadVuid
+}
+
+func (t *VolRepairTask) Finished() bool {
+	return t.State == RepairStateFinished || t.State == RepairStateFinishedInAdvance
+}
+
+func (t *VolRepairTask) Running() bool {
+	return t.State == RepairStatePrepared || t.State == RepairStateWorkCompleted
+}
+
+func (t *VolRepairTask) Copy() *VolRepairTask {
+	task := &VolRepairTask{}
+	*task = *t
+	dst := make([]VunitLocation, len(t.Sources))
+	copy(dst, t.Sources)
+	task.Sources = dst
+	return task
+}
+
+type MigrateState uint8
+
+const (
+	MigrateStateInited MigrateState = iota + 1
+	MigrateStatePrepared
+	MigrateStateWorkCompleted
+	MigrateStateFinished
+	MigrateStateFinishedInAdvance
+)
+
+type MigrateTask struct {
+	TaskID        string       `json:"task_id" bson:"_id"`                     // task id
+	State         MigrateState `json:"state" bson:"state"`                     // task state
+	WorkerRedoCnt uint8        `json:"worker_redo_cnt" bson:"worker_redo_cnt"` // worker redo task count
+
+	SourceIdc    string `json:"source_idc" bson:"source_idc"`         // source idc
+	SourceDiskID DiskID `json:"source_disk_id" bson:"source_disk_id"` // source disk id
+	SourceVuid   Vuid   `json:"source_vuid" bson:"source_vuid"`       // source volume unit id
+
+	Sources  []VunitLocation   `json:"sources" bson:"sources"`     // source volume units location
+	CodeMode codemode.CodeMode `json:"code_mode" bson:"code_mode"` // codemode
+
+	Destination VunitLocation `json:"destination" bson:"destination"` // destination volume unit location
+
+	Ctime string `json:"ctime" bson:"ctime"` // create time
+	MTime string `json:"mtime" bson:"mtime"` // modify time
+
+	FinishAdvanceReason string `json:"finish_advance_reason" bson:"finish_advance_reason"`
+	// task migrate chunk direct download first,if fail will recover chunk by ec repair
+	ForbiddenDirectDownload bool `json:"forbidden_direct_download" bson:"forbidden_direct_download"`
+}
+
+func (t *MigrateTask) GetSrc() []VunitLocation {
+	return t.Sources
+}
+
+func (t *MigrateTask) GetDest() VunitLocation {
+	return t.Destination
+}
+
+func (t *MigrateTask) SetDest(dest VunitLocation) {
+	t.Destination = dest
+}
+
+func (t *MigrateTask) DestinationDiskId() DiskID {
+	return t.Destination.DiskID
+}
+
+func (t *MigrateTask) Running() bool {
+	return t.State == MigrateStatePrepared || t.State == MigrateStateWorkCompleted
+}
+
+func (t *MigrateTask) Finished() bool {
+	return t.State == MigrateStateFinished || t.State == MigrateStateFinishedInAdvance
+}
+
+func (t *MigrateTask) Copy() *MigrateTask {
+	task := &MigrateTask{}
+	*task = *t
+	dst := make([]VunitLocation, len(t.Sources))
+	copy(dst, t.Sources)
+	task.Sources = dst
+	return task
+}
+
+func (t *MigrateTask) SrcMigDiskID() DiskID {
+	return t.SourceDiskID
+}
+
+type InspectCheckPoint struct {
+	Id       string `json:"_id" bson:"_id"`
+	StartVid Vid    `json:"start_vid" bson:"start_vid"` // min vid in current batch volumes
+	Ctime    string `json:"ctime" bson:"ctime"`
+}
+
+type InspectTask struct {
+	TaskId   string            `json:"task_id"`
+	Mode     codemode.CodeMode `json:"mode"`
+	Replicas []VunitLocation   `json:"replicas"`
+}
+
+type MissedShard struct {
+	Vuid Vuid   `json:"vuid"`
+	Bid  BlobID `json:"bid"`
+}
+
+type InspectRet struct {
+	TaskID        string         `json:"task_id"`
+	InspectErrStr string         `json:"inspect_err_str"` // inspect run success or not
+	MissedShards  []*MissedShard `json:"missed_shards"`
+}
+
+func (inspect *InspectRet) Err() error {
+	if len(inspect.InspectErrStr) == 0 {
+		return nil
+	}
+	return errors.New(inspect.InspectErrStr)
+}
+
+// ArchiveRecord archive record
+type ArchiveRecord struct {
+	TaskID      string      `bson:"_id"`
+	TaskType    string      `bson:"task_type"`
+	ArchiveTime string      `bson:"archive_time"`
+	Content     interface{} `bson:"content"`
+}
+
+type ShardRepairTask struct {
+	Bid      BlobID            `json:"bid"`
+	CodeMode codemode.CodeMode `json:"code_mode"`
+	Sources  []VunitLocation   `json:"sources"`
+	BadIdxs  []uint8           `json:"bad_idxs"` // TODO: BadIdxes
+	Reason   string            `json:"reason"`
+}
+
+func (task *ShardRepairTask) IsValid() bool {
+	return task.CodeMode.IsValid() && CheckVunitLocations(task.Sources)
+}

blobstore/common/proto/vuid.go

@@ -0,0 +1,93 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package proto
+
+import (
+	"errors"
+	"strconv"
+)
+
+type (
+	Vuid       uint64
+	VuidPrefix uint64
+)
+
+const (
+	MinEpoch = 1
+	MaxEpoch = 16777215
+	MinIndex = 0
+	MaxIndex = 255
+)
+
+func (vu Vuid) IsValid() bool {
+	return vu > InvalidVuid && IsValidEpoch(vu.Epoch()) && IsValidIndex(vu.Index())
+}
+
+func NewVuid(vid Vid, idx uint8, epoch uint32) (Vuid, error) {
+	if !IsValidEpoch(epoch) {
+		err := errors.New("fail to new vuid,Epoch is overflow")
+		return 0, err
+	}
+
+	u64 := uint64(vid)<<32 + uint64(idx)<<24 + uint64(epoch)
+	return Vuid(u64), nil
+}
+
+func EncodeVuidPrefix(vid Vid, idx uint8) VuidPrefix {
+	u64 := uint64(vid)<<32 + uint64(idx)<<24
+	return VuidPrefix(u64)
+}
+
+func EncodeVuid(v VuidPrefix, epoch uint32) Vuid {
+	u64 := uint64(v) + uint64(epoch)
+	return Vuid(u64)
+}
+
+func (v Vuid) Vid() Vid {
+	return Vid(v & 0xffffffff00000000 >> 32)
+}
+
+func (v Vuid) ToString() string {
+	return strconv.FormatUint(uint64(v), 10)
+}
+
+func (v Vuid) Index() uint8 {
+	return uint8(v & 0xff000000 >> 24)
+}
+
+func (v Vuid) Epoch() uint32 {
+	return uint32(v & 0xffffff)
+}
+
+func (v Vuid) VuidPrefix() VuidPrefix {
+	vuidPre := uint64(v) - uint64(v.Epoch())
+	return VuidPrefix(vuidPre)
+}
+
+func (v VuidPrefix) Vid() Vid {
+	return Vid(v & 0xffffffff00000000 >> 32)
+}
+
+func (v VuidPrefix) Index() uint8 {
+	return uint8(v & 0xff000000 >> 24)
+}
+
+func IsValidEpoch(epoch uint32) bool {
+	return epoch <= MaxEpoch && epoch >= MinEpoch
+}
+
+func IsValidIndex(index uint8) bool {
+	return index <= MaxIndex && index >= MinIndex
+}

blobstore/common/proto/vuid_test.go

@@ -0,0 +1,51 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package proto
+
+import (
+	"math/rand"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestVuid(t *testing.T) {
+	vid := Vid(rand.Uint32())
+	index := uint8(rand.Intn(256))
+	epoch := uint32(rand.Int31n(MaxEpoch))
+	vuid, err := NewVuid(vid, index, epoch)
+	vuidPre := EncodeVuidPrefix(vid, index)
+	assert.NoError(t, err)
+
+	assert.Equal(t, vuidPre, vuid.VuidPrefix())
+	assert.Equal(t, vid, vuidPre.Vid())
+	assert.Equal(t, index, vuidPre.Index())
+	assert.Equal(t, vid, vuid.Vid())
+	assert.Equal(t, index, vuid.Index())
+	assert.Equal(t, epoch, vuid.Epoch())
+}
+
+func TestDecodeVuid(t *testing.T) {
+	for i := 0; i < 900000; i++ {
+		TestVuid(t)
+	}
+}
+
+func TestDecodeVuid2(t *testing.T) {
+	old := Vuid(425335980033)
+	new := Vuid(116131889167)
+	t.Log(old.Vid(), old.VuidPrefix(), old.Index(), old.Epoch())
+	t.Log(new.Vid(), new.VuidPrefix(), new.Index(), new.Epoch())
+}

blobstore/common/proto/worker_types.go

@@ -0,0 +1,43 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package proto
+
+// stats task has done in worker
+type TaskStatistics struct {
+	MigDataSizeByte   uint64 `json:"mig_data_size_byte"`
+	MigShardCnt       uint64 `json:"mig_shard_cnt"`
+	TotalDataSizeByte uint64 `json:"total_data_size_byte"`
+	TotalShardCnt     uint64 `json:"total_shard_cnt"`
+	Progress          uint64 `json:"progress"`
+}
+
+func (self *TaskStatistics) Add(dataSize, shardCnt uint64) {
+	self.MigDataSizeByte += dataSize
+	self.MigShardCnt += shardCnt
+	if self.TotalDataSizeByte == 0 {
+		self.Progress = 100
+	} else {
+		self.Progress = (self.MigDataSizeByte * 100) / self.TotalDataSizeByte
+	}
+}
+
+func (self *TaskStatistics) InitTotal(totalDataSize, totalShardCnt uint64) {
+	self.TotalDataSizeByte = totalDataSize
+	self.TotalShardCnt = totalShardCnt
+}
+
+func (self *TaskStatistics) Completed() bool {
+	return self.Progress == 100
+}

blobstore/common/taskswitch/task_switch.go

@@ -0,0 +1,180 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package taskswitch
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"time"
+
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+)
+
+type ISwitcher interface {
+	Enabled() bool
+	WaitEnable()
+}
+
+// task switch name
+const (
+	DiskRepairSwitchName    = "disk_repair"
+	BalanceSwitchName       = "balance"
+	DiskDropSwitchName      = "disk_drop"
+	BlobDeleteSwitchName    = "blob_delete"
+	ShardRepairSwitchName   = "shard_repair"
+	VolumeInspectSwitchName = "volume_inspect"
+)
+
+const (
+	syncTaskStatusIntervalS = 15
+	SwitchOpen              = "true"
+	SwitchClose             = "false"
+)
+
+var (
+	ErrConflictSwitch = errors.New("switch has existed")
+	ErrNoSuchSwitch   = errors.New("no such switch")
+)
+
+type TaskSwitch struct {
+	mu      sync.Mutex
+	enabled bool
+	wg      sync.WaitGroup
+}
+
+func newTaskSwitch() *TaskSwitch {
+	c := &TaskSwitch{
+		enabled: true,
+	}
+	c.Disable()
+	return c
+}
+
+func NewEnabledTaskSwitch() *TaskSwitch {
+	taskSwitch := newTaskSwitch()
+	taskSwitch.Enable()
+	return taskSwitch
+}
+
+func (s *TaskSwitch) Enable() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.enabled {
+		return
+	}
+	s.enabled = true
+	s.wg.Done()
+}
+
+func (s *TaskSwitch) Disable() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if !s.enabled {
+		return
+	}
+	s.enabled = false
+	s.wg.Add(1)
+}
+
+func (s *TaskSwitch) Enabled() bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.enabled
+}
+
+func (s *TaskSwitch) WaitEnable() {
+	s.wg.Wait()
+}
+
+type ConfigGetter interface {
+	GetConfig(ctx context.Context, key string) (val string, err error)
+}
+
+type SwitchMgr struct {
+	switchs     map[string]*TaskSwitch
+	mu          sync.Mutex
+	cmCfgGetter ConfigGetter
+}
+
+func NewSwitchMgr(cmCli ConfigGetter) *SwitchMgr {
+	sm := SwitchMgr{
+		switchs:     make(map[string]*TaskSwitch),
+		cmCfgGetter: cmCli,
+	}
+	go sm.loopUpdate()
+	return &sm
+}
+
+func (sm *SwitchMgr) loopUpdate() {
+	for {
+		sm.update()
+		time.Sleep(syncTaskStatusIntervalS * time.Second)
+	}
+}
+
+func (sm *SwitchMgr) update() {
+	sm.mu.Lock()
+	defer sm.mu.Unlock()
+
+	span, ctx := trace.StartSpanFromContext(context.Background(), "")
+
+	for switchName, taskSwitch := range sm.switchs {
+		statusStr, err := sm.cmCfgGetter.GetConfig(ctx, switchName)
+		if err != nil {
+			span.Errorf("Get Fail switchName %s err %v", switchName, err)
+			continue
+		}
+
+		if switchStatus(statusStr) {
+			taskSwitch.Enable()
+			continue
+		}
+		taskSwitch.Disable()
+	}
+}
+
+func (sm *SwitchMgr) AddSwitch(switchName string) (*TaskSwitch, error) {
+	sm.mu.Lock()
+	defer sm.mu.Unlock()
+
+	if _, ok := sm.switchs[switchName]; ok {
+		return nil, ErrConflictSwitch
+	}
+	sm.switchs[switchName] = newTaskSwitch()
+	return sm.switchs[switchName], nil
+}
+
+func (sm *SwitchMgr) DelSwitch(switchName string) error {
+	sm.mu.Lock()
+	defer sm.mu.Unlock()
+
+	if _, ok := sm.switchs[switchName]; ok {
+		delete(sm.switchs, switchName)
+		return nil
+	}
+	return ErrNoSuchSwitch
+}
+
+func switchStatus(statusStr string) (open bool) {
+	switch statusStr {
+	case SwitchOpen:
+		return true
+	case SwitchClose:
+		return false
+	default:
+		return false
+	}
+}

blobstore/common/taskswitch/task_switch_test.go

@@ -0,0 +1,67 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package taskswitch
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestTaskSwitch(t *testing.T) {
+	ts := newTaskSwitch()
+	require.Equal(t, false, ts.Enabled())
+	ts.Enable()
+	require.Equal(t, true, ts.Enabled())
+	ts.Disable()
+	require.Equal(t, false, ts.Enabled())
+}
+
+type mockCfgGetter struct {
+	m map[string]string
+}
+
+func (cfgGetter *mockCfgGetter) GetConfig(ctx context.Context, key string) (val string, err error) {
+	if val, ok := cfgGetter.m[key]; ok {
+		return val, nil
+	}
+	return "", errors.New("no such key")
+}
+
+func TestSwitchMgr(t *testing.T) {
+	cfgGetter := mockCfgGetter{
+		m: make(map[string]string),
+	}
+	cfgGetter.m["switch1"] = SwitchOpen
+	cfgGetter.m["switch2"] = SwitchClose
+	sm := NewSwitchMgr(&cfgGetter)
+	s1, err := sm.AddSwitch("switch1")
+	require.NoError(t, err)
+	s2, err := sm.AddSwitch("switch2")
+	require.NoError(t, err)
+
+	sm.update()
+	require.Equal(t, true, s1.Enabled())
+	require.Equal(t, false, s2.Enabled())
+
+	sm.update()
+	err = sm.DelSwitch("switch1")
+	require.NoError(t, err)
+	err = sm.DelSwitch("switch2")
+	require.NoError(t, err)
+	require.Equal(t, 0, len(sm.switchs))
+}

blobstore/common/trace/ext/tags.go

@@ -0,0 +1,66 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package ext
+
+import (
+	"github.com/opentracing/opentracing-go/ext"
+)
+
+var (
+	// SpanKind (client/server or producer/consumer)
+	SpanKind              = ext.SpanKind
+	SpanKindRPCClientEnum = ext.SpanKindRPCClientEnum
+	SpanKindRPCClient     = ext.SpanKindRPCClient
+	SpanKindRPCServerEnum = ext.SpanKindRPCServerEnum
+	SpanKindRPCServer     = ext.SpanKindRPCServer
+	SpanKindProducerEnum  = ext.SpanKindProducerEnum
+	SpanKindProducer      = ext.SpanKindProducer
+	SpanKindConsumerEnum  = ext.SpanKindConsumerEnum
+	SpanKindConsumer      = ext.SpanKindConsumer
+
+	// Component name
+	Component = ext.Component
+
+	// Sampling hint
+	SamplingPriority = ext.SamplingPriority
+
+	// Peer tags
+	PeerService  = ext.PeerService
+	PeerAddress  = ext.PeerAddress
+	PeerHostname = ext.PeerHostname
+	PeerHostIPv4 = ext.PeerHostIPv4
+	PeerHostIPv6 = ext.PeerHostIPv6
+	PeerPort     = ext.PeerPort
+
+	// HTTP tags
+	HTTPUrl        = ext.HTTPUrl
+	HTTPMethod     = ext.HTTPMethod
+	HTTPStatusCode = ext.HTTPStatusCode
+
+	// DB tags
+	DBInstance  = ext.DBInstance
+	DBStatement = ext.DBStatement
+	DBType      = ext.DBType
+	DBUser      = ext.DBUser
+
+	// Message Bus Tag
+	MessageBusDestination = ext.MessageBusDestination
+
+	// Error Tag
+	Error = ext.Error
+)
+
+// SpanKindEnum represents common span types
+type SpanKindEnum ext.SpanKindEnum

blobstore/common/trace/ext/tags_test.go

@@ -0,0 +1,141 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package ext
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+)
+
+func TestPeerTags(t *testing.T) {
+	tracer := trace.NewTracer("blobstore")
+	span := tracer.StartSpan("my-trace").(trace.Span)
+	PeerService.Set(span, "my-service")
+	PeerAddress.Set(span, "my-hostname:8080")
+	PeerHostname.Set(span, "my-hostname")
+	PeerHostIPv4.Set(span, uint32(127<<24|1))
+	PeerHostIPv4.SetString(span, "127.0.0.1")
+	PeerHostIPv6.Set(span, "::")
+	PeerPort.Set(span, uint16(8080))
+	SamplingPriority.Set(span, uint16(1))
+	SpanKind.Set(span, SpanKindRPCServerEnum)
+	SpanKindRPCClient.Set(span)
+	span.Finish()
+
+	assert.Equal(t, trace.Tags{
+		"peer.service":      "my-service",
+		"peer.address":      "my-hostname:8080",
+		"peer.hostname":     "my-hostname",
+		"peer.ipv4":         "127.0.0.1",
+		"peer.ipv6":         "::",
+		"peer.port":         uint16(8080),
+		"sampling.priority": uint16(1),
+		"span.kind":         SpanKindRPCClientEnum,
+	}, span.Tags())
+}
+
+func TestHTTPTags(t *testing.T) {
+	tracer := trace.NewTracer("blobstore")
+	span := tracer.StartSpan("my-trace", SpanKindRPCServer).(trace.Span)
+	HTTPUrl.Set(span, "test.biz/uri?protocol=false")
+	HTTPMethod.Set(span, "GET")
+	HTTPStatusCode.Set(span, 301)
+	span.Finish()
+
+	assert.Equal(t, trace.Tags{
+		"http.url":         "test.biz/uri?protocol=false",
+		"http.method":      "GET",
+		"http.status_code": uint16(301),
+		"span.kind":        SpanKindRPCServerEnum,
+	}, span.Tags())
+}
+
+func TestDBTags(t *testing.T) {
+	tracer := trace.NewTracer("blobstore")
+	span := tracer.StartSpan("my-trace", SpanKindRPCClient).(trace.Span)
+	DBInstance.Set(span, "127.0.0.1:3306/customers")
+	DBStatement.Set(span, "SELECT * FROM user_table")
+	DBType.Set(span, "sql")
+	DBUser.Set(span, "customer_user")
+	span.Finish()
+
+	assert.Equal(t, trace.Tags{
+		"db.instance":  "127.0.0.1:3306/customers",
+		"db.statement": "SELECT * FROM user_table",
+		"db.type":      "sql",
+		"db.user":      "customer_user",
+		"span.kind":    SpanKindRPCClientEnum,
+	}, span.Tags())
+}
+
+func TestMiscTags(t *testing.T) {
+	tracer := trace.NewTracer("blobstore")
+	span := tracer.StartSpan("my-trace").(trace.Span)
+	Component.Set(span, "my-awesome-library")
+	SamplingPriority.Set(span, 1)
+	Error.Set(span, true)
+
+	span.Finish()
+
+	assert.Equal(t, trace.Tags{
+		"component":         "my-awesome-library",
+		"sampling.priority": uint16(1),
+		"error":             true,
+	}, span.Tags())
+}
+
+func TestRPCServerOption(t *testing.T) {
+	tracer := trace.NewTracer("blobstore")
+	parent := tracer.StartSpan("my-trace")
+	parent.SetBaggageItem("bag", "gage")
+
+	carrier := trace.HTTPHeadersCarrier{}
+	err := tracer.Inject(parent.Context(), trace.HTTPHeaders, carrier)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = tracer.Extract(trace.HTTPHeaders, carrier)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestMessageBusProducerTags(t *testing.T) {
+	tracer := trace.NewTracer("blobstore")
+	span := tracer.StartSpan("my-trace", SpanKindProducer).(trace.Span)
+	MessageBusDestination.Set(span, "topic name")
+	span.Finish()
+
+	assert.Equal(t, trace.Tags{
+		"message_bus.destination": "topic name",
+		"span.kind":               SpanKindProducerEnum,
+	}, span.Tags())
+}
+
+func TestMessageBusConsumerTags(t *testing.T) {
+	tracer := trace.NewTracer("blobstore")
+	span := tracer.StartSpan("my-trace", SpanKindConsumer).(trace.Span)
+	MessageBusDestination.Set(span, "topic name")
+	span.Finish()
+
+	assert.Equal(t, trace.Tags{
+		"message_bus.destination": "topic name",
+		"span.kind":               SpanKindConsumerEnum,
+	}, span.Tags())
+}

blobstore/common/trace/propagation.go

@@ -0,0 +1,143 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package trace
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/opentracing/opentracing-go"
+)
+
+const (
+	prefixTracer  = "blobstore-tracer-"
+	prefixBaggage = "blobstore-baggage-"
+
+	tracerFieldCount = 2
+	fieldKeyTraceID  = prefixTracer + "traceid"
+	fieldKeySpanID   = prefixTracer + "spanid"
+)
+
+var (
+	// ErrUnsupportedFormat is the alias of opentracing.ErrUnsupportedFormat.
+	ErrUnsupportedFormat = opentracing.ErrUnsupportedFormat
+
+	// ErrSpanContextNotFound is the alias of opentracing.ErrSpanContextNotFound.
+	ErrSpanContextNotFound = opentracing.ErrSpanContextNotFound
+
+	// ErrInvalidSpanContext is the alias of opentracing.ErrInvalidSpanContext.
+	ErrInvalidSpanContext = opentracing.ErrInvalidSpanContext
+
+	// ErrInvalidCarrier is the alias of opentracing.ErrInvalidCarrier.
+	ErrInvalidCarrier = opentracing.ErrInvalidCarrier
+
+	// ErrSpanContextCorrupted is the alias of opentracing.ErrSpanContextCorrupted.
+	ErrSpanContextCorrupted = opentracing.ErrSpanContextCorrupted
+)
+
+const (
+	// Binary is the alias of opentracing.Binary.
+	Binary = opentracing.Binary
+
+	// TextMap is the alias of opentracing.TextMap.
+	TextMap = opentracing.TextMap
+
+	// HTTPHeaders is the alias of opentracing.HTTPHeaders.
+	HTTPHeaders = opentracing.HTTPHeaders
+)
+
+// TextMapCarrier is the alias of opentracing.TextMapCarrier.
+type TextMapCarrier = opentracing.TextMapCarrier
+
+// HTTPHeadersCarrier is the alias of opentracing.HTTPHeadersCarrier.
+type HTTPHeadersCarrier = opentracing.HTTPHeadersCarrier
+
+// TextMapPropagator is a combined Injector and Extractor for TextMap format.
+type TextMapPropagator struct{}
+
+var defaultTexMapPropagator = TextMapPropagator{}
+
+// Inject implements Injector of TextMapPropagator
+func (t *TextMapPropagator) Inject(sc *SpanContext, carrier interface{}) error {
+	writer, ok := carrier.(opentracing.TextMapWriter)
+	if !ok {
+		return ErrInvalidCarrier
+	}
+	writer.Set(fieldKeyTraceID, sc.traceID)
+	writer.Set(fieldKeySpanID, sc.spanID.String())
+
+	sc.ForeachBaggageItems(func(k string, v []string) bool {
+		if k != internalTrackLogKey { // internal baggage will not inject
+			writer.Set(prefixBaggage+k, strings.Join(v, ","))
+		}
+		return true
+	})
+	return nil
+}
+
+// Extract implements Extractor of TextMapPropagator.
+func (t *TextMapPropagator) Extract(carrier interface{}) (opentracing.SpanContext, error) {
+	reader, ok := carrier.(opentracing.TextMapReader)
+	if !ok {
+		return nil, ErrInvalidCarrier
+	}
+	var (
+		traceID    string
+		spanID     ID
+		baggage    = make(map[string][]string)
+		fieldCount int
+		err        error
+	)
+	err = reader.ForeachKey(func(key, val string) error {
+		switch strings.ToLower(key) {
+		case fieldKeyTraceID:
+			traceID = val
+			fieldCount++
+		case fieldKeySpanID:
+			id, err := strconv.ParseUint(val, 16, 64)
+			if err != nil {
+				return ErrSpanContextCorrupted
+			}
+			spanID = ID(id)
+			fieldCount++
+		default:
+			lowerKey := strings.ToLower(key)
+			if strings.HasPrefix(lowerKey, prefixBaggage) {
+				baggage[strings.TrimPrefix(lowerKey, prefixBaggage)] = []string{val}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if fieldCount == 0 {
+		return nil, ErrSpanContextNotFound
+	}
+	if fieldCount < tracerFieldCount {
+		return nil, ErrSpanContextCorrupted
+	}
+	return &SpanContext{
+		traceID: traceID,
+		spanID:  spanID,
+		baggage: baggage,
+	}, nil
+}
+
+// GetTraceIDKey returns http header name of traceid
+func GetTraceIDKey() string {
+	return fieldKeyTraceID
+}

blobstore/common/trace/propagation_test.go

@@ -0,0 +1,76 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package trace
+
+import (
+	"bytes"
+	"context"
+	"net/http"
+	"testing"
+
+	"github.com/opentracing/opentracing-go/mocktracer"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSpanPropagator(t *testing.T) {
+	tracer := NewTracer("blobstore")
+	defer tracer.Close()
+	SetGlobalTracer(tracer)
+
+	span, _ := StartSpanFromContext(context.Background(), "test baggage")
+	defer span.Finish()
+
+	span.SetBaggageItem("k1", "v1")
+
+	carriers := []struct {
+		carrierType interface{}
+		carrier     interface{}
+	}{
+		{HTTPHeaders, HTTPHeadersCarrier(http.Header{})},
+		{TextMap, TextMapCarrier(make(map[string]string))},
+	}
+
+	for _, c := range carriers {
+		err := span.Tracer().Inject(span.Context(), c.carrierType, c.carrier)
+		assert.NoError(t, err)
+
+		sp, err := Extract(c.carrierType, c.carrier)
+		assert.NoError(t, err)
+
+		child := tracer.StartSpan("child", ChildOf(sp))
+		assert.Equal(t, "v1", child.BaggageItem("k1"))
+		assert.Equal(t, span.Context().(*SpanContext).traceID, child.Context().(*SpanContext).traceID)
+		assert.Equal(t, span.Context().(*SpanContext).spanID, child.Context().(*SpanContext).parentID)
+		child.Finish()
+	}
+
+	err := span.Tracer().Inject(span.Context(), Binary, &bytes.Buffer{})
+	assert.EqualError(t, err, ErrUnsupportedFormat.Error())
+	_, err = Extract(Binary, &bytes.Buffer{})
+	assert.EqualError(t, err, ErrUnsupportedFormat.Error())
+
+	err = tracer.Inject(mocktracer.MockSpanContext{}, Binary, &bytes.Buffer{})
+	assert.EqualError(t, err, ErrInvalidSpanContext.Error())
+
+	err = defaultTexMapPropagator.Inject(span.(*spanImpl).context, &bytes.Buffer{})
+	assert.EqualError(t, err, ErrInvalidCarrier.Error())
+	_, err = defaultTexMapPropagator.Extract(&bytes.Buffer{})
+	assert.EqualError(t, err, ErrInvalidCarrier.Error())
+
+	_, err = defaultTexMapPropagator.Extract(HTTPHeadersCarrier(http.Header{}))
+	assert.Error(t, err)
+
+	assert.Equal(t, fieldKeyTraceID, GetTraceIDKey())
+}

blobstore/common/trace/span.go

@@ -0,0 +1,331 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package trace
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/opentracing/opentracing-go"
+	ptlog "github.com/opentracing/opentracing-go/log"
+
+	"github.com/cubefs/cubefs/blobstore/util/log"
+)
+
+const (
+	maxErrorLen = 32
+)
+
+// Span extends opentracing.Span
+type Span interface {
+	opentracing.Span
+
+	// OperationName allows retrieving current operation name.
+	OperationName() string
+
+	// Tags returns tags for span
+	Tags() Tags
+
+	// Logs returns micro logs for span
+	Logs() []opentracing.LogRecord
+
+	// String returns traceID:spanID.
+	String() string
+
+	// TraceID returns traceID
+	TraceID() string
+
+	// AppendRPCTrackLog appends RPC track logs to baggage with default key fieldTrackLogKey.
+	AppendRPCTrackLog(logs []string)
+	// AppendTrackLog records cost time with startTime (duration=time.Since(startTime)) for a calling to a module and
+	// appends to baggage with default key fieldTrackLogKey.
+	AppendTrackLog(module string, startTime time.Time, err error)
+	// AppendTrackLogWithDuration records cost time with duration for a calling to a module and
+	// appends to baggage with default key fieldTrackLogKey.
+	AppendTrackLogWithDuration(module string, duration time.Duration, err error)
+	// TrackLog returns track log, calls BaggageItem with default key fieldTrackLogKey.
+	TrackLog() []string
+
+	// BaseLogger defines interface of application log apis.
+	log.BaseLogger
+}
+
+// spanImpl implements Span
+type spanImpl struct {
+	operationName string
+
+	tracer *Tracer
+
+	context *SpanContext
+
+	startTime time.Time
+	duration  time.Duration
+
+	tags Tags
+
+	logs []opentracing.LogRecord
+
+	// rootSpan, if true indicate that this span is the root of the (sub)tree
+	// of spans and parentID is empty.
+	rootSpan bool
+
+	// references for this span
+	references []opentracing.SpanReference
+
+	sync.RWMutex
+}
+
+// Finish implements opentracing.Span API
+func (s *spanImpl) Finish() {
+	s.FinishWithOptions(opentracing.FinishOptions{})
+}
+
+// FinishWithOptions implements opentracing.Span API
+func (s *spanImpl) FinishWithOptions(opts opentracing.FinishOptions) {
+	finishTime := opts.FinishTime
+	if finishTime.IsZero() {
+		finishTime = time.Now()
+	}
+	s.duration = finishTime.Sub(s.startTime)
+
+	s.Lock()
+	defer s.Unlock()
+
+	s.logs = append(s.logs, opts.LogRecords...)
+
+	for _, ld := range opts.BulkLogData {
+		s.logs = append(s.logs, ld.ToLogRecord())
+	}
+
+	// TODO report span
+}
+
+// Context implements opentracing.Span API
+func (s *spanImpl) Context() opentracing.SpanContext {
+	s.RLock()
+	defer s.RUnlock()
+
+	return s.context
+}
+
+// SetOperationName implements opentracing.Span API
+func (s *spanImpl) SetOperationName(operationName string) opentracing.Span {
+	s.Lock()
+	defer s.Unlock()
+
+	s.operationName = operationName
+	return s
+}
+
+// LogFields implements opentracing.Span API
+func (s *spanImpl) LogFields(fields ...ptlog.Field) {
+	s.Lock()
+	defer s.Unlock()
+
+	lr := opentracing.LogRecord{
+		Fields:    fields,
+		Timestamp: time.Now(),
+	}
+	s.logs = append(s.logs, lr)
+}
+
+// LogKV implements opentracing.Span API
+func (s *spanImpl) LogKV(keyValues ...interface{}) {
+	fields, err := ptlog.InterleavedKVToFields(keyValues...)
+	if err != nil {
+		s.LogFields(ptlog.Error(err), ptlog.String("function", "LogKV"))
+		return
+	}
+	s.LogFields(fields...)
+}
+
+// SetBaggageItem implements opentracing.Span API
+func (s *spanImpl) SetBaggageItem(key, value string) opentracing.Span {
+	for _, ref := range s.references {
+		spanCtx, ok := ref.ReferencedContext.(*SpanContext)
+		if !ok {
+			continue
+		}
+		spanCtx.setBaggageItem(key, []string{value})
+	}
+	s.context.setBaggageItem(key, []string{value})
+	return s
+}
+
+// BaggageItem implements opentracing.Span API
+func (s *spanImpl) BaggageItem(key string) string {
+	return strings.Join(s.context.baggageItem(key), ",")
+}
+
+// Tracer implements opentracing.Span API
+func (s *spanImpl) Tracer() opentracing.Tracer {
+	return s.tracer
+}
+
+// SetTag implements opentracing.Span API
+func (s *spanImpl) SetTag(key string, value interface{}) opentracing.Span {
+	s.Lock()
+	defer s.Unlock()
+
+	if s.tags == nil {
+		s.tags = Tags{}
+	}
+	s.tags[key] = value
+	return s
+}
+
+// Deprecated: use LogFields or LogKV (not implements)
+func (s *spanImpl) LogEvent(event string) {}
+
+// Deprecated: use LogFields or LogKV (not implements)
+func (s *spanImpl) LogEventWithPayload(event string, payload interface{}) {}
+
+// Deprecated: use LogFields or LogKV (not implements)
+func (s *spanImpl) Log(data opentracing.LogData) {}
+
+// OperationName returns operationName for span
+func (s *spanImpl) OperationName() string {
+	s.RLock()
+	defer s.RUnlock()
+
+	return s.operationName
+}
+
+// Tags returns tags for span
+func (s *spanImpl) Tags() Tags {
+	s.RLock()
+	defer s.RUnlock()
+	// copy
+	tags := make(map[string]interface{}, len(s.tags))
+	for key, value := range s.tags {
+		tags[key] = value
+	}
+	return tags
+}
+
+// Logs returns micro logs for span
+func (s *spanImpl) Logs() []opentracing.LogRecord {
+	s.RLock()
+	defer s.RUnlock()
+
+	return s.logs
+}
+
+// AppendTrackLog records cost time with startTime (duration=time.Since(startTime)) for a calling to a module and
+// appends to baggage with default key fieldTrackLogKey.
+func (s *spanImpl) AppendTrackLog(module string, startTime time.Time, err error) {
+	s.AppendTrackLogWithDuration(module, time.Since(startTime), err)
+}
+
+// AppendTrackLogWithDuration records cost time with duration for a calling to a module and
+// appends to baggage with default key fieldTrackLogKey.
+func (s *spanImpl) AppendTrackLogWithDuration(module string, duration time.Duration, err error) {
+	durMs := duration.Nanoseconds() / 1e6
+	if durMs > 0 {
+		module += ":" + strconv.FormatInt(durMs, 10)
+	}
+	if err != nil {
+		msg := err.Error()
+		if len(msg) > maxErrorLen {
+			msg = msg[:maxErrorLen]
+		}
+		module += "/" + msg
+	}
+	s.track(module)
+}
+
+// AppendRPCTrackLog appends RPC track logs to baggage with default key fieldTrackLogKey.
+func (s *spanImpl) AppendRPCTrackLog(logs []string) {
+	for _, trackLog := range logs {
+		s.track(trackLog)
+	}
+}
+
+// TrackLog returns track log, calls BaggageItem with default key fieldTrackLogKey.
+func (s *spanImpl) TrackLog() []string {
+	return s.context.trackLogs()
+}
+
+func (s *spanImpl) track(value string) {
+	for _, ref := range s.references {
+		spanCtx, ok := ref.ReferencedContext.(*SpanContext)
+		if !ok {
+			continue
+		}
+		spanCtx.append(value)
+	}
+	s.context.append(value)
+}
+
+// String returns traceID:spanID.
+func (s *spanImpl) String() string {
+	return fmt.Sprintf("%s:%s", s.context.traceID, s.context.spanID)
+}
+
+// TraceID return traceID
+func (s *spanImpl) TraceID() string {
+	return s.context.traceID
+}
+
+//-------------------------------------------------------------------
+//
+const (
+	defaultCalldepth = 3
+)
+
+func (s *spanImpl) output(lvl log.Level, v []interface{}) {
+	log.DefaultLogger.Output(s.String(), lvl, defaultCalldepth, fmt.Sprintln(v...))
+}
+
+func (s *spanImpl) outputf(lvl log.Level, format string, v []interface{}) {
+	log.DefaultLogger.Output(s.String(), lvl, defaultCalldepth, fmt.Sprintf(format, v...))
+}
+
+func (s *spanImpl) Println(v ...interface{})               { s.output(log.Linfo, v) }
+func (s *spanImpl) Printf(format string, v ...interface{}) { s.outputf(log.Linfo, format, v) }
+func (s *spanImpl) Debug(v ...interface{})                 { s.output(log.Ldebug, v) }
+func (s *spanImpl) Debugf(format string, v ...interface{}) { s.outputf(log.Ldebug, format, v) }
+func (s *spanImpl) Info(v ...interface{})                  { s.output(log.Linfo, v) }
+func (s *spanImpl) Infof(format string, v ...interface{})  { s.outputf(log.Linfo, format, v) }
+func (s *spanImpl) Warn(v ...interface{})                  { s.output(log.Lwarn, v) }
+func (s *spanImpl) Warnf(format string, v ...interface{})  { s.outputf(log.Lwarn, format, v) }
+func (s *spanImpl) Error(v ...interface{})                 { s.output(log.Lerror, v) }
+func (s *spanImpl) Errorf(format string, v ...interface{}) { s.outputf(log.Lerror, format, v) }
+
+func (s *spanImpl) Panic(v ...interface{}) {
+	str := fmt.Sprintln(v...)
+	s.output(log.Lpanic, v)
+	panic(s.String() + " -> " + str)
+}
+
+func (s *spanImpl) Panicf(format string, v ...interface{}) {
+	str := fmt.Sprintf(format, v...)
+	s.outputf(log.Lpanic, format, v)
+	panic(s.String() + " -> " + str)
+}
+
+func (s *spanImpl) Fatal(v ...interface{}) {
+	s.output(log.Lfatal, v)
+	os.Exit(1)
+}
+
+func (s *spanImpl) Fatalf(format string, v ...interface{}) {
+	s.outputf(log.Lfatal, format, v)
+	os.Exit(1)
+}

blobstore/common/trace/span_context.go

@@ -0,0 +1,135 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package trace
+
+import (
+	"fmt"
+	"math/rand"
+	"sync"
+	"time"
+)
+
+const (
+	internalTrackLogKey = "internal-baggage-key-tracklog"
+)
+
+// ID used for spanID or traceID
+type ID uint64
+
+func (id ID) String() string {
+	return fmt.Sprintf("%016x", uint64(id))
+}
+
+var (
+	seededIDGen = rand.New(rand.NewSource(time.Now().UnixNano()))
+	// The golang rand generators are *not* intrinsically thread-safe.
+	seededIDLock sync.Mutex
+)
+
+// RandomID generate ID for traceID or spanID
+func RandomID() ID {
+	seededIDLock.Lock()
+	defer seededIDLock.Unlock()
+	return ID(seededIDGen.Int63())
+}
+
+// SpanContext implements opentracing.SpanContext
+type SpanContext struct {
+	// traceID represents globally unique ID of the trace.
+	traceID string
+
+	// spanID represents span ID that must be unique within its trace.
+	spanID ID
+
+	// parentID refers to the ID of the parent span.
+	// Should be 0 if the current span is a root span.
+	parentID ID
+
+	// Distributed Context baggage.
+	baggage map[string][]string
+	sync.RWMutex
+}
+
+// ForeachBaggageItem implements opentracing.SpanContext API
+func (s *SpanContext) ForeachBaggageItem(handler func(k, v string) bool) {
+	panic("not implements")
+}
+
+// ForeachBaggageItems will called the handler function  for each baggage key/values pair.
+func (s *SpanContext) ForeachBaggageItems(handler func(k string, v []string) bool) {
+	s.Lock()
+	defer s.Unlock()
+
+	for k, v := range s.baggage {
+		if !handler(k, v) {
+			break
+		}
+	}
+}
+
+func (s *SpanContext) setBaggageItem(key string, value []string) {
+	s.Lock()
+	defer s.Unlock()
+
+	if s.baggage == nil {
+		s.baggage = map[string][]string{key: value}
+		return
+	}
+	s.baggage[key] = value
+}
+
+func (s *SpanContext) trackLogs() []string {
+	return s.baggageItemDeepCopy(internalTrackLogKey)
+}
+
+func (s *SpanContext) append(value string) {
+	s.Lock()
+	defer s.Unlock()
+
+	if s.baggage == nil {
+		s.baggage = map[string][]string{internalTrackLogKey: {value}}
+		return
+	}
+
+	if _, ok := s.baggage[internalTrackLogKey]; ok {
+		s.baggage[internalTrackLogKey] = append(s.baggage[internalTrackLogKey], value)
+		return
+	}
+	s.baggage[internalTrackLogKey] = []string{value}
+}
+
+func (s *SpanContext) baggageItem(key string) []string {
+	s.RLock()
+	defer s.RUnlock()
+
+	return s.baggage[key]
+}
+
+func (s *SpanContext) baggageItemDeepCopy(key string) (item []string) {
+	s.RLock()
+	defer s.RUnlock()
+	item = append(item, s.baggage[key]...)
+	return
+}
+
+// IsValid returns true if SpanContext is valid
+func (s *SpanContext) IsValid() bool {
+	return s.traceID != "" && s.spanID != 0
+}
+
+// IsEmpty returns true is span context is empty
+func (s *SpanContext) IsEmpty() bool {
+	return !s.IsValid() && len(s.baggage) == 0
+}

blobstore/common/trace/span_test.go

@@ -0,0 +1,215 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package trace
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/opentracing/opentracing-go"
+	ptlog "github.com/opentracing/opentracing-go/log"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/cubefs/cubefs/blobstore/util/log"
+)
+
+func TestSpan_Tags(t *testing.T) {
+	span, _ := StartSpanFromContext(context.Background(), "test tags")
+	defer span.Finish()
+
+	expectedTags := Tags{
+		"module": "worker",
+		"ip":     "127.0.0.1",
+	}
+
+	span.SetTag("module", "worker")
+	span.SetTag("ip", "127.0.0.1")
+	assert.Equal(t, span.Tags(), expectedTags)
+}
+
+func TestSpan_Logs(t *testing.T) {
+	span, _ := StartSpanFromContext(context.Background(), "test logs")
+	defer span.Finish()
+
+	expectedLogs := []struct {
+		logs []ptlog.Field
+	}{
+		{
+			logs: []ptlog.Field{ptlog.String("event", "success"), ptlog.Int("waited.millis", 20)},
+		},
+		{
+			logs: []ptlog.Field{ptlog.String("event", "failed"), ptlog.Int("waited.millis", 1500)},
+		},
+	}
+
+	for k, v := range expectedLogs {
+		span.LogFields(v.logs...)
+		assert.Equal(t, expectedLogs[k].logs, span.Logs()[k].Fields)
+	}
+	assert.Equal(t, 2, len(span.Logs()))
+
+	fields := []ptlog.Field{ptlog.String("code", "200"), ptlog.Float32("count", 100)}
+	for k, v := range fields {
+		span.LogKV(v.Key(), v.Value())
+		assert.Equal(t, fields[k].Key(), span.Logs()[k+2].Fields[0].Key())
+		assert.Equal(t, fields[k].Value(), span.Logs()[k+2].Fields[0].Value())
+	}
+	assert.Equal(t, 4, len(span.Logs()))
+
+	span.LogKV("only key")
+	assert.Equal(t, 5, len(span.Logs()))
+}
+
+func TestSpan_OperationName(t *testing.T) {
+	span, _ := StartSpanFromContext(context.Background(), "span")
+	defer span.Finish()
+
+	assert.Equal(t, "span", span.OperationName())
+	span.SetOperationName("span2")
+	assert.Equal(t, "span2", span.OperationName())
+}
+
+func TestSpan_Baggage(t *testing.T) {
+	span, ctx := StartSpanFromContext(context.Background(), "test baggage")
+	defer span.Finish()
+
+	baggages := []struct {
+		k string
+		v string
+	}{
+		{k: "k1", v: "v1"},
+		{k: "k2", v: "v2"},
+		{k: "k3", v: "v3"},
+	}
+	for _, v := range baggages {
+		span.SetBaggageItem(v.k, v.v)
+		assert.Equal(t, v.v, span.BaggageItem(v.k))
+	}
+
+	spanChild, _ := StartSpanFromContext(ctx, "child of span")
+	for _, v := range baggages {
+		assert.Equal(t, v.v, spanChild.BaggageItem(v.k))
+	}
+
+	spanChild.SetBaggageItem("k4", "v4")
+	assert.Equal(t, "v4", spanChild.BaggageItem("k4"))
+	assert.Equal(t, "v4", span.BaggageItem("k4"))
+}
+
+func TestSpan_TrackLog(t *testing.T) {
+	span, ctx := StartSpanFromContext(context.Background(), "test trackLog")
+	defer span.Finish()
+
+	span.AppendTrackLog("sleep", time.Now(), nil)
+	assert.Equal(t, []string{"sleep"}, span.TrackLog())
+
+	spanChild, _ := StartSpanFromContext(ctx, "child of span")
+	assert.Equal(t, []string{"sleep"}, spanChild.TrackLog())
+
+	spanChild.AppendTrackLog("sleep2", time.Now(), errors.New("sleep2 err"))
+	assert.Equal(t, []string{"sleep", "sleep2/sleep2 err"}, spanChild.TrackLog())
+	assert.Equal(t, []string{"sleep", "sleep2/sleep2 err"}, span.TrackLog())
+
+	spanChild.AppendRPCTrackLog([]string{"blobnode:4", "scheduler:5"})
+	assert.Equal(t, []string{"sleep", "sleep2/sleep2 err", "blobnode:4", "scheduler:5"}, spanChild.TrackLog())
+	assert.Equal(t, []string{"sleep", "sleep2/sleep2 err", "blobnode:4", "scheduler:5"}, span.TrackLog())
+
+	spanChild.AppendTrackLog("sleep3", time.Now(), nil)
+	assert.Equal(t, []string{"sleep", "sleep2/sleep2 err", "blobnode:4", "scheduler:5", "sleep3"}, span.TrackLog())
+}
+
+func TestSpan_TrackLogWithDuration(t *testing.T) {
+	span, ctx := StartSpanFromContext(context.Background(), "test trackLog")
+	defer span.Finish()
+
+	span.AppendTrackLogWithDuration("sleep", time.Millisecond, nil)
+	assert.Equal(t, []string{"sleep:1"}, span.TrackLog())
+
+	spanChild, _ := StartSpanFromContext(ctx, "child of span")
+	assert.Equal(t, []string{"sleep:1"}, spanChild.TrackLog())
+
+	spanChild.AppendTrackLogWithDuration("sleep2", 2*time.Millisecond, errors.New("sleep2 err"))
+	assert.Equal(t, []string{"sleep:1", "sleep2:2/sleep2 err"}, spanChild.TrackLog())
+	assert.Equal(t, []string{"sleep:1", "sleep2:2/sleep2 err"}, span.TrackLog())
+
+	spanChild.AppendRPCTrackLog([]string{"blobnode:4", "scheduler:5"})
+	assert.Equal(t, []string{"sleep:1", "sleep2:2/sleep2 err", "blobnode:4", "scheduler:5"}, spanChild.TrackLog())
+	assert.Equal(t, []string{"sleep:1", "sleep2:2/sleep2 err", "blobnode:4", "scheduler:5"}, span.TrackLog())
+
+	spanChild.AppendTrackLogWithDuration("sleep3", 3*time.Millisecond, nil)
+	assert.Equal(t, []string{"sleep:1", "sleep2:2/sleep2 err", "blobnode:4", "scheduler:5", "sleep3:3"}, span.TrackLog())
+}
+
+func TestSpan_BaseLogger(t *testing.T) {
+	rootSpan, ctx := StartSpanFromContext(context.Background(), "test baseLogger")
+	defer rootSpan.Finish()
+
+	logLevel := []log.Level{
+		log.Ldebug,
+		log.Linfo,
+		log.Lwarn,
+		log.Lerror,
+		log.Lpanic,
+		log.Lfatal,
+	}
+
+	for _, level := range logLevel {
+		rootSpan.Infof("set log level, level: %d", level)
+		log.SetOutputLevel(level)
+
+		span, _ := StartSpanFromContext(ctx, "test baseLogger")
+
+		span.Debug("span info:", span.String())
+		span.Debugf("spanContent info: %+v ,traceID: %s", span.Context(), span.TraceID())
+
+		span.Info("service name", span.Tracer().(*Tracer).serviceName)
+		span.Infof("start span success, name: %s", span.OperationName())
+
+		span.Warn("get spanID")
+		span.Warnf("spanID: %d", span.Context().(*SpanContext).spanID)
+
+		ctx := context.Background()
+		if spanNil := SpanFromContext(ctx); spanNil == nil {
+			span.Error("SpanFromContext failed")
+			span.Errorf("ctx: %+v, span: %+v", ctx, spanNil)
+		}
+
+		assert.Panics(t, func() {
+			if level%2 == 0 {
+				span.Panic("panic on span", span)
+			} else {
+				span.Panicf("panic on span: %p", span.Context().(*SpanContext))
+			}
+		})
+
+		span.Finish()
+	}
+}
+
+func TestSpan_FinishWithOptions(t *testing.T) {
+	span, _ := StartSpanFromContext(context.Background(), "test baseLogger")
+
+	span.FinishWithOptions(opentracing.FinishOptions{
+		LogRecords: []opentracing.LogRecord{
+			{Timestamp: time.Now()},
+			{Timestamp: time.Now()},
+		},
+		BulkLogData: []opentracing.LogData{
+			{Timestamp: time.Now()},
+		},
+	})
+}

blobstore/common/trace/tracer.go

@@ -0,0 +1,310 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package trace
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"path"
+	"time"
+
+	"github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go/ext"
+)
+
+const (
+	defaultRootSpanName   = "defaultBlobnodeRootSpanName"
+	defaultMaxLogsPerSpan = 50
+	reqidKey              = "X-Reqid"
+)
+
+// ChildOf is the alias of opentracing.ChildOf
+var ChildOf = opentracing.ChildOf
+
+// FollowsFrom is the alias of opentracing.FollowsFrom
+var FollowsFrom = opentracing.FollowsFrom
+
+// StartTime is alias of opentracing.StartTime.
+type StartTime = opentracing.StartTime
+
+// Tags are the expand of opentracing.Tags
+type Tags opentracing.Tags
+
+// Apply satisfies the StartSpanOption interface.
+func (t Tags) Apply(options *opentracing.StartSpanOptions) {
+	if options.Tags == nil {
+		options.Tags = make(opentracing.Tags)
+	}
+	for k, v := range t {
+		options.Tags[k] = v
+	}
+}
+
+// ToSlice change tags to slice
+func (t Tags) ToSlice() (ret []string) {
+	for k := range t {
+		ret = append(ret, k+":"+fmt.Sprint(t[k]))
+	}
+	return
+}
+
+// Marshal marshal tracer tags
+func (t Tags) Marshal() (ret []byte, err error) {
+	ret, err = json.Marshal(t)
+	return
+}
+
+// Tag is the alias of opentracing.Tag,
+type Tag = opentracing.Tag
+
+// Options tracer options
+type Options struct {
+	maxLogsPerSpan int
+}
+
+// Tracer implements opentracing.Tracer
+type Tracer struct {
+	serviceName string
+
+	options Options
+}
+
+// init sets default global tracer
+func init() {
+	tracer := NewTracer(path.Base(os.Args[0]))
+	SetGlobalTracer(tracer)
+}
+
+// NewTracer creates a tracer with serviceName
+func NewTracer(serviceName string, opts ...TracerOption) *Tracer {
+	t := &Tracer{
+		serviceName: serviceName,
+	}
+	for _, option := range opts {
+		option(t)
+	}
+
+	if t.options.maxLogsPerSpan <= 0 {
+		t.options.maxLogsPerSpan = defaultMaxLogsPerSpan
+	}
+
+	return t
+}
+
+// StartSpan implements StartSpan() method of opentracing.Tracer.
+// Create, start, and return a new Span with the given `operationName` and
+// incorporate the given StartSpanOption `opts`.
+func (t *Tracer) StartSpan(operationName string, options ...opentracing.StartSpanOption) opentracing.Span {
+	sso := opentracing.StartSpanOptions{}
+	for _, o := range options {
+		o.Apply(&sso)
+	}
+	return t.startSpanWithOptions(operationName, sso)
+}
+
+func (t *Tracer) startSpanWithOptions(operationName string, opts opentracing.StartSpanOptions) Span {
+	startTime := opts.StartTime
+	if startTime.IsZero() {
+		startTime = time.Now()
+	}
+
+	var (
+		hasParent  bool
+		parent     *SpanContext
+		references []opentracing.SpanReference
+		ctx        = &SpanContext{}
+	)
+
+	for _, reference := range opts.References {
+		spanCtx, ok := reference.ReferencedContext.(*SpanContext)
+		if !ok {
+			continue
+		}
+
+		if spanCtx == nil || spanCtx.IsEmpty() {
+			continue
+		}
+
+		if spanCtx.IsValid() {
+			references = append(references, reference)
+		}
+
+		if !hasParent {
+			parent = spanCtx
+			hasParent = reference.Type == opentracing.ChildOfRef
+		}
+	}
+
+	if !hasParent && parent != nil && !parent.IsEmpty() {
+		hasParent = true
+	}
+
+	if !hasParent || (parent != nil && !parent.IsValid()) {
+		ctx.traceID = RandomID().String()
+		ctx.spanID = RandomID()
+		ctx.parentID = 0
+	} else {
+		ctx.traceID = parent.traceID
+		ctx.spanID = RandomID()
+		ctx.parentID = parent.spanID
+	}
+	if hasParent {
+		// copy baggage items
+		parent.ForeachBaggageItems(func(k string, v []string) bool {
+			ctx.setBaggageItem(k, v)
+			return true
+		})
+	}
+
+	tags := opts.Tags
+
+	span := &spanImpl{
+		operationName: operationName,
+		startTime:     startTime,
+		tags:          tags,
+		context:       ctx,
+		tracer:        t,
+		references:    references,
+		duration:      0,
+	}
+	span.rootSpan = ctx.parentID == 0
+	return span
+}
+
+// Inject implements Inject() method of opentracing.Tracer
+func (t *Tracer) Inject(sc opentracing.SpanContext, format interface{}, carrier interface{}) error {
+	s, ok := sc.(*SpanContext)
+	if !ok {
+		return opentracing.ErrInvalidSpanContext
+	}
+	switch format {
+	case TextMap, HTTPHeaders:
+		return defaultTexMapPropagator.Inject(s, carrier)
+	}
+	return ErrUnsupportedFormat
+}
+
+// Extract implements Extract() method of opentracing.Tracer
+func (t *Tracer) Extract(format interface{}, carrier interface{}) (opentracing.SpanContext, error) {
+	switch format {
+	case TextMap, HTTPHeaders:
+		return defaultTexMapPropagator.Extract(carrier)
+	}
+	return nil, ErrUnsupportedFormat
+}
+
+// Close releases all resources
+func (t *Tracer) Close() error {
+	// TODO report span
+	return nil
+}
+
+// StartSpanFromContext starts and returns a Span with `operationName`, using
+// any Span found within `ctx` as a ChildOfRef. If no such parent could be
+// found, StartSpanFromContext creates a root (parentless) Span.
+func StartSpanFromContext(ctx context.Context, operationName string, opts ...opentracing.StartSpanOption) (Span, context.Context) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, operationName, opts...)
+	return span.(Span), ctx
+}
+
+// StartSpanFromContextWithTraceID starts and return a new span with `operationName` and traceID.
+func StartSpanFromContextWithTraceID(ctx context.Context, operationName string, traceID string, opts ...opentracing.StartSpanOption) (Span, context.Context) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, operationName, opts...)
+	s := span.(*spanImpl)
+	s.context.traceID = traceID
+	return s, ctx
+}
+
+// StartSpanFromHTTPHeaderSafe starts and return a Span with `operationName` and http.Request
+func StartSpanFromHTTPHeaderSafe(r *http.Request, operationName string) (Span, context.Context) {
+	spanCtx, _ := Extract(HTTPHeaders, HTTPHeadersCarrier(r.Header))
+	traceID := r.Header.Get(reqidKey)
+	if traceID == "" {
+		return StartSpanFromContext(context.Background(), operationName, ext.RPCServerOption(spanCtx))
+	}
+	return StartSpanFromContextWithTraceID(context.Background(), operationName, traceID, ext.RPCServerOption(spanCtx))
+}
+
+// ContextWithSpan returns a new `context.Context` that holds a reference to
+// the span. If span is nil, a new context without an active span is returned.
+func ContextWithSpan(ctx context.Context, span Span) context.Context {
+	return opentracing.ContextWithSpan(ctx, span)
+}
+
+// SpanFromContext returns the `Span` previously associated with `ctx`, or
+// `nil` if no such `Span` could be found.
+func SpanFromContext(ctx context.Context) Span {
+	span := opentracing.SpanFromContext(ctx)
+	s, ok := span.(Span)
+	if !ok {
+		return nil
+	}
+	return s
+}
+
+// SpanFromContextSafe returns the `Span` previously associated with `ctx`, or
+// creates a root Span with name default.
+func SpanFromContextSafe(ctx context.Context) Span {
+	span := opentracing.SpanFromContext(ctx)
+	s, ok := span.(Span)
+	if !ok || s == nil {
+		return opentracing.GlobalTracer().StartSpan(defaultRootSpanName).(Span)
+	}
+	return s
+}
+
+// SetGlobalTracer sets the [singleton] opentracing.Tracer returned by
+// GlobalTracer(). Those who use GlobalTracer (rather than directly manage an
+// opentracing.Tracer instance) should call SetGlobalTracer as early as
+// possible in main(), prior to calling the `StartSpan` global func below.
+// Prior to calling `SetGlobalTracer`, any Spans started via the `StartSpan`
+// (etc) globals are noops.
+func SetGlobalTracer(tracer *Tracer) {
+	opentracing.SetGlobalTracer(tracer)
+}
+
+// CloseGlobalTracer closes global tracer gracefully.
+func CloseGlobalTracer() {
+	tracer, ok := opentracing.GlobalTracer().(*Tracer)
+	if !ok {
+		return
+	}
+	tracer.Close()
+}
+
+// GlobalTracer returns the global singleton `Tracer` implementation.
+func GlobalTracer() *Tracer {
+	t := opentracing.GlobalTracer()
+	return t.(*Tracer)
+}
+
+// Extract returns a SpanContext instance given `format` and `carrier`.
+func Extract(format interface{}, carrier interface{}) (opentracing.SpanContext, error) {
+	return GlobalTracer().Extract(format, carrier)
+}
+
+// InjectWithHTTPHeader takes the `sm` SpanContext instance and injects it for
+// propagation within `HTTPHeadersCarrier` and `HTTPHeaders`.
+func InjectWithHTTPHeader(ctx context.Context, r *http.Request) error {
+	span := SpanFromContextSafe(ctx)
+
+	ext.SpanKindRPCClient.Set(span)
+	ext.HTTPMethod.Set(span, r.Method)
+
+	return span.Tracer().Inject(span.Context(), HTTPHeaders, HTTPHeadersCarrier(r.Header))
+}

blobstore/common/trace/tracer_options.go

@@ -0,0 +1,29 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package trace
+
+// TracerOption is a function that sets some option on the tracer
+type TracerOption func(tracer *Tracer)
+
+// TracerOptions is a factory for all available TracerOption's
+var TracerOptions tracerOptions
+
+type tracerOptions struct{}
+
+func (tracerOptions) MaxLogsPerSpan(maxLogsPerSpan int) TracerOption {
+	return func(tracer *Tracer) {
+		tracer.options.maxLogsPerSpan = maxLogsPerSpan
+	}
+}

blobstore/common/trace/tracer_test.go

@@ -0,0 +1,207 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package trace
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/opentracing/opentracing-go"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExplicitStartTime(t *testing.T) {
+	tracer := NewTracer("blobstore")
+	defer tracer.Close()
+
+	start := time.Now()
+
+	span := tracer.StartSpan("testStartTime", StartTime(start))
+	defer span.Finish()
+
+	assert.Equal(t, start, span.(*spanImpl).startTime)
+}
+
+func TestExplicitTags(t *testing.T) {
+	tracer := NewTracer("blobstore")
+	defer tracer.Close()
+
+	tags := Tags{
+		"tag1K": "tag1V",
+		"tag2K": "tag2V",
+	}
+
+	span1 := tracer.StartSpan("testTags", tags)
+	defer span1.Finish()
+
+	assert.Equal(t, tags, span1.(*spanImpl).tags)
+
+	tag1 := Tag{
+		Key:   "tag1K",
+		Value: "tag1V",
+	}
+	span2 := tracer.StartSpan("testTag", tag1).(Span)
+	defer span2.Finish()
+
+	expect := Tags{
+		"tag1K": "tag1V",
+	}
+	assert.Equal(t, expect, span2.(*spanImpl).tags)
+
+	tag2 := Tag{
+		Key:   "tag2K",
+		Value: "tag2V",
+	}
+	tag2.Set(span2)
+	assert.Equal(t, tags, span2.(*spanImpl).tags)
+}
+
+func TestExplicitReferences(t *testing.T) {
+	tracer := NewTracer("blobstore")
+	defer tracer.Close()
+
+	parentSpan := tracer.StartSpan("parent").(*spanImpl)
+	defer parentSpan.Finish()
+
+	span1 := tracer.StartSpan("child", ChildOf(parentSpan.Context())).(*spanImpl)
+	defer span1.Finish()
+
+	assert.Equal(t, 1, len(span1.references))
+	assert.Equal(t, parentSpan.context.traceID, span1.context.traceID)
+	assert.Equal(t, parentSpan.context.spanID, span1.context.parentID)
+
+	ctx := ContextWithSpan(context.Background(), span1)
+	span2 := SpanFromContext(ctx).(*spanImpl)
+
+	span3, _ := StartSpanFromContext(ctx, "child of span")
+	cs := span3.(*spanImpl)
+
+	assert.Equal(t, span1, span2)
+	assert.Equal(t, 1, len(cs.references))
+	assert.Equal(t, span1.context.traceID, cs.context.traceID)
+	assert.Equal(t, span1.context.spanID, cs.context.parentID)
+
+	newParentSpan := tracer.StartSpan("newParentSpan")
+	span4 := tracer.StartSpan("nChild", FollowsFrom(span3.Context()),
+		FollowsFrom(newParentSpan.Context())).(*spanImpl)
+
+	assert.Equal(t, 2, len(span4.references))
+
+	span5 := tracer.StartSpan("empty span context", ChildOf(&SpanContext{})).(*spanImpl)
+	assert.Equal(t, 0, len(span5.references))
+}
+
+func TestStartSpanFromContext(t *testing.T) {
+	span, ctx := StartSpanFromContext(context.Background(), "span1")
+	defer span.Finish()
+
+	assert.NotNil(t, SpanFromContext(ctx))
+	s := span.(*spanImpl)
+
+	childSpan, _ := StartSpanFromContext(ctx, "child span")
+	cs := childSpan.(*spanImpl)
+	assert.Equal(t, 1, len(cs.references))
+	assert.Equal(t, s.context.traceID, cs.context.traceID)
+	assert.Equal(t, s.context.spanID, cs.context.parentID)
+
+	// root span
+	traceID := "traceID"
+	traceID2 := "traceID2"
+	rootSpan1, _ := StartSpanFromContextWithTraceID(context.Background(), "root span1", traceID)
+	rootSpan2, _ := StartSpanFromContextWithTraceID(ctx, "root span2", traceID2)
+	rs1 := rootSpan1.(*spanImpl)
+	rs2 := rootSpan2.(*spanImpl)
+
+	assert.Equal(t, traceID, rs1.context.traceID)
+	assert.NotEqual(t, traceID, rs2.context.traceID)
+}
+
+func TestSpanFromContext(t *testing.T) {
+	ctx := context.Background()
+	assert.Nil(t, SpanFromContext(ctx))
+
+	span, ctx := StartSpanFromContext(ctx, "span1")
+	defer span.Finish()
+
+	assert.NotNil(t, SpanFromContext(ctx))
+
+	spanSafe := SpanFromContextSafe(context.Background())
+	defer spanSafe.Finish()
+
+	s := spanSafe.(*spanImpl)
+	assert.Equal(t, defaultRootSpanName, s.operationName)
+
+	spanCopy := SpanFromContextSafe(ctx)
+
+	sc := spanCopy.(*spanImpl)
+	assert.Equal(t, span.OperationName(), sc.OperationName())
+}
+
+func TestStartSpanFromHTTPHeaderSafe(t *testing.T) {
+	r := &http.Request{Header: http.Header{}}
+	traceID := "test"
+	span, _ := StartSpanFromHTTPHeaderSafe(r, "http")
+	assert.NotEqual(t, traceID, span.Context().(*SpanContext).traceID)
+
+	r.Header.Set(reqidKey, traceID)
+	span, _ = StartSpanFromHTTPHeaderSafe(r, "http")
+	assert.Equal(t, traceID, span.Context().(*SpanContext).traceID)
+}
+
+func TestNewTracer(t *testing.T) {
+	tracer := NewTracer("blobstore")
+	assert.Equal(t, defaultMaxLogsPerSpan, tracer.options.maxLogsPerSpan)
+	tracer.Close()
+
+	tracer = NewTracer("blobstore", TracerOptions.MaxLogsPerSpan(10))
+	assert.Equal(t, 10, tracer.options.maxLogsPerSpan)
+	tracer.Close()
+}
+
+func TestCloseGlobalTracer(t *testing.T) {
+	noopTracer := opentracing.NoopTracer{}
+	opentracing.SetGlobalTracer(noopTracer)
+	CloseGlobalTracer()
+
+	tracer := NewTracer("blobstore")
+	SetGlobalTracer(tracer)
+	CloseGlobalTracer()
+}
+
+func TestInject(t *testing.T) {
+	r := &http.Request{Header: http.Header{}}
+	err := InjectWithHTTPHeader(context.Background(), r)
+	assert.NoError(t, err)
+	span1, _ := StartSpanFromHTTPHeaderSafe(r, "span1")
+	firstUpper := func(s string) string {
+		if s == "" {
+			return ""
+		}
+		return strings.ToUpper(s[:1]) + s[1:]
+	}
+	assert.Equal(t, r.Header.Get(firstUpper(fieldKeyTraceID)), span1.Context().(*SpanContext).traceID)
+
+	span2, ctx := StartSpanFromContext(context.Background(), "span2")
+	r = &http.Request{Header: http.Header{}}
+	err = InjectWithHTTPHeader(ctx, r)
+	assert.NoError(t, err)
+
+	span3, _ := StartSpanFromHTTPHeaderSafe(r, "span3")
+	assert.Equal(t, r.Header.Get(firstUpper(fieldKeyTraceID)), span3.Context().(*SpanContext).traceID)
+	assert.Equal(t, span2.Context().(*SpanContext).traceID, span3.Context().(*SpanContext).traceID)
+}

blobstore/scheduler/archiver.go

@@ -0,0 +1,149 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package scheduler
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/scheduler/base"
+	"github.com/cubefs/cubefs/blobstore/scheduler/db"
+	"github.com/cubefs/cubefs/blobstore/util/closer"
+)
+
+// IArchiver define the interface of archive manager
+type IArchiver interface {
+	RegisterTables(tables ...db.IRecordSrcTbl)
+	Run()
+	closer.Closer
+}
+
+// ArchiveStoreConfig archive store config
+type ArchiveStoreConfig struct {
+	ArchiveIntervalMin int `json:"archive_interval_min"`
+	// recode will archive util delete ArchiveDelayMin
+	ArchiveDelayMin int `json:"archive_delay_min"`
+}
+
+// ArchiveStoreMgr archive store
+type ArchiveStoreMgr struct {
+	closer.Closer
+
+	archTbl db.IArchiveTable
+
+	mu        sync.Mutex
+	srcTables map[string]db.IRecordSrcTbl
+
+	cfg ArchiveStoreConfig
+}
+
+// NewArchiveStoreMgr returns archive store manager
+func NewArchiveStoreMgr(table db.IArchiveTable, cfg ArchiveStoreConfig) *ArchiveStoreMgr {
+	return &ArchiveStoreMgr{
+		Closer:    closer.New(),
+		archTbl:   table,
+		srcTables: make(map[string]db.IRecordSrcTbl),
+		cfg:       cfg,
+	}
+}
+
+// Run archive store task
+func (mgr *ArchiveStoreMgr) Run() {
+	go mgr.run()
+}
+
+func (mgr *ArchiveStoreMgr) run() {
+	t := time.NewTicker(time.Duration(mgr.cfg.ArchiveIntervalMin) * time.Minute)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-t.C:
+			mgr.storeRun()
+		case <-mgr.Closer.Done():
+			return
+		}
+	}
+}
+
+func (mgr *ArchiveStoreMgr) storeRun() {
+	_, ctx := trace.StartSpanFromContext(context.Background(), "ArchiveStoreMgr")
+
+	mgr.mu.Lock()
+	for _, src := range mgr.srcTables {
+		mgr.store(ctx, src)
+	}
+	mgr.mu.Unlock()
+}
+
+func (mgr *ArchiveStoreMgr) store(ctx context.Context, src db.IRecordSrcTbl) {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Debugf("archive store: src[%s]", src.Name())
+
+	deleteTasks, err := src.QueryMarkDeleteTasks(ctx, mgr.cfg.ArchiveDelayMin)
+	if err != nil {
+		span.Errorf("query delete tasks failed: src[%s], err[%+v]", src.Name(), err)
+		return
+	}
+
+	shouldRemove := make([]*proto.ArchiveRecord, 0, len(deleteTasks))
+	for _, task := range deleteTasks {
+		archivedRecord, err := mgr.archTbl.FindTask(ctx, task.TaskID)
+		if archivedRecord != nil {
+			span.Infof("task has been archived: task_id [%s]", task.TaskID)
+			shouldRemove = append(shouldRemove, task)
+			continue
+		}
+		if err != nil && err != base.ErrNoDocuments {
+			span.Errorf("find task in archive table failed: task_id[%s], err[%+v]", task.TaskID, err)
+			continue
+		}
+
+		err = mgr.archTbl.Insert(ctx, task)
+		if err != nil {
+			span.Errorf("insert task into archive table failed: task_id[%s], err[%+v]", task.TaskID, err)
+			continue
+		}
+
+		shouldRemove = append(shouldRemove, task)
+	}
+
+	for _, task := range shouldRemove {
+		archivedRecord, err := mgr.archTbl.FindTask(ctx, task.TaskID)
+		if archivedRecord == nil {
+			span.Warnf("task not find in archive table: task_id[%s], err[%+v]", task.TaskID, err)
+			continue
+		}
+
+		err = src.RemoveMarkDelete(ctx, task.TaskID)
+		if err != nil {
+			span.Errorf("remove task failed: task_id[%s], err[%+v]", task.TaskID, err)
+			continue
+		}
+		span.Debugf("archived: task_id[%s] task[%+v]", task.TaskID, task)
+	}
+}
+
+// RegisterTables register tables to archive.
+func (mgr *ArchiveStoreMgr) RegisterTables(tables ...db.IRecordSrcTbl) {
+	mgr.mu.Lock()
+	for _, src := range tables {
+		mgr.srcTables[src.Name()] = src
+	}
+	mgr.mu.Unlock()
+}

blobstore/scheduler/archiver_test.go

@@ -0,0 +1,110 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package scheduler
+
+import (
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/google/uuid"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/scheduler/base"
+)
+
+var testArchiveConfig = ArchiveStoreConfig{
+	ArchiveIntervalMin: defaultArchiveIntervalMin,
+	ArchiveDelayMin:    defaultArchiveDelayMin,
+}
+
+func TestArchiveStore(t *testing.T) {
+	ctr := gomock.NewController(t)
+	{
+		archiveTable := NewMockArchiveTable(ctr)
+		archiveTable.EXPECT().FindTask(any, any).AnyTimes().Return(nil, nil)
+		archiveTable.EXPECT().Insert(any, any).AnyTimes().Return(nil)
+
+		mgr := NewArchiveStoreMgr(archiveTable, testArchiveConfig)
+		defer mgr.Close()
+
+		balanceTable := NewMockMigrateTaskTable(ctr)
+		balanceTable.EXPECT().Name().AnyTimes().Return(proto.BalanceTaskType)
+		record1 := &proto.ArchiveRecord{TaskID: uuid.New().String(), TaskType: proto.BalanceTaskType, Content: proto.MigrateTask{}}
+		balanceTable.EXPECT().QueryMarkDeleteTasks(any, any).AnyTimes().Return([]*proto.ArchiveRecord{record1}, nil)
+
+		repairTaskTable := NewMockRepairTaskTable(ctr)
+		repairTaskTable.EXPECT().Name().AnyTimes().Return(proto.RepairTaskType)
+		record2 := &proto.ArchiveRecord{TaskID: uuid.New().String(), TaskType: proto.RepairTaskType, Content: proto.VolRepairTask{}}
+		repairTaskTable.EXPECT().QueryMarkDeleteTasks(any, any).AnyTimes().Return([]*proto.ArchiveRecord{record2}, nil)
+
+		mgr.RegisterTables(balanceTable, repairTaskTable)
+		mgr.Run()
+	}
+	{
+		// register twice
+		archiveTable := NewMockArchiveTable(ctr)
+		mgr := NewArchiveStoreMgr(archiveTable, testArchiveConfig)
+		defer mgr.Close()
+
+		repairTaskTable := NewMockRepairTaskTable(ctr)
+		repairTaskTable.EXPECT().Name().AnyTimes().Return(proto.RepairTaskType)
+		mgr.RegisterTables(repairTaskTable)
+		mgr.RegisterTables(repairTaskTable)
+	}
+	{
+		// QueryMarkDeleteTasks failed
+		archiveTable := NewMockArchiveTable(ctr)
+		mgr := NewArchiveStoreMgr(archiveTable, testArchiveConfig)
+		defer mgr.Close()
+
+		repairTaskTable := NewMockRepairTaskTable(ctr)
+		repairTaskTable.EXPECT().Name().AnyTimes().Return(proto.RepairTaskType)
+		repairTaskTable.EXPECT().QueryMarkDeleteTasks(any, any).AnyTimes().Return(nil, errMock)
+
+		mgr.RegisterTables(repairTaskTable)
+		mgr.storeRun()
+	}
+	{
+		archiveTable := NewMockArchiveTable(ctr)
+		mgr := NewArchiveStoreMgr(archiveTable, testArchiveConfig)
+		defer mgr.Close()
+
+		repairTaskTable := NewMockRepairTaskTable(ctr)
+		repairTaskTable.EXPECT().Name().AnyTimes().Return(proto.RepairTaskType)
+		id1 := uuid.New().String()
+		id2 := uuid.New().String()
+		task1 := proto.VolRepairTask{TaskID: id1}
+		task2 := proto.VolRepairTask{TaskID: id2}
+		record1 := &proto.ArchiveRecord{TaskID: id1, TaskType: proto.RepairTaskType, Content: task1} // task already archive and remove success
+		record2 := &proto.ArchiveRecord{TaskID: id2, TaskType: proto.RepairTaskType, Content: task2} // archive success
+		record3 := &proto.ArchiveRecord{}                                                            // find task failed
+		record4 := &proto.ArchiveRecord{}                                                            // task already archive and remove source failed
+		repairTaskTable.EXPECT().QueryMarkDeleteTasks(any, any).AnyTimes().Return([]*proto.ArchiveRecord{record1, record2, record3, record4}, nil)
+
+		archiveTable.EXPECT().FindTask(any, any).Return(record1, nil)
+		archiveTable.EXPECT().FindTask(any, any).Return(nil, base.ErrNoDocuments)
+		archiveTable.EXPECT().FindTask(any, any).Return(nil, errMock)
+		archiveTable.EXPECT().FindTask(any, any).Return(record4, nil)
+
+		archiveTable.EXPECT().Insert(any, any).Return(errMock)
+		archiveTable.EXPECT().FindTask(any, any).Return(record1, nil)
+		archiveTable.EXPECT().FindTask(any, any).Return(record4, nil)
+		repairTaskTable.EXPECT().RemoveMarkDelete(any, any).Return(nil)
+		repairTaskTable.EXPECT().RemoveMarkDelete(any, any).Return(errMock)
+
+		mgr.RegisterTables(repairTaskTable)
+		mgr.storeRun()
+	}
+}

blobstore/scheduler/balancer.go

@@ -0,0 +1,264 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package scheduler
+
+import (
+	"context"
+	"errors"
+	"sort"
+	"time"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/taskswitch"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/scheduler/base"
+	"github.com/cubefs/cubefs/blobstore/scheduler/client"
+	"github.com/cubefs/cubefs/blobstore/scheduler/db"
+	"github.com/cubefs/cubefs/blobstore/util/log"
+)
+
+// IBalancer define the interface of balance manager
+type IBalancer interface {
+	Migrator
+}
+
+const (
+	collectBalanceTaskPauseS = 5
+)
+
+var (
+	// ErrNoBalanceVunit no balance volume unit on disk
+	ErrNoBalanceVunit = errors.New("no balance volume unit on disk")
+	// ErrTooManyBalancingTasks too many balancing tasks
+	ErrTooManyBalancingTasks = errors.New("too many balancing tasks")
+)
+
+// BalanceMgrConfig balance task manager config
+type BalanceMgrConfig struct {
+	BalanceDiskCntLimit int   `json:"balance_disk_cnt_limit"`
+	MaxDiskFreeChunkCnt int64 `json:"max_disk_free_chunk_cnt"`
+	MinDiskFreeChunkCnt int64 `json:"min_disk_free_chunk_cnt"`
+	MigrateConfig
+}
+
+// BalanceMgr balance manager
+type BalanceMgr struct {
+	IMigrater
+
+	clusterTopology IClusterTopology
+	clusterMgrCli   client.ClusterMgrAPI
+
+	cfg *BalanceMgrConfig
+}
+
+// NewBalanceMgr returns balance manager
+func NewBalanceMgr(
+	clusterMgrCli client.ClusterMgrAPI,
+	volumeUpdater client.IVolumeUpdater,
+	taskSwitch taskswitch.ISwitcher,
+	clusterTopology IClusterTopology,
+	taskTbl db.IMigrateTaskTable,
+	conf *BalanceMgrConfig) *BalanceMgr {
+	mgr := &BalanceMgr{
+		clusterTopology: clusterTopology,
+		clusterMgrCli:   clusterMgrCli,
+		cfg:             conf,
+	}
+	mgr.IMigrater = NewMigrateMgr(clusterMgrCli, volumeUpdater, taskSwitch, taskTbl,
+		&conf.MigrateConfig, proto.BalanceTaskType, conf.ClusterID)
+	mgr.IMigrater.SetLockFailHandleFunc(mgr.IMigrater.FinishTaskInAdvanceWhenLockFail)
+	return mgr
+}
+
+// Run run balance task manager
+func (mgr *BalanceMgr) Run() {
+	go mgr.collectTaskLoop()
+	mgr.IMigrater.Run()
+	go mgr.clearTaskLoop()
+}
+
+// Close close balance task manager
+func (mgr *BalanceMgr) Close() {
+	mgr.clusterTopology.Close()
+	mgr.IMigrater.Close()
+}
+
+func (mgr *BalanceMgr) collectTaskLoop() {
+	t := time.NewTicker(time.Duration(mgr.cfg.CollectTaskIntervalS) * time.Second)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-t.C:
+			mgr.IMigrater.WaitEnable()
+			err := mgr.collectionTask()
+			if err == ErrTooManyBalancingTasks || err == ErrNoBalanceVunit {
+				log.Debugf("no task to collect and sleep: sleep second[%d], err[%+v]", collectBalanceTaskPauseS, err)
+				time.Sleep(time.Duration(collectBalanceTaskPauseS) * time.Second)
+			}
+		case <-mgr.IMigrater.Done():
+			return
+		}
+	}
+}
+
+func (mgr *BalanceMgr) collectionTask() (err error) {
+	span, ctx := trace.StartSpanFromContext(context.Background(), "balance_collectionTask")
+	defer span.Finish()
+
+	needBalanceDiskCnt := mgr.cfg.BalanceDiskCntLimit - mgr.IMigrater.GetMigratingDiskNum()
+	if needBalanceDiskCnt <= 0 {
+		span.Warnf("the number of balancing disk is greater than config: current[%d], conf[%d]",
+			mgr.IMigrater.GetMigratingDiskNum(), mgr.cfg.BalanceDiskCntLimit)
+		return ErrTooManyBalancingTasks
+	}
+
+	// select balance disks
+	disks := mgr.selectDisks(mgr.cfg.MaxDiskFreeChunkCnt, mgr.cfg.MinDiskFreeChunkCnt)
+	span.Debugf("select balance disks: len[%d]", len(disks))
+
+	balanceDiskCnt := 0
+	for _, disk := range disks {
+		err = mgr.genOneBalanceTask(ctx, disk)
+		if err != nil {
+			continue
+		}
+
+		balanceDiskCnt++
+		if balanceDiskCnt >= needBalanceDiskCnt {
+			break
+		}
+	}
+	// if balanceDiskCnt==0, means there is no balance volume unit on disk and need to do collect task later
+	if balanceDiskCnt == 0 {
+		span.Infof("select disks has no balance volume unit on disk: len[%d]", len(disks))
+		return ErrNoBalanceVunit
+	}
+
+	return nil
+}
+
+func (mgr *BalanceMgr) selectDisks(maxFreeChunkCnt, minFreeChunkCnt int64) []*client.DiskInfoSimple {
+	var allDisks []*client.DiskInfoSimple
+	for idcName := range mgr.clusterTopology.GetIDCs() {
+		if idcDisks := mgr.clusterTopology.GetIDCDisks(idcName); idcDisks != nil {
+			if freeChunkCntMax(idcDisks) >= maxFreeChunkCnt {
+				allDisks = append(allDisks, idcDisks...)
+			}
+		}
+	}
+
+	var selected []*client.DiskInfoSimple
+	for _, disk := range allDisks {
+		if !disk.IsHealth() {
+			continue
+		}
+		if ok := mgr.IMigrater.IsMigratingDisk(disk.DiskID); ok {
+			continue
+		}
+		if disk.FreeChunkCnt < minFreeChunkCnt {
+			selected = append(selected, disk)
+		}
+	}
+	return selected
+}
+
+func (mgr *BalanceMgr) genOneBalanceTask(ctx context.Context, diskInfo *client.DiskInfoSimple) (err error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	vuid, err := mgr.selectBalanceVunit(ctx, diskInfo.DiskID)
+	if err != nil {
+		span.Errorf("generate task source failed: disk_id[%d], err[%+v]", diskInfo.DiskID, err)
+		return
+	}
+
+	span.Debugf("select balance volume unit; vuid[%d+, volume_id[%v]", vuid, vuid.Vid())
+	task := &proto.MigrateTask{
+		TaskID:       mgr.genUniqTaskID(vuid.Vid()),
+		State:        proto.MigrateStateInited,
+		SourceIdc:    diskInfo.Idc,
+		SourceDiskID: diskInfo.DiskID,
+		SourceVuid:   vuid,
+	}
+	mgr.IMigrater.AddTask(ctx, task)
+	return
+}
+
+func (mgr *BalanceMgr) selectBalanceVunit(ctx context.Context, diskID proto.DiskID) (vuid proto.Vuid, err error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	vunits, err := mgr.clusterMgrCli.ListDiskVolumeUnits(ctx, diskID)
+	if err != nil {
+		return
+	}
+
+	sortVunitByUsed(vunits)
+
+	for i := range vunits {
+		volInfo, err := mgr.clusterMgrCli.GetVolumeInfo(ctx, vunits[i].Vuid.Vid())
+		if err != nil {
+			span.Errorf("get volume info failed: vid[%d], err[%+v]", vunits[i].Vuid.Vid(), err)
+			continue
+		}
+		if volInfo.IsIdle() {
+			return vunits[i].Vuid, nil
+		}
+	}
+	return vuid, ErrNoBalanceVunit
+}
+
+func (mgr *BalanceMgr) clearTaskLoop() {
+	t := time.NewTicker(time.Duration(mgr.cfg.CheckTaskIntervalS) * time.Second)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-t.C:
+			mgr.IMigrater.WaitEnable()
+			mgr.ClearFinishedTask()
+		case <-mgr.IMigrater.Done():
+			return
+		}
+	}
+}
+
+// ClearFinishedTask clear finished balance task
+func (mgr *BalanceMgr) ClearFinishedTask() {
+	span, ctx := trace.StartSpanFromContext(context.Background(), "balance_ClearFinishedTask")
+	defer span.Finish()
+
+	clearStates := []proto.MigrateState{proto.MigrateStateFinished, proto.MigrateStateFinishedInAdvance}
+	mgr.IMigrater.ClearTasksByStates(ctx, clearStates)
+}
+
+func (mgr *BalanceMgr) genUniqTaskID(vid proto.Vid) string {
+	return base.GenTaskID("balance", vid)
+}
+
+func sortVunitByUsed(vunits []*client.VunitInfoSimple) {
+	sort.Slice(vunits, func(i, j int) bool {
+		return vunits[i].Used < vunits[j].Used
+	})
+}
+
+func freeChunkCntMax(disks []*client.DiskInfoSimple) int64 {
+	var max int64
+	for _, disk := range disks {
+		if disk.FreeChunkCnt > max {
+			max = disk.FreeChunkCnt
+		}
+	}
+	return max
+}

blobstore/scheduler/balancer_test.go

@@ -0,0 +1,225 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package scheduler
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/require"
+
+	api "github.com/cubefs/cubefs/blobstore/api/scheduler"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/scheduler/base"
+	"github.com/cubefs/cubefs/blobstore/scheduler/client"
+	"github.com/cubefs/cubefs/blobstore/testing/mocks"
+	"github.com/cubefs/cubefs/blobstore/util/closer"
+)
+
+func newBalancer(t *testing.T) *BalanceMgr {
+	ctr := gomock.NewController(t)
+	clusterMgr := NewMockClusterMgrAPI(ctr)
+	volumeUpdater := NewMockVolumeUpdater(ctr)
+	taskSwitch := mocks.NewMockSwitcher(ctr)
+	topologyMgr := NewMockClusterTopology(ctr)
+	migrateTable := NewMockMigrateTaskTable(ctr)
+	migrater := NewMockMigrater(ctr)
+	conf := &BalanceMgrConfig{}
+	c := closer.New()
+
+	topologyMgr.EXPECT().Close().AnyTimes().Return()
+	migrater.EXPECT().StatQueueTaskCnt().AnyTimes().Return(0, 0, 0)
+	migrater.EXPECT().Close().AnyTimes().DoAndReturn(c.Close)
+	migrater.EXPECT().Done().AnyTimes().Return(c.Done())
+	migrater.EXPECT().WaitEnable().AnyTimes().Return()
+	migrater.EXPECT().Enabled().AnyTimes().Return(true)
+	mgr := NewBalanceMgr(clusterMgr, volumeUpdater, taskSwitch, topologyMgr, migrateTable, conf)
+	mgr.IMigrater = migrater
+	return mgr
+}
+
+func TestBalanceLoad(t *testing.T) {
+	mgr := newBalancer(t)
+	mgr.IMigrater.(*MockMigrater).EXPECT().Load().Return(nil)
+	err := mgr.Load()
+	require.NoError(t, err)
+}
+
+func TestBalanceRun(t *testing.T) {
+	mgr := newBalancer(t)
+	defer mgr.Close()
+
+	mgr.IMigrater.(*MockMigrater).EXPECT().Run().Return()
+	mgr.IMigrater.(*MockMigrater).EXPECT().ClearTasksByStates(any, any).AnyTimes().Return()
+	mgr.IMigrater.(*MockMigrater).EXPECT().GetMigratingDiskNum().AnyTimes().Return(1)
+	mgr.cfg.CollectTaskIntervalS = 1
+	mgr.cfg.CheckTaskIntervalS = 1
+	require.True(t, mgr.Enabled())
+	mgr.Run()
+
+	time.Sleep(1 * time.Second)
+}
+
+func TestBalanceCollectionTask(t *testing.T) {
+	{
+		mgr := newBalancer(t)
+		mgr.IMigrater.(*MockMigrater).EXPECT().GetMigratingDiskNum().AnyTimes().Return(1)
+
+		err := mgr.collectionTask()
+		require.True(t, errors.Is(err, ErrTooManyBalancingTasks))
+		mgr.Close()
+	}
+	{
+		mgr := newBalancer(t)
+		mgr.cfg.BalanceDiskCntLimit = 2
+		mgr.IMigrater.(*MockMigrater).EXPECT().GetMigratingDiskNum().AnyTimes().Return(1)
+
+		disk1 := &client.DiskInfoSimple{
+			ClusterID:    1,
+			Idc:          "z0",
+			Rack:         "rack1",
+			Host:         "127.0.0.1:8000",
+			Status:       proto.DiskStatusNormal,
+			DiskID:       1,
+			FreeChunkCnt: 10,
+			MaxChunkCnt:  700,
+		}
+		disk2 := &client.DiskInfoSimple{
+			ClusterID:    1,
+			Idc:          "z1",
+			Rack:         "rack1",
+			Host:         "127.0.0.2:8000",
+			Status:       proto.DiskStatusNormal,
+			DiskID:       2,
+			FreeChunkCnt: 100,
+			MaxChunkCnt:  700,
+		}
+		disk3 := &client.DiskInfoSimple{
+			ClusterID:    1,
+			Idc:          "z1",
+			Rack:         "rack1",
+			Host:         "127.0.0.3:8000",
+			Status:       proto.DiskStatusBroken,
+			DiskID:       3,
+			FreeChunkCnt: 20,
+			MaxChunkCnt:  700,
+		}
+		clusterTopMgr := &ClusterTopologyMgr{
+			taskStatsMgr: base.NewClusterTopologyStatisticsMgr(1, []float64{}),
+		}
+		clusterTopMgr.buildClusterTopo([]*client.DiskInfoSimple{disk1, disk2, disk3}, 1)
+		mgr.IMigrater.(*MockMigrater).EXPECT().IsMigratingDisk(any).AnyTimes().DoAndReturn(func(diskID proto.DiskID) bool {
+			return diskID == 1
+		})
+		mgr.clusterTopology = clusterTopMgr
+
+		err := mgr.collectionTask()
+		require.True(t, errors.Is(err, ErrNoBalanceVunit))
+
+		// select one task
+		mgr.cfg.MinDiskFreeChunkCnt = 101
+		volume := MockGenVolInfo(10000, codemode.EC6P6, proto.VolumeStatusIdle)
+		var units []*client.VunitInfoSimple
+		for _, unit := range volume.VunitLocations {
+			ele := client.VunitInfoSimple{
+				Vuid:   unit.Vuid,
+				DiskID: unit.DiskID,
+			}
+			units = append(units, &ele)
+		}
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(volume, nil)
+		mgr.IMigrater.(*MockMigrater).EXPECT().AddTask(any, any).Return()
+		err = mgr.collectionTask()
+		require.NoError(t, err)
+
+		// select one task and gen task failed
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, errMock)
+		err = mgr.collectionTask()
+		require.True(t, errors.Is(err, ErrNoBalanceVunit))
+
+		// select one task and gen task failed
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).AnyTimes().Return(nil, errMock)
+		err = mgr.collectionTask()
+		require.True(t, errors.Is(err, ErrNoBalanceVunit))
+	}
+}
+
+func TestBalanceAcquireTask(t *testing.T) {
+	ctx := context.Background()
+	idc := "z0"
+	mgr := newBalancer(t)
+	mgr.IMigrater.(*MockMigrater).EXPECT().AcquireTask(any, any).Return(&proto.MigrateTask{}, nil)
+	_, err := mgr.AcquireTask(ctx, idc)
+	require.NoError(t, err)
+}
+
+func TestBalanceCancelTask(t *testing.T) {
+	ctx := context.Background()
+	mgr := newBalancer(t)
+	mgr.IMigrater.(*MockMigrater).EXPECT().CancelTask(any, any).Return(nil)
+	err := mgr.CancelTask(ctx, &api.CancelTaskArgs{})
+	require.NoError(t, err)
+}
+
+func TestBalanceReclaimTask(t *testing.T) {
+	ctx := context.Background()
+	idc := "z0"
+	mgr := newBalancer(t)
+	mgr.IMigrater.(*MockMigrater).EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil)
+	t1 := mockGenMigrateTask(idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap)
+	err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{})
+	require.NoError(t, err)
+}
+
+func TestBalanceCompleteTask(t *testing.T) {
+	ctx := context.Background()
+	idc := "z0"
+	mgr := newBalancer(t)
+	mgr.IMigrater.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(nil)
+	t1 := mockGenMigrateTask(idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap)
+	err := mgr.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: idc, TaskId: t1.TaskID, Src: t1.Sources, Dest: t1.Destination})
+	require.NoError(t, err)
+
+	mgr.IMigrater.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(errMock)
+	err = mgr.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: idc, TaskId: t1.TaskID, Src: t1.Sources, Dest: t1.Destination})
+	require.True(t, errors.Is(err, errMock))
+}
+
+func TestBalanceRenewalTask(t *testing.T) {
+	ctx := context.Background()
+	idc := "z0"
+	mgr := newBalancer(t)
+	mgr.IMigrater.(*MockMigrater).EXPECT().RenewalTask(any, any, any).Return(nil)
+	err := mgr.RenewalTask(ctx, idc, "")
+	require.NoError(t, err)
+
+	mgr.IMigrater.(*MockMigrater).EXPECT().RenewalTask(any, any, any).Return(errMock)
+	err = mgr.RenewalTask(ctx, idc, "")
+	require.True(t, errors.Is(err, errMock))
+}
+
+func TestBalanceStatQueueTaskCnt(t *testing.T) {
+	mgr := newBalancer(t)
+	inited, prepared, completed := mgr.StatQueueTaskCnt()
+	require.Equal(t, 0, inited)
+	require.Equal(t, 0, prepared)
+	require.Equal(t, 0, completed)
+}

blobstore/scheduler/base/base_test.go

@@ -0,0 +1,85 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package base
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/Shopify/sarama"
+
+	"github.com/cubefs/cubefs/blobstore/scheduler/db"
+	"github.com/cubefs/cubefs/blobstore/util/log"
+)
+
+//go:generate mockgen -destination=./utils_mock_test.go -package=base -mock_names IAllocVunit=MockAllocVunit github.com/cubefs/cubefs/blobstore/scheduler/base IAllocVunit
+
+const testTopic = "test_topic"
+
+var (
+	_ db.IKafkaOffsetTable = &mockAccess{}
+
+	errMock = errors.New("mock error")
+)
+
+func init() {
+	log.SetOutputLevel(log.Lfatal)
+}
+
+func newBroker(t *testing.T) *sarama.MockBroker {
+	mockFetchResponse := sarama.NewMockFetchResponse(t, 1)
+	mockFetchResponse.SetVersion(1)
+	var msg sarama.ByteEncoder = []byte("FOO")
+	for i := 0; i < 1000; i++ {
+		mockFetchResponse.SetMessage(testTopic, 0, int64(i), msg)
+	}
+
+	broker := sarama.NewMockBrokerAddr(t, 0, "127.0.0.1:0")
+	broker.SetHandlerByMap(map[string]sarama.MockResponse{
+		"MetadataRequest": sarama.NewMockMetadataResponse(t).
+			SetBroker(broker.Addr(), broker.BrokerID()).
+			SetLeader(testTopic, 0, broker.BrokerID()),
+		"OffsetRequest": sarama.NewMockOffsetResponse(t).
+			SetOffset(testTopic, 0, sarama.OffsetOldest, 0).
+			SetOffset(testTopic, 0, sarama.OffsetNewest, 2345),
+		"FetchRequest": mockFetchResponse,
+	})
+
+	return broker
+}
+
+type mockAccess struct {
+	offsets map[string]int64
+	err     error
+}
+
+func newMockAccess(err error) *mockAccess {
+	return &mockAccess{
+		offsets: make(map[string]int64),
+		err:     err,
+	}
+}
+
+func (m *mockAccess) Set(topic string, partition int32, offset int64) error {
+	key := fmt.Sprintf("%s_%d", topic, partition)
+	m.offsets[key] = offset
+	return m.err
+}
+
+func (m *mockAccess) Get(topic string, partition int32) (int64, error) {
+	key := fmt.Sprintf("%s_%d", topic, partition)
+	return m.offsets[key], m.err
+}

blobstore/scheduler/base/kafka_consumer.go

@@ -0,0 +1,222 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package base
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/Shopify/sarama"
+	"go.mongodb.org/mongo-driver/mongo"
+
+	"github.com/cubefs/cubefs/blobstore/common/kafka"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/scheduler/db"
+)
+
+const minConsumeWaitTime = time.Millisecond * 500
+
+// IConsumer define the interface of consumer for message consume
+type IConsumer interface {
+	ConsumeMessages(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage)
+	CommitOffset(ctx context.Context) error
+}
+
+// KafkaConfig kafka config
+type KafkaConfig struct {
+	Topic      string
+	Partitions []int32
+	BrokerList []string
+}
+
+// ConsumeInfo consume info
+type ConsumeInfo struct {
+	Offset int64
+	Commit int64
+}
+
+// TopicConsumer rotate consume msg among partition consumers
+type TopicConsumer struct {
+	partitionsConsumers []IConsumer
+
+	curIdx int
+}
+
+// NewTopicConsumer returns topic round-robin partition consumer
+func NewTopicConsumer(cfg *KafkaConfig, offsetAccessor db.IKafkaOffsetTable) (IConsumer, error) {
+	consumers, err := NewKafkaPartitionConsumers(cfg, offsetAccessor)
+	if err != nil {
+		return nil, err
+	}
+	topicConsumer := &TopicConsumer{
+		partitionsConsumers: consumers,
+	}
+	return topicConsumer, err
+}
+
+// ConsumeMessages consumer messages
+func (c *TopicConsumer) ConsumeMessages(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) {
+	msgs = c.partitionsConsumers[c.curIdx].ConsumeMessages(ctx, msgCnt)
+	c.curIdx = (c.curIdx + 1) % len(c.partitionsConsumers)
+	return
+}
+
+// CommitOffset commit offset
+func (c *TopicConsumer) CommitOffset(ctx context.Context) error {
+	for _, pc := range c.partitionsConsumers {
+		if err := pc.CommitOffset(ctx); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// PartitionConsumer partition consumer
+type PartitionConsumer struct {
+	topic          string
+	partition      int32
+	consumer       sarama.PartitionConsumer
+	consumeInfo    ConsumeInfo
+	offsetAccessor db.IKafkaOffsetTable // consume offset persistence
+}
+
+// NewKafkaPartitionConsumers returns kafka partition consumers
+func NewKafkaPartitionConsumers(cfg *KafkaConfig, offsetAccessor db.IKafkaOffsetTable) ([]IConsumer, error) {
+	var consumers []IConsumer
+	consumer, err := sarama.NewConsumer(cfg.BrokerList, defaultKafkaCfg())
+	if err != nil {
+		return nil, fmt.Errorf("new consumer: err[%w]", err)
+	}
+	if len(cfg.Partitions) == 0 {
+		partitions, err := consumer.Partitions(cfg.Topic)
+		if err != nil {
+			return nil, err
+		}
+		cfg.Partitions = partitions
+	}
+	for _, partition := range cfg.Partitions {
+		partitionConsumer, err := newKafkaPartitionConsumer(consumer, cfg.Topic, partition, offsetAccessor)
+		if err != nil {
+			return nil, fmt.Errorf("new kafka partition consumer: err[%w]", err)
+		}
+		consumers = append(consumers, partitionConsumer)
+	}
+
+	return consumers, nil
+}
+
+func newKafkaPartitionConsumer(consumer sarama.Consumer, topic string, partition int32, offsetAccessor db.IKafkaOffsetTable) (*PartitionConsumer, error) {
+	kafkaConsumer := PartitionConsumer{
+		topic:          topic,
+		offsetAccessor: offsetAccessor,
+	}
+
+	partConsumeInfo, err := kafkaConsumer.loadConsumeInfo(topic, partition)
+	if err != nil {
+		return nil, fmt.Errorf("loadConsumeInfo: topic[%s], err[%w]", topic, err)
+	}
+
+	pc, err := consumer.ConsumePartition(topic, partition, partConsumeInfo.Commit)
+	if err != nil {
+		return nil, fmt.Errorf("consume partition: topic[%s], partition[%d], partConsumeInfo[%+v], err[%w]", topic, partition, partConsumeInfo, err)
+	}
+
+	kafkaConsumer.partition = partition
+	kafkaConsumer.consumer = pc
+	kafkaConsumer.consumeInfo = partConsumeInfo
+
+	return &kafkaConsumer, nil
+}
+
+// ConsumeMessages consume messages
+func (c *PartitionConsumer) ConsumeMessages(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	d := time.Millisecond / 2 * time.Duration(msgCnt) // assume each message cost 0.5 ms
+	if d < minConsumeWaitTime {
+		d = minConsumeWaitTime
+	}
+
+	ticker := time.NewTicker(d)
+	defer ticker.Stop()
+
+	start := time.Now()
+	for {
+		var err error
+		var msg *sarama.ConsumerMessage
+		select {
+		case msg = <-c.consumer.Messages():
+		case err = <-c.consumer.Errors():
+		case <-ticker.C:
+		}
+		if err != nil {
+			span.Errorf("acquire msg failed: topic[%s], partition[%d], err[%+v]", c.topic, c.partition, err)
+			break
+		}
+
+		if msg == nil {
+			span.Debugf("no message for consume and return")
+			break // consume finish,return
+		}
+
+		c.consumeInfo.Offset = msg.Offset
+		msgs = append(msgs, msg)
+		if len(msgs) >= msgCnt {
+			break
+		}
+	}
+
+	span.Debugf("consume info: topic[%s], partition[%d], time cost[%+v], consumer msg numbers[%d], offset[%d], batch msg cnt[%d]",
+		c.topic, c.partition, time.Since(start), len(msgs), c.consumeInfo.Offset, msgCnt)
+	return
+}
+
+// CommitOffset commit offset
+func (c *PartitionConsumer) CommitOffset(ctx context.Context) error {
+	span := trace.SpanFromContextSafe(ctx)
+
+	offset := c.consumeInfo.Offset
+	span.Debugf("start commit offset: offset[%d], topic[%s], partition[%d]", offset, c.topic, c.partition)
+	err := c.offsetAccessor.Set(c.topic, c.partition, offset)
+	if err != nil {
+		span.Errorf("commit offset failed: [%+v]", err)
+		return err
+	}
+	c.consumeInfo.Commit = offset
+	return nil
+}
+
+func (c *PartitionConsumer) loadConsumeInfo(topic string, pt int32) (consumeInfo ConsumeInfo, err error) {
+	commitOffset, err := c.offsetAccessor.Get(topic, pt)
+	if err != nil {
+		if err == mongo.ErrNoDocuments {
+			return ConsumeInfo{Commit: sarama.OffsetOldest, Offset: sarama.OffsetOldest}, nil
+		}
+		return
+	}
+
+	return ConsumeInfo{Commit: commitOffset + 1, Offset: commitOffset}, err
+}
+
+func defaultKafkaCfg() *sarama.Config {
+	cfg := sarama.NewConfig()
+	cfg.Version = kafka.DefaultKafkaVersion
+	cfg.Consumer.Return.Errors = true
+	cfg.Producer.Return.Successes = true
+	cfg.Producer.RequiredAcks = sarama.WaitForAll
+	cfg.Producer.Compression = sarama.CompressionSnappy
+	return cfg
+}

blobstore/scheduler/base/kafka_consumer_test.go

@@ -0,0 +1,248 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package base
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync/atomic"
+	"testing"
+
+	"github.com/Shopify/sarama"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+var (
+	_ sarama.PartitionConsumer = &mockPartitionConsumer{}
+	_ sarama.Consumer          = &mockConsumer{}
+)
+
+type mockPartitionConsumer struct {
+	topic  string
+	pid    int32
+	offset int64
+
+	msgCh chan *sarama.ConsumerMessage
+	errCh chan *sarama.ConsumerError
+}
+
+func newMockPartitionConsumer(topic string, pid int32) *mockPartitionConsumer {
+	return &mockPartitionConsumer{
+		topic:  topic,
+		pid:    pid,
+		offset: 0,
+		msgCh:  make(chan *sarama.ConsumerMessage),
+		errCh:  make(chan *sarama.ConsumerError),
+	}
+}
+
+func (m *mockPartitionConsumer) sendMsg(key, val string) {
+	m.msgCh <- &sarama.ConsumerMessage{
+		Key:       []byte(key),
+		Value:     []byte(val),
+		Topic:     m.topic,
+		Partition: m.pid,
+		Offset:    atomic.LoadInt64(&m.offset),
+	}
+	atomic.AddInt64(&m.offset, 1)
+}
+
+func (m *mockPartitionConsumer) sendErr(err error) {
+	m.errCh <- &sarama.ConsumerError{
+		Topic:     m.topic,
+		Partition: m.pid,
+		Err:       err,
+	}
+}
+
+func (m *mockPartitionConsumer) AsyncClose()                              {}
+func (m *mockPartitionConsumer) Close() error                             { return nil }
+func (m *mockPartitionConsumer) Messages() <-chan *sarama.ConsumerMessage { return m.msgCh }
+func (m *mockPartitionConsumer) Errors() <-chan *sarama.ConsumerError     { return m.errCh }
+func (m *mockPartitionConsumer) HighWaterMarkOffset() int64               { return 0 }
+
+type mockConsumer struct {
+	topics map[string][]*mockPartitionConsumer
+}
+
+func newMockConsumer() *mockConsumer {
+	return &mockConsumer{
+		topics: map[string][]*mockPartitionConsumer{
+			"topic1": newBatchpc("topic1", []int32{1, 2, 3, 4}),
+			"topic2": newBatchpc("topic2", []int32{1, 2, 3, 4}),
+		},
+	}
+}
+
+func newBatchpc(topic string, pids []int32) (ret []*mockPartitionConsumer) {
+	for _, pid := range pids {
+		pc := newMockPartitionConsumer(topic, pid)
+		ret = append(ret, pc)
+	}
+	return ret
+}
+
+func (m *mockConsumer) Run(msgCnt int) {
+	for _, pcs := range m.topics {
+		for _, pc := range pcs {
+			tmpPc := pc
+			go func() {
+				for i := 1; i <= msgCnt; i++ {
+					tmpPc.sendMsg(fmt.Sprintf("key_%d", i), fmt.Sprintf("val_%d", i))
+				}
+			}()
+		}
+	}
+}
+
+func (m *mockConsumer) getPc(topic string, pid int32) *mockPartitionConsumer {
+	pcs := m.topics[topic]
+	for _, pc := range pcs {
+		if pc.pid == pid {
+			return pc
+		}
+	}
+	return nil
+}
+
+func (m *mockConsumer) Topics() ([]string, error) {
+	var ret []string
+	for topic := range m.topics {
+		ret = append(ret, topic)
+	}
+	return ret, nil
+}
+
+func (m *mockConsumer) Partitions(topic string) ([]int32, error) {
+	var pids []int32
+	for _, pc := range m.topics[topic] {
+		pids = append(pids, pc.pid)
+	}
+	return pids, nil
+}
+
+func (m *mockConsumer) ConsumePartition(topic string, partition int32, offset int64) (sarama.PartitionConsumer, error) {
+	for _, pc := range m.topics[topic] {
+		if pc.pid == partition {
+			return pc, nil
+		}
+	}
+	return nil, errors.New("not found")
+}
+
+func (m *mockConsumer) HighWaterMarks() map[string]map[int32]int64 {
+	return nil
+}
+
+func (m *mockConsumer) Close() error {
+	return nil
+}
+
+func TestPartitionConsumer(t *testing.T) {
+	mockConsume := newMockConsumer()
+	access := newMockAccess(nil)
+	pc, err := newKafkaPartitionConsumer(mockConsume, "topic1", 1, access)
+	require.NoError(t, err)
+
+	mockConsume.Run(100)
+	msgs := pc.ConsumeMessages(context.Background(), 5)
+	require.Equal(t, 5, len(msgs))
+
+	err = pc.CommitOffset(context.Background())
+	require.NoError(t, err)
+}
+
+func TestTopicConsume(t *testing.T) {
+	const topic = "topic1"
+	var cs []IConsumer
+	mockConsume := newMockConsumer()
+	access := newMockAccess(nil)
+	for _, pid := range []int32{1, 2, 3} {
+		pc, _ := newKafkaPartitionConsumer(mockConsume, topic, pid, access)
+		cs = append(cs, pc)
+	}
+	mockConsume.Run(100)
+
+	topicConsumer := &TopicConsumer{
+		partitionsConsumers: cs,
+	}
+	for _, pid := range []int32{1, 2, 3} {
+		topicConsumer.ConsumeMessages(context.Background(), 1)
+		err := topicConsumer.CommitOffset(context.Background())
+		require.NoError(t, err)
+		off, err := access.Get(topic, pid)
+		require.NoError(t, err)
+		require.Equal(t, int64(0), off)
+	}
+
+	topicConsumer.ConsumeMessages(context.Background(), 1)
+	err := topicConsumer.CommitOffset(context.Background())
+	require.NoError(t, err)
+	off, err := access.Get(topic, 1)
+	require.NoError(t, err)
+	require.Equal(t, int64(1), off)
+}
+
+func TestConsumerError(t *testing.T) {
+	mockConsume := newMockConsumer()
+	access := newMockAccess(nil)
+	pc, err := newKafkaPartitionConsumer(mockConsume, "topic1", 1, access)
+	require.NoError(t, err)
+	pcTmp := mockConsume.getPc("topic1", 1)
+	go func() { pcTmp.sendErr(errors.New("fake error")) }()
+
+	msgs := pc.ConsumeMessages(context.Background(), 1)
+	require.Equal(t, 0, len(msgs))
+}
+
+func TestLoadConsumeInfo(t *testing.T) {
+	mockConsume := newMockConsumer()
+	access := newMockAccess(mongo.ErrNoDocuments)
+	pc, _ := newKafkaPartitionConsumer(mockConsume, "topic1", 1, access)
+	off, _ := pc.loadConsumeInfo("topic1", 1)
+	require.Equal(t, sarama.OffsetOldest, off.Offset)
+}
+
+func TestNewTopicConsumer(t *testing.T) {
+	broker := newBroker(t)
+	defer broker.Close()
+	cfg := &KafkaConfig{
+		Topic:      testTopic,
+		BrokerList: []string{broker.Addr()},
+		Partitions: []int32{0},
+	}
+
+	access := newMockAccess(nil)
+	consumer, err := NewTopicConsumer(cfg, access)
+	require.NoError(t, err)
+
+	msgs := consumer.ConsumeMessages(context.Background(), 1)
+	require.Equal(t, 1, len(msgs))
+
+	access.err = errMock
+	err = consumer.CommitOffset(context.Background())
+	require.Error(t, err)
+
+	cfg.BrokerList = []string{}
+	_, err = NewTopicConsumer(cfg, access)
+	require.Error(t, err)
+
+	cfg.Partitions = nil
+	cfg.BrokerList = []string{broker.Addr()}
+	_, err = NewTopicConsumer(cfg, access)
+	require.Error(t, err)
+}

blobstore/scheduler/base/kafka_topic_monitor.go

@@ -0,0 +1,90 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package base
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/Shopify/sarama"
+	"go.mongodb.org/mongo-driver/mongo"
+
+	"github.com/cubefs/cubefs/blobstore/common/kafka"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/scheduler/db"
+	"github.com/cubefs/cubefs/blobstore/util/log"
+)
+
+// KafkaTopicMonitor kafka monitor
+type KafkaTopicMonitor struct {
+	topic          string
+	partitions     []int32
+	offsetAccessor db.IKafkaOffsetTable
+	monitor        *kafka.Monitor
+	interval       time.Duration
+}
+
+// NewKafkaTopicMonitor returns kafka topic monitor
+func NewKafkaTopicMonitor(clusterID proto.ClusterID, cfg *KafkaConfig, offsetAccessor db.IKafkaOffsetTable, monitorIntervalS int) (*KafkaTopicMonitor, error) {
+	consumer, err := sarama.NewConsumer(cfg.BrokerList, defaultKafkaCfg())
+	if err != nil {
+		return nil, err
+	}
+
+	partitions, err := consumer.Partitions(cfg.Topic)
+	if err != nil {
+		return nil, fmt.Errorf("get partitions: err[%w]", err)
+	}
+
+	// create kafka monitor
+	monitor, err := kafka.NewKafkaMonitor(clusterID, proto.ServiceNameScheduler, cfg.BrokerList, cfg.Topic, partitions, kafka.DefauleintervalSecs)
+	if err != nil {
+		return nil, fmt.Errorf("new kafka monitor: broker list[%v], topic[%v], parts[%v], error[%w]",
+			cfg.BrokerList, cfg.Topic, partitions, err)
+	}
+
+	interval := time.Second * time.Duration(monitorIntervalS)
+	if interval <= 0 {
+		interval = time.Millisecond
+	}
+	return &KafkaTopicMonitor{
+		topic:          cfg.Topic,
+		partitions:     partitions,
+		offsetAccessor: offsetAccessor,
+		monitor:        monitor,
+		interval:       interval,
+	}, nil
+}
+
+// Run run kafka monitor
+func (m *KafkaTopicMonitor) Run() {
+	ticker := time.NewTicker(m.interval)
+	defer ticker.Stop()
+
+	for {
+		for _, partition := range m.partitions {
+			off, err := m.offsetAccessor.Get(m.topic, partition)
+			if err != nil {
+				if err != mongo.ErrNoDocuments {
+					log.Errorf("get consume offset failed: err[%v]", err)
+				}
+				continue
+			}
+			m.monitor.SetConsumeOffset(off, partition)
+		}
+
+		<-ticker.C
+	}
+}

blobstore/scheduler/base/kafka_topic_monitor_test.go

@@ -0,0 +1,46 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package base
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+func TestNewKafkaTopicMonitor(t *testing.T) {
+	broker := newBroker(t)
+	defer broker.Close()
+	cfg := &KafkaConfig{
+		Topic:      testTopic,
+		BrokerList: []string{broker.Addr()},
+		Partitions: []int32{0},
+	}
+
+	access := newMockAccess(nil)
+	monitor, err := NewKafkaTopicMonitor(proto.ClusterID(1), cfg, access, 0)
+	go func() {
+		monitor.Run()
+	}()
+	time.Sleep(time.Second * 3)
+	require.NoError(t, err)
+
+	cfg.BrokerList = []string{}
+	monitor, err = NewKafkaTopicMonitor(proto.ClusterID(1), cfg, access, 0)
+	require.Error(t, err)
+}

blobstore/scheduler/base/msg_sender.go

@@ -0,0 +1,47 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package base
+
+import "github.com/cubefs/cubefs/blobstore/common/kafka"
+
+// IProducer define the interface of producer
+type IProducer interface {
+	SendMessage(msg []byte) (err error)
+	SendMessages(msgs [][]byte) (err error)
+}
+
+type msgSender struct {
+	topic    string
+	producer kafka.MsgProducer
+}
+
+// NewMsgSender returns message sender
+func NewMsgSender(cfg *kafka.ProducerCfg) (IProducer, error) {
+	producer, err := kafka.NewProducer(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return &msgSender{topic: cfg.Topic, producer: producer}, nil
+}
+
+// SendMessage send message to mq
+func (sender *msgSender) SendMessage(msg []byte) error {
+	return sender.producer.SendMessage(sender.topic, msg)
+}
+
+// SendMessages send message batch
+func (sender *msgSender) SendMessages(msgs [][]byte) error {
+	return sender.producer.SendMessages(sender.topic, msgs)
+}

blobstore/scheduler/base/msg_sender_test.go

@@ -0,0 +1,72 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package base
+
+import (
+	"testing"
+
+	"github.com/Shopify/sarama"
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/common/kafka"
+)
+
+func TestSendMessage(t *testing.T) {
+	kafka.DefaultKafkaVersion = sarama.V0_9_0_1
+
+	seedBroker := sarama.NewMockBrokerAddr(t, 1, "127.0.0.1:0")
+	leader := sarama.NewMockBrokerAddr(t, 2, "127.0.0.1:0")
+
+	metadataResponse := new(sarama.MetadataResponse)
+	metadataResponse.AddBroker(leader.Addr(), leader.BrokerID())
+	metadataResponse.AddTopicPartition(testTopic, 0, leader.BrokerID(), nil, nil, nil, 0)
+	seedBroker.Returns(metadataResponse)
+
+	prodSuccess := new(sarama.ProduceResponse)
+	prodSuccess.AddTopicPartition(testTopic, 0, 0)
+	for i := 0; i < 10; i++ {
+		leader.Returns(prodSuccess)
+	}
+
+	msgSender, err := NewMsgSender(&kafka.ProducerCfg{BrokerList: []string{seedBroker.Addr()}, Topic: testTopic})
+	require.NoError(t, err)
+
+	err = msgSender.SendMessage([]byte("dasdada"))
+	require.NoError(t, err)
+}
+
+func TestSendMessages(t *testing.T) {
+	kafka.DefaultKafkaVersion = sarama.V0_9_0_1
+
+	seedBroker := sarama.NewMockBrokerAddr(t, 1, "127.0.0.1:0")
+	leader := sarama.NewMockBrokerAddr(t, 2, "127.0.0.1:0")
+
+	metadataResponse := new(sarama.MetadataResponse)
+	metadataResponse.AddBroker(leader.Addr(), leader.BrokerID())
+	metadataResponse.AddTopicPartition(testTopic, 0, leader.BrokerID(), nil, nil, nil, 0)
+	seedBroker.Returns(metadataResponse)
+
+	prodSuccess := new(sarama.ProduceResponse)
+	prodSuccess.AddTopicPartition(testTopic, 0, 0)
+	for i := 0; i < 10; i++ {
+		leader.Returns(prodSuccess)
+	}
+
+	msgSender, err := NewMsgSender(&kafka.ProducerCfg{BrokerList: []string{seedBroker.Addr()}, Topic: testTopic})
+	require.NoError(t, err)
+
+	err = msgSender.SendMessages([][]byte{[]byte("dasdada")})
+	require.NoError(t, err)
+}

blobstore/scheduler/base/priority_consumer.go

@@ -0,0 +1,90 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package base
+
+import (
+	"context"
+	"fmt"
+	"sort"
+
+	"github.com/Shopify/sarama"
+
+	"github.com/cubefs/cubefs/blobstore/scheduler/db"
+)
+
+type topicPriority struct {
+	topic    string
+	priority int // The larger the value, the higher the priority
+}
+
+// PriorityConsumerConfig priority consumer
+type PriorityConsumerConfig struct {
+	KafkaConfig
+	Priority int `json:"priority"`
+}
+
+type priorityConsumer struct {
+	topicConsumers      map[string]IConsumer
+	sortedTopicPriority []topicPriority // Sort from largest to smallest
+}
+
+// NewPriorityConsumer return priority consumer
+func NewPriorityConsumer(cfgs []PriorityConsumerConfig, offsetAccessor db.IKafkaOffsetTable) (IConsumer, error) {
+	multiConsumer := priorityConsumer{}
+	multiConsumer.topicConsumers = make(map[string]IConsumer, len(cfgs))
+	multiConsumer.sortedTopicPriority = make([]topicPriority, 0)
+	for _, cfg := range cfgs {
+		cs, err := NewTopicConsumer(&cfg.KafkaConfig, offsetAccessor)
+		if err != nil {
+			return nil, fmt.Errorf("new topic consumer: cfg[%+v], err[%w]", cfg.KafkaConfig, err)
+		}
+
+		multiConsumer.topicConsumers[cfg.KafkaConfig.Topic] = cs
+		multiConsumer.sortedTopicPriority = append(multiConsumer.sortedTopicPriority, topicPriority{
+			topic: cfg.KafkaConfig.Topic, priority: cfg.Priority,
+		})
+	}
+
+	sort.SliceStable(multiConsumer.sortedTopicPriority, func(i, j int) bool {
+		return multiConsumer.sortedTopicPriority[i].priority > multiConsumer.sortedTopicPriority[j].priority
+	})
+
+	return &multiConsumer, nil
+}
+
+// ConsumeMessages consume messages
+func (m *priorityConsumer) ConsumeMessages(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) {
+	remainCnt := msgCnt
+	for _, pair := range m.sortedTopicPriority {
+		consumer := m.topicConsumers[pair.topic]
+		ms := consumer.ConsumeMessages(ctx, remainCnt)
+		msgs = append(msgs, ms...)
+		if len(msgs) >= msgCnt {
+			return msgs
+		}
+		remainCnt = msgCnt - len(msgs)
+	}
+	return
+}
+
+// CommitOffset commit offset
+func (m *priorityConsumer) CommitOffset(ctx context.Context) error {
+	for _, c := range m.topicConsumers {
+		if err := c.CommitOffset(ctx); err != nil {
+			return err
+		}
+	}
+	return nil
+}

blobstore/scheduler/base/priority_consumer_test.go

@@ -0,0 +1,70 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package base
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestPriorityConsumer(t *testing.T) {
+	broker := newBroker(t)
+	defer broker.Close()
+
+	cfgs := []PriorityConsumerConfig{
+		{
+			KafkaConfig: KafkaConfig{
+				Topic:      testTopic,
+				BrokerList: []string{broker.Addr()},
+				Partitions: []int32{0},
+			},
+			Priority: 1,
+		},
+	}
+
+	mockAcc := newMockAccess(nil)
+	priorityConsumer, err := NewPriorityConsumer(cfgs, mockAcc)
+	require.NoError(t, err)
+
+	// Then: messages starting from offset 0 are consumed.
+	for i := 0; i < 10; i++ {
+		msgs := priorityConsumer.ConsumeMessages(context.Background(), 100)
+		if len(msgs) > 0 {
+			require.Equal(t, "FOO", string(msgs[0].Value))
+			err = priorityConsumer.CommitOffset(context.Background())
+			require.NoError(t, err)
+		}
+	}
+
+	mockAcc.err = errMock
+	err = priorityConsumer.CommitOffset(context.Background())
+	require.Error(t, err)
+
+	// NewPriorityConsumer with empty BrokerList
+	cfgs = []PriorityConsumerConfig{
+		{
+			KafkaConfig: KafkaConfig{
+				Topic:      testTopic,
+				BrokerList: nil,
+				Partitions: []int32{0},
+			},
+			Priority: 1,
+		},
+	}
+	_, err = NewPriorityConsumer(cfgs, mockAcc)
+	require.Error(t, err)
+}

blobstore/scheduler/base/queue.go

@@ -0,0 +1,464 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package base
+
+import (
+	"container/list"
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+// queue implement
+// task queue & worker task queue
+
+var (
+	// ErrNoSuchMessageID no such message id
+	ErrNoSuchMessageID = errors.New("no such message id")
+	// ErrUnmatchedVuids unmatched task vuids
+	ErrUnmatchedVuids    = errors.New("unmatched task vuids")
+	errNoSuchIDCQueue    = errors.New("no such idc queue")
+	errExistingMessageID = errors.New("existing message id")
+)
+
+const (
+	msgStateTodo = iota + 1
+	msgStateDoing
+)
+
+// 100 years is long enough。
+const neverTimeout = time.Duration(100*365*24) * time.Hour
+
+// Queue task queue
+type Queue struct {
+	mu    sync.RWMutex
+	todo  *list.List
+	doing *list.List
+	msgs  map[string]*list.Element
+
+	msgTimeout time.Duration // default duration of task locking
+}
+
+// NewQueue return task queue
+func NewQueue(msgTimeout time.Duration) *Queue {
+	if msgTimeout == 0 {
+		msgTimeout = neverTimeout
+	}
+	q := &Queue{
+		todo:       new(list.List),
+		doing:      new(list.List),
+		msgs:       make(map[string]*list.Element),
+		msgTimeout: msgTimeout,
+	}
+	return q
+}
+
+type msgEx struct {
+	id       string
+	state    int
+	deadline time.Time
+	msg      interface{}
+}
+
+// Push push message to queue id is uniquely identifies。
+func (q *Queue) Push(id string, msg interface{}) error {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	if _, ok := q.msgs[id]; ok {
+		return errExistingMessageID
+	}
+
+	m := &msgEx{
+		id:    id,
+		state: msgStateTodo,
+		msg:   msg,
+	}
+	elem := q.todo.PushBack(m)
+	q.msgs[id] = elem
+
+	return nil
+}
+
+// Pop  fetch a msg from queue。
+func (q *Queue) Pop() (string, interface{}, bool) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	now := time.Now()
+	for ele := q.doing.Front(); ele != nil; ele = ele.Next() {
+		m := ele.Value.(*msgEx)
+		if m.deadline.Before(now) {
+			m.deadline = now.Add(q.msgTimeout)
+			return m.id, m.msg, true
+		}
+	}
+
+	// no timeout msg in doing ,fetch from todo
+	if q.todo.Len() == 0 {
+		return "", nil, false
+	}
+	elem := q.todo.Front()
+	q.todo.Remove(elem)
+
+	m := elem.Value.(*msgEx)
+	m.state = msgStateDoing
+	m.deadline = now.Add(q.msgTimeout)
+
+	elem = q.doing.PushFront(m)
+	q.msgs[m.id] = elem
+
+	return m.id, m.msg, true
+}
+
+// Get returns message by id
+func (q *Queue) Get(id string) (interface{}, error) {
+	q.mu.RLock()
+	defer q.mu.RUnlock()
+
+	elem, ok := q.msgs[id]
+	if !ok {
+		return nil, ErrNoSuchMessageID
+	}
+	return elem.Value.(*msgEx).msg, nil
+}
+
+// Requeue :msg while get again after delay。
+func (q *Queue) Requeue(id string, delay time.Duration) error {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	if delay < 0 {
+		delay = 0
+	}
+	elem, ok := q.msgs[id]
+	if !ok {
+		return ErrNoSuchMessageID
+	}
+	m := elem.Value.(*msgEx)
+	if m.state == msgStateTodo {
+		// msg pop and new queue and reload msg, the pop msg state will chang to msgStateTodo
+		// if msg in todo queue then return success。
+		return nil
+	}
+
+	// msg in doing queue。
+	m.deadline = time.Now().Add(delay)
+	return nil
+}
+
+// Remove remove message by id
+func (q *Queue) Remove(id string) error {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	elem, ok := q.msgs[id]
+	if !ok {
+		return ErrNoSuchMessageID
+	}
+	m := elem.Value.(*msgEx)
+	switch m.state {
+	case msgStateTodo:
+		q.todo.Remove(elem)
+	case msgStateDoing:
+		q.doing.Remove(elem)
+	default:
+		panic("invalid msg state")
+	}
+	delete(q.msgs, id)
+	return nil
+}
+
+// Stats returns queue stats
+func (q *Queue) Stats() (todo, doing int) {
+	q.mu.RLock()
+	defer q.mu.RUnlock()
+
+	return q.todo.Len(), q.doing.Len()
+}
+
+// WorkerTask define worker task interface
+type WorkerTask interface {
+	GetSrc() []proto.VunitLocation
+	GetDest() proto.VunitLocation
+	SetDest(dest proto.VunitLocation)
+}
+
+// TaskQueue task queue
+type TaskQueue struct {
+	mu         sync.Mutex
+	queue      *Queue
+	retryDelay time.Duration // punish a period of time to avoid frequent failure retry。
+}
+
+// NewTaskQueue returns task queue
+func NewTaskQueue(retryDelay time.Duration) *TaskQueue {
+	return &TaskQueue{
+		queue:      NewQueue(0),
+		retryDelay: retryDelay,
+	}
+}
+
+// PushTask push task to queue
+func (q *TaskQueue) PushTask(taskID string, task WorkerTask) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	err := q.queue.Push(taskID, task)
+	if err != nil {
+		panic("unexpect push task fail " + err.Error())
+	}
+}
+
+// PopTask return args： taskID, task, flag of task exist
+func (q *TaskQueue) PopTask() (string, WorkerTask, bool) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	taskID, task, exist := q.queue.Pop()
+	if exist {
+		return taskID, task.(WorkerTask), true
+	}
+	return "", nil, false
+}
+
+// RemoveTask remove task by taskID
+func (q *TaskQueue) RemoveTask(taskID string) error {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	return q.queue.Remove(taskID)
+}
+
+// RetryTask retry task by taskID
+func (q *TaskQueue) RetryTask(taskID string) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	err := q.queue.Requeue(taskID, q.retryDelay)
+	if err != nil {
+		panic("unexpect retry task fail:" + err.Error())
+	}
+}
+
+// Query find task by taskID
+func (q *TaskQueue) Query(taskID string) (WorkerTask, bool) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	task, err := q.queue.Get(taskID)
+	if err != nil {
+		return nil, false
+	}
+	return task.(WorkerTask), true
+}
+
+// StatsTasks returns task stats
+func (q *TaskQueue) StatsTasks() (todo int, doing int) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	return q.queue.Stats()
+}
+
+// WorkerTaskQueue task queue for worker
+type WorkerTaskQueue struct {
+	mu        sync.Mutex
+	idcQueues map[string]*Queue
+
+	cancelPunishDuration time.Duration // task cancel will punish a period of time to avoid frequent failure retry
+	leaseExpiredS        time.Duration
+}
+
+// NewWorkerTaskQueue return worker task queue
+func NewWorkerTaskQueue(cancelPunishDuration time.Duration) *WorkerTaskQueue {
+	// extended lock duration of task leasing
+	leaseExpiredS := proto.TaskLeaseExpiredS * time.Second
+
+	return &WorkerTaskQueue{
+		idcQueues:            make(map[string]*Queue),
+		cancelPunishDuration: cancelPunishDuration,
+		leaseExpiredS:        leaseExpiredS,
+	}
+}
+
+// AddPreparedTask add prepared task
+func (q *WorkerTaskQueue) AddPreparedTask(idc, taskID string, wtask WorkerTask) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	idcQueue, ok := q.idcQueues[idc]
+	if !ok {
+		idcQueue = NewQueue(q.leaseExpiredS)
+		q.idcQueues[idc] = idcQueue
+	}
+	err := idcQueue.Push(taskID, wtask)
+	if err != nil {
+		panic("unexpect add prepared task fail:" + err.Error())
+	}
+}
+
+// Acquire acquire task by idc
+func (q *WorkerTaskQueue) Acquire(idc string) (taskID string, wtask WorkerTask, exist bool) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	idcQueue, ok := q.idcQueues[idc]
+	if !ok {
+		return "", nil, false
+	}
+
+	taskID, task, exist := idcQueue.Pop()
+	if exist {
+		return taskID, task.(WorkerTask), exist
+	}
+	return "", nil, false
+}
+
+// Cancel cancel task
+func (q *WorkerTaskQueue) Cancel(idc, taskID string, src []proto.VunitLocation, dst proto.VunitLocation) error {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	idcQueue, ok := q.idcQueues[idc]
+	if !ok {
+		return errNoSuchIDCQueue
+	}
+
+	task, err := idcQueue.Get(taskID)
+	if err != nil {
+		return err
+	}
+	err = checkValid(task.(WorkerTask), src, dst)
+	if err != nil {
+		return err
+	}
+
+	return idcQueue.Requeue(taskID, q.cancelPunishDuration)
+}
+
+// Reclaim reclaim task
+func (q *WorkerTaskQueue) Reclaim(idc, taskID string, src []proto.VunitLocation, oldDest, newDest proto.VunitLocation, newDiskID proto.DiskID) error {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	idcQueue, ok := q.idcQueues[idc]
+	if !ok {
+		return errNoSuchIDCQueue
+	}
+
+	task, err := idcQueue.Get(taskID)
+	if err != nil {
+		return err
+	}
+	wtask := task.(WorkerTask)
+	err = checkValid(wtask, src, oldDest)
+	if err != nil {
+		return err
+	}
+	wtask.SetDest(newDest)
+	return idcQueue.Requeue(taskID, 0)
+}
+
+// Renewal renewal task
+func (q *WorkerTaskQueue) Renewal(idc, taskID string) error {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	idcQueue, ok := q.idcQueues[idc]
+	if !ok {
+		return errNoSuchIDCQueue
+	}
+	return idcQueue.Requeue(taskID, q.leaseExpiredS)
+}
+
+// Complete complete task
+func (q *WorkerTaskQueue) Complete(idc, taskID string, src []proto.VunitLocation, dst proto.VunitLocation) (WorkerTask, error) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	idcQueue, ok := q.idcQueues[idc]
+	if !ok {
+		return nil, errNoSuchIDCQueue
+	}
+
+	task, err := idcQueue.Get(taskID)
+	if err != nil {
+		return nil, err
+	}
+
+	t := task.(WorkerTask)
+	err = checkValid(t, src, dst)
+	if err != nil {
+		return nil, err
+	}
+
+	err = idcQueue.Remove(taskID)
+	if err != nil {
+		panic(fmt.Sprintf("task %s remove form queue fail err %v", taskID, err))
+	}
+
+	return t, err
+}
+
+// StatsTasks returns task stats
+func (q *WorkerTaskQueue) StatsTasks() (todo int, doing int) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	for _, queue := range q.idcQueues {
+		todoTmp, doingTmp := queue.Stats()
+		todo += todoTmp
+		doing += doingTmp
+	}
+	return todo, doing
+}
+
+// Query find task by idc and taskID
+func (q *WorkerTaskQueue) Query(idc, taskID string) (WorkerTask, error) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	idcQueue, ok := q.idcQueues[idc]
+	if !ok {
+		return nil, errNoSuchIDCQueue
+	}
+
+	wt, err := idcQueue.Get(taskID)
+	if err != nil {
+		return nil, err
+	}
+	return wt.(WorkerTask), nil
+}
+
+// SetLeaseExpiredS set lease expired time
+func (q *WorkerTaskQueue) SetLeaseExpiredS(dura time.Duration) {
+	q.leaseExpiredS = dura
+}
+
+func checkValid(task WorkerTask, src []proto.VunitLocation, dst proto.VunitLocation) error {
+	if !vunitSliceEqual(task.GetSrc(), src) || task.GetDest() != dst {
+		return ErrUnmatchedVuids
+	}
+	return nil
+}
+
+func vunitSliceEqual(a, b []proto.VunitLocation) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}

blobstore/scheduler/base/queue_test.go

@@ -0,0 +1,262 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package base
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+func TestQueue(t *testing.T) {
+	q := NewQueue(500 * time.Millisecond)
+	msgID := "msg_1"
+	msgString := "test_msg"
+
+	// test Push
+	err := q.Push(msgID, msgString)
+	require.NoError(t, err)
+	err = q.Push(msgID, msgString)
+	require.EqualError(t, err, errExistingMessageID.Error())
+
+	msg, err := q.Get(msgID)
+	require.NoError(t, err)
+	require.Equal(t, msgString, msg.(string))
+
+	// test Pop
+	id, msg, exist := q.Pop()
+	require.Equal(t, msgID, id)
+	require.Equal(t, msgString, msg)
+	require.Equal(t, true, exist)
+	_, _, exist = q.Pop()
+	require.Equal(t, false, exist)
+	time.Sleep(time.Second)
+	id, msg, exist = q.Pop()
+	require.Equal(t, msgID, id)
+	require.Equal(t, msgString, msg)
+	require.Equal(t, true, exist)
+
+	// test Requeue
+	err = q.Requeue(msgID, 0)
+	require.NoError(t, err)
+	id, msg, exist = q.Pop()
+	require.Equal(t, msgID, id)
+	require.Equal(t, msgString, msg)
+	require.Equal(t, true, exist)
+
+	err = q.Requeue(msgID, 100*time.Millisecond)
+	require.NoError(t, err)
+	time.Sleep(100 * time.Millisecond)
+	id, msg, exist = q.Pop()
+	require.Equal(t, msgID, id)
+	require.Equal(t, msgString, msg)
+	require.Equal(t, true, exist)
+	// test q.Stats()
+	msgID2 := "msg_2"
+	msgString2 := "test_msg2"
+	err = q.Push(msgID2, msgString2)
+	require.NoError(t, err)
+	todo, doing := q.Stats()
+	require.Equal(t, 1, todo)
+	require.Equal(t, 1, doing)
+
+	// test Remove
+	err = q.Remove(msgID)
+	require.NoError(t, err)
+
+	// test no such msg id
+	noSuchMsgID := "NoSuchId"
+	_, err = q.Get(noSuchMsgID)
+	require.EqualError(t, err, ErrNoSuchMessageID.Error())
+
+	err = q.Requeue(noSuchMsgID, 0)
+	require.EqualError(t, err, ErrNoSuchMessageID.Error())
+
+	err = q.Remove(noSuchMsgID)
+	require.EqualError(t, err, ErrNoSuchMessageID.Error())
+}
+
+type mockWorkerTask struct {
+	src []proto.VunitLocation
+	dst proto.VunitLocation
+}
+
+func vunits(vuids []proto.Vuid) []proto.VunitLocation {
+	ret := []proto.VunitLocation{}
+	for _, vuid := range vuids {
+		ret = append(ret, proto.VunitLocation{Vuid: vuid, Host: "127.0.0.1:xx"})
+	}
+	return ret
+}
+
+func vunit(vuid proto.Vuid) proto.VunitLocation {
+	return proto.VunitLocation{Vuid: vuid, Host: "127.0.0.1:xx"}
+}
+
+func (t *mockWorkerTask) GetSrc() []proto.VunitLocation {
+	return t.src
+}
+
+func (t *mockWorkerTask) GetDest() proto.VunitLocation {
+	return t.dst
+}
+
+func (t *mockWorkerTask) SetDest(dstVuid proto.VunitLocation) {
+	t.dst = dstVuid
+}
+
+func TestTaskQueue(t *testing.T) {
+	// test Push
+	taskID1 := "task_id1"
+	task1 := mockWorkerTask{src: vunits([]proto.Vuid{1, 2, 3}), dst: vunit(4)}
+
+	q := NewTaskQueue(100 * time.Millisecond)
+	q.PushTask(taskID1, &task1)
+
+	_, ok := q.Query(taskID1)
+	require.Equal(t, true, ok)
+
+	// test PopTask
+	id, wt, exist := q.PopTask()
+	require.Equal(t, true, exist)
+	require.Equal(t, id, taskID1)
+	require.Equal(t, task1.GetSrc(), wt.GetSrc())
+	require.Equal(t, task1.GetDest(), wt.GetDest())
+	_, _, exist = q.PopTask()
+	require.Equal(t, false, exist)
+
+	// test RetryTask
+	q.RetryTask(taskID1)
+	time.Sleep(100 * time.Millisecond)
+	id, wt, exist = q.PopTask()
+	require.Equal(t, true, exist)
+	require.Equal(t, id, taskID1)
+	require.Equal(t, vunits([]proto.Vuid{1, 2, 3}), wt.GetSrc())
+	require.Equal(t, vunit(4), wt.GetDest())
+
+	// test Stats
+	taskID2 := "task_id2"
+	task2 := mockWorkerTask{src: vunits([]proto.Vuid{3, 4, 5}), dst: vunit(6)}
+	q.PushTask(taskID2, &task2)
+	todo, doing := q.StatsTasks()
+	require.Equal(t, 1, todo)
+	require.Equal(t, 1, doing)
+
+	// test Remove
+	err := q.RemoveTask(taskID1)
+	require.NoError(t, err)
+	err = q.RemoveTask(taskID2)
+	require.NoError(t, err)
+	todo, doing = q.StatsTasks()
+	require.Equal(t, 0, todo)
+	require.Equal(t, 0, doing)
+
+	// test no such msg id
+	noSuchTaskID := "NoSuchId"
+	err = q.RemoveTask(noSuchTaskID)
+	require.EqualError(t, err, ErrNoSuchMessageID.Error())
+}
+
+func newTestWorkerTaskQueue(cancelPunishDuration, renewDuration time.Duration) *WorkerTaskQueue {
+	return &WorkerTaskQueue{
+		idcQueues:            make(map[string]*Queue),
+		cancelPunishDuration: cancelPunishDuration,
+		leaseExpiredS:        renewDuration,
+	}
+}
+
+func TestWorkerTaskQueue(t *testing.T) {
+	taskID1 := "task_id1"
+	idc := "z0"
+	task1 := mockWorkerTask{src: vunits([]proto.Vuid{1, 2, 3}), dst: vunit(4)}
+
+	cancelPunishDuration := 100 * time.Millisecond
+	renewDuration := 200 * time.Millisecond
+
+	// test AddPreparedTask
+	wq := newTestWorkerTaskQueue(cancelPunishDuration, renewDuration)
+	wq.AddPreparedTask(idc, taskID1, &task1)
+
+	// test acquire
+	id, wt, exist := wq.Acquire(idc)
+	require.Equal(t, true, exist)
+	require.Equal(t, id, taskID1)
+	require.Equal(t, wt.GetSrc(), task1.GetSrc())
+	require.Equal(t, wt.GetDest(), task1.GetDest())
+
+	_, _, exist = wq.Acquire(idc)
+	require.Equal(t, false, exist)
+	time.Sleep(renewDuration)
+	id, wt, exist = wq.Acquire(idc)
+	require.Equal(t, true, exist)
+	require.Equal(t, id, taskID1)
+	require.Equal(t, wt.GetSrc(), task1.GetSrc())
+	require.Equal(t, wt.GetDest(), task1.GetDest())
+
+	// test Cancel
+	err := wq.Cancel(idc, taskID1, task1.GetSrc(), task1.GetDest())
+	require.NoError(t, err)
+	_, _, exist = wq.Acquire(idc)
+	require.Equal(t, false, exist)
+	time.Sleep(cancelPunishDuration)
+	id, wt, exist = wq.Acquire(idc)
+	require.Equal(t, true, exist)
+	require.Equal(t, id, taskID1)
+	require.Equal(t, wt.GetSrc(), task1.GetSrc())
+	require.Equal(t, wt.GetDest(), task1.GetDest())
+
+	// test Reclaim
+	err = wq.Reclaim(idc, taskID1, task1.GetSrc(), task1.GetDest(), vunit(6), 0)
+	require.NoError(t, err)
+	id, wt, exist = wq.Acquire(idc)
+	require.Equal(t, true, exist)
+	require.Equal(t, id, taskID1)
+	require.Equal(t, wt.GetSrc(), vunits([]proto.Vuid{1, 2, 3}))
+	require.Equal(t, wt.GetDest(), vunit(6))
+
+	// test Renewal
+	err = wq.Renewal(idc, taskID1)
+	require.NoError(t, err)
+	_, _, exist = wq.Acquire(idc)
+	require.Equal(t, false, exist)
+	time.Sleep(renewDuration)
+	id, wt, exist = wq.Acquire(idc)
+	require.Equal(t, true, exist)
+	require.Equal(t, id, taskID1)
+	require.Equal(t, wt.GetSrc(), vunits([]proto.Vuid{1, 2, 3}))
+	require.Equal(t, wt.GetDest(), vunit(6))
+	// test Complete
+	_, err = wq.Complete(idc, taskID1, vunits([]proto.Vuid{1, 2, 3}), vunit(6))
+	require.NoError(t, err)
+	todo, doing := wq.StatsTasks()
+	require.Equal(t, 0, todo)
+	require.Equal(t, 0, doing)
+
+	// test ErrUnmatchedVuids
+	taskID2 := "task_id2"
+	task2 := mockWorkerTask{src: vunits([]proto.Vuid{1, 2, 3}), dst: vunit(4)}
+	wq = NewWorkerTaskQueue(cancelPunishDuration)
+	wq.AddPreparedTask(idc, taskID2, &task2)
+
+	err = wq.Cancel(idc, taskID2, vunits([]proto.Vuid{4, 5, 6}), vunit(4))
+	require.EqualError(t, err, ErrUnmatchedVuids.Error())
+	err = wq.Reclaim(idc, taskID2, vunits([]proto.Vuid{4, 5, 6}), vunit(4), vunit(5), 0)
+	require.EqualError(t, err, ErrUnmatchedVuids.Error())
+	_, err = wq.Complete(idc, taskID2, vunits([]proto.Vuid{4, 5, 6}), vunit(4))
+	require.EqualError(t, err, ErrUnmatchedVuids.Error())
+}

blobstore/scheduler/base/statistics_metrics.go

@@ -0,0 +1,398 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package base
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/cubefs/cubefs/blobstore/common/counter"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	api "github.com/cubefs/cubefs/blobstore/scheduler/client"
+)
+
+const (
+	defaultTaskCntReportIntervalS = 15
+	namespace                     = "scheduler"
+)
+
+// Buckets default buckets for stats
+var Buckets = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000}
+
+// ClusterTopologyStatsMgr cluster topology stats manager
+type ClusterTopologyStatsMgr struct {
+	freeChunkCntRangeProHis *prometheus.HistogramVec
+}
+
+// NewClusterTopologyStatisticsMgr returns cluster topology stats manager
+func NewClusterTopologyStatisticsMgr(clusterID proto.ClusterID, buckets []float64) *ClusterTopologyStatsMgr {
+	labels := map[string]string{
+		"cluster_id": fmt.Sprintf("%d", clusterID),
+	}
+	namespace := "scheduler"
+	if len(buckets) == 0 {
+		buckets = Buckets
+	}
+	freeChunkCntRangeProHis := prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace:   namespace,
+			Name:        "free_chunk_cnt_range",
+			Help:        "free chunk cnt range",
+			Buckets:     buckets,
+			ConstLabels: labels,
+		},
+		[]string{"rack", "idc"},
+	)
+	if err := prometheus.Register(freeChunkCntRangeProHis); err != nil {
+		if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+			freeChunkCntRangeProHis = are.ExistingCollector.(*prometheus.HistogramVec)
+		} else {
+			panic(err)
+		}
+	}
+
+	return &ClusterTopologyStatsMgr{
+		freeChunkCntRangeProHis: freeChunkCntRangeProHis,
+	}
+}
+
+// ReportFreeChunk report free chunk
+func (statsMgr *ClusterTopologyStatsMgr) ReportFreeChunk(disk *api.DiskInfoSimple) {
+	statsMgr.freeChunkCntRangeProHis.WithLabelValues(disk.Rack, disk.Idc).Observe(float64(disk.FreeChunkCnt))
+}
+
+// TaskCntStats information of task running on worker
+type TaskCntStats interface {
+	StatQueueTaskCnt() (preparing, workerDoing, finishing int)
+}
+
+// TaskRunDetailInfo task run detail info
+type TaskRunDetailInfo struct {
+	Statistics   proto.TaskStatistics `json:"statistics"`
+	StartTime    time.Time            `json:"start_time"`
+	CompleteTime time.Time            `json:"complete_time"`
+	Completed    bool                 `json:"completed"`
+}
+
+// TaskStatsMgr task stats manager
+type TaskStatsMgr struct {
+	mu                  sync.Mutex
+	TaskRunInfos        map[string]TaskRunDetailInfo
+	dataSizeByteCounter counter.Counter
+	shardCntCounter     counter.Counter
+
+	dataSizeProCounter prometheus.Counter
+	shardCntProCounter prometheus.Counter
+
+	taskCntGauge *prometheus.GaugeVec
+
+	reclaimCounter prometheus.Counter
+	cancelCounter  prometheus.Counter
+
+	taskCntStats TaskCntStats
+}
+
+// NewTaskStatsMgrAndRun run task stats manager
+func NewTaskStatsMgrAndRun(clusterID proto.ClusterID, taskType string, taskCntStats TaskCntStats) *TaskStatsMgr {
+	mgr := NewTaskStatsMgr(clusterID, taskType)
+	mgr.taskCntStats = taskCntStats
+	go mgr.ReportTaskCntLoop()
+	return mgr
+}
+
+// NewTaskStatsMgr returns task stats manager
+func NewTaskStatsMgr(clusterID proto.ClusterID, taskType string) *TaskStatsMgr {
+	labels := map[string]string{
+		"cluster_id": fmt.Sprintf("%d", clusterID),
+		"task_type":  taskType,
+		"kind":       KindSuccess,
+	}
+
+	dataSizeProCounter := prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace:   namespace,
+		Subsystem:   "task",
+		Name:        "data_size",
+		Help:        "data size",
+		ConstLabels: labels,
+	})
+
+	shardCntProCounter := prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace:   namespace,
+		Subsystem:   "task",
+		Name:        "shard_cnt",
+		Help:        "shard cnt",
+		ConstLabels: labels,
+	})
+
+	taskCntGauge := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace:   namespace,
+			Subsystem:   "",
+			Name:        "task_cnt",
+			Help:        "task cnt",
+			ConstLabels: labels,
+		}, []string{"task_status"})
+
+	reclaimCounter := prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace:   namespace,
+			Subsystem:   "task",
+			Name:        "reclaim",
+			Help:        "task reclaim",
+			ConstLabels: labels,
+		})
+
+	cancelCounter := prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace:   namespace,
+			Subsystem:   "task",
+			Name:        "cancel",
+			Help:        "task cancel",
+			ConstLabels: labels,
+		})
+
+	if err := prometheus.Register(dataSizeProCounter); err != nil {
+		if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+			dataSizeProCounter = are.ExistingCollector.(prometheus.Counter)
+		} else {
+			panic(err)
+		}
+	}
+	if err := prometheus.Register(shardCntProCounter); err != nil {
+		if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+			shardCntProCounter = are.ExistingCollector.(prometheus.Counter)
+		} else {
+			panic(err)
+		}
+	}
+	if err := prometheus.Register(taskCntGauge); err != nil {
+		if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+			taskCntGauge = are.ExistingCollector.(*prometheus.GaugeVec)
+		} else {
+			panic(err)
+		}
+	}
+	if err := prometheus.Register(reclaimCounter); err != nil {
+		if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+			reclaimCounter = are.ExistingCollector.(prometheus.Counter)
+		} else {
+			panic(err)
+		}
+	}
+	if err := prometheus.Register(cancelCounter); err != nil {
+		if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+			cancelCounter = are.ExistingCollector.(prometheus.Counter)
+		} else {
+			panic(err)
+		}
+	}
+
+	mgr := &TaskStatsMgr{
+		TaskRunInfos:       make(map[string]TaskRunDetailInfo),
+		dataSizeProCounter: dataSizeProCounter,
+		shardCntProCounter: shardCntProCounter,
+		taskCntGauge:       taskCntGauge,
+		reclaimCounter:     reclaimCounter,
+		cancelCounter:      cancelCounter,
+	}
+
+	return mgr
+}
+
+// ReportTaskCntLoop report task count
+func (statsMgr *TaskStatsMgr) ReportTaskCntLoop() {
+	t := time.NewTicker(time.Duration(defaultTaskCntReportIntervalS) * time.Second)
+	for range t.C {
+		preparing, workerDoing, finishing := statsMgr.taskCntStats.StatQueueTaskCnt()
+
+		statsMgr.mu.Lock()
+		statsMgr.taskCntGauge.WithLabelValues("preparing").Set(float64(preparing))
+		statsMgr.taskCntGauge.WithLabelValues("worker_doing").Set(float64(workerDoing))
+		statsMgr.taskCntGauge.WithLabelValues("finishing").Set(float64(finishing))
+		statsMgr.mu.Unlock()
+	}
+}
+
+// ReportWorkerTaskStats report worker task stats
+func (statsMgr *TaskStatsMgr) ReportWorkerTaskStats(
+	taskID string,
+	s proto.TaskStatistics,
+	increaseDataSize,
+	increaseShardCnt int) {
+	statsMgr.mu.Lock()
+	defer statsMgr.mu.Unlock()
+
+	var taskRunInfo TaskRunDetailInfo
+	if _, ok := statsMgr.TaskRunInfos[taskID]; ok {
+		taskRunInfo = statsMgr.TaskRunInfos[taskID]
+	} else {
+		taskRunInfo.StartTime = time.Now()
+	}
+
+	taskRunInfo.Statistics = s
+	if taskRunInfo.Statistics.Completed() {
+		taskRunInfo.CompleteTime = time.Now()
+		taskRunInfo.Completed = true
+	}
+
+	statsMgr.TaskRunInfos[taskID] = taskRunInfo
+	statsMgr.dataSizeByteCounter.AddN(increaseDataSize)
+	statsMgr.shardCntCounter.AddN(increaseShardCnt)
+
+	statsMgr.dataSizeProCounter.Add(float64(increaseDataSize))
+	statsMgr.shardCntProCounter.Add(float64(increaseShardCnt))
+}
+
+// ReclaimTask reclaim task
+func (statsMgr *TaskStatsMgr) ReclaimTask() {
+	statsMgr.reclaimCounter.Inc()
+}
+
+// CancelTask cancel task
+func (statsMgr *TaskStatsMgr) CancelTask() {
+	statsMgr.cancelCounter.Inc()
+}
+
+// QueryTaskDetail find task detail info
+func (statsMgr *TaskStatsMgr) QueryTaskDetail(taskID string) (detail TaskRunDetailInfo, err error) {
+	statsMgr.mu.Lock()
+	defer statsMgr.mu.Unlock()
+
+	if info, ok := statsMgr.TaskRunInfos[taskID]; ok {
+		return info, nil
+	}
+
+	return TaskRunDetailInfo{}, errors.New("not found")
+}
+
+// Counters returns task stats counters
+func (statsMgr *TaskStatsMgr) Counters() (increaseDataSize, increaseShardCnt [counter.SLOT]int) {
+	increaseDataSize = statsMgr.dataSizeByteCounter.Show()
+	increaseShardCnt = statsMgr.shardCntCounter.Show()
+	return
+}
+
+// statistics stats
+const (
+	KindFailed  = "failed"
+	KindSuccess = "success"
+)
+
+// NewCounter returns statistics counter
+func NewCounter(clusterID proto.ClusterID, taskType string, kind string) prometheus.Counter {
+	labels := map[string]string{
+		"cluster_id": fmt.Sprintf("%d", clusterID),
+		"task_type":  taskType,
+		"kind":       kind,
+	}
+	shardCntCounter := prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace:   namespace,
+		Subsystem:   "task",
+		Name:        "shard_cnt",
+		Help:        "shard cnt",
+		ConstLabels: labels,
+	})
+	if err := prometheus.Register(shardCntCounter); err != nil {
+		if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+			return are.ExistingCollector.(prometheus.Counter)
+		}
+		panic(err)
+	}
+	return shardCntCounter
+}
+
+// ErrorStats error stats
+type ErrorStats struct {
+	lock        sync.Mutex
+	errMap      map[string]uint64
+	totalErrCnt uint64
+}
+
+// ErrorPercent error percent
+type ErrorPercent struct {
+	err     string
+	percent float64
+	errCnt  uint64
+}
+
+// NewErrorStats returns error stats
+func NewErrorStats() *ErrorStats {
+	es := ErrorStats{
+		errMap: make(map[string]uint64),
+	}
+	return &es
+}
+
+// AddFail add fail statistics
+func (es *ErrorStats) AddFail(err error) {
+	es.lock.Lock()
+	defer es.lock.Unlock()
+	es.totalErrCnt++
+
+	errStr := errStrFormat(err)
+	if _, ok := es.errMap[errStr]; !ok {
+		es.errMap[errStr] = 0
+	}
+	es.errMap[errStr]++
+}
+
+// Stats returns stats
+func (es *ErrorStats) Stats() (statsResult []ErrorPercent, totalErrCnt uint64) {
+	es.lock.Lock()
+	defer es.lock.Unlock()
+
+	var totalCnt uint64
+	for _, cnt := range es.errMap {
+		totalCnt += cnt
+	}
+
+	for err, cnt := range es.errMap {
+		percent := ErrorPercent{
+			err:     err,
+			percent: float64(cnt) / float64(totalCnt),
+			errCnt:  cnt,
+		}
+		statsResult = append(statsResult, percent)
+	}
+
+	sort.Slice(statsResult, func(i, j int) bool {
+		return statsResult[i].percent > statsResult[j].percent
+	})
+
+	return statsResult, es.totalErrCnt
+}
+
+// FormatPrint format print message
+func FormatPrint(statsInfos []ErrorPercent) (res []string) {
+	for _, info := range statsInfos {
+		res = append(res, fmt.Sprintf("%s: %0.2f%%[%d]", info.err, info.percent*100, info.errCnt))
+	}
+	return
+}
+
+func errStrFormat(err error) string {
+	if err == nil || len(err.Error()) == 0 {
+		return ""
+	}
+
+	strSlice := strings.Split(err.Error(), ":")
+	return strSlice[len(strSlice)-1]
+}

blobstore/scheduler/base/statistics_metrics_test.go

@@ -0,0 +1,108 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package base
+
+import (
+	"encoding/json"
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/common/counter"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	api "github.com/cubefs/cubefs/blobstore/scheduler/client"
+)
+
+type mockStats struct{}
+
+func (m *mockStats) StatQueueTaskCnt() (preparing, workerDoing, finishing int) {
+	return 0, 0, 0
+}
+
+func TestTaskStatisticsMgr(t *testing.T) {
+	mgr := NewTaskStatsMgrAndRun(1, proto.RepairTaskType, &mockStats{})
+	mgr.ReportWorkerTaskStats("repair_task_1", proto.TaskStatistics{}, 10, 10)
+
+	_, err := mgr.QueryTaskDetail("repair_task_1")
+	require.NoError(t, err)
+
+	increaseDataSize, increaseShardCnt := mgr.Counters()
+	var increaseDataSizeVec [counter.SLOT]int
+	increaseDataSizeVec[counter.SLOT-1] = 10
+
+	var increaseShardCntVec [counter.SLOT]int
+	increaseShardCntVec[counter.SLOT-1] = 10
+	require.Equal(t, increaseDataSizeVec, increaseDataSize)
+	require.Equal(t, increaseShardCntVec, increaseShardCnt)
+
+	mgr.ReclaimTask()
+	mgr.CancelTask()
+}
+
+func TestNewClusterTopoStatisticsMgr(t *testing.T) {
+	mgr := NewClusterTopologyStatisticsMgr(1, []float64{})
+	disk := &api.DiskInfoSimple{
+		Idc:          "z0",
+		Rack:         "test_rack",
+		FreeChunkCnt: 100,
+	}
+	mgr.ReportFreeChunk(disk)
+}
+
+func TestErrorStats(t *testing.T) {
+	err1 := errors.New("error 1")
+	err2 := errors.New("error 2")
+	err3 := errors.New("error 3")
+
+	es := NewErrorStats()
+	for range [3]struct{}{} {
+		es.AddFail(err1)
+	}
+	for range [5]struct{}{} {
+		es.AddFail(err2)
+	}
+	for range [2]struct{}{} {
+		es.AddFail(err3)
+	}
+
+	infos, _ := es.Stats()
+	res := FormatPrint(infos)
+
+	t.Log(res)
+
+	p, err := json.MarshalIndent(&res, "", "\t")
+	t.Logf("%v -> %s", err, p)
+
+	es2 := NewErrorStats()
+	infos, _ = es2.Stats()
+	p, err = json.MarshalIndent(&infos, "", "\t")
+	t.Logf("%v -> %s", err, p)
+}
+
+func TestErrStrFormat(t *testing.T) {
+	err1 := errors.New("Post http://127.0.0.1:xxx/xxx: EOF")
+	err2 := errors.New("fake error")
+	var err3 error
+
+	require.Equal(t, " EOF", errStrFormat(err1))
+	require.Equal(t, "fake error", errStrFormat(err2))
+	require.Equal(t, "", errStrFormat(err3))
+}
+
+func TestNewCounter(t *testing.T) {
+	counter := NewCounter(0, "", "")
+	require.NotNil(t, counter)
+}

blobstore/scheduler/base/task_types.go

@@ -0,0 +1,67 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package base
+
+import (
+	"errors"
+
+	"go.mongodb.org/mongo-driver/mongo"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/util/defaulter"
+)
+
+const (
+	defaultPrepareQueueRetryDelayS = 10
+	defaultCancelPunishDurationS   = 20
+	defaultFinishQueueRetryDelayS  = 10
+	defaultCollectIntervalS        = 5
+	defaultCheckTaskIntervalS      = 5
+
+	defaultWorkQueueSize = 20
+)
+
+const (
+	// EmptyDiskID empty diskID
+	EmptyDiskID = proto.DiskID(0)
+)
+
+// err use for task
+var (
+	ErrNoTaskInQueue     = errors.New("no task in queue")
+	ErrVolNotOnlyOneTask = errors.New("vol not only one task running")
+	ErrUpdateVolumeCache = errors.New("update volume cache failed")
+	ErrNoDocuments       = mongo.ErrNoDocuments
+)
+
+// TaskCommonConfig task common config
+type TaskCommonConfig struct {
+	PrepareQueueRetryDelayS int `json:"prepare_queue_retry_delay_s"`
+	FinishQueueRetryDelayS  int `json:"finish_queue_retry_delay_s"`
+	CancelPunishDurationS   int `json:"cancel_punish_duration_s"`
+	WorkQueueSize           int `json:"work_queue_size"`
+	CollectTaskIntervalS    int `json:"collect_task_interval_s"`
+	CheckTaskIntervalS      int `json:"check_task_interval_s"`
+}
+
+// CheckAndFix check and fix task common config
+func (conf *TaskCommonConfig) CheckAndFix() {
+	defaulter.LessOrEqual(&conf.PrepareQueueRetryDelayS, defaultPrepareQueueRetryDelayS)
+	defaulter.LessOrEqual(&conf.FinishQueueRetryDelayS, defaultFinishQueueRetryDelayS)
+	defaulter.LessOrEqual(&conf.CancelPunishDurationS, defaultCancelPunishDurationS)
+	defaulter.LessOrEqual(&conf.WorkQueueSize, defaultWorkQueueSize)
+	defaulter.LessOrEqual(&conf.CollectTaskIntervalS, defaultCollectIntervalS)
+	defaulter.LessOrEqual(&conf.CheckTaskIntervalS, defaultCheckTaskIntervalS)
+}

blobstore/scheduler/base/task_types_test.go

@@ -0,0 +1,32 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package base
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestCommonCheckAndFix(t *testing.T) {
+	cfg := TaskCommonConfig{}
+	cfg.CheckAndFix()
+	require.Equal(t, defaultPrepareQueueRetryDelayS, cfg.PrepareQueueRetryDelayS)
+	require.Equal(t, defaultFinishQueueRetryDelayS, cfg.FinishQueueRetryDelayS)
+	require.Equal(t, defaultCancelPunishDurationS, cfg.CancelPunishDurationS)
+	require.Equal(t, defaultWorkQueueSize, cfg.WorkQueueSize)
+	require.Equal(t, defaultCollectIntervalS, cfg.CollectTaskIntervalS)
+	require.Equal(t, defaultCheckTaskIntervalS, cfg.CheckTaskIntervalS)
+}

blobstore/scheduler/base/utils.go

@@ -0,0 +1,129 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package base
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"time"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+
+	"github.com/cubefs/cubefs/blobstore/common/counter"
+	"github.com/cubefs/cubefs/blobstore/common/errors"
+	comproto "github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/scheduler/client"
+	"github.com/cubefs/cubefs/blobstore/util/retry"
+)
+
+// IAllocVunit define the interface of clustermgr used for volume alloc
+type IAllocVunit interface {
+	AllocVolumeUnit(ctx context.Context, vuid comproto.Vuid) (ret *client.AllocVunitInfo, err error)
+}
+
+// AllocVunitSafe alloc volume unit safe
+func AllocVunitSafe(
+	ctx context.Context,
+	cli IAllocVunit,
+	vuid comproto.Vuid,
+	volReplicas []comproto.VunitLocation) (ret *client.AllocVunitInfo, err error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	allocVunit, err := cli.AllocVolumeUnit(ctx, vuid)
+	if err != nil {
+		return nil, err
+	}
+
+	// safety check
+	for _, repl := range volReplicas {
+		if repl.Vuid.Index() == allocVunit.Vuid.Index() {
+			// allow alloc on same disk with old chunk
+			continue
+		}
+		if repl.DiskID == allocVunit.DiskID {
+			span.Panic("alloc chunk and others chunks are on same disk")
+		}
+	}
+
+	return allocVunit, nil
+}
+
+// Subtraction c = a - b
+func Subtraction(a, b []comproto.Vuid) (c []comproto.Vuid) {
+	m := make(map[comproto.Vuid]struct{})
+	for _, vuid := range b {
+		m[vuid] = struct{}{}
+	}
+
+	for _, vuid := range a {
+		if _, ok := m[vuid]; !ok {
+			c = append(c, vuid)
+		}
+	}
+	return c
+}
+
+// GenTaskID return task id
+func GenTaskID(prefix string, vid comproto.Vid) string {
+	return fmt.Sprintf("%s-%d-%v", prefix, vid, primitive.NewObjectID().Hex())
+}
+
+// DataMountFormat format data
+func DataMountFormat(dataMountBytes [counter.SLOT]int) string {
+	var formatStr []string
+	for _, dataMount := range dataMountBytes {
+		formatStr = append(formatStr, bytesCntFormat(dataMount))
+	}
+	return fmt.Sprint(formatStr)
+}
+
+func bytesCntFormat(bytesCnt int) string {
+	units := []string{"B", "KB", "MB", "GB", "TB", "PB"}
+	idx := 0
+	bytesCnt2 := bytesCnt
+	for {
+		bytesCnt2 = bytesCnt2 / 1024
+		if bytesCnt2 == 0 {
+			break
+		}
+		idx++
+		if idx == 5 {
+			break
+		}
+	}
+
+	num := float64(bytesCnt) / math.Pow(float64(1024), float64(idx))
+	return fmt.Sprintf("%.3f%s", num, units[idx])
+}
+
+// ShouldAllocAndRedo return true if should alloc and redo task
+func ShouldAllocAndRedo(errCode int) bool {
+	if errCode == errors.CodeNewVuidNotMatch ||
+		errCode == errors.CodeStatChunkFailed {
+		return true
+	}
+	return false
+}
+
+func InsistOn(ctx context.Context, errMsg string, on func() error) {
+	attempt := 0
+	retry.InsistContext(ctx, time.Second, on, func(err error) {
+		attempt++
+		span := trace.SpanFromContextSafe(ctx)
+		span.Errorf("insist attempt-%d: %s %s", attempt, errMsg, err.Error())
+	})
+}

blobstore/scheduler/base/utils_mock_test.go

@@ -0,0 +1,52 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/cubefs/cubefs/blobstore/scheduler/base (interfaces: IAllocVunit)
+
+// Package base is a generated GoMock package.
+package base
+
+import (
+	context "context"
+	reflect "reflect"
+
+	proto "github.com/cubefs/cubefs/blobstore/common/proto"
+	client "github.com/cubefs/cubefs/blobstore/scheduler/client"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockAllocVunit is a mock of IAllocVunit interface.
+type MockAllocVunit struct {
+	ctrl     *gomock.Controller
+	recorder *MockAllocVunitMockRecorder
+}
+
+// MockAllocVunitMockRecorder is the mock recorder for MockAllocVunit.
+type MockAllocVunitMockRecorder struct {
+	mock *MockAllocVunit
+}
+
+// NewMockAllocVunit creates a new mock instance.
+func NewMockAllocVunit(ctrl *gomock.Controller) *MockAllocVunit {
+	mock := &MockAllocVunit{ctrl: ctrl}
+	mock.recorder = &MockAllocVunitMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockAllocVunit) EXPECT() *MockAllocVunitMockRecorder {
+	return m.recorder
+}
+
+// AllocVolumeUnit mocks base method.
+func (m *MockAllocVunit) AllocVolumeUnit(arg0 context.Context, arg1 proto.Vuid) (*client.AllocVunitInfo, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "AllocVolumeUnit", arg0, arg1)
+	ret0, _ := ret[0].(*client.AllocVunitInfo)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// AllocVolumeUnit indicates an expected call of AllocVolumeUnit.
+func (mr *MockAllocVunitMockRecorder) AllocVolumeUnit(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocVolumeUnit", reflect.TypeOf((*MockAllocVunit)(nil).AllocVolumeUnit), arg0, arg1)
+}

blobstore/scheduler/base/utils_test.go

@@ -0,0 +1,97 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package base
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/common/counter"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+	"github.com/cubefs/cubefs/blobstore/scheduler/client"
+)
+
+func TestSubtraction(t *testing.T) {
+	fromCm := []proto.Vuid{1, 2, 3, 4, 5}
+	fromDb := []proto.Vuid{1, 2, 3}
+	remain := Subtraction(fromCm, fromDb)
+	require.Equal(t, []proto.Vuid{4, 5}, remain)
+}
+
+func TestHumanRead(t *testing.T) {
+	bytesCntFormat(0)
+	bytesCntFormat((1 << 60) + 1)
+}
+
+func TestDataMountBytePrintEx(t *testing.T) {
+	dataMountBytes := [counter.SLOT]int{
+		(1 << 60) + 1,
+		2445979449, 2363491431, 2122318836,
+		4341106710, 3521119887, 3681901617,
+		3697979790, 4244637672, 3424650849,
+		3279947292, 2675967228, 3624579435,
+		4855608246, 5161093533, 5064624495,
+		4855608246, 4984233630, 512, 0,
+	}
+	DataMountFormat(dataMountBytes)
+}
+
+func TestLoopExecUntilSuccess(t *testing.T) {
+	InsistOn(context.Background(), "test", func() error {
+		return nil
+	})
+}
+
+func TestShouldAllocAndRedo(t *testing.T) {
+	err := errcode.ErrNewVuidNotMatch
+	code := rpc.DetectStatusCode(err)
+	redo := ShouldAllocAndRedo(code)
+	require.Equal(t, true, redo)
+}
+
+func TestAllocVunitSafe(t *testing.T) {
+	ctx := context.Background()
+	errMock := errors.New("fake error")
+	volumeAllocClient := NewMockAllocVunit(gomock.NewController(t))
+	vuid1, _ := proto.NewVuid(1, 1, 1)
+	volumeAllocClient.EXPECT().AllocVolumeUnit(gomock.Any(), gomock.Any()).Return(nil, errMock)
+	_, err := AllocVunitSafe(ctx, volumeAllocClient, vuid1, nil)
+	require.True(t, errors.Is(err, errMock))
+
+	vuid2, _ := proto.NewVuid(1, 1, 2)
+	vuid3, _ := proto.NewVuid(1, 2, 1)
+	volumeAllocClient.EXPECT().AllocVolumeUnit(gomock.Any(), gomock.Any()).Return(&client.AllocVunitInfo{VunitLocation: proto.VunitLocation{Vuid: vuid2, DiskID: proto.DiskID(1)}}, nil)
+	allocVunit, err := AllocVunitSafe(ctx, volumeAllocClient, vuid1, []proto.VunitLocation{{Vuid: vuid2, DiskID: proto.DiskID(1)}, {Vuid: vuid3, DiskID: proto.DiskID(2)}})
+	require.NoError(t, err)
+	require.Equal(t, vuid2, allocVunit.Vuid)
+
+	volumeAllocClient.EXPECT().AllocVolumeUnit(gomock.Any(), gomock.Any()).Return(&client.AllocVunitInfo{VunitLocation: proto.VunitLocation{Vuid: vuid2, DiskID: proto.DiskID(2)}}, nil)
+	require.Panics(t, func() {
+		AllocVunitSafe(ctx, volumeAllocClient, vuid1, []proto.VunitLocation{{Vuid: vuid2, DiskID: proto.DiskID(1)}, {Vuid: vuid3, DiskID: proto.DiskID(2)}})
+	})
+}
+
+func TestGenTaskID(t *testing.T) {
+	prefix := "test-"
+	id := GenTaskID(prefix, proto.Vid(1))
+	require.True(t, strings.HasPrefix(id, prefix))
+}

blobstore/scheduler/base/volume_task_locker.go

@@ -0,0 +1,77 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package base
+
+import (
+	"context"
+	"errors"
+	"sync"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+)
+
+// make sure only one task in same volume to run in cluster
+var (
+	// ErrVidTaskConflict vid task conflict
+	ErrVidTaskConflict = errors.New("vid task conflict")
+)
+
+// VolTaskLocker volume task locker
+type VolTaskLocker struct {
+	taskMap map[proto.Vid]struct{}
+	mu      sync.Mutex
+}
+
+// TryLock try lock task volume and return error if there is task doing
+func (m *VolTaskLocker) TryLock(ctx context.Context, vid proto.Vid) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	span := trace.SpanFromContextSafe(ctx)
+	span.Infof("vid %d mutex try lock", vid)
+
+	if _, ok := m.taskMap[vid]; ok {
+		return ErrVidTaskConflict
+	}
+	m.taskMap[vid] = struct{}{}
+	return nil
+}
+
+// Unlock unlock task volume
+func (m *VolTaskLocker) Unlock(ctx context.Context, vid proto.Vid) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	span := trace.SpanFromContextSafe(ctx)
+	span.Infof("vid %d mutex unlock", vid)
+
+	delete(m.taskMap, vid)
+}
+
+var volTaskLocker *VolTaskLocker
+
+// NewVolTaskLockerOnce singleton mode:make sure only one instance in global
+var NewVolTaskLockerOnce sync.Once
+
+// VolTaskLockerInst ensure that only one background task is executing on the same volume
+func VolTaskLockerInst() *VolTaskLocker {
+	NewVolTaskLockerOnce.Do(func() {
+		volTaskLocker = &VolTaskLocker{
+			taskMap: make(map[proto.Vid]struct{}),
+		}
+	})
+	return volTaskLocker
+}

blobstore/scheduler/base/volume_task_locker_test.go

@@ -0,0 +1,46 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package base
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+func MockEmptyVolTaskLocker() {
+	VolTaskLockerInst().mu.Lock()
+	defer VolTaskLockerInst().mu.Unlock()
+	VolTaskLockerInst().taskMap = make(map[proto.Vid]struct{})
+}
+
+func TestVolTaskLocker(t *testing.T) {
+	MockEmptyVolTaskLocker()
+	ctx := context.Background()
+
+	mu := VolTaskLockerInst()
+	mu2 := VolTaskLockerInst()
+	require.Equal(t, mu, mu2)
+	err := mu.TryLock(ctx, 1)
+	require.NoError(t, err)
+	err = mu.TryLock(ctx, 1)
+	require.EqualError(t, err, ErrVidTaskConflict.Error())
+	mu.Unlock(ctx, 1)
+	err = mu.TryLock(ctx, 1)
+	require.NoError(t, err)
+}

blobstore/scheduler/base_mock_test.go

@@ -0,0 +1,115 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/cubefs/cubefs/blobstore/scheduler/base (interfaces: IConsumer,IProducer)
+
+// Package scheduler is a generated GoMock package.
+package scheduler
+
+import (
+	context "context"
+	reflect "reflect"
+
+	sarama "github.com/Shopify/sarama"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockConsumer is a mock of IConsumer interface.
+type MockConsumer struct {
+	ctrl     *gomock.Controller
+	recorder *MockConsumerMockRecorder
+}
+
+// MockConsumerMockRecorder is the mock recorder for MockConsumer.
+type MockConsumerMockRecorder struct {
+	mock *MockConsumer
+}
+
+// NewMockConsumer creates a new mock instance.
+func NewMockConsumer(ctrl *gomock.Controller) *MockConsumer {
+	mock := &MockConsumer{ctrl: ctrl}
+	mock.recorder = &MockConsumerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockConsumer) EXPECT() *MockConsumerMockRecorder {
+	return m.recorder
+}
+
+// CommitOffset mocks base method.
+func (m *MockConsumer) CommitOffset(arg0 context.Context) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "CommitOffset", arg0)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// CommitOffset indicates an expected call of CommitOffset.
+func (mr *MockConsumerMockRecorder) CommitOffset(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CommitOffset", reflect.TypeOf((*MockConsumer)(nil).CommitOffset), arg0)
+}
+
+// ConsumeMessages mocks base method.
+func (m *MockConsumer) ConsumeMessages(arg0 context.Context, arg1 int) []*sarama.ConsumerMessage {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ConsumeMessages", arg0, arg1)
+	ret0, _ := ret[0].([]*sarama.ConsumerMessage)
+	return ret0
+}
+
+// ConsumeMessages indicates an expected call of ConsumeMessages.
+func (mr *MockConsumerMockRecorder) ConsumeMessages(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConsumeMessages", reflect.TypeOf((*MockConsumer)(nil).ConsumeMessages), arg0, arg1)
+}
+
+// MockProducer is a mock of IProducer interface.
+type MockProducer struct {
+	ctrl     *gomock.Controller
+	recorder *MockProducerMockRecorder
+}
+
+// MockProducerMockRecorder is the mock recorder for MockProducer.
+type MockProducerMockRecorder struct {
+	mock *MockProducer
+}
+
+// NewMockProducer creates a new mock instance.
+func NewMockProducer(ctrl *gomock.Controller) *MockProducer {
+	mock := &MockProducer{ctrl: ctrl}
+	mock.recorder = &MockProducerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockProducer) EXPECT() *MockProducerMockRecorder {
+	return m.recorder
+}
+
+// SendMessage mocks base method.
+func (m *MockProducer) SendMessage(arg0 []byte) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "SendMessage", arg0)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// SendMessage indicates an expected call of SendMessage.
+func (mr *MockProducerMockRecorder) SendMessage(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessage", reflect.TypeOf((*MockProducer)(nil).SendMessage), arg0)
+}
+
+// SendMessages mocks base method.
+func (m *MockProducer) SendMessages(arg0 [][]byte) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "SendMessages", arg0)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// SendMessages indicates an expected call of SendMessages.
+func (mr *MockProducerMockRecorder) SendMessages(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessages", reflect.TypeOf((*MockProducer)(nil).SendMessages), arg0)
+}

blobstore/scheduler/blob_deleter.go

@@ -0,0 +1,716 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package scheduler
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"sync"
+	"time"
+
+	"github.com/Shopify/sarama"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/cubefs/cubefs/blobstore/common/counter"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/kafka"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/recordlog"
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+	"github.com/cubefs/cubefs/blobstore/common/taskswitch"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/scheduler/base"
+	"github.com/cubefs/cubefs/blobstore/scheduler/client"
+	"github.com/cubefs/cubefs/blobstore/scheduler/db"
+	"github.com/cubefs/cubefs/blobstore/util/taskpool"
+)
+
+// ITaskRunner define the interface of task running status.
+type ITaskRunner interface {
+	Enabled() bool
+	RunTask()
+	GetTaskStats() (success, failed [counter.SLOT]int)
+	GetErrorStats() (errStats []string, totalErrCnt uint64)
+}
+
+type deleteStatus int
+
+// blob delete status
+const (
+	DelDone = deleteStatus(iota)
+	DelDelay
+	DelFailed
+	DelUnexpect
+)
+
+// ErrVunitLengthNotEqual vunit length not equal
+var ErrVunitLengthNotEqual = errors.New("vunit length not equal")
+
+type deleteStageMgr struct {
+	l         sync.Mutex
+	delStages map[proto.BlobID]*proto.BlobDeleteStage
+}
+
+func (dsm *deleteStageMgr) clear() {
+	dsm.l.Lock()
+	defer dsm.l.Unlock()
+	dsm.delStages = make(map[proto.BlobID]*proto.BlobDeleteStage)
+}
+
+func (dsm *deleteStageMgr) setBlobDelStage(bid proto.BlobID, stage proto.BlobDeleteStage) {
+	dsm.l.Lock()
+	defer dsm.l.Unlock()
+	stageCopy := stage.Copy()
+	dsm.delStages[bid] = &stageCopy
+}
+
+func (dsm *deleteStageMgr) getBlobDelStage(bid proto.BlobID) proto.BlobDeleteStage {
+	dsm.l.Lock()
+	defer dsm.l.Unlock()
+	if stage, exist := dsm.delStages[bid]; exist {
+		return stage.Copy()
+	}
+	return proto.BlobDeleteStage{}
+}
+
+func (dsm *deleteStageMgr) setShardDelStage(bid proto.BlobID, vuid proto.Vuid, stage proto.DeleteStage) {
+	dsm.l.Lock()
+	defer dsm.l.Unlock()
+	if dsm.delStages == nil {
+		dsm.delStages = make(map[proto.BlobID]*proto.BlobDeleteStage)
+	}
+
+	if _, exist := dsm.delStages[bid]; !exist {
+		dsm.delStages[bid] = &proto.BlobDeleteStage{}
+	}
+	dbs := dsm.delStages[bid]
+	dbs.SetStage(vuid.Index(), stage)
+}
+
+func (dsm *deleteStageMgr) hasMarkDel(bid proto.BlobID, vuid proto.Vuid) bool {
+	dsm.l.Lock()
+	defer dsm.l.Unlock()
+	return dsm.stageEqual(bid, vuid, proto.MarkDelStage)
+}
+
+func (dsm *deleteStageMgr) stageEqual(bid proto.BlobID, vuid proto.Vuid, target proto.DeleteStage) bool {
+	if ds, exist := dsm.delStages[bid]; exist {
+		s, ok := ds.Stage(vuid)
+		if ok && s == target {
+			return true
+		}
+	}
+	return false
+}
+
+type delShardRet struct {
+	err  error
+	vuid proto.Vuid
+}
+
+type delBlobRet struct {
+	status deleteStatus
+	err    error
+	delMsg *proto.DeleteMsg
+}
+
+// DelDoc is a delete doc information for logging in dellog
+type DelDoc struct {
+	ClusterID     proto.ClusterID `json:"cid"`
+	Bid           proto.BlobID    `json:"bid"`
+	Vid           proto.Vid       `json:"vid"`
+	Retry         int             `json:"retry"`
+	Time          int64           `json:"t"`
+	ReqID         string          `json:"rid"`
+	ActualDelTime int64           `json:"del_at"` // unix time in S
+}
+
+func toDelDoc(msg proto.DeleteMsg) DelDoc {
+	return DelDoc{
+		ClusterID:     msg.ClusterID,
+		Bid:           msg.Bid,
+		Vid:           msg.Vid,
+		Retry:         msg.Retry,
+		Time:          msg.Time,
+		ReqID:         msg.ReqId,
+		ActualDelTime: time.Now().Unix(),
+	}
+}
+
+// BlobDeleteConfig is blob delete config
+type BlobDeleteConfig struct {
+	ClusterID proto.ClusterID
+
+	TaskPoolSize int `json:"task_pool_size"`
+
+	FailMsgConsumeIntervalMs int64 `json:"fail_msg_consume_interval_ms"`
+
+	NormalHandleBatchCnt int `json:"normal_handle_batch_cnt"`
+	FailHandleBatchCnt   int `json:"fail_handle_batch_cnt"`
+
+	SafeDelayTimeH int64            `json:"safe_delay_time_h"`
+	DeleteLog      recordlog.Config `json:"delete_log"`
+
+	Kafka BlobDeleteKafkaConfig `json:"-"`
+}
+
+func (cfg *BlobDeleteConfig) normalConsumerConfig() *base.KafkaConfig {
+	return &base.KafkaConfig{
+		Topic:      cfg.Kafka.Normal.Topic,
+		Partitions: cfg.Kafka.Normal.Partitions,
+		BrokerList: cfg.Kafka.BrokerList,
+	}
+}
+
+func (cfg *BlobDeleteConfig) failedConsumerConfig() *base.KafkaConfig {
+	return &base.KafkaConfig{
+		Topic:      cfg.Kafka.Failed.Topic,
+		Partitions: cfg.Kafka.Failed.Partitions,
+		BrokerList: cfg.Kafka.BrokerList,
+	}
+}
+
+func (cfg *BlobDeleteConfig) failedProducerConfig() *kafka.ProducerCfg {
+	return &kafka.ProducerCfg{
+		BrokerList: cfg.Kafka.BrokerList,
+		Topic:      cfg.Kafka.Failed.Topic,
+		TimeoutMs:  cfg.Kafka.FailMsgSenderTimeoutMs,
+	}
+}
+
+// BlobDeleteMgr is blob delete manager
+type BlobDeleteMgr struct {
+	taskSwitch *taskswitch.TaskSwitch
+
+	normalConsumer *deleteTopicConsumer
+	failConsumer   *deleteTopicConsumer
+
+	delSuccessCounter      prometheus.Counter
+	delSuccessCounterByMin *counter.Counter
+	delFailCounter         prometheus.Counter
+	delFailCounterByMin    *counter.Counter
+	errStatsDistribution   *base.ErrorStats
+}
+
+// NewBlobDeleteMgr returns blob delete manager
+func NewBlobDeleteMgr(
+	cfg *BlobDeleteConfig,
+	volCache IVolumeCache,
+	offAccessor db.IKafkaOffsetTable,
+	blobnodeCli client.BlobnodeAPI,
+	switchMgr *taskswitch.SwitchMgr,
+) (*BlobDeleteMgr, error) {
+	normalTopicConsumers, err := base.NewKafkaPartitionConsumers(cfg.normalConsumerConfig(), offAccessor)
+	if err != nil {
+		return nil, err
+	}
+
+	failTopicConsumers, err := base.NewKafkaPartitionConsumers(cfg.failedConsumerConfig(), offAccessor)
+	if err != nil {
+		return nil, err
+	}
+
+	failMsgSender, err := base.NewMsgSender(cfg.failedProducerConfig())
+	if err != nil {
+		return nil, err
+	}
+
+	taskSwitch, err := switchMgr.AddSwitch(taskswitch.BlobDeleteSwitchName)
+	if err != nil {
+		return nil, err
+	}
+
+	delLogger, err := recordlog.NewEncoder(&cfg.DeleteLog)
+	if err != nil {
+		return nil, err
+	}
+
+	tp := taskpool.New(cfg.TaskPoolSize, cfg.TaskPoolSize)
+
+	mgr := &BlobDeleteMgr{
+		taskSwitch:             taskSwitch,
+		delSuccessCounter:      base.NewCounter(cfg.ClusterID, "delete", base.KindSuccess),
+		delFailCounter:         base.NewCounter(cfg.ClusterID, "delete", base.KindFailed),
+		errStatsDistribution:   base.NewErrorStats(),
+		delSuccessCounterByMin: &counter.Counter{},
+		delFailCounterByMin:    &counter.Counter{},
+	}
+
+	normalTopicConsumer := &deleteTopicConsumer{
+		taskSwitch: taskSwitch,
+
+		taskPool:          &tp,
+		topicConsumers:    normalTopicConsumers,
+		consumeBatchCnt:   cfg.NormalHandleBatchCnt,
+		consumeIntervalMs: time.Duration(0),
+		safeDelayTime:     time.Duration(cfg.SafeDelayTimeH) * time.Hour,
+		volCache:          volCache,
+		blobnodeCli:       blobnodeCli,
+		failMsgSender:     failMsgSender,
+
+		delSuccessCounter:      mgr.delSuccessCounter,
+		delSuccessCounterByMin: mgr.delSuccessCounterByMin,
+		delFailCounter:         mgr.delFailCounter,
+		delFailCounterByMin:    mgr.delFailCounterByMin,
+		errStatsDistribution:   mgr.errStatsDistribution,
+
+		delLogger: delLogger,
+	}
+
+	failTopicConsumer := &deleteTopicConsumer{
+		taskSwitch: taskSwitch,
+
+		taskPool:          &tp,
+		topicConsumers:    failTopicConsumers,
+		consumeBatchCnt:   cfg.FailHandleBatchCnt,
+		consumeIntervalMs: time.Duration(cfg.FailMsgConsumeIntervalMs) * time.Millisecond,
+		safeDelayTime:     time.Duration(cfg.SafeDelayTimeH) * time.Hour,
+		volCache:          volCache,
+		blobnodeCli:       blobnodeCli,
+		failMsgSender:     failMsgSender,
+
+		delSuccessCounter:      mgr.delSuccessCounter,
+		delSuccessCounterByMin: mgr.delSuccessCounterByMin,
+		delFailCounter:         mgr.delFailCounter,
+		delFailCounterByMin:    mgr.delFailCounterByMin,
+		errStatsDistribution:   mgr.errStatsDistribution,
+
+		delLogger: delLogger,
+	}
+
+	mgr.normalConsumer = normalTopicConsumer
+	mgr.failConsumer = failTopicConsumer
+
+	return mgr, nil
+}
+
+// RunTask consumers delete messages
+func (mgr *BlobDeleteMgr) RunTask() {
+	mgr.normalConsumer.run()
+	mgr.failConsumer.run()
+}
+
+// Enabled returns return if delete task switch is enable, otherwise returns false
+func (mgr *BlobDeleteMgr) Enabled() bool {
+	return mgr.taskSwitch.Enabled()
+}
+
+// GetTaskStats returns task stats
+func (mgr *BlobDeleteMgr) GetTaskStats() (success [counter.SLOT]int, failed [counter.SLOT]int) {
+	return mgr.delSuccessCounterByMin.Show(), mgr.delFailCounterByMin.Show()
+}
+
+// GetErrorStats returns error stats
+func (mgr *BlobDeleteMgr) GetErrorStats() (errStats []string, totalErrCnt uint64) {
+	statsResult, totalErrCnt := mgr.errStatsDistribution.Stats()
+	return base.FormatPrint(statsResult), totalErrCnt
+}
+
+type deleteTopicConsumer struct {
+	taskSwitch *taskswitch.TaskSwitch
+
+	taskPool          *taskpool.TaskPool
+	topicConsumers    []base.IConsumer
+	consumeBatchCnt   int
+	consumeIntervalMs time.Duration
+	safeDelayTime     time.Duration
+
+	volCache    IVolumeCache
+	blobnodeCli client.BlobnodeAPI
+
+	failMsgSender base.IProducer
+	dsm           deleteStageMgr
+
+	// stats
+	delSuccessCounter      prometheus.Counter
+	delSuccessCounterByMin *counter.Counter
+	delFailCounter         prometheus.Counter
+	delFailCounterByMin    *counter.Counter
+	errStatsDistribution   *base.ErrorStats
+
+	// delete log
+	delLogger recordlog.Encoder
+}
+
+func (d *deleteTopicConsumer) run() {
+	for _, consumer := range d.topicConsumers {
+		go func(consumer base.IConsumer) {
+			for {
+				d.taskSwitch.WaitEnable()
+				d.consumeAndDelete(consumer, d.consumeBatchCnt)
+				if d.consumeIntervalMs != time.Duration(0) {
+					time.Sleep(d.consumeIntervalMs)
+				}
+			}
+		}(consumer)
+	}
+}
+
+func (d *deleteTopicConsumer) consumeAndDelete(consumer base.IConsumer, batchCnt int) {
+	span, ctx := trace.StartSpanFromContext(context.Background(), "consumeAndDelete")
+	defer span.Finish()
+
+	if batchCnt <= 0 {
+		batchCnt = 1
+	}
+
+	msgs := consumer.ConsumeMessages(ctx, batchCnt)
+	d.handleMsgBatch(ctx, msgs)
+
+	base.InsistOn(ctx, "deleter consumer.CommitOffset", func() error {
+		return consumer.CommitOffset(ctx)
+	})
+}
+
+func (d *deleteTopicConsumer) handleMsgBatch(ctx context.Context, mqMsgs []*sarama.ConsumerMessage) {
+	span := trace.SpanFromContextSafe(ctx)
+	ctx = trace.ContextWithSpan(ctx, span)
+
+	span.Infof("handle delete msg: len[%d]", len(mqMsgs))
+
+	var msgs []*proto.DeleteMsg
+	if len(mqMsgs) != 0 {
+		ms := unmarshalMsgs(mqMsgs)
+		msgs = DeduplicateMsgs(ctx, ms)
+		span.Infof("deduplicate messages: len[%d]", len(msgs))
+	}
+
+	if len(msgs) != 0 {
+		// clear delete stage before handle batch msgs
+		span.Debugf("dsm clear before delete")
+		d.dsm.clear()
+		for _, m := range msgs {
+			span.Debugf("set blob delete stage: %+v", m.BlobDelStages)
+			d.dsm.setBlobDelStage(m.Bid, m.BlobDelStages)
+		}
+	}
+
+	for len(msgs) != 0 {
+		finishCh := make(chan delBlobRet, len(msgs))
+		for _, m := range msgs {
+			func(delMsg *proto.DeleteMsg) {
+				d.taskPool.Run(func() {
+					d.handleOneMsg(ctx, delMsg, finishCh)
+				})
+			}(m)
+		}
+
+		var delayMsgs []*proto.DeleteMsg
+		var maxDelayMsgTimeStamp int64 = 0
+		for i := 0; i < len(msgs); i++ {
+			ret := <-finishCh
+			switch ret.status {
+			case DelDone:
+				span.Debugf("delete success: vid[%d], bid[%d], reqid[%s]", ret.delMsg.Vid, ret.delMsg.Bid, ret.delMsg.ReqId)
+				d.delSuccessCounterByMin.Add()
+				d.delSuccessCounter.Inc()
+
+			case DelFailed:
+				span.Warnf("delete failed and send msg to fail queue: vid[%d], bid[%d], reqid[%s], retry[%d], err[%+v]",
+					ret.delMsg.Vid, ret.delMsg.Bid, ret.delMsg.ReqId, ret.delMsg.Retry, ret.err)
+				d.delFailCounter.Inc()
+				d.delFailCounterByMin.Add()
+				d.errStatsDistribution.AddFail(ret.err)
+
+				base.InsistOn(ctx, "deleter send2FailQueue", func() error {
+					return d.send2FailQueue(ctx, *ret.delMsg)
+				})
+
+			case DelDelay:
+				if ret.delMsg.Time > maxDelayMsgTimeStamp {
+					maxDelayMsgTimeStamp = ret.delMsg.Time
+				}
+				delayMsgs = append(delayMsgs, ret.delMsg)
+
+			case DelUnexpect:
+				span.Warnf("unexpected result will ignore: msg[%+v], err[%+v]", ret.delMsg, ret.err)
+			}
+		}
+
+		if len(delayMsgs) == 0 {
+			return
+		}
+
+		sleepDuration := d.delayDuration(maxDelayMsgTimeStamp)
+		span.Warnf("blob is protected: util[%+v], sleep[%+v]", time.Unix(maxDelayMsgTimeStamp, 0).Add(d.safeDelayTime), sleepDuration)
+		time.Sleep(sleepDuration)
+		msgs = delayMsgs
+	}
+}
+
+func (d *deleteTopicConsumer) delayDuration(delTimeStamp int64) time.Duration {
+	start := time.Unix(delTimeStamp, 0)
+	now := time.Now()
+	return start.Add(d.safeDelayTime).Sub(now)
+}
+
+func (d *deleteTopicConsumer) handleOneMsg(ctx context.Context, delMsg *proto.DeleteMsg, finishCh chan<- delBlobRet) {
+	if !delMsg.IsValid() {
+		finishCh <- delBlobRet{
+			status: DelUnexpect,
+			err:    proto.ErrInvalidMsg,
+		}
+		return
+	}
+
+	now := time.Now().UTC()
+	if now.Sub(time.Unix(delMsg.Time, 0)) < d.safeDelayTime {
+		finishCh <- delBlobRet{
+			status: DelDelay,
+			delMsg: delMsg,
+		}
+		return
+	}
+	pSpan := trace.SpanFromContextSafe(ctx)
+	pSpan.Infof("start delete msg: [%+v]", delMsg)
+
+	span, tmpCtx := trace.StartSpanFromContextWithTraceID(context.Background(), "handleDeleteMsg", delMsg.ReqId)
+	err := d.deleteWithCheckVolConsistency(tmpCtx, delMsg.Vid, delMsg.Bid)
+	if err != nil {
+		finishCh <- delBlobRet{
+			status: DelFailed,
+			err:    err,
+			delMsg: delMsg,
+		}
+		return
+	}
+
+	delDoc := toDelDoc(*delMsg)
+	err = d.delLogger.Encode(delDoc)
+	if err != nil {
+		span.Warnf("write delete log failed: vid[%d], bid[%d], err[%+v]", delDoc.Vid, delDoc.Bid, err)
+	}
+
+	finishCh <- delBlobRet{
+		status: DelDone,
+		delMsg: delMsg,
+	}
+}
+
+func (d *deleteTopicConsumer) deleteWithCheckVolConsistency(ctx context.Context, vid proto.Vid, bid proto.BlobID) error {
+	return DoubleCheckedRun(ctx, d.volCache, vid, func(info *client.VolumeInfoSimple) error {
+		return d.deleteBlob(ctx, info, bid)
+	})
+}
+
+func (d *deleteTopicConsumer) deleteBlob(ctx context.Context, volInfo *client.VolumeInfoSimple, bid proto.BlobID) (err error) {
+	newVol, err := d.markDelBlob(ctx, volInfo, bid)
+	if err != nil {
+		return
+	}
+
+	_, err = d.delBlob(ctx, newVol, bid)
+	return err
+}
+
+func (d *deleteTopicConsumer) markDelBlob(ctx context.Context, volInfo *client.VolumeInfoSimple, bid proto.BlobID) (*client.VolumeInfoSimple, error) {
+	return d.deleteShards(ctx, volInfo, bid, true)
+}
+
+func (d *deleteTopicConsumer) delBlob(ctx context.Context, volInfo *client.VolumeInfoSimple, bid proto.BlobID) (*client.VolumeInfoSimple, error) {
+	return d.deleteShards(ctx, volInfo, bid, false)
+}
+
+func (d *deleteTopicConsumer) deleteShards(
+	ctx context.Context,
+	volInfo *client.VolumeInfoSimple,
+	bid proto.BlobID,
+	markDelete bool) (new *client.VolumeInfoSimple, err error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	var updateAndRetryShards []proto.Vuid
+	locations := volInfo.VunitLocations
+	vid := volInfo.Vid
+	retCh := make(chan delShardRet, len(locations))
+
+	span.Debugf("delete blob: vid[%d], bid[%d], markDelete[%+v]", vid, bid, markDelete)
+	for _, location := range locations {
+		span.Debugf("delete shards: location[%+v]", location)
+		go func(ctx context.Context, location proto.VunitLocation, bid proto.BlobID, markDelete bool) {
+			err := d.deleteShard(ctx, location, bid, markDelete)
+			retCh <- delShardRet{err: err, vuid: location.Vuid}
+		}(ctx, location, bid, markDelete)
+	}
+
+	for i := 0; i < len(locations); i++ {
+		ret := <-retCh
+		if ret.err != nil {
+			errCode := rpc.DetectStatusCode(ret.err)
+			if shouldUpdateVolumeErr(errCode) {
+				span.Errorf("delete shard failed will retry: bid[%d], vuid[%d], markDelete[%+v], code[%d], err[%+v]",
+					bid, ret.vuid, markDelete, errCode, ret.err)
+				updateAndRetryShards = append(updateAndRetryShards, ret.vuid)
+				err = ret.err
+				continue
+			}
+
+			span.Errorf("delete shard failed: bid[%d], vuid[%d], markDelete[%+v], code[%d], err[%+v]",
+				bid, ret.vuid, markDelete, errCode, ret.err)
+			return volInfo, ret.err
+		}
+	}
+
+	if len(updateAndRetryShards) == 0 {
+		span.Debugf("delete blob success: vid[%d], bid[%d], markDelete[%+v] ", vid, bid, markDelete)
+		return volInfo, nil
+	}
+
+	span.Infof("bid delete will update and retry: len updateAndRetryShards[%d]", len(updateAndRetryShards))
+	// update volCache
+	newVolInfo, updateVolErr := d.volCache.Update(vid)
+	if updateVolErr != nil || newVolInfo.EqualWith(volInfo) {
+		// if update volInfo failed or volInfo not updated, don't need retry
+		span.Warnf("new volInfo is same or volCache.Update failed: vid[%d], err[%+v]", volInfo.Vid, updateVolErr)
+		return volInfo, err
+	}
+
+	if len(newVolInfo.VunitLocations) != len(locations) {
+		span.Warnf("vid locations len not equal: vid[%d], old len[%d], new len[%d]", len(locations), len(newVolInfo.VunitLocations))
+		return volInfo, ErrVunitLengthNotEqual
+	}
+
+	for _, oldVuid := range updateAndRetryShards {
+		idx := oldVuid.Index()
+		newLocation := newVolInfo.VunitLocations[idx]
+		span.Debugf("start retry delete shard: bid[%d]", bid)
+		err := d.deleteShard(ctx, newLocation, bid, markDelete)
+		if err != nil {
+			span.Errorf("retry delete shard: bid[%d], new location[%+v], markDelete[%+v], err[%+v]",
+				bid, newLocation, markDelete, err)
+			return newVolInfo, err
+		}
+	}
+
+	span.Debugf("delete blob success: vid[%d], bid[%d], markDelete[%+v]", vid, bid, markDelete)
+	return newVolInfo, nil
+}
+
+func (d *deleteTopicConsumer) deleteShard(ctx context.Context, location proto.VunitLocation, bid proto.BlobID, markDelete bool) (err error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	// in order to prevent missing delete task,
+	// regardless of whether it is deleted or not, it will be deleted,
+	// just skip mark delete when has mark deleted
+	if d.hasMarkDeleted(location.Vuid, bid, markDelete) {
+		span.Infof("bid has mark deleted and skip: bid[%d], location[%+v]", bid, location)
+		return nil
+	}
+
+	var stage proto.DeleteStage
+	if markDelete {
+		stage = proto.MarkDelStage
+		err = d.blobnodeCli.MarkDelete(ctx, location, bid)
+	} else {
+		stage = proto.DelStage
+		err = d.blobnodeCli.Delete(ctx, location, bid)
+	}
+
+	defer func() {
+		if err == nil {
+			span.Debugf("delete shard set stage: location[%+v], stage[%d]", location, stage)
+			d.dsm.setShardDelStage(bid, location.Vuid, stage)
+			return
+		}
+
+		if shouldBackToInitStage(err) {
+			d.dsm.setShardDelStage(bid, location.Vuid, proto.InitStage)
+		}
+	}()
+
+	if err != nil {
+		errCode := rpc.DetectStatusCode(err)
+		if assumeDeleteSuccess(errCode) {
+			span.Debugf("delete bid failed but assume success: bid[%d], location[%+v], err[%+v] ",
+				bid, location, err)
+			return nil
+		}
+	}
+
+	return
+}
+
+func (d *deleteTopicConsumer) hasMarkDeleted(vuid proto.Vuid, bid proto.BlobID, markDelete bool) bool {
+	return markDelete && d.dsm.hasMarkDel(bid, vuid)
+}
+
+func shouldBackToInitStage(err error) bool {
+	//Simple handling, for all deletion errors, delete tasks are all redone from InitStage
+	//todo:analyze the error codes carefully to determine which ones need to be back to init stage
+	return err != nil
+}
+
+func (d *deleteTopicConsumer) send2FailQueue(ctx context.Context, msg proto.DeleteMsg) error {
+	span := trace.SpanFromContextSafe(ctx)
+
+	// set delete stage
+	delStage := d.dsm.getBlobDelStage(msg.Bid)
+	span.Debugf("send to fail queue: bid[%d], try[%d], delete stages[%+v]", msg.Bid, msg.Retry, delStage)
+	msg.SetDeleteStage(delStage)
+	span.Debugf("delete stage: [%+v]", msg.BlobDelStages)
+
+	msg.Retry++
+	b, err := json.Marshal(msg)
+	if err != nil {
+		// just panic if marsh fail
+		span.Panicf("\"send to fail queue json.Marshal failed: msg[%+v], err[%+v]", msg, err)
+	}
+
+	err = d.failMsgSender.SendMessage(b)
+	if err != nil {
+		span.Errorf("failMsgSender.SendMessage failed: err[%+v]", b)
+		return err
+	}
+	return nil
+}
+
+// for error code judgment
+func shouldUpdateVolumeErr(errCode int) bool {
+	return errCode == errcode.CodeDiskBroken ||
+		errCode == errcode.CodeVuidNotFound ||
+		errCode == errcode.CodeDiskNotFound
+}
+
+func assumeDeleteSuccess(errCode int) bool {
+	return errCode == errcode.CodeBidNotFound ||
+		errCode == errcode.CodeShardMarkDeleted
+}
+
+func unmarshalMsgs(msgs []*sarama.ConsumerMessage) (delMsgs []*proto.DeleteMsg) {
+	for _, msg := range msgs {
+		var delMsg proto.DeleteMsg
+		err := json.Unmarshal(msg.Value, &delMsg)
+		if err != nil {
+			continue
+		}
+		delMsgs = append(delMsgs, &delMsg)
+	}
+	return delMsgs
+}
+
+// DeduplicateMsgs deduplicate delete messages
+func DeduplicateMsgs(ctx context.Context, delMsgs []*proto.DeleteMsg) (msgs []*proto.DeleteMsg) {
+	span := trace.SpanFromContextSafe(ctx)
+	bids := make(map[proto.BlobID]struct{})
+	for _, m := range delMsgs {
+		if _, exist := bids[m.Bid]; !exist {
+			msgs = append(msgs, m)
+			bids[m.Bid] = struct{}{}
+			continue
+		}
+		span.Infof("msg dropped due to same task: msg[%+v]", m)
+	}
+	return
+}

blobstore/scheduler/blob_deleter_test.go

@@ -0,0 +1,363 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package scheduler
+
+import (
+	"context"
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/Shopify/sarama"
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/common/counter"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/recordlog"
+	"github.com/cubefs/cubefs/blobstore/common/taskswitch"
+	"github.com/cubefs/cubefs/blobstore/scheduler/base"
+	"github.com/cubefs/cubefs/blobstore/scheduler/client"
+	"github.com/cubefs/cubefs/blobstore/testing/mocks"
+	"github.com/cubefs/cubefs/blobstore/util/taskpool"
+)
+
+func newDeleteTopicConsumer(t *testing.T) *deleteTopicConsumer {
+	ctr := gomock.NewController(t)
+	clusterMgrCli := NewMockClusterMgrAPI(ctr)
+	clusterMgrCli.EXPECT().GetConfig(any, any).AnyTimes().Return("", nil)
+
+	volCache := NewMockVolumeCache(ctr)
+	volCache.EXPECT().Get(any).AnyTimes().DoAndReturn(
+		func(vid proto.Vid) (*client.VolumeInfoSimple, error) {
+			return &client.VolumeInfoSimple{Vid: vid}, nil
+		},
+	)
+
+	switchMgr := taskswitch.NewSwitchMgr(clusterMgrCli)
+	taskSwitch, err := switchMgr.AddSwitch(taskswitch.BlobDeleteSwitchName)
+	require.NoError(t, err)
+
+	blobnodeCli := NewMockBlobnodeAPI(ctr)
+	blobnodeCli.EXPECT().MarkDelete(any, any, any).AnyTimes().Return(nil)
+	blobnodeCli.EXPECT().Delete(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
+
+	producer := NewMockProducer(ctr)
+	producer.EXPECT().SendMessage(any).AnyTimes().Return(nil)
+	consumer := NewMockConsumer(ctr)
+
+	delLogger := mocks.NewMockEncoder(ctr)
+	delLogger.EXPECT().Close().AnyTimes().Return(nil)
+	delLogger.EXPECT().Encode(any).AnyTimes().Return(nil)
+	tp := taskpool.New(2, 2)
+
+	return &deleteTopicConsumer{
+		taskSwitch:     taskSwitch,
+		topicConsumers: []base.IConsumer{consumer},
+		taskPool:       &tp,
+
+		consumeIntervalMs: time.Duration(0),
+		safeDelayTime:     time.Hour,
+		volCache:          volCache,
+		blobnodeCli:       blobnodeCli,
+		failMsgSender:     producer,
+
+		delSuccessCounter:    base.NewCounter(1, "delete", base.KindSuccess),
+		delFailCounter:       base.NewCounter(1, "delete", base.KindFailed),
+		errStatsDistribution: base.NewErrorStats(),
+		delLogger:            delLogger,
+
+		delSuccessCounterByMin: &counter.Counter{},
+		delFailCounterByMin:    &counter.Counter{},
+	}
+}
+
+func TestDeleteTopicConsumer(t *testing.T) {
+	ctr := gomock.NewController(t)
+	mockTopicConsumeDelete := newDeleteTopicConsumer(t)
+
+	consumer := mockTopicConsumeDelete.topicConsumers[0].(*MockConsumer)
+	consumer.EXPECT().CommitOffset(any).AnyTimes().Return(nil)
+
+	{
+		// nothing todo
+		consumer.EXPECT().ConsumeMessages(any, any).Return([]*sarama.ConsumerMessage{})
+		mockTopicConsumeDelete.consumeAndDelete(consumer, 0)
+	}
+	{
+		// return one invalid message
+		consumer.EXPECT().ConsumeMessages(any, any).DoAndReturn(
+			func(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) {
+				msg := proto.DeleteMsg{}
+				msgByte, _ := json.Marshal(msg)
+				kafkaMgs := &sarama.ConsumerMessage{
+					Value: msgByte,
+				}
+				return []*sarama.ConsumerMessage{kafkaMgs}
+			},
+		)
+		mockTopicConsumeDelete.consumeAndDelete(consumer, 1)
+	}
+	{
+		// return 2 same messages and consume one time
+		consumer.EXPECT().ConsumeMessages(any, any).DoAndReturn(
+			func(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) {
+				msg := proto.DeleteMsg{Bid: 1, Vid: 1, ReqId: "123456"}
+				msgByte, _ := json.Marshal(msg)
+				kafkaMgs := &sarama.ConsumerMessage{
+					Value: msgByte,
+				}
+				return []*sarama.ConsumerMessage{kafkaMgs, kafkaMgs}
+			},
+		)
+		mockTopicConsumeDelete.consumeAndDelete(consumer, 2)
+	}
+	{
+		// return 2 diff messages adn consume success
+		consumer.EXPECT().ConsumeMessages(any, any).DoAndReturn(
+			func(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) {
+				msg := proto.DeleteMsg{Bid: 2, Vid: 2, ReqId: "msg1"}
+				msgByte, _ := json.Marshal(msg)
+				kafkaMgs := &sarama.ConsumerMessage{
+					Value: msgByte,
+				}
+
+				msg2 := proto.DeleteMsg{Bid: 1, Vid: 1, ReqId: "msg2"}
+				msgByte2, _ := json.Marshal(msg2)
+				kafkaMgs2 := &sarama.ConsumerMessage{
+					Value: msgByte2,
+				}
+				return []*sarama.ConsumerMessage{kafkaMgs, kafkaMgs2}
+			},
+		)
+		mockTopicConsumeDelete.consumeAndDelete(consumer, 2)
+	}
+	{
+		// return one message and delete protected
+		oldCache := mockTopicConsumeDelete.volCache
+		volCache := NewMockVolumeCache(ctr)
+		volCache.EXPECT().Get(any).AnyTimes().DoAndReturn(
+			func(vid proto.Vid) (*client.VolumeInfoSimple, error) {
+				return &client.VolumeInfoSimple{
+					Vid:            vid,
+					VunitLocations: []proto.VunitLocation{{Vuid: 1}},
+				}, nil
+			},
+		)
+		mockTopicConsumeDelete.volCache = volCache
+
+		consumer.EXPECT().ConsumeMessages(any, any).DoAndReturn(
+			func(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) {
+				msg := proto.DeleteMsg{
+					Bid:   2,
+					Vid:   2,
+					ReqId: "msg with volume return",
+					Time:  time.Now().Unix() - 1,
+				}
+				msgByte, _ := json.Marshal(msg)
+				kafkaMgs := &sarama.ConsumerMessage{
+					Value: msgByte,
+				}
+				return []*sarama.ConsumerMessage{kafkaMgs}
+			},
+		)
+		mockTopicConsumeDelete.safeDelayTime = 2 * time.Second
+		mockTopicConsumeDelete.consumeAndDelete(consumer, 2)
+		mockTopicConsumeDelete.volCache = oldCache
+	}
+	{
+		// return one message and blobnode delete failed
+		oldCache := mockTopicConsumeDelete.volCache
+		volCache := NewMockVolumeCache(ctr)
+		volCache.EXPECT().Get(any).AnyTimes().DoAndReturn(
+			func(vid proto.Vid) (*client.VolumeInfoSimple, error) {
+				return &client.VolumeInfoSimple{
+					Vid:            vid,
+					VunitLocations: []proto.VunitLocation{{Vuid: 1}},
+				}, nil
+			},
+		)
+		mockTopicConsumeDelete.volCache = volCache
+
+		oldBlobNode := mockTopicConsumeDelete.blobnodeCli
+		blobnodeCli := NewMockBlobnodeAPI(ctr)
+		blobnodeCli.EXPECT().MarkDelete(any, any, any).AnyTimes().Return(errMock)
+		mockTopicConsumeDelete.blobnodeCli = blobnodeCli
+
+		consumer.EXPECT().ConsumeMessages(any, any).DoAndReturn(
+			func(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) {
+				msg := proto.DeleteMsg{Bid: 2, Vid: 2, ReqId: "delete failed"}
+				msgByte, _ := json.Marshal(msg)
+				kafkaMgs := &sarama.ConsumerMessage{
+					Value: msgByte,
+				}
+				return []*sarama.ConsumerMessage{kafkaMgs}
+			},
+		)
+		mockTopicConsumeDelete.consumeAndDelete(consumer, 2)
+		mockTopicConsumeDelete.volCache = oldCache
+		mockTopicConsumeDelete.blobnodeCli = oldBlobNode
+	}
+	{
+		// return one message and blobnode return ErrDiskBroken
+		oldCache := mockTopicConsumeDelete.volCache
+		volCache := NewMockVolumeCache(ctr)
+		volCache.EXPECT().Get(any).AnyTimes().DoAndReturn(
+			func(vid proto.Vid) (*client.VolumeInfoSimple, error) {
+				return &client.VolumeInfoSimple{
+					Vid:            vid,
+					VunitLocations: []proto.VunitLocation{{Vuid: 1}},
+				}, nil
+			},
+		)
+		volCache.EXPECT().Update(any).AnyTimes().DoAndReturn(
+			func(vid proto.Vid) (*client.VolumeInfoSimple, error) {
+				return &client.VolumeInfoSimple{
+					Vid:            vid,
+					VunitLocations: []proto.VunitLocation{{Vuid: 1}},
+				}, nil
+			},
+		)
+		mockTopicConsumeDelete.volCache = volCache
+
+		oldBlobNode := mockTopicConsumeDelete.blobnodeCli
+		blobnodeCli := NewMockBlobnodeAPI(ctr)
+		blobnodeCli.EXPECT().MarkDelete(any, any, any).AnyTimes().Return(errcode.ErrDiskBroken)
+		mockTopicConsumeDelete.blobnodeCli = blobnodeCli
+
+		consumer.EXPECT().ConsumeMessages(any, any).DoAndReturn(
+			func(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) {
+				msg := proto.DeleteMsg{Bid: 2, Vid: 2, ReqId: "delete failed"}
+				msgByte, _ := json.Marshal(msg)
+				kafkaMgs := &sarama.ConsumerMessage{
+					Value: msgByte,
+				}
+				return []*sarama.ConsumerMessage{kafkaMgs}
+			},
+		)
+		mockTopicConsumeDelete.consumeAndDelete(consumer, 2)
+		mockTopicConsumeDelete.volCache = oldCache
+		mockTopicConsumeDelete.blobnodeCli = oldBlobNode
+	}
+	{
+		// return one message, blobnode return ErrDiskBroken, and volCache update not eql
+		oldCache := mockTopicConsumeDelete.volCache
+		volCache := NewMockVolumeCache(ctr)
+		volCache.EXPECT().Get(any).AnyTimes().DoAndReturn(
+			func(vid proto.Vid) (*client.VolumeInfoSimple, error) {
+				return &client.VolumeInfoSimple{
+					Vid:            vid,
+					VunitLocations: []proto.VunitLocation{{Vuid: 1}},
+				}, nil
+			},
+		)
+		volCache.EXPECT().Update(any).DoAndReturn(
+			func(vid proto.Vid) (*client.VolumeInfoSimple, error) {
+				return &client.VolumeInfoSimple{
+					Vid:            vid,
+					VunitLocations: []proto.VunitLocation{{Vuid: 1}, {Vuid: 2}},
+				}, nil
+			},
+		)
+		mockTopicConsumeDelete.volCache = volCache
+
+		oldBlobNode := mockTopicConsumeDelete.blobnodeCli
+		blobnodeCli := NewMockBlobnodeAPI(ctr)
+		blobnodeCli.EXPECT().MarkDelete(any, any, any).AnyTimes().Return(errcode.ErrDiskBroken)
+		mockTopicConsumeDelete.blobnodeCli = blobnodeCli
+
+		consumer.EXPECT().ConsumeMessages(any, any).AnyTimes().DoAndReturn(
+			func(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) {
+				msg := proto.DeleteMsg{Bid: 2, Vid: 2, ReqId: "delete failed"}
+				msgByte, _ := json.Marshal(msg)
+				kafkaMgs := &sarama.ConsumerMessage{
+					Value: msgByte,
+				}
+				return []*sarama.ConsumerMessage{kafkaMgs}
+			},
+		)
+		mockTopicConsumeDelete.consumeAndDelete(consumer, 2)
+
+		volCache.EXPECT().Update(any).AnyTimes().DoAndReturn(
+			func(vid proto.Vid) (*client.VolumeInfoSimple, error) {
+				return &client.VolumeInfoSimple{
+					Vid:            vid,
+					VunitLocations: []proto.VunitLocation{{Vuid: 2}},
+				}, nil
+			},
+		)
+		mockTopicConsumeDelete.volCache = volCache
+		mockTopicConsumeDelete.consumeAndDelete(consumer, 2)
+
+		mockTopicConsumeDelete.volCache = oldCache
+		mockTopicConsumeDelete.blobnodeCli = oldBlobNode
+	}
+}
+
+// comment temporary
+func TestNewDeleteMgr(t *testing.T) {
+	ctr := gomock.NewController(t)
+	broker0 := NewBroker(t)
+	defer broker0.Close()
+
+	testDir, err := ioutil.TempDir(os.TempDir(), "delete_log")
+	require.NoError(t, err)
+	defer os.RemoveAll(testDir)
+
+	blobCfg := &BlobDeleteConfig{
+		ClusterID:            0,
+		TaskPoolSize:         2,
+		NormalHandleBatchCnt: 10,
+		FailHandleBatchCnt:   10,
+		DeleteLog: recordlog.Config{
+			Dir:       testDir,
+			ChunkBits: 22,
+		},
+		Kafka: BlobDeleteKafkaConfig{
+			BrokerList: []string{broker0.Addr()},
+			Normal: TopicConfig{
+				Topic:      testTopic,
+				Partitions: []int32{0},
+			},
+			Failed: TopicConfig{
+				Topic:      testTopic,
+				Partitions: []int32{0},
+			},
+			FailMsgSenderTimeoutMs: 0,
+		},
+	}
+
+	clusterMgrCli := NewMockClusterMgrAPI(ctr)
+	clusterMgrCli.EXPECT().GetConfig(any, any).AnyTimes().Return("", errMock)
+	volCache := NewMockVolumeCache(ctr)
+	blobnodeCli := NewMockBlobnodeAPI(ctr)
+	accessor := NewMockKafkaOffsetTable(ctr)
+	accessor.EXPECT().Get(any, any).AnyTimes().Return(int64(0), nil)
+	accessor.EXPECT().Set(any, any, any).AnyTimes().Return(nil)
+	switchMgr := taskswitch.NewSwitchMgr(clusterMgrCli)
+
+	service, err := NewBlobDeleteMgr(blobCfg, volCache, accessor, blobnodeCli, switchMgr)
+	require.NoError(t, err)
+
+	// run task
+	service.RunTask()
+
+	// get stats
+	service.GetTaskStats()
+	service.GetErrorStats()
+}

blobstore/scheduler/client/blobnode.go

@@ -0,0 +1,62 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package client
+
+import (
+	"context"
+
+	api "github.com/cubefs/cubefs/blobstore/api/blobnode"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+// BlobnodeAPI interface of blobnode client deleter api
+type BlobnodeAPI interface {
+	MarkDelete(ctx context.Context, location proto.VunitLocation, bid proto.BlobID) error
+	Delete(ctx context.Context, location proto.VunitLocation, bid proto.BlobID) error
+	RepairShard(ctx context.Context, host string, task proto.ShardRepairTask) error
+}
+
+type blobnodeClient struct {
+	client api.StorageAPI
+}
+
+// NewBlobnodeClient returns blobnode client
+func NewBlobnodeClient(cfg *api.Config) BlobnodeAPI {
+	return &blobnodeClient{api.New(cfg)}
+}
+
+func (c *blobnodeClient) RepairShard(ctx context.Context, host string, task proto.ShardRepairTask) error {
+	return c.client.RepairShard(ctx, host, &api.ShardRepairArgs{
+		Task: task,
+	})
+}
+
+// MarkDelete mark delete blob
+func (c *blobnodeClient) MarkDelete(ctx context.Context, location proto.VunitLocation, bid proto.BlobID) error {
+	return c.client.MarkDeleteShard(ctx, location.Host, &api.DeleteShardArgs{
+		DiskID: location.DiskID,
+		Vuid:   location.Vuid,
+		Bid:    bid,
+	})
+}
+
+// Delete delete blob
+func (c *blobnodeClient) Delete(ctx context.Context, location proto.VunitLocation, bid proto.BlobID) error {
+	return c.client.DeleteShard(ctx, location.Host, &api.DeleteShardArgs{
+		DiskID: location.DiskID,
+		Vuid:   location.Vuid,
+		Bid:    bid,
+	})
+}

blobstore/scheduler/client/blobnode_test.go

@@ -0,0 +1,44 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package client
+
+import (
+	"context"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/require"
+
+	api "github.com/cubefs/cubefs/blobstore/api/blobnode"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/testing/mocks"
+)
+
+func TestBlobnode(t *testing.T) {
+	any := gomock.Any()
+	ctx := context.Background()
+
+	cli := NewBlobnodeClient(&api.Config{}).(*blobnodeClient)
+	client := mocks.NewMockStorageAPI(gomock.NewController(t))
+	client.EXPECT().MarkDeleteShard(any, any, any).Return(nil)
+	client.EXPECT().DeleteShard(any, any, any).Return(nil)
+	cli.client = client
+
+	err := cli.MarkDelete(ctx, proto.VunitLocation{}, proto.BlobID(1))
+	require.NoError(t, err)
+
+	err = cli.Delete(ctx, proto.VunitLocation{}, proto.BlobID(1))
+	require.NoError(t, err)
+}

blobstore/scheduler/client/clustermgr.go

@@ -0,0 +1,645 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package client
+
+import (
+	"context"
+	"sync"
+
+	"github.com/cubefs/cubefs/blobstore/api/blobnode"
+	cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/util/log"
+)
+
+// ClusterMgrAPI define the interface of clustermgr used by scheduler
+type ClusterMgrAPI interface {
+	GetConfig(ctx context.Context, key string) (val string, err error)
+
+	// volume
+	GetVolumeInfo(ctx context.Context, Vid proto.Vid) (ret *VolumeInfoSimple, err error)
+	LockVolume(ctx context.Context, Vid proto.Vid) (err error)
+	UnlockVolume(ctx context.Context, Vid proto.Vid) (err error)
+	UpdateVolume(ctx context.Context, newVuid, oldVuid proto.Vuid, newDiskID proto.DiskID) (err error)
+	AllocVolumeUnit(ctx context.Context, vuid proto.Vuid) (ret *AllocVunitInfo, err error)
+	ReleaseVolumeUnit(ctx context.Context, vuid proto.Vuid, diskID proto.DiskID) (err error)
+	ListDiskVolumeUnits(ctx context.Context, diskID proto.DiskID) (ret []*VunitInfoSimple, err error)
+	ListVolume(ctx context.Context, marker proto.Vid, count int) (volInfo []*VolumeInfoSimple, retVid proto.Vid, err error)
+
+	// disk
+	ListClusterDisks(ctx context.Context) (disks []*DiskInfoSimple, err error)
+	ListBrokenDisks(ctx context.Context, count int) (disks []*DiskInfoSimple, err error)
+	ListRepairingDisks(ctx context.Context) (disks []*DiskInfoSimple, err error)
+	ListDropDisks(ctx context.Context) (disks []*DiskInfoSimple, err error)
+	SetDiskRepairing(ctx context.Context, diskID proto.DiskID) (err error)
+	SetDiskRepaired(ctx context.Context, diskID proto.DiskID) (err error)
+	SetDiskDropped(ctx context.Context, diskID proto.DiskID) (err error)
+	GetDiskInfo(ctx context.Context, diskID proto.DiskID) (ret *DiskInfoSimple, err error)
+
+	// register
+	Register(ctx context.Context, info RegisterInfo) error
+	GetService(ctx context.Context, name string, clusterID proto.ClusterID) (hosts []string, err error)
+}
+
+var (
+	defaultListDiskNum    = 1000
+	defaultListDiskMarker = proto.DiskID(0)
+)
+
+// VolumeInfoSimple volume info used by scheduler
+type VolumeInfoSimple struct {
+	Vid            proto.Vid             `json:"vid"`
+	CodeMode       codemode.CodeMode     `json:"code_mode"`
+	Status         proto.VolumeStatus    `json:"status"`
+	VunitLocations []proto.VunitLocation `json:"vunit_locations"`
+}
+
+// EqualWith returns whether equal with another.
+func (vol *VolumeInfoSimple) EqualWith(volInfo *VolumeInfoSimple) bool {
+	if len(vol.VunitLocations) != len(volInfo.VunitLocations) {
+		return false
+	}
+	if vol.Vid != volInfo.Vid ||
+		vol.CodeMode != volInfo.CodeMode ||
+		vol.Status != volInfo.Status {
+		return false
+	}
+	for i := range vol.VunitLocations {
+		if vol.VunitLocations[i] != volInfo.VunitLocations[i] {
+			return false
+		}
+	}
+	return true
+}
+
+// IsIdle returns true if volume is idle
+func (vol *VolumeInfoSimple) IsIdle() bool {
+	return vol.Status == proto.VolumeStatusIdle
+}
+
+// IsActive returns true if volume is active
+func (vol *VolumeInfoSimple) IsActive() bool {
+	return vol.Status == proto.VolumeStatusActive
+}
+
+func (vol *VolumeInfoSimple) set(info *cmapi.VolumeInfo) {
+	vol.Vid = info.Vid
+	vol.CodeMode = info.CodeMode
+	vol.Status = info.Status
+	vol.VunitLocations = make([]proto.VunitLocation, len(info.Units))
+
+	// check volume info
+	codeModeInfo := info.CodeMode.Tactic()
+	vunitCnt := codeModeInfo.N + codeModeInfo.M + codeModeInfo.L
+	if len(info.Units) != vunitCnt {
+		log.Panicf("volume %d info unexpect", info.Vid)
+	}
+
+	diskIDMap := make(map[proto.DiskID]struct{}, vunitCnt)
+	for _, repl := range info.Units {
+		if _, ok := diskIDMap[repl.DiskID]; ok {
+			log.Panicf("vid %d many chunks on same disk", info.Vid)
+		}
+		diskIDMap[repl.DiskID] = struct{}{}
+	}
+
+	for i := 0; i < len(info.Units); i++ {
+		vol.VunitLocations[i] = proto.VunitLocation{
+			Vuid:   info.Units[i].Vuid,
+			Host:   info.Units[i].Host,
+			DiskID: info.Units[i].DiskID,
+		}
+	}
+}
+
+// AllocVunitInfo volume unit info for alloc
+type AllocVunitInfo struct {
+	proto.VunitLocation
+}
+
+// Location returns volume unit location
+func (vunit *AllocVunitInfo) Location() proto.VunitLocation {
+	return vunit.VunitLocation
+}
+
+func (vunit *AllocVunitInfo) set(info *cmapi.AllocVolumeUnit, host string) {
+	vunit.Vuid = info.Vuid
+	vunit.DiskID = info.DiskID
+	vunit.Host = host
+}
+
+// VunitInfoSimple volume unit simple info
+type VunitInfoSimple struct {
+	Vuid   proto.Vuid   `json:"vuid"`
+	DiskID proto.DiskID `json:"disk_id"`
+	Host   string       `json:"host"`
+	Used   uint64       `json:"used"`
+}
+
+func (vunit *VunitInfoSimple) set(info *cmapi.VolumeUnitInfo, host string) {
+	vunit.Vuid = info.Vuid
+	vunit.DiskID = info.DiskID
+	vunit.Host = host
+	vunit.Used = info.Used
+}
+
+// DiskInfoSimple disk simple info
+type DiskInfoSimple struct {
+	ClusterID    proto.ClusterID  `json:"cluster_id"`
+	DiskID       proto.DiskID     `json:"disk_id"`
+	Idc          string           `json:"idc"`
+	Rack         string           `json:"rack"`
+	Host         string           `json:"host"`
+	Status       proto.DiskStatus `json:"status"`
+	Readonly     bool             `json:"readonly"`
+	UsedChunkCnt int64            `json:"used_chunk_cnt"`
+	MaxChunkCnt  int64            `json:"max_chunk_cnt"`
+	FreeChunkCnt int64            `json:"free_chunk_cnt"`
+}
+
+// IsHealth return true if disk is health
+func (disk *DiskInfoSimple) IsHealth() bool {
+	return disk.Status == proto.DiskStatusNormal
+}
+
+// IsBroken return true if disk is broken
+func (disk *DiskInfoSimple) IsBroken() bool {
+	return disk.Status == proto.DiskStatusBroken
+}
+
+// IsDropped return true if disk is dropped
+func (disk *DiskInfoSimple) IsDropped() bool {
+	return disk.Status == proto.DiskStatusDropped
+}
+
+// CanDropped  disk can drop when disk is normal or has repaired or has dropped
+// for simplicity we not allow to set disk status dropped
+// when disk is repairing
+func (disk *DiskInfoSimple) CanDropped() bool {
+	if disk.Status == proto.DiskStatusNormal ||
+		disk.Status == proto.DiskStatusRepaired ||
+		disk.Status == proto.DiskStatusDropped {
+		return true
+	}
+	return false
+}
+
+func (disk *DiskInfoSimple) set(info *blobnode.DiskInfo) {
+	disk.ClusterID = info.ClusterID
+	disk.Idc = info.Idc
+	disk.Rack = info.Rack
+	disk.Host = info.Host
+	disk.DiskID = info.DiskID
+	disk.Status = info.Status
+	disk.Readonly = info.Readonly
+	disk.UsedChunkCnt = info.UsedChunkCnt
+	disk.MaxChunkCnt = info.MaxChunkCnt
+	disk.FreeChunkCnt = info.FreeChunkCnt
+}
+
+// RegisterInfo register info use for clustermgr
+type RegisterInfo struct {
+	ClusterID          uint64 `json:"cluster_id"`
+	Name               string `json:"name"`
+	Host               string `json:"host"`
+	Idc                string `json:"idc"`
+	HeartbeatIntervalS uint32 `json:"heartbeat_interval_s"`
+	HeartbeatTicks     uint32 `json:"heartbeat_ticks"`
+	ExpiresTicks       uint32 `json:"expires_ticks"`
+}
+
+// IClusterManager define the interface of clustermgr
+type IClusterManager interface {
+	GetConfig(ctx context.Context, key string) (ret string, err error)
+	GetVolumeInfo(ctx context.Context, args *cmapi.GetVolumeArgs) (ret *cmapi.VolumeInfo, err error)
+	LockVolume(ctx context.Context, args *cmapi.LockVolumeArgs) (err error)
+	UnlockVolume(ctx context.Context, args *cmapi.UnlockVolumeArgs) (err error)
+	UpdateVolume(ctx context.Context, args *cmapi.UpdateVolumeArgs) (err error)
+	AllocVolumeUnit(ctx context.Context, args *cmapi.AllocVolumeUnitArgs) (ret *cmapi.AllocVolumeUnit, err error)
+	ReleaseVolumeUnit(ctx context.Context, args *cmapi.ReleaseVolumeUnitArgs) (err error)
+	ListVolumeUnit(ctx context.Context, args *cmapi.ListVolumeUnitArgs) ([]*cmapi.VolumeUnitInfo, error)
+	ListVolume(ctx context.Context, args *cmapi.ListVolumeArgs) (ret cmapi.ListVolumes, err error)
+	ListDisk(ctx context.Context, args *cmapi.ListOptionArgs) (ret cmapi.ListDiskRet, err error)
+	ListDroppingDisk(ctx context.Context) (ret []*blobnode.DiskInfo, err error)
+	SetDisk(ctx context.Context, id proto.DiskID, status proto.DiskStatus) (err error)
+	DiskInfo(ctx context.Context, id proto.DiskID) (ret *blobnode.DiskInfo, err error)
+	DroppedDisk(ctx context.Context, id proto.DiskID) (err error)
+	RegisterService(ctx context.Context, node cmapi.ServiceNode, tickInterval, heartbeatTicks, expiresTicks uint32) (err error)
+	GetService(ctx context.Context, args cmapi.GetServiceArgs) (info cmapi.ServiceInfo, err error)
+}
+
+// clustermgrClient clustermgr client
+type clustermgrClient struct {
+	client IClusterManager
+	rwLock sync.RWMutex
+}
+
+func NewClusterMgrClient(conf *cmapi.Config) ClusterMgrAPI {
+	return &clustermgrClient{
+		client: cmapi.New(conf),
+		rwLock: sync.RWMutex{},
+	}
+}
+
+// GetConfig returns config by config key
+func (c *clustermgrClient) GetConfig(ctx context.Context, key string) (val string, err error) {
+	c.rwLock.RLock()
+	defer c.rwLock.RUnlock()
+
+	span := trace.SpanFromContextSafe(ctx)
+
+	span.Debugf("get config: args key[%s]", key)
+	ret, err := c.client.GetConfig(ctx, key)
+	if err != nil {
+		span.Errorf("get config failed: err[%+v]", err)
+		return
+	}
+	span.Debugf("get config ret: config[%s]", ret)
+	return ret, err
+}
+
+// GetVolumeInfo returns volume info
+func (c *clustermgrClient) GetVolumeInfo(ctx context.Context, vid proto.Vid) (*VolumeInfoSimple, error) {
+	c.rwLock.RLock()
+	defer c.rwLock.RUnlock()
+
+	span := trace.SpanFromContextSafe(ctx)
+
+	span.Debugf("get volume info: args vid[%d]", vid)
+	info, err := c.client.GetVolumeInfo(ctx, &cmapi.GetVolumeArgs{Vid: vid})
+	if err != nil {
+		span.Errorf("get volume info failed: err[%+v]", err)
+		return nil, err
+	}
+	span.Debugf("get volume info ret: volume[%+v]", *info)
+	ret := &VolumeInfoSimple{}
+	ret.set(info)
+	return ret, nil
+}
+
+// LockVolume lock volume
+func (c *clustermgrClient) LockVolume(ctx context.Context, vid proto.Vid) (err error) {
+	c.rwLock.Lock()
+	defer c.rwLock.Unlock()
+
+	span := trace.SpanFromContextSafe(ctx)
+
+	span.Debugf("lock volume: args vid[%d]", vid)
+	err = c.client.LockVolume(ctx, &cmapi.LockVolumeArgs{Vid: vid})
+	span.Debugf("lock volume ret: err[%+v]", err)
+	return
+}
+
+// UnlockVolume unlock volume
+func (c *clustermgrClient) UnlockVolume(ctx context.Context, vid proto.Vid) (err error) {
+	c.rwLock.Lock()
+	defer c.rwLock.Unlock()
+
+	span := trace.SpanFromContextSafe(ctx)
+
+	span.Debugf("unlock volume: args vid[%d]", vid)
+	err = c.client.UnlockVolume(ctx, &cmapi.UnlockVolumeArgs{Vid: vid})
+	span.Debugf("unlock volume ret: err[%+v]", err)
+	if rpc.DetectStatusCode(err) == errcode.CodeUnlockNotAllow {
+		span.Infof("unlock volume failed but deem lock success: err[%+v], code[%d]", err, rpc.DetectStatusCode(err))
+		return nil
+	}
+
+	return
+}
+
+// UpdateVolume update volume
+func (c *clustermgrClient) UpdateVolume(ctx context.Context, newVuid, oldVuid proto.Vuid, newDiskID proto.DiskID) (err error) {
+	c.rwLock.Lock()
+	defer c.rwLock.Unlock()
+
+	span := trace.SpanFromContextSafe(ctx)
+
+	span.Infof("update volume: args new vuid[%d], old vuid[%d], new disk_id[%d]", newVuid, oldVuid, newDiskID)
+	err = c.client.UpdateVolume(ctx, &cmapi.UpdateVolumeArgs{NewVuid: newVuid, OldVuid: oldVuid, NewDiskID: newDiskID})
+	span.Infof("update volume ret: err %+v", err)
+	return
+}
+
+// AllocVolumeUnit alloc volume unit
+func (c *clustermgrClient) AllocVolumeUnit(ctx context.Context, vuid proto.Vuid) (*AllocVunitInfo, error) {
+	c.rwLock.Lock()
+	defer c.rwLock.Unlock()
+
+	span := trace.SpanFromContextSafe(ctx)
+
+	span.Debugf("alloc volume unit: args vuid[%d]", vuid)
+	ret := &AllocVunitInfo{}
+	info, err := c.client.AllocVolumeUnit(ctx, &cmapi.AllocVolumeUnitArgs{Vuid: vuid})
+	if err != nil {
+		span.Errorf("alloc volume unit failed: err[%+v]", err)
+		return nil, err
+	}
+	span.Debugf("alloc volume unit ret: unit[%+v]", *info)
+
+	diskInfo, err := c.client.DiskInfo(ctx, info.DiskID)
+	if err != nil {
+		return nil, err
+	}
+	span.Debugf("get disk info ret: disk[%+v]", diskInfo)
+
+	ret.set(info, diskInfo.Host)
+	return ret, err
+}
+
+// ReleaseVolumeUnit release volume unit
+func (c *clustermgrClient) ReleaseVolumeUnit(ctx context.Context, vuid proto.Vuid, diskID proto.DiskID) (err error) {
+	c.rwLock.Lock()
+	defer c.rwLock.Unlock()
+
+	span := trace.SpanFromContextSafe(ctx)
+
+	span.Debugf("release volume unit: args vuid[%d], disk_id[%d]", vuid, diskID)
+	err = c.client.ReleaseVolumeUnit(ctx, &cmapi.ReleaseVolumeUnitArgs{Vuid: vuid, DiskID: diskID})
+	span.Debugf("release volume unit ret: err[%+v]", err)
+
+	return
+}
+
+// ListDiskVolumeUnits list disk volume units
+func (c *clustermgrClient) ListDiskVolumeUnits(ctx context.Context, diskID proto.DiskID) (rets []*VunitInfoSimple, err error) {
+	c.rwLock.RLock()
+	defer c.rwLock.RUnlock()
+
+	span := trace.SpanFromContextSafe(ctx)
+
+	span.Debugf("list disk volume units: args disk_id[%d]", diskID)
+	infos, err := c.client.ListVolumeUnit(ctx, &cmapi.ListVolumeUnitArgs{DiskID: diskID})
+	if err != nil {
+		span.Errorf("list disk volume units failed: err[%+v]", err)
+		return nil, err
+	}
+
+	for idx, info := range infos {
+		span.Debugf("list disk volume units ret: idx[%d], info[%+v]", idx, *info)
+	}
+
+	diskInfo, err := c.client.DiskInfo(ctx, diskID)
+	if err != nil {
+		span.Errorf("get disk info failed: err[%+v]", err)
+		return nil, err
+	}
+	span.Debugf("get disk info ret: disk[%+v]", *diskInfo)
+
+	for _, info := range infos {
+		ele := VunitInfoSimple{}
+		ele.set(info, diskInfo.Host)
+		rets = append(rets, &ele)
+	}
+	return rets, nil
+}
+
+// ListVolume list volume
+func (c *clustermgrClient) ListVolume(ctx context.Context, marker proto.Vid, count int) (rets []*VolumeInfoSimple, nextVid proto.Vid, err error) {
+	c.rwLock.RLock()
+	defer c.rwLock.RUnlock()
+
+	vols, err := c.client.ListVolume(ctx, &cmapi.ListVolumeArgs{Marker: marker, Count: count})
+	if err != nil {
+		return
+	}
+	for index := range vols.Volumes {
+		ret := &VolumeInfoSimple{}
+		ret.set(vols.Volumes[index])
+		rets = append(rets, ret)
+	}
+	nextVid = vols.Marker
+	return
+}
+
+// ListClusterDisks list all disks
+func (c *clustermgrClient) ListClusterDisks(ctx context.Context) (disks []*DiskInfoSimple, err error) {
+	c.rwLock.RLock()
+	defer c.rwLock.RUnlock()
+	return c.listAllDisks(ctx, proto.DiskStatusNormal)
+}
+
+// ListBrokenDisks list all broken disks
+func (c *clustermgrClient) ListBrokenDisks(ctx context.Context, count int) (disks []*DiskInfoSimple, err error) {
+	c.rwLock.RLock()
+	defer c.rwLock.RUnlock()
+	return c.listDisks(ctx, proto.DiskStatusBroken, count)
+}
+
+// ListRepairingDisks list repairing disks
+func (c *clustermgrClient) ListRepairingDisks(ctx context.Context) (disks []*DiskInfoSimple, err error) {
+	c.rwLock.RLock()
+	defer c.rwLock.RUnlock()
+	return c.listAllDisks(ctx, proto.DiskStatusRepairing)
+}
+
+func (c *clustermgrClient) listAllDisks(ctx context.Context, status proto.DiskStatus) (disks []*DiskInfoSimple, err error) {
+	span := trace.SpanFromContextSafe(ctx)
+	marker := defaultListDiskMarker
+	for {
+		args := &cmapi.ListOptionArgs{
+			Status: status,
+			Count:  defaultListDiskNum,
+			Marker: marker,
+		}
+		selectDisks, selectMarker, err := c.listDisk(ctx, args)
+		if err != nil {
+			span.Errorf("list disk failed: err[%+v]", err)
+			return nil, err
+		}
+
+		marker = selectMarker
+		disks = append(disks, selectDisks...)
+		if marker == defaultListDiskMarker {
+			break
+		}
+	}
+	return
+}
+
+func (c *clustermgrClient) listDisks(ctx context.Context, status proto.DiskStatus, count int) (disks []*DiskInfoSimple, err error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	marker := defaultListDiskMarker
+	needDiskCount := count
+	for {
+		args := &cmapi.ListOptionArgs{
+			Status: status,
+			Count:  needDiskCount,
+			Marker: marker,
+		}
+		selectDisks, selectMarker, err := c.listDisk(ctx, args)
+		if err != nil {
+			span.Errorf("list disk failed: err[%+v]", err)
+			return nil, err
+		}
+
+		marker = selectMarker
+		disks = append(disks, selectDisks...)
+		needDiskCount -= len(disks)
+		if marker == defaultListDiskMarker || needDiskCount <= 0 {
+			break
+		}
+	}
+	return
+}
+
+func (c *clustermgrClient) listDisk(ctx context.Context, args *cmapi.ListOptionArgs) (disks []*DiskInfoSimple, marker proto.DiskID, err error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	span.Debugf("list disk: args[%+v]", *args)
+	infos, err := c.client.ListDisk(ctx, args)
+	if err != nil {
+		span.Errorf("list disk failed: err[%+v]", err)
+		return nil, defaultListDiskMarker, err
+	}
+	marker = infos.Marker
+	for _, info := range infos.Disks {
+		span.Debugf("list disk ret: disk[%+v]", *info)
+		ele := DiskInfoSimple{}
+		ele.set(info)
+		disks = append(disks, &ele)
+	}
+	return
+}
+
+// ListDropDisks list drop disks, may contain {DiskStatusNormal,DiskStatusReadOnly,DiskStatusBroken,DiskStatusRepairing,DiskStatusRepaired} disks
+func (c *clustermgrClient) ListDropDisks(ctx context.Context) (disks []*DiskInfoSimple, err error) {
+	c.rwLock.RLock()
+	defer c.rwLock.RUnlock()
+
+	span := trace.SpanFromContextSafe(ctx)
+
+	infos, err := c.client.ListDroppingDisk(ctx)
+	if err != nil {
+		span.Errorf("list drop disks failed: err[%+v]", err)
+		return nil, err
+	}
+	span.Infof("list drop disks: len[%d]", len(infos))
+	for _, info := range infos {
+		span.Debugf("list drop disks ret: disk[%+v]", *info)
+		disk := DiskInfoSimple{}
+		disk.set(info)
+		span.Infof("disk status: [%s]", disk.Status.String())
+		if disk.IsHealth() {
+			disks = append(disks, &disk)
+		}
+	}
+	return disks, nil
+}
+
+// SetDiskRepairing set disk repairing
+func (c *clustermgrClient) SetDiskRepairing(ctx context.Context, diskID proto.DiskID) (err error) {
+	c.rwLock.Lock()
+	defer c.rwLock.Unlock()
+
+	span := trace.SpanFromContextSafe(ctx)
+
+	span.Debugf("set disk repairing: args disk_id[%d], status[%s]", diskID, proto.DiskStatusRepairing.String())
+	err = c.setDiskStatus(ctx, diskID, proto.DiskStatusRepairing)
+	span.Debugf("set disk repairing ret: err[%+v]", err)
+	return
+}
+
+// SetDiskRepaired set disk repaired
+func (c *clustermgrClient) SetDiskRepaired(ctx context.Context, diskID proto.DiskID) (err error) {
+	c.rwLock.Lock()
+	defer c.rwLock.Unlock()
+
+	span := trace.SpanFromContextSafe(ctx)
+
+	span.Debugf("set disk repaired: args disk_id[%d], disk status[%s]", diskID, proto.DiskStatusRepaired.String())
+	err = c.setDiskStatus(ctx, diskID, proto.DiskStatusRepaired)
+	span.Debugf("set disk repaired ret: err[%+v]", err)
+	return
+}
+
+// SetDiskDropped set disk dropped
+func (c *clustermgrClient) SetDiskDropped(ctx context.Context, diskID proto.DiskID) (err error) {
+	c.rwLock.Lock()
+	defer c.rwLock.Unlock()
+	span := trace.SpanFromContextSafe(ctx)
+
+	info, err := c.client.DiskInfo(ctx, diskID)
+	if err != nil {
+		span.Errorf("get disk info failed: disk_id[%d], err[%+v]", diskID, err)
+		return err
+	}
+
+	disk := &DiskInfoSimple{}
+	disk.set(info)
+	if disk.IsDropped() {
+		return nil
+	}
+
+	if !disk.CanDropped() {
+		return errcode.ErrCanNotDropped
+	}
+
+	span.Debugf("set disk dropped: args disk_id[%d], status[%s]", diskID, proto.DiskStatusDropped.String())
+	err = c.client.DroppedDisk(ctx, diskID)
+	span.Debugf("set disk dropped ret: err[%+v]", err)
+	return
+}
+
+func (c *clustermgrClient) setDiskStatus(ctx context.Context, diskID proto.DiskID, status proto.DiskStatus) (err error) {
+	return c.client.SetDisk(ctx, diskID, status)
+}
+
+// GetDiskInfo returns disk info
+func (c *clustermgrClient) GetDiskInfo(ctx context.Context, diskID proto.DiskID) (ret *DiskInfoSimple, err error) {
+	c.rwLock.RLock()
+	defer c.rwLock.RUnlock()
+
+	span := trace.SpanFromContextSafe(ctx)
+
+	span.Debugf("get disk info: args disk_id[%d]", diskID)
+	info, err := c.client.DiskInfo(ctx, diskID)
+	if err != nil {
+		span.Errorf("get disk info failed: err[%+v]", err)
+		return nil, err
+	}
+	span.Debugf("get disk info ret: disk[%+v]", *info)
+	ret = &DiskInfoSimple{}
+	ret.set(info)
+	return ret, nil
+}
+
+func (c *clustermgrClient) Register(ctx context.Context, info RegisterInfo) error {
+	node := cmapi.ServiceNode{
+		ClusterID: info.ClusterID,
+		Name:      info.Name,
+		Host:      info.Host,
+		Idc:       info.Idc,
+	}
+	return c.client.RegisterService(ctx, node, info.HeartbeatIntervalS, info.HeartbeatTicks, info.ExpiresTicks)
+}
+
+func (c *clustermgrClient) GetService(ctx context.Context, name string, clusterID proto.ClusterID) (hosts []string, err error) {
+	svrInfos, err := c.client.GetService(ctx, cmapi.GetServiceArgs{Name: name})
+	if err != nil {
+		return nil, err
+	}
+	for _, s := range svrInfos.Nodes {
+		if clusterID == proto.ClusterID(s.ClusterID) {
+			hosts = append(hosts, s.Host)
+		}
+	}
+	return
+}

blobstore/scheduler/client/clustermgr_mock_test.go

@@ -0,0 +1,271 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/cubefs/cubefs/blobstore/scheduler/client (interfaces: IClusterManager)
+
+// Package client is a generated GoMock package.
+package client
+
+import (
+	context "context"
+	reflect "reflect"
+
+	blobnode "github.com/cubefs/cubefs/blobstore/api/blobnode"
+	clustermgr "github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	proto "github.com/cubefs/cubefs/blobstore/common/proto"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockClusterManager is a mock of IClusterManager interface.
+type MockClusterManager struct {
+	ctrl     *gomock.Controller
+	recorder *MockClusterManagerMockRecorder
+}
+
+// MockClusterManagerMockRecorder is the mock recorder for MockClusterManager.
+type MockClusterManagerMockRecorder struct {
+	mock *MockClusterManager
+}
+
+// NewMockClusterManager creates a new mock instance.
+func NewMockClusterManager(ctrl *gomock.Controller) *MockClusterManager {
+	mock := &MockClusterManager{ctrl: ctrl}
+	mock.recorder = &MockClusterManagerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockClusterManager) EXPECT() *MockClusterManagerMockRecorder {
+	return m.recorder
+}
+
+// AllocVolumeUnit mocks base method.
+func (m *MockClusterManager) AllocVolumeUnit(arg0 context.Context, arg1 *clustermgr.AllocVolumeUnitArgs) (*clustermgr.AllocVolumeUnit, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "AllocVolumeUnit", arg0, arg1)
+	ret0, _ := ret[0].(*clustermgr.AllocVolumeUnit)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// AllocVolumeUnit indicates an expected call of AllocVolumeUnit.
+func (mr *MockClusterManagerMockRecorder) AllocVolumeUnit(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocVolumeUnit", reflect.TypeOf((*MockClusterManager)(nil).AllocVolumeUnit), arg0, arg1)
+}
+
+// DiskInfo mocks base method.
+func (m *MockClusterManager) DiskInfo(arg0 context.Context, arg1 proto.DiskID) (*blobnode.DiskInfo, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "DiskInfo", arg0, arg1)
+	ret0, _ := ret[0].(*blobnode.DiskInfo)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// DiskInfo indicates an expected call of DiskInfo.
+func (mr *MockClusterManagerMockRecorder) DiskInfo(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiskInfo", reflect.TypeOf((*MockClusterManager)(nil).DiskInfo), arg0, arg1)
+}
+
+// DroppedDisk mocks base method.
+func (m *MockClusterManager) DroppedDisk(arg0 context.Context, arg1 proto.DiskID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "DroppedDisk", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// DroppedDisk indicates an expected call of DroppedDisk.
+func (mr *MockClusterManagerMockRecorder) DroppedDisk(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DroppedDisk", reflect.TypeOf((*MockClusterManager)(nil).DroppedDisk), arg0, arg1)
+}
+
+// GetConfig mocks base method.
+func (m *MockClusterManager) GetConfig(arg0 context.Context, arg1 string) (string, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetConfig", arg0, arg1)
+	ret0, _ := ret[0].(string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetConfig indicates an expected call of GetConfig.
+func (mr *MockClusterManagerMockRecorder) GetConfig(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfig", reflect.TypeOf((*MockClusterManager)(nil).GetConfig), arg0, arg1)
+}
+
+// GetService mocks base method.
+func (m *MockClusterManager) GetService(arg0 context.Context, arg1 clustermgr.GetServiceArgs) (clustermgr.ServiceInfo, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetService", arg0, arg1)
+	ret0, _ := ret[0].(clustermgr.ServiceInfo)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetService indicates an expected call of GetService.
+func (mr *MockClusterManagerMockRecorder) GetService(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetService", reflect.TypeOf((*MockClusterManager)(nil).GetService), arg0, arg1)
+}
+
+// GetVolumeInfo mocks base method.
+func (m *MockClusterManager) GetVolumeInfo(arg0 context.Context, arg1 *clustermgr.GetVolumeArgs) (*clustermgr.VolumeInfo, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetVolumeInfo", arg0, arg1)
+	ret0, _ := ret[0].(*clustermgr.VolumeInfo)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetVolumeInfo indicates an expected call of GetVolumeInfo.
+func (mr *MockClusterManagerMockRecorder) GetVolumeInfo(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVolumeInfo", reflect.TypeOf((*MockClusterManager)(nil).GetVolumeInfo), arg0, arg1)
+}
+
+// ListDisk mocks base method.
+func (m *MockClusterManager) ListDisk(arg0 context.Context, arg1 *clustermgr.ListOptionArgs) (clustermgr.ListDiskRet, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ListDisk", arg0, arg1)
+	ret0, _ := ret[0].(clustermgr.ListDiskRet)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ListDisk indicates an expected call of ListDisk.
+func (mr *MockClusterManagerMockRecorder) ListDisk(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDisk", reflect.TypeOf((*MockClusterManager)(nil).ListDisk), arg0, arg1)
+}
+
+// ListDroppingDisk mocks base method.
+func (m *MockClusterManager) ListDroppingDisk(arg0 context.Context) ([]*blobnode.DiskInfo, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ListDroppingDisk", arg0)
+	ret0, _ := ret[0].([]*blobnode.DiskInfo)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ListDroppingDisk indicates an expected call of ListDroppingDisk.
+func (mr *MockClusterManagerMockRecorder) ListDroppingDisk(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDroppingDisk", reflect.TypeOf((*MockClusterManager)(nil).ListDroppingDisk), arg0)
+}
+
+// ListVolume mocks base method.
+func (m *MockClusterManager) ListVolume(arg0 context.Context, arg1 *clustermgr.ListVolumeArgs) (clustermgr.ListVolumes, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ListVolume", arg0, arg1)
+	ret0, _ := ret[0].(clustermgr.ListVolumes)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ListVolume indicates an expected call of ListVolume.
+func (mr *MockClusterManagerMockRecorder) ListVolume(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListVolume", reflect.TypeOf((*MockClusterManager)(nil).ListVolume), arg0, arg1)
+}
+
+// ListVolumeUnit mocks base method.
+func (m *MockClusterManager) ListVolumeUnit(arg0 context.Context, arg1 *clustermgr.ListVolumeUnitArgs) ([]*clustermgr.VolumeUnitInfo, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ListVolumeUnit", arg0, arg1)
+	ret0, _ := ret[0].([]*clustermgr.VolumeUnitInfo)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ListVolumeUnit indicates an expected call of ListVolumeUnit.
+func (mr *MockClusterManagerMockRecorder) ListVolumeUnit(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListVolumeUnit", reflect.TypeOf((*MockClusterManager)(nil).ListVolumeUnit), arg0, arg1)
+}
+
+// LockVolume mocks base method.
+func (m *MockClusterManager) LockVolume(arg0 context.Context, arg1 *clustermgr.LockVolumeArgs) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "LockVolume", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// LockVolume indicates an expected call of LockVolume.
+func (mr *MockClusterManagerMockRecorder) LockVolume(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LockVolume", reflect.TypeOf((*MockClusterManager)(nil).LockVolume), arg0, arg1)
+}
+
+// RegisterService mocks base method.
+func (m *MockClusterManager) RegisterService(arg0 context.Context, arg1 clustermgr.ServiceNode, arg2, arg3, arg4 uint32) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "RegisterService", arg0, arg1, arg2, arg3, arg4)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// RegisterService indicates an expected call of RegisterService.
+func (mr *MockClusterManagerMockRecorder) RegisterService(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterService", reflect.TypeOf((*MockClusterManager)(nil).RegisterService), arg0, arg1, arg2, arg3, arg4)
+}
+
+// ReleaseVolumeUnit mocks base method.
+func (m *MockClusterManager) ReleaseVolumeUnit(arg0 context.Context, arg1 *clustermgr.ReleaseVolumeUnitArgs) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ReleaseVolumeUnit", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// ReleaseVolumeUnit indicates an expected call of ReleaseVolumeUnit.
+func (mr *MockClusterManagerMockRecorder) ReleaseVolumeUnit(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReleaseVolumeUnit", reflect.TypeOf((*MockClusterManager)(nil).ReleaseVolumeUnit), arg0, arg1)
+}
+
+// SetDisk mocks base method.
+func (m *MockClusterManager) SetDisk(arg0 context.Context, arg1 proto.DiskID, arg2 proto.DiskStatus) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "SetDisk", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// SetDisk indicates an expected call of SetDisk.
+func (mr *MockClusterManagerMockRecorder) SetDisk(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDisk", reflect.TypeOf((*MockClusterManager)(nil).SetDisk), arg0, arg1, arg2)
+}
+
+// UnlockVolume mocks base method.
+func (m *MockClusterManager) UnlockVolume(arg0 context.Context, arg1 *clustermgr.UnlockVolumeArgs) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UnlockVolume", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// UnlockVolume indicates an expected call of UnlockVolume.
+func (mr *MockClusterManagerMockRecorder) UnlockVolume(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnlockVolume", reflect.TypeOf((*MockClusterManager)(nil).UnlockVolume), arg0, arg1)
+}
+
+// UpdateVolume mocks base method.
+func (m *MockClusterManager) UpdateVolume(arg0 context.Context, arg1 *clustermgr.UpdateVolumeArgs) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateVolume", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// UpdateVolume indicates an expected call of UpdateVolume.
+func (mr *MockClusterManagerMockRecorder) UpdateVolume(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateVolume", reflect.TypeOf((*MockClusterManager)(nil).UpdateVolume), arg0, arg1)
+}

blobstore/scheduler/client/clustermgr_test.go

@@ -0,0 +1,305 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package client
+
+//go:generate mockgen -destination=./clustermgr_mock_test.go -package=client -mock_names IClusterManager=MockClusterManager github.com/cubefs/cubefs/blobstore/scheduler/client IClusterManager
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/api/blobnode"
+	cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/taskswitch"
+	"github.com/cubefs/cubefs/blobstore/util/log"
+)
+
+var (
+	defaultVolumeListMarker = proto.Vid(0)
+	defaultDiskListMarker   = proto.DiskID(0)
+)
+
+func init() {
+	log.SetOutputLevel(log.Lfatal)
+}
+
+func MockGenVolInfo(vid proto.Vid, mode codemode.CodeMode, status proto.VolumeStatus) *cmapi.VolumeInfo {
+	cmInfo := mode.Tactic()
+	vunitCnt := cmInfo.M + cmInfo.N + cmInfo.L
+	host := "127.0.0.0:xxx"
+	locations := make([]cmapi.Unit, vunitCnt)
+	var idx uint8
+	for i := 0; i < vunitCnt; i++ {
+		locations[i].Vuid, _ = proto.NewVuid(vid, idx, 1)
+		locations[i].Host = host
+		locations[i].DiskID = proto.DiskID(locations[i].Vuid)
+		idx++
+	}
+
+	return &cmapi.VolumeInfo{
+		Units: locations,
+		VolumeInfoBase: cmapi.VolumeInfoBase{
+			Vid:      vid,
+			CodeMode: mode,
+			Status:   status,
+		},
+	}
+}
+
+func TestClustermgrClient(t *testing.T) {
+	cli := NewClusterMgrClient(&cmapi.Config{}).(*clustermgrClient)
+	mockCli := NewMockClusterManager(gomock.NewController(t))
+	cli.client = mockCli
+
+	ctx := context.Background()
+	any := gomock.Any()
+	errMock := errors.New("fake error")
+	{
+		// get config
+		cli.client.(*MockClusterManager).EXPECT().GetConfig(any, any).Return("", errMock)
+		_, err := cli.GetConfig(ctx, "config")
+		require.True(t, errors.Is(err, errMock))
+
+		cli.client.(*MockClusterManager).EXPECT().GetConfig(any, any).Return(taskswitch.SwitchOpen, nil)
+		enable, err := cli.GetConfig(ctx, "config")
+		require.NoError(t, err)
+		require.Equal(t, "true", enable)
+	}
+	{
+		// get volume info
+		cli.client.(*MockClusterManager).EXPECT().GetVolumeInfo(any, any).Return(nil, errMock)
+		_, err := cli.GetVolumeInfo(ctx, proto.Vid(1))
+		require.True(t, errors.Is(err, errMock))
+
+		volume := MockGenVolInfo(10, codemode.EC6P6, proto.VolumeStatusIdle)
+		volume2 := MockGenVolInfo(10, codemode.EC6P6, proto.VolumeStatusActive)
+		cli.client.(*MockClusterManager).EXPECT().GetVolumeInfo(any, any).Return(volume, nil)
+		cli.client.(*MockClusterManager).EXPECT().GetVolumeInfo(any, any).Return(volume2, nil)
+		vol, err := cli.GetVolumeInfo(ctx, proto.Vid(1))
+		require.NoError(t, err)
+		vol2, err := cli.GetVolumeInfo(ctx, proto.Vid(2))
+		require.NoError(t, err)
+		require.Equal(t, vol.Vid, volume.Vid)
+		require.Equal(t, vol.CodeMode, volume.CodeMode)
+		require.Equal(t, vol.Status, volume.Status)
+		require.Equal(t, len(vol.VunitLocations), len(volume.Units))
+		require.True(t, vol.IsIdle())
+		require.True(t, vol2.IsActive())
+		require.True(t, vol.EqualWith(vol))
+		require.False(t, vol.EqualWith(vol2))
+	}
+	{
+		// lock volume
+		cli.client.(*MockClusterManager).EXPECT().LockVolume(any, any).Return(nil)
+		err := cli.LockVolume(ctx, proto.Vid(1))
+		require.NoError(t, err)
+	}
+	{
+		// unlock volume
+		cli.client.(*MockClusterManager).EXPECT().UnlockVolume(any, any).Return(nil)
+		err := cli.UnlockVolume(ctx, proto.Vid(1))
+		require.NoError(t, err)
+
+		cli.client.(*MockClusterManager).EXPECT().UnlockVolume(any, any).Return(errcode.ErrUnlockNotAllow)
+		err = cli.UnlockVolume(ctx, proto.Vid(1))
+		require.NoError(t, err)
+
+		cli.client.(*MockClusterManager).EXPECT().UnlockVolume(any, any).Return(errMock)
+		err = cli.UnlockVolume(ctx, proto.Vid(1))
+		require.True(t, errors.Is(err, errMock))
+	}
+	{
+		// update volume
+		cli.client.(*MockClusterManager).EXPECT().UpdateVolume(any, any).Return(nil)
+		err := cli.UpdateVolume(ctx, proto.Vuid(2), proto.Vuid(1), proto.DiskID(1))
+		require.NoError(t, err)
+	}
+	{
+		// update volume
+		cli.client.(*MockClusterManager).EXPECT().AllocVolumeUnit(any, any).Return(nil, errMock)
+		_, err := cli.AllocVolumeUnit(ctx, proto.Vuid(2))
+		require.True(t, errors.Is(err, errMock))
+
+		unit := &cmapi.AllocVolumeUnit{Vuid: proto.Vuid(3), DiskID: proto.DiskID(2)}
+		cli.client.(*MockClusterManager).EXPECT().AllocVolumeUnit(any, any).Return(unit, nil)
+		cli.client.(*MockClusterManager).EXPECT().DiskInfo(any, any).Return(nil, errMock)
+		_, err = cli.AllocVolumeUnit(ctx, proto.Vuid(2))
+		require.True(t, errors.Is(err, errMock))
+
+		cli.client.(*MockClusterManager).EXPECT().AllocVolumeUnit(any, any).Return(unit, nil)
+		cli.client.(*MockClusterManager).EXPECT().DiskInfo(any, any).Return(&blobnode.DiskInfo{Host: "127.0.0.1:xxx"}, nil)
+		allocUnit, err := cli.AllocVolumeUnit(ctx, proto.Vuid(2))
+		require.NoError(t, err)
+		require.Equal(t, unit.Vuid, allocUnit.Location().Vuid)
+	}
+	{
+		// release volume unit
+		cli.client.(*MockClusterManager).EXPECT().ReleaseVolumeUnit(any, any).Return(nil)
+		err := cli.ReleaseVolumeUnit(ctx, proto.Vuid(2), proto.DiskID(1))
+		require.NoError(t, err)
+	}
+	{
+		// list disk volume units
+		cli.client.(*MockClusterManager).EXPECT().ListVolumeUnit(any, any).Return(nil, errMock)
+		_, err := cli.ListDiskVolumeUnits(ctx, proto.DiskID(1))
+		require.True(t, errors.Is(err, errMock))
+
+		unit := &cmapi.VolumeUnitInfo{Vuid: proto.Vuid(3), DiskID: proto.DiskID(2)}
+		cli.client.(*MockClusterManager).EXPECT().ListVolumeUnit(any, any).Return([]*cmapi.VolumeUnitInfo{unit}, nil)
+		cli.client.(*MockClusterManager).EXPECT().DiskInfo(any, any).Return(nil, errMock)
+		_, err = cli.ListDiskVolumeUnits(ctx, proto.DiskID(1))
+		require.True(t, errors.Is(err, errMock))
+
+		cli.client.(*MockClusterManager).EXPECT().ListVolumeUnit(any, any).Return([]*cmapi.VolumeUnitInfo{unit}, nil)
+		cli.client.(*MockClusterManager).EXPECT().DiskInfo(any, any).Return(&blobnode.DiskInfo{Host: "127.0.0.1:xxx"}, nil)
+		units, err := cli.ListDiskVolumeUnits(ctx, proto.DiskID(1))
+		require.NoError(t, err)
+		require.Equal(t, 1, len(units))
+	}
+	{
+		// list volume
+		cli.client.(*MockClusterManager).EXPECT().ListVolume(any, any).Return(cmapi.ListVolumes{}, errMock)
+		_, _, err := cli.ListVolume(ctx, defaultVolumeListMarker, 10)
+		require.True(t, errors.Is(err, errMock))
+
+		cli.client.(*MockClusterManager).EXPECT().ListVolume(any, any).Return(cmapi.ListVolumes{}, nil)
+		rets, _, err := cli.ListVolume(ctx, defaultVolumeListMarker, 10)
+		require.NoError(t, err)
+		require.Equal(t, 0, len(rets))
+
+		volume := MockGenVolInfo(10, codemode.EC6P6, proto.VolumeStatusIdle)
+		cli.client.(*MockClusterManager).EXPECT().ListVolume(any, any).Return(cmapi.ListVolumes{Volumes: []*cmapi.VolumeInfo{volume}, Marker: defaultVolumeListMarker}, nil)
+		rets, marker, err := cli.ListVolume(ctx, defaultVolumeListMarker, 10)
+		require.NoError(t, err)
+		require.Equal(t, 1, len(rets))
+		require.Equal(t, marker, defaultVolumeListMarker)
+	}
+	{
+		// list cluster disk
+		cli.client.(*MockClusterManager).EXPECT().ListDisk(any, any).Return(cmapi.ListDiskRet{}, errMock)
+		_, err := cli.ListClusterDisks(ctx)
+		require.True(t, errors.Is(err, errMock))
+	}
+	{
+		// list broken disk
+		cli.client.(*MockClusterManager).EXPECT().ListDisk(any, any).Return(cmapi.ListDiskRet{}, errMock)
+		_, err := cli.ListBrokenDisks(ctx, 1)
+		require.True(t, errors.Is(err, errMock))
+	}
+	{
+		// list repair disk
+		cli.client.(*MockClusterManager).EXPECT().ListDisk(any, any).Return(cmapi.ListDiskRet{}, errMock)
+		_, err := cli.ListRepairingDisks(ctx)
+		require.True(t, errors.Is(err, errMock))
+	}
+	{
+		// list all disk
+		cli.client.(*MockClusterManager).EXPECT().ListDisk(any, any).Return(cmapi.ListDiskRet{}, errMock)
+		_, err := cli.listAllDisks(ctx, proto.DiskStatusNormal)
+		require.True(t, errors.Is(err, errMock))
+
+		disk1 := &blobnode.DiskInfo{Host: "127.0.0.1:xxx", Status: proto.DiskStatusNormal}
+		cli.client.(*MockClusterManager).EXPECT().ListDisk(any, any).Return(cmapi.ListDiskRet{Disks: []*blobnode.DiskInfo{disk1}, Marker: defaultDiskListMarker}, nil)
+		disks, err := cli.listAllDisks(ctx, proto.DiskStatusNormal)
+		require.NoError(t, err)
+		require.Equal(t, 1, len(disks))
+	}
+	{
+		// list disks
+		cli.client.(*MockClusterManager).EXPECT().ListDisk(any, any).Return(cmapi.ListDiskRet{}, errMock)
+		_, err := cli.listDisks(ctx, proto.DiskStatusNormal, 1)
+		require.True(t, errors.Is(err, errMock))
+
+		disk1 := &blobnode.DiskInfo{Host: "127.0.0.1:xxx", Status: proto.DiskStatusNormal}
+		disk2 := &blobnode.DiskInfo{Host: "127.0.0.1:xxx", Status: proto.DiskStatusNormal}
+		cli.client.(*MockClusterManager).EXPECT().ListDisk(any, any).Return(cmapi.ListDiskRet{Disks: []*blobnode.DiskInfo{disk1}, Marker: proto.DiskID(2)}, nil)
+		cli.client.(*MockClusterManager).EXPECT().ListDisk(any, any).Return(cmapi.ListDiskRet{Disks: []*blobnode.DiskInfo{disk2}, Marker: defaultDiskListMarker}, nil)
+		disks, err := cli.listDisks(ctx, proto.DiskStatusNormal, 2)
+		require.NoError(t, err)
+		require.Equal(t, 2, len(disks))
+	}
+	{
+		// list drop disk
+		cli.client.(*MockClusterManager).EXPECT().ListDroppingDisk(any).Return(nil, errMock)
+		_, err := cli.ListDropDisks(ctx)
+		require.True(t, errors.Is(err, errMock))
+
+		disk1 := &blobnode.DiskInfo{Host: "127.0.0.1:xxx", Status: proto.DiskStatusNormal}
+		cli.client.(*MockClusterManager).EXPECT().ListDroppingDisk(any).Return([]*blobnode.DiskInfo{disk1}, nil)
+		disks, err := cli.ListDropDisks(ctx)
+		require.NoError(t, err)
+		require.Equal(t, 1, len(disks))
+	}
+	{
+		// set disk repair
+		cli.client.(*MockClusterManager).EXPECT().SetDisk(any, any, any).Return(nil)
+		err := cli.SetDiskRepairing(ctx, proto.DiskID(1))
+		require.NoError(t, err)
+	}
+	{
+		// set disk repaired
+		cli.client.(*MockClusterManager).EXPECT().SetDisk(any, any, any).Return(nil)
+		err := cli.SetDiskRepaired(ctx, proto.DiskID(1))
+		require.NoError(t, err)
+	}
+	{
+		// set disk repaired
+		cli.client.(*MockClusterManager).EXPECT().DiskInfo(any, any).Return(nil, errMock)
+		err := cli.SetDiskDropped(ctx, proto.DiskID(1))
+		require.True(t, errors.Is(err, errMock))
+
+		disk1 := &blobnode.DiskInfo{Host: "127.0.0.1:xxx", Status: proto.DiskStatusDropped}
+		cli.client.(*MockClusterManager).EXPECT().DiskInfo(any, any).Return(disk1, nil)
+		err = cli.SetDiskDropped(ctx, proto.DiskID(1))
+		require.NoError(t, err)
+
+		disk2 := &blobnode.DiskInfo{Host: "127.0.0.1:xxx", Status: proto.DiskStatusRepairing}
+		cli.client.(*MockClusterManager).EXPECT().DiskInfo(any, any).Return(disk2, nil)
+		err = cli.SetDiskDropped(ctx, proto.DiskID(1))
+		require.True(t, errors.Is(err, errcode.ErrCanNotDropped))
+
+		disk3 := &blobnode.DiskInfo{Host: "127.0.0.1:xxx", Status: proto.DiskStatusNormal}
+		cli.client.(*MockClusterManager).EXPECT().DiskInfo(any, any).Return(disk3, nil)
+		cli.client.(*MockClusterManager).EXPECT().DroppedDisk(any, any).Return(nil)
+		err = cli.SetDiskDropped(ctx, proto.DiskID(1))
+		require.NoError(t, err)
+	}
+	{
+		// get disk info
+		cli.client.(*MockClusterManager).EXPECT().DiskInfo(any, any).Return(nil, errMock)
+		_, err := cli.GetDiskInfo(ctx, proto.DiskID(1))
+		require.True(t, errors.Is(err, errMock))
+
+		disk1 := &blobnode.DiskInfo{Host: "127.0.0.1:xxx", Status: proto.DiskStatusDropped}
+		cli.client.(*MockClusterManager).EXPECT().DiskInfo(any, any).Return(disk1, nil)
+		disk, err := cli.GetDiskInfo(ctx, proto.DiskID(1))
+		require.NoError(t, err)
+		require.Equal(t, disk.Status, disk1.Status)
+		require.False(t, disk.IsBroken())
+	}
+	{
+		// register service
+		cli.client.(*MockClusterManager).EXPECT().RegisterService(any, any, any, any, any).Return(nil)
+		err := cli.Register(ctx, RegisterInfo{})
+		require.NoError(t, err)
+	}
+}

blobstore/scheduler/client/proxy.go

@@ -0,0 +1,58 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package client
+
+import (
+	"context"
+
+	cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	api "github.com/cubefs/cubefs/blobstore/api/proxy"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+)
+
+// ProxyAPI define the interface of proxy used by scheduler
+type ProxyAPI interface {
+	SendShardRepairMsg(ctx context.Context, vid proto.Vid, bid proto.BlobID, badIdx []uint8) error
+}
+
+// proxyClient proxy client
+type proxyClient struct {
+	client    api.LbMsgSender
+	clusterID proto.ClusterID
+}
+
+// NewProxyClient returns proxy client
+func NewProxyClient(cfg *api.LbConfig, clusterMgr *cmapi.Client, clusterID proto.ClusterID) ProxyAPI {
+	return &proxyClient{client: api.NewMQLbClient(cfg, clusterMgr, clusterID), clusterID: clusterID}
+}
+
+// SendShardRepairMsg send shard repair message
+func (c *proxyClient) SendShardRepairMsg(ctx context.Context, vid proto.Vid, bid proto.BlobID, badIdx []uint8) error {
+	pSpan := trace.SpanFromContextSafe(ctx)
+	span, ctx := trace.StartSpanFromContextWithTraceID(context.Background(), "SendShardRepairMsg", pSpan.TraceID())
+	span.Debugf("send shard repair msg vid %d bid %d badIdx %+v", vid, bid, badIdx)
+
+	err := c.client.SendShardRepairMsg(ctx, &api.ShardRepairArgs{
+		ClusterID: c.clusterID,
+		Bid:       bid,
+		Vid:       vid,
+		BadIdxes:  badIdx,
+		Reason:    "inspect",
+	})
+
+	span.Debugf("send shard repair msg ret err %+v", err)
+	return err
+}

blobstore/scheduler/client/proxy_test.go

@@ -0,0 +1,36 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package client
+
+import (
+	"context"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/testing/mocks"
+)
+
+func TestMQProxy(t *testing.T) {
+	mqcli := mocks.NewMockProxyLbRpcClient(gomock.NewController(t))
+	mqcli.EXPECT().SendShardRepairMsg(gomock.Any(), gomock.Any()).Return(nil)
+	cli := &proxyClient{
+		client:    mqcli,
+		clusterID: 1,
+	}
+	err := cli.SendShardRepairMsg(context.Background(), 0, 0, []uint8{0})
+	require.NoError(t, err)
+}

blobstore/scheduler/client/volume_update.go

@@ -0,0 +1,34 @@
+package client
+
+import (
+	"context"
+
+	api "github.com/cubefs/cubefs/blobstore/api/scheduler"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+// IVolumeUpdater update volume cache
+type IVolumeUpdater interface {
+	UpdateFollowerVolumeCache(ctx context.Context, host string, vid proto.Vid) (err error)
+	UpdateLeaderVolumeCache(ctx context.Context, vid proto.Vid) (err error)
+}
+
+type volumeUpdater struct {
+	LeaderHost string
+	Client     api.IVolumeUpdater
+}
+
+func NewVolumeUpdater(cfg *api.Config, host string) IVolumeUpdater {
+	return &volumeUpdater{
+		LeaderHost: host,
+		Client:     api.NewVolumeUpdater(cfg),
+	}
+}
+
+func (v *volumeUpdater) UpdateLeaderVolumeCache(ctx context.Context, vid proto.Vid) (err error) {
+	return v.Client.UpdateVol(ctx, v.LeaderHost, vid)
+}
+
+func (v *volumeUpdater) UpdateFollowerVolumeCache(ctx context.Context, host string, vid proto.Vid) (err error) {
+	return v.Client.UpdateVol(ctx, host, vid)
+}

blobstore/scheduler/client/volume_update_test.go

@@ -0,0 +1,46 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package client
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/require"
+
+	api "github.com/cubefs/cubefs/blobstore/api/scheduler"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/testing/mocks"
+)
+
+func TestVolumeUpdate(t *testing.T) {
+	any := gomock.Any()
+	errMock := errors.New("fake error")
+	cli := NewVolumeUpdater(&api.Config{}, "127.0.0.1:xxx").(*volumeUpdater)
+
+	ctx := context.Background()
+	schedulerCli := mocks.NewMockIScheduler(gomock.NewController(t))
+	schedulerCli.EXPECT().UpdateVol(any, any, any).Return(nil)
+	cli.Client = schedulerCli
+
+	err := cli.UpdateLeaderVolumeCache(ctx, proto.Vid(1))
+	require.NoError(t, err)
+
+	schedulerCli.EXPECT().UpdateVol(any, any, any).Return(errMock)
+	err = cli.UpdateFollowerVolumeCache(ctx, "127.0.0.1:xxx", proto.Vid(1))
+	require.True(t, errors.Is(err, errMock))
+}

blobstore/scheduler/client_mock_test.go

@@ -0,0 +1,468 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/cubefs/cubefs/blobstore/scheduler/client (interfaces: ClusterMgrAPI,BlobnodeAPI,IVolumeUpdater,ProxyAPI)
+
+// Package scheduler is a generated GoMock package.
+package scheduler
+
+import (
+	context "context"
+	reflect "reflect"
+
+	proto "github.com/cubefs/cubefs/blobstore/common/proto"
+	client "github.com/cubefs/cubefs/blobstore/scheduler/client"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockClusterMgrAPI is a mock of ClusterMgrAPI interface.
+type MockClusterMgrAPI struct {
+	ctrl     *gomock.Controller
+	recorder *MockClusterMgrAPIMockRecorder
+}
+
+// MockClusterMgrAPIMockRecorder is the mock recorder for MockClusterMgrAPI.
+type MockClusterMgrAPIMockRecorder struct {
+	mock *MockClusterMgrAPI
+}
+
+// NewMockClusterMgrAPI creates a new mock instance.
+func NewMockClusterMgrAPI(ctrl *gomock.Controller) *MockClusterMgrAPI {
+	mock := &MockClusterMgrAPI{ctrl: ctrl}
+	mock.recorder = &MockClusterMgrAPIMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockClusterMgrAPI) EXPECT() *MockClusterMgrAPIMockRecorder {
+	return m.recorder
+}
+
+// AllocVolumeUnit mocks base method.
+func (m *MockClusterMgrAPI) AllocVolumeUnit(arg0 context.Context, arg1 proto.Vuid) (*client.AllocVunitInfo, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "AllocVolumeUnit", arg0, arg1)
+	ret0, _ := ret[0].(*client.AllocVunitInfo)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// AllocVolumeUnit indicates an expected call of AllocVolumeUnit.
+func (mr *MockClusterMgrAPIMockRecorder) AllocVolumeUnit(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocVolumeUnit", reflect.TypeOf((*MockClusterMgrAPI)(nil).AllocVolumeUnit), arg0, arg1)
+}
+
+// GetConfig mocks base method.
+func (m *MockClusterMgrAPI) GetConfig(arg0 context.Context, arg1 string) (string, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetConfig", arg0, arg1)
+	ret0, _ := ret[0].(string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetConfig indicates an expected call of GetConfig.
+func (mr *MockClusterMgrAPIMockRecorder) GetConfig(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfig", reflect.TypeOf((*MockClusterMgrAPI)(nil).GetConfig), arg0, arg1)
+}
+
+// GetDiskInfo mocks base method.
+func (m *MockClusterMgrAPI) GetDiskInfo(arg0 context.Context, arg1 proto.DiskID) (*client.DiskInfoSimple, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetDiskInfo", arg0, arg1)
+	ret0, _ := ret[0].(*client.DiskInfoSimple)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetDiskInfo indicates an expected call of GetDiskInfo.
+func (mr *MockClusterMgrAPIMockRecorder) GetDiskInfo(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDiskInfo", reflect.TypeOf((*MockClusterMgrAPI)(nil).GetDiskInfo), arg0, arg1)
+}
+
+// GetService mocks base method.
+func (m *MockClusterMgrAPI) GetService(arg0 context.Context, arg1 string, arg2 proto.ClusterID) ([]string, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetService", arg0, arg1, arg2)
+	ret0, _ := ret[0].([]string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetService indicates an expected call of GetService.
+func (mr *MockClusterMgrAPIMockRecorder) GetService(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetService", reflect.TypeOf((*MockClusterMgrAPI)(nil).GetService), arg0, arg1, arg2)
+}
+
+// GetVolumeInfo mocks base method.
+func (m *MockClusterMgrAPI) GetVolumeInfo(arg0 context.Context, arg1 proto.Vid) (*client.VolumeInfoSimple, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetVolumeInfo", arg0, arg1)
+	ret0, _ := ret[0].(*client.VolumeInfoSimple)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetVolumeInfo indicates an expected call of GetVolumeInfo.
+func (mr *MockClusterMgrAPIMockRecorder) GetVolumeInfo(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVolumeInfo", reflect.TypeOf((*MockClusterMgrAPI)(nil).GetVolumeInfo), arg0, arg1)
+}
+
+// ListBrokenDisks mocks base method.
+func (m *MockClusterMgrAPI) ListBrokenDisks(arg0 context.Context, arg1 int) ([]*client.DiskInfoSimple, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ListBrokenDisks", arg0, arg1)
+	ret0, _ := ret[0].([]*client.DiskInfoSimple)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ListBrokenDisks indicates an expected call of ListBrokenDisks.
+func (mr *MockClusterMgrAPIMockRecorder) ListBrokenDisks(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListBrokenDisks", reflect.TypeOf((*MockClusterMgrAPI)(nil).ListBrokenDisks), arg0, arg1)
+}
+
+// ListClusterDisks mocks base method.
+func (m *MockClusterMgrAPI) ListClusterDisks(arg0 context.Context) ([]*client.DiskInfoSimple, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ListClusterDisks", arg0)
+	ret0, _ := ret[0].([]*client.DiskInfoSimple)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ListClusterDisks indicates an expected call of ListClusterDisks.
+func (mr *MockClusterMgrAPIMockRecorder) ListClusterDisks(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListClusterDisks", reflect.TypeOf((*MockClusterMgrAPI)(nil).ListClusterDisks), arg0)
+}
+
+// ListDiskVolumeUnits mocks base method.
+func (m *MockClusterMgrAPI) ListDiskVolumeUnits(arg0 context.Context, arg1 proto.DiskID) ([]*client.VunitInfoSimple, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ListDiskVolumeUnits", arg0, arg1)
+	ret0, _ := ret[0].([]*client.VunitInfoSimple)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ListDiskVolumeUnits indicates an expected call of ListDiskVolumeUnits.
+func (mr *MockClusterMgrAPIMockRecorder) ListDiskVolumeUnits(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDiskVolumeUnits", reflect.TypeOf((*MockClusterMgrAPI)(nil).ListDiskVolumeUnits), arg0, arg1)
+}
+
+// ListDropDisks mocks base method.
+func (m *MockClusterMgrAPI) ListDropDisks(arg0 context.Context) ([]*client.DiskInfoSimple, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ListDropDisks", arg0)
+	ret0, _ := ret[0].([]*client.DiskInfoSimple)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ListDropDisks indicates an expected call of ListDropDisks.
+func (mr *MockClusterMgrAPIMockRecorder) ListDropDisks(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDropDisks", reflect.TypeOf((*MockClusterMgrAPI)(nil).ListDropDisks), arg0)
+}
+
+// ListRepairingDisks mocks base method.
+func (m *MockClusterMgrAPI) ListRepairingDisks(arg0 context.Context) ([]*client.DiskInfoSimple, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ListRepairingDisks", arg0)
+	ret0, _ := ret[0].([]*client.DiskInfoSimple)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ListRepairingDisks indicates an expected call of ListRepairingDisks.
+func (mr *MockClusterMgrAPIMockRecorder) ListRepairingDisks(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRepairingDisks", reflect.TypeOf((*MockClusterMgrAPI)(nil).ListRepairingDisks), arg0)
+}
+
+// ListVolume mocks base method.
+func (m *MockClusterMgrAPI) ListVolume(arg0 context.Context, arg1 proto.Vid, arg2 int) ([]*client.VolumeInfoSimple, proto.Vid, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ListVolume", arg0, arg1, arg2)
+	ret0, _ := ret[0].([]*client.VolumeInfoSimple)
+	ret1, _ := ret[1].(proto.Vid)
+	ret2, _ := ret[2].(error)
+	return ret0, ret1, ret2
+}
+
+// ListVolume indicates an expected call of ListVolume.
+func (mr *MockClusterMgrAPIMockRecorder) ListVolume(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListVolume", reflect.TypeOf((*MockClusterMgrAPI)(nil).ListVolume), arg0, arg1, arg2)
+}
+
+// LockVolume mocks base method.
+func (m *MockClusterMgrAPI) LockVolume(arg0 context.Context, arg1 proto.Vid) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "LockVolume", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// LockVolume indicates an expected call of LockVolume.
+func (mr *MockClusterMgrAPIMockRecorder) LockVolume(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LockVolume", reflect.TypeOf((*MockClusterMgrAPI)(nil).LockVolume), arg0, arg1)
+}
+
+// Register mocks base method.
+func (m *MockClusterMgrAPI) Register(arg0 context.Context, arg1 client.RegisterInfo) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Register", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Register indicates an expected call of Register.
+func (mr *MockClusterMgrAPIMockRecorder) Register(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Register", reflect.TypeOf((*MockClusterMgrAPI)(nil).Register), arg0, arg1)
+}
+
+// ReleaseVolumeUnit mocks base method.
+func (m *MockClusterMgrAPI) ReleaseVolumeUnit(arg0 context.Context, arg1 proto.Vuid, arg2 proto.DiskID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ReleaseVolumeUnit", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// ReleaseVolumeUnit indicates an expected call of ReleaseVolumeUnit.
+func (mr *MockClusterMgrAPIMockRecorder) ReleaseVolumeUnit(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReleaseVolumeUnit", reflect.TypeOf((*MockClusterMgrAPI)(nil).ReleaseVolumeUnit), arg0, arg1, arg2)
+}
+
+// SetDiskDropped mocks base method.
+func (m *MockClusterMgrAPI) SetDiskDropped(arg0 context.Context, arg1 proto.DiskID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "SetDiskDropped", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// SetDiskDropped indicates an expected call of SetDiskDropped.
+func (mr *MockClusterMgrAPIMockRecorder) SetDiskDropped(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDiskDropped", reflect.TypeOf((*MockClusterMgrAPI)(nil).SetDiskDropped), arg0, arg1)
+}
+
+// SetDiskRepaired mocks base method.
+func (m *MockClusterMgrAPI) SetDiskRepaired(arg0 context.Context, arg1 proto.DiskID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "SetDiskRepaired", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// SetDiskRepaired indicates an expected call of SetDiskRepaired.
+func (mr *MockClusterMgrAPIMockRecorder) SetDiskRepaired(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDiskRepaired", reflect.TypeOf((*MockClusterMgrAPI)(nil).SetDiskRepaired), arg0, arg1)
+}
+
+// SetDiskRepairing mocks base method.
+func (m *MockClusterMgrAPI) SetDiskRepairing(arg0 context.Context, arg1 proto.DiskID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "SetDiskRepairing", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// SetDiskRepairing indicates an expected call of SetDiskRepairing.
+func (mr *MockClusterMgrAPIMockRecorder) SetDiskRepairing(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDiskRepairing", reflect.TypeOf((*MockClusterMgrAPI)(nil).SetDiskRepairing), arg0, arg1)
+}
+
+// UnlockVolume mocks base method.
+func (m *MockClusterMgrAPI) UnlockVolume(arg0 context.Context, arg1 proto.Vid) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UnlockVolume", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// UnlockVolume indicates an expected call of UnlockVolume.
+func (mr *MockClusterMgrAPIMockRecorder) UnlockVolume(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnlockVolume", reflect.TypeOf((*MockClusterMgrAPI)(nil).UnlockVolume), arg0, arg1)
+}
+
+// UpdateVolume mocks base method.
+func (m *MockClusterMgrAPI) UpdateVolume(arg0 context.Context, arg1, arg2 proto.Vuid, arg3 proto.DiskID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateVolume", arg0, arg1, arg2, arg3)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// UpdateVolume indicates an expected call of UpdateVolume.
+func (mr *MockClusterMgrAPIMockRecorder) UpdateVolume(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateVolume", reflect.TypeOf((*MockClusterMgrAPI)(nil).UpdateVolume), arg0, arg1, arg2, arg3)
+}
+
+// MockBlobnodeAPI is a mock of BlobnodeAPI interface.
+type MockBlobnodeAPI struct {
+	ctrl     *gomock.Controller
+	recorder *MockBlobnodeAPIMockRecorder
+}
+
+// MockBlobnodeAPIMockRecorder is the mock recorder for MockBlobnodeAPI.
+type MockBlobnodeAPIMockRecorder struct {
+	mock *MockBlobnodeAPI
+}
+
+// NewMockBlobnodeAPI creates a new mock instance.
+func NewMockBlobnodeAPI(ctrl *gomock.Controller) *MockBlobnodeAPI {
+	mock := &MockBlobnodeAPI{ctrl: ctrl}
+	mock.recorder = &MockBlobnodeAPIMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockBlobnodeAPI) EXPECT() *MockBlobnodeAPIMockRecorder {
+	return m.recorder
+}
+
+// Delete mocks base method.
+func (m *MockBlobnodeAPI) Delete(arg0 context.Context, arg1 proto.VunitLocation, arg2 proto.BlobID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Delete", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Delete indicates an expected call of Delete.
+func (mr *MockBlobnodeAPIMockRecorder) Delete(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockBlobnodeAPI)(nil).Delete), arg0, arg1, arg2)
+}
+
+// MarkDelete mocks base method.
+func (m *MockBlobnodeAPI) MarkDelete(arg0 context.Context, arg1 proto.VunitLocation, arg2 proto.BlobID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "MarkDelete", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// MarkDelete indicates an expected call of MarkDelete.
+func (mr *MockBlobnodeAPIMockRecorder) MarkDelete(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkDelete", reflect.TypeOf((*MockBlobnodeAPI)(nil).MarkDelete), arg0, arg1, arg2)
+}
+
+// RepairShard mocks base method.
+func (m *MockBlobnodeAPI) RepairShard(arg0 context.Context, arg1 string, arg2 proto.ShardRepairTask) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "RepairShard", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// RepairShard indicates an expected call of RepairShard.
+func (mr *MockBlobnodeAPIMockRecorder) RepairShard(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RepairShard", reflect.TypeOf((*MockBlobnodeAPI)(nil).RepairShard), arg0, arg1, arg2)
+}
+
+// MockVolumeUpdater is a mock of IVolumeUpdater interface.
+type MockVolumeUpdater struct {
+	ctrl     *gomock.Controller
+	recorder *MockVolumeUpdaterMockRecorder
+}
+
+// MockVolumeUpdaterMockRecorder is the mock recorder for MockVolumeUpdater.
+type MockVolumeUpdaterMockRecorder struct {
+	mock *MockVolumeUpdater
+}
+
+// NewMockVolumeUpdater creates a new mock instance.
+func NewMockVolumeUpdater(ctrl *gomock.Controller) *MockVolumeUpdater {
+	mock := &MockVolumeUpdater{ctrl: ctrl}
+	mock.recorder = &MockVolumeUpdaterMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockVolumeUpdater) EXPECT() *MockVolumeUpdaterMockRecorder {
+	return m.recorder
+}
+
+// UpdateFollowerVolumeCache mocks base method.
+func (m *MockVolumeUpdater) UpdateFollowerVolumeCache(arg0 context.Context, arg1 string, arg2 proto.Vid) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateFollowerVolumeCache", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// UpdateFollowerVolumeCache indicates an expected call of UpdateFollowerVolumeCache.
+func (mr *MockVolumeUpdaterMockRecorder) UpdateFollowerVolumeCache(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateFollowerVolumeCache", reflect.TypeOf((*MockVolumeUpdater)(nil).UpdateFollowerVolumeCache), arg0, arg1, arg2)
+}
+
+// UpdateLeaderVolumeCache mocks base method.
+func (m *MockVolumeUpdater) UpdateLeaderVolumeCache(arg0 context.Context, arg1 proto.Vid) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateLeaderVolumeCache", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// UpdateLeaderVolumeCache indicates an expected call of UpdateLeaderVolumeCache.
+func (mr *MockVolumeUpdaterMockRecorder) UpdateLeaderVolumeCache(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateLeaderVolumeCache", reflect.TypeOf((*MockVolumeUpdater)(nil).UpdateLeaderVolumeCache), arg0, arg1)
+}
+
+// MockMqProxyAPI is a mock of ProxyAPI interface.
+type MockMqProxyAPI struct {
+	ctrl     *gomock.Controller
+	recorder *MockMqProxyAPIMockRecorder
+}
+
+// MockMqProxyAPIMockRecorder is the mock recorder for MockMqProxyAPI.
+type MockMqProxyAPIMockRecorder struct {
+	mock *MockMqProxyAPI
+}
+
+// NewMockMqProxyAPI creates a new mock instance.
+func NewMockMqProxyAPI(ctrl *gomock.Controller) *MockMqProxyAPI {
+	mock := &MockMqProxyAPI{ctrl: ctrl}
+	mock.recorder = &MockMqProxyAPIMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockMqProxyAPI) EXPECT() *MockMqProxyAPIMockRecorder {
+	return m.recorder
+}
+
+// SendShardRepairMsg mocks base method.
+func (m *MockMqProxyAPI) SendShardRepairMsg(arg0 context.Context, arg1 proto.Vid, arg2 proto.BlobID, arg3 []byte) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "SendShardRepairMsg", arg0, arg1, arg2, arg3)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// SendShardRepairMsg indicates an expected call of SendShardRepairMsg.
+func (mr *MockMqProxyAPIMockRecorder) SendShardRepairMsg(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendShardRepairMsg", reflect.TypeOf((*MockMqProxyAPI)(nil).SendShardRepairMsg), arg0, arg1, arg2, arg3)
+}

blobstore/scheduler/cluster_topology.go

@@ -0,0 +1,238 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package scheduler
+
+import (
+	"context"
+	"sort"
+	"time"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/scheduler/base"
+	"github.com/cubefs/cubefs/blobstore/scheduler/client"
+	"github.com/cubefs/cubefs/blobstore/util/closer"
+	"github.com/cubefs/cubefs/blobstore/util/log"
+)
+
+// IClusterTopology define the interface og cluster topology
+type IClusterTopology interface {
+	GetIDCs() map[string]*IDC
+	GetIDCDisks(idc string) (disks []*client.DiskInfoSimple)
+	closer.Closer
+}
+
+type clusterTopoConf struct {
+	ClusterID               proto.ClusterID
+	UpdateInterval          time.Duration
+	FreeChunkCounterBuckets []float64
+}
+
+// ClusterTopology cluster topology
+type ClusterTopology struct {
+	clusterID    proto.ClusterID
+	idcMap       map[string]*IDC
+	diskMap      map[string][]*client.DiskInfoSimple
+	FreeChunkCnt int64
+	MaxChunkCnt  int64
+}
+
+// IDC idc info
+type IDC struct {
+	name         string
+	rackMap      map[string]*Rack
+	FreeChunkCnt int64
+	MaxChunkCnt  int64
+}
+
+// Rack rack info
+type Rack struct {
+	name         string
+	diskMap      map[string]*Host
+	FreeChunkCnt int64
+	MaxChunkCnt  int64
+}
+
+// Host host info
+type Host struct {
+	host         string                   // ip+port
+	disks        []*client.DiskInfoSimple // disk list
+	FreeChunkCnt int64                    // host free chunk count
+	MaxChunkCnt  int64                    // total chunk count
+}
+
+// ClusterTopologyMgr cluster topology manager
+type ClusterTopologyMgr struct {
+	closer.Closer
+	updateInterval time.Duration
+	clusterID      proto.ClusterID
+
+	clusterMgrCli client.ClusterMgrAPI
+
+	clusterTopo  *ClusterTopology
+	taskStatsMgr *base.ClusterTopologyStatsMgr
+}
+
+// NewClusterTopologyMgr returns cluster topology manager
+func NewClusterTopologyMgr(topologyClient client.ClusterMgrAPI, conf *clusterTopoConf) IClusterTopology {
+	mgr := &ClusterTopologyMgr{
+		Closer:         closer.New(),
+		updateInterval: conf.UpdateInterval,
+		clusterID:      conf.ClusterID,
+
+		clusterMgrCli: topologyClient,
+
+		clusterTopo: &ClusterTopology{
+			idcMap:  make(map[string]*IDC),
+			diskMap: make(map[string][]*client.DiskInfoSimple),
+		},
+		taskStatsMgr: base.NewClusterTopologyStatisticsMgr(conf.ClusterID, conf.FreeChunkCounterBuckets),
+	}
+	go mgr.loopUpdate()
+	return mgr
+}
+
+func (m *ClusterTopologyMgr) loopUpdate() {
+	t := time.NewTicker(m.updateInterval)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-t.C:
+			m.updateClusterTopology()
+		case <-m.Closer.Done():
+			return
+		}
+	}
+}
+
+func (m *ClusterTopologyMgr) updateClusterTopology() {
+	span, ctx := trace.StartSpanFromContext(context.Background(), "updateClusterTopology")
+
+	disks, err := m.clusterMgrCli.ListClusterDisks(ctx)
+	if err != nil {
+		span.Errorf("update cluster topology failed: err[%+v]", err)
+	}
+
+	m.buildClusterTopo(disks, m.clusterID)
+}
+
+// GetIDCs returns IDCs
+func (m *ClusterTopologyMgr) GetIDCs() map[string]*IDC {
+	return m.clusterTopo.idcMap
+}
+
+// GetIDCDisks returns disks with IDC
+func (m *ClusterTopologyMgr) GetIDCDisks(idc string) (disks []*client.DiskInfoSimple) {
+	return m.clusterTopo.diskMap[idc]
+}
+
+// ReportFreeChunkCnt report free chunk cnt
+func (m *ClusterTopologyMgr) ReportFreeChunkCnt(disk *client.DiskInfoSimple) {
+	m.taskStatsMgr.ReportFreeChunk(disk)
+}
+
+func (m *ClusterTopologyMgr) buildClusterTopo(disks []*client.DiskInfoSimple, clusterID proto.ClusterID) {
+	cluster := &ClusterTopology{
+		clusterID: clusterID,
+		idcMap:    make(map[string]*IDC),
+		diskMap:   make(map[string][]*client.DiskInfoSimple),
+	}
+
+	for i := range disks {
+		if cluster.clusterID != disks[i].ClusterID {
+			log.Errorf("the disk does not belong to this cluster: cluster_id[%d], disk[%+v]", cluster.clusterID, disks[i])
+			continue
+		}
+		cluster.addDisk(disks[i])
+		m.ReportFreeChunkCnt(disks[i])
+	}
+
+	for idc := range cluster.diskMap {
+		sortDiskByFreeChunkCnt(cluster.diskMap[idc])
+	}
+	m.clusterTopo = cluster
+}
+
+func (cluster *ClusterTopology) addDisk(disk *client.DiskInfoSimple) {
+	cluster.addDiskToCluster(disk)
+	cluster.addDiskToDiskMap(disk)
+	cluster.addDiskToIdc(disk)
+	cluster.addDiskToRack(disk)
+	cluster.addDiskToHost(disk)
+}
+
+func (cluster *ClusterTopology) addDiskToCluster(disk *client.DiskInfoSimple) {
+	// statistics cluster chunk info
+	cluster.FreeChunkCnt += disk.FreeChunkCnt
+	cluster.MaxChunkCnt += disk.MaxChunkCnt
+}
+
+func (cluster *ClusterTopology) addDiskToDiskMap(disk *client.DiskInfoSimple) {
+	if _, ok := cluster.diskMap[disk.Idc]; !ok {
+		var disks []*client.DiskInfoSimple
+		cluster.diskMap[disk.Idc] = disks
+	}
+	cluster.diskMap[disk.Idc] = append(cluster.diskMap[disk.Idc], disk)
+}
+
+func (cluster *ClusterTopology) addDiskToIdc(disk *client.DiskInfoSimple) {
+	idcName := disk.Idc
+	if _, ok := cluster.idcMap[idcName]; !ok {
+		cluster.idcMap[idcName] = &IDC{
+			name:    idcName,
+			rackMap: make(map[string]*Rack),
+		}
+	}
+	// statistics idc chunk info
+	cluster.idcMap[idcName].FreeChunkCnt += disk.FreeChunkCnt
+	cluster.idcMap[idcName].MaxChunkCnt += disk.MaxChunkCnt
+}
+
+func (cluster *ClusterTopology) addDiskToRack(disk *client.DiskInfoSimple) {
+	idc := cluster.idcMap[disk.Idc]
+	rackName := disk.Rack
+	if _, ok := idc.rackMap[rackName]; !ok {
+		idc.rackMap[rackName] = &Rack{
+			name:    rackName,
+			diskMap: make(map[string]*Host),
+		}
+	}
+	// statistics rack chunk info
+	idc.rackMap[rackName].FreeChunkCnt += disk.FreeChunkCnt
+	idc.rackMap[rackName].MaxChunkCnt += disk.MaxChunkCnt
+}
+
+func (cluster *ClusterTopology) addDiskToHost(disk *client.DiskInfoSimple) {
+	rack := cluster.idcMap[disk.Idc].rackMap[disk.Rack]
+	if _, ok := rack.diskMap[disk.Host]; !ok {
+		var disks []*client.DiskInfoSimple
+		rack.diskMap[disk.Host] = &Host{
+			host:  disk.Host,
+			disks: disks,
+		}
+	}
+	rack.diskMap[disk.Host].disks = append(rack.diskMap[disk.Host].disks, disk)
+
+	// statistics host chunk info
+	rack.diskMap[disk.Host].FreeChunkCnt += disk.FreeChunkCnt
+	rack.diskMap[disk.Host].MaxChunkCnt += disk.MaxChunkCnt
+}
+
+func sortDiskByFreeChunkCnt(disks []*client.DiskInfoSimple) {
+	sort.Slice(disks, func(i, j int) bool {
+		return disks[i].FreeChunkCnt < disks[j].FreeChunkCnt
+	})
+}

blobstore/scheduler/cluster_topology_test.go

@@ -0,0 +1,114 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package scheduler
+
+import (
+	"testing"
+	"time"
+
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/scheduler/base"
+	"github.com/cubefs/cubefs/blobstore/scheduler/client"
+)
+
+var (
+	topoDisk1 = &client.DiskInfoSimple{
+		ClusterID:    1,
+		Idc:          "z0",
+		Rack:         "rack1",
+		Host:         "127.0.0.1:8000",
+		DiskID:       1,
+		FreeChunkCnt: 10,
+		MaxChunkCnt:  700,
+	}
+	topoDisk2 = &client.DiskInfoSimple{
+		ClusterID:    1,
+		Idc:          "z0",
+		Rack:         "rack1",
+		Host:         "127.0.0.2:8000",
+		DiskID:       2,
+		FreeChunkCnt: 100,
+		MaxChunkCnt:  700,
+	}
+	topoDisk3 = &client.DiskInfoSimple{
+		ClusterID:    1,
+		Idc:          "z1",
+		Rack:         "rack1",
+		Host:         "127.0.0.3:8000",
+		DiskID:       3,
+		FreeChunkCnt: 20,
+		MaxChunkCnt:  700,
+	}
+	topoDisk4 = &client.DiskInfoSimple{
+		ClusterID:    1,
+		Idc:          "z1",
+		Rack:         "rack2",
+		Host:         "127.0.0.4:8000",
+		DiskID:       4,
+		FreeChunkCnt: 5,
+		MaxChunkCnt:  700,
+	}
+	topoDisk5 = &client.DiskInfoSimple{
+		ClusterID:    1,
+		Idc:          "z2",
+		Rack:         "rack2",
+		Host:         "127.0.0.4:8000",
+		DiskID:       5,
+		FreeChunkCnt: 200,
+		MaxChunkCnt:  700,
+	}
+	topoDisk6 = &client.DiskInfoSimple{
+		ClusterID:    123,
+		Idc:          "z2",
+		Rack:         "rack2",
+		Host:         "127.0.0.4:8000",
+		DiskID:       5,
+		FreeChunkCnt: 200,
+		MaxChunkCnt:  700,
+	}
+
+	topoDisks = []*client.DiskInfoSimple{topoDisk1, topoDisk2, topoDisk3, topoDisk4, topoDisk5, topoDisk6}
+)
+
+func TestNewClusterTopologyMgr(t *testing.T) {
+	clusterTopMgr := &ClusterTopologyMgr{
+		taskStatsMgr: base.NewClusterTopologyStatisticsMgr(1, []float64{}),
+	}
+	clusterTopMgr.buildClusterTopo(topoDisks, 1)
+	require.Equal(t, 3, len(clusterTopMgr.GetIDCs()))
+	disks := clusterTopMgr.GetIDCDisks("z0")
+	require.Equal(t, 2, len(disks))
+	disks = clusterTopMgr.GetIDCDisks("z1")
+	require.Equal(t, 2, len(disks))
+	disks = clusterTopMgr.GetIDCDisks("z2")
+	require.Equal(t, 1, len(disks))
+	disks = clusterTopMgr.GetIDCDisks("z3")
+	require.True(t, disks == nil)
+
+	ctr := gomock.NewController(t)
+	clusterMgrCli := NewMockClusterMgrAPI(ctr)
+	clusterMgrCli.EXPECT().ListClusterDisks(any).AnyTimes().Return(nil, errMock)
+	conf := &clusterTopoConf{
+		ClusterID:      1,
+		UpdateInterval: 1 * time.Microsecond,
+	}
+	mgr := NewClusterTopologyMgr(clusterMgrCli, conf)
+	defer mgr.Close()
+
+	// wait topology update
+	time.Sleep(2 * time.Microsecond)
+}

blobstore/scheduler/config.go

@@ -0,0 +1,306 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package scheduler
+
+import (
+	"github.com/cubefs/cubefs/blobstore/api/blobnode"
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/api/proxy"
+	"github.com/cubefs/cubefs/blobstore/cmd"
+	"github.com/cubefs/cubefs/blobstore/common/mongoutil"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+	"github.com/cubefs/cubefs/blobstore/scheduler/db"
+	"github.com/cubefs/cubefs/blobstore/util/defaulter"
+)
+
+const (
+	defaultTopologyUpdateIntervalMin  = 1
+	defaultVolumeCacheUpdateIntervalS = 10
+	defaultArchiveDelayMin            = 5
+	defaultArchiveIntervalMin         = 5
+	defaultRetryHostsCnt              = 1
+	defaultClientTimeoutMs            = int64(1000)
+	defaultHostSyncIntervalMs         = int64(1000)
+	defaultMongoTimeoutMs             = int64(3000)
+
+	defaultBalanceDiskCntLimit = 100
+	defaultMaxDiskFreeChunkCnt = int64(1024)
+	defaultMinDiskFreeChunkCnt = int64(20)
+
+	defaultInspectIntervalS  = 1
+	defaultListVolIntervalMs = 10
+	defaultListVolStep       = 100
+	defaultInspectBatch      = 1000
+	defaultInspectTimeoutMs  = 10000
+
+	defaultTaskPoolSize             = 10
+	defaultHandleBatchCnt           = 100
+	defaultFailMsgConsumeIntervalMs = int64(10000)
+	defaultDeleteLogChunkSize       = uint(29)
+	defaultDeleteDelayH             = int64(72)
+	defaultDeleteNoDelay            = int64(0)
+
+	defaultTickInterval   = uint32(1)
+	defaultHeartbeatTicks = uint32(30)
+	defaultExpiresTicks   = uint32(60)
+
+	defaultDatabase               = "scheduler"
+	defaultBalanceTable           = "balance_tbl"
+	defaultDiskDropTable          = "disk_drop_tbl"
+	defaultRepairTable            = "repair_tbl"
+	defaultInspectCheckPointTable = "inspect_checkpoint_tbl"
+	defaultManualMigrateTable     = "manual_migrate_tbl"
+	defaultArchiveTasksTable      = "archive_tasks_tbl"
+	defaultKafkaOffsetTable       = "kafka_offset_tbl"
+	defaultOrphanedShardTable     = "orphaned_shard_tbl"
+
+	defaultShardRepairNormalTopic   = "shard_repair"
+	defaultShardRepairPriorityTopic = "shard_repair_prior"
+	defaultShardRepairFailedTopic   = "shard_repair_failed"
+
+	defaultBlobDeleteNormalTopic = "blob_delete"
+	defaultBlobDeleteFailedTopic = "blob_delete_failed"
+)
+
+var defaultWriteConfig = mongoutil.DefaultWriteConfig
+
+// Config service config
+type Config struct {
+	cmd.Config
+
+	ClusterID proto.ClusterID `json:"cluster_id"`
+	Services  Services        `json:"services"`
+
+	Database                   db.Config `json:"database"`
+	TopologyUpdateIntervalMin  int       `json:"topology_update_interval_min"`
+	VolumeCacheUpdateIntervalS int       `json:"volume_cache_update_interval_s"`
+	FreeChunkCounterBuckets    []float64 `json:"free_chunk_counter_buckets"`
+
+	ClusterMgr clustermgr.Config `json:"clustermgr"`
+	Proxy      proxy.LbConfig    `json:"proxy"`
+	Blobnode   blobnode.Config   `json:"blobnode"`
+	Scheduler  rpc.Config        `json:"scheduler"`
+
+	Balance       BalanceMgrConfig    `json:"balance"`
+	DiskDrop      DiskDropMgrConfig   `json:"disk_drop"`
+	DiskRepair    DiskRepairMgrCfg    `json:"disk_repair"`
+	VolumeInspect VolumeInspectMgrCfg `json:"volume_inspect"`
+	Archive       ArchiveStoreConfig  `json:"archive"`
+
+	Kafka       KafkaConfig       `json:"kafka"`
+	ShardRepair ShardRepairConfig `json:"shard_repair"`
+	BlobDelete  BlobDeleteConfig  `json:"blob_delete"`
+
+	ServiceRegister ServiceRegisterConfig `json:"service_register"`
+}
+
+// ServiceRegisterConfig is service register info
+type ServiceRegisterConfig struct {
+	TickInterval   uint32 `json:"tick_interval"`
+	HeartbeatTicks uint32 `json:"heartbeat_ticks"`
+	ExpiresTicks   uint32 `json:"expires_ticks"`
+	Idc            string `json:"idc"`
+	Host           string `json:"host"`
+}
+
+// ShardRepairKafkaConfig is kafka config of shard repair
+type ShardRepairKafkaConfig struct {
+	BrokerList             []string    `json:"-"`
+	FailMsgSenderTimeoutMs int64       `json:"-"`
+	Normal                 TopicConfig `json:"normal"`
+	Failed                 TopicConfig `json:"failed"`
+	Priority               TopicConfig `json:"priority"`
+}
+
+// BlobDeleteKafkaConfig is kafka config of blob delete
+type BlobDeleteKafkaConfig struct {
+	BrokerList             []string    `json:"-"`
+	FailMsgSenderTimeoutMs int64       `json:"-"`
+	Normal                 TopicConfig `json:"normal"`
+	Failed                 TopicConfig `json:"failed"`
+}
+
+// KafkaConfig kafka config
+type KafkaConfig struct {
+	BrokerList             []string               `json:"broker_list"`
+	FailMsgSenderTimeoutMs int64                  `json:"fail_msg_sender_timeout_ms"`
+	ShardRepair            ShardRepairKafkaConfig `json:"shard_repair"`
+	BlobDelete             BlobDeleteKafkaConfig  `json:"blob_delete"`
+}
+
+// TopicConfig topic config
+type TopicConfig struct {
+	Topic      string  `json:"topic"`
+	Partitions []int32 `json:"partitions"`
+}
+
+type Services struct {
+	Leader  uint64            `json:"leader"`
+	NodeID  uint64            `json:"node_id"`
+	Members map[uint64]string `json:"members"`
+}
+
+func (c *Config) IsLeader() bool {
+	return c.Services.Leader == c.Services.NodeID
+}
+
+func (c *Config) Leader() string {
+	return c.Services.Members[c.Services.Leader]
+}
+
+func (c *Config) Follower() []string {
+	var followers []string
+	for k, v := range c.Services.Members {
+		if k != c.Services.Leader {
+			followers = append(followers, v)
+		}
+	}
+	return followers
+}
+
+func (c *Config) fixServices() error {
+	if len(c.Services.Members) < 1 {
+		return errInvalidMembers
+	}
+	if _, ok := c.Services.Members[c.Services.Leader]; !ok {
+		return errInvalidLeader
+	}
+	if _, ok := c.Services.Members[c.Services.NodeID]; !ok {
+		return errInvalidNodeID
+	}
+	return nil
+}
+
+func (c *Config) fixConfig() (err error) {
+	if c.ClusterID == 0 {
+		return errIllegalClusterID
+	}
+	if err := c.fixServices(); err != nil {
+		return err
+	}
+	defaulter.LessOrEqual(&c.TopologyUpdateIntervalMin, defaultTopologyUpdateIntervalMin)
+	defaulter.LessOrEqual(&c.VolumeCacheUpdateIntervalS, defaultVolumeCacheUpdateIntervalS)
+	c.fixClientConfig()
+	c.fixDataBaseConfig()
+	c.fixKafkaConfig()
+	c.fixBalanceConfig()
+	c.fixDiskDropConfig()
+	c.fixRepairConfig()
+	c.fixInspectConfig()
+	c.fixShardRepairConfig()
+	c.fixBlobDeleteConfig()
+	c.fixArchiveStoreConfig()
+	c.fixRegisterConfig()
+	return nil
+}
+
+func (c *Config) fixClientConfig() {
+	defaulter.LessOrEqual(&c.Proxy.ClientTimeoutMs, defaultClientTimeoutMs)
+	defaulter.LessOrEqual(&c.Proxy.HostSyncIntervalMs, defaultHostSyncIntervalMs)
+	defaulter.LessOrEqual(&c.Proxy.RetryHostsCnt, defaultRetryHostsCnt)
+	defaulter.LessOrEqual(&c.Blobnode.ClientTimeoutMs, defaultClientTimeoutMs)
+	defaulter.LessOrEqual(&c.Scheduler.ClientTimeoutMs, defaultClientTimeoutMs)
+}
+
+func (c *Config) fixDataBaseConfig() {
+	if c.Database.Mongo.WriteConcern == nil {
+		c.Database.Mongo.WriteConcern = &defaultWriteConfig
+	}
+	defaulter.LessOrEqual(&c.Database.Mongo.TimeoutMs, defaultMongoTimeoutMs)
+	defaulter.Empty(&c.Database.DBName, defaultDatabase)
+	defaulter.Empty(&c.Database.BalanceTable, defaultBalanceTable)
+	defaulter.Empty(&c.Database.DiskDropTable, defaultDiskDropTable)
+	defaulter.Empty(&c.Database.RepairTable, defaultRepairTable)
+	defaulter.Empty(&c.Database.InspectCheckPointTable, defaultInspectCheckPointTable)
+	defaulter.Empty(&c.Database.ManualMigrateTable, defaultManualMigrateTable)
+	defaulter.Empty(&c.Database.ArchiveTasksTable, defaultArchiveTasksTable)
+	defaulter.Empty(&c.Database.KafkaOffsetTable, defaultKafkaOffsetTable)
+	defaulter.Empty(&c.Database.OrphanShardTable, defaultOrphanedShardTable)
+}
+
+func (c *Config) fixKafkaConfig() {
+	defaulter.Empty(&c.Kafka.BlobDelete.Normal.Topic, defaultBlobDeleteNormalTopic)
+	defaulter.Empty(&c.Kafka.BlobDelete.Failed.Topic, defaultBlobDeleteFailedTopic)
+	defaulter.Empty(&c.Kafka.ShardRepair.Normal.Topic, defaultShardRepairNormalTopic)
+	defaulter.Empty(&c.Kafka.ShardRepair.Priority.Topic, defaultShardRepairPriorityTopic)
+	defaulter.Empty(&c.Kafka.ShardRepair.Failed.Topic, defaultShardRepairFailedTopic)
+	defaulter.LessOrEqual(&c.Kafka.FailMsgSenderTimeoutMs, defaultClientTimeoutMs)
+	c.BlobDelete.Kafka.FailMsgSenderTimeoutMs = c.Kafka.FailMsgSenderTimeoutMs
+	c.ShardRepair.Kafka.FailMsgSenderTimeoutMs = c.Kafka.FailMsgSenderTimeoutMs
+	c.Kafka.ShardRepair.BrokerList = c.Kafka.BrokerList
+	c.Kafka.BlobDelete.BrokerList = c.Kafka.BrokerList
+}
+
+func (c *Config) fixRepairConfig() {
+	c.DiskRepair.ClusterID = c.ClusterID
+	c.DiskRepair.CheckAndFix()
+}
+
+func (c *Config) fixBalanceConfig() {
+	c.Balance.ClusterID = c.ClusterID
+	defaulter.LessOrEqual(&c.Balance.BalanceDiskCntLimit, defaultBalanceDiskCntLimit)
+	defaulter.LessOrEqual(&c.Balance.MaxDiskFreeChunkCnt, defaultMaxDiskFreeChunkCnt)
+	defaulter.LessOrEqual(&c.Balance.MinDiskFreeChunkCnt, defaultMinDiskFreeChunkCnt)
+	c.Balance.CheckAndFix()
+}
+
+func (c *Config) fixDiskDropConfig() {
+	c.DiskDrop.ClusterID = c.ClusterID
+	c.DiskDrop.CheckAndFix()
+}
+
+func (c *Config) fixInspectConfig() {
+	defaulter.LessOrEqual(&c.VolumeInspect.TimeoutMs, defaultInspectTimeoutMs)
+	defaulter.LessOrEqual(&c.VolumeInspect.ListVolStep, defaultListVolStep)
+	defaulter.LessOrEqual(&c.VolumeInspect.ListVolIntervalMs, defaultListVolIntervalMs)
+	defaulter.LessOrEqual(&c.VolumeInspect.InspectBatch, defaultInspectBatch)
+	if c.VolumeInspect.InspectBatch < c.VolumeInspect.ListVolStep {
+		c.VolumeInspect.InspectBatch = c.VolumeInspect.ListVolStep
+	}
+	defaulter.LessOrEqual(&c.VolumeInspect.InspectIntervalS, defaultInspectIntervalS)
+}
+
+func (c *Config) fixShardRepairConfig() {
+	c.ShardRepair.ClusterID = c.ClusterID
+	defaulter.LessOrEqual(&c.ShardRepair.TaskPoolSize, defaultTaskPoolSize)
+	defaulter.LessOrEqual(&c.ShardRepair.NormalHandleBatchCnt, defaultHandleBatchCnt)
+	defaulter.LessOrEqual(&c.ShardRepair.FailHandleBatchCnt, defaultHandleBatchCnt)
+	defaulter.LessOrEqual(&c.ShardRepair.FailMsgConsumeIntervalMs, defaultFailMsgConsumeIntervalMs)
+	c.ShardRepair.Kafka = c.Kafka.ShardRepair
+}
+
+func (c *Config) fixBlobDeleteConfig() {
+	c.BlobDelete.ClusterID = c.ClusterID
+	defaulter.LessOrEqual(&c.BlobDelete.TaskPoolSize, defaultTaskPoolSize)
+	defaulter.LessOrEqual(&c.BlobDelete.NormalHandleBatchCnt, defaultHandleBatchCnt)
+	defaulter.LessOrEqual(&c.BlobDelete.FailHandleBatchCnt, defaultHandleBatchCnt)
+	defaulter.LessOrEqual(&c.BlobDelete.FailMsgConsumeIntervalMs, defaultFailMsgConsumeIntervalMs)
+	defaulter.LessOrEqual(&c.BlobDelete.DeleteLog.ChunkBits, defaultDeleteLogChunkSize)
+	defaulter.Equal(&c.BlobDelete.SafeDelayTimeH, defaultDeleteDelayH)
+	defaulter.Less(&c.BlobDelete.SafeDelayTimeH, defaultDeleteNoDelay)
+	c.BlobDelete.Kafka = c.Kafka.BlobDelete
+}
+
+func (c *Config) fixArchiveStoreConfig() {
+	defaulter.LessOrEqual(&c.Archive.ArchiveDelayMin, defaultArchiveDelayMin)
+	defaulter.LessOrEqual(&c.Archive.ArchiveIntervalMin, defaultArchiveIntervalMin)
+}
+
+func (c *Config) fixRegisterConfig() {
+	defaulter.LessOrEqual(&c.ServiceRegister.TickInterval, defaultTickInterval)
+	defaulter.LessOrEqual(&c.ServiceRegister.HeartbeatTicks, defaultHeartbeatTicks)
+	defaulter.LessOrEqual(&c.ServiceRegister.ExpiresTicks, defaultExpiresTicks)
+}

blobstore/scheduler/config_test.go

@@ -0,0 +1,55 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package scheduler
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestConfigCheckAndFix(t *testing.T) {
+	cfg := &Config{}
+	err := cfg.fixConfig()
+	require.Error(t, err, errIllegalClusterID.Error())
+
+	cfg.ClusterID = 1
+	err = cfg.fixConfig()
+	require.Error(t, err, errInvalidMembers)
+
+	cfg.Services.Members = map[uint64]string{1: "127.0.0.1:9800"}
+	err = cfg.fixConfig()
+	require.Error(t, err, errInvalidLeader)
+
+	cfg.Services.Leader = 1
+	err = cfg.fixConfig()
+	require.Error(t, err, errInvalidNodeID)
+
+	cfg.Services.NodeID = 1
+	err = cfg.fixConfig()
+	require.NoError(t, err)
+	require.True(t, cfg.IsLeader())
+	require.Equal(t, "127.0.0.1:9800", cfg.Leader())
+	require.Nil(t, cfg.Follower())
+	require.Equal(t, defaultDeleteDelayH, cfg.BlobDelete.SafeDelayTimeH)
+	cfg.Services.Members[2] = "127.0.0.1:9880"
+	require.Equal(t, "127.0.0.1:9880", cfg.Follower()[0])
+
+	cfg.Services.NodeID = 1
+	cfg.BlobDelete.SafeDelayTimeH = -1
+	err = cfg.fixConfig()
+	require.NoError(t, err)
+	require.Equal(t, defaultDeleteNoDelay, cfg.BlobDelete.SafeDelayTimeH)
+}

blobstore/scheduler/db/database.go

@@ -0,0 +1,127 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package db
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/x/bsonx"
+
+	"github.com/cubefs/cubefs/blobstore/common/mongoutil"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+const (
+	// deleteMark for mark delete
+	deleteMark = "delete_mark"
+)
+
+// Config mongo config
+type Config struct {
+	Mongo  mongoutil.Config `json:"mongo"`
+	DBName string           `json:"db_name"`
+
+	BalanceTable           string `json:"balance_table"`
+	DiskDropTable          string `json:"disk_drop_table"`
+	ManualMigrateTable     string `json:"manual_migrate_table"`
+	RepairTable            string `json:"repair_table"`
+	InspectCheckPointTable string `json:"inspect_checkpoint_table"`
+	OrphanShardTable       string `json:"orphaned_shard_table"`
+	KafkaOffsetTable       string `json:"kafka_offset_table"`
+
+	ArchiveTasksTable string `json:"archive_tasks_table"`
+}
+
+// Database used for database operate
+type Database struct {
+	DB *mongo.Database
+
+	BalanceTable       IMigrateTaskTable
+	DiskDropTable      IMigrateTaskTable
+	ManualMigrateTable IMigrateTaskTable
+	RepairTaskTable    IRepairTaskTable
+
+	KafkaOffsetTable IKafkaOffsetTable
+	OrphanShardTable IOrphanShardTable
+
+	InspectCheckPointTable IInspectCheckPointTable
+
+	ArchiveTable IArchiveTable
+}
+
+// OpenDatabase open database
+func OpenDatabase(conf *Config) (tables *Database, err error) {
+	client, err := mongoutil.GetClient(conf.Mongo)
+	if err != nil {
+		return nil, err
+	}
+	db := client.Database(conf.DBName)
+	tables = &Database{DB: db}
+
+	if tables.BalanceTable, err = openMigrateTbl(
+		mustCreateCollection(db, conf.BalanceTable),
+		proto.BalanceTaskType); err != nil {
+		return nil, err
+	}
+	if tables.DiskDropTable, err = openMigrateTbl(
+		mustCreateCollection(db, conf.DiskDropTable),
+		proto.DiskDropTaskType); err != nil {
+		return nil, err
+	}
+	if tables.ManualMigrateTable, err = openMigrateTbl(
+		mustCreateCollection(db, conf.ManualMigrateTable),
+		proto.ManualMigrateType); err != nil {
+		return nil, err
+	}
+	if tables.RepairTaskTable, err = OpenRepairTaskTbl(
+		mustCreateCollection(db, conf.RepairTable),
+		proto.RepairTaskType); err != nil {
+		return nil, err
+	}
+	if tables.InspectCheckPointTable, err = OpenInspectCheckPointTbl(
+		mustCreateCollection(db, conf.InspectCheckPointTable)); err != nil {
+		return nil, err
+	}
+	if tables.ArchiveTable, err = openArchiveTbl(
+		mustCreateCollection(db, conf.ArchiveTasksTable)); err != nil {
+		return nil, err
+	}
+	tables.KafkaOffsetTable = openKafkaOffsetTable(mustCreateCollection(db, conf.KafkaOffsetTable))
+	tables.OrphanShardTable = openOrphanedShardTable(mustCreateCollection(db, conf.OrphanShardTable))
+
+	return
+}
+
+func mustCreateCollection(db *mongo.Database, collName string) *mongo.Collection {
+	err := db.RunCommand(context.Background(), bsonx.Doc{{Key: "create", Value: bsonx.String(collName)}}).Err()
+	if err == nil || strings.Contains(err.Error(), "already exists") {
+		return db.Collection(collName)
+	}
+	panic(fmt.Sprintf("create collection error: %v", err))
+}
+
+func deleteBson() bson.M {
+	return bson.M{"$set": bson.M{deleteMark: true, "del_time": time.Now().Unix()}}
+}
+
+func inDelayTime(delTime int64, delayMin int) bool {
+	now := time.Now()
+	return now.Sub(time.Unix(delTime, 0)) <= time.Duration(delayMin)*time.Minute
+}

blobstore/scheduler/db/disk_repair_table.go

@@ -0,0 +1,151 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package db
+
+import (
+	"context"
+	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+)
+
+// IRepairTaskTable define the interface of db used by disk repair
+type IRepairTaskTable interface {
+	Insert(ctx context.Context, t *proto.VolRepairTask) error
+	Update(ctx context.Context, t *proto.VolRepairTask) error
+	Find(ctx context.Context, taskID string) (task *proto.VolRepairTask, err error)
+	FindByDiskID(ctx context.Context, diskID proto.DiskID) (tasks []*proto.VolRepairTask, err error)
+	FindAll(ctx context.Context) (tasks []*proto.VolRepairTask, err error)
+	MarkDeleteByDiskID(ctx context.Context, diskID proto.DiskID) error
+
+	IRecordSrcTbl
+}
+
+// RepairTaskTable disk repair task table
+type RepairTaskTbl struct {
+	coll *mongo.Collection
+	name string
+}
+
+// OpenRepairTaskTbl open disk repair task table
+func OpenRepairTaskTbl(coll *mongo.Collection, name string) (IRepairTaskTable, error) {
+	tbl := &RepairTaskTbl{
+		coll: coll,
+		name: name,
+	}
+	return tbl, nil
+}
+
+// Insert insert task
+func (tbl *RepairTaskTbl) Insert(ctx context.Context, t *proto.VolRepairTask) error {
+	t.Ctime = time.Now().String()
+	t.MTime = t.Ctime
+	trace.SpanFromContextSafe(ctx).Debugf("insert task %+v", t)
+	_, err := tbl.coll.InsertOne(ctx, t)
+	return err
+}
+
+// Update update task
+func (tbl *RepairTaskTbl) Update(ctx context.Context, t *proto.VolRepairTask) error {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Debugf("update repair task tbl task %+v", *t)
+
+	t.MTime = time.Now().String()
+	return tbl.coll.FindOneAndReplace(ctx, bson.M{"_id": t.TaskID}, t).Err()
+}
+
+// Find find task by taskID
+func (tbl *RepairTaskTbl) Find(ctx context.Context, taskID string) (task *proto.VolRepairTask, err error) {
+	err = tbl.coll.FindOne(ctx, bson.M{"_id": taskID, deleteMark: bson.M{"$ne": true}}).Decode(&task)
+	return
+}
+
+// FindByDiskID find task by diskID
+func (tbl *RepairTaskTbl) FindByDiskID(ctx context.Context, diskID proto.DiskID) (tasks []*proto.VolRepairTask, err error) {
+	cursor, err := tbl.coll.Find(ctx, bson.M{"repair_disk_id": diskID, deleteMark: bson.M{"$ne": true}})
+	if err != nil {
+		return nil, err
+	}
+	err = cursor.All(ctx, &tasks)
+	return tasks, err
+}
+
+// FindAll return all tasks
+func (tbl *RepairTaskTbl) FindAll(ctx context.Context) (tasks []*proto.VolRepairTask, err error) {
+	cursor, err := tbl.coll.Find(ctx, bson.M{deleteMark: bson.M{"$ne": true}})
+	if err != nil {
+		return nil, err
+	}
+	err = cursor.All(ctx, &tasks)
+	return tasks, err
+}
+
+// MarkDeleteByDiskID mark delete task by diskID
+func (tbl *RepairTaskTbl) MarkDeleteByDiskID(ctx context.Context, diskID proto.DiskID) error {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Debugf("mark delete by disk_id %d", diskID)
+
+	_, err := tbl.coll.UpdateMany(ctx, bson.M{"repair_disk_id": diskID}, deleteBson())
+	return err
+}
+
+// QueryMarkDeleteTasks find mark delete tasks
+func (tbl *RepairTaskTbl) QueryMarkDeleteTasks(ctx context.Context, delayMin int) (records []*proto.ArchiveRecord, err error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	type VolRepairTaskEx struct {
+		proto.VolRepairTask `bson:",inline"`
+		DelTime             int64 `bson:"del_time"`
+	}
+	var tasks []*VolRepairTaskEx
+	cursor, err := tbl.coll.Find(ctx, bson.M{deleteMark: true})
+	if err != nil {
+		return nil, err
+	}
+	err = cursor.All(ctx, &tasks)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, task := range tasks {
+		if inDelayTime(task.DelTime, delayMin) {
+			span.Debugf("task_id %s is in delay time", task.TaskID)
+			continue
+		}
+
+		r := &proto.ArchiveRecord{
+			TaskID:   task.TaskID,
+			TaskType: tbl.Name(),
+			Content:  task,
+		}
+		records = append(records, r)
+	}
+	return records, nil
+}
+
+// RemoveMarkDelete remove mark delete task by taskID
+func (tbl *RepairTaskTbl) RemoveMarkDelete(ctx context.Context, taskID string) error {
+	_, err := tbl.coll.DeleteOne(ctx, bson.M{"_id": taskID, deleteMark: true})
+	return err
+}
+
+// Name return repair table name
+func (tbl *RepairTaskTbl) Name() string {
+	return tbl.name
+}

blobstore/scheduler/db/inspect_table.go

@@ -0,0 +1,67 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package db
+
+import (
+	"context"
+	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+)
+
+// record the current checkpoint of inspect which save min vid in last batch volumes
+// service will start inspect worker from checkpoint when service start
+const inspectID = "inspect_checkpoint"
+
+// IInspectCheckPointTable define the interface of db used by inspect
+type IInspectCheckPointTable interface {
+	GetCheckPoint(ctx context.Context) (ck *proto.InspectCheckPoint, err error)
+	SaveCheckPoint(ctx context.Context, startVid proto.Vid) error
+}
+
+// InspectCheckPointTable inspect check point table
+type InspectCheckPointTbl struct {
+	coll *mongo.Collection
+}
+
+// OpenInspectCheckPointTbl returns inspect check point table
+func OpenInspectCheckPointTbl(coll *mongo.Collection) (IInspectCheckPointTable, error) {
+	return &InspectCheckPointTbl{
+		coll: coll,
+	}, nil
+}
+
+// GetCheckPoint returns check point
+func (tbl *InspectCheckPointTbl) GetCheckPoint(ctx context.Context) (ck *proto.InspectCheckPoint, err error) {
+	err = tbl.coll.FindOne(ctx, bson.M{}).Decode(&ck)
+	return ck, err
+}
+
+// SaveCheckPoint save check point
+func (tbl *InspectCheckPointTbl) SaveCheckPoint(ctx context.Context, startVid proto.Vid) error {
+	ck := proto.InspectCheckPoint{
+		Id:       inspectID,
+		StartVid: startVid,
+		Ctime:    time.Now().String(),
+	}
+	trace.SpanFromContextSafe(ctx).Infof("save checkpoint %+v", ck)
+	_, err := tbl.coll.ReplaceOne(ctx, bson.M{"_id": inspectID}, ck, options.Replace().SetUpsert(true))
+	return err
+}

blobstore/scheduler/db/kafka_offset_table.go

@@ -0,0 +1,62 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package db
+
+import (
+	"context"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+// IKafkaOffsetTable define interface of kafka offset table use by delete or repair message consume.
+type IKafkaOffsetTable interface {
+	Set(topic string, partition int32, offset int64) error
+	Get(topic string, partition int32) (int64, error)
+}
+
+type kafkaOffset struct {
+	Topic     string `bson:"topic"`
+	Partition int32  `bson:"partition"`
+	Offset    int64  `bson:"offset"`
+}
+
+type kafkaOffsetTable struct {
+	coll *mongo.Collection
+}
+
+func openKafkaOffsetTable(coll *mongo.Collection) IKafkaOffsetTable {
+	return &kafkaOffsetTable{coll: coll}
+}
+
+func (t *kafkaOffsetTable) Set(topic string, partition int32, off int64) error {
+	info := kafkaOffset{Topic: topic, Partition: partition, Offset: off}
+	selector := bson.M{"topic": topic, "partition": partition}
+
+	update := bson.M{
+		"$set": info,
+	}
+	opts := options.Update().SetUpsert(true)
+	_, err := t.coll.UpdateOne(context.Background(), selector, update, opts)
+	return err
+}
+
+func (t *kafkaOffsetTable) Get(topic string, partition int32) (int64, error) {
+	infos := kafkaOffset{}
+	selector := bson.M{"topic": topic, "partition": partition}
+	err := t.coll.FindOne(context.Background(), &selector).Decode(&infos)
+	return infos.Offset, err
+}

blobstore/scheduler/db/migrate_task_table.go

@@ -0,0 +1,197 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package db
+
+import (
+	"context"
+	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"go.mongodb.org/mongo-driver/x/bsonx"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+)
+
+// IMigrateTaskTable define the interface of db use by migrate
+type IMigrateTaskTable interface {
+	Insert(ctx context.Context, task *proto.MigrateTask) error
+	Delete(ctx context.Context, taskID string) error
+	MarkDeleteByDiskID(ctx context.Context, diskID proto.DiskID) error
+	MarkDeleteByStates(ctx context.Context, states []proto.MigrateState) error
+	Update(ctx context.Context, oldState proto.MigrateState, task *proto.MigrateTask) error
+	Find(ctx context.Context, taskID string) (task *proto.MigrateTask, err error)
+	FindByDiskID(ctx context.Context, diskID proto.DiskID) (tasks []*proto.MigrateTask, err error)
+	FindAll(ctx context.Context) (tasks []*proto.MigrateTask, err error)
+
+	IRecordSrcTbl
+}
+
+// MigrateTaskTbl migrate table
+type MigrateTaskTbl struct {
+	coll *mongo.Collection
+	name string
+}
+
+// openMigrateTbl open migrate tables
+func openMigrateTbl(coll *mongo.Collection, name string) (IMigrateTaskTable, error) {
+	tbl := &MigrateTaskTbl{
+		coll: coll,
+		name: name,
+	}
+	err := tbl.createIndex()
+	return tbl, err
+}
+
+func (tbl *MigrateTaskTbl) createIndex() error {
+	ctx := context.Background()
+	opts := options.CreateIndexes().SetMaxTime(10 * time.Second)
+	mods := []mongo.IndexModel{
+		{
+			Keys:    bsonx.Doc{{Key: "state", Value: bsonx.Int32(-1)}},
+			Options: options.Index().SetName("_state_").SetBackground(true),
+		},
+		{
+			Keys:    bsonx.Doc{{Key: deleteMark, Value: bsonx.Int32(-1)}},
+			Options: options.Index().SetName("_delete_mark_").SetBackground(true),
+		},
+		{
+			Keys:    bsonx.Doc{{Key: "source_disk_id", Value: bsonx.Int32(-1)}},
+			Options: options.Index().SetName("_source_disk_id_").SetBackground(true),
+		},
+	}
+
+	_, err := tbl.coll.Indexes().CreateMany(ctx, mods, opts)
+	return err
+}
+
+// Insert insert task to db
+func (tbl *MigrateTaskTbl) Insert(ctx context.Context, task *proto.MigrateTask) error {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Debugf("DB:insert task, taskId: %s, %+v", task.TaskID, task)
+
+	task.Ctime = time.Now().String()
+	task.MTime = task.Ctime
+	_, err := tbl.coll.InsertOne(ctx, task)
+	return err
+}
+
+// Update update task
+func (tbl *MigrateTaskTbl) Update(ctx context.Context, oldState proto.MigrateState, task *proto.MigrateTask) error {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Debugf("DB:update task, taskId: %s,state: %d", task.TaskID, task.State)
+
+	task.MTime = time.Now().String()
+
+	states := []proto.MigrateState{oldState, task.State}
+	return tbl.coll.FindOneAndReplace(ctx, bson.M{"_id": task.TaskID, "state": bson.M{"$in": states}}, task).Err()
+}
+
+// Delete delete task
+func (tbl *MigrateTaskTbl) Delete(ctx context.Context, taskID string) error {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Debugf("DB:delete task, taskID: %s", taskID)
+
+	_, err := tbl.coll.UpdateOne(ctx, bson.M{"_id": taskID}, deleteBson())
+	return err
+}
+
+// MarkDeleteByDiskID mark delete task by diskID
+func (tbl *MigrateTaskTbl) MarkDeleteByDiskID(ctx context.Context, diskID proto.DiskID) error {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Debugf("delete db task by diskID %d", diskID)
+
+	_, err := tbl.coll.UpdateMany(ctx, bson.M{"source_disk_id": diskID}, deleteBson())
+	return err
+}
+
+// MarkDeleteByStates mark delete task by status
+func (tbl *MigrateTaskTbl) MarkDeleteByStates(ctx context.Context, states []proto.MigrateState) error {
+	_, err := tbl.coll.UpdateMany(ctx, bson.M{"state": bson.M{"$in": states}, deleteMark: bson.M{"$ne": true}}, deleteBson())
+	return err
+}
+
+// FindAll returns all un mark delete task
+func (tbl *MigrateTaskTbl) FindAll(ctx context.Context) (tasks []*proto.MigrateTask, err error) {
+	cursor, err := tbl.coll.Find(ctx, bson.M{deleteMark: bson.M{"$ne": true}})
+	if err != nil {
+		return nil, err
+	}
+	err = cursor.All(ctx, &tasks)
+	return tasks, err
+}
+
+// Find find task by taskID
+func (tbl *MigrateTaskTbl) Find(ctx context.Context, taskID string) (task *proto.MigrateTask, err error) {
+	err = tbl.coll.FindOne(ctx, bson.M{"_id": taskID, deleteMark: bson.M{"$ne": true}}).Decode(&task)
+	return
+}
+
+// FindByDiskID find task by diskID
+func (tbl *MigrateTaskTbl) FindByDiskID(ctx context.Context, diskID proto.DiskID) (tasks []*proto.MigrateTask, err error) {
+	cursor, err := tbl.coll.Find(ctx, bson.M{"source_disk_id": diskID, deleteMark: bson.M{"$ne": true}})
+	if err != nil {
+		return nil, err
+	}
+	err = cursor.All(ctx, &tasks)
+	return tasks, err
+}
+
+// QueryMarkDeleteTasks find mark delete task for archive
+func (tbl *MigrateTaskTbl) QueryMarkDeleteTasks(ctx context.Context, delayMin int) (records []*proto.ArchiveRecord, err error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	type MigrateTaskEx struct {
+		proto.MigrateTask `bson:",inline"`
+		DelTime           int64 `bson:"del_time"`
+	}
+	var tasks []*MigrateTaskEx
+	cursor, err := tbl.coll.Find(ctx, bson.M{deleteMark: true})
+	if err != nil {
+		return nil, err
+	}
+	err = cursor.All(ctx, &tasks)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, task := range tasks {
+		if inDelayTime(task.DelTime, delayMin) {
+			span.Debugf("task_id %s is in delay time", task.TaskID)
+			continue
+		}
+
+		r := &proto.ArchiveRecord{
+			TaskID:   task.TaskID,
+			TaskType: tbl.Name(),
+			Content:  task,
+		}
+		records = append(records, r)
+	}
+	return records, nil
+}
+
+// RemoveMarkDelete remove mark delete task
+func (tbl *MigrateTaskTbl) RemoveMarkDelete(ctx context.Context, taskID string) error {
+	_, err := tbl.coll.DeleteOne(ctx, bson.M{"_id": taskID, deleteMark: true})
+	return err
+}
+
+// Name return table name
+func (tbl *MigrateTaskTbl) Name() string {
+	return tbl.name
+}

blobstore/scheduler/db/orphan_shard_table.go

@@ -0,0 +1,48 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package db
+
+import (
+	"context"
+
+	"go.mongodb.org/mongo-driver/mongo"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+// IOrphanShardTable define the interface to save orphan shard record.
+type IOrphanShardTable interface {
+	Save(shard OrphanShard) error
+}
+
+// OrphanShard orphan shard identification.
+type OrphanShard struct {
+	ClusterID proto.ClusterID `bson:"cluster_id"`
+	Vid       proto.Vid       `bson:"vid"`
+	Bid       proto.BlobID    `bson:"bid"`
+}
+
+type orphanShardTable struct {
+	coll *mongo.Collection
+}
+
+func openOrphanedShardTable(coll *mongo.Collection) IOrphanShardTable {
+	return &orphanShardTable{coll: coll}
+}
+
+func (t *orphanShardTable) Save(shard OrphanShard) error {
+	_, err := t.coll.InsertOne(context.Background(), shard)
+	return err
+}

blobstore/scheduler/db/task_archive_table.go

@@ -0,0 +1,67 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package db
+
+import (
+	"context"
+	"time"
+
+	"github.com/globalsign/mgo/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+)
+
+//duties:transfer the deleted records to the archive table
+
+// IArchiveTable define the interface of db use by archive
+type IArchiveTable interface {
+	Insert(ctx context.Context, record *proto.ArchiveRecord) error
+	FindTask(ctx context.Context, taskID string) (record *proto.ArchiveRecord, err error)
+}
+
+// IRecordSrcTbl define the interface of source record table used by archive
+type IRecordSrcTbl interface {
+	QueryMarkDeleteTasks(ctx context.Context, delayMin int) (records []*proto.ArchiveRecord, err error)
+	RemoveMarkDelete(ctx context.Context, taskID string) error
+	Name() string
+}
+
+type archiveTbl struct {
+	coll *mongo.Collection
+}
+
+func openArchiveTbl(coll *mongo.Collection) (IArchiveTable, error) {
+	return &archiveTbl{
+		coll: coll,
+	}, nil
+}
+
+// Insert insert record
+func (tbl *archiveTbl) Insert(ctx context.Context, record *proto.ArchiveRecord) error {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Debugf("archiveTbl:insert task %s", record.TaskID)
+
+	record.ArchiveTime = time.Now().String()
+	_, err := tbl.coll.InsertOne(ctx, record)
+	return err
+}
+
+// FindTask find task by taskID
+func (tbl *archiveTbl) FindTask(ctx context.Context, taskID string) (record *proto.ArchiveRecord, err error) {
+	err = tbl.coll.FindOne(ctx, bson.M{"_id": taskID}).Decode(&record)
+	return
+}

blobstore/scheduler/db_mock_test.go

@@ -0,0 +1,541 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/cubefs/cubefs/blobstore/scheduler/db (interfaces: IKafkaOffsetTable,IOrphanShardTable,IArchiveTable,IMigrateTaskTable,IRepairTaskTable,IInspectCheckPointTable)
+
+// Package scheduler is a generated GoMock package.
+package scheduler
+
+import (
+	context "context"
+	reflect "reflect"
+
+	proto "github.com/cubefs/cubefs/blobstore/common/proto"
+	db "github.com/cubefs/cubefs/blobstore/scheduler/db"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockKafkaOffsetTable is a mock of IKafkaOffsetTable interface.
+type MockKafkaOffsetTable struct {
+	ctrl     *gomock.Controller
+	recorder *MockKafkaOffsetTableMockRecorder
+}
+
+// MockKafkaOffsetTableMockRecorder is the mock recorder for MockKafkaOffsetTable.
+type MockKafkaOffsetTableMockRecorder struct {
+	mock *MockKafkaOffsetTable
+}
+
+// NewMockKafkaOffsetTable creates a new mock instance.
+func NewMockKafkaOffsetTable(ctrl *gomock.Controller) *MockKafkaOffsetTable {
+	mock := &MockKafkaOffsetTable{ctrl: ctrl}
+	mock.recorder = &MockKafkaOffsetTableMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockKafkaOffsetTable) EXPECT() *MockKafkaOffsetTableMockRecorder {
+	return m.recorder
+}
+
+// Get mocks base method.
+func (m *MockKafkaOffsetTable) Get(arg0 string, arg1 int32) (int64, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Get", arg0, arg1)
+	ret0, _ := ret[0].(int64)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Get indicates an expected call of Get.
+func (mr *MockKafkaOffsetTableMockRecorder) Get(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockKafkaOffsetTable)(nil).Get), arg0, arg1)
+}
+
+// Set mocks base method.
+func (m *MockKafkaOffsetTable) Set(arg0 string, arg1 int32, arg2 int64) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Set", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Set indicates an expected call of Set.
+func (mr *MockKafkaOffsetTableMockRecorder) Set(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockKafkaOffsetTable)(nil).Set), arg0, arg1, arg2)
+}
+
+// MockOrphanShardTable is a mock of IOrphanShardTable interface.
+type MockOrphanShardTable struct {
+	ctrl     *gomock.Controller
+	recorder *MockOrphanShardTableMockRecorder
+}
+
+// MockOrphanShardTableMockRecorder is the mock recorder for MockOrphanShardTable.
+type MockOrphanShardTableMockRecorder struct {
+	mock *MockOrphanShardTable
+}
+
+// NewMockOrphanShardTable creates a new mock instance.
+func NewMockOrphanShardTable(ctrl *gomock.Controller) *MockOrphanShardTable {
+	mock := &MockOrphanShardTable{ctrl: ctrl}
+	mock.recorder = &MockOrphanShardTableMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockOrphanShardTable) EXPECT() *MockOrphanShardTableMockRecorder {
+	return m.recorder
+}
+
+// Save mocks base method.
+func (m *MockOrphanShardTable) Save(arg0 db.OrphanShard) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Save", arg0)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Save indicates an expected call of Save.
+func (mr *MockOrphanShardTableMockRecorder) Save(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Save", reflect.TypeOf((*MockOrphanShardTable)(nil).Save), arg0)
+}
+
+// MockArchiveTable is a mock of IArchiveTable interface.
+type MockArchiveTable struct {
+	ctrl     *gomock.Controller
+	recorder *MockArchiveTableMockRecorder
+}
+
+// MockArchiveTableMockRecorder is the mock recorder for MockArchiveTable.
+type MockArchiveTableMockRecorder struct {
+	mock *MockArchiveTable
+}
+
+// NewMockArchiveTable creates a new mock instance.
+func NewMockArchiveTable(ctrl *gomock.Controller) *MockArchiveTable {
+	mock := &MockArchiveTable{ctrl: ctrl}
+	mock.recorder = &MockArchiveTableMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockArchiveTable) EXPECT() *MockArchiveTableMockRecorder {
+	return m.recorder
+}
+
+// FindTask mocks base method.
+func (m *MockArchiveTable) FindTask(arg0 context.Context, arg1 string) (*proto.ArchiveRecord, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "FindTask", arg0, arg1)
+	ret0, _ := ret[0].(*proto.ArchiveRecord)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// FindTask indicates an expected call of FindTask.
+func (mr *MockArchiveTableMockRecorder) FindTask(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindTask", reflect.TypeOf((*MockArchiveTable)(nil).FindTask), arg0, arg1)
+}
+
+// Insert mocks base method.
+func (m *MockArchiveTable) Insert(arg0 context.Context, arg1 *proto.ArchiveRecord) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Insert", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Insert indicates an expected call of Insert.
+func (mr *MockArchiveTableMockRecorder) Insert(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Insert", reflect.TypeOf((*MockArchiveTable)(nil).Insert), arg0, arg1)
+}
+
+// MockMigrateTaskTable is a mock of IMigrateTaskTable interface.
+type MockMigrateTaskTable struct {
+	ctrl     *gomock.Controller
+	recorder *MockMigrateTaskTableMockRecorder
+}
+
+// MockMigrateTaskTableMockRecorder is the mock recorder for MockMigrateTaskTable.
+type MockMigrateTaskTableMockRecorder struct {
+	mock *MockMigrateTaskTable
+}
+
+// NewMockMigrateTaskTable creates a new mock instance.
+func NewMockMigrateTaskTable(ctrl *gomock.Controller) *MockMigrateTaskTable {
+	mock := &MockMigrateTaskTable{ctrl: ctrl}
+	mock.recorder = &MockMigrateTaskTableMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockMigrateTaskTable) EXPECT() *MockMigrateTaskTableMockRecorder {
+	return m.recorder
+}
+
+// Delete mocks base method.
+func (m *MockMigrateTaskTable) Delete(arg0 context.Context, arg1 string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Delete", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Delete indicates an expected call of Delete.
+func (mr *MockMigrateTaskTableMockRecorder) Delete(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockMigrateTaskTable)(nil).Delete), arg0, arg1)
+}
+
+// Find mocks base method.
+func (m *MockMigrateTaskTable) Find(arg0 context.Context, arg1 string) (*proto.MigrateTask, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Find", arg0, arg1)
+	ret0, _ := ret[0].(*proto.MigrateTask)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Find indicates an expected call of Find.
+func (mr *MockMigrateTaskTableMockRecorder) Find(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Find", reflect.TypeOf((*MockMigrateTaskTable)(nil).Find), arg0, arg1)
+}
+
+// FindAll mocks base method.
+func (m *MockMigrateTaskTable) FindAll(arg0 context.Context) ([]*proto.MigrateTask, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "FindAll", arg0)
+	ret0, _ := ret[0].([]*proto.MigrateTask)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// FindAll indicates an expected call of FindAll.
+func (mr *MockMigrateTaskTableMockRecorder) FindAll(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindAll", reflect.TypeOf((*MockMigrateTaskTable)(nil).FindAll), arg0)
+}
+
+// FindByDiskID mocks base method.
+func (m *MockMigrateTaskTable) FindByDiskID(arg0 context.Context, arg1 proto.DiskID) ([]*proto.MigrateTask, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "FindByDiskID", arg0, arg1)
+	ret0, _ := ret[0].([]*proto.MigrateTask)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// FindByDiskID indicates an expected call of FindByDiskID.
+func (mr *MockMigrateTaskTableMockRecorder) FindByDiskID(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindByDiskID", reflect.TypeOf((*MockMigrateTaskTable)(nil).FindByDiskID), arg0, arg1)
+}
+
+// Insert mocks base method.
+func (m *MockMigrateTaskTable) Insert(arg0 context.Context, arg1 *proto.MigrateTask) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Insert", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Insert indicates an expected call of Insert.
+func (mr *MockMigrateTaskTableMockRecorder) Insert(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Insert", reflect.TypeOf((*MockMigrateTaskTable)(nil).Insert), arg0, arg1)
+}
+
+// MarkDeleteByDiskID mocks base method.
+func (m *MockMigrateTaskTable) MarkDeleteByDiskID(arg0 context.Context, arg1 proto.DiskID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "MarkDeleteByDiskID", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// MarkDeleteByDiskID indicates an expected call of MarkDeleteByDiskID.
+func (mr *MockMigrateTaskTableMockRecorder) MarkDeleteByDiskID(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkDeleteByDiskID", reflect.TypeOf((*MockMigrateTaskTable)(nil).MarkDeleteByDiskID), arg0, arg1)
+}
+
+// MarkDeleteByStates mocks base method.
+func (m *MockMigrateTaskTable) MarkDeleteByStates(arg0 context.Context, arg1 []proto.MigrateState) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "MarkDeleteByStates", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// MarkDeleteByStates indicates an expected call of MarkDeleteByStates.
+func (mr *MockMigrateTaskTableMockRecorder) MarkDeleteByStates(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkDeleteByStates", reflect.TypeOf((*MockMigrateTaskTable)(nil).MarkDeleteByStates), arg0, arg1)
+}
+
+// Name mocks base method.
+func (m *MockMigrateTaskTable) Name() string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Name")
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// Name indicates an expected call of Name.
+func (mr *MockMigrateTaskTableMockRecorder) Name() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockMigrateTaskTable)(nil).Name))
+}
+
+// QueryMarkDeleteTasks mocks base method.
+func (m *MockMigrateTaskTable) QueryMarkDeleteTasks(arg0 context.Context, arg1 int) ([]*proto.ArchiveRecord, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "QueryMarkDeleteTasks", arg0, arg1)
+	ret0, _ := ret[0].([]*proto.ArchiveRecord)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// QueryMarkDeleteTasks indicates an expected call of QueryMarkDeleteTasks.
+func (mr *MockMigrateTaskTableMockRecorder) QueryMarkDeleteTasks(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueryMarkDeleteTasks", reflect.TypeOf((*MockMigrateTaskTable)(nil).QueryMarkDeleteTasks), arg0, arg1)
+}
+
+// RemoveMarkDelete mocks base method.
+func (m *MockMigrateTaskTable) RemoveMarkDelete(arg0 context.Context, arg1 string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "RemoveMarkDelete", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// RemoveMarkDelete indicates an expected call of RemoveMarkDelete.
+func (mr *MockMigrateTaskTableMockRecorder) RemoveMarkDelete(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveMarkDelete", reflect.TypeOf((*MockMigrateTaskTable)(nil).RemoveMarkDelete), arg0, arg1)
+}
+
+// Update mocks base method.
+func (m *MockMigrateTaskTable) Update(arg0 context.Context, arg1 proto.MigrateState, arg2 *proto.MigrateTask) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Update", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Update indicates an expected call of Update.
+func (mr *MockMigrateTaskTableMockRecorder) Update(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockMigrateTaskTable)(nil).Update), arg0, arg1, arg2)
+}
+
+// MockRepairTaskTable is a mock of IRepairTaskTable interface.
+type MockRepairTaskTable struct {
+	ctrl     *gomock.Controller
+	recorder *MockRepairTaskTableMockRecorder
+}
+
+// MockRepairTaskTableMockRecorder is the mock recorder for MockRepairTaskTable.
+type MockRepairTaskTableMockRecorder struct {
+	mock *MockRepairTaskTable
+}
+
+// NewMockRepairTaskTable creates a new mock instance.
+func NewMockRepairTaskTable(ctrl *gomock.Controller) *MockRepairTaskTable {
+	mock := &MockRepairTaskTable{ctrl: ctrl}
+	mock.recorder = &MockRepairTaskTableMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockRepairTaskTable) EXPECT() *MockRepairTaskTableMockRecorder {
+	return m.recorder
+}
+
+// Find mocks base method.
+func (m *MockRepairTaskTable) Find(arg0 context.Context, arg1 string) (*proto.VolRepairTask, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Find", arg0, arg1)
+	ret0, _ := ret[0].(*proto.VolRepairTask)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Find indicates an expected call of Find.
+func (mr *MockRepairTaskTableMockRecorder) Find(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Find", reflect.TypeOf((*MockRepairTaskTable)(nil).Find), arg0, arg1)
+}
+
+// FindAll mocks base method.
+func (m *MockRepairTaskTable) FindAll(arg0 context.Context) ([]*proto.VolRepairTask, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "FindAll", arg0)
+	ret0, _ := ret[0].([]*proto.VolRepairTask)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// FindAll indicates an expected call of FindAll.
+func (mr *MockRepairTaskTableMockRecorder) FindAll(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindAll", reflect.TypeOf((*MockRepairTaskTable)(nil).FindAll), arg0)
+}
+
+// FindByDiskID mocks base method.
+func (m *MockRepairTaskTable) FindByDiskID(arg0 context.Context, arg1 proto.DiskID) ([]*proto.VolRepairTask, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "FindByDiskID", arg0, arg1)
+	ret0, _ := ret[0].([]*proto.VolRepairTask)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// FindByDiskID indicates an expected call of FindByDiskID.
+func (mr *MockRepairTaskTableMockRecorder) FindByDiskID(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindByDiskID", reflect.TypeOf((*MockRepairTaskTable)(nil).FindByDiskID), arg0, arg1)
+}
+
+// Insert mocks base method.
+func (m *MockRepairTaskTable) Insert(arg0 context.Context, arg1 *proto.VolRepairTask) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Insert", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Insert indicates an expected call of Insert.
+func (mr *MockRepairTaskTableMockRecorder) Insert(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Insert", reflect.TypeOf((*MockRepairTaskTable)(nil).Insert), arg0, arg1)
+}
+
+// MarkDeleteByDiskID mocks base method.
+func (m *MockRepairTaskTable) MarkDeleteByDiskID(arg0 context.Context, arg1 proto.DiskID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "MarkDeleteByDiskID", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// MarkDeleteByDiskID indicates an expected call of MarkDeleteByDiskID.
+func (mr *MockRepairTaskTableMockRecorder) MarkDeleteByDiskID(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkDeleteByDiskID", reflect.TypeOf((*MockRepairTaskTable)(nil).MarkDeleteByDiskID), arg0, arg1)
+}
+
+// Name mocks base method.
+func (m *MockRepairTaskTable) Name() string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Name")
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// Name indicates an expected call of Name.
+func (mr *MockRepairTaskTableMockRecorder) Name() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockRepairTaskTable)(nil).Name))
+}
+
+// QueryMarkDeleteTasks mocks base method.
+func (m *MockRepairTaskTable) QueryMarkDeleteTasks(arg0 context.Context, arg1 int) ([]*proto.ArchiveRecord, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "QueryMarkDeleteTasks", arg0, arg1)
+	ret0, _ := ret[0].([]*proto.ArchiveRecord)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// QueryMarkDeleteTasks indicates an expected call of QueryMarkDeleteTasks.
+func (mr *MockRepairTaskTableMockRecorder) QueryMarkDeleteTasks(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueryMarkDeleteTasks", reflect.TypeOf((*MockRepairTaskTable)(nil).QueryMarkDeleteTasks), arg0, arg1)
+}
+
+// RemoveMarkDelete mocks base method.
+func (m *MockRepairTaskTable) RemoveMarkDelete(arg0 context.Context, arg1 string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "RemoveMarkDelete", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// RemoveMarkDelete indicates an expected call of RemoveMarkDelete.
+func (mr *MockRepairTaskTableMockRecorder) RemoveMarkDelete(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveMarkDelete", reflect.TypeOf((*MockRepairTaskTable)(nil).RemoveMarkDelete), arg0, arg1)
+}
+
+// Update mocks base method.
+func (m *MockRepairTaskTable) Update(arg0 context.Context, arg1 *proto.VolRepairTask) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Update", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Update indicates an expected call of Update.
+func (mr *MockRepairTaskTableMockRecorder) Update(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockRepairTaskTable)(nil).Update), arg0, arg1)
+}
+
+// MockInspectCheckPointTable is a mock of IInspectCheckPointTable interface.
+type MockInspectCheckPointTable struct {
+	ctrl     *gomock.Controller
+	recorder *MockInspectCheckPointTableMockRecorder
+}
+
+// MockInspectCheckPointTableMockRecorder is the mock recorder for MockInspectCheckPointTable.
+type MockInspectCheckPointTableMockRecorder struct {
+	mock *MockInspectCheckPointTable
+}
+
+// NewMockInspectCheckPointTable creates a new mock instance.
+func NewMockInspectCheckPointTable(ctrl *gomock.Controller) *MockInspectCheckPointTable {
+	mock := &MockInspectCheckPointTable{ctrl: ctrl}
+	mock.recorder = &MockInspectCheckPointTableMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockInspectCheckPointTable) EXPECT() *MockInspectCheckPointTableMockRecorder {
+	return m.recorder
+}
+
+// GetCheckPoint mocks base method.
+func (m *MockInspectCheckPointTable) GetCheckPoint(arg0 context.Context) (*proto.InspectCheckPoint, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetCheckPoint", arg0)
+	ret0, _ := ret[0].(*proto.InspectCheckPoint)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetCheckPoint indicates an expected call of GetCheckPoint.
+func (mr *MockInspectCheckPointTableMockRecorder) GetCheckPoint(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCheckPoint", reflect.TypeOf((*MockInspectCheckPointTable)(nil).GetCheckPoint), arg0)
+}
+
+// SaveCheckPoint mocks base method.
+func (m *MockInspectCheckPointTable) SaveCheckPoint(arg0 context.Context, arg1 proto.Vid) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "SaveCheckPoint", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// SaveCheckPoint indicates an expected call of SaveCheckPoint.
+func (mr *MockInspectCheckPointTableMockRecorder) SaveCheckPoint(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveCheckPoint", reflect.TypeOf((*MockInspectCheckPointTable)(nil).SaveCheckPoint), arg0, arg1)
+}

blobstore/scheduler/disk_droper.go

@@ -0,0 +1,380 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package scheduler
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/taskswitch"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/scheduler/base"
+	"github.com/cubefs/cubefs/blobstore/scheduler/client"
+	"github.com/cubefs/cubefs/blobstore/scheduler/db"
+	"github.com/cubefs/cubefs/blobstore/util/log"
+)
+
+// IDiskDroper define the interface of disk drop manager
+type IDiskDroper interface {
+	Migrator
+	Progress(ctx context.Context) (repairingDiskID proto.DiskID, total, repaired int)
+}
+
+// DiskDropMgrConfig disk drop manager config
+type DiskDropMgrConfig struct {
+	MigrateConfig
+}
+
+// DiskDropMgr disk drop manager
+type DiskDropMgr struct {
+	IMigrater
+
+	mu             sync.Mutex
+	dropDisk       *client.DiskInfoSimple
+	droppingDiskID proto.DiskID
+	clusterMgrCli  client.ClusterMgrAPI
+	hasRevised     bool
+	cfg            *DiskDropMgrConfig
+}
+
+// NewDiskDropMgr returns disk drop manager
+func NewDiskDropMgr(
+	clusterMgrCli client.ClusterMgrAPI,
+	volumeUpdater client.IVolumeUpdater,
+	taskSwitch taskswitch.ISwitcher,
+	taskTbl db.IMigrateTaskTable,
+	conf *DiskDropMgrConfig) *DiskDropMgr {
+	mgr := &DiskDropMgr{
+		clusterMgrCli: clusterMgrCli,
+		cfg:           conf,
+	}
+	mgr.IMigrater = NewMigrateMgr(clusterMgrCli, volumeUpdater, taskSwitch, taskTbl,
+		&conf.MigrateConfig, proto.DiskDropTaskType, conf.ClusterID)
+	return mgr
+}
+
+// Load load disk drop task from database
+func (mgr *DiskDropMgr) Load() (err error) {
+	ctx := context.Background()
+	allTasks, err := mgr.IMigrater.FindAll(ctx)
+	if err != nil {
+		return err
+	}
+	if len(allTasks) == 0 {
+		log.Infof("no drop tasks in db")
+		return
+	}
+
+	droppingDiskID := allTasks[0].SrcMigDiskID()
+	tasks, err := mgr.IMigrater.FindByDiskID(ctx, droppingDiskID)
+	if err != nil {
+		return err
+	}
+	if len(allTasks) != len(tasks) {
+		panic("can not allow many disk dropping")
+	}
+
+	mgr.setDroppingDiskID(droppingDiskID)
+
+	return mgr.IMigrater.Load()
+}
+
+// Run run disk drop task
+func (mgr *DiskDropMgr) Run() {
+	go mgr.collectTaskLoop()
+	mgr.IMigrater.Run()
+	go mgr.checkDroppedAndClearLoop()
+}
+
+// collectTaskLoop collect disk drop task loop
+func (mgr *DiskDropMgr) collectTaskLoop() {
+	t := time.NewTicker(time.Duration(mgr.cfg.CollectTaskIntervalS) * time.Second)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-t.C:
+			mgr.IMigrater.WaitEnable()
+			mgr.collectTask()
+		case <-mgr.IMigrater.Done():
+			return
+		}
+	}
+}
+
+func (mgr *DiskDropMgr) collectTask() {
+	span, ctx := trace.StartSpanFromContext(context.Background(), "disk_drop.collectTask")
+	defer span.Finish()
+
+	if !mgr.hasRevised && mgr.hasDroppingDisk() {
+		err := mgr.reviseDropTask(ctx, mgr.getDroppingDiskID())
+		if err == nil {
+			span.Infof("drop collect revise tasks success")
+			mgr.hasRevised = true
+			mgr.setDroppingDiskID(mgr.getDroppingDiskID())
+			return
+		}
+		span.Errorf("drop collect revise task failed: err[%+v]", err)
+		return
+	}
+
+	if mgr.hasDroppingDisk() {
+		return
+	}
+
+	dropDisk, err := mgr.acquireDropDisk(ctx)
+	if err != nil {
+		span.Info("acquire drop disk failed: err[%+v]", err)
+		return
+	}
+
+	if dropDisk == nil {
+		return
+	}
+
+	err = mgr.genDiskDropTasks(ctx, dropDisk.DiskID, dropDisk.Idc)
+	if err != nil {
+		span.Errorf("drop collect drop task failed: err[%+v]", err)
+		return
+	}
+
+	mgr.setDroppingDiskID(dropDisk.DiskID)
+}
+
+func (mgr *DiskDropMgr) reviseDropTask(ctx context.Context, diskID proto.DiskID) error {
+	span := trace.SpanFromContextSafe(ctx)
+
+	diskInfo, err := mgr.clusterMgrCli.GetDiskInfo(ctx, diskID)
+	if err != nil {
+		span.Errorf("get disk info failed: err[%+v]", err)
+		return err
+	}
+
+	err = mgr.genDiskDropTasks(ctx, diskInfo.DiskID, diskInfo.Idc)
+	if err != nil {
+		span.Errorf("gen disk drop tasks failed: err[%+v]", err)
+		return err
+	}
+	return nil
+}
+
+func (mgr *DiskDropMgr) genDiskDropTasks(ctx context.Context, diskID proto.DiskID, diskIdc string) error {
+	span := trace.SpanFromContextSafe(ctx)
+
+	span.Infof("start generate disk drop tasks: disk_id[%d], disk_idc[%s]", diskID, diskIdc)
+
+	vuidsDb, err := mgr.dropVuidsFromDb(ctx, diskID)
+	if err != nil {
+		span.Errorf("get drop vuids from db failed: err[%+v]", err)
+		return err
+	}
+	span.Infof("drop vuids from db success: len[%d]", len(vuidsDb))
+
+	vuidsCm, err := mgr.dropVuidsFromCm(ctx, diskID)
+	if err != nil {
+		span.Errorf("get drop vuid from clustermgr failed: err[%+v]", err)
+		return err
+	}
+	span.Infof("drop vuids from clustermgr success: len[%d]", len(vuidsCm))
+
+	remain := base.Subtraction(vuidsCm, vuidsDb)
+	span.Infof("should gen tasks: remain len[%d]", len(remain))
+	for _, vuid := range remain {
+		mgr.initOneTask(ctx, vuid, diskID, diskIdc)
+		span.Infof("init drop task success: vuid[%d]", vuid)
+	}
+	return nil
+}
+
+func (mgr *DiskDropMgr) dropVuidsFromDb(ctx context.Context, diskID proto.DiskID) (drops []proto.Vuid, err error) {
+	tasks, err := mgr.IMigrater.FindByDiskID(ctx, diskID)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, t := range tasks {
+		drops = append(drops, t.SourceVuid)
+	}
+	return drops, nil
+}
+
+func (mgr *DiskDropMgr) dropVuidsFromCm(ctx context.Context, diskID proto.DiskID) (drops []proto.Vuid, err error) {
+	vunits, err := mgr.clusterMgrCli.ListDiskVolumeUnits(ctx, diskID)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, vunit := range vunits {
+		drops = append(drops, vunit.Vuid)
+	}
+	return drops, nil
+}
+
+func (mgr *DiskDropMgr) initOneTask(ctx context.Context, src proto.Vuid, dropDiskID proto.DiskID, diskIDC string) {
+	vid := src.Vid()
+	t := proto.MigrateTask{
+		TaskID:       mgr.genUniqTaskID(vid),
+		State:        proto.MigrateStateInited,
+		SourceDiskID: dropDiskID,
+		SourceIdc:    diskIDC,
+		SourceVuid:   src,
+	}
+	mgr.IMigrater.AddTask(ctx, &t)
+}
+
+func (mgr *DiskDropMgr) acquireDropDisk(ctx context.Context) (*client.DiskInfoSimple, error) {
+	// it will retry when break in collectTask,
+	// should make sure acquire same disk
+	if mgr.dropDisk != nil {
+		return mgr.dropDisk, nil
+	}
+
+	dropDisks, err := mgr.clusterMgrCli.ListDropDisks(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if len(dropDisks) == 0 {
+		return nil, nil
+	}
+
+	mgr.dropDisk = dropDisks[0]
+	return mgr.dropDisk, nil
+}
+
+func (mgr *DiskDropMgr) checkDroppedAndClearLoop() {
+	t := time.NewTicker(time.Duration(mgr.cfg.CheckTaskIntervalS) * time.Second)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-t.C:
+			mgr.IMigrater.WaitEnable()
+			mgr.checkDroppedAndClear()
+		case <-mgr.IMigrater.Done():
+			return
+		}
+	}
+}
+
+func (mgr *DiskDropMgr) checkDroppedAndClear() {
+	diskID := mgr.getDroppingDiskID()
+
+	span, ctx := trace.StartSpanFromContext(context.Background(), "disk_drop.checkDroppedAndClear")
+	defer span.Finish()
+
+	if !mgr.hasDroppingDisk() {
+		return
+	}
+	if mgr.checkDropped(ctx, diskID) {
+		err := mgr.clusterMgrCli.SetDiskDropped(ctx, diskID)
+		if err != nil {
+			span.Errorf("set disk dropped failed: err[%+v]", err)
+			return
+		}
+		span.Infof("start clear dropped disk: disk_id[%d]", diskID)
+		mgr.clearTasksByDiskID(ctx, diskID)
+		mgr.emptyDroppingDiskID()
+	}
+}
+
+func (mgr *DiskDropMgr) checkDropped(ctx context.Context, diskID proto.DiskID) bool {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Infof("check dropped: disk_id[%d]", diskID)
+
+	tasks, err := mgr.IMigrater.FindByDiskID(ctx, diskID)
+	if err != nil {
+		span.Errorf("find all tasks failed: disk_id[%d], err[%+v]", diskID, err)
+		return false
+	}
+	for _, task := range tasks {
+		if !task.Finished() {
+			return false
+		}
+	}
+	span.Infof("disk drop has finished: disk_id[%d], task len[%d]", diskID, len(tasks))
+
+	vunitInfos, err := mgr.clusterMgrCli.ListDiskVolumeUnits(ctx, diskID)
+	if err != nil {
+		span.Errorf("list disk volume units failed: disk_id[%s], err[%+v]", diskID, err)
+		return false
+	}
+	if len(vunitInfos) != 0 {
+		// it may be occur when migration done and repair tasks generate concurrent, list volume units may not return the migrate unit
+		span.Warnf("clustermgr has some volume unit not repair and revise again: disk_id[%d], volume units len[%d]", diskID, len(vunitInfos))
+		if err = mgr.reviseDropTask(ctx, diskID); err != nil {
+			span.Errorf("revise repair task failed: err[%+v]", err)
+		}
+		return false
+	}
+	return true
+}
+
+func (mgr *DiskDropMgr) clearTasksByDiskID(ctx context.Context, diskID proto.DiskID) {
+	mgr.IMigrater.ClearTasksByDiskID(ctx, diskID)
+}
+
+func (mgr *DiskDropMgr) setDroppingDiskID(diskID proto.DiskID) {
+	mgr.mu.Lock()
+	defer mgr.mu.Unlock()
+	mgr.droppingDiskID = diskID
+}
+
+func (mgr *DiskDropMgr) emptyDroppingDiskID() {
+	mgr.mu.Lock()
+	defer mgr.mu.Unlock()
+	mgr.droppingDiskID = base.EmptyDiskID
+	mgr.dropDisk = nil
+}
+
+func (mgr *DiskDropMgr) getDroppingDiskID() proto.DiskID {
+	mgr.mu.Lock()
+	defer mgr.mu.Unlock()
+	return mgr.droppingDiskID
+}
+
+func (mgr *DiskDropMgr) hasDroppingDisk() bool {
+	mgr.mu.Lock()
+	defer mgr.mu.Unlock()
+	return mgr.droppingDiskID != base.EmptyDiskID
+}
+
+func (mgr *DiskDropMgr) genUniqTaskID(vid proto.Vid) string {
+	return base.GenTaskID("disk_drop", vid)
+}
+
+// Progress returns disk drop progress
+func (mgr *DiskDropMgr) Progress(ctx context.Context) (dropDiskID proto.DiskID, total, dropped int) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	dropDiskID = mgr.getDroppingDiskID()
+	if dropDiskID == base.EmptyDiskID {
+		return base.EmptyDiskID, 0, 0
+	}
+
+	allTasks, err := mgr.IMigrater.FindByDiskID(ctx, dropDiskID)
+	if err != nil {
+		span.Errorf("find all task failed: err[%+v]", err)
+		return dropDiskID, 0, 0
+	}
+	total = len(allTasks)
+	for _, task := range allTasks {
+		if task.Finished() {
+			dropped++
+		}
+	}
+	return dropDiskID, total, dropped
+}

blobstore/scheduler/disk_droper_test.go

@@ -0,0 +1,364 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package scheduler
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/require"
+
+	api "github.com/cubefs/cubefs/blobstore/api/scheduler"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/scheduler/base"
+	"github.com/cubefs/cubefs/blobstore/scheduler/client"
+	"github.com/cubefs/cubefs/blobstore/testing/mocks"
+	"github.com/cubefs/cubefs/blobstore/util/closer"
+)
+
+func newDiskDroper(t *testing.T) *DiskDropMgr {
+	ctr := gomock.NewController(t)
+	clusterMgr := NewMockClusterMgrAPI(ctr)
+	volumeUpdater := NewMockVolumeUpdater(ctr)
+	taskSwitch := mocks.NewMockSwitcher(ctr)
+	migrateTable := NewMockMigrateTaskTable(ctr)
+	conf := &DiskDropMgrConfig{}
+	c := closer.New()
+
+	migrater := NewMockMigrater(ctr)
+	migrater.EXPECT().StatQueueTaskCnt().AnyTimes().Return(0, 0, 0)
+	migrater.EXPECT().Close().AnyTimes().DoAndReturn(c.Close)
+	migrater.EXPECT().Done().AnyTimes().Return(c.Done())
+	mgr := NewDiskDropMgr(clusterMgr, volumeUpdater, taskSwitch, migrateTable, conf)
+	mgr.IMigrater = migrater
+	return mgr
+}
+
+func TestDiskDropLoad(t *testing.T) {
+	{
+		mgr := newDiskDroper(t)
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindAll(any).Return(nil, errMock)
+		err := mgr.Load()
+		require.True(t, errors.Is(err, errMock))
+	}
+	{
+		mgr := newDiskDroper(t)
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindAll(any).Return(nil, nil)
+		err := mgr.Load()
+		require.NoError(t, err)
+	}
+	{
+		mgr := newDiskDroper(t)
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindAll(any).Return([]*proto.MigrateTask{{SourceDiskID: proto.DiskID(1)}, {SourceDiskID: proto.DiskID(2)}}, nil)
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return(nil, errMock)
+		err := mgr.Load()
+		require.True(t, errors.Is(err, errMock))
+	}
+	{
+		mgr := newDiskDroper(t)
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindAll(any).Return([]*proto.MigrateTask{{SourceDiskID: proto.DiskID(1)}, {SourceDiskID: proto.DiskID(2)}}, nil)
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return([]*proto.MigrateTask{{SourceDiskID: proto.DiskID(1)}}, nil)
+		require.Panics(t, func() {
+			mgr.Load()
+		})
+	}
+	{
+		mgr := newDiskDroper(t)
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindAll(any).Return([]*proto.MigrateTask{{SourceDiskID: proto.DiskID(1)}}, nil)
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return([]*proto.MigrateTask{{SourceDiskID: proto.DiskID(1)}}, nil)
+		mgr.IMigrater.(*MockMigrater).EXPECT().Load().Return(nil)
+		err := mgr.Load()
+		require.NoError(t, err)
+	}
+}
+
+func TestDiskDropRun(t *testing.T) {
+	mgr := newDiskDroper(t)
+	defer mgr.Close()
+
+	mgr.IMigrater.(*MockMigrater).EXPECT().WaitEnable().AnyTimes().Return()
+	mgr.IMigrater.(*MockMigrater).EXPECT().Enabled().AnyTimes().Return(true)
+	mgr.IMigrater.(*MockMigrater).EXPECT().Run().Return()
+	mgr.IMigrater.(*MockMigrater).EXPECT().ClearTasksByStates(any, any).AnyTimes().Return()
+	mgr.IMigrater.(*MockMigrater).EXPECT().GetMigratingDiskNum().AnyTimes().Return(1)
+	mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDropDisks(any).AnyTimes().Return(nil, errMock)
+	mgr.cfg.CollectTaskIntervalS = 1
+	mgr.cfg.CheckTaskIntervalS = 1
+	require.True(t, mgr.Enabled())
+	mgr.Run()
+
+	time.Sleep(1 * time.Second)
+}
+
+func TestDiskDropCollectTask(t *testing.T) {
+	{
+		// reviseDropTask failed
+		mgr := newDiskDroper(t)
+		mgr.hasRevised = false
+		mgr.droppingDiskID = proto.DiskID(1)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(nil, errMock)
+		mgr.collectTask()
+	}
+	{
+		// genDiskDropTasks failed
+		mgr := newDiskDroper(t)
+		mgr.hasRevised = false
+		mgr.droppingDiskID = proto.DiskID(1)
+
+		// find in db failed
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(&client.DiskInfoSimple{DiskID: mgr.droppingDiskID}, nil)
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return(nil, errMock)
+		mgr.collectTask()
+
+		// find in cm failed
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(&client.DiskInfoSimple{DiskID: mgr.droppingDiskID}, nil)
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return(nil, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, errMock)
+		mgr.collectTask()
+
+		// genDiskDropTasks success
+		volume := MockGenVolInfo(10005, codemode.EC6P6, proto.VolumeStatusIdle)
+		var units []*client.VunitInfoSimple
+		for _, unit := range volume.VunitLocations {
+			ele := client.VunitInfoSimple{
+				Vuid:   unit.Vuid,
+				DiskID: unit.DiskID,
+			}
+			units = append(units, &ele)
+		}
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(&client.DiskInfoSimple{DiskID: mgr.droppingDiskID}, nil)
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return(nil, nil)
+		mgr.IMigrater.(*MockMigrater).EXPECT().AddTask(any, any).AnyTimes().Return()
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil)
+		mgr.collectTask()
+	}
+	{
+		mgr := newDiskDroper(t)
+		mgr.hasRevised = true
+		mgr.droppingDiskID = proto.DiskID(1)
+		mgr.collectTask()
+	}
+	{
+		// acquireDropDisk
+		mgr := newDiskDroper(t)
+		mgr.hasRevised = true
+
+		// list drop disk failed
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDropDisks(any).Return(nil, errMock)
+		mgr.collectTask()
+
+		// no drop disk
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDropDisks(any).Return(nil, nil)
+		mgr.collectTask()
+	}
+	{
+		mgr := newDiskDroper(t)
+		mgr.hasRevised = true
+		disk1 := &client.DiskInfoSimple{
+			ClusterID:    1,
+			Idc:          "z0",
+			Rack:         "rack1",
+			Host:         "127.0.0.1:8000",
+			Status:       proto.DiskStatusNormal,
+			DiskID:       1,
+			FreeChunkCnt: 10,
+			MaxChunkCnt:  700,
+		}
+		volume := MockGenVolInfo(10005, codemode.EC6P6, proto.VolumeStatusIdle)
+		var units []*client.VunitInfoSimple
+		for _, unit := range volume.VunitLocations {
+			ele := client.VunitInfoSimple{
+				Vuid:   unit.Vuid,
+				DiskID: unit.DiskID,
+			}
+			units = append(units, &ele)
+		}
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return(nil, nil)
+		mgr.IMigrater.(*MockMigrater).EXPECT().AddTask(any, any).AnyTimes().Return()
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDropDisks(any).Return([]*client.DiskInfoSimple{disk1}, nil)
+		mgr.collectTask()
+	}
+	{
+		mgr := newDiskDroper(t)
+		mgr.hasRevised = true
+		disk1 := &client.DiskInfoSimple{
+			ClusterID:    1,
+			Idc:          "z0",
+			Rack:         "rack1",
+			Host:         "127.0.0.1:8000",
+			Status:       proto.DiskStatusNormal,
+			DiskID:       1,
+			FreeChunkCnt: 10,
+			MaxChunkCnt:  700,
+		}
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDropDisks(any).Return([]*client.DiskInfoSimple{disk1}, nil)
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return(nil, errMock)
+		mgr.collectTask()
+	}
+}
+
+func TestDiskDropCheckDroppedAndClear(t *testing.T) {
+	{
+		mgr := newDiskDroper(t)
+		mgr.checkDroppedAndClear()
+	}
+	{
+		// check dropped return false
+		mgr := newDiskDroper(t)
+		mgr.droppingDiskID = proto.DiskID(1)
+
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return(nil, errMock)
+		mgr.checkDroppedAndClear()
+		require.True(t, mgr.hasDroppingDisk())
+
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return(nil, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, errMock)
+		mgr.checkDroppedAndClear()
+		require.True(t, mgr.hasDroppingDisk())
+
+		task1 := &proto.MigrateTask{State: proto.MigrateStatePrepared}
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return([]*proto.MigrateTask{task1}, nil)
+		mgr.checkDroppedAndClear()
+		require.True(t, mgr.hasDroppingDisk())
+
+		task2 := &proto.MigrateTask{State: proto.MigrateStateFinished}
+		task3 := &proto.MigrateTask{State: proto.MigrateStateFinishedInAdvance}
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return([]*proto.MigrateTask{task1, task2, task3}, nil)
+		mgr.checkDroppedAndClear()
+		require.True(t, mgr.hasDroppingDisk())
+	}
+	{
+		// check dropped return true
+		mgr := newDiskDroper(t)
+		mgr.droppingDiskID = proto.DiskID(1)
+		task1 := &proto.MigrateTask{State: proto.MigrateStateFinished}
+		task2 := &proto.MigrateTask{State: proto.MigrateStateFinishedInAdvance}
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return([]*proto.MigrateTask{task1, task2}, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().SetDiskDropped(any, any).Return(errMock)
+		mgr.checkDroppedAndClear()
+		require.True(t, mgr.hasDroppingDisk())
+
+		volume := MockGenVolInfo(10, codemode.EC6P6, proto.VolumeStatusIdle)
+		var units []*client.VunitInfoSimple
+		for _, unit := range volume.VunitLocations {
+			ele := client.VunitInfoSimple{
+				Vuid:   unit.Vuid,
+				DiskID: unit.DiskID,
+			}
+			units = append(units, &ele)
+		}
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return([]*proto.MigrateTask{task1, task2}, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(nil, errMock)
+		mgr.checkDroppedAndClear()
+		require.True(t, mgr.hasDroppingDisk())
+
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return([]*proto.MigrateTask{task1, task2}, nil)
+		mgr.IMigrater.(*MockMigrater).EXPECT().ClearTasksByDiskID(any, any).Return()
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().SetDiskDropped(any, any).Return(nil)
+		mgr.checkDroppedAndClear()
+		require.False(t, mgr.hasDroppingDisk())
+	}
+}
+
+func TestDiskDropAcquireTask(t *testing.T) {
+	ctx := context.Background()
+	idc := "z0"
+	mgr := newDiskDroper(t)
+	mgr.IMigrater.(*MockMigrater).EXPECT().AcquireTask(any, any).Return(&proto.MigrateTask{}, nil)
+	_, err := mgr.AcquireTask(ctx, idc)
+	require.NoError(t, err)
+}
+
+func TestDiskDropCancelTask(t *testing.T) {
+	ctx := context.Background()
+	mgr := newDiskDroper(t)
+	mgr.IMigrater.(*MockMigrater).EXPECT().CancelTask(any, any).Return(nil)
+	err := mgr.CancelTask(ctx, &api.CancelTaskArgs{})
+	require.NoError(t, err)
+}
+
+func TestDiskDropReclaimTask(t *testing.T) {
+	ctx := context.Background()
+	idc := "z0"
+	mgr := newDiskDroper(t)
+	mgr.IMigrater.(*MockMigrater).EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil)
+	t1 := mockGenMigrateTask(idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap)
+	err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{})
+	require.NoError(t, err)
+}
+
+func TestDiskDropCompleteTask(t *testing.T) {
+	ctx := context.Background()
+	idc := "z0"
+	mgr := newDiskDroper(t)
+	mgr.IMigrater.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(nil)
+	t1 := mockGenMigrateTask(idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap)
+	err := mgr.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: idc, TaskId: t1.TaskID, Src: t1.Sources, Dest: t1.Destination})
+	require.NoError(t, err)
+
+	mgr.IMigrater.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(errMock)
+	err = mgr.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: idc, TaskId: t1.TaskID, Src: t1.Sources, Dest: t1.Destination})
+	require.True(t, errors.Is(err, errMock))
+}
+
+func TestDiskDropRenewalTask(t *testing.T) {
+	ctx := context.Background()
+	idc := "z0"
+	mgr := newDiskDroper(t)
+	mgr.IMigrater.(*MockMigrater).EXPECT().RenewalTask(any, any, any).Return(nil)
+	err := mgr.RenewalTask(ctx, idc, "")
+	require.NoError(t, err)
+
+	mgr.IMigrater.(*MockMigrater).EXPECT().RenewalTask(any, any, any).Return(errMock)
+	err = mgr.RenewalTask(ctx, idc, "")
+	require.True(t, errors.Is(err, errMock))
+}
+
+func TestDiskDropProgress(t *testing.T) {
+	ctx := context.Background()
+	{
+		mgr := newDiskDroper(t)
+		diskID, _, _ := mgr.Progress(ctx)
+		require.Equal(t, base.EmptyDiskID, diskID)
+	}
+	{
+		mgr := newDiskDroper(t)
+		mgr.droppingDiskID = proto.DiskID(1)
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return(nil, errMock)
+		diskID, _, _ := mgr.Progress(ctx)
+		require.Equal(t, proto.DiskID(1), diskID)
+	}
+	{
+		mgr := newDiskDroper(t)
+		diskID := proto.DiskID(1)
+		mgr.droppingDiskID = diskID
+		task1 := &proto.MigrateTask{State: proto.MigrateStatePrepared, SourceDiskID: diskID}
+		task2 := &proto.MigrateTask{State: proto.MigrateStateFinished, SourceDiskID: diskID}
+		task3 := &proto.MigrateTask{State: proto.MigrateStateFinishedInAdvance, SourceDiskID: diskID}
+		mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return([]*proto.MigrateTask{task1, task2, task3}, nil)
+		doingDisk, tatal, dropped := mgr.Progress(ctx)
+		require.Equal(t, diskID, doingDisk)
+		require.Equal(t, 3, tatal)
+		require.Equal(t, 2, dropped)
+	}
+}

blobstore/scheduler/disk_repairer.go

@@ -0,0 +1,848 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package scheduler
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	api "github.com/cubefs/cubefs/blobstore/api/scheduler"
+	"github.com/cubefs/cubefs/blobstore/common/counter"
+	"github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+	"github.com/cubefs/cubefs/blobstore/common/taskswitch"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/scheduler/base"
+	"github.com/cubefs/cubefs/blobstore/scheduler/client"
+	"github.com/cubefs/cubefs/blobstore/scheduler/db"
+	"github.com/cubefs/cubefs/blobstore/util/closer"
+	"github.com/cubefs/cubefs/blobstore/util/log"
+)
+
+// IDiskRepairer define the interface of disk repair manager
+type IDiskRepairer interface {
+	AcquireTask(ctx context.Context, idc string) (task *proto.VolRepairTask, err error)
+	CancelTask(ctx context.Context, args *api.CancelTaskArgs) error
+	CompleteTask(ctx context.Context, args *api.CompleteTaskArgs) error
+	ReclaimTask(ctx context.Context, idc, taskID string,
+		src []proto.VunitLocation, oldDst proto.VunitLocation, newDst *client.AllocVunitInfo) error
+	RenewalTask(ctx context.Context, idc, taskID string) error
+	QueryTask(ctx context.Context, taskID string) (*api.RepairTaskDetail, error)
+	ReportWorkerTaskStats(st *api.TaskReportArgs)
+	StatQueueTaskCnt() (inited, prepared, completed int)
+	Stats() api.MigrateTasksStat
+	Progress(ctx context.Context) (repairingDiskID proto.DiskID, total, repaired int)
+	Enabled() bool
+	Load() error
+	Run()
+	closer.Closer
+}
+
+const (
+	prepareIntervalS = 1
+	finishIntervalS  = 5
+)
+
+// DiskRepairMgrCfg repair manager config
+type DiskRepairMgrCfg struct {
+	ClusterID proto.ClusterID `json:"cluster_id"`
+	base.TaskCommonConfig
+}
+
+// DiskRepairMgr repair task manager
+type DiskRepairMgr struct {
+	closer.Closer
+	repairingDiskID proto.DiskID // only supports repair one disk at the same time temporarily
+	brokenDisk      *client.DiskInfoSimple
+
+	mu sync.Mutex
+
+	taskTbl db.IRepairTaskTable
+
+	prepareQueue *base.TaskQueue
+	workQueue    *base.WorkerTaskQueue
+	finishQueue  *base.TaskQueue
+
+	clusterMgrCli client.ClusterMgrAPI
+
+	taskSwitch taskswitch.ISwitcher
+
+	// for stats
+	finishTaskCounter counter.Counter
+	taskStatsMgr      *base.TaskStatsMgr
+
+	hasRevised bool
+	cfg        *DiskRepairMgrCfg
+}
+
+// NewRepairMgr returns repair manager
+func NewRepairMgr(cfg *DiskRepairMgrCfg, taskSwitch taskswitch.ISwitcher,
+	taskTbl db.IRepairTaskTable, cmCli client.ClusterMgrAPI) *DiskRepairMgr {
+	mgr := &DiskRepairMgr{
+		Closer:       closer.New(),
+		taskTbl:      taskTbl,
+		prepareQueue: base.NewTaskQueue(time.Duration(cfg.PrepareQueueRetryDelayS) * time.Second),
+		workQueue:    base.NewWorkerTaskQueue(time.Duration(cfg.CancelPunishDurationS) * time.Second),
+		finishQueue:  base.NewTaskQueue(time.Duration(cfg.FinishQueueRetryDelayS) * time.Second),
+
+		clusterMgrCli: cmCli,
+		taskSwitch:    taskSwitch,
+		cfg:           cfg,
+
+		hasRevised: false,
+	}
+	mgr.taskStatsMgr = base.NewTaskStatsMgrAndRun(cfg.ClusterID, proto.RepairTaskType, mgr)
+	return mgr
+}
+
+// Load load repair task from database
+func (mgr *DiskRepairMgr) Load() error {
+	ctx := context.Background()
+
+	allTasks, err := mgr.taskTbl.FindAll(ctx)
+	if err != nil {
+		return err
+	}
+	log.Infof("repair load tasks: len[%d]", len(allTasks))
+
+	if len(allTasks) == 0 {
+		return nil
+	}
+
+	repairingDiskID := allTasks[0].RepairDiskID
+	tasks, err := mgr.taskTbl.FindByDiskID(ctx, repairingDiskID)
+	if err != nil {
+		return err
+	}
+	if len(allTasks) != len(tasks) {
+		panic("can not allow many disk repairing")
+	}
+
+	mgr.setRepairingDiskID(repairingDiskID)
+
+	for _, t := range tasks {
+		if t.Running() {
+			err = base.VolTaskLockerInst().TryLock(ctx, t.Vid())
+			if err != nil {
+				log.Panicf("repair task conflict: task[%+v], err[%+v]",
+					t, err.Error())
+			}
+		}
+
+		log.Infof("load task success: task_id[%s], state[%d]", t.TaskID, t.State)
+		switch t.State {
+		case proto.RepairStateInited:
+			mgr.prepareQueue.PushTask(t.TaskID, t)
+		case proto.RepairStatePrepared:
+			mgr.workQueue.AddPreparedTask(t.BrokenDiskIDC, t.TaskID, t)
+		case proto.RepairStateWorkCompleted:
+			mgr.finishQueue.PushTask(t.TaskID, t)
+		case proto.RepairStateFinished, proto.RepairStateFinishedInAdvance:
+			continue
+		default:
+			panic("unexpect repair state")
+		}
+	}
+
+	return nil
+}
+
+// Run run repair task includes collect/prepare/finish/check phase
+func (mgr *DiskRepairMgr) Run() {
+	go mgr.collectTaskLoop()
+	go mgr.prepareTaskLoop()
+	go mgr.finishTaskLoop()
+	go mgr.checkRepairedAndClearLoop()
+}
+
+func (mgr *DiskRepairMgr) Enabled() bool {
+	return mgr.taskSwitch.Enabled()
+}
+
+func (mgr *DiskRepairMgr) collectTaskLoop() {
+	t := time.NewTicker(time.Duration(mgr.cfg.CollectTaskIntervalS) * time.Second)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-t.C:
+			mgr.taskSwitch.WaitEnable()
+			mgr.collectTask()
+		case <-mgr.Closer.Done():
+			return
+		}
+	}
+}
+
+func (mgr *DiskRepairMgr) collectTask() {
+	span, ctx := trace.StartSpanFromContext(context.Background(), "disk_repair.collectTask")
+	defer span.Finish()
+
+	// revise repair tasks to make sure data consistency when services start
+	if !mgr.hasRevised && mgr.hasRepairingDisk() {
+		span.Infof("first collect task will revise repair task")
+		err := mgr.reviseRepairTask(ctx, mgr.getRepairingDiskID())
+		if err == nil {
+			span.Infof("firstCollectTask finished")
+			mgr.hasRevised = true
+		}
+		return
+	}
+
+	if mgr.hasRepairingDisk() {
+		span.Infof("disk is repairing and skip collect task: disk_id[%d]", mgr.getRepairingDiskID())
+		return
+	}
+
+	brokenDisk, err := mgr.acquireBrokenDisk(ctx)
+	if err != nil {
+		span.Errorf("acquire broken disk failed: err[%+v]", err)
+		return
+	}
+	if brokenDisk == nil {
+		return
+	}
+
+	err = mgr.genDiskRepairTasks(ctx, brokenDisk.DiskID, brokenDisk.Idc)
+	if err != nil {
+		span.Errorf("generate disk repair tasks failed: err[%+v]", err)
+		return
+	}
+
+	base.InsistOn(ctx, "set disk diskId %d repairing failed", func() error {
+		return mgr.clusterMgrCli.SetDiskRepairing(ctx, brokenDisk.DiskID)
+	})
+
+	mgr.setRepairingDiskID(brokenDisk.DiskID)
+}
+
+func (mgr *DiskRepairMgr) reviseRepairTask(ctx context.Context, diskID proto.DiskID) error {
+	span := trace.SpanFromContextSafe(ctx)
+
+	diskInfo, err := mgr.clusterMgrCli.GetDiskInfo(ctx, diskID)
+	if err != nil {
+		span.Errorf("get disk info failed: err[%+v]", err)
+		return err
+	}
+
+	if err = mgr.genDiskRepairTasks(ctx, diskID, diskInfo.Idc); err != nil {
+		span.Errorf("generate disk repair tasks failed: err[%+v]", err)
+		return err
+	}
+
+	if diskInfo.IsBroken() {
+		execMsg := fmt.Sprintf("set disk diskId %d repairing", mgr.getRepairingDiskID())
+		base.InsistOn(ctx, execMsg, func() error {
+			return mgr.clusterMgrCli.SetDiskRepairing(ctx, diskID)
+		})
+	}
+	return nil
+}
+
+func (mgr *DiskRepairMgr) genDiskRepairTasks(ctx context.Context, diskID proto.DiskID, diskIdc string) error {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Infof("start generate disk repair tasks: disk_id[%d], disk_idc[%s]", diskID, diskIdc)
+
+	vuidsDb, err := mgr.badVuidsFromDb(ctx, diskID)
+	if err != nil {
+		span.Errorf("get bad vuids from db failed: err[%+v]", err)
+		return err
+	}
+	span.Infof("bad vuids from db: len[%d]", len(vuidsDb))
+
+	vuidsCm, err := mgr.badVuidsFromClusterMgr(ctx, diskID)
+	if err != nil {
+		span.Errorf("get bad vuid from clustermgr failed: err[%+v]", err)
+		return err
+	}
+	span.Infof("bad vuid from clustermgr: len[%d]", len(vuidsCm))
+
+	remain := base.Subtraction(vuidsCm, vuidsDb)
+	span.Infof("should gen tasks remain: len[%d]", len(remain))
+	for _, vuid := range remain {
+		mgr.initOneTask(ctx, vuid, diskID, diskIdc)
+	}
+	return nil
+}
+
+func (mgr *DiskRepairMgr) badVuidsFromDb(ctx context.Context, diskID proto.DiskID) (bads []proto.Vuid, err error) {
+	tasks, err := mgr.taskTbl.FindByDiskID(ctx, diskID)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, t := range tasks {
+		bads = append(bads, t.RepairVuid())
+	}
+	return bads, nil
+}
+
+func (mgr *DiskRepairMgr) badVuidsFromClusterMgr(ctx context.Context, diskID proto.DiskID) (bads []proto.Vuid, err error) {
+	vunits, err := mgr.clusterMgrCli.ListDiskVolumeUnits(ctx, diskID)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, vunit := range vunits {
+		bads = append(bads, vunit.Vuid)
+	}
+	return bads, nil
+}
+
+func (mgr *DiskRepairMgr) initOneTask(ctx context.Context, badVuid proto.Vuid, brokenDiskID proto.DiskID, brokenDiskIdc string) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	vid := badVuid.Vid()
+	t := proto.VolRepairTask{
+		TaskID:       mgr.genUniqTaskID(vid),
+		State:        proto.RepairStateInited,
+		RepairDiskID: brokenDiskID,
+
+		BadVuid: badVuid,
+		BadIdx:  badVuid.Index(),
+
+		BrokenDiskIDC: brokenDiskIdc,
+		TriggerBy:     proto.BrokenDiskTrigger,
+	}
+	base.InsistOn(ctx, "repair init one task insert task to tbl", func() error {
+		return mgr.taskTbl.Insert(ctx, &t)
+	})
+
+	mgr.prepareQueue.PushTask(t.TaskID, &t)
+	span.Infof("init repair task success %+v", t)
+}
+
+func (mgr *DiskRepairMgr) genUniqTaskID(vid proto.Vid) string {
+	return base.GenTaskID("disk-repair", vid)
+}
+
+func (mgr *DiskRepairMgr) acquireBrokenDisk(ctx context.Context) (*client.DiskInfoSimple, error) {
+	// can not assume request cm to acquire broken disk is the same disk
+	// because break in generate tasks(eg. generate task return an error),
+	// and reentry(not because of starting of service) need the same disk
+	// cache last broken disk acquired from cm
+	if mgr.brokenDisk != nil {
+		return mgr.brokenDisk, nil
+	}
+
+	brokenDisks, err := mgr.clusterMgrCli.ListBrokenDisks(ctx, 1)
+	if err != nil {
+		return nil, err
+	}
+	if len(brokenDisks) == 0 {
+		return nil, nil
+	}
+
+	mgr.brokenDisk = brokenDisks[0]
+	return mgr.brokenDisk, nil
+}
+
+func (mgr *DiskRepairMgr) prepareTaskLoop() {
+	for {
+		mgr.taskSwitch.WaitEnable()
+		todo, doing := mgr.workQueue.StatsTasks()
+		if !mgr.hasRepairingDisk() || todo+doing >= mgr.cfg.WorkQueueSize {
+			time.Sleep(1 * time.Second)
+			continue
+		}
+
+		err := mgr.popTaskAndPrepare()
+		if err == base.ErrNoTaskInQueue {
+			time.Sleep(time.Duration(prepareIntervalS) * time.Second)
+		}
+	}
+}
+
+func (mgr *DiskRepairMgr) popTaskAndPrepare() error {
+	_, task, exist := mgr.prepareQueue.PopTask()
+	if !exist {
+		return base.ErrNoTaskInQueue
+	}
+
+	var err error
+	span, ctx := trace.StartSpanFromContext(context.Background(), "disk_repair.popTaskAndPrepare")
+	defer span.Finish()
+
+	defer func() {
+		if err != nil {
+			span.Errorf("prepare task failed  and retry task: task_id[%s], err[%+v]", task.(*proto.VolRepairTask).TaskID, err)
+			mgr.prepareQueue.RetryTask(task.(*proto.VolRepairTask).TaskID)
+		}
+	}()
+
+	//why:avoid to change task in queue
+	t := task.(*proto.VolRepairTask).Copy()
+	span.Infof("pop task: task_id[%s], task[%+v]", t.TaskID, t)
+	// whether vid has another running task
+	err = base.VolTaskLockerInst().TryLock(ctx, t.Vid())
+	if err != nil {
+		span.Warnf("tryLock failed: vid[%d]", t.Vid())
+		return base.ErrVolNotOnlyOneTask
+	}
+	defer func() {
+		if err != nil {
+			span.Errorf("prepare task failed: task_id[%s], err[%+v]", t.TaskID, err)
+			base.VolTaskLockerInst().Unlock(ctx, t.Vid())
+		}
+	}()
+
+	err = mgr.prepareTask(t)
+	if err != nil {
+		span.Errorf("prepare task failed: task_id[%s], err[%+v]", t.TaskID, err)
+		return err
+	}
+
+	span.Infof("prepare task success: task_id[%s]", t.TaskID)
+	return nil
+}
+
+func (mgr *DiskRepairMgr) prepareTask(t *proto.VolRepairTask) error {
+	span, ctx := trace.StartSpanFromContext(
+		context.Background(),
+		"DiskRepairMgr.prepareTask")
+	defer span.Finish()
+
+	span.Infof("start prepare repair task: task_id[%s], task[%+v]", t.TaskID, t)
+
+	volInfo, err := mgr.clusterMgrCli.GetVolumeInfo(ctx, t.Vid())
+	if err != nil {
+		span.Errorf("prepare task get volume info failed: err[%+v]", err)
+		return err
+	}
+
+	// 1.check necessity of generating current task
+	badVuid := t.RepairVuid()
+	if volInfo.VunitLocations[t.BadIdx].Vuid != badVuid {
+		span.Infof("repair task finish in advance: task_id[%s]", t.TaskID)
+		mgr.finishTaskInAdvance(ctx, t)
+		return nil
+	}
+
+	// 2.generate src and destination for task & task persist
+	allocDstVunit, err := base.AllocVunitSafe(ctx, mgr.clusterMgrCli, badVuid, t.Sources)
+	if err != nil {
+		span.Errorf("repair alloc volume unit failed: err[%+v]", err)
+		return err
+	}
+
+	t.CodeMode = volInfo.CodeMode
+	t.Sources = volInfo.VunitLocations
+	t.Destination = allocDstVunit.Location()
+	t.State = proto.RepairStatePrepared
+	base.InsistOn(ctx, "repair prepare task update task tbl", func() error {
+		return mgr.taskTbl.Update(ctx, t)
+	})
+
+	mgr.sendToWorkQueue(t)
+	return nil
+}
+
+func (mgr *DiskRepairMgr) sendToWorkQueue(t *proto.VolRepairTask) {
+	mgr.workQueue.AddPreparedTask(t.BrokenDiskIDC, t.TaskID, t)
+	mgr.prepareQueue.RemoveTask(t.TaskID)
+}
+
+func (mgr *DiskRepairMgr) finishTaskInAdvance(ctx context.Context, t *proto.VolRepairTask) {
+	t.State = proto.RepairStateFinishedInAdvance
+	base.InsistOn(ctx, "repair finish task in advance update task tbl", func() error {
+		return mgr.taskTbl.Update(ctx, t)
+	})
+
+	mgr.finishTaskCounter.Add()
+	mgr.prepareQueue.RemoveTask(t.TaskID)
+	base.VolTaskLockerInst().Unlock(ctx, t.Vid())
+}
+
+func (mgr *DiskRepairMgr) finishTaskLoop() {
+	for {
+		mgr.taskSwitch.WaitEnable()
+		err := mgr.popTaskAndFinish()
+		if err == base.ErrNoTaskInQueue {
+			time.Sleep(time.Duration(finishIntervalS) * time.Second)
+		}
+	}
+}
+
+func (mgr *DiskRepairMgr) popTaskAndFinish() error {
+	_, task, exist := mgr.finishQueue.PopTask()
+	if !exist {
+		return base.ErrNoTaskInQueue
+	}
+
+	span, ctx := trace.StartSpanFromContext(context.Background(), "disk_repair.popTaskAndFinish")
+	defer span.Finish()
+
+	t := task.(*proto.VolRepairTask).Copy()
+	err := mgr.finishTask(ctx, t)
+	if err != nil {
+		span.Errorf("finish task failed: err[%+v]", err)
+		return err
+	}
+
+	span.Infof("finish task success: task_id[%s]", t.TaskID)
+	return nil
+}
+
+func (mgr *DiskRepairMgr) finishTask(ctx context.Context, task *proto.VolRepairTask) (retErr error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	defer func() {
+		if retErr != nil {
+			mgr.finishQueue.RetryTask(task.TaskID)
+		}
+	}()
+
+	if task.State != proto.RepairStateWorkCompleted {
+		span.Panicf("task state not expect: task_id[%s], expect state[%d], actual state[%d]", proto.RepairStateWorkCompleted, task.State)
+	}
+	// complete stage can not make sure to save task info to db,
+	// finish stage make sure to save task info to db
+	// execute update volume mapping relation when can not save task with completed state is dangerous
+	// because if process restart will reload task and redo by worker
+	// worker will write data to chunk which is online
+	base.InsistOn(ctx, "repair finish task update task state completed", func() error {
+		return mgr.taskTbl.Update(ctx, task)
+	})
+
+	newVuid := task.Destination.Vuid
+	oldVuid := task.RepairVuid()
+	err := mgr.clusterMgrCli.UpdateVolume(ctx, newVuid, oldVuid, task.NewDiskId())
+	if err != nil {
+		span.Errorf("update volume failed: err[%+v]", err)
+		return mgr.handleUpdateVolMappingFail(ctx, task, err)
+	}
+
+	task.State = proto.RepairStateFinished
+	base.InsistOn(ctx, "repair finish task update task state finished", func() error {
+		return mgr.taskTbl.Update(ctx, task)
+	})
+
+	mgr.finishTaskCounter.Add()
+	// 1.remove task in memory
+	// 2.release lock of volume task
+	mgr.finishQueue.RemoveTask(task.TaskID)
+	base.VolTaskLockerInst().Unlock(ctx, task.Vid())
+
+	return nil
+}
+
+func (mgr *DiskRepairMgr) handleUpdateVolMappingFail(ctx context.Context, task *proto.VolRepairTask, err error) error {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Infof("handle update vol mapping failed: task_id[%s], state[%d], dest vuid[%d]", task.TaskID, task.State, task.Destination.Vuid)
+
+	code := rpc.DetectStatusCode(err)
+	if code == errors.CodeOldVuidNotMatch {
+		span.Panicf("change volume unit relationship got unexpected err")
+	}
+
+	if base.ShouldAllocAndRedo(code) {
+		span.Infof("realloc vunit and redo: task_id[%s]", task.TaskID)
+
+		newVunit, err := base.AllocVunitSafe(ctx, mgr.clusterMgrCli, task.BadVuid, task.Sources)
+		if err != nil {
+			span.Errorf("realloc failed: vuid[%d], err[%+v]", task.BadVuid, err)
+			return err
+		}
+		task.SetDest(newVunit.Location())
+		task.State = proto.RepairStatePrepared
+		task.WorkerRedoCnt++
+
+		base.InsistOn(ctx, "repair redo task update task tbl", func() error {
+			return mgr.taskTbl.Update(ctx, task)
+		})
+
+		mgr.finishQueue.RemoveTask(task.TaskID)
+		mgr.workQueue.AddPreparedTask(task.BrokenDiskIDC, task.TaskID, task)
+		span.Infof("task redo again:  task_id[%v]", task.TaskID)
+		return nil
+	}
+
+	return err
+}
+
+func (mgr *DiskRepairMgr) checkRepairedAndClearLoop() {
+	t := time.NewTicker(time.Duration(mgr.cfg.CheckTaskIntervalS) * time.Second)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-t.C:
+			mgr.taskSwitch.WaitEnable()
+			mgr.checkRepairedAndClear()
+		case <-mgr.Closer.Done():
+			return
+		}
+	}
+}
+
+func (mgr *DiskRepairMgr) checkRepairedAndClear() {
+	diskID := mgr.getRepairingDiskID()
+	span, ctx := trace.StartSpanFromContext(context.Background(), "disk_repair.checkRepairedAndClear")
+	defer span.Finish()
+
+	if !mgr.hasRepairingDisk() {
+		return
+	}
+
+	span.Infof("check repaired: disk_id[%d]", diskID)
+	if mgr.checkRepaired(ctx, diskID) {
+		err := mgr.clusterMgrCli.SetDiskRepaired(ctx, diskID)
+		if err != nil {
+			return
+		}
+		span.Infof("disk repaired will start clear: disk_id[%d]", diskID)
+		mgr.clearTasksByDiskID(diskID)
+		mgr.emptyRepairingDiskID()
+	}
+}
+
+func (mgr *DiskRepairMgr) checkRepaired(ctx context.Context, diskID proto.DiskID) bool {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Infof("check repaired: disk_id[%d]", diskID)
+
+	tasks, err := mgr.taskTbl.FindByDiskID(ctx, diskID)
+	if err != nil {
+		span.Errorf("check repaired and find task failed: disk_iD[%d], err[%+v]", diskID, err)
+		return false
+	}
+	for _, task := range tasks {
+		if !task.Finished() {
+			return false
+		}
+	}
+
+	span.Infof("disk repair has finished: disk_id[%d], task len[%d]", diskID, len(tasks))
+
+	vunitInfos, err := mgr.clusterMgrCli.ListDiskVolumeUnits(ctx, diskID)
+	if err != nil {
+		span.Errorf("check repaired list disk volume units failed: disk_id[%s], err[%+v]", diskID, err)
+		return false
+	}
+	if len(vunitInfos) != 0 {
+		// it may be occur when migration done and repair tasks generate concurrent, list volume units may not return the migrate unit
+		span.Warnf("clustermgr has some volume unit not repair and revise again: disk_id[%d], volume units len[%d]", diskID, len(vunitInfos))
+		if err = mgr.reviseRepairTask(ctx, diskID); err != nil {
+			span.Errorf("revise repair task failed: err[%+v]", err)
+		}
+		return false
+	}
+	return true
+}
+
+func (mgr *DiskRepairMgr) clearTasksByDiskID(diskID proto.DiskID) {
+	span, ctx := trace.StartSpanFromContext(context.Background(), "disk_repair.clearTasksByDiskID")
+	defer span.Finish()
+
+	base.InsistOn(ctx, "repair clear task by diskID", func() error {
+		return mgr.taskTbl.MarkDeleteByDiskID(ctx, diskID)
+	})
+}
+
+func (mgr *DiskRepairMgr) setRepairingDiskID(diskID proto.DiskID) {
+	mgr.mu.Lock()
+	defer mgr.mu.Unlock()
+	mgr.repairingDiskID = diskID
+	mgr.brokenDisk = nil
+}
+
+func (mgr *DiskRepairMgr) emptyRepairingDiskID() {
+	mgr.mu.Lock()
+	defer mgr.mu.Unlock()
+	mgr.repairingDiskID = base.EmptyDiskID
+}
+
+func (mgr *DiskRepairMgr) getRepairingDiskID() proto.DiskID {
+	mgr.mu.Lock()
+	defer mgr.mu.Unlock()
+	return mgr.repairingDiskID
+}
+
+func (mgr *DiskRepairMgr) hasRepairingDisk() bool {
+	mgr.mu.Lock()
+	defer mgr.mu.Unlock()
+	return mgr.repairingDiskID != base.EmptyDiskID
+}
+
+// AcquireTask acquire repair task
+func (mgr *DiskRepairMgr) AcquireTask(ctx context.Context, idc string) (*proto.VolRepairTask, error) {
+	if !mgr.taskSwitch.Enabled() {
+		return nil, proto.ErrTaskPaused
+	}
+
+	_, task, _ := mgr.workQueue.Acquire(idc)
+	if task != nil {
+		t := task.(*proto.VolRepairTask)
+		return t, nil
+	}
+	return nil, proto.ErrTaskEmpty
+}
+
+// CancelTask cancel repair task
+func (mgr *DiskRepairMgr) CancelTask(ctx context.Context, args *api.CancelTaskArgs) error {
+	span := trace.SpanFromContextSafe(ctx)
+
+	err := mgr.workQueue.Cancel(args.IDC, args.TaskId, args.Src, args.Dest)
+	if err != nil {
+		span.Errorf("cancel repair failed: task_id[%s], err[%+v]", args.TaskId, err)
+	}
+
+	mgr.taskStatsMgr.CancelTask()
+
+	return err
+}
+
+// ReclaimTask reclaim repair task
+func (mgr *DiskRepairMgr) ReclaimTask(ctx context.Context,
+	idc, taskID string,
+	src []proto.VunitLocation,
+	oldDst proto.VunitLocation,
+	newDst *client.AllocVunitInfo) error {
+	span := trace.SpanFromContextSafe(ctx)
+
+	err := mgr.workQueue.Reclaim(idc, taskID, src, oldDst, newDst.Location(), newDst.DiskID)
+	if err != nil {
+		// task has finished,because only complete will remove task from queue
+		span.Errorf("reclaim repair task failed: task_id[%s], err[%+v]", taskID, err)
+		return err
+	}
+
+	task, err := mgr.workQueue.Query(idc, taskID)
+	if err != nil {
+		span.Errorf("found task in workQueue failed: idc[%s], task_id[%s], err[%+v]", idc, taskID, err)
+		return err
+	}
+
+	err = mgr.taskTbl.Update(ctx, task.(*proto.VolRepairTask))
+	if err != nil {
+		span.Warnf("update reclaim task failed: task_id[%s], err[%+v]", taskID, err)
+	}
+
+	mgr.taskStatsMgr.ReclaimTask()
+	return nil
+}
+
+// CompleteTask complete repair task
+func (mgr *DiskRepairMgr) CompleteTask(ctx context.Context, args *api.CompleteTaskArgs) error {
+	span := trace.SpanFromContextSafe(ctx)
+
+	completeTask, err := mgr.workQueue.Complete(args.IDC, args.TaskId, args.Src, args.Dest)
+	if err != nil {
+		span.Errorf("complete repair task failed: task_id[%s], err[%+v]", args.TaskId, err)
+		return err
+	}
+
+	t := completeTask.(*proto.VolRepairTask)
+	t.State = proto.RepairStateWorkCompleted
+
+	mgr.finishQueue.PushTask(args.TaskId, t)
+	// as complete func is face to svr api, so can not loop save task
+	// to db until success, it will make saving task info to be difficult,
+	// that delay saving task info in finish stage is a simply way
+	return nil
+}
+
+// RenewalTask renewal repair task
+func (mgr *DiskRepairMgr) RenewalTask(ctx context.Context, idc, taskID string) error {
+	if !mgr.taskSwitch.Enabled() {
+		// renewal task stopping will touch off worker to stop task
+		return proto.ErrTaskPaused
+	}
+
+	span := trace.SpanFromContextSafe(ctx)
+	err := mgr.workQueue.Renewal(idc, taskID)
+	if err != nil {
+		span.Warnf("renewal repair task failed: task_id[%s], err[%+v]", taskID, err)
+	}
+
+	return err
+}
+
+// ReportWorkerTaskStats reports task stats
+func (mgr *DiskRepairMgr) ReportWorkerTaskStats(st *api.TaskReportArgs) {
+	mgr.taskStatsMgr.ReportWorkerTaskStats(st.TaskId, st.TaskStats, st.IncreaseDataSizeByte, st.IncreaseShardCnt)
+}
+
+// QueryTask return task statistics
+func (mgr *DiskRepairMgr) QueryTask(ctx context.Context, taskID string) (*api.RepairTaskDetail, error) {
+	detail := &api.RepairTaskDetail{}
+	taskInfo, err := mgr.taskTbl.Find(ctx, taskID)
+	if err != nil {
+		return detail, err
+	}
+	detail.TaskInfo = *taskInfo
+
+	detailRunInfo, err := mgr.taskStatsMgr.QueryTaskDetail(taskID)
+	if err != nil {
+		return detail, nil
+	}
+	detail.RunStats = detailRunInfo.Statistics
+	return detail, nil
+}
+
+// StatQueueTaskCnt returns task queue stats
+func (mgr *DiskRepairMgr) StatQueueTaskCnt() (inited, prepared, completed int) {
+	todo, doing := mgr.prepareQueue.StatsTasks()
+	inited = todo + doing
+
+	todo, doing = mgr.workQueue.StatsTasks()
+	prepared = todo + doing
+
+	todo, doing = mgr.finishQueue.StatsTasks()
+	completed = todo + doing
+	return
+}
+
+// Stats returns task stats
+func (mgr *DiskRepairMgr) Stats() api.MigrateTasksStat {
+	preparing, workerDoing, finishing := mgr.StatQueueTaskCnt()
+	finishedCnt := mgr.finishTaskCounter.Show()
+	increaseDataSize, increaseShardCnt := mgr.taskStatsMgr.Counters()
+	return api.MigrateTasksStat{
+		PreparingCnt:   preparing,
+		WorkerDoingCnt: workerDoing,
+		FinishingCnt:   finishing,
+		StatsPerMin: api.PerMinStats{
+			FinishedCnt:    fmt.Sprint(finishedCnt),
+			DataAmountByte: base.DataMountFormat(increaseDataSize),
+			ShardCnt:       fmt.Sprint(increaseShardCnt),
+		},
+	}
+}
+
+// Progress repair manager progress
+func (mgr *DiskRepairMgr) Progress(ctx context.Context) (repairingDiskID proto.DiskID, total, repaired int) {
+	span := trace.SpanFromContextSafe(ctx)
+	repairingDiskID = mgr.getRepairingDiskID()
+	if repairingDiskID == base.EmptyDiskID {
+		return base.EmptyDiskID, 0, 0
+	}
+
+	allTasks, err := mgr.taskTbl.FindByDiskID(ctx, repairingDiskID)
+	if err != nil {
+		span.Errorf("find all task failed: err[%+v]", err)
+		return repairingDiskID, 0, 0
+	}
+	total = len(allTasks)
+	for _, task := range allTasks {
+		if task.Finished() {
+			repaired++
+		}
+	}
+
+	return repairingDiskID, total, repaired
+}

blobstore/scheduler/disk_repairer_test.go

@@ -0,0 +1,721 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package scheduler
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/require"
+
+	api "github.com/cubefs/cubefs/blobstore/api/scheduler"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/scheduler/base"
+	"github.com/cubefs/cubefs/blobstore/scheduler/client"
+	"github.com/cubefs/cubefs/blobstore/testing/mocks"
+)
+
+func newMockVolInfoMap() map[proto.Vid]*client.VolumeInfoSimple {
+	return map[proto.Vid]*client.VolumeInfoSimple{
+		1: MockGenVolInfo(1, codemode.EC6P6, proto.VolumeStatusIdle),
+		2: MockGenVolInfo(2, codemode.EC6P10L2, proto.VolumeStatusIdle),
+		3: MockGenVolInfo(3, codemode.EC6P10L2, proto.VolumeStatusActive),
+		4: MockGenVolInfo(4, codemode.EC6P6, proto.VolumeStatusLock),
+		5: MockGenVolInfo(5, codemode.EC6P6, proto.VolumeStatusLock),
+
+		6: MockGenVolInfo(6, codemode.EC6P6, proto.VolumeStatusLock),
+		7: MockGenVolInfo(7, codemode.EC6P6, proto.VolumeStatusLock),
+	}
+}
+
+func newDiskRepairer(t *testing.T) *DiskRepairMgr {
+	ctr := gomock.NewController(t)
+	clusterMgr := NewMockClusterMgrAPI(ctr)
+	taskSwitch := mocks.NewMockSwitcher(ctr)
+	repairTable := NewMockRepairTaskTable(ctr)
+	conf := &DiskRepairMgrCfg{
+		TaskCommonConfig: base.TaskCommonConfig{
+			CollectTaskIntervalS: 1,
+			CheckTaskIntervalS:   1,
+		},
+	}
+	return NewRepairMgr(conf, taskSwitch, repairTable, clusterMgr)
+}
+
+func TestDiskRepairerLoad(t *testing.T) {
+	{
+		mgr := newDiskRepairer(t)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindAll(any).Return(nil, errMock)
+		err := mgr.Load()
+		require.True(t, errors.Is(err, errMock))
+	}
+	{
+		mgr := newDiskRepairer(t)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindAll(any).Return(nil, nil)
+		err := mgr.Load()
+		require.NoError(t, err)
+	}
+	{
+		mgr := newDiskRepairer(t)
+		t1 := mockGenVolRepairTask(1, proto.RepairStateInited, 1, newMockVolInfoMap())
+		t2 := mockGenVolRepairTask(2, proto.RepairStatePrepared, 2, newMockVolInfoMap())
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindAll(any).Return([]*proto.VolRepairTask{t1, t2}, nil)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return(nil, errMock)
+		err := mgr.Load()
+		require.True(t, errors.Is(err, errMock))
+	}
+	{
+		mgr := newDiskRepairer(t)
+		t1 := mockGenVolRepairTask(1, proto.RepairStateInited, 1, newMockVolInfoMap())
+		t2 := mockGenVolRepairTask(2, proto.RepairStatePrepared, 2, newMockVolInfoMap())
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindAll(any).Return([]*proto.VolRepairTask{t1, t2}, nil)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1}, nil)
+		require.Panics(t, func() {
+			mgr.Load()
+		})
+	}
+	{
+		mgr := newDiskRepairer(t)
+		t1 := mockGenVolRepairTask(1, proto.RepairStateInited, 1, newMockVolInfoMap())
+		t2 := mockGenVolRepairTask(2, proto.RepairStatePrepared, 1, newMockVolInfoMap())
+		t3 := mockGenVolRepairTask(3, proto.RepairStateFinishedInAdvance, 1, newMockVolInfoMap())
+		t4 := mockGenVolRepairTask(4, proto.RepairStateWorkCompleted, 1, newMockVolInfoMap())
+		t5 := mockGenVolRepairTask(5, proto.RepairStateFinished, 1, newMockVolInfoMap())
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindAll(any).Return([]*proto.VolRepairTask{t1, t2, t3, t4, t5}, nil)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1, t2, t3, t4, t5}, nil)
+		err := mgr.Load()
+		require.NoError(t, err)
+	}
+	{
+		mgr := newDiskRepairer(t)
+		t1 := mockGenVolRepairTask(1, proto.RepairStateInited, 1, newMockVolInfoMap())
+		t2 := mockGenVolRepairTask(2, proto.RepairStatePrepared, 2, newMockVolInfoMap())
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindAll(any).Return([]*proto.VolRepairTask{t1, t2}, nil)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1, t2}, nil)
+		require.Panics(t, func() {
+			mgr.Load()
+		})
+	}
+	{
+		mgr := newDiskRepairer(t)
+		t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap())
+		t2 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap())
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindAll(any).Return([]*proto.VolRepairTask{t1, t2}, nil)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1, t2}, nil)
+		require.Panics(t, func() {
+			mgr.Load()
+		})
+	}
+	{
+		mgr := newDiskRepairer(t)
+		t1 := mockGenVolRepairTask(1, proto.RepairState(111), 1, newMockVolInfoMap())
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindAll(any).Return([]*proto.VolRepairTask{t1}, nil)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1}, nil)
+		require.Panics(t, func() {
+			mgr.Load()
+		})
+	}
+}
+
+func TestDiskRepairerRun(t *testing.T) {
+	mgr := newDiskRepairer(t)
+	defer mgr.Close()
+
+	mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().WaitEnable().AnyTimes().Return()
+	mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().AnyTimes().Return(true)
+
+	mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).AnyTimes().Return(nil, errMock)
+	require.True(t, mgr.Enabled())
+	mgr.hasRevised = true
+	mgr.repairingDiskID = proto.DiskID(1)
+
+	mgr.Run()
+	time.Sleep(1 * time.Second)
+}
+
+func TestDiskRepairerCollectTask(t *testing.T) {
+	{
+		mgr := newDiskRepairer(t)
+		mgr.hasRevised = false
+		mgr.repairingDiskID = proto.DiskID(1)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(nil, errMock)
+		mgr.collectTask()
+	}
+	{
+		mgr := newDiskRepairer(t)
+		mgr.hasRevised = false
+		mgr.repairingDiskID = proto.DiskID(1)
+		// genDiskRepairTasks failed
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(&client.DiskInfoSimple{DiskID: mgr.repairingDiskID, Status: proto.DiskStatusBroken}, nil)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return(nil, errMock)
+		mgr.collectTask()
+
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return(nil, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(&client.DiskInfoSimple{DiskID: mgr.repairingDiskID, Status: proto.DiskStatusBroken}, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, errMock)
+		mgr.collectTask()
+
+		// gen task success
+		volume := MockGenVolInfo(10, codemode.EC6P6, proto.VolumeStatusIdle)
+		var units []*client.VunitInfoSimple
+		for _, unit := range volume.VunitLocations {
+			ele := client.VunitInfoSimple{
+				Vuid:   unit.Vuid,
+				DiskID: unit.DiskID,
+			}
+			units = append(units, &ele)
+		}
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return(nil, nil)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Insert(any, any).AnyTimes().Return(nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(&client.DiskInfoSimple{DiskID: mgr.repairingDiskID, Status: proto.DiskStatusBroken}, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().SetDiskRepairing(any, any).Return(nil)
+		mgr.collectTask()
+		todo, doing := mgr.prepareQueue.StatsTasks()
+		require.Equal(t, 12, todo+doing)
+		require.Equal(t, true, mgr.hasRevised)
+	}
+	{
+		mgr := newDiskRepairer(t)
+		mgr.hasRevised = true
+		mgr.repairingDiskID = proto.DiskID(0)
+
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListBrokenDisks(any, any).Return(nil, errMock)
+		mgr.collectTask()
+
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListBrokenDisks(any, any).Return(nil, nil)
+		mgr.collectTask()
+
+		disk1 := &client.DiskInfoSimple{
+			ClusterID:    1,
+			Idc:          "z0",
+			Rack:         "rack1",
+			Host:         "127.0.0.1:8000",
+			Status:       proto.DiskStatusBroken,
+			DiskID:       proto.DiskID(1),
+			FreeChunkCnt: 10,
+			MaxChunkCnt:  700,
+		}
+
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListBrokenDisks(any, any).Return([]*client.DiskInfoSimple{disk1}, nil)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return(nil, errMock)
+		mgr.collectTask()
+
+	}
+	{
+		mgr := newDiskRepairer(t)
+		mgr.hasRevised = true
+		mgr.repairingDiskID = proto.DiskID(0)
+
+		disk1 := &client.DiskInfoSimple{
+			ClusterID:    1,
+			Idc:          "z0",
+			Rack:         "rack1",
+			Host:         "127.0.0.1:8000",
+			Status:       proto.DiskStatusBroken,
+			DiskID:       proto.DiskID(1),
+			FreeChunkCnt: 10,
+			MaxChunkCnt:  700,
+		}
+
+		volume := MockGenVolInfo(10, codemode.EC6P6, proto.VolumeStatusIdle)
+		var units []*client.VunitInfoSimple
+		for _, unit := range volume.VunitLocations {
+			ele := client.VunitInfoSimple{
+				Vuid:   unit.Vuid,
+				DiskID: unit.DiskID,
+			}
+			units = append(units, &ele)
+		}
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListBrokenDisks(any, any).Return([]*client.DiskInfoSimple{disk1}, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().SetDiskRepairing(any, any).Return(nil)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return(nil, nil)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Insert(any, any).AnyTimes().Return(nil)
+		mgr.collectTask()
+		todo, doing := mgr.prepareQueue.StatsTasks()
+		require.Equal(t, disk1.DiskID, mgr.repairingDiskID)
+		require.Equal(t, 12, todo+doing)
+	}
+	{
+		mgr := newDiskRepairer(t)
+		mgr.hasRevised = true
+		mgr.repairingDiskID = proto.DiskID(0)
+
+		disk1 := &client.DiskInfoSimple{
+			ClusterID:    1,
+			Idc:          "z0",
+			Rack:         "rack1",
+			Host:         "127.0.0.1:8000",
+			Status:       proto.DiskStatusBroken,
+			DiskID:       proto.DiskID(1),
+			FreeChunkCnt: 10,
+			MaxChunkCnt:  700,
+		}
+
+		volume := MockGenVolInfo(10, codemode.EC6P6, proto.VolumeStatusIdle)
+		var units []*client.VunitInfoSimple
+		for _, unit := range volume.VunitLocations {
+			ele := client.VunitInfoSimple{
+				Vuid:   unit.Vuid,
+				DiskID: unit.DiskID,
+			}
+			units = append(units, &ele)
+		}
+		t1 := &proto.VolRepairTask{
+			TaskID:  base.GenTaskID("disk-repair", volume.Vid),
+			BadVuid: units[0].Vuid,
+		}
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListBrokenDisks(any, any).Return([]*client.DiskInfoSimple{disk1}, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().SetDiskRepairing(any, any).Return(nil)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1}, nil)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Insert(any, any).AnyTimes().Return(nil)
+		mgr.collectTask()
+		todo, doing := mgr.prepareQueue.StatsTasks()
+		require.Equal(t, disk1.DiskID, mgr.repairingDiskID)
+		require.Equal(t, 11, todo+doing)
+	}
+}
+
+func TestDiskRepairerPopTaskAndPrepare(t *testing.T) {
+	{
+		mgr := newDiskRepairer(t)
+		err := mgr.popTaskAndPrepare()
+		require.True(t, errors.Is(err, base.ErrNoTaskInQueue))
+	}
+	{
+		mgr := newDiskRepairer(t)
+		mgr.hasRevised = true
+		mgr.repairingDiskID = proto.DiskID(0)
+
+		disk1 := &client.DiskInfoSimple{
+			ClusterID:    1,
+			Idc:          "z0",
+			Rack:         "rack1",
+			Host:         "127.0.0.1:8000",
+			Status:       proto.DiskStatusBroken,
+			DiskID:       proto.DiskID(1),
+			FreeChunkCnt: 10,
+			MaxChunkCnt:  700,
+		}
+
+		volume := MockGenVolInfo(10, codemode.EC6P6, proto.VolumeStatusIdle)
+		var units []*client.VunitInfoSimple
+		for _, unit := range volume.VunitLocations {
+			ele := client.VunitInfoSimple{
+				Vuid:   unit.Vuid,
+				DiskID: unit.DiskID,
+			}
+			units = append(units, &ele)
+		}
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListBrokenDisks(any, any).Return([]*client.DiskInfoSimple{disk1}, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().SetDiskRepairing(any, any).Return(nil)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return(nil, nil)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Insert(any, any).AnyTimes().Return(nil)
+		mgr.collectTask()
+		todo, doing := mgr.prepareQueue.StatsTasks()
+		require.Equal(t, disk1.DiskID, mgr.repairingDiskID)
+		require.Equal(t, 12, todo+doing)
+
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(nil, errMock)
+		err := mgr.popTaskAndPrepare()
+		require.True(t, errors.Is(err, errMock))
+
+		// finish in advance
+		volume.VunitLocations[0].Vuid = volume.VunitLocations[0].Vuid + 1
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(volume, nil)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Update(any, any).Return(nil)
+		err = mgr.popTaskAndPrepare()
+		todo, doing = mgr.prepareQueue.StatsTasks()
+
+		require.NoError(t, err)
+		require.Equal(t, 11, todo+doing)
+
+		// alloc volume unit failed
+		volume.VunitLocations[0].Vuid = volume.VunitLocations[0].Vuid - 1
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(any, any).Return(nil, errMock)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(volume, nil)
+		err = mgr.popTaskAndPrepare()
+		require.True(t, errors.Is(err, errMock))
+
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(any, any).DoAndReturn(func(ctx context.Context, vuid proto.Vuid) (*client.AllocVunitInfo, error) {
+			vid := vuid.Vid()
+			idx := vuid.Index()
+			epoch := vuid.Epoch()
+			epoch++
+			newVuid, _ := proto.NewVuid(vid, idx, epoch)
+			return &client.AllocVunitInfo{
+				VunitLocation: proto.VunitLocation{Vuid: newVuid},
+			}, nil
+		})
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(volume, nil)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Update(any, any).Return(nil)
+		err = mgr.popTaskAndPrepare()
+		require.NoError(t, err)
+
+		todo, doing = mgr.prepareQueue.StatsTasks()
+		require.Equal(t, 10, todo+doing)
+		todo, doing = mgr.workQueue.StatsTasks()
+		require.Equal(t, 1, todo+doing)
+	}
+}
+
+func TestDiskRepairerPopTaskAndFinish(t *testing.T) {
+	{
+		mgr := newDiskRepairer(t)
+		err := mgr.popTaskAndFinish()
+		require.True(t, errors.Is(err, base.ErrNoTaskInQueue))
+	}
+	{
+		mgr := newDiskRepairer(t)
+		t1 := mockGenVolRepairTask(1, proto.RepairStateFinished, 1, newMockVolInfoMap())
+		mgr.finishQueue.PushTask(t1.TaskID, t1)
+		require.Panics(t, func() {
+			mgr.popTaskAndFinish()
+		})
+	}
+	{
+		mgr := newDiskRepairer(t)
+		t1 := mockGenVolRepairTask(1, proto.RepairStateWorkCompleted, 1, newMockVolInfoMap())
+		mgr.finishQueue.PushTask(t1.TaskID, t1)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Update(any, any).Return(nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateVolume(any, any, any, any).Return(errMock)
+		err := mgr.popTaskAndFinish()
+		require.True(t, errors.Is(err, errMock))
+	}
+	{
+		mgr := newDiskRepairer(t)
+		t1 := mockGenVolRepairTask(1, proto.RepairStateWorkCompleted, 1, newMockVolInfoMap())
+		mgr.finishQueue.PushTask(t1.TaskID, t1)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Update(any, any).Return(nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateVolume(any, any, any, any).Return(errcode.ErrOldVuidNotMatch)
+		require.Panics(t, func() {
+			mgr.popTaskAndFinish()
+		})
+	}
+	{
+		mgr := newDiskRepairer(t)
+		t1 := mockGenVolRepairTask(1, proto.RepairStateWorkCompleted, 1, newMockVolInfoMap())
+		mgr.finishQueue.PushTask(t1.TaskID, t1)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Update(any, any).Return(nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateVolume(any, any, any, any).Return(errcode.ErrNewVuidNotMatch)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(any, any).Return(nil, errMock)
+		err := mgr.popTaskAndFinish()
+		require.True(t, errors.Is(err, errMock))
+	}
+	{
+		mgr := newDiskRepairer(t)
+		t1 := mockGenVolRepairTask(1, proto.RepairStateWorkCompleted, 1, newMockVolInfoMap())
+		mgr.finishQueue.PushTask(t1.TaskID, t1)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Update(any, any).Times(2).Return(nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateVolume(any, any, any, any).Return(errcode.ErrNewVuidNotMatch)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(any, any).DoAndReturn(func(ctx context.Context, vuid proto.Vuid) (*client.AllocVunitInfo, error) {
+			vid := vuid.Vid()
+			idx := vuid.Index()
+			epoch := vuid.Epoch()
+			epoch++
+			newVuid, _ := proto.NewVuid(vid, idx, epoch)
+			return &client.AllocVunitInfo{
+				VunitLocation: proto.VunitLocation{Vuid: newVuid},
+			}, nil
+		})
+		err := mgr.popTaskAndFinish()
+		require.NoError(t, err)
+		todo, doing := mgr.finishQueue.StatsTasks()
+		require.Equal(t, 0, todo+doing)
+		todo, doing = mgr.workQueue.StatsTasks()
+		require.Equal(t, 1, todo+doing)
+	}
+	{
+		mgr := newDiskRepairer(t)
+		t1 := mockGenVolRepairTask(1, proto.RepairStateWorkCompleted, 1, newMockVolInfoMap())
+		mgr.finishQueue.PushTask(t1.TaskID, t1)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Update(any, any).Times(2).Return(nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateVolume(any, any, any, any).Return(errcode.ErrStatChunkFailed)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(any, any).DoAndReturn(func(ctx context.Context, vuid proto.Vuid) (*client.AllocVunitInfo, error) {
+			vid := vuid.Vid()
+			idx := vuid.Index()
+			epoch := vuid.Epoch()
+			epoch++
+			newVuid, _ := proto.NewVuid(vid, idx, epoch)
+			return &client.AllocVunitInfo{
+				VunitLocation: proto.VunitLocation{Vuid: newVuid},
+			}, nil
+		})
+		err := mgr.popTaskAndFinish()
+		require.NoError(t, err)
+		todo, doing := mgr.finishQueue.StatsTasks()
+		require.Equal(t, 0, todo+doing)
+		todo, doing = mgr.workQueue.StatsTasks()
+		require.Equal(t, 1, todo+doing)
+	}
+	{
+		mgr := newDiskRepairer(t)
+		t1 := mockGenVolRepairTask(1, proto.RepairStateWorkCompleted, 1, newMockVolInfoMap())
+		mgr.finishQueue.PushTask(t1.TaskID, t1)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Update(any, any).Times(2).Return(nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateVolume(any, any, any, any).Return(nil)
+		err := mgr.popTaskAndFinish()
+		require.NoError(t, err)
+		todo, doing := mgr.finishQueue.StatsTasks()
+		require.Equal(t, 0, todo+doing)
+	}
+}
+
+func TestDiskRepairerCheckRepairedAndClear(t *testing.T) {
+	{
+		mgr := newDiskRepairer(t)
+		mgr.checkRepairedAndClear()
+	}
+	{
+		mgr := newDiskRepairer(t)
+		mgr.repairingDiskID = proto.DiskID(1)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return(nil, errMock)
+		mgr.checkRepairedAndClear()
+	}
+	{
+		mgr := newDiskRepairer(t)
+		mgr.repairingDiskID = proto.DiskID(1)
+		volume := MockGenVolInfo(10, codemode.EC6P6, proto.VolumeStatusIdle)
+		var units []*client.VunitInfoSimple
+		for _, unit := range volume.VunitLocations {
+			ele := client.VunitInfoSimple{
+				Vuid:   unit.Vuid,
+				DiskID: unit.DiskID,
+			}
+			units = append(units, &ele)
+		}
+		t1 := &proto.VolRepairTask{
+			TaskID:  base.GenTaskID("disk-repair", volume.Vid),
+			BadVuid: units[0].Vuid,
+		}
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1}, nil)
+		mgr.checkRepairedAndClear()
+
+		t1.State = proto.RepairStateFinished
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1}, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, errMock)
+		mgr.checkRepairedAndClear()
+
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1}, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(nil, errMock)
+		mgr.checkRepairedAndClear()
+
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1}, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().SetDiskRepaired(any, any).Return(errMock)
+		mgr.checkRepairedAndClear()
+
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1}, nil)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().MarkDeleteByDiskID(any, any).Return(nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().SetDiskRepaired(any, any).Return(nil)
+		mgr.checkRepairedAndClear()
+		require.False(t, mgr.hasRepairingDisk())
+	}
+}
+
+func TestDiskRepairerAcquireTask(t *testing.T) {
+	ctx := context.Background()
+	idc := "z0"
+	{
+		mgr := newDiskRepairer(t)
+		mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(false)
+		_, err := mgr.AcquireTask(ctx, idc)
+		require.True(t, errors.Is(err, proto.ErrTaskPaused))
+	}
+	{
+		mgr := newDiskRepairer(t)
+		mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(true)
+		_, err := mgr.AcquireTask(ctx, idc)
+		require.True(t, errors.Is(err, proto.ErrTaskEmpty))
+	}
+	{
+		mgr := newDiskRepairer(t)
+		mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(true)
+		t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap())
+		mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1)
+		_, err := mgr.AcquireTask(ctx, idc)
+		require.NoError(t, err)
+	}
+}
+
+func TestDiskRepairerCancelTask(t *testing.T) {
+	ctx := context.Background()
+	idc := "z0"
+	{
+		mgr := newDiskRepairer(t)
+		err := mgr.CancelTask(ctx, &api.CancelTaskArgs{})
+		require.Error(t, err)
+	}
+	{
+		mgr := newDiskRepairer(t)
+		t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap())
+		mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1)
+
+		err := mgr.CancelTask(ctx, &api.CancelTaskArgs{})
+		require.Error(t, err)
+	}
+}
+
+func TestDiskRepairerReclaimTask(t *testing.T) {
+	ctx := context.Background()
+	idc := "z0"
+	{
+		mgr := newDiskRepairer(t)
+		t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap())
+		err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{})
+		require.Error(t, err)
+	}
+	{
+		mgr := newDiskRepairer(t)
+		t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap())
+		mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Update(any, any).Return(errMock)
+		err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{})
+		require.NoError(t, err)
+	}
+	{
+		mgr := newDiskRepairer(t)
+		t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap())
+		mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Update(any, any).Return(nil)
+		err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{})
+		require.NoError(t, err)
+	}
+}
+
+func TestDiskRepairerCompleteTask(t *testing.T) {
+	ctx := context.Background()
+	idc := "z0"
+	{
+		mgr := newDiskRepairer(t)
+		t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap())
+		err := mgr.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: idc, TaskId: t1.TaskID, Src: t1.Sources, Dest: t1.Destination})
+		require.Error(t, err)
+	}
+	{
+		mgr := newDiskRepairer(t)
+		t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap())
+		mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1)
+		err := mgr.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: idc, TaskId: t1.TaskID, Src: t1.Sources, Dest: t1.Destination})
+		require.NoError(t, err)
+		todo, doing := mgr.finishQueue.StatsTasks()
+		require.Equal(t, 1, todo+doing)
+		todo, doing = mgr.workQueue.StatsTasks()
+		require.Equal(t, 0, todo+doing)
+	}
+}
+
+func TestDiskRepairerRenewalTask(t *testing.T) {
+	ctx := context.Background()
+	idc := "z0"
+	{
+		mgr := newDiskRepairer(t)
+		mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(false)
+		err := mgr.RenewalTask(ctx, idc, "")
+		require.True(t, errors.Is(err, proto.ErrTaskPaused))
+	}
+	{
+		mgr := newDiskRepairer(t)
+		mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(true)
+		err := mgr.RenewalTask(ctx, idc, "")
+		require.Error(t, err)
+	}
+	{
+		mgr := newDiskRepairer(t)
+		mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(true)
+		t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap())
+		mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1)
+		err := mgr.RenewalTask(ctx, idc, t1.TaskID)
+		require.NoError(t, err)
+	}
+}
+
+func TestDiskRepairerStats(t *testing.T) {
+	mgr := newDiskRepairer(t)
+	mgr.Stats()
+}
+
+func TestDiskRepairerStatQueueTaskCnt(t *testing.T) {
+	mgr := newDiskRepairer(t)
+	inited, prepared, completed := mgr.StatQueueTaskCnt()
+	require.Equal(t, 0, inited)
+	require.Equal(t, 0, prepared)
+	require.Equal(t, 0, completed)
+}
+
+func TestDiskRepairerQueryTask(t *testing.T) {
+	ctx := context.Background()
+	taskID := "task"
+	{
+		mgr := newDiskRepairer(t)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Find(any, any).Return(nil, errMock)
+		_, err := mgr.QueryTask(ctx, taskID)
+		require.True(t, errors.Is(err, errMock))
+	}
+	{
+		mgr := newDiskRepairer(t)
+		t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap())
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Find(any, any).Return(t1, nil)
+		_, err := mgr.QueryTask(ctx, taskID)
+		require.NoError(t, err)
+	}
+}
+
+func TestDiskRepairerReportWorkerTaskStats(t *testing.T) {
+	mgr := newDiskRepairer(t)
+	mgr.ReportWorkerTaskStats(&api.TaskReportArgs{
+		TaskId:               "task",
+		IncreaseDataSizeByte: 1,
+		IncreaseShardCnt:     1,
+	})
+}
+
+func TestDiskRepairerProgress(t *testing.T) {
+	ctx := context.Background()
+	{
+		mgr := newDiskRepairer(t)
+		diskID, _, _ := mgr.Progress(ctx)
+		require.Equal(t, base.EmptyDiskID, diskID)
+	}
+	{
+		mgr := newDiskRepairer(t)
+		mgr.repairingDiskID = proto.DiskID(1)
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return(nil, errMock)
+		diskID, _, _ := mgr.Progress(ctx)
+		require.Equal(t, proto.DiskID(1), diskID)
+	}
+	{
+		mgr := newDiskRepairer(t)
+		mgr.repairingDiskID = proto.DiskID(1)
+		t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap())
+		t2 := mockGenVolRepairTask(2, proto.RepairStateFinished, 1, newMockVolInfoMap())
+		t3 := mockGenVolRepairTask(3, proto.RepairStateFinishedInAdvance, 1, newMockVolInfoMap())
+		mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1, t2, t3}, nil)
+		diskID, total, repaired := mgr.Progress(ctx)
+		require.Equal(t, proto.DiskID(1), diskID)
+		require.Equal(t, 3, total)
+		require.Equal(t, 2, repaired)
+	}
+}

blobstore/scheduler/manual_migrater.go

@@ -0,0 +1,95 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package scheduler
+
+import (
+	"context"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/taskswitch"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/scheduler/base"
+	"github.com/cubefs/cubefs/blobstore/scheduler/client"
+	"github.com/cubefs/cubefs/blobstore/scheduler/db"
+)
+
+// IManualMigrater interface of manual migrater
+type IManualMigrater interface {
+	Migrator
+	AddManualTask(ctx context.Context, vuid proto.Vuid, forbiddenDirectDownload bool) (err error)
+}
+
+// ManualMigrateMgr manual migrate manager
+type ManualMigrateMgr struct {
+	IMigrater
+
+	clusterMgrCli client.ClusterMgrAPI
+}
+
+// NewManualMigrateMgr returns manual migrate manager
+func NewManualMigrateMgr(clusterMgrCli client.ClusterMgrAPI, volumeUpdater client.IVolumeUpdater,
+	taskTbl db.IMigrateTaskTable, clusterID proto.ClusterID) *ManualMigrateMgr {
+	mgr := &ManualMigrateMgr{
+		clusterMgrCli: clusterMgrCli,
+	}
+	cfg := defaultMigrateConfig(clusterID)
+
+	mgr.IMigrater = NewMigrateMgr(clusterMgrCli, volumeUpdater, taskswitch.NewEnabledTaskSwitch(), taskTbl,
+		&cfg, proto.ManualMigrateType, clusterID)
+	mgr.IMigrater.SetLockFailHandleFunc(mgr.IMigrater.FinishTaskInAdvanceWhenLockFail)
+	return mgr
+}
+
+// AddManualTask add manual migrate task
+func (mgr *ManualMigrateMgr) AddManualTask(ctx context.Context, vuid proto.Vuid, forbiddenDirectDownload bool) (err error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	volume, err := mgr.clusterMgrCli.GetVolumeInfo(ctx, vuid.Vid())
+	if err != nil {
+		span.Errorf("get volume failed: vid[%d], err[%+v]", vuid.Vid(), err)
+		return err
+	}
+	diskID := volume.VunitLocations[vuid.Index()].DiskID
+	disk, err := mgr.clusterMgrCli.GetDiskInfo(ctx, diskID)
+	if err != nil {
+		span.Errorf("get disk info failed:  disk_id[%d], err[%+v]", err)
+		return err
+	}
+
+	task := &proto.MigrateTask{
+		TaskID:                  mgr.genUniqTaskID(vuid.Vid()),
+		State:                   proto.MigrateStateInited,
+		SourceIdc:               disk.Idc,
+		SourceDiskID:            disk.DiskID,
+		SourceVuid:              vuid,
+		ForbiddenDirectDownload: forbiddenDirectDownload,
+	}
+	mgr.IMigrater.AddTask(ctx, task)
+
+	span.Debugf("add manual migrate task success: task_info[%+v]", task)
+	return nil
+}
+
+func (mgr *ManualMigrateMgr) genUniqTaskID(vid proto.Vid) string {
+	return base.GenTaskID("manual_migrate", vid)
+}
+
+func defaultMigrateConfig(clusterID proto.ClusterID) MigrateConfig {
+	cfg := MigrateConfig{
+		ClusterID: clusterID,
+	}
+	cfg.CheckAndFix()
+	return cfg
+}

blobstore/scheduler/manual_migrater_test.go

@@ -0,0 +1,134 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package scheduler
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/require"
+
+	api "github.com/cubefs/cubefs/blobstore/api/scheduler"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/scheduler/client"
+)
+
+func newManualMigrater(t *testing.T) *ManualMigrateMgr {
+	ctr := gomock.NewController(t)
+	clusterMgr := NewMockClusterMgrAPI(ctr)
+	volumeUpdater := NewMockVolumeUpdater(ctr)
+	migrateTable := NewMockMigrateTaskTable(ctr)
+	migrater := NewMockMigrater(ctr)
+	mgr := NewManualMigrateMgr(clusterMgr, volumeUpdater, migrateTable, proto.ClusterID(1))
+	mgr.IMigrater = migrater
+	return mgr
+}
+
+func TestManualMigrateLoad(t *testing.T) {
+	mgr := newManualMigrater(t)
+	mgr.IMigrater.(*MockMigrater).EXPECT().Load().Return(nil)
+	err := mgr.Load()
+	require.NoError(t, err)
+}
+
+func TestManualMigrateRun(t *testing.T) {
+	mgr := newManualMigrater(t)
+	mgr.IMigrater.(*MockMigrater).EXPECT().Run().Return()
+	mgr.Run()
+}
+
+func TestManualMigrateAddTask(t *testing.T) {
+	ctx := context.Background()
+	{
+		mgr := newManualMigrater(t)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(nil, errMock)
+		err := mgr.AddManualTask(ctx, proto.Vuid(1), false)
+		require.True(t, errors.Is(err, errMock))
+	}
+	{
+		mgr := newManualMigrater(t)
+		volume := MockGenVolInfo(10001, codemode.EC6P6, proto.VolumeStatusIdle)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(volume, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(nil, errMock)
+		err := mgr.AddManualTask(ctx, proto.Vuid(1), false)
+		require.True(t, errors.Is(err, errMock))
+	}
+	{
+		mgr := newManualMigrater(t)
+		volume := MockGenVolInfo(10001, codemode.EC6P6, proto.VolumeStatusIdle)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(volume, nil)
+		mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(&client.DiskInfoSimple{}, nil)
+		mgr.IMigrater.(*MockMigrater).EXPECT().AddTask(any, any).Return()
+		err := mgr.AddManualTask(ctx, proto.Vuid(1), false)
+		require.NoError(t, err)
+	}
+}
+
+func TestManualMigrateAcquireTask(t *testing.T) {
+	ctx := context.Background()
+	idc := "z0"
+	mgr := newManualMigrater(t)
+	mgr.IMigrater.(*MockMigrater).EXPECT().AcquireTask(any, any).Return(&proto.MigrateTask{}, nil)
+	_, err := mgr.AcquireTask(ctx, idc)
+	require.NoError(t, err)
+}
+
+func TestManualMigrateCancelTask(t *testing.T) {
+	ctx := context.Background()
+	mgr := newManualMigrater(t)
+	mgr.IMigrater.(*MockMigrater).EXPECT().CancelTask(any, any).Return(nil)
+	err := mgr.CancelTask(ctx, &api.CancelTaskArgs{})
+	require.NoError(t, err)
+}
+
+func TestManualMigrateReclaimTask(t *testing.T) {
+	ctx := context.Background()
+	idc := "z0"
+	mgr := newManualMigrater(t)
+	mgr.IMigrater.(*MockMigrater).EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil)
+	t1 := mockGenMigrateTask(idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap)
+	err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{})
+	require.NoError(t, err)
+}
+
+func TestManualMigrateCompleteTask(t *testing.T) {
+	ctx := context.Background()
+	idc := "z0"
+	mgr := newManualMigrater(t)
+	mgr.IMigrater.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(nil)
+	t1 := mockGenMigrateTask(idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap)
+	err := mgr.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: idc, TaskId: t1.TaskID, Src: t1.Sources, Dest: t1.Destination})
+	require.NoError(t, err)
+
+	mgr.IMigrater.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(errMock)
+	err = mgr.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: idc, TaskId: t1.TaskID, Src: t1.Sources, Dest: t1.Destination})
+	require.True(t, errors.Is(err, errMock))
+}
+
+func TestManualMigrateRenewalTask(t *testing.T) {
+	ctx := context.Background()
+	idc := "z0"
+	mgr := newManualMigrater(t)
+	mgr.IMigrater.(*MockMigrater).EXPECT().RenewalTask(any, any, any).Return(nil)
+	err := mgr.RenewalTask(ctx, idc, "")
+	require.NoError(t, err)
+
+	mgr.IMigrater.(*MockMigrater).EXPECT().RenewalTask(any, any, any).Return(errMock)
+	err = mgr.RenewalTask(ctx, idc, "")
+	require.True(t, errors.Is(err, errMock))
+}