diff --git a/blobstore/api/proxy/allocator.go b/blobstore/api/proxy/allocator.go new file mode 100644 index 000000000..17ba9b7a4 --- /dev/null +++ b/blobstore/api/proxy/allocator.go @@ -0,0 +1,50 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package proxy + +import ( + "context" + + "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +type Allocator interface { + VolumeAlloc(ctx context.Context, host string, args *AllocVolsArgs) (ret []AllocRet, err error) +} + +type ListVolsArgs struct { + CodeMode codemode.CodeMode `json:"code_mode"` +} + +type VolumeList struct { + Vids []proto.Vid `json:"vids"` + Volumes []clustermgr.AllocVolumeInfo `json:"volumes"` +} + +type AllocRet struct { + BidStart proto.BlobID `json:"bid_start"` + BidEnd proto.BlobID `json:"bid_end"` + Vid proto.Vid `json:"vid"` +} + +type AllocVolsArgs struct { + Fsize uint64 `json:"fsize"` + CodeMode codemode.CodeMode `json:"code_mode"` + BidCount uint64 `json:"bid_count"` + Excludes []proto.Vid `json:"excludes"` + Discards []proto.Vid `json:"discards"` +} diff --git a/blobstore/api/proxy/client.go b/blobstore/api/proxy/client.go new file mode 100644 index 000000000..a0f10a685 --- /dev/null +++ b/blobstore/api/proxy/client.go @@ -0,0 +1,52 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package proxy + +import ( + "context" + + "github.com/cubefs/cubefs/blobstore/common/rpc" +) + +type Config struct { + rpc.Config +} + +type client struct { + rpc.Client +} + +type Client interface { + MsgSender + Allocator +} + +func New(cfg *Config) Client { + return &client{rpc.NewClient(&cfg.Config)} +} + +func (c *client) VolumeAlloc(ctx context.Context, host string, args *AllocVolsArgs) (ret []AllocRet, err error) { + ret = make([]AllocRet, 0) + err = c.PostWith(ctx, host+"/volume/alloc", &ret, args) + return +} + +func (c *client) SendShardRepairMsg(ctx context.Context, host string, args *ShardRepairArgs) error { + return c.PostWith(ctx, host+"/repairmsg", nil, args) +} + +func (c *client) SendDeleteMsg(ctx context.Context, host string, args *DeleteArgs) error { + return c.PostWith(ctx, host+"/deletemsg", nil, args) +} diff --git a/blobstore/api/proxy/client_test.go b/blobstore/api/proxy/client_test.go new file mode 100644 index 000000000..444443295 --- /dev/null +++ b/blobstore/api/proxy/client_test.go @@ -0,0 +1,154 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package proxy + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/common/rpc" +) + +func TestClient_VolumeAlloc(t *testing.T) { + cli := New(&Config{}) + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer mockServer.Close() + args := &AllocVolsArgs{Fsize: 8, CodeMode: 2, BidCount: 1} + ret, err := cli.VolumeAlloc(context.Background(), mockServer.URL, args) + require.NoError(t, err) + require.Equal(t, make([]AllocRet, 0), ret) +} + +func TestLbClient_SendShardRepairMsg(t *testing.T) { + mqproxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(200) + })) + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(200) + w.Write([]byte(fmt.Sprintf(`{"nodes":[{"cluster_id":1,"name":"PROXY","host":"%s","idc":"z0"}]}`, mqproxyServer.URL))) + })) + defer func() { + s.Close() + mqproxyServer.Close() + }() + + cmCfg := clustermgr.Config{LbConfig: rpc.LbConfig{ + Hosts: []string{s.URL}, + }} + cm := clustermgr.New(&cmCfg) + cli := NewMQLbClient(&LbConfig{ + Config: Config{}, + RetryHostsCnt: 0, + HostSyncIntervalMs: 0, + }, cm, 1) + + err := cli.SendShardRepairMsg(context.Background(), &ShardRepairArgs{ + ClusterID: 0, + Bid: 0, + Vid: 0, + BadIdxes: nil, + Reason: "test", + }) + require.NoError(t, err) +} + +func TestLbClient_SendShardRepairMsg_failed(t *testing.T) { + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(200) + w.Write([]byte(fmt.Sprintf(`{"nodes":[{"cluster_id":1,"name":"PROXY","host":"%s","idc":"z0"}]}`, "abc.com"))) + })) + defer s.Close() + + cmCfg := clustermgr.Config{LbConfig: rpc.LbConfig{ + Hosts: []string{s.URL}, + }} + cm := clustermgr.New(&cmCfg) + cli := NewMQLbClient(&LbConfig{ + Config: Config{}, + RetryHostsCnt: 0, + HostSyncIntervalMs: 0, + }, cm, 1) + + err := cli.SendShardRepairMsg(context.Background(), &ShardRepairArgs{ + ClusterID: 0, + Bid: 0, + Vid: 0, + BadIdxes: nil, + Reason: "test", + }) + require.Error(t, err) +} + +func TestLbClient_BlobDelete_failed(t *testing.T) { + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(200) + w.Write([]byte(fmt.Sprintf(`{"nodes":[{"cluster_id":1,"name":"PROXY","host":"%s","idc":"z0"}]}`, "abc.com"))) + })) + defer s.Close() + + cmCfg := clustermgr.Config{LbConfig: rpc.LbConfig{ + Hosts: []string{s.URL}, + }} + cm := clustermgr.New(&cmCfg) + cli := NewMQLbClient(&LbConfig{ + Config: Config{}, + RetryHostsCnt: 0, + HostSyncIntervalMs: 0, + }, cm, 1) + + err := cli.SendDeleteMsg(context.Background(), &DeleteArgs{ + ClusterID: 0, + Blobs: nil, + }) + require.Error(t, err) +} + +func TestLbClient_BlobDelete(t *testing.T) { + mqproxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(200) + })) + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(200) + w.Write([]byte(fmt.Sprintf(`{"nodes":[{"cluster_id":1,"name":"PROXY","host":"%s","idc":"z0"}]}`, mqproxyServer.URL))) + })) + defer func() { + s.Close() + mqproxyServer.Close() + }() + + cmCfg := clustermgr.Config{LbConfig: rpc.LbConfig{ + Hosts: []string{s.URL}, + }} + cm := clustermgr.New(&cmCfg) + cli := NewMQLbClient(&LbConfig{ + Config: Config{}, + RetryHostsCnt: 0, + HostSyncIntervalMs: 0, + }, cm, 1) + + err := cli.SendDeleteMsg(context.Background(), &DeleteArgs{ + ClusterID: 0, + Blobs: nil, + }) + require.NoError(t, err) +} diff --git a/blobstore/api/proxy/lb_client.go b/blobstore/api/proxy/lb_client.go new file mode 100644 index 000000000..37920bffe --- /dev/null +++ b/blobstore/api/proxy/lb_client.go @@ -0,0 +1,120 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package proxy + +import ( + "context" + + "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/util/errors" + "github.com/cubefs/cubefs/blobstore/util/selector" +) + +var errNoServiceAvailable = errors.New("no service available") + +type LbConfig struct { + Config + + RetryHostsCnt int `json:"retry_hosts_cnt"` + HostSyncIntervalMs int64 `json:"host_sync_interval_ms"` +} + +type lbClient struct { + Client + selector selector.Selector + retryHostsCnt int +} + +func NewMQLbClient(cfg *LbConfig, service clustermgr.APIService, clusterID proto.ClusterID) LbMsgSender { + hostGetter := func() ([]string, error) { + svrInfos, err := service.GetService(context.Background(), clustermgr.GetServiceArgs{Name: proto.ServiceNameProxy}) + if err != nil { + return nil, err + } + + var hosts []string + for _, s := range svrInfos.Nodes { + if clusterID == proto.ClusterID(s.ClusterID) { + hosts = append(hosts, s.Host) + } + } + if len(hosts) == 0 { + return nil, errNoServiceAvailable + } + + return hosts, nil + } + + if cfg.HostSyncIntervalMs == 0 { + cfg.HostSyncIntervalMs = 1000 + } + if cfg.RetryHostsCnt == 0 { + cfg.RetryHostsCnt = 1 + } + + return &lbClient{ + retryHostsCnt: cfg.RetryHostsCnt, + Client: New(&cfg.Config), + selector: selector.NewSelectorWithGetter(cfg.HostSyncIntervalMs, hostGetter), + } +} + +func (c *lbClient) SendDeleteMsg(ctx context.Context, args *DeleteArgs) (err error) { + span := trace.SpanFromContextSafe(ctx) + + hosts := c.selector.GetRandomN(c.retryHostsCnt) + if len(hosts) == 0 { + return errNoServiceAvailable + } + for _, h := range hosts { + err = c.Client.SendDeleteMsg(ctx, h, args) + if err == nil || !shouldRetry(err) { + return err + } + span.Errorf("send delete message failed, host: %s, args: %+v, err:%+v", h, args, err) + } + + return err +} + +func (c *lbClient) SendShardRepairMsg(ctx context.Context, args *ShardRepairArgs) (err error) { + span := trace.SpanFromContextSafe(ctx) + ctx = trace.ContextWithSpan(ctx, span) + + hosts := c.selector.GetRandomN(c.retryHostsCnt) + if len(hosts) == 0 { + return errNoServiceAvailable + } + for _, h := range hosts { + err = c.Client.SendShardRepairMsg(ctx, h, args) + if err == nil || !shouldRetry(err) { + return err + } + span.Errorf("seed shard repair failed, host: %s, args: %+v, err:%+v", h, args, err) + } + + return err +} + +func shouldRetry(err error) bool { + if err == nil { + return false // success + } + _, ok := err.(rpc.HTTPError) + return ok +} diff --git a/blobstore/api/proxy/mq.go b/blobstore/api/proxy/mq.go new file mode 100644 index 000000000..c97dbe180 --- /dev/null +++ b/blobstore/api/proxy/mq.go @@ -0,0 +1,49 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package proxy + +import ( + "context" + + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +type MsgSender interface { + SendDeleteMsg(ctx context.Context, host string, args *DeleteArgs) error + SendShardRepairMsg(ctx context.Context, host string, args *ShardRepairArgs) error +} + +type LbMsgSender interface { + SendDeleteMsg(ctx context.Context, args *DeleteArgs) error + SendShardRepairMsg(ctx context.Context, args *ShardRepairArgs) error +} + +type DeleteArgs struct { + ClusterID proto.ClusterID `json:"cluster_id"` + Blobs []BlobDelete `json:"blobs"` +} + +type BlobDelete struct { + Bid proto.BlobID `json:"bid"` + Vid proto.Vid `json:"vid"` +} + +type ShardRepairArgs struct { + ClusterID proto.ClusterID `json:"cluster_id"` + Bid proto.BlobID `json:"bid"` + Vid proto.Vid `json:"vid"` + BadIdxes []uint8 `json:"bad_idxes"` + Reason string `json:"reason"` +} diff --git a/blobstore/api/scheduler/client.go b/blobstore/api/scheduler/client.go new file mode 100644 index 000000000..d86d17bb5 --- /dev/null +++ b/blobstore/api/scheduler/client.go @@ -0,0 +1,131 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + + cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/util/errors" + "github.com/cubefs/cubefs/blobstore/util/selector" +) + +// defined http server path. +const ( + PathStats = "/stats" + PathLeaderStats = "/leader/stats" + + PathTaskAcquire = "/task/acquire" + PathTaskReclaim = "/task/reclaim" + PathTaskCancel = "/task/cancel" + PathTaskComplete = "/task/complete" + PathTaskReport = "/task/report" + PathTaskRenewal = "/task/renewal" + PathInspectComplete = "/inspect/complete" + PathInspectAcquire = "/inspect/acquire" + PathManualMigrateTaskAdd = "/manual/migrate/task/add" + + PathBalanceTaskDetail = "/balance/task/detail" + PathRepairTaskDetail = "/repair/task/detail" + PathDropTaskDetail = "/drop/task/detail" + PathManualMigrateTaskDetail = "/manual/migrate/task/detail" + + PathUpdateVolume = "/update/vol" +) + +var errNoServiceAvailable = errors.New("no service available") + +type IScheduler interface { + AcquireTask(ctx context.Context, args *AcquireArgs) (ret *WorkerTask, err error) + AcquireInspectTask(ctx context.Context) (ret *WorkerInspectTask, err error) + // report alive tasks + RenewalTask(ctx context.Context, args *TaskRenewalArgs) (ret *TaskRenewalRet, err error) + ReportTask(ctx context.Context, args *TaskReportArgs) (err error) + ReclaimTask(ctx context.Context, args *ReclaimTaskArgs) (err error) + CancelTask(ctx context.Context, args *CancelTaskArgs) (err error) + CompleteTask(ctx context.Context, args *CompleteTaskArgs) (err error) + CompleteInspect(ctx context.Context, args *CompleteInspectArgs) (err error) + + // stats + DiskRepairTaskDetail(ctx context.Context, args *TaskStatArgs) (ret RepairTaskDetail, err error) + BalanceTaskDetail(ctx context.Context, args *TaskStatArgs) (ret MigrateTaskDetail, err error) + DiskDropTaskDetail(ctx context.Context, args *TaskStatArgs) (ret MigrateTaskDetail, err error) + ManualMigrateTaskDetail(ctx context.Context, args *TaskStatArgs) (ret MigrateTaskDetail, err error) + Stats(ctx context.Context, host string) (ret TasksStat, err error) + LeaderStats(ctx context.Context) (ret TasksStat, err error) + + // add manual migrate task + AddManualMigrateTask(ctx context.Context, args *AddManualMigrateArgs) (err error) + IVolumeUpdater +} + +type IVolumeUpdater interface { + UpdateVol(ctx context.Context, host string, vid proto.Vid) (err error) +} + +// UpdateVolumeArgs argument of volume to update. +type UpdateVolumeArgs struct { + Vid proto.Vid `json:"vid"` +} + +type Config struct { + HostSyncIntervalMs int64 `json:"host_sync_interval_ms"` + rpc.Config +} + +type client struct { + selector selector.Selector + rpc.Client +} + +func NewVolumeUpdater(cfg *Config) IVolumeUpdater { + return &client{ + Client: rpc.NewClient(&cfg.Config), + } +} + +func New(cfg *Config, service cmapi.APIService, clusterID proto.ClusterID) IScheduler { + hostGetter := func() ([]string, error) { + svrInfos, err := service.GetService(context.Background(), cmapi.GetServiceArgs{Name: proto.ServiceNameScheduler}) + if err != nil { + return nil, err + } + + var hosts []string + for _, s := range svrInfos.Nodes { + if clusterID == proto.ClusterID(s.ClusterID) { + hosts = append(hosts, s.Host) + } + } + if len(hosts) == 0 { + return nil, errNoServiceAvailable + } + + return hosts, nil + } + if cfg.HostSyncIntervalMs == 0 { + cfg.HostSyncIntervalMs = 1000 + } + return &client{ + selector: selector.NewSelectorWithGetter(cfg.HostSyncIntervalMs, hostGetter), + Client: rpc.NewClient(&cfg.Config), + } +} + +func (c *client) UpdateVol(ctx context.Context, host string, vid proto.Vid) (err error) { + return c.PostWith(ctx, host+PathUpdateVolume, nil, UpdateVolumeArgs{Vid: vid}) +} diff --git a/blobstore/api/scheduler/task.go b/blobstore/api/scheduler/task.go new file mode 100644 index 000000000..f4a4227b2 --- /dev/null +++ b/blobstore/api/scheduler/task.go @@ -0,0 +1,349 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + + "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +type AcquireArgs struct { + IDC string `json:"idc"` +} + +type WorkerTask struct { + TaskType string `json:"task_type"` // task type + Repair *proto.VolRepairTask `json:"repair"` // repair task + Balance *proto.MigrateTask `json:"balance"` // balance task + DiskDrop *proto.MigrateTask `json:"disk_drop"` // disk drop task + ManualMigrate *proto.MigrateTask `json:"manual_migrate"` // manual migrate task +} + +func (task *WorkerTask) IsValid() bool { + var ( + mode codemode.CodeMode + destination proto.VunitLocation + srcs []proto.VunitLocation + ) + switch task.TaskType { + case proto.RepairTaskType: + mode = task.Repair.CodeMode + destination = task.Repair.Destination + srcs = task.Repair.Sources + case proto.BalanceTaskType: + mode = task.Balance.CodeMode + destination = task.Balance.Destination + srcs = task.Balance.Sources + case proto.DiskDropTaskType: + mode = task.DiskDrop.CodeMode + destination = task.DiskDrop.Destination + srcs = task.DiskDrop.Sources + case proto.ManualMigrateType: + mode = task.ManualMigrate.CodeMode + destination = task.ManualMigrate.Destination + srcs = task.ManualMigrate.Sources + default: + return false + } + + if !mode.IsValid() { + return false + } + // check destination + if !proto.CheckVunitLocations([]proto.VunitLocation{destination}) { + return false + } + // check sources + if !proto.CheckVunitLocations(srcs) { + return false + } + + return true +} + +func (c *client) AcquireTask(ctx context.Context, args *AcquireArgs) (ret *WorkerTask, err error) { + err = c.request(func(host string) error { + return c.GetWith(ctx, host+PathTaskAcquire+"?idc="+args.IDC, &ret) + }) + return +} + +type WorkerInspectTask struct { + Task *proto.InspectTask `json:"task"` +} + +func (task *WorkerInspectTask) IsValid() bool { + if !task.Task.Mode.IsValid() { + return false + } + + if !proto.CheckVunitLocations(task.Task.Replicas) { + return false + } + + return true +} + +func (c *client) AcquireInspectTask(ctx context.Context) (*WorkerInspectTask, error) { + ret := WorkerInspectTask{ + Task: &proto.InspectTask{}, + } + err := c.request(func(host string) error { + return c.GetWith(ctx, host+PathInspectAcquire, &ret) + }) + return &ret, err +} + +type TaskRenewalArgs struct { + IDC string `json:"idc"` + Repair map[string]struct{} `json:"repair"` + Balance map[string]struct{} `json:"balance"` + DiskDrop map[string]struct{} `json:"disk_drop"` + ManualMigrate map[string]struct{} `json:"manual_migrate"` +} + +type TaskRenewalRet struct { + Repair map[string]string `json:"repair"` + Balance map[string]string `json:"balance"` + DiskDrop map[string]string `json:"disk_drop"` + ManualMigrate map[string]string `json:"manual_migrate"` +} + +func (c *client) RenewalTask(ctx context.Context, args *TaskRenewalArgs) (ret *TaskRenewalRet, err error) { + err = c.request(func(host string) error { + return c.PostWith(ctx, host+PathTaskRenewal, &ret, args) + }) + return +} + +type TaskReportArgs struct { + TaskType string `json:"task_type"` + TaskId string `json:"task_id"` + + TaskStats proto.TaskStatistics `json:"task_stats"` + IncreaseDataSizeByte int `json:"increase_data_size_byte"` + IncreaseShardCnt int `json:"increase_shard_cnt"` +} + +func (c *client) ReportTask(ctx context.Context, args *TaskReportArgs) (err error) { + return c.request(func(host string) error { + return c.PostWith(ctx, host+PathTaskReport, nil, args) + }) +} + +type ReclaimTaskArgs struct { + TaskId string `json:"task_id"` + IDC string `json:"idc"` + TaskType string `json:"task_type"` + Src []proto.VunitLocation `json:"src"` + Dest proto.VunitLocation `json:"dest"` + Reason string `json:"reason"` +} + +func (c *client) ReclaimTask(ctx context.Context, args *ReclaimTaskArgs) (err error) { + return c.request(func(host string) error { + return c.PostWith(ctx, host+PathTaskReclaim, nil, args) + }) +} + +type CancelTaskArgs struct { + TaskId string `json:"task_id"` + IDC string `json:"idc"` + TaskType string `json:"task_type"` + Src []proto.VunitLocation `json:"src"` + Dest proto.VunitLocation `json:"dest"` + Reason string `json:"reason"` +} + +func (c *client) CancelTask(ctx context.Context, args *CancelTaskArgs) (err error) { + return c.request(func(host string) error { + return c.PostWith(ctx, host+PathTaskCancel, nil, args) + }) +} + +type CompleteTaskArgs struct { + TaskId string `json:"task_id"` + IDC string `json:"idc"` + TaskType string `json:"task_type"` + Src []proto.VunitLocation `json:"src"` + Dest proto.VunitLocation `json:"dest"` +} + +func (c *client) CompleteTask(ctx context.Context, args *CompleteTaskArgs) (err error) { + return c.request(func(host string) error { + return c.PostWith(ctx, host+PathTaskComplete, nil, args) + }) +} + +type CompleteInspectArgs struct { + *proto.InspectRet +} + +func (c *client) CompleteInspect(ctx context.Context, args *CompleteInspectArgs) (err error) { + return c.request(func(host string) error { + return c.PostWith(ctx, host+PathInspectComplete, nil, args) + }) +} + +type AddManualMigrateArgs struct { + Vuid proto.Vuid `json:"vuid"` + DirectDownload bool `json:"direct_download"` +} + +func (args *AddManualMigrateArgs) Valid() bool { + return args.Vuid.IsValid() +} + +func (c *client) AddManualMigrateTask(ctx context.Context, args *AddManualMigrateArgs) (err error) { + return c.request(func(host string) error { + return c.PostWith(ctx, host+PathManualMigrateTaskAdd, nil, args) + }) +} + +// for task stat +type TaskStatArgs struct { + TaskId string `json:"task_id"` +} + +type RepairTaskDetail struct { + TaskInfo proto.VolRepairTask `json:"task_info"` + RunStats proto.TaskStatistics `json:"run_stats"` +} + +type MigrateTaskDetail struct { + TaskInfo proto.MigrateTask `json:"task_info"` + RunStats proto.TaskStatistics `json:"run_stats"` +} + +type PerMinStats struct { + FinishedCnt string `json:"finished_cnt"` + ShardCnt string `json:"shard_cnt"` + DataAmountByte string `json:"data_amount_byte"` +} + +type DiskRepairTasksStat struct { + Enable bool `json:"enable"` + RepairingDiskID proto.DiskID `json:"repairing_disk_id"` + TotalTasksCnt int `json:"total_tasks_cnt"` + RepairedTasksCnt int `json:"repaired_tasks_cnt"` + MigrateTasksStat +} + +type MigrateTasksStat struct { + PreparingCnt int `json:"preparing_cnt"` + WorkerDoingCnt int `json:"worker_doing_cnt"` + FinishingCnt int `json:"finishing_cnt"` + StatsPerMin PerMinStats `json:"stats_per_min"` +} + +type DiskDropTasksStat struct { + Enable bool `json:"enable"` + DroppingDiskID proto.DiskID `json:"dropping_disk_id"` + TotalTasksCnt int `json:"total_tasks_cnt"` + DroppedTasksCnt int `json:"dropped_tasks_cnt"` + MigrateTasksStat +} + +type BalanceTasksStat struct { + Enable bool `json:"enable"` + MigrateTasksStat +} + +type ManualMigrateTasksStat struct { + MigrateTasksStat +} + +type VolumeInspectTasksStat struct { + Enable bool `json:"enable"` + FinishedPerMin string `json:"finished_per_min"` + TimeOutPerMin string `json:"time_out_per_min"` +} + +// RunnerStat shard repair and blob delete stat +type RunnerStat struct { + Enable bool `json:"enable"` + SuccessPerMin string `json:"success_per_min"` + FailedPerMin string `json:"failed_per_min"` + TotalErrCnt uint64 `json:"total_err_cnt"` + ErrStats []string `json:"err_stats"` +} + +type TasksStat struct { + DiskRepair *DiskRepairTasksStat `json:"disk_repair,omitempty"` + DiskDrop *DiskDropTasksStat `json:"disk_drop,omitempty"` + Balance *BalanceTasksStat `json:"balance,omitempty"` + ManualMigrate *ManualMigrateTasksStat `json:"manual_migrate,omitempty"` + VolumeInspect *VolumeInspectTasksStat `json:"volume_inspect,omitempty"` + ShardRepair *RunnerStat `json:"shard_repair"` + BlobDelete *RunnerStat `json:"blob_delete"` +} + +func (c *client) DiskRepairTaskDetail(ctx context.Context, args *TaskStatArgs) (ret RepairTaskDetail, err error) { + err = c.request(func(host string) error { + return c.PostWith(ctx, host+PathRepairTaskDetail, &ret, args) + }) + return +} + +func (c *client) BalanceTaskDetail(ctx context.Context, args *TaskStatArgs) (ret MigrateTaskDetail, err error) { + err = c.request(func(host string) error { + return c.PostWith(ctx, host+PathBalanceTaskDetail, &ret, args) + }) + return +} + +func (c *client) DiskDropTaskDetail(ctx context.Context, args *TaskStatArgs) (ret MigrateTaskDetail, err error) { + err = c.request(func(host string) error { + return c.PostWith(ctx, host+PathDropTaskDetail, &ret, args) + }) + return +} + +func (c *client) ManualMigrateTaskDetail(ctx context.Context, args *TaskStatArgs) (ret MigrateTaskDetail, err error) { + err = c.request(func(host string) error { + return c.PostWith(ctx, host+PathManualMigrateTaskDetail, &ret, args) + }) + return +} + +func (c *client) Stats(ctx context.Context, host string) (ret TasksStat, err error) { + err = c.GetWith(ctx, host+PathStats, &ret) + return +} + +func (c *client) LeaderStats(ctx context.Context) (ret TasksStat, err error) { + err = c.request(func(host string) error { + return c.GetWith(ctx, host+PathLeaderStats, &ret) + }) + return +} + +func (c *client) selectHost() (string, error) { + hosts := c.selector.GetRandomN(1) + if len(hosts) == 0 { + return "", errNoServiceAvailable + } + return hosts[0], nil +} + +func (c *client) request(req func(host string) error) error { + host, err := c.selectHost() + if err != nil { + return err + } + return req(host) +} diff --git a/blobstore/cmd/cmd.go b/blobstore/cmd/cmd.go new file mode 100644 index 000000000..e432779f4 --- /dev/null +++ b/blobstore/cmd/cmd.go @@ -0,0 +1,223 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cmd + +import ( + "context" + "io" + "net" + "net/http" + "os" + "os/signal" + "runtime" + "syscall" + "time" + + "gopkg.in/natefinch/lumberjack.v2" + + "github.com/cubefs/cubefs/blobstore/common/config" + "github.com/cubefs/cubefs/blobstore/common/profile" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/rpc/auditlog" + "github.com/cubefs/cubefs/blobstore/common/rpc/auth" + "github.com/cubefs/cubefs/blobstore/util/graceful" + "github.com/cubefs/cubefs/blobstore/util/log" + + // module release init functions + _ "github.com/cubefs/cubefs/blobstore/util/version" +) + +const ( + defaultShutdownTimeoutS = 30 +) + +type LogConfig struct { + Level log.Level `json:"level"` + Filename string `json:"filename"` + MaxSize int `json:"maxsize"` + MaxAge int `json:"maxage"` + MaxBackups int `json:"maxbackups"` +} + +type Config struct { + MaxProcs int `json:"max_procs"` + LogConf LogConfig `json:"log"` + BindAddr string `json:"bind_addr"` + ShutdownTimeoutS int `json:"shutdown_timeout_s"` + + AuditLog auditlog.Config `json:"auditlog"` + Auth auth.Config `json:"auth"` +} + +type Module struct { + Name string + InitConfig func(args []string) (*Config, error) + SetUp func() (*rpc.Router, []rpc.ProgressHandler) + TearDown func() + graceful bool +} + +var mod *Module + +func RegisterModule(m *Module) { + mod = m + mod.graceful = false +} + +func RegisterGracefulModule(m *Module) { + mod = m + mod.graceful = true +} + +func newLogWriter(cfg *LogConfig) io.Writer { + maxsize := cfg.MaxSize + if maxsize == 0 { + maxsize = 1024 + } + maxage := cfg.MaxAge + if maxage == 0 { + maxage = 7 + } + maxbackups := cfg.MaxBackups + if maxbackups == 0 { + maxbackups = 7 + } + return &lumberjack.Logger{ + Filename: cfg.Filename, + MaxSize: maxsize, + MaxAge: maxage, + MaxBackups: maxbackups, + LocalTime: true, + } +} + +func Main(args []string) { + cfg, err := mod.InitConfig(args) + if err != nil { + log.Fatalf("init config error: %v", err) + } + if cfg.MaxProcs > 0 { + runtime.GOMAXPROCS(cfg.MaxProcs) + } + log.SetOutputLevel(cfg.LogConf.Level) + registerLogLevel() + if cfg.LogConf.Filename != "" { + log.SetOutput(newLogWriter(&cfg.LogConf)) + } + if cfg.ShutdownTimeoutS <= 0 { + cfg.ShutdownTimeoutS = defaultShutdownTimeoutS + } + + lh, logf, err := auditlog.Open(mod.Name, &cfg.AuditLog) + if err != nil { + log.Fatal("failed to open auditlog:", err) + } + defer logf.Close() + + ctx, cancel1 := context.WithCancel(context.Background()) + defer cancel1() + config.HotReload(ctx, config.ConfName()) + + if mod.graceful { + programEntry := func(state *graceful.State) { + router, handlers := mod.SetUp() + + httpServer := &http.Server{ + Addr: cfg.BindAddr, + Handler: reorderMiddleWareHandlers(router, lh, cfg.BindAddr, cfg.Auth, handlers), + } + + log.Info("server is running at:", cfg.BindAddr) + go func() { + if err := httpServer.Serve(state.ListenerFds[0].(*net.TCPListener)); err != nil && err != http.ErrServerClosed { + log.Fatal("server exits:", err) + } + }() + + // wait for signal + <-state.CloseCh + log.Info("graceful shutdown...") + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(cfg.ShutdownTimeoutS)*time.Second) + defer cancel() + httpServer.Shutdown(ctx) + + if mod.TearDown != nil { + mod.TearDown() + } + } + graceful.Run(&graceful.Config{ + Entry: programEntry, + ListenAddresses: []string{cfg.BindAddr}, + }) + return + } + + router, handlers := mod.SetUp() + + httpServer := &http.Server{ + Addr: cfg.BindAddr, + Handler: reorderMiddleWareHandlers(router, lh, cfg.BindAddr, cfg.Auth, handlers), + } + + log.Info("Server is running at", cfg.BindAddr) + go func() { + if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatalf("Server exits, err: %v", err) + } + }() + + // wait for signal + ch := make(chan os.Signal, 1) + signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT) + sig := <-ch + log.Infof("receive signal: %s, stop service...", sig.String()) + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(cfg.ShutdownTimeoutS)*time.Second) + defer cancel() + httpServer.Shutdown(ctx) + + if mod.TearDown != nil { + mod.TearDown() + } +} + +// reorderMiddleWareHandlers +// the order of handlers in MiddlewareHandler has some constraints: +// 1. the first is router, +// 2. the second is AuditLog handler, +// 3. the third is profile handler if config, +// 4. the fourth is Auth handler if config, +// 5. others self define handlers by modules. +func reorderMiddleWareHandlers(r *rpc.Router, lh rpc.ProgressHandler, profileAddr string, authCfg auth.Config, handlers []rpc.ProgressHandler) (mux http.Handler) { + hs := append([]rpc.ProgressHandler{}, lh) + if profileHandler := profile.NewProfileHandler(profileAddr); profileHandler != nil { + hs = append(hs, profileHandler) + } + if authCfg.EnableAuth && authCfg.Secret != "" { + hs = append(hs, auth.NewAuthHandler(&authCfg)) + } + hs = append(hs, handlers...) + + return rpc.MiddlewareHandlerWith(r, hs...) +} + +func registerLogLevel() { + logLevelPath, logLevelHandler := log.ChangeDefaultLevelHandler() + profile.HandleFunc(http.MethodPost, logLevelPath, func(c *rpc.Context) { + logLevelHandler.ServeHTTP(c.Writer, c.Request) + }) + profile.HandleFunc(http.MethodGet, logLevelPath, func(c *rpc.Context) { + logLevelHandler.ServeHTTP(c.Writer, c.Request) + }) +} diff --git a/blobstore/cmd/scheduler/main.go b/blobstore/cmd/scheduler/main.go new file mode 100644 index 000000000..c56141d5d --- /dev/null +++ b/blobstore/cmd/scheduler/main.go @@ -0,0 +1,27 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package main + +import ( + "os" + + "github.com/cubefs/cubefs/blobstore/cmd" + + _ "github.com/cubefs/cubefs/blobstore/scheduler" +) + +func main() { + cmd.Main(os.Args) +} diff --git a/blobstore/cmd/scheduler/scheduler.conf b/blobstore/cmd/scheduler/scheduler.conf new file mode 100644 index 000000000..75c45a30e --- /dev/null +++ b/blobstore/cmd/scheduler/scheduler.conf @@ -0,0 +1,35 @@ +{ + "bind_addr": ":9800", + "cluster_id": 1, + "services": { + "leader": 1, + "node_id": 1, + "members": {"1": "127.0.0.1:9800"} + }, + "service_register": { + "host": "http://127.0.0.1:9800", + "idc": "z0" + }, + "clustermgr": { + "hosts": ["http://127.0.0.1:10110", "http://127.0.0.1:10111", "http://127.0.0.1:10112"] + }, + "database": { + "mongo": { + "uri": "mongodb://127.0.0.1:27017" + } + }, + "kafka": { + "broker_list": ["127.0.0.1:9092"] + }, + "blob_delete": { + "delete_log": { + "dir": "./delete_log" + } + }, + "log": { + "level": 0 + }, + "auditlog": { + "logdir": "/tmp/scheduler/" + } +} \ No newline at end of file diff --git a/blobstore/cmd/scheduler/scheduler.follower.conf b/blobstore/cmd/scheduler/scheduler.follower.conf new file mode 100644 index 000000000..d100f507e --- /dev/null +++ b/blobstore/cmd/scheduler/scheduler.follower.conf @@ -0,0 +1,35 @@ +{ + "bind_addr": ":9880", + "cluster_id": 1, + "services": { + "leader": 1, + "node_id": 2, + "members": {"1": "127.0.0.1:9800", "2": "127.0.0.1:9880"} + }, + "service_register": { + "host": "http://127.0.0.1:9880", + "idc": "z0" + }, + "clustermgr": { + "hosts": ["http://127.0.0.1:10110", "http://127.0.0.1:10111", "http://127.0.0.1:10112"] + }, + "database": { + "mongo": { + "uri": "mongodb://127.0.0.1:27017" + } + }, + "kafka": { + "broker_list": ["127.0.0.1:9092"] + }, + "blob_delete": { + "delete_log": { + "dir": "./delete_log" + } + }, + "log": { + "level": 0 + }, + "auditlog": { + "logdir": "./auditlog/scheduler" + } +} diff --git a/blobstore/cmd/scheduler/scheduler.leader.conf b/blobstore/cmd/scheduler/scheduler.leader.conf new file mode 100644 index 000000000..0a6d11bac --- /dev/null +++ b/blobstore/cmd/scheduler/scheduler.leader.conf @@ -0,0 +1,35 @@ +{ + "bind_addr": ":9800", + "cluster_id": 1, + "services": { + "leader": 1, + "node_id": 1, + "members": {"1": "127.0.0.1:9800", "2": "127.0.0.1:9880"} + }, + "service_register": { + "host": "http://127.0.0.1:9800", + "idc": "z0" + }, + "clustermgr": { + "hosts": ["http://127.0.0.1:10110", "http://127.0.0.1:10111", "http://127.0.0.1:10112"] + }, + "database": { + "mongo": { + "uri": "mongodb://127.0.0.1:27017" + } + }, + "kafka": { + "broker_list": ["127.0.0.1:9092"] + }, + "blob_delete": { + "delete_log": { + "dir": "./delete_log" + } + }, + "log": { + "level": 0 + }, + "auditlog": { + "logdir": "./auditlog/scheduler" + } +} diff --git a/blobstore/common/errors/access.go b/blobstore/common/errors/access.go new file mode 100644 index 000000000..ff0a4e46b --- /dev/null +++ b/blobstore/common/errors/access.go @@ -0,0 +1,35 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package errors + +// code for access +const ( + CodeAccessReadRequestBody = 466 // read request body error + CodeAccessReadConflictBody = 499 // read conflict body error + CodeAccessUnexpect = 550 // unexpect + CodeAccessServiceDiscovery = 551 // service discovery for access api client + CodeAccessLimited = 552 // read write limited for access api client + CodeAccessExceedSize = 553 // exceed max size +) + +// errro of access +var ( + ErrAccessReadRequestBody = Error(CodeAccessReadRequestBody) + ErrAccessReadConflictBody = Error(CodeAccessReadConflictBody) + ErrAccessUnexpect = Error(CodeAccessUnexpect) + ErrAccessServiceDiscovery = Error(CodeAccessServiceDiscovery) + ErrAccessLimited = Error(CodeAccessLimited) + ErrAccessExceedSize = Error(CodeAccessExceedSize) +) diff --git a/blobstore/common/errors/allocator.go b/blobstore/common/errors/allocator.go new file mode 100644 index 000000000..6ac305edc --- /dev/null +++ b/blobstore/common/errors/allocator.go @@ -0,0 +1,25 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package errors + +const ( + CodeNoAvaliableVolume = 801 + CodeAllocBidFromCm = 802 +) + +var ( + ErrNoAvaliableVolume = Error(CodeNoAvaliableVolume) + ErrAllocBidFromCm = Error(CodeAllocBidFromCm) +) diff --git a/blobstore/common/errors/blobnode.go b/blobstore/common/errors/blobnode.go new file mode 100644 index 000000000..837d5d540 --- /dev/null +++ b/blobstore/common/errors/blobnode.go @@ -0,0 +1,80 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package errors + +const ( + CodeInvalidParam = 600 + CodeAlreadyExist = 601 + CodeOutOfLimit = 602 + CodeInternal = 603 + CodeOverload = 604 + + CodeDiskNotFound = 611 + CodeDiskBroken = 613 + CodeInvalidDiskId = 614 + CodeDiskNoSpace = 615 + + CodeVuidNotFound = 621 + CodeVUIDReadonly = 622 + CodeVUIDRelease = 623 + CodeVuidNotMatch = 624 + CodeChunkNotReadonly = 625 + CodeChunkNotNormal = 626 + CodeChunkNoSpace = 627 + CodeChunkCompacting = 628 + CodeInvalidChunkId = 630 + CodeTooManyChunks = 632 + CodeChunkInuse = 633 + + CodeBidNotFound = 651 + CodeShardSizeTooLarge = 652 + CodeShardNotMarkDelete = 653 + CodeShardMarkDeleted = 654 + CodeShardInvalidOffset = 655 + CodeShardListExceedLimit = 656 + CodeShardInvalidBid = 657 +) + +var ( + ErrInvalidParam = Error(CodeInvalidParam) + ErrAlreadyExist = Error(CodeAlreadyExist) + ErrOutOfLimit = Error(CodeOutOfLimit) + ErrOverload = Error(CodeOverload) + + ErrNoSuchDisk = Error(CodeDiskNotFound) + ErrDiskBroken = Error(CodeDiskBroken) + ErrDiskNoSpace = Error(CodeDiskNoSpace) + ErrInvalidDiskId = Error(CodeInvalidDiskId) + + ErrNoSuchVuid = Error(CodeVuidNotFound) + ErrReadonlyVUID = Error(CodeVUIDReadonly) + ErrReleaseVUID = Error(CodeVUIDRelease) + ErrVuidNotMatch = Error(CodeVuidNotMatch) + ErrChunkNotReadonly = Error(CodeChunkNotReadonly) + ErrChunkNotNormal = Error(CodeChunkNotNormal) + ErrChunkNoSpace = Error(CodeChunkNoSpace) + ErrChunkInCompact = Error(CodeChunkCompacting) + ErrInvalidChunkId = Error(CodeInvalidChunkId) + ErrTooManyChunks = Error(CodeTooManyChunks) + ErrChunkInuse = Error(CodeChunkInuse) + + ErrNoSuchBid = Error(CodeBidNotFound) + ErrShardSizeTooLarge = Error(CodeShardSizeTooLarge) + ErrShardNotMarkDelete = Error(CodeShardNotMarkDelete) + ErrShardMarkDeleted = Error(CodeShardMarkDeleted) + ErrShardInvalidOffset = Error(CodeShardInvalidOffset) + ErrShardListExceedLimit = Error(CodeShardListExceedLimit) + ErrShardInvalidBid = Error(CodeShardInvalidBid) +) diff --git a/blobstore/common/errors/clustermgr.go b/blobstore/common/errors/clustermgr.go new file mode 100644 index 000000000..0ef8bbeb9 --- /dev/null +++ b/blobstore/common/errors/clustermgr.go @@ -0,0 +1,87 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package errors + +const ( + CodeCMUnexpect = 900 + CodeActiveVolume = 901 + CodeLockNotAllow = 902 + CodeUnlockNotAllow = 903 + CodeVolumeNotExist = 904 + CodeVolumeStatusNotAcceptable = 905 + CodeRaftPropose = 906 + CodeNoLeader = 907 + CodeRaftReadIndex = 908 + CodeUpdateVolumeParamInvalid = 909 + CodeDuplicatedMemberInfo = 910 + CodeCMDiskNotFound = 911 + CodeInvalidDiskStatus = 912 + CodeChangeDiskStatusNotAllow = 913 + CodeConcurrentAllocVolumeUnit = 914 + CodeOverMaxVolumeThreshold = 915 + CodeAllocVolumeInvalidParams = 916 + CodeNoAvailableVolume = 917 + CodeOldVuidNotMatch = 918 + CodeNewVuidNotMatch = 919 + CodeNewDiskIDNotMatch = 920 + CodeConfigArgument = 921 + CodeInvalidClusterID = 922 + CodeInvalidIDC = 923 + CodeVolumeUnitNotExist = 924 + CodeRegisterServiceInvalidParams = 925 + CodeDiskAbnormalOrNotReadOnly = 926 + CodeStatChunkFailed = 927 + CodeInvalidCodeMode = 928 + CodeRetainVolumeNotAlloc = 929 + CodeDroppedDiskHasVolumeUnit = 930 + CodeNotSupportIdle = 931 + CodeDiskIsDropping = 932 +) + +var ( + ErrCMUnexpect = Error(CodeCMUnexpect) + ErrActiveVolume = Error(CodeActiveVolume) + ErrLockNotAllow = Error(CodeLockNotAllow) + ErrUnlockNotAllow = Error(CodeUnlockNotAllow) + ErrVolumeNotExist = Error(CodeVolumeNotExist) + ErrVolumeStatusNotAcceptable = Error(CodeVolumeStatusNotAcceptable) + ErrRaftPropose = Error(CodeRaftPropose) + ErrNoLeader = Error(CodeNoLeader) + ErrRaftReadIndex = Error(CodeRaftReadIndex) + ErrUpdateVolumeParamInvalid = Error(CodeUpdateVolumeParamInvalid) + ErrDuplicatedMemberInfo = Error(CodeDuplicatedMemberInfo) + ErrCMDiskNotFound = Error(CodeCMDiskNotFound) + ErrInvalidStatus = Error(CodeInvalidDiskStatus) + ErrChangeDiskStatusNotAllow = Error(CodeChangeDiskStatusNotAllow) + ErrConcurrentAllocVolumeUnit = Error(CodeConcurrentAllocVolumeUnit) + ErrOverMaxVolumeThreshold = Error(CodeOverMaxVolumeThreshold) + ErrNoAvailableVolume = Error(CodeNoAvailableVolume) + ErrAllocVolumeInvalidParams = Error(CodeAllocVolumeInvalidParams) + ErrOldVuidNotMatch = Error(CodeOldVuidNotMatch) + ErrNewVuidNotMatch = Error(CodeNewVuidNotMatch) + ErrNewDiskIDNotMatch = Error(CodeNewDiskIDNotMatch) + ErrConfigArgument = Error(CodeConfigArgument) + ErrInvalidClusterID = Error(CodeInvalidClusterID) + ErrInvalidIDC = Error(CodeInvalidIDC) + ErrVolumeUnitNotExist = Error(CodeVolumeUnitNotExist) + ErrRegisterServiceInvalidParams = Error(CodeRegisterServiceInvalidParams) + ErrDiskAbnormalOrNotReadOnly = Error(CodeDiskAbnormalOrNotReadOnly) + ErrStatChunkFailed = Error(CodeStatChunkFailed) + ErrInvalidCodeMode = Error(CodeInvalidCodeMode) + ErrRetainVolumeNotAlloc = Error(CodeRetainVolumeNotAlloc) + ErrDroppedDiskHasVolumeUnit = Error(CodeDroppedDiskHasVolumeUnit) + ErrNotSupportIdle = Error(CodeNotSupportIdle) + ErrDiskIsDropping = Error(CodeDiskIsDropping) +) diff --git a/blobstore/common/errors/common.go b/blobstore/common/errors/common.go new file mode 100644 index 000000000..0c459d628 --- /dev/null +++ b/blobstore/common/errors/common.go @@ -0,0 +1,42 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package errors + +import ( + "errors" + "net/http" + + "github.com/cubefs/cubefs/blobstore/common/rpc" +) + +var ( + // 2xx + ErrExist = newError(http.StatusCreated, "Data Already Exist") + + // 4xx + ErrIllegalArguments = newError(http.StatusBadRequest, "Illegal Arguments") + ErrNotFound = newError(http.StatusNotFound, "Not Found") + ErrRequestTimeout = newError(http.StatusRequestTimeout, "Request Timeout") + ErrRequestedRangeNotSatisfiable = newError(http.StatusRequestedRangeNotSatisfiable, "Request Range Not Satisfiable") + ErrRequestNotAllow = newError(http.StatusBadRequest, "Request Not Allow") + ErrReaderError = newError(499, "Reader Error") + + // 5xx errUnexpected - unexpected error, requires manual intervention. + ErrUnexpected = newError(http.StatusInternalServerError, "Unexpected Error") +) + +func newError(status int, msg string) *rpc.Error { + return rpc.NewError(status, "", errors.New(msg)) +} diff --git a/blobstore/common/errors/errors.go b/blobstore/common/errors/errors.go new file mode 100644 index 000000000..7a849ba31 --- /dev/null +++ b/blobstore/common/errors/errors.go @@ -0,0 +1,170 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package errors + +import ( + "net/http" + + "github.com/cubefs/cubefs/blobstore/common/rpc" +) + +// access 550-599 +// blobnode 600-699 +// background service 700-799 +// allocator 800-899 +// clusterMgr 900-999 + +// Error http status code for all application +type Error int + +var _ rpc.HTTPError = Error(0) + +// Error implements error and rpc.HTTPError +func (e Error) Error() string { + return errCodeMap[int(e)] +} + +// StatusCode implements rpc.HTTPError +func (e Error) StatusCode() int { + return int(e) +} + +// ErrorCode implements rpc.HTTPError +func (e Error) ErrorCode() string { + return "" +} + +var errCodeMap = map[int]string{ + // access + CodeAccessReadRequestBody: "access read request body", + CodeAccessReadConflictBody: "access read conflict body", + CodeAccessUnexpect: "access unexpected error", + CodeAccessServiceDiscovery: "access client service discovery disconnect", + CodeAccessLimited: "access limited", + CodeAccessExceedSize: "access exceed object size", + + // clustermgr + CodeCMUnexpect: "cm: unexpected error", + CodeActiveVolume: "volume is activity status", + CodeLockNotAllow: "lock volume not allow", + CodeUnlockNotAllow: "unlock volume not allow", + CodeVolumeNotExist: "volume not exist", + CodeVolumeStatusNotAcceptable: "volume status not acceptable", + CodeRaftPropose: "raft propose error", + CodeNoLeader: "no leader", + CodeRaftReadIndex: "raft read index error", + CodeUpdateVolumeParamInvalid: "update volume params invalid", + CodeDuplicatedMemberInfo: "duplicated member info", + CodeCMDiskNotFound: "disk not found", + CodeInvalidDiskStatus: "invalid status", + CodeChangeDiskStatusNotAllow: "not allow to change status back", + CodeConcurrentAllocVolumeUnit: "alloc volume unit concurrently", + CodeOverMaxVolumeThreshold: "allocator request alloc volume over max threshold", + CodeNoAvailableVolume: "no available volume", + CodeAllocVolumeInvalidParams: "alloc volume request params is invalid", + CodeOldVuidNotMatch: "update volume unit, old vuid not match", + CodeNewVuidNotMatch: "update volume unit, new vuid not match", + CodeNewDiskIDNotMatch: "update volume unit, new diskID not match", + CodeConfigArgument: "config argument marshal error", + CodeInvalidClusterID: "request params error, invalid clusterID", + CodeInvalidIDC: "request params error,invalid idc", + CodeVolumeUnitNotExist: "volume unit not exist", + CodeDiskAbnormalOrNotReadOnly: "disk is abnormal or not readonly, can't add into dropping list", + CodeStatChunkFailed: "stat blob node chunk failed", + CodeInvalidCodeMode: "request alloc volume codeMode not invalid", + CodeRetainVolumeNotAlloc: "retain volume is not alloc", + CodeDroppedDiskHasVolumeUnit: "dropped disk still has volume unit remain, migrate them firstly", + CodeNotSupportIdle: "list volume v2 not support idle status", + CodeDiskIsDropping: "dropping disk not allow change state or set readonly", + + // background + CodeNotingTodo: "nothing to do", + CodeDestReplicaBad: "dest replica is bad can not repair", + CodeOrphanShard: "shard is an orphan", + CodeIllegalTask: "illegal task", + CodeNoInspect: "no inspect mgr instance", + CodeClusterIDNotMatch: "clusterId not match", + CodeRegisterServiceInvalidParams: "register service params is invalid", + CodeRequestLimited: "request limited", + + // allocator + CodeNoAvaliableVolume: "this codemode has no avaliable volume", + CodeAllocBidFromCm: "alloc bid from clustermgr error", + + // blobnode + CodeInvalidParam: "blobnode: invalid params", + CodeAlreadyExist: "blobnode: entry already exist", + CodeOutOfLimit: "blobnode: out of limit", + CodeInternal: "blobnode: internal error", + CodeOverload: "blobnode: service is overload", + + CodeDiskNotFound: "disk not found", + CodeDiskBroken: "disk is broken", + CodeInvalidDiskId: "disk id is invalid", + CodeDiskNoSpace: "disk no space", + + CodeVuidNotFound: "vuid not found", + CodeVUIDReadonly: "vuid readonly", + CodeVUIDRelease: "vuid released", + CodeVuidNotMatch: "vuid not match", + CodeChunkNotReadonly: "chunk must readonly", + CodeChunkNotNormal: "chunk must normal", + CodeChunkNoSpace: "chunk no space", + CodeChunkCompacting: "chunk is compacting", + CodeInvalidChunkId: "chunk id is invalid", + CodeTooManyChunks: "too many chunks", + CodeChunkInuse: "chunk in use", + + CodeBidNotFound: "bid not found", + CodeShardSizeTooLarge: "shard size too large", + CodeShardNotMarkDelete: "shard must mark delete", + CodeShardMarkDeleted: "shard already mark delete", + CodeShardInvalidOffset: "shard offset is invalid", + CodeShardInvalidBid: "shard key bid is invalid", + CodeShardListExceedLimit: "shard list exceed the limit", +} + +// HTTPError make rpc.HTTPError +func HTTPError(statusCode int, errCode string, err error) error { + return rpc.NewError(statusCode, errCode, err) +} + +// Error2HTTPError transfer error to rpc.HTTPError +func Error2HTTPError(err error) error { + if err == nil { + return nil + } + if e, ok := err.(rpc.HTTPError); ok { + return e + } + if code, ok := err.(Error); ok { + return code + } + return rpc.NewError(http.StatusInternalServerError, "ServerError", err) +} + +// DetectCode detect code +func DetectCode(err error) int { + if err == nil { + return http.StatusOK + } + if code, ok := err.(Error); ok { + return int(code) + } + if httpErr, ok := err.(rpc.HTTPError); ok { + return httpErr.StatusCode() + } + return http.StatusInternalServerError +} diff --git a/blobstore/common/errors/scheduler_sys.go b/blobstore/common/errors/scheduler_sys.go new file mode 100644 index 000000000..6e54a1ae2 --- /dev/null +++ b/blobstore/common/errors/scheduler_sys.go @@ -0,0 +1,59 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package errors + +import ( + "errors" +) + +const ( + CodeNotingTodo = 700 + CodeDestReplicaBad = 702 + CodeOrphanShard = 703 + CodeIllegalTask = 704 + CodeNoInspect = 705 + CodeClusterIDNotMatch = 706 + CodeRequestLimited = 707 +) + +// common +var ( + ErrRequestLimited = Error(CodeRequestLimited) +) + +// scheduler +var ( + ErrNoSuchService = errors.New("no such service") + ErrIllegalTaskType = errors.New("illegal task type") + ErrCanNotDropped = errors.New("disk can not dropped") + + // error code + ErrNothingTodo = Error(CodeNotingTodo) + ErrNoInspect = Error(CodeNoInspect) +) + +// worker +var ( + ErrShardMayBeLost = errors.New("shard may be lost") + // error code + ErrOrphanShard = Error(CodeOrphanShard) + ErrIllegalTask = Error(CodeIllegalTask) + ErrDestReplicaBad = Error(CodeDestReplicaBad) +) + +// +var ( + ErrClusterIDNotMatch = Error(CodeClusterIDNotMatch) +) diff --git a/blobstore/common/kafka/monitor.go b/blobstore/common/kafka/monitor.go new file mode 100644 index 000000000..9b24a5089 --- /dev/null +++ b/blobstore/common/kafka/monitor.go @@ -0,0 +1,221 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package kafka + +import ( + "fmt" + "sync" + "time" + + "github.com/Shopify/sarama" + "github.com/prometheus/client_golang/prometheus" + + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +var mockTestKafkaClient sarama.Client + +func newKafkaOffsetGauge() *prometheus.GaugeVec { + gaugeOpts := prometheus.GaugeOpts{ + Namespace: "kafka", + Subsystem: "topic_partition", + Name: "offset", + Help: "monitor kafka newest oldest and consume offset", + } + labelNames := []string{"module_name", "cluster_id", "topic", "partition", "type"} + kafkaOffsetGaugeVec := prometheus.NewGaugeVec(gaugeOpts, labelNames) + + err := prometheus.Register(kafkaOffsetGaugeVec) + if err == nil { + return kafkaOffsetGaugeVec + } + if are, ok := err.(prometheus.AlreadyRegisteredError); ok { + return are.ExistingCollector.(*prometheus.GaugeVec) + } + panic(err) +} + +func newKafkaLatencyGauge() *prometheus.GaugeVec { + gaugeOpts := prometheus.GaugeOpts{ + Namespace: "kafka", + Subsystem: "topic_partition", + Name: "consume_lag", + Help: "monitor kafka latency", + } + labelNames := []string{"module_name", "cluster_id", "topic", "partition"} + kafkaLatencyGaugeVec := prometheus.NewGaugeVec(gaugeOpts, labelNames) + + err := prometheus.Register(kafkaLatencyGaugeVec) + if err == nil { + return kafkaLatencyGaugeVec + } + if are, ok := err.(prometheus.AlreadyRegisteredError); ok { + return are.ExistingCollector.(*prometheus.GaugeVec) + } + panic(err) +} + +type offsetMap struct { + offsetMap map[int32]int64 + OffsetLock sync.RWMutex +} + +func newOffsetMap() *offsetMap { + m := make(map[int32]int64) + retOffsetMap := offsetMap{offsetMap: m} + return &retOffsetMap +} + +func (o *offsetMap) getOffset(pid int32) int64 { + o.OffsetLock.RLock() + defer o.OffsetLock.RUnlock() + if _, ok := o.offsetMap[pid]; ok { + return o.offsetMap[pid] + } + return 0 +} + +func (o *offsetMap) setOffset(offset int64, pid int32) { + o.OffsetLock.Lock() + defer o.OffsetLock.Unlock() + o.offsetMap[pid] = offset +} + +type Monitor struct { + clusterID proto.ClusterID + kafkaClient sarama.Client + topic string + pids []int32 + newestOffsetMap *offsetMap + oldestOffsetMap *offsetMap + consumeOffsetMap *offsetMap + kafkaOffAcquireIntervalSecs int64 + offsetGauge *prometheus.GaugeVec + latencyGauge *prometheus.GaugeVec + moduleName string +} + +const DefauleintervalSecs = 60 + +func NewKafkaMonitor( + clusterID proto.ClusterID, + moduleName string, + brokerHosts []string, + topic string, + pids []int32, + intervalSecs int64) (*Monitor, error, +) { + monitor := Monitor{ + clusterID: clusterID, + topic: topic, + pids: pids, + newestOffsetMap: newOffsetMap(), + oldestOffsetMap: newOffsetMap(), + consumeOffsetMap: newOffsetMap(), + kafkaOffAcquireIntervalSecs: DefauleintervalSecs, + moduleName: moduleName, + } + + if intervalSecs == 0 { + intervalSecs = DefauleintervalSecs + } + monitor.kafkaOffAcquireIntervalSecs = intervalSecs + + if mockTestKafkaClient != nil { + monitor.kafkaClient = mockTestKafkaClient + } else { + client, err := sarama.NewClient(brokerHosts, nil) + if err != nil { + return nil, err + } + monitor.kafkaClient = client + } + + monitor.offsetGauge = newKafkaOffsetGauge() + monitor.latencyGauge = newKafkaLatencyGauge() + + go monitor.loopAcquireKafkaOffset() + + return &monitor, nil +} + +func (monitor *Monitor) loopAcquireKafkaOffset() { + for { + for _, pid := range monitor.pids { + newestOffset, err := monitor.kafkaClient.GetOffset(monitor.topic, pid, sarama.OffsetNewest) + if err != nil { + log.Error(fmt.Sprintf("get newest offset fail topic %v pid %v ", monitor.topic, pid)) + continue + } + log.Debug("loopAcquireKafkaOffset newestOffset:", newestOffset) + monitor.newestOffsetMap.setOffset(newestOffset, pid) + + oldestOffset, err := monitor.kafkaClient.GetOffset(monitor.topic, pid, sarama.OffsetOldest) + if err != nil { + log.Error(fmt.Sprintf("get oldest offset fail topic %v pid %v ", monitor.topic, pid)) + continue + } + log.Debug("loopAcquireKafkaOffset oldestOffset:", oldestOffset) + monitor.oldestOffsetMap.setOffset(oldestOffset, pid) + } + + monitor.report() + time.Sleep(time.Duration(monitor.kafkaOffAcquireIntervalSecs) * time.Second) + } +} + +func (monitor *Monitor) report() { + for _, pid := range monitor.pids { + oldestOffset := monitor.oldestOffsetMap.getOffset(pid) + newestOffset := monitor.newestOffsetMap.getOffset(pid) + consumeOffset := monitor.consumeOffsetMap.getOffset(pid) + latency := newestOffset - consumeOffset - 1 //-1，because the newestOffset is the next message offset + if latency < 0 { + latency = 0 + } + + monitor.reportOffsetMetric(pid, string("oldest"), float64(oldestOffset)) + monitor.reportOffsetMetric(pid, string("newest"), float64(newestOffset)) + monitor.reportOffsetMetric(pid, string("consume"), float64(consumeOffset)) + monitor.reportLatencyMetric(pid, float64(latency)) + log.Debug("Report...") + } +} + +func (monitor *Monitor) reportOffsetMetric(pid int32, metricType string, val float64) { + labels := prometheus.Labels{ + "module_name": monitor.moduleName, + "cluster_id": monitor.clusterID.ToString(), + "topic": monitor.topic, + "partition": fmt.Sprintf("%d", pid), + "type": metricType, + } + monitor.offsetGauge.With(labels).Set(val) +} + +func (monitor *Monitor) reportLatencyMetric(pid int32, val float64) { + labels := prometheus.Labels{ + "module_name": monitor.moduleName, + "cluster_id": monitor.clusterID.ToString(), + "topic": monitor.topic, + "partition": fmt.Sprintf("%d", pid), + } + monitor.latencyGauge.With(labels).Set(val) +} + +func (monitor *Monitor) SetConsumeOffset(consumerOff int64, pid int32) { + monitor.consumeOffsetMap.setOffset(consumerOff, pid) +} diff --git a/blobstore/common/kafka/monitor_test.go b/blobstore/common/kafka/monitor_test.go new file mode 100644 index 000000000..b9d45592d --- /dev/null +++ b/blobstore/common/kafka/monitor_test.go @@ -0,0 +1,103 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package kafka + +import ( + "testing" + + "github.com/cubefs/cubefs/blobstore/common/proto" + + "github.com/Shopify/sarama" +) + +type MockKafkaClient struct{} + +func (c *MockKafkaClient) Config() *sarama.Config { + return nil +} + +func (c *MockKafkaClient) Controller() (*sarama.Broker, error) { + return nil, nil +} + +func (c *MockKafkaClient) Brokers() []*sarama.Broker { + return nil +} + +func (c *MockKafkaClient) Topics() ([]string, error) { + return nil, nil +} + +func (c *MockKafkaClient) Partitions(topic string) ([]int32, error) { + return nil, nil +} + +func (c *MockKafkaClient) WritablePartitions(topic string) ([]int32, error) { + return nil, nil +} + +func (c *MockKafkaClient) Leader(topic string, partitionID int32) (*sarama.Broker, error) { + return nil, nil +} + +func (c *MockKafkaClient) Replicas(topic string, partitionID int32) ([]int32, error) { + return nil, nil +} + +func (c *MockKafkaClient) InSyncReplicas(topic string, partitionID int32) ([]int32, error) { + return nil, nil +} + +func (c *MockKafkaClient) OfflineReplicas(topic string, partitionID int32) ([]int32, error) { + return nil, nil +} + +func (c *MockKafkaClient) RefreshMetadata(topics ...string) error { + return nil +} + +func (c *MockKafkaClient) GetOffset(topic string, partitionID int32, time int64) (int64, error) { + if time == sarama.OffsetNewest { + return 100, nil + } + return 1, nil +} + +func (c *MockKafkaClient) Coordinator(consumerGroup string) (*sarama.Broker, error) { + return nil, nil +} + +func (c *MockKafkaClient) RefreshCoordinator(consumerGroup string) error { + return nil +} + +func (c *MockKafkaClient) InitProducerID() (*sarama.InitProducerIDResponse, error) { + return nil, nil +} + +func (c *MockKafkaClient) Close() error { + return nil +} + +func (c *MockKafkaClient) Closed() bool { + return true +} + +func TestSetConsumeOffset(t *testing.T) { + brokens := []string{"127.0.01:9092"} + mockTestKafkaClient = &MockKafkaClient{} + monitor, _ := NewKafkaMonitor(proto.ClusterID(1), "", brokens, "Test_monitor", []int32{0, 1, 2}, 10) + monitor.SetConsumeOffset(1, 1) +} diff --git a/blobstore/common/kafka/msg_sender.go b/blobstore/common/kafka/msg_sender.go new file mode 100644 index 000000000..5e16bc1d0 --- /dev/null +++ b/blobstore/common/kafka/msg_sender.go @@ -0,0 +1,85 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package kafka + +import ( + "time" + + "github.com/Shopify/sarama" + + "github.com/cubefs/cubefs/blobstore/util/log" +) + +var DefaultKafkaVersion = sarama.V0_10_0_0 + +type MsgProducer interface { + SendMessage(topic string, msg []byte) (err error) + SendMessages(topic string, msgs [][]byte) (err error) +} + +type ProducerCfg struct { + BrokerList []string `json:"broker_list"` + Topic string `json:"topic"` + TimeoutMs int64 `json:"timeout_ms"` +} + +type Producer struct { + sarama.SyncProducer +} + +func defaultCfg() *sarama.Config { + config := sarama.NewConfig() + config.Producer.Return.Successes = true + config.Producer.RequiredAcks = sarama.WaitForAll + config.Version = DefaultKafkaVersion + return config +} + +func NewProducer(cfg *ProducerCfg) (*Producer, error) { + config := defaultCfg() + if cfg.TimeoutMs <= 0 { + cfg.TimeoutMs = 1000 + } + config.Producer.Timeout = time.Duration(cfg.TimeoutMs) * time.Millisecond + + producer, err := sarama.NewSyncProducer(cfg.BrokerList, config) + if err != nil { + log.Error("sarama.NewClient:", err) + return nil, err + } + return &Producer{producer}, err +} + +func (p *Producer) SendMessage(topic string, msg []byte) (err error) { + m := &sarama.ProducerMessage{ + Topic: topic, + Timestamp: time.Now(), + Value: sarama.ByteEncoder(msg), + } + _, _, err = p.SyncProducer.SendMessage(m) + return err +} + +func (p *Producer) SendMessages(topic string, msgs [][]byte) (err error) { + sendMsgs := make([]*sarama.ProducerMessage, len(msgs)) + for idx, msg := range msgs { + sendMsgs[idx] = &sarama.ProducerMessage{ + Topic: topic, + Timestamp: time.Now(), + Value: sarama.ByteEncoder(msg), + } + } + return p.SyncProducer.SendMessages(sendMsgs) +} diff --git a/blobstore/common/mongoutil/mongoutil.go b/blobstore/common/mongoutil/mongoutil.go new file mode 100644 index 000000000..c717c64bb --- /dev/null +++ b/blobstore/common/mongoutil/mongoutil.go @@ -0,0 +1,110 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package mongoutil + +import ( + "context" + "errors" + "strings" + "time" + + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/mongo/readconcern" + "go.mongodb.org/mongo-driver/mongo/writeconcern" +) + +// Config for mongo client +type Config struct { + URI string `json:"uri"` + TimeoutMs int64 `json:"timeout_ms"` + WriteConcern *WriteConcernConfig `json:"write_concern"` + ReadConcern string `json:"read_concern"` +} + +// for detail：https://docs.mongodb.com/manual/reference/write-concern/。 +type WriteConcernConfig struct { + Majority bool `json:"majority"` + TimeoutMs int64 `json:"timeout_ms"` +} + +var DefaultWriteConfig = WriteConcernConfig{ + Majority: true, + TimeoutMs: 3000, +} + +const ( + // ReadConcernLocal mean https://docs.mongodb.com/manual/reference/read-concern-local/#readconcern.%22local%22 + ReadConcernLocal = "local" + // ReadConcernAvailable mean https://docs.mongodb.com/manual/reference/read-concern-available/#readconcern.%22available%22 + ReadConcernAvailable = "available" + // ReadConcernMajority mean https://docs.mongodb.com/manual/reference/read-concern-majority/#readconcern.%22majority%22 + ReadConcernMajority = "majority" + // ReadConcernLinearizable mean https://docs.mongodb.com/manual/reference/read-concern-linearizable/#readconcern.%22linearizable%22 + ReadConcernLinearizable = "linearizable" + // ReadConcernSnapshot mean https://docs.mongodb.com/manual/reference/read-concern-snapshot/#readconcern.%22snapshot%22 + ReadConcernSnapshot = "snapshot" +) + +func checkValidWriteConcern(s string) error { + switch s { + case ReadConcernLocal, ReadConcernAvailable, ReadConcernMajority, ReadConcernLinearizable, ReadConcernSnapshot: + return nil + default: + return errors.New("invalid write concern") + } +} + +func GetClient(conf Config) (*mongo.Client, error) { + opt := options.Client().ApplyURI(conf.URI) + if conf.TimeoutMs > 0 { + timeoutDur := time.Duration(conf.TimeoutMs) * time.Millisecond + opt.SetConnectTimeout(timeoutDur) + opt.SetServerSelectionTimeout(timeoutDur) + opt.SetSocketTimeout(timeoutDur) + } + if wcConf := conf.WriteConcern; wcConf != nil { + var wcOpts []writeconcern.Option + if wcConf.Majority { + wcOpts = append(wcOpts, writeconcern.WMajority()) + } + if wcConf.TimeoutMs > 0 { + wcOpts = append(wcOpts, writeconcern.WTimeout(time.Duration(wcConf.TimeoutMs)*time.Millisecond)) + } + wc := writeconcern.New(wcOpts...) + opt.SetWriteConcern(wc) + } + if conf.ReadConcern != "" { + if err := checkValidWriteConcern(conf.ReadConcern); err != nil { + return nil, err + } + opt.SetReadConcern(readconcern.New(readconcern.Level(conf.ReadConcern))) + } + + client, err := mongo.NewClient(opt) + if err != nil { + return nil, err + } + err = client.Connect(context.TODO()) + if err != nil { + return nil, err + } + return client, nil +} + +// IsDupError : err is mongo E11000?。 +func IsDupError(err error) bool { + return strings.Contains(err.Error(), "E11000") +} diff --git a/blobstore/common/proto/basic.go b/blobstore/common/proto/basic.go new file mode 100644 index 000000000..f60ff4dab --- /dev/null +++ b/blobstore/common/proto/basic.go @@ -0,0 +1,77 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package proto + +import ( + "encoding/binary" + "fmt" + "strconv" + "strings" +) + +// basic type for all module +type ( + DiskID uint32 + BlobID uint64 + Vid uint32 + ClusterID uint32 +) + +func (id DiskID) Encode() []byte { + key := make([]byte, 4) + binary.BigEndian.PutUint32(key, uint32(id)) + return key +} + +func (id *DiskID) Decode(b []byte) DiskID { + key := binary.BigEndian.Uint32(b) + *id = DiskID(key) + return *id +} + +func (id DiskID) ToString() string { + return strconv.FormatUint(uint64(id), 10) +} + +func (vid Vid) ToString() string { + return strconv.FormatUint(uint64(vid), 10) +} + +func (id ClusterID) ToString() string { + return strconv.FormatUint(uint64(id), 10) +} + +const seqToken = ";" + +// EncodeToken encode host and vid to a string token. +func EncodeToken(host string, vid Vid) (token string) { + return fmt.Sprintf("%s%s%s", host, seqToken, strconv.FormatUint(uint64(vid), 10)) +} + +// DecodeToken decode host and vid from the token. +func DecodeToken(token string) (host string, vid Vid, err error) { + parts := strings.SplitN(token, seqToken, 2) + if len(parts) != 2 { + err = fmt.Errorf("invalid token %s", token) + return + } + host = parts[0] + vidU32, err := strconv.ParseUint(parts[1], 10, 32) + if err != nil { + return + } + vid = Vid(vidU32) + return +} diff --git a/blobstore/common/proto/basic_test.go b/blobstore/common/proto/basic_test.go new file mode 100644 index 000000000..0e6d58001 --- /dev/null +++ b/blobstore/common/proto/basic_test.go @@ -0,0 +1,66 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package proto_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +func TestProtoDiskID(t *testing.T) { + id := proto.DiskID(9) + bytes := []byte{0x00, 0x00, 0x00, 0x09} + require.Equal(t, uint32(9), uint32(id)) + require.Equal(t, bytes, id.Encode()) + require.Equal(t, "9", id.ToString()) + + var dec proto.DiskID + require.Equal(t, id, dec.Decode(bytes)) + require.Equal(t, id, dec) +} + +func TestProtoVID(t *testing.T) { + id := proto.Vid(123) + require.Equal(t, "123", id.ToString()) +} + +func TestProtoClusterID(t *testing.T) { + id := proto.ClusterID(10) + require.Equal(t, "10", id.ToString()) +} + +func TestProtoToken(t *testing.T) { + host := "127.0.0.1:80" + vid := proto.Vid(123) + token := "127.0.0.1:80;123" + require.Equal(t, token, proto.EncodeToken(host, vid)) + { + newHost, newVid, err := proto.DecodeToken(token) + require.NoError(t, err) + require.Equal(t, host, newHost) + require.Equal(t, vid, newVid) + } + { + _, _, err := proto.DecodeToken(host) + require.Error(t, err) + } + { + _, _, err := proto.DecodeToken(token + ";") + require.Error(t, err) + } +} diff --git a/blobstore/common/proto/const.go b/blobstore/common/proto/const.go new file mode 100644 index 000000000..abf98379b --- /dev/null +++ b/blobstore/common/proto/const.go @@ -0,0 +1,108 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package proto + +import ( + "math" +) + +// service names +const ( + ServiceNameBlobNode = "BLOBNODE" + ServiceNameProxy = "PROXY" + ServiceNameScheduler = "SCHEDULER" +) + +type DiskStatus uint8 + +// disk status +const ( + DiskStatusNormal = DiskStatus(iota + 1) // 1 + DiskStatusBroken // 2 + DiskStatusRepairing // 3 + DiskStatusRepaired // 4 + DiskStatusDropped // 5 + DiskStatusMax // 6 +) + +func (status DiskStatus) IsValid() bool { + return status >= DiskStatusNormal && status < DiskStatusMax +} + +func (status DiskStatus) String() string { + switch status { + case DiskStatusNormal: + return "normal" + case DiskStatusBroken: + return "broken" + case DiskStatusRepairing: + return "repairing" + case DiskStatusRepaired: + return "repaired" + case DiskStatusDropped: + return "dropped" + default: + return "unknown" + } +} + +const ( + InvalidDiskID = DiskID(0) + InValidBlobID = BlobID(0) + InvalidCrc32 = uint32(0) + InvalidVid = Vid(0) + InvalidVuid = Vuid(0) +) + +const ( + MaxBlobID = BlobID(math.MaxUint64) +) + +// volume status +type VolumeStatus uint8 + +func (status VolumeStatus) IsValid() bool { + return status > volumeStatusMin && status < volumeStatusMax +} + +func (status VolumeStatus) String() string { + switch status { + case VolumeStatusIdle: + return "idle" + case VolumeStatusActive: + return "active" + case VolumeStatusLock: + return "lock" + case VolumeStatusUnlocking: + return "unlocking" + } + return "unknown" +} + +const ( + volumeStatusMin = VolumeStatus(iota) + VolumeStatusIdle + VolumeStatusActive + VolumeStatusLock + VolumeStatusUnlocking + volumeStatusMax +) + +// config key +const ( + CodeModeConfigKey = "code_mode" + VolumeReserveSizeKey = "volume_reserve_size" + VolumeChunkSizeKey = "volume_chunk_size" +) diff --git a/blobstore/common/proto/const_test.go b/blobstore/common/proto/const_test.go new file mode 100644 index 000000000..ae8ddbe82 --- /dev/null +++ b/blobstore/common/proto/const_test.go @@ -0,0 +1,43 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package proto_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +func TestProtoDiskStatus(t *testing.T) { + for st := proto.DiskStatusNormal; st < proto.DiskStatusMax; st++ { + require.True(t, st.IsValid()) + t.Logf("disk st %d -> %s", st, st) + } + st := proto.DiskStatus(0xff) + require.False(t, st.IsValid()) + t.Logf("disk st %d -> %s", st, st) +} + +func TestProtoVolumeStatus(t *testing.T) { + for st := proto.VolumeStatusIdle; st <= proto.VolumeStatusUnlocking; st++ { + require.True(t, st.IsValid()) + t.Logf("volume st %d -> %s", st, st) + } + st := proto.VolumeStatus(0xff) + require.False(t, st.IsValid()) + t.Logf("volume st %d -> %s", st, st) +} diff --git a/blobstore/common/proto/mqproxy_types.go b/blobstore/common/proto/mqproxy_types.go new file mode 100644 index 000000000..a46ee3174 --- /dev/null +++ b/blobstore/common/proto/mqproxy_types.go @@ -0,0 +1,103 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package proto + +import ( + "github.com/cubefs/cubefs/blobstore/util/errors" +) + +var ErrInvalidMsg = errors.New("msg is invalid") + +type DeleteStage byte + +const ( + InitStage DeleteStage = iota + MarkDelStage + DelStage +) + +type BlobDeleteStage struct { + Stages map[uint8]DeleteStage `json:"stages"` +} + +func (s *BlobDeleteStage) SetStage(vuidIdx uint8, stage DeleteStage) { + if s.Stages == nil { + s.Stages = make(map[uint8]DeleteStage) + } + s.Stages[vuidIdx] = stage +} + +func (s *BlobDeleteStage) Stage(vuid Vuid) (DeleteStage, bool) { + stage, exist := s.Stages[vuid.Index()] + return stage, exist +} + +func (s *BlobDeleteStage) Copy() BlobDeleteStage { + myCopy := BlobDeleteStage{} + myCopy.Stages = make(map[uint8]DeleteStage) + for k, v := range s.Stages { + myCopy.Stages[k] = v + } + return myCopy +} + +type DeleteMsg struct { + ClusterID ClusterID `json:"cluster_id"` + Bid BlobID `json:"bid"` + Vid Vid `json:"vid"` + Retry int `json:"retry"` + Time int64 `json:"time"` + ReqId string `json:"req_id"` + BlobDelStages BlobDeleteStage `json:"blob_del_stages"` +} + +func (msg *DeleteMsg) IsValid() bool { + if msg.Bid == InValidBlobID { + return false + } + if msg.Vid == InvalidVid { + return false + } + return true +} + +func (msg *DeleteMsg) SetDeleteStage(stage BlobDeleteStage) { + for idx, s := range stage.Stages { + msg.BlobDelStages.SetStage(idx, s) + } +} + +type ShardRepairMsg struct { + ClusterID ClusterID `json:"cluster_id"` + Bid BlobID `json:"bid"` + Vid Vid `json:"vid"` + BadIdx []uint8 `json:"bad_idx"` + Retry int `json:"retry"` + Reason string `json:"reason"` + ReqId string `json:"req_id"` +} + +func (msg *ShardRepairMsg) IsValid() bool { + if msg.Bid == InValidBlobID { + return false + } + if msg.Vid == InvalidVid { + return false + } + if len(msg.BadIdx) == 0 { + return false + } + return true +} diff --git a/blobstore/common/proto/mqproxy_types_test.go b/blobstore/common/proto/mqproxy_types_test.go new file mode 100644 index 000000000..9009a6404 --- /dev/null +++ b/blobstore/common/proto/mqproxy_types_test.go @@ -0,0 +1,114 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package proto + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestShardRepairMsg_IsValid(t *testing.T) { + msg := ShardRepairMsg{ + ClusterID: 1, + Bid: 1, + Vid: 1, + BadIdx: []uint8{1}, + Retry: 0, + Reason: "access", + } + require.Equal(t, true, msg.IsValid()) + + msg = ShardRepairMsg{ + ClusterID: 1, + Bid: 0, + Vid: 1, + BadIdx: []uint8{1}, + Retry: 0, + Reason: "access", + } + require.Equal(t, false, msg.IsValid()) + + msg = ShardRepairMsg{ + ClusterID: 1, + Bid: 1, + Vid: 0, + BadIdx: []uint8{1}, + Retry: 0, + Reason: "access", + } + require.Equal(t, false, msg.IsValid()) + + msg = ShardRepairMsg{ + ClusterID: 1, + Bid: 1, + Vid: 1, + BadIdx: []uint8{}, + Retry: 0, + Reason: "access", + } + require.Equal(t, false, msg.IsValid()) +} + +func TestDeleteMsg_IsValid(t *testing.T) { + msg := DeleteMsg{ + ClusterID: 1, + Bid: 1, + Vid: 1, + Retry: 0, + Time: 0, + } + require.Equal(t, true, msg.IsValid()) + + msg = DeleteMsg{ + ClusterID: 1, + Bid: 0, + Vid: 1, + Retry: 0, + Time: 0, + } + require.Equal(t, false, msg.IsValid()) + + msg = DeleteMsg{ + ClusterID: 1, + Bid: 1, + Vid: 0, + Retry: 0, + Time: 0, + } + require.Equal(t, false, msg.IsValid()) +} + +func TestMsgMarshal(t *testing.T) { + stags := BlobDeleteStage{} + stags.SetStage(1, 1) + stags.SetStage(2, 1) + msg := DeleteMsg{ + ClusterID: 1, + Bid: 1, + Vid: 0, + Retry: 0, + Time: 0, + BlobDelStages: stags, + } + b, err := json.Marshal(msg) + require.NoError(t, err) + + var delMsg DeleteMsg + err = json.Unmarshal(b, &delMsg) + require.NoError(t, err) + t.Logf("del msg %+v", delMsg) +} diff --git a/blobstore/common/proto/scheduler_types.go b/blobstore/common/proto/scheduler_types.go new file mode 100644 index 000000000..c9760b1a8 --- /dev/null +++ b/blobstore/common/proto/scheduler_types.go @@ -0,0 +1,269 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package proto + +import ( + "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/cubefs/cubefs/blobstore/util/errors" +) + +var ( + ErrTaskPaused = errors.New("task has paused") + ErrTaskEmpty = errors.New("no task to run") +) + +const ( + // TaskRenewalPeriodS + RenewalTimeoutS < TaskLeaseExpiredS + TaskRenewalPeriodS = 5 // worker alive tasks renewal period + RenewalTimeoutS = 1 // timeout of worker task renewal + TaskLeaseExpiredS = 10 // task lease duration in scheduler +) + +type VunitLocation struct { + Vuid Vuid `json:"vuid" bson:"vuid"` + Host string `json:"host" bson:"host"` + DiskID DiskID `json:"disk_id" bson:"disk_id"` +} + +// for task check +func CheckVunitLocations(locations []VunitLocation) bool { + if len(locations) == 0 { + return false + } + + for _, l := range locations { + if l.Vuid == InvalidVuid || l.Host == "" || l.DiskID == InvalidDiskID { + return false + } + } + return true +} + +const ( + RepairTaskType = "repair_task" + BalanceTaskType = "balance_task" + DiskDropTaskType = "disk_drop_task" + ManualMigrateType = "manual_migrate" +) + +var _taskType = map[string]struct{}{ + RepairTaskType: {}, + BalanceTaskType: {}, + DiskDropTaskType: {}, + ManualMigrateType: {}, +} + +func ValidTaskType(task string) bool { + _, ok := _taskType[task] + return ok +} + +type RepairState uint8 + +const ( + RepairStateInited RepairState = iota + 1 + RepairStatePrepared + RepairStateWorkCompleted + RepairStateFinished + RepairStateFinishedInAdvance +) + +const ( + BrokenDiskTrigger = 0 + BrokenStripeTrigger = 1 +) + +type VolRepairTask struct { + TaskID string `json:"task_id" bson:"_id"` + State RepairState `json:"state" bson:"state"` + WorkerRedoCnt uint8 `json:"worker_redo_cnt" bson:"worker_redo_cnt"` + RepairDiskID DiskID `json:"repair_disk_id" bson:"repair_disk_id"` + + CodeMode codemode.CodeMode `json:"code_mode" bson:"code_mode"` + Sources []VunitLocation `json:"sources" bson:"sources"` // include all replicas of volumes + Destination VunitLocation `json:"destination" bson:"destination"` + + BadVuid Vuid `json:"bad_vuid"` + BadIdx uint8 `json:"bad_idx" bson:"bad_idx"` // index of repair replica in volume replicas + + BrokenDiskIDC string `json:"broken_disk_idc"` + + Ctime string `json:"ctime" bson:"ctime"` // task create time + MTime string `json:"mtime" bson:"mtime"` // task modify time + + // BrokenDiskTrigger: trigger by broken disk, + // BrokenStripeTrigger: trigger by stripe which has broken replica + TriggerBy int `json:"trigger_by" bson:"trigger_by"` +} + +func (t *VolRepairTask) GetSrc() []VunitLocation { + return t.Sources +} + +func (t *VolRepairTask) GetDest() VunitLocation { + return t.Destination +} + +func (t *VolRepairTask) SetDest(dst VunitLocation) { + t.Destination = dst +} + +func (t *VolRepairTask) NewDiskId() DiskID { + return t.Destination.DiskID +} + +func (t *VolRepairTask) Vid() Vid { + return t.BadVuid.Vid() +} + +func (t *VolRepairTask) RepairVuid() Vuid { + return t.BadVuid +} + +func (t *VolRepairTask) Finished() bool { + return t.State == RepairStateFinished || t.State == RepairStateFinishedInAdvance +} + +func (t *VolRepairTask) Running() bool { + return t.State == RepairStatePrepared || t.State == RepairStateWorkCompleted +} + +func (t *VolRepairTask) Copy() *VolRepairTask { + task := &VolRepairTask{} + *task = *t + dst := make([]VunitLocation, len(t.Sources)) + copy(dst, t.Sources) + task.Sources = dst + return task +} + +type MigrateState uint8 + +const ( + MigrateStateInited MigrateState = iota + 1 + MigrateStatePrepared + MigrateStateWorkCompleted + MigrateStateFinished + MigrateStateFinishedInAdvance +) + +type MigrateTask struct { + TaskID string `json:"task_id" bson:"_id"` // task id + State MigrateState `json:"state" bson:"state"` // task state + WorkerRedoCnt uint8 `json:"worker_redo_cnt" bson:"worker_redo_cnt"` // worker redo task count + + SourceIdc string `json:"source_idc" bson:"source_idc"` // source idc + SourceDiskID DiskID `json:"source_disk_id" bson:"source_disk_id"` // source disk id + SourceVuid Vuid `json:"source_vuid" bson:"source_vuid"` // source volume unit id + + Sources []VunitLocation `json:"sources" bson:"sources"` // source volume units location + CodeMode codemode.CodeMode `json:"code_mode" bson:"code_mode"` // codemode + + Destination VunitLocation `json:"destination" bson:"destination"` // destination volume unit location + + Ctime string `json:"ctime" bson:"ctime"` // create time + MTime string `json:"mtime" bson:"mtime"` // modify time + + FinishAdvanceReason string `json:"finish_advance_reason" bson:"finish_advance_reason"` + // task migrate chunk direct download first,if fail will recover chunk by ec repair + ForbiddenDirectDownload bool `json:"forbidden_direct_download" bson:"forbidden_direct_download"` +} + +func (t *MigrateTask) GetSrc() []VunitLocation { + return t.Sources +} + +func (t *MigrateTask) GetDest() VunitLocation { + return t.Destination +} + +func (t *MigrateTask) SetDest(dest VunitLocation) { + t.Destination = dest +} + +func (t *MigrateTask) DestinationDiskId() DiskID { + return t.Destination.DiskID +} + +func (t *MigrateTask) Running() bool { + return t.State == MigrateStatePrepared || t.State == MigrateStateWorkCompleted +} + +func (t *MigrateTask) Finished() bool { + return t.State == MigrateStateFinished || t.State == MigrateStateFinishedInAdvance +} + +func (t *MigrateTask) Copy() *MigrateTask { + task := &MigrateTask{} + *task = *t + dst := make([]VunitLocation, len(t.Sources)) + copy(dst, t.Sources) + task.Sources = dst + return task +} + +func (t *MigrateTask) SrcMigDiskID() DiskID { + return t.SourceDiskID +} + +type InspectCheckPoint struct { + Id string `json:"_id" bson:"_id"` + StartVid Vid `json:"start_vid" bson:"start_vid"` // min vid in current batch volumes + Ctime string `json:"ctime" bson:"ctime"` +} + +type InspectTask struct { + TaskId string `json:"task_id"` + Mode codemode.CodeMode `json:"mode"` + Replicas []VunitLocation `json:"replicas"` +} + +type MissedShard struct { + Vuid Vuid `json:"vuid"` + Bid BlobID `json:"bid"` +} + +type InspectRet struct { + TaskID string `json:"task_id"` + InspectErrStr string `json:"inspect_err_str"` // inspect run success or not + MissedShards []*MissedShard `json:"missed_shards"` +} + +func (inspect *InspectRet) Err() error { + if len(inspect.InspectErrStr) == 0 { + return nil + } + return errors.New(inspect.InspectErrStr) +} + +// ArchiveRecord archive record +type ArchiveRecord struct { + TaskID string `bson:"_id"` + TaskType string `bson:"task_type"` + ArchiveTime string `bson:"archive_time"` + Content interface{} `bson:"content"` +} + +type ShardRepairTask struct { + Bid BlobID `json:"bid"` + CodeMode codemode.CodeMode `json:"code_mode"` + Sources []VunitLocation `json:"sources"` + BadIdxs []uint8 `json:"bad_idxs"` // TODO: BadIdxes + Reason string `json:"reason"` +} + +func (task *ShardRepairTask) IsValid() bool { + return task.CodeMode.IsValid() && CheckVunitLocations(task.Sources) +} diff --git a/blobstore/common/proto/vuid.go b/blobstore/common/proto/vuid.go new file mode 100644 index 000000000..7327f9da6 --- /dev/null +++ b/blobstore/common/proto/vuid.go @@ -0,0 +1,93 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package proto + +import ( + "errors" + "strconv" +) + +type ( + Vuid uint64 + VuidPrefix uint64 +) + +const ( + MinEpoch = 1 + MaxEpoch = 16777215 + MinIndex = 0 + MaxIndex = 255 +) + +func (vu Vuid) IsValid() bool { + return vu > InvalidVuid && IsValidEpoch(vu.Epoch()) && IsValidIndex(vu.Index()) +} + +func NewVuid(vid Vid, idx uint8, epoch uint32) (Vuid, error) { + if !IsValidEpoch(epoch) { + err := errors.New("fail to new vuid,Epoch is overflow") + return 0, err + } + + u64 := uint64(vid)<<32 + uint64(idx)<<24 + uint64(epoch) + return Vuid(u64), nil +} + +func EncodeVuidPrefix(vid Vid, idx uint8) VuidPrefix { + u64 := uint64(vid)<<32 + uint64(idx)<<24 + return VuidPrefix(u64) +} + +func EncodeVuid(v VuidPrefix, epoch uint32) Vuid { + u64 := uint64(v) + uint64(epoch) + return Vuid(u64) +} + +func (v Vuid) Vid() Vid { + return Vid(v & 0xffffffff00000000 >> 32) +} + +func (v Vuid) ToString() string { + return strconv.FormatUint(uint64(v), 10) +} + +func (v Vuid) Index() uint8 { + return uint8(v & 0xff000000 >> 24) +} + +func (v Vuid) Epoch() uint32 { + return uint32(v & 0xffffff) +} + +func (v Vuid) VuidPrefix() VuidPrefix { + vuidPre := uint64(v) - uint64(v.Epoch()) + return VuidPrefix(vuidPre) +} + +func (v VuidPrefix) Vid() Vid { + return Vid(v & 0xffffffff00000000 >> 32) +} + +func (v VuidPrefix) Index() uint8 { + return uint8(v & 0xff000000 >> 24) +} + +func IsValidEpoch(epoch uint32) bool { + return epoch <= MaxEpoch && epoch >= MinEpoch +} + +func IsValidIndex(index uint8) bool { + return index <= MaxIndex && index >= MinIndex +} diff --git a/blobstore/common/proto/vuid_test.go b/blobstore/common/proto/vuid_test.go new file mode 100644 index 000000000..38da30e3e --- /dev/null +++ b/blobstore/common/proto/vuid_test.go @@ -0,0 +1,51 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package proto + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestVuid(t *testing.T) { + vid := Vid(rand.Uint32()) + index := uint8(rand.Intn(256)) + epoch := uint32(rand.Int31n(MaxEpoch)) + vuid, err := NewVuid(vid, index, epoch) + vuidPre := EncodeVuidPrefix(vid, index) + assert.NoError(t, err) + + assert.Equal(t, vuidPre, vuid.VuidPrefix()) + assert.Equal(t, vid, vuidPre.Vid()) + assert.Equal(t, index, vuidPre.Index()) + assert.Equal(t, vid, vuid.Vid()) + assert.Equal(t, index, vuid.Index()) + assert.Equal(t, epoch, vuid.Epoch()) +} + +func TestDecodeVuid(t *testing.T) { + for i := 0; i < 900000; i++ { + TestVuid(t) + } +} + +func TestDecodeVuid2(t *testing.T) { + old := Vuid(425335980033) + new := Vuid(116131889167) + t.Log(old.Vid(), old.VuidPrefix(), old.Index(), old.Epoch()) + t.Log(new.Vid(), new.VuidPrefix(), new.Index(), new.Epoch()) +} diff --git a/blobstore/common/proto/worker_types.go b/blobstore/common/proto/worker_types.go new file mode 100644 index 000000000..ffd71afaa --- /dev/null +++ b/blobstore/common/proto/worker_types.go @@ -0,0 +1,43 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package proto + +// stats task has done in worker +type TaskStatistics struct { + MigDataSizeByte uint64 `json:"mig_data_size_byte"` + MigShardCnt uint64 `json:"mig_shard_cnt"` + TotalDataSizeByte uint64 `json:"total_data_size_byte"` + TotalShardCnt uint64 `json:"total_shard_cnt"` + Progress uint64 `json:"progress"` +} + +func (self *TaskStatistics) Add(dataSize, shardCnt uint64) { + self.MigDataSizeByte += dataSize + self.MigShardCnt += shardCnt + if self.TotalDataSizeByte == 0 { + self.Progress = 100 + } else { + self.Progress = (self.MigDataSizeByte * 100) / self.TotalDataSizeByte + } +} + +func (self *TaskStatistics) InitTotal(totalDataSize, totalShardCnt uint64) { + self.TotalDataSizeByte = totalDataSize + self.TotalShardCnt = totalShardCnt +} + +func (self *TaskStatistics) Completed() bool { + return self.Progress == 100 +} diff --git a/blobstore/common/taskswitch/task_switch.go b/blobstore/common/taskswitch/task_switch.go new file mode 100644 index 000000000..104b81f59 --- /dev/null +++ b/blobstore/common/taskswitch/task_switch.go @@ -0,0 +1,180 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package taskswitch + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/cubefs/cubefs/blobstore/common/trace" +) + +type ISwitcher interface { + Enabled() bool + WaitEnable() +} + +// task switch name +const ( + DiskRepairSwitchName = "disk_repair" + BalanceSwitchName = "balance" + DiskDropSwitchName = "disk_drop" + BlobDeleteSwitchName = "blob_delete" + ShardRepairSwitchName = "shard_repair" + VolumeInspectSwitchName = "volume_inspect" +) + +const ( + syncTaskStatusIntervalS = 15 + SwitchOpen = "true" + SwitchClose = "false" +) + +var ( + ErrConflictSwitch = errors.New("switch has existed") + ErrNoSuchSwitch = errors.New("no such switch") +) + +type TaskSwitch struct { + mu sync.Mutex + enabled bool + wg sync.WaitGroup +} + +func newTaskSwitch() *TaskSwitch { + c := &TaskSwitch{ + enabled: true, + } + c.Disable() + return c +} + +func NewEnabledTaskSwitch() *TaskSwitch { + taskSwitch := newTaskSwitch() + taskSwitch.Enable() + return taskSwitch +} + +func (s *TaskSwitch) Enable() { + s.mu.Lock() + defer s.mu.Unlock() + if s.enabled { + return + } + s.enabled = true + s.wg.Done() +} + +func (s *TaskSwitch) Disable() { + s.mu.Lock() + defer s.mu.Unlock() + if !s.enabled { + return + } + s.enabled = false + s.wg.Add(1) +} + +func (s *TaskSwitch) Enabled() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.enabled +} + +func (s *TaskSwitch) WaitEnable() { + s.wg.Wait() +} + +type ConfigGetter interface { + GetConfig(ctx context.Context, key string) (val string, err error) +} + +type SwitchMgr struct { + switchs map[string]*TaskSwitch + mu sync.Mutex + cmCfgGetter ConfigGetter +} + +func NewSwitchMgr(cmCli ConfigGetter) *SwitchMgr { + sm := SwitchMgr{ + switchs: make(map[string]*TaskSwitch), + cmCfgGetter: cmCli, + } + go sm.loopUpdate() + return &sm +} + +func (sm *SwitchMgr) loopUpdate() { + for { + sm.update() + time.Sleep(syncTaskStatusIntervalS * time.Second) + } +} + +func (sm *SwitchMgr) update() { + sm.mu.Lock() + defer sm.mu.Unlock() + + span, ctx := trace.StartSpanFromContext(context.Background(), "") + + for switchName, taskSwitch := range sm.switchs { + statusStr, err := sm.cmCfgGetter.GetConfig(ctx, switchName) + if err != nil { + span.Errorf("Get Fail switchName %s err %v", switchName, err) + continue + } + + if switchStatus(statusStr) { + taskSwitch.Enable() + continue + } + taskSwitch.Disable() + } +} + +func (sm *SwitchMgr) AddSwitch(switchName string) (*TaskSwitch, error) { + sm.mu.Lock() + defer sm.mu.Unlock() + + if _, ok := sm.switchs[switchName]; ok { + return nil, ErrConflictSwitch + } + sm.switchs[switchName] = newTaskSwitch() + return sm.switchs[switchName], nil +} + +func (sm *SwitchMgr) DelSwitch(switchName string) error { + sm.mu.Lock() + defer sm.mu.Unlock() + + if _, ok := sm.switchs[switchName]; ok { + delete(sm.switchs, switchName) + return nil + } + return ErrNoSuchSwitch +} + +func switchStatus(statusStr string) (open bool) { + switch statusStr { + case SwitchOpen: + return true + case SwitchClose: + return false + default: + return false + } +} diff --git a/blobstore/common/taskswitch/task_switch_test.go b/blobstore/common/taskswitch/task_switch_test.go new file mode 100644 index 000000000..8dbc49212 --- /dev/null +++ b/blobstore/common/taskswitch/task_switch_test.go @@ -0,0 +1,67 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package taskswitch + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestTaskSwitch(t *testing.T) { + ts := newTaskSwitch() + require.Equal(t, false, ts.Enabled()) + ts.Enable() + require.Equal(t, true, ts.Enabled()) + ts.Disable() + require.Equal(t, false, ts.Enabled()) +} + +type mockCfgGetter struct { + m map[string]string +} + +func (cfgGetter *mockCfgGetter) GetConfig(ctx context.Context, key string) (val string, err error) { + if val, ok := cfgGetter.m[key]; ok { + return val, nil + } + return "", errors.New("no such key") +} + +func TestSwitchMgr(t *testing.T) { + cfgGetter := mockCfgGetter{ + m: make(map[string]string), + } + cfgGetter.m["switch1"] = SwitchOpen + cfgGetter.m["switch2"] = SwitchClose + sm := NewSwitchMgr(&cfgGetter) + s1, err := sm.AddSwitch("switch1") + require.NoError(t, err) + s2, err := sm.AddSwitch("switch2") + require.NoError(t, err) + + sm.update() + require.Equal(t, true, s1.Enabled()) + require.Equal(t, false, s2.Enabled()) + + sm.update() + err = sm.DelSwitch("switch1") + require.NoError(t, err) + err = sm.DelSwitch("switch2") + require.NoError(t, err) + require.Equal(t, 0, len(sm.switchs)) +} diff --git a/blobstore/common/trace/ext/tags.go b/blobstore/common/trace/ext/tags.go new file mode 100644 index 000000000..0abd13de2 --- /dev/null +++ b/blobstore/common/trace/ext/tags.go @@ -0,0 +1,66 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package ext + +import ( + "github.com/opentracing/opentracing-go/ext" +) + +var ( + // SpanKind (client/server or producer/consumer) + SpanKind = ext.SpanKind + SpanKindRPCClientEnum = ext.SpanKindRPCClientEnum + SpanKindRPCClient = ext.SpanKindRPCClient + SpanKindRPCServerEnum = ext.SpanKindRPCServerEnum + SpanKindRPCServer = ext.SpanKindRPCServer + SpanKindProducerEnum = ext.SpanKindProducerEnum + SpanKindProducer = ext.SpanKindProducer + SpanKindConsumerEnum = ext.SpanKindConsumerEnum + SpanKindConsumer = ext.SpanKindConsumer + + // Component name + Component = ext.Component + + // Sampling hint + SamplingPriority = ext.SamplingPriority + + // Peer tags + PeerService = ext.PeerService + PeerAddress = ext.PeerAddress + PeerHostname = ext.PeerHostname + PeerHostIPv4 = ext.PeerHostIPv4 + PeerHostIPv6 = ext.PeerHostIPv6 + PeerPort = ext.PeerPort + + // HTTP tags + HTTPUrl = ext.HTTPUrl + HTTPMethod = ext.HTTPMethod + HTTPStatusCode = ext.HTTPStatusCode + + // DB tags + DBInstance = ext.DBInstance + DBStatement = ext.DBStatement + DBType = ext.DBType + DBUser = ext.DBUser + + // Message Bus Tag + MessageBusDestination = ext.MessageBusDestination + + // Error Tag + Error = ext.Error +) + +// SpanKindEnum represents common span types +type SpanKindEnum ext.SpanKindEnum diff --git a/blobstore/common/trace/ext/tags_test.go b/blobstore/common/trace/ext/tags_test.go new file mode 100644 index 000000000..345863c6c --- /dev/null +++ b/blobstore/common/trace/ext/tags_test.go @@ -0,0 +1,141 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package ext + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/cubefs/cubefs/blobstore/common/trace" +) + +func TestPeerTags(t *testing.T) { + tracer := trace.NewTracer("blobstore") + span := tracer.StartSpan("my-trace").(trace.Span) + PeerService.Set(span, "my-service") + PeerAddress.Set(span, "my-hostname:8080") + PeerHostname.Set(span, "my-hostname") + PeerHostIPv4.Set(span, uint32(127<<24|1)) + PeerHostIPv4.SetString(span, "127.0.0.1") + PeerHostIPv6.Set(span, "::") + PeerPort.Set(span, uint16(8080)) + SamplingPriority.Set(span, uint16(1)) + SpanKind.Set(span, SpanKindRPCServerEnum) + SpanKindRPCClient.Set(span) + span.Finish() + + assert.Equal(t, trace.Tags{ + "peer.service": "my-service", + "peer.address": "my-hostname:8080", + "peer.hostname": "my-hostname", + "peer.ipv4": "127.0.0.1", + "peer.ipv6": "::", + "peer.port": uint16(8080), + "sampling.priority": uint16(1), + "span.kind": SpanKindRPCClientEnum, + }, span.Tags()) +} + +func TestHTTPTags(t *testing.T) { + tracer := trace.NewTracer("blobstore") + span := tracer.StartSpan("my-trace", SpanKindRPCServer).(trace.Span) + HTTPUrl.Set(span, "test.biz/uri?protocol=false") + HTTPMethod.Set(span, "GET") + HTTPStatusCode.Set(span, 301) + span.Finish() + + assert.Equal(t, trace.Tags{ + "http.url": "test.biz/uri?protocol=false", + "http.method": "GET", + "http.status_code": uint16(301), + "span.kind": SpanKindRPCServerEnum, + }, span.Tags()) +} + +func TestDBTags(t *testing.T) { + tracer := trace.NewTracer("blobstore") + span := tracer.StartSpan("my-trace", SpanKindRPCClient).(trace.Span) + DBInstance.Set(span, "127.0.0.1:3306/customers") + DBStatement.Set(span, "SELECT * FROM user_table") + DBType.Set(span, "sql") + DBUser.Set(span, "customer_user") + span.Finish() + + assert.Equal(t, trace.Tags{ + "db.instance": "127.0.0.1:3306/customers", + "db.statement": "SELECT * FROM user_table", + "db.type": "sql", + "db.user": "customer_user", + "span.kind": SpanKindRPCClientEnum, + }, span.Tags()) +} + +func TestMiscTags(t *testing.T) { + tracer := trace.NewTracer("blobstore") + span := tracer.StartSpan("my-trace").(trace.Span) + Component.Set(span, "my-awesome-library") + SamplingPriority.Set(span, 1) + Error.Set(span, true) + + span.Finish() + + assert.Equal(t, trace.Tags{ + "component": "my-awesome-library", + "sampling.priority": uint16(1), + "error": true, + }, span.Tags()) +} + +func TestRPCServerOption(t *testing.T) { + tracer := trace.NewTracer("blobstore") + parent := tracer.StartSpan("my-trace") + parent.SetBaggageItem("bag", "gage") + + carrier := trace.HTTPHeadersCarrier{} + err := tracer.Inject(parent.Context(), trace.HTTPHeaders, carrier) + if err != nil { + t.Fatal(err) + } + + _, err = tracer.Extract(trace.HTTPHeaders, carrier) + if err != nil { + t.Fatal(err) + } +} + +func TestMessageBusProducerTags(t *testing.T) { + tracer := trace.NewTracer("blobstore") + span := tracer.StartSpan("my-trace", SpanKindProducer).(trace.Span) + MessageBusDestination.Set(span, "topic name") + span.Finish() + + assert.Equal(t, trace.Tags{ + "message_bus.destination": "topic name", + "span.kind": SpanKindProducerEnum, + }, span.Tags()) +} + +func TestMessageBusConsumerTags(t *testing.T) { + tracer := trace.NewTracer("blobstore") + span := tracer.StartSpan("my-trace", SpanKindConsumer).(trace.Span) + MessageBusDestination.Set(span, "topic name") + span.Finish() + + assert.Equal(t, trace.Tags{ + "message_bus.destination": "topic name", + "span.kind": SpanKindConsumerEnum, + }, span.Tags()) +} diff --git a/blobstore/common/trace/propagation.go b/blobstore/common/trace/propagation.go new file mode 100644 index 000000000..a488a68a9 --- /dev/null +++ b/blobstore/common/trace/propagation.go @@ -0,0 +1,143 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package trace + +import ( + "strconv" + "strings" + + "github.com/opentracing/opentracing-go" +) + +const ( + prefixTracer = "blobstore-tracer-" + prefixBaggage = "blobstore-baggage-" + + tracerFieldCount = 2 + fieldKeyTraceID = prefixTracer + "traceid" + fieldKeySpanID = prefixTracer + "spanid" +) + +var ( + // ErrUnsupportedFormat is the alias of opentracing.ErrUnsupportedFormat. + ErrUnsupportedFormat = opentracing.ErrUnsupportedFormat + + // ErrSpanContextNotFound is the alias of opentracing.ErrSpanContextNotFound. + ErrSpanContextNotFound = opentracing.ErrSpanContextNotFound + + // ErrInvalidSpanContext is the alias of opentracing.ErrInvalidSpanContext. + ErrInvalidSpanContext = opentracing.ErrInvalidSpanContext + + // ErrInvalidCarrier is the alias of opentracing.ErrInvalidCarrier. + ErrInvalidCarrier = opentracing.ErrInvalidCarrier + + // ErrSpanContextCorrupted is the alias of opentracing.ErrSpanContextCorrupted. + ErrSpanContextCorrupted = opentracing.ErrSpanContextCorrupted +) + +const ( + // Binary is the alias of opentracing.Binary. + Binary = opentracing.Binary + + // TextMap is the alias of opentracing.TextMap. + TextMap = opentracing.TextMap + + // HTTPHeaders is the alias of opentracing.HTTPHeaders. + HTTPHeaders = opentracing.HTTPHeaders +) + +// TextMapCarrier is the alias of opentracing.TextMapCarrier. +type TextMapCarrier = opentracing.TextMapCarrier + +// HTTPHeadersCarrier is the alias of opentracing.HTTPHeadersCarrier. +type HTTPHeadersCarrier = opentracing.HTTPHeadersCarrier + +// TextMapPropagator is a combined Injector and Extractor for TextMap format. +type TextMapPropagator struct{} + +var defaultTexMapPropagator = TextMapPropagator{} + +// Inject implements Injector of TextMapPropagator +func (t *TextMapPropagator) Inject(sc *SpanContext, carrier interface{}) error { + writer, ok := carrier.(opentracing.TextMapWriter) + if !ok { + return ErrInvalidCarrier + } + writer.Set(fieldKeyTraceID, sc.traceID) + writer.Set(fieldKeySpanID, sc.spanID.String()) + + sc.ForeachBaggageItems(func(k string, v []string) bool { + if k != internalTrackLogKey { // internal baggage will not inject + writer.Set(prefixBaggage+k, strings.Join(v, ",")) + } + return true + }) + return nil +} + +// Extract implements Extractor of TextMapPropagator. +func (t *TextMapPropagator) Extract(carrier interface{}) (opentracing.SpanContext, error) { + reader, ok := carrier.(opentracing.TextMapReader) + if !ok { + return nil, ErrInvalidCarrier + } + var ( + traceID string + spanID ID + baggage = make(map[string][]string) + fieldCount int + err error + ) + err = reader.ForeachKey(func(key, val string) error { + switch strings.ToLower(key) { + case fieldKeyTraceID: + traceID = val + fieldCount++ + case fieldKeySpanID: + id, err := strconv.ParseUint(val, 16, 64) + if err != nil { + return ErrSpanContextCorrupted + } + spanID = ID(id) + fieldCount++ + default: + lowerKey := strings.ToLower(key) + if strings.HasPrefix(lowerKey, prefixBaggage) { + baggage[strings.TrimPrefix(lowerKey, prefixBaggage)] = []string{val} + } + } + return nil + }) + if err != nil { + return nil, err + } + + if fieldCount == 0 { + return nil, ErrSpanContextNotFound + } + if fieldCount < tracerFieldCount { + return nil, ErrSpanContextCorrupted + } + return &SpanContext{ + traceID: traceID, + spanID: spanID, + baggage: baggage, + }, nil +} + +// GetTraceIDKey returns http header name of traceid +func GetTraceIDKey() string { + return fieldKeyTraceID +} diff --git a/blobstore/common/trace/propagation_test.go b/blobstore/common/trace/propagation_test.go new file mode 100644 index 000000000..afbd4d704 --- /dev/null +++ b/blobstore/common/trace/propagation_test.go @@ -0,0 +1,76 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package trace + +import ( + "bytes" + "context" + "net/http" + "testing" + + "github.com/opentracing/opentracing-go/mocktracer" + "github.com/stretchr/testify/assert" +) + +func TestSpanPropagator(t *testing.T) { + tracer := NewTracer("blobstore") + defer tracer.Close() + SetGlobalTracer(tracer) + + span, _ := StartSpanFromContext(context.Background(), "test baggage") + defer span.Finish() + + span.SetBaggageItem("k1", "v1") + + carriers := []struct { + carrierType interface{} + carrier interface{} + }{ + {HTTPHeaders, HTTPHeadersCarrier(http.Header{})}, + {TextMap, TextMapCarrier(make(map[string]string))}, + } + + for _, c := range carriers { + err := span.Tracer().Inject(span.Context(), c.carrierType, c.carrier) + assert.NoError(t, err) + + sp, err := Extract(c.carrierType, c.carrier) + assert.NoError(t, err) + + child := tracer.StartSpan("child", ChildOf(sp)) + assert.Equal(t, "v1", child.BaggageItem("k1")) + assert.Equal(t, span.Context().(*SpanContext).traceID, child.Context().(*SpanContext).traceID) + assert.Equal(t, span.Context().(*SpanContext).spanID, child.Context().(*SpanContext).parentID) + child.Finish() + } + + err := span.Tracer().Inject(span.Context(), Binary, &bytes.Buffer{}) + assert.EqualError(t, err, ErrUnsupportedFormat.Error()) + _, err = Extract(Binary, &bytes.Buffer{}) + assert.EqualError(t, err, ErrUnsupportedFormat.Error()) + + err = tracer.Inject(mocktracer.MockSpanContext{}, Binary, &bytes.Buffer{}) + assert.EqualError(t, err, ErrInvalidSpanContext.Error()) + + err = defaultTexMapPropagator.Inject(span.(*spanImpl).context, &bytes.Buffer{}) + assert.EqualError(t, err, ErrInvalidCarrier.Error()) + _, err = defaultTexMapPropagator.Extract(&bytes.Buffer{}) + assert.EqualError(t, err, ErrInvalidCarrier.Error()) + + _, err = defaultTexMapPropagator.Extract(HTTPHeadersCarrier(http.Header{})) + assert.Error(t, err) + + assert.Equal(t, fieldKeyTraceID, GetTraceIDKey()) +} diff --git a/blobstore/common/trace/span.go b/blobstore/common/trace/span.go new file mode 100644 index 000000000..581d85620 --- /dev/null +++ b/blobstore/common/trace/span.go @@ -0,0 +1,331 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package trace + +import ( + "fmt" + "os" + "strconv" + "strings" + "sync" + "time" + + "github.com/opentracing/opentracing-go" + ptlog "github.com/opentracing/opentracing-go/log" + + "github.com/cubefs/cubefs/blobstore/util/log" +) + +const ( + maxErrorLen = 32 +) + +// Span extends opentracing.Span +type Span interface { + opentracing.Span + + // OperationName allows retrieving current operation name. + OperationName() string + + // Tags returns tags for span + Tags() Tags + + // Logs returns micro logs for span + Logs() []opentracing.LogRecord + + // String returns traceID:spanID. + String() string + + // TraceID returns traceID + TraceID() string + + // AppendRPCTrackLog appends RPC track logs to baggage with default key fieldTrackLogKey. + AppendRPCTrackLog(logs []string) + // AppendTrackLog records cost time with startTime (duration=time.Since(startTime)) for a calling to a module and + // appends to baggage with default key fieldTrackLogKey. + AppendTrackLog(module string, startTime time.Time, err error) + // AppendTrackLogWithDuration records cost time with duration for a calling to a module and + // appends to baggage with default key fieldTrackLogKey. + AppendTrackLogWithDuration(module string, duration time.Duration, err error) + // TrackLog returns track log, calls BaggageItem with default key fieldTrackLogKey. + TrackLog() []string + + // BaseLogger defines interface of application log apis. + log.BaseLogger +} + +// spanImpl implements Span +type spanImpl struct { + operationName string + + tracer *Tracer + + context *SpanContext + + startTime time.Time + duration time.Duration + + tags Tags + + logs []opentracing.LogRecord + + // rootSpan, if true indicate that this span is the root of the (sub)tree + // of spans and parentID is empty. + rootSpan bool + + // references for this span + references []opentracing.SpanReference + + sync.RWMutex +} + +// Finish implements opentracing.Span API +func (s *spanImpl) Finish() { + s.FinishWithOptions(opentracing.FinishOptions{}) +} + +// FinishWithOptions implements opentracing.Span API +func (s *spanImpl) FinishWithOptions(opts opentracing.FinishOptions) { + finishTime := opts.FinishTime + if finishTime.IsZero() { + finishTime = time.Now() + } + s.duration = finishTime.Sub(s.startTime) + + s.Lock() + defer s.Unlock() + + s.logs = append(s.logs, opts.LogRecords...) + + for _, ld := range opts.BulkLogData { + s.logs = append(s.logs, ld.ToLogRecord()) + } + + // TODO report span +} + +// Context implements opentracing.Span API +func (s *spanImpl) Context() opentracing.SpanContext { + s.RLock() + defer s.RUnlock() + + return s.context +} + +// SetOperationName implements opentracing.Span API +func (s *spanImpl) SetOperationName(operationName string) opentracing.Span { + s.Lock() + defer s.Unlock() + + s.operationName = operationName + return s +} + +// LogFields implements opentracing.Span API +func (s *spanImpl) LogFields(fields ...ptlog.Field) { + s.Lock() + defer s.Unlock() + + lr := opentracing.LogRecord{ + Fields: fields, + Timestamp: time.Now(), + } + s.logs = append(s.logs, lr) +} + +// LogKV implements opentracing.Span API +func (s *spanImpl) LogKV(keyValues ...interface{}) { + fields, err := ptlog.InterleavedKVToFields(keyValues...) + if err != nil { + s.LogFields(ptlog.Error(err), ptlog.String("function", "LogKV")) + return + } + s.LogFields(fields...) +} + +// SetBaggageItem implements opentracing.Span API +func (s *spanImpl) SetBaggageItem(key, value string) opentracing.Span { + for _, ref := range s.references { + spanCtx, ok := ref.ReferencedContext.(*SpanContext) + if !ok { + continue + } + spanCtx.setBaggageItem(key, []string{value}) + } + s.context.setBaggageItem(key, []string{value}) + return s +} + +// BaggageItem implements opentracing.Span API +func (s *spanImpl) BaggageItem(key string) string { + return strings.Join(s.context.baggageItem(key), ",") +} + +// Tracer implements opentracing.Span API +func (s *spanImpl) Tracer() opentracing.Tracer { + return s.tracer +} + +// SetTag implements opentracing.Span API +func (s *spanImpl) SetTag(key string, value interface{}) opentracing.Span { + s.Lock() + defer s.Unlock() + + if s.tags == nil { + s.tags = Tags{} + } + s.tags[key] = value + return s +} + +// Deprecated: use LogFields or LogKV (not implements) +func (s *spanImpl) LogEvent(event string) {} + +// Deprecated: use LogFields or LogKV (not implements) +func (s *spanImpl) LogEventWithPayload(event string, payload interface{}) {} + +// Deprecated: use LogFields or LogKV (not implements) +func (s *spanImpl) Log(data opentracing.LogData) {} + +// OperationName returns operationName for span +func (s *spanImpl) OperationName() string { + s.RLock() + defer s.RUnlock() + + return s.operationName +} + +// Tags returns tags for span +func (s *spanImpl) Tags() Tags { + s.RLock() + defer s.RUnlock() + // copy + tags := make(map[string]interface{}, len(s.tags)) + for key, value := range s.tags { + tags[key] = value + } + return tags +} + +// Logs returns micro logs for span +func (s *spanImpl) Logs() []opentracing.LogRecord { + s.RLock() + defer s.RUnlock() + + return s.logs +} + +// AppendTrackLog records cost time with startTime (duration=time.Since(startTime)) for a calling to a module and +// appends to baggage with default key fieldTrackLogKey. +func (s *spanImpl) AppendTrackLog(module string, startTime time.Time, err error) { + s.AppendTrackLogWithDuration(module, time.Since(startTime), err) +} + +// AppendTrackLogWithDuration records cost time with duration for a calling to a module and +// appends to baggage with default key fieldTrackLogKey. +func (s *spanImpl) AppendTrackLogWithDuration(module string, duration time.Duration, err error) { + durMs := duration.Nanoseconds() / 1e6 + if durMs > 0 { + module += ":" + strconv.FormatInt(durMs, 10) + } + if err != nil { + msg := err.Error() + if len(msg) > maxErrorLen { + msg = msg[:maxErrorLen] + } + module += "/" + msg + } + s.track(module) +} + +// AppendRPCTrackLog appends RPC track logs to baggage with default key fieldTrackLogKey. +func (s *spanImpl) AppendRPCTrackLog(logs []string) { + for _, trackLog := range logs { + s.track(trackLog) + } +} + +// TrackLog returns track log, calls BaggageItem with default key fieldTrackLogKey. +func (s *spanImpl) TrackLog() []string { + return s.context.trackLogs() +} + +func (s *spanImpl) track(value string) { + for _, ref := range s.references { + spanCtx, ok := ref.ReferencedContext.(*SpanContext) + if !ok { + continue + } + spanCtx.append(value) + } + s.context.append(value) +} + +// String returns traceID:spanID. +func (s *spanImpl) String() string { + return fmt.Sprintf("%s:%s", s.context.traceID, s.context.spanID) +} + +// TraceID return traceID +func (s *spanImpl) TraceID() string { + return s.context.traceID +} + +//------------------------------------------------------------------- +// +const ( + defaultCalldepth = 3 +) + +func (s *spanImpl) output(lvl log.Level, v []interface{}) { + log.DefaultLogger.Output(s.String(), lvl, defaultCalldepth, fmt.Sprintln(v...)) +} + +func (s *spanImpl) outputf(lvl log.Level, format string, v []interface{}) { + log.DefaultLogger.Output(s.String(), lvl, defaultCalldepth, fmt.Sprintf(format, v...)) +} + +func (s *spanImpl) Println(v ...interface{}) { s.output(log.Linfo, v) } +func (s *spanImpl) Printf(format string, v ...interface{}) { s.outputf(log.Linfo, format, v) } +func (s *spanImpl) Debug(v ...interface{}) { s.output(log.Ldebug, v) } +func (s *spanImpl) Debugf(format string, v ...interface{}) { s.outputf(log.Ldebug, format, v) } +func (s *spanImpl) Info(v ...interface{}) { s.output(log.Linfo, v) } +func (s *spanImpl) Infof(format string, v ...interface{}) { s.outputf(log.Linfo, format, v) } +func (s *spanImpl) Warn(v ...interface{}) { s.output(log.Lwarn, v) } +func (s *spanImpl) Warnf(format string, v ...interface{}) { s.outputf(log.Lwarn, format, v) } +func (s *spanImpl) Error(v ...interface{}) { s.output(log.Lerror, v) } +func (s *spanImpl) Errorf(format string, v ...interface{}) { s.outputf(log.Lerror, format, v) } + +func (s *spanImpl) Panic(v ...interface{}) { + str := fmt.Sprintln(v...) + s.output(log.Lpanic, v) + panic(s.String() + " -> " + str) +} + +func (s *spanImpl) Panicf(format string, v ...interface{}) { + str := fmt.Sprintf(format, v...) + s.outputf(log.Lpanic, format, v) + panic(s.String() + " -> " + str) +} + +func (s *spanImpl) Fatal(v ...interface{}) { + s.output(log.Lfatal, v) + os.Exit(1) +} + +func (s *spanImpl) Fatalf(format string, v ...interface{}) { + s.outputf(log.Lfatal, format, v) + os.Exit(1) +} diff --git a/blobstore/common/trace/span_context.go b/blobstore/common/trace/span_context.go new file mode 100644 index 000000000..80df00088 --- /dev/null +++ b/blobstore/common/trace/span_context.go @@ -0,0 +1,135 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package trace + +import ( + "fmt" + "math/rand" + "sync" + "time" +) + +const ( + internalTrackLogKey = "internal-baggage-key-tracklog" +) + +// ID used for spanID or traceID +type ID uint64 + +func (id ID) String() string { + return fmt.Sprintf("%016x", uint64(id)) +} + +var ( + seededIDGen = rand.New(rand.NewSource(time.Now().UnixNano())) + // The golang rand generators are *not* intrinsically thread-safe. + seededIDLock sync.Mutex +) + +// RandomID generate ID for traceID or spanID +func RandomID() ID { + seededIDLock.Lock() + defer seededIDLock.Unlock() + return ID(seededIDGen.Int63()) +} + +// SpanContext implements opentracing.SpanContext +type SpanContext struct { + // traceID represents globally unique ID of the trace. + traceID string + + // spanID represents span ID that must be unique within its trace. + spanID ID + + // parentID refers to the ID of the parent span. + // Should be 0 if the current span is a root span. + parentID ID + + // Distributed Context baggage. + baggage map[string][]string + sync.RWMutex +} + +// ForeachBaggageItem implements opentracing.SpanContext API +func (s *SpanContext) ForeachBaggageItem(handler func(k, v string) bool) { + panic("not implements") +} + +// ForeachBaggageItems will called the handler function for each baggage key/values pair. +func (s *SpanContext) ForeachBaggageItems(handler func(k string, v []string) bool) { + s.Lock() + defer s.Unlock() + + for k, v := range s.baggage { + if !handler(k, v) { + break + } + } +} + +func (s *SpanContext) setBaggageItem(key string, value []string) { + s.Lock() + defer s.Unlock() + + if s.baggage == nil { + s.baggage = map[string][]string{key: value} + return + } + s.baggage[key] = value +} + +func (s *SpanContext) trackLogs() []string { + return s.baggageItemDeepCopy(internalTrackLogKey) +} + +func (s *SpanContext) append(value string) { + s.Lock() + defer s.Unlock() + + if s.baggage == nil { + s.baggage = map[string][]string{internalTrackLogKey: {value}} + return + } + + if _, ok := s.baggage[internalTrackLogKey]; ok { + s.baggage[internalTrackLogKey] = append(s.baggage[internalTrackLogKey], value) + return + } + s.baggage[internalTrackLogKey] = []string{value} +} + +func (s *SpanContext) baggageItem(key string) []string { + s.RLock() + defer s.RUnlock() + + return s.baggage[key] +} + +func (s *SpanContext) baggageItemDeepCopy(key string) (item []string) { + s.RLock() + defer s.RUnlock() + item = append(item, s.baggage[key]...) + return +} + +// IsValid returns true if SpanContext is valid +func (s *SpanContext) IsValid() bool { + return s.traceID != "" && s.spanID != 0 +} + +// IsEmpty returns true is span context is empty +func (s *SpanContext) IsEmpty() bool { + return !s.IsValid() && len(s.baggage) == 0 +} diff --git a/blobstore/common/trace/span_test.go b/blobstore/common/trace/span_test.go new file mode 100644 index 000000000..2120eb4b7 --- /dev/null +++ b/blobstore/common/trace/span_test.go @@ -0,0 +1,215 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package trace + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/opentracing/opentracing-go" + ptlog "github.com/opentracing/opentracing-go/log" + "github.com/stretchr/testify/assert" + + "github.com/cubefs/cubefs/blobstore/util/log" +) + +func TestSpan_Tags(t *testing.T) { + span, _ := StartSpanFromContext(context.Background(), "test tags") + defer span.Finish() + + expectedTags := Tags{ + "module": "worker", + "ip": "127.0.0.1", + } + + span.SetTag("module", "worker") + span.SetTag("ip", "127.0.0.1") + assert.Equal(t, span.Tags(), expectedTags) +} + +func TestSpan_Logs(t *testing.T) { + span, _ := StartSpanFromContext(context.Background(), "test logs") + defer span.Finish() + + expectedLogs := []struct { + logs []ptlog.Field + }{ + { + logs: []ptlog.Field{ptlog.String("event", "success"), ptlog.Int("waited.millis", 20)}, + }, + { + logs: []ptlog.Field{ptlog.String("event", "failed"), ptlog.Int("waited.millis", 1500)}, + }, + } + + for k, v := range expectedLogs { + span.LogFields(v.logs...) + assert.Equal(t, expectedLogs[k].logs, span.Logs()[k].Fields) + } + assert.Equal(t, 2, len(span.Logs())) + + fields := []ptlog.Field{ptlog.String("code", "200"), ptlog.Float32("count", 100)} + for k, v := range fields { + span.LogKV(v.Key(), v.Value()) + assert.Equal(t, fields[k].Key(), span.Logs()[k+2].Fields[0].Key()) + assert.Equal(t, fields[k].Value(), span.Logs()[k+2].Fields[0].Value()) + } + assert.Equal(t, 4, len(span.Logs())) + + span.LogKV("only key") + assert.Equal(t, 5, len(span.Logs())) +} + +func TestSpan_OperationName(t *testing.T) { + span, _ := StartSpanFromContext(context.Background(), "span") + defer span.Finish() + + assert.Equal(t, "span", span.OperationName()) + span.SetOperationName("span2") + assert.Equal(t, "span2", span.OperationName()) +} + +func TestSpan_Baggage(t *testing.T) { + span, ctx := StartSpanFromContext(context.Background(), "test baggage") + defer span.Finish() + + baggages := []struct { + k string + v string + }{ + {k: "k1", v: "v1"}, + {k: "k2", v: "v2"}, + {k: "k3", v: "v3"}, + } + for _, v := range baggages { + span.SetBaggageItem(v.k, v.v) + assert.Equal(t, v.v, span.BaggageItem(v.k)) + } + + spanChild, _ := StartSpanFromContext(ctx, "child of span") + for _, v := range baggages { + assert.Equal(t, v.v, spanChild.BaggageItem(v.k)) + } + + spanChild.SetBaggageItem("k4", "v4") + assert.Equal(t, "v4", spanChild.BaggageItem("k4")) + assert.Equal(t, "v4", span.BaggageItem("k4")) +} + +func TestSpan_TrackLog(t *testing.T) { + span, ctx := StartSpanFromContext(context.Background(), "test trackLog") + defer span.Finish() + + span.AppendTrackLog("sleep", time.Now(), nil) + assert.Equal(t, []string{"sleep"}, span.TrackLog()) + + spanChild, _ := StartSpanFromContext(ctx, "child of span") + assert.Equal(t, []string{"sleep"}, spanChild.TrackLog()) + + spanChild.AppendTrackLog("sleep2", time.Now(), errors.New("sleep2 err")) + assert.Equal(t, []string{"sleep", "sleep2/sleep2 err"}, spanChild.TrackLog()) + assert.Equal(t, []string{"sleep", "sleep2/sleep2 err"}, span.TrackLog()) + + spanChild.AppendRPCTrackLog([]string{"blobnode:4", "scheduler:5"}) + assert.Equal(t, []string{"sleep", "sleep2/sleep2 err", "blobnode:4", "scheduler:5"}, spanChild.TrackLog()) + assert.Equal(t, []string{"sleep", "sleep2/sleep2 err", "blobnode:4", "scheduler:5"}, span.TrackLog()) + + spanChild.AppendTrackLog("sleep3", time.Now(), nil) + assert.Equal(t, []string{"sleep", "sleep2/sleep2 err", "blobnode:4", "scheduler:5", "sleep3"}, span.TrackLog()) +} + +func TestSpan_TrackLogWithDuration(t *testing.T) { + span, ctx := StartSpanFromContext(context.Background(), "test trackLog") + defer span.Finish() + + span.AppendTrackLogWithDuration("sleep", time.Millisecond, nil) + assert.Equal(t, []string{"sleep:1"}, span.TrackLog()) + + spanChild, _ := StartSpanFromContext(ctx, "child of span") + assert.Equal(t, []string{"sleep:1"}, spanChild.TrackLog()) + + spanChild.AppendTrackLogWithDuration("sleep2", 2*time.Millisecond, errors.New("sleep2 err")) + assert.Equal(t, []string{"sleep:1", "sleep2:2/sleep2 err"}, spanChild.TrackLog()) + assert.Equal(t, []string{"sleep:1", "sleep2:2/sleep2 err"}, span.TrackLog()) + + spanChild.AppendRPCTrackLog([]string{"blobnode:4", "scheduler:5"}) + assert.Equal(t, []string{"sleep:1", "sleep2:2/sleep2 err", "blobnode:4", "scheduler:5"}, spanChild.TrackLog()) + assert.Equal(t, []string{"sleep:1", "sleep2:2/sleep2 err", "blobnode:4", "scheduler:5"}, span.TrackLog()) + + spanChild.AppendTrackLogWithDuration("sleep3", 3*time.Millisecond, nil) + assert.Equal(t, []string{"sleep:1", "sleep2:2/sleep2 err", "blobnode:4", "scheduler:5", "sleep3:3"}, span.TrackLog()) +} + +func TestSpan_BaseLogger(t *testing.T) { + rootSpan, ctx := StartSpanFromContext(context.Background(), "test baseLogger") + defer rootSpan.Finish() + + logLevel := []log.Level{ + log.Ldebug, + log.Linfo, + log.Lwarn, + log.Lerror, + log.Lpanic, + log.Lfatal, + } + + for _, level := range logLevel { + rootSpan.Infof("set log level, level: %d", level) + log.SetOutputLevel(level) + + span, _ := StartSpanFromContext(ctx, "test baseLogger") + + span.Debug("span info:", span.String()) + span.Debugf("spanContent info: %+v ,traceID: %s", span.Context(), span.TraceID()) + + span.Info("service name", span.Tracer().(*Tracer).serviceName) + span.Infof("start span success, name: %s", span.OperationName()) + + span.Warn("get spanID") + span.Warnf("spanID: %d", span.Context().(*SpanContext).spanID) + + ctx := context.Background() + if spanNil := SpanFromContext(ctx); spanNil == nil { + span.Error("SpanFromContext failed") + span.Errorf("ctx: %+v, span: %+v", ctx, spanNil) + } + + assert.Panics(t, func() { + if level%2 == 0 { + span.Panic("panic on span", span) + } else { + span.Panicf("panic on span: %p", span.Context().(*SpanContext)) + } + }) + + span.Finish() + } +} + +func TestSpan_FinishWithOptions(t *testing.T) { + span, _ := StartSpanFromContext(context.Background(), "test baseLogger") + + span.FinishWithOptions(opentracing.FinishOptions{ + LogRecords: []opentracing.LogRecord{ + {Timestamp: time.Now()}, + {Timestamp: time.Now()}, + }, + BulkLogData: []opentracing.LogData{ + {Timestamp: time.Now()}, + }, + }) +} diff --git a/blobstore/common/trace/tracer.go b/blobstore/common/trace/tracer.go new file mode 100644 index 000000000..063e1ed86 --- /dev/null +++ b/blobstore/common/trace/tracer.go @@ -0,0 +1,310 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package trace + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "path" + "time" + + "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go/ext" +) + +const ( + defaultRootSpanName = "defaultBlobnodeRootSpanName" + defaultMaxLogsPerSpan = 50 + reqidKey = "X-Reqid" +) + +// ChildOf is the alias of opentracing.ChildOf +var ChildOf = opentracing.ChildOf + +// FollowsFrom is the alias of opentracing.FollowsFrom +var FollowsFrom = opentracing.FollowsFrom + +// StartTime is alias of opentracing.StartTime. +type StartTime = opentracing.StartTime + +// Tags are the expand of opentracing.Tags +type Tags opentracing.Tags + +// Apply satisfies the StartSpanOption interface. +func (t Tags) Apply(options *opentracing.StartSpanOptions) { + if options.Tags == nil { + options.Tags = make(opentracing.Tags) + } + for k, v := range t { + options.Tags[k] = v + } +} + +// ToSlice change tags to slice +func (t Tags) ToSlice() (ret []string) { + for k := range t { + ret = append(ret, k+":"+fmt.Sprint(t[k])) + } + return +} + +// Marshal marshal tracer tags +func (t Tags) Marshal() (ret []byte, err error) { + ret, err = json.Marshal(t) + return +} + +// Tag is the alias of opentracing.Tag, +type Tag = opentracing.Tag + +// Options tracer options +type Options struct { + maxLogsPerSpan int +} + +// Tracer implements opentracing.Tracer +type Tracer struct { + serviceName string + + options Options +} + +// init sets default global tracer +func init() { + tracer := NewTracer(path.Base(os.Args[0])) + SetGlobalTracer(tracer) +} + +// NewTracer creates a tracer with serviceName +func NewTracer(serviceName string, opts ...TracerOption) *Tracer { + t := &Tracer{ + serviceName: serviceName, + } + for _, option := range opts { + option(t) + } + + if t.options.maxLogsPerSpan <= 0 { + t.options.maxLogsPerSpan = defaultMaxLogsPerSpan + } + + return t +} + +// StartSpan implements StartSpan() method of opentracing.Tracer. +// Create, start, and return a new Span with the given `operationName` and +// incorporate the given StartSpanOption `opts`. +func (t *Tracer) StartSpan(operationName string, options ...opentracing.StartSpanOption) opentracing.Span { + sso := opentracing.StartSpanOptions{} + for _, o := range options { + o.Apply(&sso) + } + return t.startSpanWithOptions(operationName, sso) +} + +func (t *Tracer) startSpanWithOptions(operationName string, opts opentracing.StartSpanOptions) Span { + startTime := opts.StartTime + if startTime.IsZero() { + startTime = time.Now() + } + + var ( + hasParent bool + parent *SpanContext + references []opentracing.SpanReference + ctx = &SpanContext{} + ) + + for _, reference := range opts.References { + spanCtx, ok := reference.ReferencedContext.(*SpanContext) + if !ok { + continue + } + + if spanCtx == nil || spanCtx.IsEmpty() { + continue + } + + if spanCtx.IsValid() { + references = append(references, reference) + } + + if !hasParent { + parent = spanCtx + hasParent = reference.Type == opentracing.ChildOfRef + } + } + + if !hasParent && parent != nil && !parent.IsEmpty() { + hasParent = true + } + + if !hasParent || (parent != nil && !parent.IsValid()) { + ctx.traceID = RandomID().String() + ctx.spanID = RandomID() + ctx.parentID = 0 + } else { + ctx.traceID = parent.traceID + ctx.spanID = RandomID() + ctx.parentID = parent.spanID + } + if hasParent { + // copy baggage items + parent.ForeachBaggageItems(func(k string, v []string) bool { + ctx.setBaggageItem(k, v) + return true + }) + } + + tags := opts.Tags + + span := &spanImpl{ + operationName: operationName, + startTime: startTime, + tags: tags, + context: ctx, + tracer: t, + references: references, + duration: 0, + } + span.rootSpan = ctx.parentID == 0 + return span +} + +// Inject implements Inject() method of opentracing.Tracer +func (t *Tracer) Inject(sc opentracing.SpanContext, format interface{}, carrier interface{}) error { + s, ok := sc.(*SpanContext) + if !ok { + return opentracing.ErrInvalidSpanContext + } + switch format { + case TextMap, HTTPHeaders: + return defaultTexMapPropagator.Inject(s, carrier) + } + return ErrUnsupportedFormat +} + +// Extract implements Extract() method of opentracing.Tracer +func (t *Tracer) Extract(format interface{}, carrier interface{}) (opentracing.SpanContext, error) { + switch format { + case TextMap, HTTPHeaders: + return defaultTexMapPropagator.Extract(carrier) + } + return nil, ErrUnsupportedFormat +} + +// Close releases all resources +func (t *Tracer) Close() error { + // TODO report span + return nil +} + +// StartSpanFromContext starts and returns a Span with `operationName`, using +// any Span found within `ctx` as a ChildOfRef. If no such parent could be +// found, StartSpanFromContext creates a root (parentless) Span. +func StartSpanFromContext(ctx context.Context, operationName string, opts ...opentracing.StartSpanOption) (Span, context.Context) { + span, ctx := opentracing.StartSpanFromContext(ctx, operationName, opts...) + return span.(Span), ctx +} + +// StartSpanFromContextWithTraceID starts and return a new span with `operationName` and traceID. +func StartSpanFromContextWithTraceID(ctx context.Context, operationName string, traceID string, opts ...opentracing.StartSpanOption) (Span, context.Context) { + span, ctx := opentracing.StartSpanFromContext(ctx, operationName, opts...) + s := span.(*spanImpl) + s.context.traceID = traceID + return s, ctx +} + +// StartSpanFromHTTPHeaderSafe starts and return a Span with `operationName` and http.Request +func StartSpanFromHTTPHeaderSafe(r *http.Request, operationName string) (Span, context.Context) { + spanCtx, _ := Extract(HTTPHeaders, HTTPHeadersCarrier(r.Header)) + traceID := r.Header.Get(reqidKey) + if traceID == "" { + return StartSpanFromContext(context.Background(), operationName, ext.RPCServerOption(spanCtx)) + } + return StartSpanFromContextWithTraceID(context.Background(), operationName, traceID, ext.RPCServerOption(spanCtx)) +} + +// ContextWithSpan returns a new `context.Context` that holds a reference to +// the span. If span is nil, a new context without an active span is returned. +func ContextWithSpan(ctx context.Context, span Span) context.Context { + return opentracing.ContextWithSpan(ctx, span) +} + +// SpanFromContext returns the `Span` previously associated with `ctx`, or +// `nil` if no such `Span` could be found. +func SpanFromContext(ctx context.Context) Span { + span := opentracing.SpanFromContext(ctx) + s, ok := span.(Span) + if !ok { + return nil + } + return s +} + +// SpanFromContextSafe returns the `Span` previously associated with `ctx`, or +// creates a root Span with name default. +func SpanFromContextSafe(ctx context.Context) Span { + span := opentracing.SpanFromContext(ctx) + s, ok := span.(Span) + if !ok || s == nil { + return opentracing.GlobalTracer().StartSpan(defaultRootSpanName).(Span) + } + return s +} + +// SetGlobalTracer sets the [singleton] opentracing.Tracer returned by +// GlobalTracer(). Those who use GlobalTracer (rather than directly manage an +// opentracing.Tracer instance) should call SetGlobalTracer as early as +// possible in main(), prior to calling the `StartSpan` global func below. +// Prior to calling `SetGlobalTracer`, any Spans started via the `StartSpan` +// (etc) globals are noops. +func SetGlobalTracer(tracer *Tracer) { + opentracing.SetGlobalTracer(tracer) +} + +// CloseGlobalTracer closes global tracer gracefully. +func CloseGlobalTracer() { + tracer, ok := opentracing.GlobalTracer().(*Tracer) + if !ok { + return + } + tracer.Close() +} + +// GlobalTracer returns the global singleton `Tracer` implementation. +func GlobalTracer() *Tracer { + t := opentracing.GlobalTracer() + return t.(*Tracer) +} + +// Extract returns a SpanContext instance given `format` and `carrier`. +func Extract(format interface{}, carrier interface{}) (opentracing.SpanContext, error) { + return GlobalTracer().Extract(format, carrier) +} + +// InjectWithHTTPHeader takes the `sm` SpanContext instance and injects it for +// propagation within `HTTPHeadersCarrier` and `HTTPHeaders`. +func InjectWithHTTPHeader(ctx context.Context, r *http.Request) error { + span := SpanFromContextSafe(ctx) + + ext.SpanKindRPCClient.Set(span) + ext.HTTPMethod.Set(span, r.Method) + + return span.Tracer().Inject(span.Context(), HTTPHeaders, HTTPHeadersCarrier(r.Header)) +} diff --git a/blobstore/common/trace/tracer_options.go b/blobstore/common/trace/tracer_options.go new file mode 100644 index 000000000..bfd42be77 --- /dev/null +++ b/blobstore/common/trace/tracer_options.go @@ -0,0 +1,29 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package trace + +// TracerOption is a function that sets some option on the tracer +type TracerOption func(tracer *Tracer) + +// TracerOptions is a factory for all available TracerOption's +var TracerOptions tracerOptions + +type tracerOptions struct{} + +func (tracerOptions) MaxLogsPerSpan(maxLogsPerSpan int) TracerOption { + return func(tracer *Tracer) { + tracer.options.maxLogsPerSpan = maxLogsPerSpan + } +} diff --git a/blobstore/common/trace/tracer_test.go b/blobstore/common/trace/tracer_test.go new file mode 100644 index 000000000..8a5a2e9a6 --- /dev/null +++ b/blobstore/common/trace/tracer_test.go @@ -0,0 +1,207 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package trace + +import ( + "context" + "net/http" + "strings" + "testing" + "time" + + "github.com/opentracing/opentracing-go" + "github.com/stretchr/testify/assert" +) + +func TestExplicitStartTime(t *testing.T) { + tracer := NewTracer("blobstore") + defer tracer.Close() + + start := time.Now() + + span := tracer.StartSpan("testStartTime", StartTime(start)) + defer span.Finish() + + assert.Equal(t, start, span.(*spanImpl).startTime) +} + +func TestExplicitTags(t *testing.T) { + tracer := NewTracer("blobstore") + defer tracer.Close() + + tags := Tags{ + "tag1K": "tag1V", + "tag2K": "tag2V", + } + + span1 := tracer.StartSpan("testTags", tags) + defer span1.Finish() + + assert.Equal(t, tags, span1.(*spanImpl).tags) + + tag1 := Tag{ + Key: "tag1K", + Value: "tag1V", + } + span2 := tracer.StartSpan("testTag", tag1).(Span) + defer span2.Finish() + + expect := Tags{ + "tag1K": "tag1V", + } + assert.Equal(t, expect, span2.(*spanImpl).tags) + + tag2 := Tag{ + Key: "tag2K", + Value: "tag2V", + } + tag2.Set(span2) + assert.Equal(t, tags, span2.(*spanImpl).tags) +} + +func TestExplicitReferences(t *testing.T) { + tracer := NewTracer("blobstore") + defer tracer.Close() + + parentSpan := tracer.StartSpan("parent").(*spanImpl) + defer parentSpan.Finish() + + span1 := tracer.StartSpan("child", ChildOf(parentSpan.Context())).(*spanImpl) + defer span1.Finish() + + assert.Equal(t, 1, len(span1.references)) + assert.Equal(t, parentSpan.context.traceID, span1.context.traceID) + assert.Equal(t, parentSpan.context.spanID, span1.context.parentID) + + ctx := ContextWithSpan(context.Background(), span1) + span2 := SpanFromContext(ctx).(*spanImpl) + + span3, _ := StartSpanFromContext(ctx, "child of span") + cs := span3.(*spanImpl) + + assert.Equal(t, span1, span2) + assert.Equal(t, 1, len(cs.references)) + assert.Equal(t, span1.context.traceID, cs.context.traceID) + assert.Equal(t, span1.context.spanID, cs.context.parentID) + + newParentSpan := tracer.StartSpan("newParentSpan") + span4 := tracer.StartSpan("nChild", FollowsFrom(span3.Context()), + FollowsFrom(newParentSpan.Context())).(*spanImpl) + + assert.Equal(t, 2, len(span4.references)) + + span5 := tracer.StartSpan("empty span context", ChildOf(&SpanContext{})).(*spanImpl) + assert.Equal(t, 0, len(span5.references)) +} + +func TestStartSpanFromContext(t *testing.T) { + span, ctx := StartSpanFromContext(context.Background(), "span1") + defer span.Finish() + + assert.NotNil(t, SpanFromContext(ctx)) + s := span.(*spanImpl) + + childSpan, _ := StartSpanFromContext(ctx, "child span") + cs := childSpan.(*spanImpl) + assert.Equal(t, 1, len(cs.references)) + assert.Equal(t, s.context.traceID, cs.context.traceID) + assert.Equal(t, s.context.spanID, cs.context.parentID) + + // root span + traceID := "traceID" + traceID2 := "traceID2" + rootSpan1, _ := StartSpanFromContextWithTraceID(context.Background(), "root span1", traceID) + rootSpan2, _ := StartSpanFromContextWithTraceID(ctx, "root span2", traceID2) + rs1 := rootSpan1.(*spanImpl) + rs2 := rootSpan2.(*spanImpl) + + assert.Equal(t, traceID, rs1.context.traceID) + assert.NotEqual(t, traceID, rs2.context.traceID) +} + +func TestSpanFromContext(t *testing.T) { + ctx := context.Background() + assert.Nil(t, SpanFromContext(ctx)) + + span, ctx := StartSpanFromContext(ctx, "span1") + defer span.Finish() + + assert.NotNil(t, SpanFromContext(ctx)) + + spanSafe := SpanFromContextSafe(context.Background()) + defer spanSafe.Finish() + + s := spanSafe.(*spanImpl) + assert.Equal(t, defaultRootSpanName, s.operationName) + + spanCopy := SpanFromContextSafe(ctx) + + sc := spanCopy.(*spanImpl) + assert.Equal(t, span.OperationName(), sc.OperationName()) +} + +func TestStartSpanFromHTTPHeaderSafe(t *testing.T) { + r := &http.Request{Header: http.Header{}} + traceID := "test" + span, _ := StartSpanFromHTTPHeaderSafe(r, "http") + assert.NotEqual(t, traceID, span.Context().(*SpanContext).traceID) + + r.Header.Set(reqidKey, traceID) + span, _ = StartSpanFromHTTPHeaderSafe(r, "http") + assert.Equal(t, traceID, span.Context().(*SpanContext).traceID) +} + +func TestNewTracer(t *testing.T) { + tracer := NewTracer("blobstore") + assert.Equal(t, defaultMaxLogsPerSpan, tracer.options.maxLogsPerSpan) + tracer.Close() + + tracer = NewTracer("blobstore", TracerOptions.MaxLogsPerSpan(10)) + assert.Equal(t, 10, tracer.options.maxLogsPerSpan) + tracer.Close() +} + +func TestCloseGlobalTracer(t *testing.T) { + noopTracer := opentracing.NoopTracer{} + opentracing.SetGlobalTracer(noopTracer) + CloseGlobalTracer() + + tracer := NewTracer("blobstore") + SetGlobalTracer(tracer) + CloseGlobalTracer() +} + +func TestInject(t *testing.T) { + r := &http.Request{Header: http.Header{}} + err := InjectWithHTTPHeader(context.Background(), r) + assert.NoError(t, err) + span1, _ := StartSpanFromHTTPHeaderSafe(r, "span1") + firstUpper := func(s string) string { + if s == "" { + return "" + } + return strings.ToUpper(s[:1]) + s[1:] + } + assert.Equal(t, r.Header.Get(firstUpper(fieldKeyTraceID)), span1.Context().(*SpanContext).traceID) + + span2, ctx := StartSpanFromContext(context.Background(), "span2") + r = &http.Request{Header: http.Header{}} + err = InjectWithHTTPHeader(ctx, r) + assert.NoError(t, err) + + span3, _ := StartSpanFromHTTPHeaderSafe(r, "span3") + assert.Equal(t, r.Header.Get(firstUpper(fieldKeyTraceID)), span3.Context().(*SpanContext).traceID) + assert.Equal(t, span2.Context().(*SpanContext).traceID, span3.Context().(*SpanContext).traceID) +} diff --git a/blobstore/scheduler/archiver.go b/blobstore/scheduler/archiver.go new file mode 100644 index 000000000..87b9432c4 --- /dev/null +++ b/blobstore/scheduler/archiver.go @@ -0,0 +1,149 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "sync" + "time" + + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/scheduler/base" + "github.com/cubefs/cubefs/blobstore/scheduler/db" + "github.com/cubefs/cubefs/blobstore/util/closer" +) + +// IArchiver define the interface of archive manager +type IArchiver interface { + RegisterTables(tables ...db.IRecordSrcTbl) + Run() + closer.Closer +} + +// ArchiveStoreConfig archive store config +type ArchiveStoreConfig struct { + ArchiveIntervalMin int `json:"archive_interval_min"` + // recode will archive util delete ArchiveDelayMin + ArchiveDelayMin int `json:"archive_delay_min"` +} + +// ArchiveStoreMgr archive store +type ArchiveStoreMgr struct { + closer.Closer + + archTbl db.IArchiveTable + + mu sync.Mutex + srcTables map[string]db.IRecordSrcTbl + + cfg ArchiveStoreConfig +} + +// NewArchiveStoreMgr returns archive store manager +func NewArchiveStoreMgr(table db.IArchiveTable, cfg ArchiveStoreConfig) *ArchiveStoreMgr { + return &ArchiveStoreMgr{ + Closer: closer.New(), + archTbl: table, + srcTables: make(map[string]db.IRecordSrcTbl), + cfg: cfg, + } +} + +// Run archive store task +func (mgr *ArchiveStoreMgr) Run() { + go mgr.run() +} + +func (mgr *ArchiveStoreMgr) run() { + t := time.NewTicker(time.Duration(mgr.cfg.ArchiveIntervalMin) * time.Minute) + defer t.Stop() + + for { + select { + case <-t.C: + mgr.storeRun() + case <-mgr.Closer.Done(): + return + } + } +} + +func (mgr *ArchiveStoreMgr) storeRun() { + _, ctx := trace.StartSpanFromContext(context.Background(), "ArchiveStoreMgr") + + mgr.mu.Lock() + for _, src := range mgr.srcTables { + mgr.store(ctx, src) + } + mgr.mu.Unlock() +} + +func (mgr *ArchiveStoreMgr) store(ctx context.Context, src db.IRecordSrcTbl) { + span := trace.SpanFromContextSafe(ctx) + span.Debugf("archive store: src[%s]", src.Name()) + + deleteTasks, err := src.QueryMarkDeleteTasks(ctx, mgr.cfg.ArchiveDelayMin) + if err != nil { + span.Errorf("query delete tasks failed: src[%s], err[%+v]", src.Name(), err) + return + } + + shouldRemove := make([]*proto.ArchiveRecord, 0, len(deleteTasks)) + for _, task := range deleteTasks { + archivedRecord, err := mgr.archTbl.FindTask(ctx, task.TaskID) + if archivedRecord != nil { + span.Infof("task has been archived: task_id [%s]", task.TaskID) + shouldRemove = append(shouldRemove, task) + continue + } + if err != nil && err != base.ErrNoDocuments { + span.Errorf("find task in archive table failed: task_id[%s], err[%+v]", task.TaskID, err) + continue + } + + err = mgr.archTbl.Insert(ctx, task) + if err != nil { + span.Errorf("insert task into archive table failed: task_id[%s], err[%+v]", task.TaskID, err) + continue + } + + shouldRemove = append(shouldRemove, task) + } + + for _, task := range shouldRemove { + archivedRecord, err := mgr.archTbl.FindTask(ctx, task.TaskID) + if archivedRecord == nil { + span.Warnf("task not find in archive table: task_id[%s], err[%+v]", task.TaskID, err) + continue + } + + err = src.RemoveMarkDelete(ctx, task.TaskID) + if err != nil { + span.Errorf("remove task failed: task_id[%s], err[%+v]", task.TaskID, err) + continue + } + span.Debugf("archived: task_id[%s] task[%+v]", task.TaskID, task) + } +} + +// RegisterTables register tables to archive. +func (mgr *ArchiveStoreMgr) RegisterTables(tables ...db.IRecordSrcTbl) { + mgr.mu.Lock() + for _, src := range tables { + mgr.srcTables[src.Name()] = src + } + mgr.mu.Unlock() +} diff --git a/blobstore/scheduler/archiver_test.go b/blobstore/scheduler/archiver_test.go new file mode 100644 index 000000000..80ab3f60a --- /dev/null +++ b/blobstore/scheduler/archiver_test.go @@ -0,0 +1,110 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "testing" + + "github.com/golang/mock/gomock" + "github.com/google/uuid" + + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/scheduler/base" +) + +var testArchiveConfig = ArchiveStoreConfig{ + ArchiveIntervalMin: defaultArchiveIntervalMin, + ArchiveDelayMin: defaultArchiveDelayMin, +} + +func TestArchiveStore(t *testing.T) { + ctr := gomock.NewController(t) + { + archiveTable := NewMockArchiveTable(ctr) + archiveTable.EXPECT().FindTask(any, any).AnyTimes().Return(nil, nil) + archiveTable.EXPECT().Insert(any, any).AnyTimes().Return(nil) + + mgr := NewArchiveStoreMgr(archiveTable, testArchiveConfig) + defer mgr.Close() + + balanceTable := NewMockMigrateTaskTable(ctr) + balanceTable.EXPECT().Name().AnyTimes().Return(proto.BalanceTaskType) + record1 := &proto.ArchiveRecord{TaskID: uuid.New().String(), TaskType: proto.BalanceTaskType, Content: proto.MigrateTask{}} + balanceTable.EXPECT().QueryMarkDeleteTasks(any, any).AnyTimes().Return([]*proto.ArchiveRecord{record1}, nil) + + repairTaskTable := NewMockRepairTaskTable(ctr) + repairTaskTable.EXPECT().Name().AnyTimes().Return(proto.RepairTaskType) + record2 := &proto.ArchiveRecord{TaskID: uuid.New().String(), TaskType: proto.RepairTaskType, Content: proto.VolRepairTask{}} + repairTaskTable.EXPECT().QueryMarkDeleteTasks(any, any).AnyTimes().Return([]*proto.ArchiveRecord{record2}, nil) + + mgr.RegisterTables(balanceTable, repairTaskTable) + mgr.Run() + } + { + // register twice + archiveTable := NewMockArchiveTable(ctr) + mgr := NewArchiveStoreMgr(archiveTable, testArchiveConfig) + defer mgr.Close() + + repairTaskTable := NewMockRepairTaskTable(ctr) + repairTaskTable.EXPECT().Name().AnyTimes().Return(proto.RepairTaskType) + mgr.RegisterTables(repairTaskTable) + mgr.RegisterTables(repairTaskTable) + } + { + // QueryMarkDeleteTasks failed + archiveTable := NewMockArchiveTable(ctr) + mgr := NewArchiveStoreMgr(archiveTable, testArchiveConfig) + defer mgr.Close() + + repairTaskTable := NewMockRepairTaskTable(ctr) + repairTaskTable.EXPECT().Name().AnyTimes().Return(proto.RepairTaskType) + repairTaskTable.EXPECT().QueryMarkDeleteTasks(any, any).AnyTimes().Return(nil, errMock) + + mgr.RegisterTables(repairTaskTable) + mgr.storeRun() + } + { + archiveTable := NewMockArchiveTable(ctr) + mgr := NewArchiveStoreMgr(archiveTable, testArchiveConfig) + defer mgr.Close() + + repairTaskTable := NewMockRepairTaskTable(ctr) + repairTaskTable.EXPECT().Name().AnyTimes().Return(proto.RepairTaskType) + id1 := uuid.New().String() + id2 := uuid.New().String() + task1 := proto.VolRepairTask{TaskID: id1} + task2 := proto.VolRepairTask{TaskID: id2} + record1 := &proto.ArchiveRecord{TaskID: id1, TaskType: proto.RepairTaskType, Content: task1} // task already archive and remove success + record2 := &proto.ArchiveRecord{TaskID: id2, TaskType: proto.RepairTaskType, Content: task2} // archive success + record3 := &proto.ArchiveRecord{} // find task failed + record4 := &proto.ArchiveRecord{} // task already archive and remove source failed + repairTaskTable.EXPECT().QueryMarkDeleteTasks(any, any).AnyTimes().Return([]*proto.ArchiveRecord{record1, record2, record3, record4}, nil) + + archiveTable.EXPECT().FindTask(any, any).Return(record1, nil) + archiveTable.EXPECT().FindTask(any, any).Return(nil, base.ErrNoDocuments) + archiveTable.EXPECT().FindTask(any, any).Return(nil, errMock) + archiveTable.EXPECT().FindTask(any, any).Return(record4, nil) + + archiveTable.EXPECT().Insert(any, any).Return(errMock) + archiveTable.EXPECT().FindTask(any, any).Return(record1, nil) + archiveTable.EXPECT().FindTask(any, any).Return(record4, nil) + repairTaskTable.EXPECT().RemoveMarkDelete(any, any).Return(nil) + repairTaskTable.EXPECT().RemoveMarkDelete(any, any).Return(errMock) + + mgr.RegisterTables(repairTaskTable) + mgr.storeRun() + } +} diff --git a/blobstore/scheduler/balancer.go b/blobstore/scheduler/balancer.go new file mode 100644 index 000000000..fb5eb7d77 --- /dev/null +++ b/blobstore/scheduler/balancer.go @@ -0,0 +1,264 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "errors" + "sort" + "time" + + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/taskswitch" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/scheduler/base" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/scheduler/db" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +// IBalancer define the interface of balance manager +type IBalancer interface { + Migrator +} + +const ( + collectBalanceTaskPauseS = 5 +) + +var ( + // ErrNoBalanceVunit no balance volume unit on disk + ErrNoBalanceVunit = errors.New("no balance volume unit on disk") + // ErrTooManyBalancingTasks too many balancing tasks + ErrTooManyBalancingTasks = errors.New("too many balancing tasks") +) + +// BalanceMgrConfig balance task manager config +type BalanceMgrConfig struct { + BalanceDiskCntLimit int `json:"balance_disk_cnt_limit"` + MaxDiskFreeChunkCnt int64 `json:"max_disk_free_chunk_cnt"` + MinDiskFreeChunkCnt int64 `json:"min_disk_free_chunk_cnt"` + MigrateConfig +} + +// BalanceMgr balance manager +type BalanceMgr struct { + IMigrater + + clusterTopology IClusterTopology + clusterMgrCli client.ClusterMgrAPI + + cfg *BalanceMgrConfig +} + +// NewBalanceMgr returns balance manager +func NewBalanceMgr( + clusterMgrCli client.ClusterMgrAPI, + volumeUpdater client.IVolumeUpdater, + taskSwitch taskswitch.ISwitcher, + clusterTopology IClusterTopology, + taskTbl db.IMigrateTaskTable, + conf *BalanceMgrConfig) *BalanceMgr { + mgr := &BalanceMgr{ + clusterTopology: clusterTopology, + clusterMgrCli: clusterMgrCli, + cfg: conf, + } + mgr.IMigrater = NewMigrateMgr(clusterMgrCli, volumeUpdater, taskSwitch, taskTbl, + &conf.MigrateConfig, proto.BalanceTaskType, conf.ClusterID) + mgr.IMigrater.SetLockFailHandleFunc(mgr.IMigrater.FinishTaskInAdvanceWhenLockFail) + return mgr +} + +// Run run balance task manager +func (mgr *BalanceMgr) Run() { + go mgr.collectTaskLoop() + mgr.IMigrater.Run() + go mgr.clearTaskLoop() +} + +// Close close balance task manager +func (mgr *BalanceMgr) Close() { + mgr.clusterTopology.Close() + mgr.IMigrater.Close() +} + +func (mgr *BalanceMgr) collectTaskLoop() { + t := time.NewTicker(time.Duration(mgr.cfg.CollectTaskIntervalS) * time.Second) + defer t.Stop() + + for { + select { + case <-t.C: + mgr.IMigrater.WaitEnable() + err := mgr.collectionTask() + if err == ErrTooManyBalancingTasks || err == ErrNoBalanceVunit { + log.Debugf("no task to collect and sleep: sleep second[%d], err[%+v]", collectBalanceTaskPauseS, err) + time.Sleep(time.Duration(collectBalanceTaskPauseS) * time.Second) + } + case <-mgr.IMigrater.Done(): + return + } + } +} + +func (mgr *BalanceMgr) collectionTask() (err error) { + span, ctx := trace.StartSpanFromContext(context.Background(), "balance_collectionTask") + defer span.Finish() + + needBalanceDiskCnt := mgr.cfg.BalanceDiskCntLimit - mgr.IMigrater.GetMigratingDiskNum() + if needBalanceDiskCnt <= 0 { + span.Warnf("the number of balancing disk is greater than config: current[%d], conf[%d]", + mgr.IMigrater.GetMigratingDiskNum(), mgr.cfg.BalanceDiskCntLimit) + return ErrTooManyBalancingTasks + } + + // select balance disks + disks := mgr.selectDisks(mgr.cfg.MaxDiskFreeChunkCnt, mgr.cfg.MinDiskFreeChunkCnt) + span.Debugf("select balance disks: len[%d]", len(disks)) + + balanceDiskCnt := 0 + for _, disk := range disks { + err = mgr.genOneBalanceTask(ctx, disk) + if err != nil { + continue + } + + balanceDiskCnt++ + if balanceDiskCnt >= needBalanceDiskCnt { + break + } + } + // if balanceDiskCnt==0, means there is no balance volume unit on disk and need to do collect task later + if balanceDiskCnt == 0 { + span.Infof("select disks has no balance volume unit on disk: len[%d]", len(disks)) + return ErrNoBalanceVunit + } + + return nil +} + +func (mgr *BalanceMgr) selectDisks(maxFreeChunkCnt, minFreeChunkCnt int64) []*client.DiskInfoSimple { + var allDisks []*client.DiskInfoSimple + for idcName := range mgr.clusterTopology.GetIDCs() { + if idcDisks := mgr.clusterTopology.GetIDCDisks(idcName); idcDisks != nil { + if freeChunkCntMax(idcDisks) >= maxFreeChunkCnt { + allDisks = append(allDisks, idcDisks...) + } + } + } + + var selected []*client.DiskInfoSimple + for _, disk := range allDisks { + if !disk.IsHealth() { + continue + } + if ok := mgr.IMigrater.IsMigratingDisk(disk.DiskID); ok { + continue + } + if disk.FreeChunkCnt < minFreeChunkCnt { + selected = append(selected, disk) + } + } + return selected +} + +func (mgr *BalanceMgr) genOneBalanceTask(ctx context.Context, diskInfo *client.DiskInfoSimple) (err error) { + span := trace.SpanFromContextSafe(ctx) + + vuid, err := mgr.selectBalanceVunit(ctx, diskInfo.DiskID) + if err != nil { + span.Errorf("generate task source failed: disk_id[%d], err[%+v]", diskInfo.DiskID, err) + return + } + + span.Debugf("select balance volume unit; vuid[%d+, volume_id[%v]", vuid, vuid.Vid()) + task := &proto.MigrateTask{ + TaskID: mgr.genUniqTaskID(vuid.Vid()), + State: proto.MigrateStateInited, + SourceIdc: diskInfo.Idc, + SourceDiskID: diskInfo.DiskID, + SourceVuid: vuid, + } + mgr.IMigrater.AddTask(ctx, task) + return +} + +func (mgr *BalanceMgr) selectBalanceVunit(ctx context.Context, diskID proto.DiskID) (vuid proto.Vuid, err error) { + span := trace.SpanFromContextSafe(ctx) + + vunits, err := mgr.clusterMgrCli.ListDiskVolumeUnits(ctx, diskID) + if err != nil { + return + } + + sortVunitByUsed(vunits) + + for i := range vunits { + volInfo, err := mgr.clusterMgrCli.GetVolumeInfo(ctx, vunits[i].Vuid.Vid()) + if err != nil { + span.Errorf("get volume info failed: vid[%d], err[%+v]", vunits[i].Vuid.Vid(), err) + continue + } + if volInfo.IsIdle() { + return vunits[i].Vuid, nil + } + } + return vuid, ErrNoBalanceVunit +} + +func (mgr *BalanceMgr) clearTaskLoop() { + t := time.NewTicker(time.Duration(mgr.cfg.CheckTaskIntervalS) * time.Second) + defer t.Stop() + + for { + select { + case <-t.C: + mgr.IMigrater.WaitEnable() + mgr.ClearFinishedTask() + case <-mgr.IMigrater.Done(): + return + } + } +} + +// ClearFinishedTask clear finished balance task +func (mgr *BalanceMgr) ClearFinishedTask() { + span, ctx := trace.StartSpanFromContext(context.Background(), "balance_ClearFinishedTask") + defer span.Finish() + + clearStates := []proto.MigrateState{proto.MigrateStateFinished, proto.MigrateStateFinishedInAdvance} + mgr.IMigrater.ClearTasksByStates(ctx, clearStates) +} + +func (mgr *BalanceMgr) genUniqTaskID(vid proto.Vid) string { + return base.GenTaskID("balance", vid) +} + +func sortVunitByUsed(vunits []*client.VunitInfoSimple) { + sort.Slice(vunits, func(i, j int) bool { + return vunits[i].Used < vunits[j].Used + }) +} + +func freeChunkCntMax(disks []*client.DiskInfoSimple) int64 { + var max int64 + for _, disk := range disks { + if disk.FreeChunkCnt > max { + max = disk.FreeChunkCnt + } + } + return max +} diff --git a/blobstore/scheduler/balancer_test.go b/blobstore/scheduler/balancer_test.go new file mode 100644 index 000000000..2cf12873d --- /dev/null +++ b/blobstore/scheduler/balancer_test.go @@ -0,0 +1,225 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + api "github.com/cubefs/cubefs/blobstore/api/scheduler" + "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/scheduler/base" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/testing/mocks" + "github.com/cubefs/cubefs/blobstore/util/closer" +) + +func newBalancer(t *testing.T) *BalanceMgr { + ctr := gomock.NewController(t) + clusterMgr := NewMockClusterMgrAPI(ctr) + volumeUpdater := NewMockVolumeUpdater(ctr) + taskSwitch := mocks.NewMockSwitcher(ctr) + topologyMgr := NewMockClusterTopology(ctr) + migrateTable := NewMockMigrateTaskTable(ctr) + migrater := NewMockMigrater(ctr) + conf := &BalanceMgrConfig{} + c := closer.New() + + topologyMgr.EXPECT().Close().AnyTimes().Return() + migrater.EXPECT().StatQueueTaskCnt().AnyTimes().Return(0, 0, 0) + migrater.EXPECT().Close().AnyTimes().DoAndReturn(c.Close) + migrater.EXPECT().Done().AnyTimes().Return(c.Done()) + migrater.EXPECT().WaitEnable().AnyTimes().Return() + migrater.EXPECT().Enabled().AnyTimes().Return(true) + mgr := NewBalanceMgr(clusterMgr, volumeUpdater, taskSwitch, topologyMgr, migrateTable, conf) + mgr.IMigrater = migrater + return mgr +} + +func TestBalanceLoad(t *testing.T) { + mgr := newBalancer(t) + mgr.IMigrater.(*MockMigrater).EXPECT().Load().Return(nil) + err := mgr.Load() + require.NoError(t, err) +} + +func TestBalanceRun(t *testing.T) { + mgr := newBalancer(t) + defer mgr.Close() + + mgr.IMigrater.(*MockMigrater).EXPECT().Run().Return() + mgr.IMigrater.(*MockMigrater).EXPECT().ClearTasksByStates(any, any).AnyTimes().Return() + mgr.IMigrater.(*MockMigrater).EXPECT().GetMigratingDiskNum().AnyTimes().Return(1) + mgr.cfg.CollectTaskIntervalS = 1 + mgr.cfg.CheckTaskIntervalS = 1 + require.True(t, mgr.Enabled()) + mgr.Run() + + time.Sleep(1 * time.Second) +} + +func TestBalanceCollectionTask(t *testing.T) { + { + mgr := newBalancer(t) + mgr.IMigrater.(*MockMigrater).EXPECT().GetMigratingDiskNum().AnyTimes().Return(1) + + err := mgr.collectionTask() + require.True(t, errors.Is(err, ErrTooManyBalancingTasks)) + mgr.Close() + } + { + mgr := newBalancer(t) + mgr.cfg.BalanceDiskCntLimit = 2 + mgr.IMigrater.(*MockMigrater).EXPECT().GetMigratingDiskNum().AnyTimes().Return(1) + + disk1 := &client.DiskInfoSimple{ + ClusterID: 1, + Idc: "z0", + Rack: "rack1", + Host: "127.0.0.1:8000", + Status: proto.DiskStatusNormal, + DiskID: 1, + FreeChunkCnt: 10, + MaxChunkCnt: 700, + } + disk2 := &client.DiskInfoSimple{ + ClusterID: 1, + Idc: "z1", + Rack: "rack1", + Host: "127.0.0.2:8000", + Status: proto.DiskStatusNormal, + DiskID: 2, + FreeChunkCnt: 100, + MaxChunkCnt: 700, + } + disk3 := &client.DiskInfoSimple{ + ClusterID: 1, + Idc: "z1", + Rack: "rack1", + Host: "127.0.0.3:8000", + Status: proto.DiskStatusBroken, + DiskID: 3, + FreeChunkCnt: 20, + MaxChunkCnt: 700, + } + clusterTopMgr := &ClusterTopologyMgr{ + taskStatsMgr: base.NewClusterTopologyStatisticsMgr(1, []float64{}), + } + clusterTopMgr.buildClusterTopo([]*client.DiskInfoSimple{disk1, disk2, disk3}, 1) + mgr.IMigrater.(*MockMigrater).EXPECT().IsMigratingDisk(any).AnyTimes().DoAndReturn(func(diskID proto.DiskID) bool { + return diskID == 1 + }) + mgr.clusterTopology = clusterTopMgr + + err := mgr.collectionTask() + require.True(t, errors.Is(err, ErrNoBalanceVunit)) + + // select one task + mgr.cfg.MinDiskFreeChunkCnt = 101 + volume := MockGenVolInfo(10000, codemode.EC6P6, proto.VolumeStatusIdle) + var units []*client.VunitInfoSimple + for _, unit := range volume.VunitLocations { + ele := client.VunitInfoSimple{ + Vuid: unit.Vuid, + DiskID: unit.DiskID, + } + units = append(units, &ele) + } + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(volume, nil) + mgr.IMigrater.(*MockMigrater).EXPECT().AddTask(any, any).Return() + err = mgr.collectionTask() + require.NoError(t, err) + + // select one task and gen task failed + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, errMock) + err = mgr.collectionTask() + require.True(t, errors.Is(err, ErrNoBalanceVunit)) + + // select one task and gen task failed + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).AnyTimes().Return(nil, errMock) + err = mgr.collectionTask() + require.True(t, errors.Is(err, ErrNoBalanceVunit)) + } +} + +func TestBalanceAcquireTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + mgr := newBalancer(t) + mgr.IMigrater.(*MockMigrater).EXPECT().AcquireTask(any, any).Return(&proto.MigrateTask{}, nil) + _, err := mgr.AcquireTask(ctx, idc) + require.NoError(t, err) +} + +func TestBalanceCancelTask(t *testing.T) { + ctx := context.Background() + mgr := newBalancer(t) + mgr.IMigrater.(*MockMigrater).EXPECT().CancelTask(any, any).Return(nil) + err := mgr.CancelTask(ctx, &api.CancelTaskArgs{}) + require.NoError(t, err) +} + +func TestBalanceReclaimTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + mgr := newBalancer(t) + mgr.IMigrater.(*MockMigrater).EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil) + t1 := mockGenMigrateTask(idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap) + err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{}) + require.NoError(t, err) +} + +func TestBalanceCompleteTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + mgr := newBalancer(t) + mgr.IMigrater.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(nil) + t1 := mockGenMigrateTask(idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap) + err := mgr.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: idc, TaskId: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + require.NoError(t, err) + + mgr.IMigrater.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(errMock) + err = mgr.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: idc, TaskId: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + require.True(t, errors.Is(err, errMock)) +} + +func TestBalanceRenewalTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + mgr := newBalancer(t) + mgr.IMigrater.(*MockMigrater).EXPECT().RenewalTask(any, any, any).Return(nil) + err := mgr.RenewalTask(ctx, idc, "") + require.NoError(t, err) + + mgr.IMigrater.(*MockMigrater).EXPECT().RenewalTask(any, any, any).Return(errMock) + err = mgr.RenewalTask(ctx, idc, "") + require.True(t, errors.Is(err, errMock)) +} + +func TestBalanceStatQueueTaskCnt(t *testing.T) { + mgr := newBalancer(t) + inited, prepared, completed := mgr.StatQueueTaskCnt() + require.Equal(t, 0, inited) + require.Equal(t, 0, prepared) + require.Equal(t, 0, completed) +} diff --git a/blobstore/scheduler/base/base_test.go b/blobstore/scheduler/base/base_test.go new file mode 100644 index 000000000..76eed9863 --- /dev/null +++ b/blobstore/scheduler/base/base_test.go @@ -0,0 +1,85 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package base + +import ( + "errors" + "fmt" + "testing" + + "github.com/Shopify/sarama" + + "github.com/cubefs/cubefs/blobstore/scheduler/db" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +//go:generate mockgen -destination=./utils_mock_test.go -package=base -mock_names IAllocVunit=MockAllocVunit github.com/cubefs/cubefs/blobstore/scheduler/base IAllocVunit + +const testTopic = "test_topic" + +var ( + _ db.IKafkaOffsetTable = &mockAccess{} + + errMock = errors.New("mock error") +) + +func init() { + log.SetOutputLevel(log.Lfatal) +} + +func newBroker(t *testing.T) *sarama.MockBroker { + mockFetchResponse := sarama.NewMockFetchResponse(t, 1) + mockFetchResponse.SetVersion(1) + var msg sarama.ByteEncoder = []byte("FOO") + for i := 0; i < 1000; i++ { + mockFetchResponse.SetMessage(testTopic, 0, int64(i), msg) + } + + broker := sarama.NewMockBrokerAddr(t, 0, "127.0.0.1:0") + broker.SetHandlerByMap(map[string]sarama.MockResponse{ + "MetadataRequest": sarama.NewMockMetadataResponse(t). + SetBroker(broker.Addr(), broker.BrokerID()). + SetLeader(testTopic, 0, broker.BrokerID()), + "OffsetRequest": sarama.NewMockOffsetResponse(t). + SetOffset(testTopic, 0, sarama.OffsetOldest, 0). + SetOffset(testTopic, 0, sarama.OffsetNewest, 2345), + "FetchRequest": mockFetchResponse, + }) + + return broker +} + +type mockAccess struct { + offsets map[string]int64 + err error +} + +func newMockAccess(err error) *mockAccess { + return &mockAccess{ + offsets: make(map[string]int64), + err: err, + } +} + +func (m *mockAccess) Set(topic string, partition int32, offset int64) error { + key := fmt.Sprintf("%s_%d", topic, partition) + m.offsets[key] = offset + return m.err +} + +func (m *mockAccess) Get(topic string, partition int32) (int64, error) { + key := fmt.Sprintf("%s_%d", topic, partition) + return m.offsets[key], m.err +} diff --git a/blobstore/scheduler/base/kafka_consumer.go b/blobstore/scheduler/base/kafka_consumer.go new file mode 100644 index 000000000..8bedc3189 --- /dev/null +++ b/blobstore/scheduler/base/kafka_consumer.go @@ -0,0 +1,222 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package base + +import ( + "context" + "fmt" + "time" + + "github.com/Shopify/sarama" + "go.mongodb.org/mongo-driver/mongo" + + "github.com/cubefs/cubefs/blobstore/common/kafka" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/scheduler/db" +) + +const minConsumeWaitTime = time.Millisecond * 500 + +// IConsumer define the interface of consumer for message consume +type IConsumer interface { + ConsumeMessages(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) + CommitOffset(ctx context.Context) error +} + +// KafkaConfig kafka config +type KafkaConfig struct { + Topic string + Partitions []int32 + BrokerList []string +} + +// ConsumeInfo consume info +type ConsumeInfo struct { + Offset int64 + Commit int64 +} + +// TopicConsumer rotate consume msg among partition consumers +type TopicConsumer struct { + partitionsConsumers []IConsumer + + curIdx int +} + +// NewTopicConsumer returns topic round-robin partition consumer +func NewTopicConsumer(cfg *KafkaConfig, offsetAccessor db.IKafkaOffsetTable) (IConsumer, error) { + consumers, err := NewKafkaPartitionConsumers(cfg, offsetAccessor) + if err != nil { + return nil, err + } + topicConsumer := &TopicConsumer{ + partitionsConsumers: consumers, + } + return topicConsumer, err +} + +// ConsumeMessages consumer messages +func (c *TopicConsumer) ConsumeMessages(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) { + msgs = c.partitionsConsumers[c.curIdx].ConsumeMessages(ctx, msgCnt) + c.curIdx = (c.curIdx + 1) % len(c.partitionsConsumers) + return +} + +// CommitOffset commit offset +func (c *TopicConsumer) CommitOffset(ctx context.Context) error { + for _, pc := range c.partitionsConsumers { + if err := pc.CommitOffset(ctx); err != nil { + return err + } + } + return nil +} + +// PartitionConsumer partition consumer +type PartitionConsumer struct { + topic string + partition int32 + consumer sarama.PartitionConsumer + consumeInfo ConsumeInfo + offsetAccessor db.IKafkaOffsetTable // consume offset persistence +} + +// NewKafkaPartitionConsumers returns kafka partition consumers +func NewKafkaPartitionConsumers(cfg *KafkaConfig, offsetAccessor db.IKafkaOffsetTable) ([]IConsumer, error) { + var consumers []IConsumer + consumer, err := sarama.NewConsumer(cfg.BrokerList, defaultKafkaCfg()) + if err != nil { + return nil, fmt.Errorf("new consumer: err[%w]", err) + } + if len(cfg.Partitions) == 0 { + partitions, err := consumer.Partitions(cfg.Topic) + if err != nil { + return nil, err + } + cfg.Partitions = partitions + } + for _, partition := range cfg.Partitions { + partitionConsumer, err := newKafkaPartitionConsumer(consumer, cfg.Topic, partition, offsetAccessor) + if err != nil { + return nil, fmt.Errorf("new kafka partition consumer: err[%w]", err) + } + consumers = append(consumers, partitionConsumer) + } + + return consumers, nil +} + +func newKafkaPartitionConsumer(consumer sarama.Consumer, topic string, partition int32, offsetAccessor db.IKafkaOffsetTable) (*PartitionConsumer, error) { + kafkaConsumer := PartitionConsumer{ + topic: topic, + offsetAccessor: offsetAccessor, + } + + partConsumeInfo, err := kafkaConsumer.loadConsumeInfo(topic, partition) + if err != nil { + return nil, fmt.Errorf("loadConsumeInfo: topic[%s], err[%w]", topic, err) + } + + pc, err := consumer.ConsumePartition(topic, partition, partConsumeInfo.Commit) + if err != nil { + return nil, fmt.Errorf("consume partition: topic[%s], partition[%d], partConsumeInfo[%+v], err[%w]", topic, partition, partConsumeInfo, err) + } + + kafkaConsumer.partition = partition + kafkaConsumer.consumer = pc + kafkaConsumer.consumeInfo = partConsumeInfo + + return &kafkaConsumer, nil +} + +// ConsumeMessages consume messages +func (c *PartitionConsumer) ConsumeMessages(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) { + span := trace.SpanFromContextSafe(ctx) + + d := time.Millisecond / 2 * time.Duration(msgCnt) // assume each message cost 0.5 ms + if d < minConsumeWaitTime { + d = minConsumeWaitTime + } + + ticker := time.NewTicker(d) + defer ticker.Stop() + + start := time.Now() + for { + var err error + var msg *sarama.ConsumerMessage + select { + case msg = <-c.consumer.Messages(): + case err = <-c.consumer.Errors(): + case <-ticker.C: + } + if err != nil { + span.Errorf("acquire msg failed: topic[%s], partition[%d], err[%+v]", c.topic, c.partition, err) + break + } + + if msg == nil { + span.Debugf("no message for consume and return") + break // consume finish,return + } + + c.consumeInfo.Offset = msg.Offset + msgs = append(msgs, msg) + if len(msgs) >= msgCnt { + break + } + } + + span.Debugf("consume info: topic[%s], partition[%d], time cost[%+v], consumer msg numbers[%d], offset[%d], batch msg cnt[%d]", + c.topic, c.partition, time.Since(start), len(msgs), c.consumeInfo.Offset, msgCnt) + return +} + +// CommitOffset commit offset +func (c *PartitionConsumer) CommitOffset(ctx context.Context) error { + span := trace.SpanFromContextSafe(ctx) + + offset := c.consumeInfo.Offset + span.Debugf("start commit offset: offset[%d], topic[%s], partition[%d]", offset, c.topic, c.partition) + err := c.offsetAccessor.Set(c.topic, c.partition, offset) + if err != nil { + span.Errorf("commit offset failed: [%+v]", err) + return err + } + c.consumeInfo.Commit = offset + return nil +} + +func (c *PartitionConsumer) loadConsumeInfo(topic string, pt int32) (consumeInfo ConsumeInfo, err error) { + commitOffset, err := c.offsetAccessor.Get(topic, pt) + if err != nil { + if err == mongo.ErrNoDocuments { + return ConsumeInfo{Commit: sarama.OffsetOldest, Offset: sarama.OffsetOldest}, nil + } + return + } + + return ConsumeInfo{Commit: commitOffset + 1, Offset: commitOffset}, err +} + +func defaultKafkaCfg() *sarama.Config { + cfg := sarama.NewConfig() + cfg.Version = kafka.DefaultKafkaVersion + cfg.Consumer.Return.Errors = true + cfg.Producer.Return.Successes = true + cfg.Producer.RequiredAcks = sarama.WaitForAll + cfg.Producer.Compression = sarama.CompressionSnappy + return cfg +} diff --git a/blobstore/scheduler/base/kafka_consumer_test.go b/blobstore/scheduler/base/kafka_consumer_test.go new file mode 100644 index 000000000..4de019c5d --- /dev/null +++ b/blobstore/scheduler/base/kafka_consumer_test.go @@ -0,0 +1,248 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package base + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + "testing" + + "github.com/Shopify/sarama" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/mongo" +) + +var ( + _ sarama.PartitionConsumer = &mockPartitionConsumer{} + _ sarama.Consumer = &mockConsumer{} +) + +type mockPartitionConsumer struct { + topic string + pid int32 + offset int64 + + msgCh chan *sarama.ConsumerMessage + errCh chan *sarama.ConsumerError +} + +func newMockPartitionConsumer(topic string, pid int32) *mockPartitionConsumer { + return &mockPartitionConsumer{ + topic: topic, + pid: pid, + offset: 0, + msgCh: make(chan *sarama.ConsumerMessage), + errCh: make(chan *sarama.ConsumerError), + } +} + +func (m *mockPartitionConsumer) sendMsg(key, val string) { + m.msgCh <- &sarama.ConsumerMessage{ + Key: []byte(key), + Value: []byte(val), + Topic: m.topic, + Partition: m.pid, + Offset: atomic.LoadInt64(&m.offset), + } + atomic.AddInt64(&m.offset, 1) +} + +func (m *mockPartitionConsumer) sendErr(err error) { + m.errCh <- &sarama.ConsumerError{ + Topic: m.topic, + Partition: m.pid, + Err: err, + } +} + +func (m *mockPartitionConsumer) AsyncClose() {} +func (m *mockPartitionConsumer) Close() error { return nil } +func (m *mockPartitionConsumer) Messages() <-chan *sarama.ConsumerMessage { return m.msgCh } +func (m *mockPartitionConsumer) Errors() <-chan *sarama.ConsumerError { return m.errCh } +func (m *mockPartitionConsumer) HighWaterMarkOffset() int64 { return 0 } + +type mockConsumer struct { + topics map[string][]*mockPartitionConsumer +} + +func newMockConsumer() *mockConsumer { + return &mockConsumer{ + topics: map[string][]*mockPartitionConsumer{ + "topic1": newBatchpc("topic1", []int32{1, 2, 3, 4}), + "topic2": newBatchpc("topic2", []int32{1, 2, 3, 4}), + }, + } +} + +func newBatchpc(topic string, pids []int32) (ret []*mockPartitionConsumer) { + for _, pid := range pids { + pc := newMockPartitionConsumer(topic, pid) + ret = append(ret, pc) + } + return ret +} + +func (m *mockConsumer) Run(msgCnt int) { + for _, pcs := range m.topics { + for _, pc := range pcs { + tmpPc := pc + go func() { + for i := 1; i <= msgCnt; i++ { + tmpPc.sendMsg(fmt.Sprintf("key_%d", i), fmt.Sprintf("val_%d", i)) + } + }() + } + } +} + +func (m *mockConsumer) getPc(topic string, pid int32) *mockPartitionConsumer { + pcs := m.topics[topic] + for _, pc := range pcs { + if pc.pid == pid { + return pc + } + } + return nil +} + +func (m *mockConsumer) Topics() ([]string, error) { + var ret []string + for topic := range m.topics { + ret = append(ret, topic) + } + return ret, nil +} + +func (m *mockConsumer) Partitions(topic string) ([]int32, error) { + var pids []int32 + for _, pc := range m.topics[topic] { + pids = append(pids, pc.pid) + } + return pids, nil +} + +func (m *mockConsumer) ConsumePartition(topic string, partition int32, offset int64) (sarama.PartitionConsumer, error) { + for _, pc := range m.topics[topic] { + if pc.pid == partition { + return pc, nil + } + } + return nil, errors.New("not found") +} + +func (m *mockConsumer) HighWaterMarks() map[string]map[int32]int64 { + return nil +} + +func (m *mockConsumer) Close() error { + return nil +} + +func TestPartitionConsumer(t *testing.T) { + mockConsume := newMockConsumer() + access := newMockAccess(nil) + pc, err := newKafkaPartitionConsumer(mockConsume, "topic1", 1, access) + require.NoError(t, err) + + mockConsume.Run(100) + msgs := pc.ConsumeMessages(context.Background(), 5) + require.Equal(t, 5, len(msgs)) + + err = pc.CommitOffset(context.Background()) + require.NoError(t, err) +} + +func TestTopicConsume(t *testing.T) { + const topic = "topic1" + var cs []IConsumer + mockConsume := newMockConsumer() + access := newMockAccess(nil) + for _, pid := range []int32{1, 2, 3} { + pc, _ := newKafkaPartitionConsumer(mockConsume, topic, pid, access) + cs = append(cs, pc) + } + mockConsume.Run(100) + + topicConsumer := &TopicConsumer{ + partitionsConsumers: cs, + } + for _, pid := range []int32{1, 2, 3} { + topicConsumer.ConsumeMessages(context.Background(), 1) + err := topicConsumer.CommitOffset(context.Background()) + require.NoError(t, err) + off, err := access.Get(topic, pid) + require.NoError(t, err) + require.Equal(t, int64(0), off) + } + + topicConsumer.ConsumeMessages(context.Background(), 1) + err := topicConsumer.CommitOffset(context.Background()) + require.NoError(t, err) + off, err := access.Get(topic, 1) + require.NoError(t, err) + require.Equal(t, int64(1), off) +} + +func TestConsumerError(t *testing.T) { + mockConsume := newMockConsumer() + access := newMockAccess(nil) + pc, err := newKafkaPartitionConsumer(mockConsume, "topic1", 1, access) + require.NoError(t, err) + pcTmp := mockConsume.getPc("topic1", 1) + go func() { pcTmp.sendErr(errors.New("fake error")) }() + + msgs := pc.ConsumeMessages(context.Background(), 1) + require.Equal(t, 0, len(msgs)) +} + +func TestLoadConsumeInfo(t *testing.T) { + mockConsume := newMockConsumer() + access := newMockAccess(mongo.ErrNoDocuments) + pc, _ := newKafkaPartitionConsumer(mockConsume, "topic1", 1, access) + off, _ := pc.loadConsumeInfo("topic1", 1) + require.Equal(t, sarama.OffsetOldest, off.Offset) +} + +func TestNewTopicConsumer(t *testing.T) { + broker := newBroker(t) + defer broker.Close() + cfg := &KafkaConfig{ + Topic: testTopic, + BrokerList: []string{broker.Addr()}, + Partitions: []int32{0}, + } + + access := newMockAccess(nil) + consumer, err := NewTopicConsumer(cfg, access) + require.NoError(t, err) + + msgs := consumer.ConsumeMessages(context.Background(), 1) + require.Equal(t, 1, len(msgs)) + + access.err = errMock + err = consumer.CommitOffset(context.Background()) + require.Error(t, err) + + cfg.BrokerList = []string{} + _, err = NewTopicConsumer(cfg, access) + require.Error(t, err) + + cfg.Partitions = nil + cfg.BrokerList = []string{broker.Addr()} + _, err = NewTopicConsumer(cfg, access) + require.Error(t, err) +} diff --git a/blobstore/scheduler/base/kafka_topic_monitor.go b/blobstore/scheduler/base/kafka_topic_monitor.go new file mode 100644 index 000000000..599b9cd6e --- /dev/null +++ b/blobstore/scheduler/base/kafka_topic_monitor.go @@ -0,0 +1,90 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package base + +import ( + "fmt" + "time" + + "github.com/Shopify/sarama" + "go.mongodb.org/mongo-driver/mongo" + + "github.com/cubefs/cubefs/blobstore/common/kafka" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/scheduler/db" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +// KafkaTopicMonitor kafka monitor +type KafkaTopicMonitor struct { + topic string + partitions []int32 + offsetAccessor db.IKafkaOffsetTable + monitor *kafka.Monitor + interval time.Duration +} + +// NewKafkaTopicMonitor returns kafka topic monitor +func NewKafkaTopicMonitor(clusterID proto.ClusterID, cfg *KafkaConfig, offsetAccessor db.IKafkaOffsetTable, monitorIntervalS int) (*KafkaTopicMonitor, error) { + consumer, err := sarama.NewConsumer(cfg.BrokerList, defaultKafkaCfg()) + if err != nil { + return nil, err + } + + partitions, err := consumer.Partitions(cfg.Topic) + if err != nil { + return nil, fmt.Errorf("get partitions: err[%w]", err) + } + + // create kafka monitor + monitor, err := kafka.NewKafkaMonitor(clusterID, proto.ServiceNameScheduler, cfg.BrokerList, cfg.Topic, partitions, kafka.DefauleintervalSecs) + if err != nil { + return nil, fmt.Errorf("new kafka monitor: broker list[%v], topic[%v], parts[%v], error[%w]", + cfg.BrokerList, cfg.Topic, partitions, err) + } + + interval := time.Second * time.Duration(monitorIntervalS) + if interval <= 0 { + interval = time.Millisecond + } + return &KafkaTopicMonitor{ + topic: cfg.Topic, + partitions: partitions, + offsetAccessor: offsetAccessor, + monitor: monitor, + interval: interval, + }, nil +} + +// Run run kafka monitor +func (m *KafkaTopicMonitor) Run() { + ticker := time.NewTicker(m.interval) + defer ticker.Stop() + + for { + for _, partition := range m.partitions { + off, err := m.offsetAccessor.Get(m.topic, partition) + if err != nil { + if err != mongo.ErrNoDocuments { + log.Errorf("get consume offset failed: err[%v]", err) + } + continue + } + m.monitor.SetConsumeOffset(off, partition) + } + + <-ticker.C + } +} diff --git a/blobstore/scheduler/base/kafka_topic_monitor_test.go b/blobstore/scheduler/base/kafka_topic_monitor_test.go new file mode 100644 index 000000000..812563095 --- /dev/null +++ b/blobstore/scheduler/base/kafka_topic_monitor_test.go @@ -0,0 +1,46 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package base + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +func TestNewKafkaTopicMonitor(t *testing.T) { + broker := newBroker(t) + defer broker.Close() + cfg := &KafkaConfig{ + Topic: testTopic, + BrokerList: []string{broker.Addr()}, + Partitions: []int32{0}, + } + + access := newMockAccess(nil) + monitor, err := NewKafkaTopicMonitor(proto.ClusterID(1), cfg, access, 0) + go func() { + monitor.Run() + }() + time.Sleep(time.Second * 3) + require.NoError(t, err) + + cfg.BrokerList = []string{} + monitor, err = NewKafkaTopicMonitor(proto.ClusterID(1), cfg, access, 0) + require.Error(t, err) +} diff --git a/blobstore/scheduler/base/msg_sender.go b/blobstore/scheduler/base/msg_sender.go new file mode 100644 index 000000000..0c64ca490 --- /dev/null +++ b/blobstore/scheduler/base/msg_sender.go @@ -0,0 +1,47 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package base + +import "github.com/cubefs/cubefs/blobstore/common/kafka" + +// IProducer define the interface of producer +type IProducer interface { + SendMessage(msg []byte) (err error) + SendMessages(msgs [][]byte) (err error) +} + +type msgSender struct { + topic string + producer kafka.MsgProducer +} + +// NewMsgSender returns message sender +func NewMsgSender(cfg *kafka.ProducerCfg) (IProducer, error) { + producer, err := kafka.NewProducer(cfg) + if err != nil { + return nil, err + } + return &msgSender{topic: cfg.Topic, producer: producer}, nil +} + +// SendMessage send message to mq +func (sender *msgSender) SendMessage(msg []byte) error { + return sender.producer.SendMessage(sender.topic, msg) +} + +// SendMessages send message batch +func (sender *msgSender) SendMessages(msgs [][]byte) error { + return sender.producer.SendMessages(sender.topic, msgs) +} diff --git a/blobstore/scheduler/base/msg_sender_test.go b/blobstore/scheduler/base/msg_sender_test.go new file mode 100644 index 000000000..fffa28d8a --- /dev/null +++ b/blobstore/scheduler/base/msg_sender_test.go @@ -0,0 +1,72 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package base + +import ( + "testing" + + "github.com/Shopify/sarama" + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/common/kafka" +) + +func TestSendMessage(t *testing.T) { + kafka.DefaultKafkaVersion = sarama.V0_9_0_1 + + seedBroker := sarama.NewMockBrokerAddr(t, 1, "127.0.0.1:0") + leader := sarama.NewMockBrokerAddr(t, 2, "127.0.0.1:0") + + metadataResponse := new(sarama.MetadataResponse) + metadataResponse.AddBroker(leader.Addr(), leader.BrokerID()) + metadataResponse.AddTopicPartition(testTopic, 0, leader.BrokerID(), nil, nil, nil, 0) + seedBroker.Returns(metadataResponse) + + prodSuccess := new(sarama.ProduceResponse) + prodSuccess.AddTopicPartition(testTopic, 0, 0) + for i := 0; i < 10; i++ { + leader.Returns(prodSuccess) + } + + msgSender, err := NewMsgSender(&kafka.ProducerCfg{BrokerList: []string{seedBroker.Addr()}, Topic: testTopic}) + require.NoError(t, err) + + err = msgSender.SendMessage([]byte("dasdada")) + require.NoError(t, err) +} + +func TestSendMessages(t *testing.T) { + kafka.DefaultKafkaVersion = sarama.V0_9_0_1 + + seedBroker := sarama.NewMockBrokerAddr(t, 1, "127.0.0.1:0") + leader := sarama.NewMockBrokerAddr(t, 2, "127.0.0.1:0") + + metadataResponse := new(sarama.MetadataResponse) + metadataResponse.AddBroker(leader.Addr(), leader.BrokerID()) + metadataResponse.AddTopicPartition(testTopic, 0, leader.BrokerID(), nil, nil, nil, 0) + seedBroker.Returns(metadataResponse) + + prodSuccess := new(sarama.ProduceResponse) + prodSuccess.AddTopicPartition(testTopic, 0, 0) + for i := 0; i < 10; i++ { + leader.Returns(prodSuccess) + } + + msgSender, err := NewMsgSender(&kafka.ProducerCfg{BrokerList: []string{seedBroker.Addr()}, Topic: testTopic}) + require.NoError(t, err) + + err = msgSender.SendMessages([][]byte{[]byte("dasdada")}) + require.NoError(t, err) +} diff --git a/blobstore/scheduler/base/priority_consumer.go b/blobstore/scheduler/base/priority_consumer.go new file mode 100644 index 000000000..ccdc3f4b3 --- /dev/null +++ b/blobstore/scheduler/base/priority_consumer.go @@ -0,0 +1,90 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package base + +import ( + "context" + "fmt" + "sort" + + "github.com/Shopify/sarama" + + "github.com/cubefs/cubefs/blobstore/scheduler/db" +) + +type topicPriority struct { + topic string + priority int // The larger the value, the higher the priority +} + +// PriorityConsumerConfig priority consumer +type PriorityConsumerConfig struct { + KafkaConfig + Priority int `json:"priority"` +} + +type priorityConsumer struct { + topicConsumers map[string]IConsumer + sortedTopicPriority []topicPriority // Sort from largest to smallest +} + +// NewPriorityConsumer return priority consumer +func NewPriorityConsumer(cfgs []PriorityConsumerConfig, offsetAccessor db.IKafkaOffsetTable) (IConsumer, error) { + multiConsumer := priorityConsumer{} + multiConsumer.topicConsumers = make(map[string]IConsumer, len(cfgs)) + multiConsumer.sortedTopicPriority = make([]topicPriority, 0) + for _, cfg := range cfgs { + cs, err := NewTopicConsumer(&cfg.KafkaConfig, offsetAccessor) + if err != nil { + return nil, fmt.Errorf("new topic consumer: cfg[%+v], err[%w]", cfg.KafkaConfig, err) + } + + multiConsumer.topicConsumers[cfg.KafkaConfig.Topic] = cs + multiConsumer.sortedTopicPriority = append(multiConsumer.sortedTopicPriority, topicPriority{ + topic: cfg.KafkaConfig.Topic, priority: cfg.Priority, + }) + } + + sort.SliceStable(multiConsumer.sortedTopicPriority, func(i, j int) bool { + return multiConsumer.sortedTopicPriority[i].priority > multiConsumer.sortedTopicPriority[j].priority + }) + + return &multiConsumer, nil +} + +// ConsumeMessages consume messages +func (m *priorityConsumer) ConsumeMessages(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) { + remainCnt := msgCnt + for _, pair := range m.sortedTopicPriority { + consumer := m.topicConsumers[pair.topic] + ms := consumer.ConsumeMessages(ctx, remainCnt) + msgs = append(msgs, ms...) + if len(msgs) >= msgCnt { + return msgs + } + remainCnt = msgCnt - len(msgs) + } + return +} + +// CommitOffset commit offset +func (m *priorityConsumer) CommitOffset(ctx context.Context) error { + for _, c := range m.topicConsumers { + if err := c.CommitOffset(ctx); err != nil { + return err + } + } + return nil +} diff --git a/blobstore/scheduler/base/priority_consumer_test.go b/blobstore/scheduler/base/priority_consumer_test.go new file mode 100644 index 000000000..0aa72dc1f --- /dev/null +++ b/blobstore/scheduler/base/priority_consumer_test.go @@ -0,0 +1,70 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package base + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPriorityConsumer(t *testing.T) { + broker := newBroker(t) + defer broker.Close() + + cfgs := []PriorityConsumerConfig{ + { + KafkaConfig: KafkaConfig{ + Topic: testTopic, + BrokerList: []string{broker.Addr()}, + Partitions: []int32{0}, + }, + Priority: 1, + }, + } + + mockAcc := newMockAccess(nil) + priorityConsumer, err := NewPriorityConsumer(cfgs, mockAcc) + require.NoError(t, err) + + // Then: messages starting from offset 0 are consumed. + for i := 0; i < 10; i++ { + msgs := priorityConsumer.ConsumeMessages(context.Background(), 100) + if len(msgs) > 0 { + require.Equal(t, "FOO", string(msgs[0].Value)) + err = priorityConsumer.CommitOffset(context.Background()) + require.NoError(t, err) + } + } + + mockAcc.err = errMock + err = priorityConsumer.CommitOffset(context.Background()) + require.Error(t, err) + + // NewPriorityConsumer with empty BrokerList + cfgs = []PriorityConsumerConfig{ + { + KafkaConfig: KafkaConfig{ + Topic: testTopic, + BrokerList: nil, + Partitions: []int32{0}, + }, + Priority: 1, + }, + } + _, err = NewPriorityConsumer(cfgs, mockAcc) + require.Error(t, err) +} diff --git a/blobstore/scheduler/base/queue.go b/blobstore/scheduler/base/queue.go new file mode 100644 index 000000000..dd6a743e5 --- /dev/null +++ b/blobstore/scheduler/base/queue.go @@ -0,0 +1,464 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package base + +import ( + "container/list" + "errors" + "fmt" + "sync" + "time" + + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +// queue implement +// task queue & worker task queue + +var ( + // ErrNoSuchMessageID no such message id + ErrNoSuchMessageID = errors.New("no such message id") + // ErrUnmatchedVuids unmatched task vuids + ErrUnmatchedVuids = errors.New("unmatched task vuids") + errNoSuchIDCQueue = errors.New("no such idc queue") + errExistingMessageID = errors.New("existing message id") +) + +const ( + msgStateTodo = iota + 1 + msgStateDoing +) + +// 100 years is long enough。 +const neverTimeout = time.Duration(100*365*24) * time.Hour + +// Queue task queue +type Queue struct { + mu sync.RWMutex + todo *list.List + doing *list.List + msgs map[string]*list.Element + + msgTimeout time.Duration // default duration of task locking +} + +// NewQueue return task queue +func NewQueue(msgTimeout time.Duration) *Queue { + if msgTimeout == 0 { + msgTimeout = neverTimeout + } + q := &Queue{ + todo: new(list.List), + doing: new(list.List), + msgs: make(map[string]*list.Element), + msgTimeout: msgTimeout, + } + return q +} + +type msgEx struct { + id string + state int + deadline time.Time + msg interface{} +} + +// Push push message to queue id is uniquely identifies。 +func (q *Queue) Push(id string, msg interface{}) error { + q.mu.Lock() + defer q.mu.Unlock() + + if _, ok := q.msgs[id]; ok { + return errExistingMessageID + } + + m := &msgEx{ + id: id, + state: msgStateTodo, + msg: msg, + } + elem := q.todo.PushBack(m) + q.msgs[id] = elem + + return nil +} + +// Pop fetch a msg from queue。 +func (q *Queue) Pop() (string, interface{}, bool) { + q.mu.Lock() + defer q.mu.Unlock() + + now := time.Now() + for ele := q.doing.Front(); ele != nil; ele = ele.Next() { + m := ele.Value.(*msgEx) + if m.deadline.Before(now) { + m.deadline = now.Add(q.msgTimeout) + return m.id, m.msg, true + } + } + + // no timeout msg in doing ,fetch from todo + if q.todo.Len() == 0 { + return "", nil, false + } + elem := q.todo.Front() + q.todo.Remove(elem) + + m := elem.Value.(*msgEx) + m.state = msgStateDoing + m.deadline = now.Add(q.msgTimeout) + + elem = q.doing.PushFront(m) + q.msgs[m.id] = elem + + return m.id, m.msg, true +} + +// Get returns message by id +func (q *Queue) Get(id string) (interface{}, error) { + q.mu.RLock() + defer q.mu.RUnlock() + + elem, ok := q.msgs[id] + if !ok { + return nil, ErrNoSuchMessageID + } + return elem.Value.(*msgEx).msg, nil +} + +// Requeue :msg while get again after delay。 +func (q *Queue) Requeue(id string, delay time.Duration) error { + q.mu.Lock() + defer q.mu.Unlock() + + if delay < 0 { + delay = 0 + } + elem, ok := q.msgs[id] + if !ok { + return ErrNoSuchMessageID + } + m := elem.Value.(*msgEx) + if m.state == msgStateTodo { + // msg pop and new queue and reload msg, the pop msg state will chang to msgStateTodo + // if msg in todo queue then return success。 + return nil + } + + // msg in doing queue。 + m.deadline = time.Now().Add(delay) + return nil +} + +// Remove remove message by id +func (q *Queue) Remove(id string) error { + q.mu.Lock() + defer q.mu.Unlock() + + elem, ok := q.msgs[id] + if !ok { + return ErrNoSuchMessageID + } + m := elem.Value.(*msgEx) + switch m.state { + case msgStateTodo: + q.todo.Remove(elem) + case msgStateDoing: + q.doing.Remove(elem) + default: + panic("invalid msg state") + } + delete(q.msgs, id) + return nil +} + +// Stats returns queue stats +func (q *Queue) Stats() (todo, doing int) { + q.mu.RLock() + defer q.mu.RUnlock() + + return q.todo.Len(), q.doing.Len() +} + +// WorkerTask define worker task interface +type WorkerTask interface { + GetSrc() []proto.VunitLocation + GetDest() proto.VunitLocation + SetDest(dest proto.VunitLocation) +} + +// TaskQueue task queue +type TaskQueue struct { + mu sync.Mutex + queue *Queue + retryDelay time.Duration // punish a period of time to avoid frequent failure retry。 +} + +// NewTaskQueue returns task queue +func NewTaskQueue(retryDelay time.Duration) *TaskQueue { + return &TaskQueue{ + queue: NewQueue(0), + retryDelay: retryDelay, + } +} + +// PushTask push task to queue +func (q *TaskQueue) PushTask(taskID string, task WorkerTask) { + q.mu.Lock() + defer q.mu.Unlock() + err := q.queue.Push(taskID, task) + if err != nil { + panic("unexpect push task fail " + err.Error()) + } +} + +// PopTask return args： taskID, task, flag of task exist +func (q *TaskQueue) PopTask() (string, WorkerTask, bool) { + q.mu.Lock() + defer q.mu.Unlock() + taskID, task, exist := q.queue.Pop() + if exist { + return taskID, task.(WorkerTask), true + } + return "", nil, false +} + +// RemoveTask remove task by taskID +func (q *TaskQueue) RemoveTask(taskID string) error { + q.mu.Lock() + defer q.mu.Unlock() + return q.queue.Remove(taskID) +} + +// RetryTask retry task by taskID +func (q *TaskQueue) RetryTask(taskID string) { + q.mu.Lock() + defer q.mu.Unlock() + + err := q.queue.Requeue(taskID, q.retryDelay) + if err != nil { + panic("unexpect retry task fail:" + err.Error()) + } +} + +// Query find task by taskID +func (q *TaskQueue) Query(taskID string) (WorkerTask, bool) { + q.mu.Lock() + defer q.mu.Unlock() + task, err := q.queue.Get(taskID) + if err != nil { + return nil, false + } + return task.(WorkerTask), true +} + +// StatsTasks returns task stats +func (q *TaskQueue) StatsTasks() (todo int, doing int) { + q.mu.Lock() + defer q.mu.Unlock() + return q.queue.Stats() +} + +// WorkerTaskQueue task queue for worker +type WorkerTaskQueue struct { + mu sync.Mutex + idcQueues map[string]*Queue + + cancelPunishDuration time.Duration // task cancel will punish a period of time to avoid frequent failure retry + leaseExpiredS time.Duration +} + +// NewWorkerTaskQueue return worker task queue +func NewWorkerTaskQueue(cancelPunishDuration time.Duration) *WorkerTaskQueue { + // extended lock duration of task leasing + leaseExpiredS := proto.TaskLeaseExpiredS * time.Second + + return &WorkerTaskQueue{ + idcQueues: make(map[string]*Queue), + cancelPunishDuration: cancelPunishDuration, + leaseExpiredS: leaseExpiredS, + } +} + +// AddPreparedTask add prepared task +func (q *WorkerTaskQueue) AddPreparedTask(idc, taskID string, wtask WorkerTask) { + q.mu.Lock() + defer q.mu.Unlock() + + idcQueue, ok := q.idcQueues[idc] + if !ok { + idcQueue = NewQueue(q.leaseExpiredS) + q.idcQueues[idc] = idcQueue + } + err := idcQueue.Push(taskID, wtask) + if err != nil { + panic("unexpect add prepared task fail:" + err.Error()) + } +} + +// Acquire acquire task by idc +func (q *WorkerTaskQueue) Acquire(idc string) (taskID string, wtask WorkerTask, exist bool) { + q.mu.Lock() + defer q.mu.Unlock() + + idcQueue, ok := q.idcQueues[idc] + if !ok { + return "", nil, false + } + + taskID, task, exist := idcQueue.Pop() + if exist { + return taskID, task.(WorkerTask), exist + } + return "", nil, false +} + +// Cancel cancel task +func (q *WorkerTaskQueue) Cancel(idc, taskID string, src []proto.VunitLocation, dst proto.VunitLocation) error { + q.mu.Lock() + defer q.mu.Unlock() + + idcQueue, ok := q.idcQueues[idc] + if !ok { + return errNoSuchIDCQueue + } + + task, err := idcQueue.Get(taskID) + if err != nil { + return err + } + err = checkValid(task.(WorkerTask), src, dst) + if err != nil { + return err + } + + return idcQueue.Requeue(taskID, q.cancelPunishDuration) +} + +// Reclaim reclaim task +func (q *WorkerTaskQueue) Reclaim(idc, taskID string, src []proto.VunitLocation, oldDest, newDest proto.VunitLocation, newDiskID proto.DiskID) error { + q.mu.Lock() + defer q.mu.Unlock() + + idcQueue, ok := q.idcQueues[idc] + if !ok { + return errNoSuchIDCQueue + } + + task, err := idcQueue.Get(taskID) + if err != nil { + return err + } + wtask := task.(WorkerTask) + err = checkValid(wtask, src, oldDest) + if err != nil { + return err + } + wtask.SetDest(newDest) + return idcQueue.Requeue(taskID, 0) +} + +// Renewal renewal task +func (q *WorkerTaskQueue) Renewal(idc, taskID string) error { + q.mu.Lock() + defer q.mu.Unlock() + idcQueue, ok := q.idcQueues[idc] + if !ok { + return errNoSuchIDCQueue + } + return idcQueue.Requeue(taskID, q.leaseExpiredS) +} + +// Complete complete task +func (q *WorkerTaskQueue) Complete(idc, taskID string, src []proto.VunitLocation, dst proto.VunitLocation) (WorkerTask, error) { + q.mu.Lock() + defer q.mu.Unlock() + + idcQueue, ok := q.idcQueues[idc] + if !ok { + return nil, errNoSuchIDCQueue + } + + task, err := idcQueue.Get(taskID) + if err != nil { + return nil, err + } + + t := task.(WorkerTask) + err = checkValid(t, src, dst) + if err != nil { + return nil, err + } + + err = idcQueue.Remove(taskID) + if err != nil { + panic(fmt.Sprintf("task %s remove form queue fail err %v", taskID, err)) + } + + return t, err +} + +// StatsTasks returns task stats +func (q *WorkerTaskQueue) StatsTasks() (todo int, doing int) { + q.mu.Lock() + defer q.mu.Unlock() + for _, queue := range q.idcQueues { + todoTmp, doingTmp := queue.Stats() + todo += todoTmp + doing += doingTmp + } + return todo, doing +} + +// Query find task by idc and taskID +func (q *WorkerTaskQueue) Query(idc, taskID string) (WorkerTask, error) { + q.mu.Lock() + defer q.mu.Unlock() + + idcQueue, ok := q.idcQueues[idc] + if !ok { + return nil, errNoSuchIDCQueue + } + + wt, err := idcQueue.Get(taskID) + if err != nil { + return nil, err + } + return wt.(WorkerTask), nil +} + +// SetLeaseExpiredS set lease expired time +func (q *WorkerTaskQueue) SetLeaseExpiredS(dura time.Duration) { + q.leaseExpiredS = dura +} + +func checkValid(task WorkerTask, src []proto.VunitLocation, dst proto.VunitLocation) error { + if !vunitSliceEqual(task.GetSrc(), src) || task.GetDest() != dst { + return ErrUnmatchedVuids + } + return nil +} + +func vunitSliceEqual(a, b []proto.VunitLocation) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/blobstore/scheduler/base/queue_test.go b/blobstore/scheduler/base/queue_test.go new file mode 100644 index 000000000..e5ae53873 --- /dev/null +++ b/blobstore/scheduler/base/queue_test.go @@ -0,0 +1,262 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package base + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +func TestQueue(t *testing.T) { + q := NewQueue(500 * time.Millisecond) + msgID := "msg_1" + msgString := "test_msg" + + // test Push + err := q.Push(msgID, msgString) + require.NoError(t, err) + err = q.Push(msgID, msgString) + require.EqualError(t, err, errExistingMessageID.Error()) + + msg, err := q.Get(msgID) + require.NoError(t, err) + require.Equal(t, msgString, msg.(string)) + + // test Pop + id, msg, exist := q.Pop() + require.Equal(t, msgID, id) + require.Equal(t, msgString, msg) + require.Equal(t, true, exist) + _, _, exist = q.Pop() + require.Equal(t, false, exist) + time.Sleep(time.Second) + id, msg, exist = q.Pop() + require.Equal(t, msgID, id) + require.Equal(t, msgString, msg) + require.Equal(t, true, exist) + + // test Requeue + err = q.Requeue(msgID, 0) + require.NoError(t, err) + id, msg, exist = q.Pop() + require.Equal(t, msgID, id) + require.Equal(t, msgString, msg) + require.Equal(t, true, exist) + + err = q.Requeue(msgID, 100*time.Millisecond) + require.NoError(t, err) + time.Sleep(100 * time.Millisecond) + id, msg, exist = q.Pop() + require.Equal(t, msgID, id) + require.Equal(t, msgString, msg) + require.Equal(t, true, exist) + // test q.Stats() + msgID2 := "msg_2" + msgString2 := "test_msg2" + err = q.Push(msgID2, msgString2) + require.NoError(t, err) + todo, doing := q.Stats() + require.Equal(t, 1, todo) + require.Equal(t, 1, doing) + + // test Remove + err = q.Remove(msgID) + require.NoError(t, err) + + // test no such msg id + noSuchMsgID := "NoSuchId" + _, err = q.Get(noSuchMsgID) + require.EqualError(t, err, ErrNoSuchMessageID.Error()) + + err = q.Requeue(noSuchMsgID, 0) + require.EqualError(t, err, ErrNoSuchMessageID.Error()) + + err = q.Remove(noSuchMsgID) + require.EqualError(t, err, ErrNoSuchMessageID.Error()) +} + +type mockWorkerTask struct { + src []proto.VunitLocation + dst proto.VunitLocation +} + +func vunits(vuids []proto.Vuid) []proto.VunitLocation { + ret := []proto.VunitLocation{} + for _, vuid := range vuids { + ret = append(ret, proto.VunitLocation{Vuid: vuid, Host: "127.0.0.1:xx"}) + } + return ret +} + +func vunit(vuid proto.Vuid) proto.VunitLocation { + return proto.VunitLocation{Vuid: vuid, Host: "127.0.0.1:xx"} +} + +func (t *mockWorkerTask) GetSrc() []proto.VunitLocation { + return t.src +} + +func (t *mockWorkerTask) GetDest() proto.VunitLocation { + return t.dst +} + +func (t *mockWorkerTask) SetDest(dstVuid proto.VunitLocation) { + t.dst = dstVuid +} + +func TestTaskQueue(t *testing.T) { + // test Push + taskID1 := "task_id1" + task1 := mockWorkerTask{src: vunits([]proto.Vuid{1, 2, 3}), dst: vunit(4)} + + q := NewTaskQueue(100 * time.Millisecond) + q.PushTask(taskID1, &task1) + + _, ok := q.Query(taskID1) + require.Equal(t, true, ok) + + // test PopTask + id, wt, exist := q.PopTask() + require.Equal(t, true, exist) + require.Equal(t, id, taskID1) + require.Equal(t, task1.GetSrc(), wt.GetSrc()) + require.Equal(t, task1.GetDest(), wt.GetDest()) + _, _, exist = q.PopTask() + require.Equal(t, false, exist) + + // test RetryTask + q.RetryTask(taskID1) + time.Sleep(100 * time.Millisecond) + id, wt, exist = q.PopTask() + require.Equal(t, true, exist) + require.Equal(t, id, taskID1) + require.Equal(t, vunits([]proto.Vuid{1, 2, 3}), wt.GetSrc()) + require.Equal(t, vunit(4), wt.GetDest()) + + // test Stats + taskID2 := "task_id2" + task2 := mockWorkerTask{src: vunits([]proto.Vuid{3, 4, 5}), dst: vunit(6)} + q.PushTask(taskID2, &task2) + todo, doing := q.StatsTasks() + require.Equal(t, 1, todo) + require.Equal(t, 1, doing) + + // test Remove + err := q.RemoveTask(taskID1) + require.NoError(t, err) + err = q.RemoveTask(taskID2) + require.NoError(t, err) + todo, doing = q.StatsTasks() + require.Equal(t, 0, todo) + require.Equal(t, 0, doing) + + // test no such msg id + noSuchTaskID := "NoSuchId" + err = q.RemoveTask(noSuchTaskID) + require.EqualError(t, err, ErrNoSuchMessageID.Error()) +} + +func newTestWorkerTaskQueue(cancelPunishDuration, renewDuration time.Duration) *WorkerTaskQueue { + return &WorkerTaskQueue{ + idcQueues: make(map[string]*Queue), + cancelPunishDuration: cancelPunishDuration, + leaseExpiredS: renewDuration, + } +} + +func TestWorkerTaskQueue(t *testing.T) { + taskID1 := "task_id1" + idc := "z0" + task1 := mockWorkerTask{src: vunits([]proto.Vuid{1, 2, 3}), dst: vunit(4)} + + cancelPunishDuration := 100 * time.Millisecond + renewDuration := 200 * time.Millisecond + + // test AddPreparedTask + wq := newTestWorkerTaskQueue(cancelPunishDuration, renewDuration) + wq.AddPreparedTask(idc, taskID1, &task1) + + // test acquire + id, wt, exist := wq.Acquire(idc) + require.Equal(t, true, exist) + require.Equal(t, id, taskID1) + require.Equal(t, wt.GetSrc(), task1.GetSrc()) + require.Equal(t, wt.GetDest(), task1.GetDest()) + + _, _, exist = wq.Acquire(idc) + require.Equal(t, false, exist) + time.Sleep(renewDuration) + id, wt, exist = wq.Acquire(idc) + require.Equal(t, true, exist) + require.Equal(t, id, taskID1) + require.Equal(t, wt.GetSrc(), task1.GetSrc()) + require.Equal(t, wt.GetDest(), task1.GetDest()) + + // test Cancel + err := wq.Cancel(idc, taskID1, task1.GetSrc(), task1.GetDest()) + require.NoError(t, err) + _, _, exist = wq.Acquire(idc) + require.Equal(t, false, exist) + time.Sleep(cancelPunishDuration) + id, wt, exist = wq.Acquire(idc) + require.Equal(t, true, exist) + require.Equal(t, id, taskID1) + require.Equal(t, wt.GetSrc(), task1.GetSrc()) + require.Equal(t, wt.GetDest(), task1.GetDest()) + + // test Reclaim + err = wq.Reclaim(idc, taskID1, task1.GetSrc(), task1.GetDest(), vunit(6), 0) + require.NoError(t, err) + id, wt, exist = wq.Acquire(idc) + require.Equal(t, true, exist) + require.Equal(t, id, taskID1) + require.Equal(t, wt.GetSrc(), vunits([]proto.Vuid{1, 2, 3})) + require.Equal(t, wt.GetDest(), vunit(6)) + + // test Renewal + err = wq.Renewal(idc, taskID1) + require.NoError(t, err) + _, _, exist = wq.Acquire(idc) + require.Equal(t, false, exist) + time.Sleep(renewDuration) + id, wt, exist = wq.Acquire(idc) + require.Equal(t, true, exist) + require.Equal(t, id, taskID1) + require.Equal(t, wt.GetSrc(), vunits([]proto.Vuid{1, 2, 3})) + require.Equal(t, wt.GetDest(), vunit(6)) + // test Complete + _, err = wq.Complete(idc, taskID1, vunits([]proto.Vuid{1, 2, 3}), vunit(6)) + require.NoError(t, err) + todo, doing := wq.StatsTasks() + require.Equal(t, 0, todo) + require.Equal(t, 0, doing) + + // test ErrUnmatchedVuids + taskID2 := "task_id2" + task2 := mockWorkerTask{src: vunits([]proto.Vuid{1, 2, 3}), dst: vunit(4)} + wq = NewWorkerTaskQueue(cancelPunishDuration) + wq.AddPreparedTask(idc, taskID2, &task2) + + err = wq.Cancel(idc, taskID2, vunits([]proto.Vuid{4, 5, 6}), vunit(4)) + require.EqualError(t, err, ErrUnmatchedVuids.Error()) + err = wq.Reclaim(idc, taskID2, vunits([]proto.Vuid{4, 5, 6}), vunit(4), vunit(5), 0) + require.EqualError(t, err, ErrUnmatchedVuids.Error()) + _, err = wq.Complete(idc, taskID2, vunits([]proto.Vuid{4, 5, 6}), vunit(4)) + require.EqualError(t, err, ErrUnmatchedVuids.Error()) +} diff --git a/blobstore/scheduler/base/statistics_metrics.go b/blobstore/scheduler/base/statistics_metrics.go new file mode 100644 index 000000000..93b76f3b8 --- /dev/null +++ b/blobstore/scheduler/base/statistics_metrics.go @@ -0,0 +1,398 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package base + +import ( + "errors" + "fmt" + "sort" + "strings" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/cubefs/cubefs/blobstore/common/counter" + "github.com/cubefs/cubefs/blobstore/common/proto" + api "github.com/cubefs/cubefs/blobstore/scheduler/client" +) + +const ( + defaultTaskCntReportIntervalS = 15 + namespace = "scheduler" +) + +// Buckets default buckets for stats +var Buckets = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000} + +// ClusterTopologyStatsMgr cluster topology stats manager +type ClusterTopologyStatsMgr struct { + freeChunkCntRangeProHis *prometheus.HistogramVec +} + +// NewClusterTopologyStatisticsMgr returns cluster topology stats manager +func NewClusterTopologyStatisticsMgr(clusterID proto.ClusterID, buckets []float64) *ClusterTopologyStatsMgr { + labels := map[string]string{ + "cluster_id": fmt.Sprintf("%d", clusterID), + } + namespace := "scheduler" + if len(buckets) == 0 { + buckets = Buckets + } + freeChunkCntRangeProHis := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: namespace, + Name: "free_chunk_cnt_range", + Help: "free chunk cnt range", + Buckets: buckets, + ConstLabels: labels, + }, + []string{"rack", "idc"}, + ) + if err := prometheus.Register(freeChunkCntRangeProHis); err != nil { + if are, ok := err.(prometheus.AlreadyRegisteredError); ok { + freeChunkCntRangeProHis = are.ExistingCollector.(*prometheus.HistogramVec) + } else { + panic(err) + } + } + + return &ClusterTopologyStatsMgr{ + freeChunkCntRangeProHis: freeChunkCntRangeProHis, + } +} + +// ReportFreeChunk report free chunk +func (statsMgr *ClusterTopologyStatsMgr) ReportFreeChunk(disk *api.DiskInfoSimple) { + statsMgr.freeChunkCntRangeProHis.WithLabelValues(disk.Rack, disk.Idc).Observe(float64(disk.FreeChunkCnt)) +} + +// TaskCntStats information of task running on worker +type TaskCntStats interface { + StatQueueTaskCnt() (preparing, workerDoing, finishing int) +} + +// TaskRunDetailInfo task run detail info +type TaskRunDetailInfo struct { + Statistics proto.TaskStatistics `json:"statistics"` + StartTime time.Time `json:"start_time"` + CompleteTime time.Time `json:"complete_time"` + Completed bool `json:"completed"` +} + +// TaskStatsMgr task stats manager +type TaskStatsMgr struct { + mu sync.Mutex + TaskRunInfos map[string]TaskRunDetailInfo + dataSizeByteCounter counter.Counter + shardCntCounter counter.Counter + + dataSizeProCounter prometheus.Counter + shardCntProCounter prometheus.Counter + + taskCntGauge *prometheus.GaugeVec + + reclaimCounter prometheus.Counter + cancelCounter prometheus.Counter + + taskCntStats TaskCntStats +} + +// NewTaskStatsMgrAndRun run task stats manager +func NewTaskStatsMgrAndRun(clusterID proto.ClusterID, taskType string, taskCntStats TaskCntStats) *TaskStatsMgr { + mgr := NewTaskStatsMgr(clusterID, taskType) + mgr.taskCntStats = taskCntStats + go mgr.ReportTaskCntLoop() + return mgr +} + +// NewTaskStatsMgr returns task stats manager +func NewTaskStatsMgr(clusterID proto.ClusterID, taskType string) *TaskStatsMgr { + labels := map[string]string{ + "cluster_id": fmt.Sprintf("%d", clusterID), + "task_type": taskType, + "kind": KindSuccess, + } + + dataSizeProCounter := prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "task", + Name: "data_size", + Help: "data size", + ConstLabels: labels, + }) + + shardCntProCounter := prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "task", + Name: "shard_cnt", + Help: "shard cnt", + ConstLabels: labels, + }) + + taskCntGauge := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: "", + Name: "task_cnt", + Help: "task cnt", + ConstLabels: labels, + }, []string{"task_status"}) + + reclaimCounter := prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "task", + Name: "reclaim", + Help: "task reclaim", + ConstLabels: labels, + }) + + cancelCounter := prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "task", + Name: "cancel", + Help: "task cancel", + ConstLabels: labels, + }) + + if err := prometheus.Register(dataSizeProCounter); err != nil { + if are, ok := err.(prometheus.AlreadyRegisteredError); ok { + dataSizeProCounter = are.ExistingCollector.(prometheus.Counter) + } else { + panic(err) + } + } + if err := prometheus.Register(shardCntProCounter); err != nil { + if are, ok := err.(prometheus.AlreadyRegisteredError); ok { + shardCntProCounter = are.ExistingCollector.(prometheus.Counter) + } else { + panic(err) + } + } + if err := prometheus.Register(taskCntGauge); err != nil { + if are, ok := err.(prometheus.AlreadyRegisteredError); ok { + taskCntGauge = are.ExistingCollector.(*prometheus.GaugeVec) + } else { + panic(err) + } + } + if err := prometheus.Register(reclaimCounter); err != nil { + if are, ok := err.(prometheus.AlreadyRegisteredError); ok { + reclaimCounter = are.ExistingCollector.(prometheus.Counter) + } else { + panic(err) + } + } + if err := prometheus.Register(cancelCounter); err != nil { + if are, ok := err.(prometheus.AlreadyRegisteredError); ok { + cancelCounter = are.ExistingCollector.(prometheus.Counter) + } else { + panic(err) + } + } + + mgr := &TaskStatsMgr{ + TaskRunInfos: make(map[string]TaskRunDetailInfo), + dataSizeProCounter: dataSizeProCounter, + shardCntProCounter: shardCntProCounter, + taskCntGauge: taskCntGauge, + reclaimCounter: reclaimCounter, + cancelCounter: cancelCounter, + } + + return mgr +} + +// ReportTaskCntLoop report task count +func (statsMgr *TaskStatsMgr) ReportTaskCntLoop() { + t := time.NewTicker(time.Duration(defaultTaskCntReportIntervalS) * time.Second) + for range t.C { + preparing, workerDoing, finishing := statsMgr.taskCntStats.StatQueueTaskCnt() + + statsMgr.mu.Lock() + statsMgr.taskCntGauge.WithLabelValues("preparing").Set(float64(preparing)) + statsMgr.taskCntGauge.WithLabelValues("worker_doing").Set(float64(workerDoing)) + statsMgr.taskCntGauge.WithLabelValues("finishing").Set(float64(finishing)) + statsMgr.mu.Unlock() + } +} + +// ReportWorkerTaskStats report worker task stats +func (statsMgr *TaskStatsMgr) ReportWorkerTaskStats( + taskID string, + s proto.TaskStatistics, + increaseDataSize, + increaseShardCnt int) { + statsMgr.mu.Lock() + defer statsMgr.mu.Unlock() + + var taskRunInfo TaskRunDetailInfo + if _, ok := statsMgr.TaskRunInfos[taskID]; ok { + taskRunInfo = statsMgr.TaskRunInfos[taskID] + } else { + taskRunInfo.StartTime = time.Now() + } + + taskRunInfo.Statistics = s + if taskRunInfo.Statistics.Completed() { + taskRunInfo.CompleteTime = time.Now() + taskRunInfo.Completed = true + } + + statsMgr.TaskRunInfos[taskID] = taskRunInfo + statsMgr.dataSizeByteCounter.AddN(increaseDataSize) + statsMgr.shardCntCounter.AddN(increaseShardCnt) + + statsMgr.dataSizeProCounter.Add(float64(increaseDataSize)) + statsMgr.shardCntProCounter.Add(float64(increaseShardCnt)) +} + +// ReclaimTask reclaim task +func (statsMgr *TaskStatsMgr) ReclaimTask() { + statsMgr.reclaimCounter.Inc() +} + +// CancelTask cancel task +func (statsMgr *TaskStatsMgr) CancelTask() { + statsMgr.cancelCounter.Inc() +} + +// QueryTaskDetail find task detail info +func (statsMgr *TaskStatsMgr) QueryTaskDetail(taskID string) (detail TaskRunDetailInfo, err error) { + statsMgr.mu.Lock() + defer statsMgr.mu.Unlock() + + if info, ok := statsMgr.TaskRunInfos[taskID]; ok { + return info, nil + } + + return TaskRunDetailInfo{}, errors.New("not found") +} + +// Counters returns task stats counters +func (statsMgr *TaskStatsMgr) Counters() (increaseDataSize, increaseShardCnt [counter.SLOT]int) { + increaseDataSize = statsMgr.dataSizeByteCounter.Show() + increaseShardCnt = statsMgr.shardCntCounter.Show() + return +} + +// statistics stats +const ( + KindFailed = "failed" + KindSuccess = "success" +) + +// NewCounter returns statistics counter +func NewCounter(clusterID proto.ClusterID, taskType string, kind string) prometheus.Counter { + labels := map[string]string{ + "cluster_id": fmt.Sprintf("%d", clusterID), + "task_type": taskType, + "kind": kind, + } + shardCntCounter := prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "task", + Name: "shard_cnt", + Help: "shard cnt", + ConstLabels: labels, + }) + if err := prometheus.Register(shardCntCounter); err != nil { + if are, ok := err.(prometheus.AlreadyRegisteredError); ok { + return are.ExistingCollector.(prometheus.Counter) + } + panic(err) + } + return shardCntCounter +} + +// ErrorStats error stats +type ErrorStats struct { + lock sync.Mutex + errMap map[string]uint64 + totalErrCnt uint64 +} + +// ErrorPercent error percent +type ErrorPercent struct { + err string + percent float64 + errCnt uint64 +} + +// NewErrorStats returns error stats +func NewErrorStats() *ErrorStats { + es := ErrorStats{ + errMap: make(map[string]uint64), + } + return &es +} + +// AddFail add fail statistics +func (es *ErrorStats) AddFail(err error) { + es.lock.Lock() + defer es.lock.Unlock() + es.totalErrCnt++ + + errStr := errStrFormat(err) + if _, ok := es.errMap[errStr]; !ok { + es.errMap[errStr] = 0 + } + es.errMap[errStr]++ +} + +// Stats returns stats +func (es *ErrorStats) Stats() (statsResult []ErrorPercent, totalErrCnt uint64) { + es.lock.Lock() + defer es.lock.Unlock() + + var totalCnt uint64 + for _, cnt := range es.errMap { + totalCnt += cnt + } + + for err, cnt := range es.errMap { + percent := ErrorPercent{ + err: err, + percent: float64(cnt) / float64(totalCnt), + errCnt: cnt, + } + statsResult = append(statsResult, percent) + } + + sort.Slice(statsResult, func(i, j int) bool { + return statsResult[i].percent > statsResult[j].percent + }) + + return statsResult, es.totalErrCnt +} + +// FormatPrint format print message +func FormatPrint(statsInfos []ErrorPercent) (res []string) { + for _, info := range statsInfos { + res = append(res, fmt.Sprintf("%s: %0.2f%%[%d]", info.err, info.percent*100, info.errCnt)) + } + return +} + +func errStrFormat(err error) string { + if err == nil || len(err.Error()) == 0 { + return "" + } + + strSlice := strings.Split(err.Error(), ":") + return strSlice[len(strSlice)-1] +} diff --git a/blobstore/scheduler/base/statistics_metrics_test.go b/blobstore/scheduler/base/statistics_metrics_test.go new file mode 100644 index 000000000..bc7f5462f --- /dev/null +++ b/blobstore/scheduler/base/statistics_metrics_test.go @@ -0,0 +1,108 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package base + +import ( + "encoding/json" + "errors" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/common/counter" + "github.com/cubefs/cubefs/blobstore/common/proto" + api "github.com/cubefs/cubefs/blobstore/scheduler/client" +) + +type mockStats struct{} + +func (m *mockStats) StatQueueTaskCnt() (preparing, workerDoing, finishing int) { + return 0, 0, 0 +} + +func TestTaskStatisticsMgr(t *testing.T) { + mgr := NewTaskStatsMgrAndRun(1, proto.RepairTaskType, &mockStats{}) + mgr.ReportWorkerTaskStats("repair_task_1", proto.TaskStatistics{}, 10, 10) + + _, err := mgr.QueryTaskDetail("repair_task_1") + require.NoError(t, err) + + increaseDataSize, increaseShardCnt := mgr.Counters() + var increaseDataSizeVec [counter.SLOT]int + increaseDataSizeVec[counter.SLOT-1] = 10 + + var increaseShardCntVec [counter.SLOT]int + increaseShardCntVec[counter.SLOT-1] = 10 + require.Equal(t, increaseDataSizeVec, increaseDataSize) + require.Equal(t, increaseShardCntVec, increaseShardCnt) + + mgr.ReclaimTask() + mgr.CancelTask() +} + +func TestNewClusterTopoStatisticsMgr(t *testing.T) { + mgr := NewClusterTopologyStatisticsMgr(1, []float64{}) + disk := &api.DiskInfoSimple{ + Idc: "z0", + Rack: "test_rack", + FreeChunkCnt: 100, + } + mgr.ReportFreeChunk(disk) +} + +func TestErrorStats(t *testing.T) { + err1 := errors.New("error 1") + err2 := errors.New("error 2") + err3 := errors.New("error 3") + + es := NewErrorStats() + for range [3]struct{}{} { + es.AddFail(err1) + } + for range [5]struct{}{} { + es.AddFail(err2) + } + for range [2]struct{}{} { + es.AddFail(err3) + } + + infos, _ := es.Stats() + res := FormatPrint(infos) + + t.Log(res) + + p, err := json.MarshalIndent(&res, "", "\t") + t.Logf("%v -> %s", err, p) + + es2 := NewErrorStats() + infos, _ = es2.Stats() + p, err = json.MarshalIndent(&infos, "", "\t") + t.Logf("%v -> %s", err, p) +} + +func TestErrStrFormat(t *testing.T) { + err1 := errors.New("Post http://127.0.0.1:xxx/xxx: EOF") + err2 := errors.New("fake error") + var err3 error + + require.Equal(t, " EOF", errStrFormat(err1)) + require.Equal(t, "fake error", errStrFormat(err2)) + require.Equal(t, "", errStrFormat(err3)) +} + +func TestNewCounter(t *testing.T) { + counter := NewCounter(0, "", "") + require.NotNil(t, counter) +} diff --git a/blobstore/scheduler/base/task_types.go b/blobstore/scheduler/base/task_types.go new file mode 100644 index 000000000..d63ed268a --- /dev/null +++ b/blobstore/scheduler/base/task_types.go @@ -0,0 +1,67 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package base + +import ( + "errors" + + "go.mongodb.org/mongo-driver/mongo" + + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/util/defaulter" +) + +const ( + defaultPrepareQueueRetryDelayS = 10 + defaultCancelPunishDurationS = 20 + defaultFinishQueueRetryDelayS = 10 + defaultCollectIntervalS = 5 + defaultCheckTaskIntervalS = 5 + + defaultWorkQueueSize = 20 +) + +const ( + // EmptyDiskID empty diskID + EmptyDiskID = proto.DiskID(0) +) + +// err use for task +var ( + ErrNoTaskInQueue = errors.New("no task in queue") + ErrVolNotOnlyOneTask = errors.New("vol not only one task running") + ErrUpdateVolumeCache = errors.New("update volume cache failed") + ErrNoDocuments = mongo.ErrNoDocuments +) + +// TaskCommonConfig task common config +type TaskCommonConfig struct { + PrepareQueueRetryDelayS int `json:"prepare_queue_retry_delay_s"` + FinishQueueRetryDelayS int `json:"finish_queue_retry_delay_s"` + CancelPunishDurationS int `json:"cancel_punish_duration_s"` + WorkQueueSize int `json:"work_queue_size"` + CollectTaskIntervalS int `json:"collect_task_interval_s"` + CheckTaskIntervalS int `json:"check_task_interval_s"` +} + +// CheckAndFix check and fix task common config +func (conf *TaskCommonConfig) CheckAndFix() { + defaulter.LessOrEqual(&conf.PrepareQueueRetryDelayS, defaultPrepareQueueRetryDelayS) + defaulter.LessOrEqual(&conf.FinishQueueRetryDelayS, defaultFinishQueueRetryDelayS) + defaulter.LessOrEqual(&conf.CancelPunishDurationS, defaultCancelPunishDurationS) + defaulter.LessOrEqual(&conf.WorkQueueSize, defaultWorkQueueSize) + defaulter.LessOrEqual(&conf.CollectTaskIntervalS, defaultCollectIntervalS) + defaulter.LessOrEqual(&conf.CheckTaskIntervalS, defaultCheckTaskIntervalS) +} diff --git a/blobstore/scheduler/base/task_types_test.go b/blobstore/scheduler/base/task_types_test.go new file mode 100644 index 000000000..59fe13ebb --- /dev/null +++ b/blobstore/scheduler/base/task_types_test.go @@ -0,0 +1,32 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package base + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCommonCheckAndFix(t *testing.T) { + cfg := TaskCommonConfig{} + cfg.CheckAndFix() + require.Equal(t, defaultPrepareQueueRetryDelayS, cfg.PrepareQueueRetryDelayS) + require.Equal(t, defaultFinishQueueRetryDelayS, cfg.FinishQueueRetryDelayS) + require.Equal(t, defaultCancelPunishDurationS, cfg.CancelPunishDurationS) + require.Equal(t, defaultWorkQueueSize, cfg.WorkQueueSize) + require.Equal(t, defaultCollectIntervalS, cfg.CollectTaskIntervalS) + require.Equal(t, defaultCheckTaskIntervalS, cfg.CheckTaskIntervalS) +} diff --git a/blobstore/scheduler/base/utils.go b/blobstore/scheduler/base/utils.go new file mode 100644 index 000000000..30a762aa5 --- /dev/null +++ b/blobstore/scheduler/base/utils.go @@ -0,0 +1,129 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package base + +import ( + "context" + "fmt" + "math" + "time" + + "go.mongodb.org/mongo-driver/bson/primitive" + + "github.com/cubefs/cubefs/blobstore/common/counter" + "github.com/cubefs/cubefs/blobstore/common/errors" + comproto "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/util/retry" +) + +// IAllocVunit define the interface of clustermgr used for volume alloc +type IAllocVunit interface { + AllocVolumeUnit(ctx context.Context, vuid comproto.Vuid) (ret *client.AllocVunitInfo, err error) +} + +// AllocVunitSafe alloc volume unit safe +func AllocVunitSafe( + ctx context.Context, + cli IAllocVunit, + vuid comproto.Vuid, + volReplicas []comproto.VunitLocation) (ret *client.AllocVunitInfo, err error) { + span := trace.SpanFromContextSafe(ctx) + + allocVunit, err := cli.AllocVolumeUnit(ctx, vuid) + if err != nil { + return nil, err + } + + // safety check + for _, repl := range volReplicas { + if repl.Vuid.Index() == allocVunit.Vuid.Index() { + // allow alloc on same disk with old chunk + continue + } + if repl.DiskID == allocVunit.DiskID { + span.Panic("alloc chunk and others chunks are on same disk") + } + } + + return allocVunit, nil +} + +// Subtraction c = a - b +func Subtraction(a, b []comproto.Vuid) (c []comproto.Vuid) { + m := make(map[comproto.Vuid]struct{}) + for _, vuid := range b { + m[vuid] = struct{}{} + } + + for _, vuid := range a { + if _, ok := m[vuid]; !ok { + c = append(c, vuid) + } + } + return c +} + +// GenTaskID return task id +func GenTaskID(prefix string, vid comproto.Vid) string { + return fmt.Sprintf("%s-%d-%v", prefix, vid, primitive.NewObjectID().Hex()) +} + +// DataMountFormat format data +func DataMountFormat(dataMountBytes [counter.SLOT]int) string { + var formatStr []string + for _, dataMount := range dataMountBytes { + formatStr = append(formatStr, bytesCntFormat(dataMount)) + } + return fmt.Sprint(formatStr) +} + +func bytesCntFormat(bytesCnt int) string { + units := []string{"B", "KB", "MB", "GB", "TB", "PB"} + idx := 0 + bytesCnt2 := bytesCnt + for { + bytesCnt2 = bytesCnt2 / 1024 + if bytesCnt2 == 0 { + break + } + idx++ + if idx == 5 { + break + } + } + + num := float64(bytesCnt) / math.Pow(float64(1024), float64(idx)) + return fmt.Sprintf("%.3f%s", num, units[idx]) +} + +// ShouldAllocAndRedo return true if should alloc and redo task +func ShouldAllocAndRedo(errCode int) bool { + if errCode == errors.CodeNewVuidNotMatch || + errCode == errors.CodeStatChunkFailed { + return true + } + return false +} + +func InsistOn(ctx context.Context, errMsg string, on func() error) { + attempt := 0 + retry.InsistContext(ctx, time.Second, on, func(err error) { + attempt++ + span := trace.SpanFromContextSafe(ctx) + span.Errorf("insist attempt-%d: %s %s", attempt, errMsg, err.Error()) + }) +} diff --git a/blobstore/scheduler/base/utils_mock_test.go b/blobstore/scheduler/base/utils_mock_test.go new file mode 100644 index 000000000..ca0226af9 --- /dev/null +++ b/blobstore/scheduler/base/utils_mock_test.go @@ -0,0 +1,52 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cubefs/cubefs/blobstore/scheduler/base (interfaces: IAllocVunit) + +// Package base is a generated GoMock package. +package base + +import ( + context "context" + reflect "reflect" + + proto "github.com/cubefs/cubefs/blobstore/common/proto" + client "github.com/cubefs/cubefs/blobstore/scheduler/client" + gomock "github.com/golang/mock/gomock" +) + +// MockAllocVunit is a mock of IAllocVunit interface. +type MockAllocVunit struct { + ctrl *gomock.Controller + recorder *MockAllocVunitMockRecorder +} + +// MockAllocVunitMockRecorder is the mock recorder for MockAllocVunit. +type MockAllocVunitMockRecorder struct { + mock *MockAllocVunit +} + +// NewMockAllocVunit creates a new mock instance. +func NewMockAllocVunit(ctrl *gomock.Controller) *MockAllocVunit { + mock := &MockAllocVunit{ctrl: ctrl} + mock.recorder = &MockAllocVunitMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAllocVunit) EXPECT() *MockAllocVunitMockRecorder { + return m.recorder +} + +// AllocVolumeUnit mocks base method. +func (m *MockAllocVunit) AllocVolumeUnit(arg0 context.Context, arg1 proto.Vuid) (*client.AllocVunitInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AllocVolumeUnit", arg0, arg1) + ret0, _ := ret[0].(*client.AllocVunitInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AllocVolumeUnit indicates an expected call of AllocVolumeUnit. +func (mr *MockAllocVunitMockRecorder) AllocVolumeUnit(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocVolumeUnit", reflect.TypeOf((*MockAllocVunit)(nil).AllocVolumeUnit), arg0, arg1) +} diff --git a/blobstore/scheduler/base/utils_test.go b/blobstore/scheduler/base/utils_test.go new file mode 100644 index 000000000..44338ee3c --- /dev/null +++ b/blobstore/scheduler/base/utils_test.go @@ -0,0 +1,97 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package base + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/common/counter" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/scheduler/client" +) + +func TestSubtraction(t *testing.T) { + fromCm := []proto.Vuid{1, 2, 3, 4, 5} + fromDb := []proto.Vuid{1, 2, 3} + remain := Subtraction(fromCm, fromDb) + require.Equal(t, []proto.Vuid{4, 5}, remain) +} + +func TestHumanRead(t *testing.T) { + bytesCntFormat(0) + bytesCntFormat((1 << 60) + 1) +} + +func TestDataMountBytePrintEx(t *testing.T) { + dataMountBytes := [counter.SLOT]int{ + (1 << 60) + 1, + 2445979449, 2363491431, 2122318836, + 4341106710, 3521119887, 3681901617, + 3697979790, 4244637672, 3424650849, + 3279947292, 2675967228, 3624579435, + 4855608246, 5161093533, 5064624495, + 4855608246, 4984233630, 512, 0, + } + DataMountFormat(dataMountBytes) +} + +func TestLoopExecUntilSuccess(t *testing.T) { + InsistOn(context.Background(), "test", func() error { + return nil + }) +} + +func TestShouldAllocAndRedo(t *testing.T) { + err := errcode.ErrNewVuidNotMatch + code := rpc.DetectStatusCode(err) + redo := ShouldAllocAndRedo(code) + require.Equal(t, true, redo) +} + +func TestAllocVunitSafe(t *testing.T) { + ctx := context.Background() + errMock := errors.New("fake error") + volumeAllocClient := NewMockAllocVunit(gomock.NewController(t)) + vuid1, _ := proto.NewVuid(1, 1, 1) + volumeAllocClient.EXPECT().AllocVolumeUnit(gomock.Any(), gomock.Any()).Return(nil, errMock) + _, err := AllocVunitSafe(ctx, volumeAllocClient, vuid1, nil) + require.True(t, errors.Is(err, errMock)) + + vuid2, _ := proto.NewVuid(1, 1, 2) + vuid3, _ := proto.NewVuid(1, 2, 1) + volumeAllocClient.EXPECT().AllocVolumeUnit(gomock.Any(), gomock.Any()).Return(&client.AllocVunitInfo{VunitLocation: proto.VunitLocation{Vuid: vuid2, DiskID: proto.DiskID(1)}}, nil) + allocVunit, err := AllocVunitSafe(ctx, volumeAllocClient, vuid1, []proto.VunitLocation{{Vuid: vuid2, DiskID: proto.DiskID(1)}, {Vuid: vuid3, DiskID: proto.DiskID(2)}}) + require.NoError(t, err) + require.Equal(t, vuid2, allocVunit.Vuid) + + volumeAllocClient.EXPECT().AllocVolumeUnit(gomock.Any(), gomock.Any()).Return(&client.AllocVunitInfo{VunitLocation: proto.VunitLocation{Vuid: vuid2, DiskID: proto.DiskID(2)}}, nil) + require.Panics(t, func() { + AllocVunitSafe(ctx, volumeAllocClient, vuid1, []proto.VunitLocation{{Vuid: vuid2, DiskID: proto.DiskID(1)}, {Vuid: vuid3, DiskID: proto.DiskID(2)}}) + }) +} + +func TestGenTaskID(t *testing.T) { + prefix := "test-" + id := GenTaskID(prefix, proto.Vid(1)) + require.True(t, strings.HasPrefix(id, prefix)) +} diff --git a/blobstore/scheduler/base/volume_task_locker.go b/blobstore/scheduler/base/volume_task_locker.go new file mode 100644 index 000000000..5937a015e --- /dev/null +++ b/blobstore/scheduler/base/volume_task_locker.go @@ -0,0 +1,77 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package base + +import ( + "context" + "errors" + "sync" + + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/trace" +) + +// make sure only one task in same volume to run in cluster +var ( + // ErrVidTaskConflict vid task conflict + ErrVidTaskConflict = errors.New("vid task conflict") +) + +// VolTaskLocker volume task locker +type VolTaskLocker struct { + taskMap map[proto.Vid]struct{} + mu sync.Mutex +} + +// TryLock try lock task volume and return error if there is task doing +func (m *VolTaskLocker) TryLock(ctx context.Context, vid proto.Vid) error { + m.mu.Lock() + defer m.mu.Unlock() + + span := trace.SpanFromContextSafe(ctx) + span.Infof("vid %d mutex try lock", vid) + + if _, ok := m.taskMap[vid]; ok { + return ErrVidTaskConflict + } + m.taskMap[vid] = struct{}{} + return nil +} + +// Unlock unlock task volume +func (m *VolTaskLocker) Unlock(ctx context.Context, vid proto.Vid) { + m.mu.Lock() + defer m.mu.Unlock() + + span := trace.SpanFromContextSafe(ctx) + span.Infof("vid %d mutex unlock", vid) + + delete(m.taskMap, vid) +} + +var volTaskLocker *VolTaskLocker + +// NewVolTaskLockerOnce singleton mode:make sure only one instance in global +var NewVolTaskLockerOnce sync.Once + +// VolTaskLockerInst ensure that only one background task is executing on the same volume +func VolTaskLockerInst() *VolTaskLocker { + NewVolTaskLockerOnce.Do(func() { + volTaskLocker = &VolTaskLocker{ + taskMap: make(map[proto.Vid]struct{}), + } + }) + return volTaskLocker +} diff --git a/blobstore/scheduler/base/volume_task_locker_test.go b/blobstore/scheduler/base/volume_task_locker_test.go new file mode 100644 index 000000000..7a9a59798 --- /dev/null +++ b/blobstore/scheduler/base/volume_task_locker_test.go @@ -0,0 +1,46 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package base + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +func MockEmptyVolTaskLocker() { + VolTaskLockerInst().mu.Lock() + defer VolTaskLockerInst().mu.Unlock() + VolTaskLockerInst().taskMap = make(map[proto.Vid]struct{}) +} + +func TestVolTaskLocker(t *testing.T) { + MockEmptyVolTaskLocker() + ctx := context.Background() + + mu := VolTaskLockerInst() + mu2 := VolTaskLockerInst() + require.Equal(t, mu, mu2) + err := mu.TryLock(ctx, 1) + require.NoError(t, err) + err = mu.TryLock(ctx, 1) + require.EqualError(t, err, ErrVidTaskConflict.Error()) + mu.Unlock(ctx, 1) + err = mu.TryLock(ctx, 1) + require.NoError(t, err) +} diff --git a/blobstore/scheduler/base_mock_test.go b/blobstore/scheduler/base_mock_test.go new file mode 100644 index 000000000..f4316742d --- /dev/null +++ b/blobstore/scheduler/base_mock_test.go @@ -0,0 +1,115 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cubefs/cubefs/blobstore/scheduler/base (interfaces: IConsumer,IProducer) + +// Package scheduler is a generated GoMock package. +package scheduler + +import ( + context "context" + reflect "reflect" + + sarama "github.com/Shopify/sarama" + gomock "github.com/golang/mock/gomock" +) + +// MockConsumer is a mock of IConsumer interface. +type MockConsumer struct { + ctrl *gomock.Controller + recorder *MockConsumerMockRecorder +} + +// MockConsumerMockRecorder is the mock recorder for MockConsumer. +type MockConsumerMockRecorder struct { + mock *MockConsumer +} + +// NewMockConsumer creates a new mock instance. +func NewMockConsumer(ctrl *gomock.Controller) *MockConsumer { + mock := &MockConsumer{ctrl: ctrl} + mock.recorder = &MockConsumerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockConsumer) EXPECT() *MockConsumerMockRecorder { + return m.recorder +} + +// CommitOffset mocks base method. +func (m *MockConsumer) CommitOffset(arg0 context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CommitOffset", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// CommitOffset indicates an expected call of CommitOffset. +func (mr *MockConsumerMockRecorder) CommitOffset(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CommitOffset", reflect.TypeOf((*MockConsumer)(nil).CommitOffset), arg0) +} + +// ConsumeMessages mocks base method. +func (m *MockConsumer) ConsumeMessages(arg0 context.Context, arg1 int) []*sarama.ConsumerMessage { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ConsumeMessages", arg0, arg1) + ret0, _ := ret[0].([]*sarama.ConsumerMessage) + return ret0 +} + +// ConsumeMessages indicates an expected call of ConsumeMessages. +func (mr *MockConsumerMockRecorder) ConsumeMessages(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConsumeMessages", reflect.TypeOf((*MockConsumer)(nil).ConsumeMessages), arg0, arg1) +} + +// MockProducer is a mock of IProducer interface. +type MockProducer struct { + ctrl *gomock.Controller + recorder *MockProducerMockRecorder +} + +// MockProducerMockRecorder is the mock recorder for MockProducer. +type MockProducerMockRecorder struct { + mock *MockProducer +} + +// NewMockProducer creates a new mock instance. +func NewMockProducer(ctrl *gomock.Controller) *MockProducer { + mock := &MockProducer{ctrl: ctrl} + mock.recorder = &MockProducerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockProducer) EXPECT() *MockProducerMockRecorder { + return m.recorder +} + +// SendMessage mocks base method. +func (m *MockProducer) SendMessage(arg0 []byte) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendMessage", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendMessage indicates an expected call of SendMessage. +func (mr *MockProducerMockRecorder) SendMessage(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessage", reflect.TypeOf((*MockProducer)(nil).SendMessage), arg0) +} + +// SendMessages mocks base method. +func (m *MockProducer) SendMessages(arg0 [][]byte) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendMessages", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendMessages indicates an expected call of SendMessages. +func (mr *MockProducerMockRecorder) SendMessages(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessages", reflect.TypeOf((*MockProducer)(nil).SendMessages), arg0) +} diff --git a/blobstore/scheduler/blob_deleter.go b/blobstore/scheduler/blob_deleter.go new file mode 100644 index 000000000..1091ae001 --- /dev/null +++ b/blobstore/scheduler/blob_deleter.go @@ -0,0 +1,716 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "encoding/json" + "errors" + "sync" + "time" + + "github.com/Shopify/sarama" + "github.com/prometheus/client_golang/prometheus" + + "github.com/cubefs/cubefs/blobstore/common/counter" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/kafka" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/recordlog" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/taskswitch" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/scheduler/base" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/scheduler/db" + "github.com/cubefs/cubefs/blobstore/util/taskpool" +) + +// ITaskRunner define the interface of task running status. +type ITaskRunner interface { + Enabled() bool + RunTask() + GetTaskStats() (success, failed [counter.SLOT]int) + GetErrorStats() (errStats []string, totalErrCnt uint64) +} + +type deleteStatus int + +// blob delete status +const ( + DelDone = deleteStatus(iota) + DelDelay + DelFailed + DelUnexpect +) + +// ErrVunitLengthNotEqual vunit length not equal +var ErrVunitLengthNotEqual = errors.New("vunit length not equal") + +type deleteStageMgr struct { + l sync.Mutex + delStages map[proto.BlobID]*proto.BlobDeleteStage +} + +func (dsm *deleteStageMgr) clear() { + dsm.l.Lock() + defer dsm.l.Unlock() + dsm.delStages = make(map[proto.BlobID]*proto.BlobDeleteStage) +} + +func (dsm *deleteStageMgr) setBlobDelStage(bid proto.BlobID, stage proto.BlobDeleteStage) { + dsm.l.Lock() + defer dsm.l.Unlock() + stageCopy := stage.Copy() + dsm.delStages[bid] = &stageCopy +} + +func (dsm *deleteStageMgr) getBlobDelStage(bid proto.BlobID) proto.BlobDeleteStage { + dsm.l.Lock() + defer dsm.l.Unlock() + if stage, exist := dsm.delStages[bid]; exist { + return stage.Copy() + } + return proto.BlobDeleteStage{} +} + +func (dsm *deleteStageMgr) setShardDelStage(bid proto.BlobID, vuid proto.Vuid, stage proto.DeleteStage) { + dsm.l.Lock() + defer dsm.l.Unlock() + if dsm.delStages == nil { + dsm.delStages = make(map[proto.BlobID]*proto.BlobDeleteStage) + } + + if _, exist := dsm.delStages[bid]; !exist { + dsm.delStages[bid] = &proto.BlobDeleteStage{} + } + dbs := dsm.delStages[bid] + dbs.SetStage(vuid.Index(), stage) +} + +func (dsm *deleteStageMgr) hasMarkDel(bid proto.BlobID, vuid proto.Vuid) bool { + dsm.l.Lock() + defer dsm.l.Unlock() + return dsm.stageEqual(bid, vuid, proto.MarkDelStage) +} + +func (dsm *deleteStageMgr) stageEqual(bid proto.BlobID, vuid proto.Vuid, target proto.DeleteStage) bool { + if ds, exist := dsm.delStages[bid]; exist { + s, ok := ds.Stage(vuid) + if ok && s == target { + return true + } + } + return false +} + +type delShardRet struct { + err error + vuid proto.Vuid +} + +type delBlobRet struct { + status deleteStatus + err error + delMsg *proto.DeleteMsg +} + +// DelDoc is a delete doc information for logging in dellog +type DelDoc struct { + ClusterID proto.ClusterID `json:"cid"` + Bid proto.BlobID `json:"bid"` + Vid proto.Vid `json:"vid"` + Retry int `json:"retry"` + Time int64 `json:"t"` + ReqID string `json:"rid"` + ActualDelTime int64 `json:"del_at"` // unix time in S +} + +func toDelDoc(msg proto.DeleteMsg) DelDoc { + return DelDoc{ + ClusterID: msg.ClusterID, + Bid: msg.Bid, + Vid: msg.Vid, + Retry: msg.Retry, + Time: msg.Time, + ReqID: msg.ReqId, + ActualDelTime: time.Now().Unix(), + } +} + +// BlobDeleteConfig is blob delete config +type BlobDeleteConfig struct { + ClusterID proto.ClusterID + + TaskPoolSize int `json:"task_pool_size"` + + FailMsgConsumeIntervalMs int64 `json:"fail_msg_consume_interval_ms"` + + NormalHandleBatchCnt int `json:"normal_handle_batch_cnt"` + FailHandleBatchCnt int `json:"fail_handle_batch_cnt"` + + SafeDelayTimeH int64 `json:"safe_delay_time_h"` + DeleteLog recordlog.Config `json:"delete_log"` + + Kafka BlobDeleteKafkaConfig `json:"-"` +} + +func (cfg *BlobDeleteConfig) normalConsumerConfig() *base.KafkaConfig { + return &base.KafkaConfig{ + Topic: cfg.Kafka.Normal.Topic, + Partitions: cfg.Kafka.Normal.Partitions, + BrokerList: cfg.Kafka.BrokerList, + } +} + +func (cfg *BlobDeleteConfig) failedConsumerConfig() *base.KafkaConfig { + return &base.KafkaConfig{ + Topic: cfg.Kafka.Failed.Topic, + Partitions: cfg.Kafka.Failed.Partitions, + BrokerList: cfg.Kafka.BrokerList, + } +} + +func (cfg *BlobDeleteConfig) failedProducerConfig() *kafka.ProducerCfg { + return &kafka.ProducerCfg{ + BrokerList: cfg.Kafka.BrokerList, + Topic: cfg.Kafka.Failed.Topic, + TimeoutMs: cfg.Kafka.FailMsgSenderTimeoutMs, + } +} + +// BlobDeleteMgr is blob delete manager +type BlobDeleteMgr struct { + taskSwitch *taskswitch.TaskSwitch + + normalConsumer *deleteTopicConsumer + failConsumer *deleteTopicConsumer + + delSuccessCounter prometheus.Counter + delSuccessCounterByMin *counter.Counter + delFailCounter prometheus.Counter + delFailCounterByMin *counter.Counter + errStatsDistribution *base.ErrorStats +} + +// NewBlobDeleteMgr returns blob delete manager +func NewBlobDeleteMgr( + cfg *BlobDeleteConfig, + volCache IVolumeCache, + offAccessor db.IKafkaOffsetTable, + blobnodeCli client.BlobnodeAPI, + switchMgr *taskswitch.SwitchMgr, +) (*BlobDeleteMgr, error) { + normalTopicConsumers, err := base.NewKafkaPartitionConsumers(cfg.normalConsumerConfig(), offAccessor) + if err != nil { + return nil, err + } + + failTopicConsumers, err := base.NewKafkaPartitionConsumers(cfg.failedConsumerConfig(), offAccessor) + if err != nil { + return nil, err + } + + failMsgSender, err := base.NewMsgSender(cfg.failedProducerConfig()) + if err != nil { + return nil, err + } + + taskSwitch, err := switchMgr.AddSwitch(taskswitch.BlobDeleteSwitchName) + if err != nil { + return nil, err + } + + delLogger, err := recordlog.NewEncoder(&cfg.DeleteLog) + if err != nil { + return nil, err + } + + tp := taskpool.New(cfg.TaskPoolSize, cfg.TaskPoolSize) + + mgr := &BlobDeleteMgr{ + taskSwitch: taskSwitch, + delSuccessCounter: base.NewCounter(cfg.ClusterID, "delete", base.KindSuccess), + delFailCounter: base.NewCounter(cfg.ClusterID, "delete", base.KindFailed), + errStatsDistribution: base.NewErrorStats(), + delSuccessCounterByMin: &counter.Counter{}, + delFailCounterByMin: &counter.Counter{}, + } + + normalTopicConsumer := &deleteTopicConsumer{ + taskSwitch: taskSwitch, + + taskPool: &tp, + topicConsumers: normalTopicConsumers, + consumeBatchCnt: cfg.NormalHandleBatchCnt, + consumeIntervalMs: time.Duration(0), + safeDelayTime: time.Duration(cfg.SafeDelayTimeH) * time.Hour, + volCache: volCache, + blobnodeCli: blobnodeCli, + failMsgSender: failMsgSender, + + delSuccessCounter: mgr.delSuccessCounter, + delSuccessCounterByMin: mgr.delSuccessCounterByMin, + delFailCounter: mgr.delFailCounter, + delFailCounterByMin: mgr.delFailCounterByMin, + errStatsDistribution: mgr.errStatsDistribution, + + delLogger: delLogger, + } + + failTopicConsumer := &deleteTopicConsumer{ + taskSwitch: taskSwitch, + + taskPool: &tp, + topicConsumers: failTopicConsumers, + consumeBatchCnt: cfg.FailHandleBatchCnt, + consumeIntervalMs: time.Duration(cfg.FailMsgConsumeIntervalMs) * time.Millisecond, + safeDelayTime: time.Duration(cfg.SafeDelayTimeH) * time.Hour, + volCache: volCache, + blobnodeCli: blobnodeCli, + failMsgSender: failMsgSender, + + delSuccessCounter: mgr.delSuccessCounter, + delSuccessCounterByMin: mgr.delSuccessCounterByMin, + delFailCounter: mgr.delFailCounter, + delFailCounterByMin: mgr.delFailCounterByMin, + errStatsDistribution: mgr.errStatsDistribution, + + delLogger: delLogger, + } + + mgr.normalConsumer = normalTopicConsumer + mgr.failConsumer = failTopicConsumer + + return mgr, nil +} + +// RunTask consumers delete messages +func (mgr *BlobDeleteMgr) RunTask() { + mgr.normalConsumer.run() + mgr.failConsumer.run() +} + +// Enabled returns return if delete task switch is enable, otherwise returns false +func (mgr *BlobDeleteMgr) Enabled() bool { + return mgr.taskSwitch.Enabled() +} + +// GetTaskStats returns task stats +func (mgr *BlobDeleteMgr) GetTaskStats() (success [counter.SLOT]int, failed [counter.SLOT]int) { + return mgr.delSuccessCounterByMin.Show(), mgr.delFailCounterByMin.Show() +} + +// GetErrorStats returns error stats +func (mgr *BlobDeleteMgr) GetErrorStats() (errStats []string, totalErrCnt uint64) { + statsResult, totalErrCnt := mgr.errStatsDistribution.Stats() + return base.FormatPrint(statsResult), totalErrCnt +} + +type deleteTopicConsumer struct { + taskSwitch *taskswitch.TaskSwitch + + taskPool *taskpool.TaskPool + topicConsumers []base.IConsumer + consumeBatchCnt int + consumeIntervalMs time.Duration + safeDelayTime time.Duration + + volCache IVolumeCache + blobnodeCli client.BlobnodeAPI + + failMsgSender base.IProducer + dsm deleteStageMgr + + // stats + delSuccessCounter prometheus.Counter + delSuccessCounterByMin *counter.Counter + delFailCounter prometheus.Counter + delFailCounterByMin *counter.Counter + errStatsDistribution *base.ErrorStats + + // delete log + delLogger recordlog.Encoder +} + +func (d *deleteTopicConsumer) run() { + for _, consumer := range d.topicConsumers { + go func(consumer base.IConsumer) { + for { + d.taskSwitch.WaitEnable() + d.consumeAndDelete(consumer, d.consumeBatchCnt) + if d.consumeIntervalMs != time.Duration(0) { + time.Sleep(d.consumeIntervalMs) + } + } + }(consumer) + } +} + +func (d *deleteTopicConsumer) consumeAndDelete(consumer base.IConsumer, batchCnt int) { + span, ctx := trace.StartSpanFromContext(context.Background(), "consumeAndDelete") + defer span.Finish() + + if batchCnt <= 0 { + batchCnt = 1 + } + + msgs := consumer.ConsumeMessages(ctx, batchCnt) + d.handleMsgBatch(ctx, msgs) + + base.InsistOn(ctx, "deleter consumer.CommitOffset", func() error { + return consumer.CommitOffset(ctx) + }) +} + +func (d *deleteTopicConsumer) handleMsgBatch(ctx context.Context, mqMsgs []*sarama.ConsumerMessage) { + span := trace.SpanFromContextSafe(ctx) + ctx = trace.ContextWithSpan(ctx, span) + + span.Infof("handle delete msg: len[%d]", len(mqMsgs)) + + var msgs []*proto.DeleteMsg + if len(mqMsgs) != 0 { + ms := unmarshalMsgs(mqMsgs) + msgs = DeduplicateMsgs(ctx, ms) + span.Infof("deduplicate messages: len[%d]", len(msgs)) + } + + if len(msgs) != 0 { + // clear delete stage before handle batch msgs + span.Debugf("dsm clear before delete") + d.dsm.clear() + for _, m := range msgs { + span.Debugf("set blob delete stage: %+v", m.BlobDelStages) + d.dsm.setBlobDelStage(m.Bid, m.BlobDelStages) + } + } + + for len(msgs) != 0 { + finishCh := make(chan delBlobRet, len(msgs)) + for _, m := range msgs { + func(delMsg *proto.DeleteMsg) { + d.taskPool.Run(func() { + d.handleOneMsg(ctx, delMsg, finishCh) + }) + }(m) + } + + var delayMsgs []*proto.DeleteMsg + var maxDelayMsgTimeStamp int64 = 0 + for i := 0; i < len(msgs); i++ { + ret := <-finishCh + switch ret.status { + case DelDone: + span.Debugf("delete success: vid[%d], bid[%d], reqid[%s]", ret.delMsg.Vid, ret.delMsg.Bid, ret.delMsg.ReqId) + d.delSuccessCounterByMin.Add() + d.delSuccessCounter.Inc() + + case DelFailed: + span.Warnf("delete failed and send msg to fail queue: vid[%d], bid[%d], reqid[%s], retry[%d], err[%+v]", + ret.delMsg.Vid, ret.delMsg.Bid, ret.delMsg.ReqId, ret.delMsg.Retry, ret.err) + d.delFailCounter.Inc() + d.delFailCounterByMin.Add() + d.errStatsDistribution.AddFail(ret.err) + + base.InsistOn(ctx, "deleter send2FailQueue", func() error { + return d.send2FailQueue(ctx, *ret.delMsg) + }) + + case DelDelay: + if ret.delMsg.Time > maxDelayMsgTimeStamp { + maxDelayMsgTimeStamp = ret.delMsg.Time + } + delayMsgs = append(delayMsgs, ret.delMsg) + + case DelUnexpect: + span.Warnf("unexpected result will ignore: msg[%+v], err[%+v]", ret.delMsg, ret.err) + } + } + + if len(delayMsgs) == 0 { + return + } + + sleepDuration := d.delayDuration(maxDelayMsgTimeStamp) + span.Warnf("blob is protected: util[%+v], sleep[%+v]", time.Unix(maxDelayMsgTimeStamp, 0).Add(d.safeDelayTime), sleepDuration) + time.Sleep(sleepDuration) + msgs = delayMsgs + } +} + +func (d *deleteTopicConsumer) delayDuration(delTimeStamp int64) time.Duration { + start := time.Unix(delTimeStamp, 0) + now := time.Now() + return start.Add(d.safeDelayTime).Sub(now) +} + +func (d *deleteTopicConsumer) handleOneMsg(ctx context.Context, delMsg *proto.DeleteMsg, finishCh chan<- delBlobRet) { + if !delMsg.IsValid() { + finishCh <- delBlobRet{ + status: DelUnexpect, + err: proto.ErrInvalidMsg, + } + return + } + + now := time.Now().UTC() + if now.Sub(time.Unix(delMsg.Time, 0)) < d.safeDelayTime { + finishCh <- delBlobRet{ + status: DelDelay, + delMsg: delMsg, + } + return + } + pSpan := trace.SpanFromContextSafe(ctx) + pSpan.Infof("start delete msg: [%+v]", delMsg) + + span, tmpCtx := trace.StartSpanFromContextWithTraceID(context.Background(), "handleDeleteMsg", delMsg.ReqId) + err := d.deleteWithCheckVolConsistency(tmpCtx, delMsg.Vid, delMsg.Bid) + if err != nil { + finishCh <- delBlobRet{ + status: DelFailed, + err: err, + delMsg: delMsg, + } + return + } + + delDoc := toDelDoc(*delMsg) + err = d.delLogger.Encode(delDoc) + if err != nil { + span.Warnf("write delete log failed: vid[%d], bid[%d], err[%+v]", delDoc.Vid, delDoc.Bid, err) + } + + finishCh <- delBlobRet{ + status: DelDone, + delMsg: delMsg, + } +} + +func (d *deleteTopicConsumer) deleteWithCheckVolConsistency(ctx context.Context, vid proto.Vid, bid proto.BlobID) error { + return DoubleCheckedRun(ctx, d.volCache, vid, func(info *client.VolumeInfoSimple) error { + return d.deleteBlob(ctx, info, bid) + }) +} + +func (d *deleteTopicConsumer) deleteBlob(ctx context.Context, volInfo *client.VolumeInfoSimple, bid proto.BlobID) (err error) { + newVol, err := d.markDelBlob(ctx, volInfo, bid) + if err != nil { + return + } + + _, err = d.delBlob(ctx, newVol, bid) + return err +} + +func (d *deleteTopicConsumer) markDelBlob(ctx context.Context, volInfo *client.VolumeInfoSimple, bid proto.BlobID) (*client.VolumeInfoSimple, error) { + return d.deleteShards(ctx, volInfo, bid, true) +} + +func (d *deleteTopicConsumer) delBlob(ctx context.Context, volInfo *client.VolumeInfoSimple, bid proto.BlobID) (*client.VolumeInfoSimple, error) { + return d.deleteShards(ctx, volInfo, bid, false) +} + +func (d *deleteTopicConsumer) deleteShards( + ctx context.Context, + volInfo *client.VolumeInfoSimple, + bid proto.BlobID, + markDelete bool) (new *client.VolumeInfoSimple, err error) { + span := trace.SpanFromContextSafe(ctx) + + var updateAndRetryShards []proto.Vuid + locations := volInfo.VunitLocations + vid := volInfo.Vid + retCh := make(chan delShardRet, len(locations)) + + span.Debugf("delete blob: vid[%d], bid[%d], markDelete[%+v]", vid, bid, markDelete) + for _, location := range locations { + span.Debugf("delete shards: location[%+v]", location) + go func(ctx context.Context, location proto.VunitLocation, bid proto.BlobID, markDelete bool) { + err := d.deleteShard(ctx, location, bid, markDelete) + retCh <- delShardRet{err: err, vuid: location.Vuid} + }(ctx, location, bid, markDelete) + } + + for i := 0; i < len(locations); i++ { + ret := <-retCh + if ret.err != nil { + errCode := rpc.DetectStatusCode(ret.err) + if shouldUpdateVolumeErr(errCode) { + span.Errorf("delete shard failed will retry: bid[%d], vuid[%d], markDelete[%+v], code[%d], err[%+v]", + bid, ret.vuid, markDelete, errCode, ret.err) + updateAndRetryShards = append(updateAndRetryShards, ret.vuid) + err = ret.err + continue + } + + span.Errorf("delete shard failed: bid[%d], vuid[%d], markDelete[%+v], code[%d], err[%+v]", + bid, ret.vuid, markDelete, errCode, ret.err) + return volInfo, ret.err + } + } + + if len(updateAndRetryShards) == 0 { + span.Debugf("delete blob success: vid[%d], bid[%d], markDelete[%+v] ", vid, bid, markDelete) + return volInfo, nil + } + + span.Infof("bid delete will update and retry: len updateAndRetryShards[%d]", len(updateAndRetryShards)) + // update volCache + newVolInfo, updateVolErr := d.volCache.Update(vid) + if updateVolErr != nil || newVolInfo.EqualWith(volInfo) { + // if update volInfo failed or volInfo not updated, don't need retry + span.Warnf("new volInfo is same or volCache.Update failed: vid[%d], err[%+v]", volInfo.Vid, updateVolErr) + return volInfo, err + } + + if len(newVolInfo.VunitLocations) != len(locations) { + span.Warnf("vid locations len not equal: vid[%d], old len[%d], new len[%d]", len(locations), len(newVolInfo.VunitLocations)) + return volInfo, ErrVunitLengthNotEqual + } + + for _, oldVuid := range updateAndRetryShards { + idx := oldVuid.Index() + newLocation := newVolInfo.VunitLocations[idx] + span.Debugf("start retry delete shard: bid[%d]", bid) + err := d.deleteShard(ctx, newLocation, bid, markDelete) + if err != nil { + span.Errorf("retry delete shard: bid[%d], new location[%+v], markDelete[%+v], err[%+v]", + bid, newLocation, markDelete, err) + return newVolInfo, err + } + } + + span.Debugf("delete blob success: vid[%d], bid[%d], markDelete[%+v]", vid, bid, markDelete) + return newVolInfo, nil +} + +func (d *deleteTopicConsumer) deleteShard(ctx context.Context, location proto.VunitLocation, bid proto.BlobID, markDelete bool) (err error) { + span := trace.SpanFromContextSafe(ctx) + + // in order to prevent missing delete task, + // regardless of whether it is deleted or not, it will be deleted, + // just skip mark delete when has mark deleted + if d.hasMarkDeleted(location.Vuid, bid, markDelete) { + span.Infof("bid has mark deleted and skip: bid[%d], location[%+v]", bid, location) + return nil + } + + var stage proto.DeleteStage + if markDelete { + stage = proto.MarkDelStage + err = d.blobnodeCli.MarkDelete(ctx, location, bid) + } else { + stage = proto.DelStage + err = d.blobnodeCli.Delete(ctx, location, bid) + } + + defer func() { + if err == nil { + span.Debugf("delete shard set stage: location[%+v], stage[%d]", location, stage) + d.dsm.setShardDelStage(bid, location.Vuid, stage) + return + } + + if shouldBackToInitStage(err) { + d.dsm.setShardDelStage(bid, location.Vuid, proto.InitStage) + } + }() + + if err != nil { + errCode := rpc.DetectStatusCode(err) + if assumeDeleteSuccess(errCode) { + span.Debugf("delete bid failed but assume success: bid[%d], location[%+v], err[%+v] ", + bid, location, err) + return nil + } + } + + return +} + +func (d *deleteTopicConsumer) hasMarkDeleted(vuid proto.Vuid, bid proto.BlobID, markDelete bool) bool { + return markDelete && d.dsm.hasMarkDel(bid, vuid) +} + +func shouldBackToInitStage(err error) bool { + //Simple handling, for all deletion errors, delete tasks are all redone from InitStage + //todo:analyze the error codes carefully to determine which ones need to be back to init stage + return err != nil +} + +func (d *deleteTopicConsumer) send2FailQueue(ctx context.Context, msg proto.DeleteMsg) error { + span := trace.SpanFromContextSafe(ctx) + + // set delete stage + delStage := d.dsm.getBlobDelStage(msg.Bid) + span.Debugf("send to fail queue: bid[%d], try[%d], delete stages[%+v]", msg.Bid, msg.Retry, delStage) + msg.SetDeleteStage(delStage) + span.Debugf("delete stage: [%+v]", msg.BlobDelStages) + + msg.Retry++ + b, err := json.Marshal(msg) + if err != nil { + // just panic if marsh fail + span.Panicf("\"send to fail queue json.Marshal failed: msg[%+v], err[%+v]", msg, err) + } + + err = d.failMsgSender.SendMessage(b) + if err != nil { + span.Errorf("failMsgSender.SendMessage failed: err[%+v]", b) + return err + } + return nil +} + +// for error code judgment +func shouldUpdateVolumeErr(errCode int) bool { + return errCode == errcode.CodeDiskBroken || + errCode == errcode.CodeVuidNotFound || + errCode == errcode.CodeDiskNotFound +} + +func assumeDeleteSuccess(errCode int) bool { + return errCode == errcode.CodeBidNotFound || + errCode == errcode.CodeShardMarkDeleted +} + +func unmarshalMsgs(msgs []*sarama.ConsumerMessage) (delMsgs []*proto.DeleteMsg) { + for _, msg := range msgs { + var delMsg proto.DeleteMsg + err := json.Unmarshal(msg.Value, &delMsg) + if err != nil { + continue + } + delMsgs = append(delMsgs, &delMsg) + } + return delMsgs +} + +// DeduplicateMsgs deduplicate delete messages +func DeduplicateMsgs(ctx context.Context, delMsgs []*proto.DeleteMsg) (msgs []*proto.DeleteMsg) { + span := trace.SpanFromContextSafe(ctx) + bids := make(map[proto.BlobID]struct{}) + for _, m := range delMsgs { + if _, exist := bids[m.Bid]; !exist { + msgs = append(msgs, m) + bids[m.Bid] = struct{}{} + continue + } + span.Infof("msg dropped due to same task: msg[%+v]", m) + } + return +} diff --git a/blobstore/scheduler/blob_deleter_test.go b/blobstore/scheduler/blob_deleter_test.go new file mode 100644 index 000000000..caaa37bfa --- /dev/null +++ b/blobstore/scheduler/blob_deleter_test.go @@ -0,0 +1,363 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "encoding/json" + "io/ioutil" + "os" + "testing" + "time" + + "github.com/Shopify/sarama" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/common/counter" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/recordlog" + "github.com/cubefs/cubefs/blobstore/common/taskswitch" + "github.com/cubefs/cubefs/blobstore/scheduler/base" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/testing/mocks" + "github.com/cubefs/cubefs/blobstore/util/taskpool" +) + +func newDeleteTopicConsumer(t *testing.T) *deleteTopicConsumer { + ctr := gomock.NewController(t) + clusterMgrCli := NewMockClusterMgrAPI(ctr) + clusterMgrCli.EXPECT().GetConfig(any, any).AnyTimes().Return("", nil) + + volCache := NewMockVolumeCache(ctr) + volCache.EXPECT().Get(any).AnyTimes().DoAndReturn( + func(vid proto.Vid) (*client.VolumeInfoSimple, error) { + return &client.VolumeInfoSimple{Vid: vid}, nil + }, + ) + + switchMgr := taskswitch.NewSwitchMgr(clusterMgrCli) + taskSwitch, err := switchMgr.AddSwitch(taskswitch.BlobDeleteSwitchName) + require.NoError(t, err) + + blobnodeCli := NewMockBlobnodeAPI(ctr) + blobnodeCli.EXPECT().MarkDelete(any, any, any).AnyTimes().Return(nil) + blobnodeCli.EXPECT().Delete(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil) + + producer := NewMockProducer(ctr) + producer.EXPECT().SendMessage(any).AnyTimes().Return(nil) + consumer := NewMockConsumer(ctr) + + delLogger := mocks.NewMockEncoder(ctr) + delLogger.EXPECT().Close().AnyTimes().Return(nil) + delLogger.EXPECT().Encode(any).AnyTimes().Return(nil) + tp := taskpool.New(2, 2) + + return &deleteTopicConsumer{ + taskSwitch: taskSwitch, + topicConsumers: []base.IConsumer{consumer}, + taskPool: &tp, + + consumeIntervalMs: time.Duration(0), + safeDelayTime: time.Hour, + volCache: volCache, + blobnodeCli: blobnodeCli, + failMsgSender: producer, + + delSuccessCounter: base.NewCounter(1, "delete", base.KindSuccess), + delFailCounter: base.NewCounter(1, "delete", base.KindFailed), + errStatsDistribution: base.NewErrorStats(), + delLogger: delLogger, + + delSuccessCounterByMin: &counter.Counter{}, + delFailCounterByMin: &counter.Counter{}, + } +} + +func TestDeleteTopicConsumer(t *testing.T) { + ctr := gomock.NewController(t) + mockTopicConsumeDelete := newDeleteTopicConsumer(t) + + consumer := mockTopicConsumeDelete.topicConsumers[0].(*MockConsumer) + consumer.EXPECT().CommitOffset(any).AnyTimes().Return(nil) + + { + // nothing todo + consumer.EXPECT().ConsumeMessages(any, any).Return([]*sarama.ConsumerMessage{}) + mockTopicConsumeDelete.consumeAndDelete(consumer, 0) + } + { + // return one invalid message + consumer.EXPECT().ConsumeMessages(any, any).DoAndReturn( + func(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) { + msg := proto.DeleteMsg{} + msgByte, _ := json.Marshal(msg) + kafkaMgs := &sarama.ConsumerMessage{ + Value: msgByte, + } + return []*sarama.ConsumerMessage{kafkaMgs} + }, + ) + mockTopicConsumeDelete.consumeAndDelete(consumer, 1) + } + { + // return 2 same messages and consume one time + consumer.EXPECT().ConsumeMessages(any, any).DoAndReturn( + func(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) { + msg := proto.DeleteMsg{Bid: 1, Vid: 1, ReqId: "123456"} + msgByte, _ := json.Marshal(msg) + kafkaMgs := &sarama.ConsumerMessage{ + Value: msgByte, + } + return []*sarama.ConsumerMessage{kafkaMgs, kafkaMgs} + }, + ) + mockTopicConsumeDelete.consumeAndDelete(consumer, 2) + } + { + // return 2 diff messages adn consume success + consumer.EXPECT().ConsumeMessages(any, any).DoAndReturn( + func(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) { + msg := proto.DeleteMsg{Bid: 2, Vid: 2, ReqId: "msg1"} + msgByte, _ := json.Marshal(msg) + kafkaMgs := &sarama.ConsumerMessage{ + Value: msgByte, + } + + msg2 := proto.DeleteMsg{Bid: 1, Vid: 1, ReqId: "msg2"} + msgByte2, _ := json.Marshal(msg2) + kafkaMgs2 := &sarama.ConsumerMessage{ + Value: msgByte2, + } + return []*sarama.ConsumerMessage{kafkaMgs, kafkaMgs2} + }, + ) + mockTopicConsumeDelete.consumeAndDelete(consumer, 2) + } + { + // return one message and delete protected + oldCache := mockTopicConsumeDelete.volCache + volCache := NewMockVolumeCache(ctr) + volCache.EXPECT().Get(any).AnyTimes().DoAndReturn( + func(vid proto.Vid) (*client.VolumeInfoSimple, error) { + return &client.VolumeInfoSimple{ + Vid: vid, + VunitLocations: []proto.VunitLocation{{Vuid: 1}}, + }, nil + }, + ) + mockTopicConsumeDelete.volCache = volCache + + consumer.EXPECT().ConsumeMessages(any, any).DoAndReturn( + func(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) { + msg := proto.DeleteMsg{ + Bid: 2, + Vid: 2, + ReqId: "msg with volume return", + Time: time.Now().Unix() - 1, + } + msgByte, _ := json.Marshal(msg) + kafkaMgs := &sarama.ConsumerMessage{ + Value: msgByte, + } + return []*sarama.ConsumerMessage{kafkaMgs} + }, + ) + mockTopicConsumeDelete.safeDelayTime = 2 * time.Second + mockTopicConsumeDelete.consumeAndDelete(consumer, 2) + mockTopicConsumeDelete.volCache = oldCache + } + { + // return one message and blobnode delete failed + oldCache := mockTopicConsumeDelete.volCache + volCache := NewMockVolumeCache(ctr) + volCache.EXPECT().Get(any).AnyTimes().DoAndReturn( + func(vid proto.Vid) (*client.VolumeInfoSimple, error) { + return &client.VolumeInfoSimple{ + Vid: vid, + VunitLocations: []proto.VunitLocation{{Vuid: 1}}, + }, nil + }, + ) + mockTopicConsumeDelete.volCache = volCache + + oldBlobNode := mockTopicConsumeDelete.blobnodeCli + blobnodeCli := NewMockBlobnodeAPI(ctr) + blobnodeCli.EXPECT().MarkDelete(any, any, any).AnyTimes().Return(errMock) + mockTopicConsumeDelete.blobnodeCli = blobnodeCli + + consumer.EXPECT().ConsumeMessages(any, any).DoAndReturn( + func(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) { + msg := proto.DeleteMsg{Bid: 2, Vid: 2, ReqId: "delete failed"} + msgByte, _ := json.Marshal(msg) + kafkaMgs := &sarama.ConsumerMessage{ + Value: msgByte, + } + return []*sarama.ConsumerMessage{kafkaMgs} + }, + ) + mockTopicConsumeDelete.consumeAndDelete(consumer, 2) + mockTopicConsumeDelete.volCache = oldCache + mockTopicConsumeDelete.blobnodeCli = oldBlobNode + } + { + // return one message and blobnode return ErrDiskBroken + oldCache := mockTopicConsumeDelete.volCache + volCache := NewMockVolumeCache(ctr) + volCache.EXPECT().Get(any).AnyTimes().DoAndReturn( + func(vid proto.Vid) (*client.VolumeInfoSimple, error) { + return &client.VolumeInfoSimple{ + Vid: vid, + VunitLocations: []proto.VunitLocation{{Vuid: 1}}, + }, nil + }, + ) + volCache.EXPECT().Update(any).AnyTimes().DoAndReturn( + func(vid proto.Vid) (*client.VolumeInfoSimple, error) { + return &client.VolumeInfoSimple{ + Vid: vid, + VunitLocations: []proto.VunitLocation{{Vuid: 1}}, + }, nil + }, + ) + mockTopicConsumeDelete.volCache = volCache + + oldBlobNode := mockTopicConsumeDelete.blobnodeCli + blobnodeCli := NewMockBlobnodeAPI(ctr) + blobnodeCli.EXPECT().MarkDelete(any, any, any).AnyTimes().Return(errcode.ErrDiskBroken) + mockTopicConsumeDelete.blobnodeCli = blobnodeCli + + consumer.EXPECT().ConsumeMessages(any, any).DoAndReturn( + func(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) { + msg := proto.DeleteMsg{Bid: 2, Vid: 2, ReqId: "delete failed"} + msgByte, _ := json.Marshal(msg) + kafkaMgs := &sarama.ConsumerMessage{ + Value: msgByte, + } + return []*sarama.ConsumerMessage{kafkaMgs} + }, + ) + mockTopicConsumeDelete.consumeAndDelete(consumer, 2) + mockTopicConsumeDelete.volCache = oldCache + mockTopicConsumeDelete.blobnodeCli = oldBlobNode + } + { + // return one message, blobnode return ErrDiskBroken, and volCache update not eql + oldCache := mockTopicConsumeDelete.volCache + volCache := NewMockVolumeCache(ctr) + volCache.EXPECT().Get(any).AnyTimes().DoAndReturn( + func(vid proto.Vid) (*client.VolumeInfoSimple, error) { + return &client.VolumeInfoSimple{ + Vid: vid, + VunitLocations: []proto.VunitLocation{{Vuid: 1}}, + }, nil + }, + ) + volCache.EXPECT().Update(any).DoAndReturn( + func(vid proto.Vid) (*client.VolumeInfoSimple, error) { + return &client.VolumeInfoSimple{ + Vid: vid, + VunitLocations: []proto.VunitLocation{{Vuid: 1}, {Vuid: 2}}, + }, nil + }, + ) + mockTopicConsumeDelete.volCache = volCache + + oldBlobNode := mockTopicConsumeDelete.blobnodeCli + blobnodeCli := NewMockBlobnodeAPI(ctr) + blobnodeCli.EXPECT().MarkDelete(any, any, any).AnyTimes().Return(errcode.ErrDiskBroken) + mockTopicConsumeDelete.blobnodeCli = blobnodeCli + + consumer.EXPECT().ConsumeMessages(any, any).AnyTimes().DoAndReturn( + func(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) { + msg := proto.DeleteMsg{Bid: 2, Vid: 2, ReqId: "delete failed"} + msgByte, _ := json.Marshal(msg) + kafkaMgs := &sarama.ConsumerMessage{ + Value: msgByte, + } + return []*sarama.ConsumerMessage{kafkaMgs} + }, + ) + mockTopicConsumeDelete.consumeAndDelete(consumer, 2) + + volCache.EXPECT().Update(any).AnyTimes().DoAndReturn( + func(vid proto.Vid) (*client.VolumeInfoSimple, error) { + return &client.VolumeInfoSimple{ + Vid: vid, + VunitLocations: []proto.VunitLocation{{Vuid: 2}}, + }, nil + }, + ) + mockTopicConsumeDelete.volCache = volCache + mockTopicConsumeDelete.consumeAndDelete(consumer, 2) + + mockTopicConsumeDelete.volCache = oldCache + mockTopicConsumeDelete.blobnodeCli = oldBlobNode + } +} + +// comment temporary +func TestNewDeleteMgr(t *testing.T) { + ctr := gomock.NewController(t) + broker0 := NewBroker(t) + defer broker0.Close() + + testDir, err := ioutil.TempDir(os.TempDir(), "delete_log") + require.NoError(t, err) + defer os.RemoveAll(testDir) + + blobCfg := &BlobDeleteConfig{ + ClusterID: 0, + TaskPoolSize: 2, + NormalHandleBatchCnt: 10, + FailHandleBatchCnt: 10, + DeleteLog: recordlog.Config{ + Dir: testDir, + ChunkBits: 22, + }, + Kafka: BlobDeleteKafkaConfig{ + BrokerList: []string{broker0.Addr()}, + Normal: TopicConfig{ + Topic: testTopic, + Partitions: []int32{0}, + }, + Failed: TopicConfig{ + Topic: testTopic, + Partitions: []int32{0}, + }, + FailMsgSenderTimeoutMs: 0, + }, + } + + clusterMgrCli := NewMockClusterMgrAPI(ctr) + clusterMgrCli.EXPECT().GetConfig(any, any).AnyTimes().Return("", errMock) + volCache := NewMockVolumeCache(ctr) + blobnodeCli := NewMockBlobnodeAPI(ctr) + accessor := NewMockKafkaOffsetTable(ctr) + accessor.EXPECT().Get(any, any).AnyTimes().Return(int64(0), nil) + accessor.EXPECT().Set(any, any, any).AnyTimes().Return(nil) + switchMgr := taskswitch.NewSwitchMgr(clusterMgrCli) + + service, err := NewBlobDeleteMgr(blobCfg, volCache, accessor, blobnodeCli, switchMgr) + require.NoError(t, err) + + // run task + service.RunTask() + + // get stats + service.GetTaskStats() + service.GetErrorStats() +} diff --git a/blobstore/scheduler/client/blobnode.go b/blobstore/scheduler/client/blobnode.go new file mode 100644 index 000000000..e173cb8bf --- /dev/null +++ b/blobstore/scheduler/client/blobnode.go @@ -0,0 +1,62 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package client + +import ( + "context" + + api "github.com/cubefs/cubefs/blobstore/api/blobnode" + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +// BlobnodeAPI interface of blobnode client deleter api +type BlobnodeAPI interface { + MarkDelete(ctx context.Context, location proto.VunitLocation, bid proto.BlobID) error + Delete(ctx context.Context, location proto.VunitLocation, bid proto.BlobID) error + RepairShard(ctx context.Context, host string, task proto.ShardRepairTask) error +} + +type blobnodeClient struct { + client api.StorageAPI +} + +// NewBlobnodeClient returns blobnode client +func NewBlobnodeClient(cfg *api.Config) BlobnodeAPI { + return &blobnodeClient{api.New(cfg)} +} + +func (c *blobnodeClient) RepairShard(ctx context.Context, host string, task proto.ShardRepairTask) error { + return c.client.RepairShard(ctx, host, &api.ShardRepairArgs{ + Task: task, + }) +} + +// MarkDelete mark delete blob +func (c *blobnodeClient) MarkDelete(ctx context.Context, location proto.VunitLocation, bid proto.BlobID) error { + return c.client.MarkDeleteShard(ctx, location.Host, &api.DeleteShardArgs{ + DiskID: location.DiskID, + Vuid: location.Vuid, + Bid: bid, + }) +} + +// Delete delete blob +func (c *blobnodeClient) Delete(ctx context.Context, location proto.VunitLocation, bid proto.BlobID) error { + return c.client.DeleteShard(ctx, location.Host, &api.DeleteShardArgs{ + DiskID: location.DiskID, + Vuid: location.Vuid, + Bid: bid, + }) +} diff --git a/blobstore/scheduler/client/blobnode_test.go b/blobstore/scheduler/client/blobnode_test.go new file mode 100644 index 000000000..cc1e4c35b --- /dev/null +++ b/blobstore/scheduler/client/blobnode_test.go @@ -0,0 +1,44 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package client + +import ( + "context" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + api "github.com/cubefs/cubefs/blobstore/api/blobnode" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/testing/mocks" +) + +func TestBlobnode(t *testing.T) { + any := gomock.Any() + ctx := context.Background() + + cli := NewBlobnodeClient(&api.Config{}).(*blobnodeClient) + client := mocks.NewMockStorageAPI(gomock.NewController(t)) + client.EXPECT().MarkDeleteShard(any, any, any).Return(nil) + client.EXPECT().DeleteShard(any, any, any).Return(nil) + cli.client = client + + err := cli.MarkDelete(ctx, proto.VunitLocation{}, proto.BlobID(1)) + require.NoError(t, err) + + err = cli.Delete(ctx, proto.VunitLocation{}, proto.BlobID(1)) + require.NoError(t, err) +} diff --git a/blobstore/scheduler/client/clustermgr.go b/blobstore/scheduler/client/clustermgr.go new file mode 100644 index 000000000..025ff183b --- /dev/null +++ b/blobstore/scheduler/client/clustermgr.go @@ -0,0 +1,645 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package client + +import ( + "context" + "sync" + + "github.com/cubefs/cubefs/blobstore/api/blobnode" + cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/common/codemode" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +// ClusterMgrAPI define the interface of clustermgr used by scheduler +type ClusterMgrAPI interface { + GetConfig(ctx context.Context, key string) (val string, err error) + + // volume + GetVolumeInfo(ctx context.Context, Vid proto.Vid) (ret *VolumeInfoSimple, err error) + LockVolume(ctx context.Context, Vid proto.Vid) (err error) + UnlockVolume(ctx context.Context, Vid proto.Vid) (err error) + UpdateVolume(ctx context.Context, newVuid, oldVuid proto.Vuid, newDiskID proto.DiskID) (err error) + AllocVolumeUnit(ctx context.Context, vuid proto.Vuid) (ret *AllocVunitInfo, err error) + ReleaseVolumeUnit(ctx context.Context, vuid proto.Vuid, diskID proto.DiskID) (err error) + ListDiskVolumeUnits(ctx context.Context, diskID proto.DiskID) (ret []*VunitInfoSimple, err error) + ListVolume(ctx context.Context, marker proto.Vid, count int) (volInfo []*VolumeInfoSimple, retVid proto.Vid, err error) + + // disk + ListClusterDisks(ctx context.Context) (disks []*DiskInfoSimple, err error) + ListBrokenDisks(ctx context.Context, count int) (disks []*DiskInfoSimple, err error) + ListRepairingDisks(ctx context.Context) (disks []*DiskInfoSimple, err error) + ListDropDisks(ctx context.Context) (disks []*DiskInfoSimple, err error) + SetDiskRepairing(ctx context.Context, diskID proto.DiskID) (err error) + SetDiskRepaired(ctx context.Context, diskID proto.DiskID) (err error) + SetDiskDropped(ctx context.Context, diskID proto.DiskID) (err error) + GetDiskInfo(ctx context.Context, diskID proto.DiskID) (ret *DiskInfoSimple, err error) + + // register + Register(ctx context.Context, info RegisterInfo) error + GetService(ctx context.Context, name string, clusterID proto.ClusterID) (hosts []string, err error) +} + +var ( + defaultListDiskNum = 1000 + defaultListDiskMarker = proto.DiskID(0) +) + +// VolumeInfoSimple volume info used by scheduler +type VolumeInfoSimple struct { + Vid proto.Vid `json:"vid"` + CodeMode codemode.CodeMode `json:"code_mode"` + Status proto.VolumeStatus `json:"status"` + VunitLocations []proto.VunitLocation `json:"vunit_locations"` +} + +// EqualWith returns whether equal with another. +func (vol *VolumeInfoSimple) EqualWith(volInfo *VolumeInfoSimple) bool { + if len(vol.VunitLocations) != len(volInfo.VunitLocations) { + return false + } + if vol.Vid != volInfo.Vid || + vol.CodeMode != volInfo.CodeMode || + vol.Status != volInfo.Status { + return false + } + for i := range vol.VunitLocations { + if vol.VunitLocations[i] != volInfo.VunitLocations[i] { + return false + } + } + return true +} + +// IsIdle returns true if volume is idle +func (vol *VolumeInfoSimple) IsIdle() bool { + return vol.Status == proto.VolumeStatusIdle +} + +// IsActive returns true if volume is active +func (vol *VolumeInfoSimple) IsActive() bool { + return vol.Status == proto.VolumeStatusActive +} + +func (vol *VolumeInfoSimple) set(info *cmapi.VolumeInfo) { + vol.Vid = info.Vid + vol.CodeMode = info.CodeMode + vol.Status = info.Status + vol.VunitLocations = make([]proto.VunitLocation, len(info.Units)) + + // check volume info + codeModeInfo := info.CodeMode.Tactic() + vunitCnt := codeModeInfo.N + codeModeInfo.M + codeModeInfo.L + if len(info.Units) != vunitCnt { + log.Panicf("volume %d info unexpect", info.Vid) + } + + diskIDMap := make(map[proto.DiskID]struct{}, vunitCnt) + for _, repl := range info.Units { + if _, ok := diskIDMap[repl.DiskID]; ok { + log.Panicf("vid %d many chunks on same disk", info.Vid) + } + diskIDMap[repl.DiskID] = struct{}{} + } + + for i := 0; i < len(info.Units); i++ { + vol.VunitLocations[i] = proto.VunitLocation{ + Vuid: info.Units[i].Vuid, + Host: info.Units[i].Host, + DiskID: info.Units[i].DiskID, + } + } +} + +// AllocVunitInfo volume unit info for alloc +type AllocVunitInfo struct { + proto.VunitLocation +} + +// Location returns volume unit location +func (vunit *AllocVunitInfo) Location() proto.VunitLocation { + return vunit.VunitLocation +} + +func (vunit *AllocVunitInfo) set(info *cmapi.AllocVolumeUnit, host string) { + vunit.Vuid = info.Vuid + vunit.DiskID = info.DiskID + vunit.Host = host +} + +// VunitInfoSimple volume unit simple info +type VunitInfoSimple struct { + Vuid proto.Vuid `json:"vuid"` + DiskID proto.DiskID `json:"disk_id"` + Host string `json:"host"` + Used uint64 `json:"used"` +} + +func (vunit *VunitInfoSimple) set(info *cmapi.VolumeUnitInfo, host string) { + vunit.Vuid = info.Vuid + vunit.DiskID = info.DiskID + vunit.Host = host + vunit.Used = info.Used +} + +// DiskInfoSimple disk simple info +type DiskInfoSimple struct { + ClusterID proto.ClusterID `json:"cluster_id"` + DiskID proto.DiskID `json:"disk_id"` + Idc string `json:"idc"` + Rack string `json:"rack"` + Host string `json:"host"` + Status proto.DiskStatus `json:"status"` + Readonly bool `json:"readonly"` + UsedChunkCnt int64 `json:"used_chunk_cnt"` + MaxChunkCnt int64 `json:"max_chunk_cnt"` + FreeChunkCnt int64 `json:"free_chunk_cnt"` +} + +// IsHealth return true if disk is health +func (disk *DiskInfoSimple) IsHealth() bool { + return disk.Status == proto.DiskStatusNormal +} + +// IsBroken return true if disk is broken +func (disk *DiskInfoSimple) IsBroken() bool { + return disk.Status == proto.DiskStatusBroken +} + +// IsDropped return true if disk is dropped +func (disk *DiskInfoSimple) IsDropped() bool { + return disk.Status == proto.DiskStatusDropped +} + +// CanDropped disk can drop when disk is normal or has repaired or has dropped +// for simplicity we not allow to set disk status dropped +// when disk is repairing +func (disk *DiskInfoSimple) CanDropped() bool { + if disk.Status == proto.DiskStatusNormal || + disk.Status == proto.DiskStatusRepaired || + disk.Status == proto.DiskStatusDropped { + return true + } + return false +} + +func (disk *DiskInfoSimple) set(info *blobnode.DiskInfo) { + disk.ClusterID = info.ClusterID + disk.Idc = info.Idc + disk.Rack = info.Rack + disk.Host = info.Host + disk.DiskID = info.DiskID + disk.Status = info.Status + disk.Readonly = info.Readonly + disk.UsedChunkCnt = info.UsedChunkCnt + disk.MaxChunkCnt = info.MaxChunkCnt + disk.FreeChunkCnt = info.FreeChunkCnt +} + +// RegisterInfo register info use for clustermgr +type RegisterInfo struct { + ClusterID uint64 `json:"cluster_id"` + Name string `json:"name"` + Host string `json:"host"` + Idc string `json:"idc"` + HeartbeatIntervalS uint32 `json:"heartbeat_interval_s"` + HeartbeatTicks uint32 `json:"heartbeat_ticks"` + ExpiresTicks uint32 `json:"expires_ticks"` +} + +// IClusterManager define the interface of clustermgr +type IClusterManager interface { + GetConfig(ctx context.Context, key string) (ret string, err error) + GetVolumeInfo(ctx context.Context, args *cmapi.GetVolumeArgs) (ret *cmapi.VolumeInfo, err error) + LockVolume(ctx context.Context, args *cmapi.LockVolumeArgs) (err error) + UnlockVolume(ctx context.Context, args *cmapi.UnlockVolumeArgs) (err error) + UpdateVolume(ctx context.Context, args *cmapi.UpdateVolumeArgs) (err error) + AllocVolumeUnit(ctx context.Context, args *cmapi.AllocVolumeUnitArgs) (ret *cmapi.AllocVolumeUnit, err error) + ReleaseVolumeUnit(ctx context.Context, args *cmapi.ReleaseVolumeUnitArgs) (err error) + ListVolumeUnit(ctx context.Context, args *cmapi.ListVolumeUnitArgs) ([]*cmapi.VolumeUnitInfo, error) + ListVolume(ctx context.Context, args *cmapi.ListVolumeArgs) (ret cmapi.ListVolumes, err error) + ListDisk(ctx context.Context, args *cmapi.ListOptionArgs) (ret cmapi.ListDiskRet, err error) + ListDroppingDisk(ctx context.Context) (ret []*blobnode.DiskInfo, err error) + SetDisk(ctx context.Context, id proto.DiskID, status proto.DiskStatus) (err error) + DiskInfo(ctx context.Context, id proto.DiskID) (ret *blobnode.DiskInfo, err error) + DroppedDisk(ctx context.Context, id proto.DiskID) (err error) + RegisterService(ctx context.Context, node cmapi.ServiceNode, tickInterval, heartbeatTicks, expiresTicks uint32) (err error) + GetService(ctx context.Context, args cmapi.GetServiceArgs) (info cmapi.ServiceInfo, err error) +} + +// clustermgrClient clustermgr client +type clustermgrClient struct { + client IClusterManager + rwLock sync.RWMutex +} + +func NewClusterMgrClient(conf *cmapi.Config) ClusterMgrAPI { + return &clustermgrClient{ + client: cmapi.New(conf), + rwLock: sync.RWMutex{}, + } +} + +// GetConfig returns config by config key +func (c *clustermgrClient) GetConfig(ctx context.Context, key string) (val string, err error) { + c.rwLock.RLock() + defer c.rwLock.RUnlock() + + span := trace.SpanFromContextSafe(ctx) + + span.Debugf("get config: args key[%s]", key) + ret, err := c.client.GetConfig(ctx, key) + if err != nil { + span.Errorf("get config failed: err[%+v]", err) + return + } + span.Debugf("get config ret: config[%s]", ret) + return ret, err +} + +// GetVolumeInfo returns volume info +func (c *clustermgrClient) GetVolumeInfo(ctx context.Context, vid proto.Vid) (*VolumeInfoSimple, error) { + c.rwLock.RLock() + defer c.rwLock.RUnlock() + + span := trace.SpanFromContextSafe(ctx) + + span.Debugf("get volume info: args vid[%d]", vid) + info, err := c.client.GetVolumeInfo(ctx, &cmapi.GetVolumeArgs{Vid: vid}) + if err != nil { + span.Errorf("get volume info failed: err[%+v]", err) + return nil, err + } + span.Debugf("get volume info ret: volume[%+v]", *info) + ret := &VolumeInfoSimple{} + ret.set(info) + return ret, nil +} + +// LockVolume lock volume +func (c *clustermgrClient) LockVolume(ctx context.Context, vid proto.Vid) (err error) { + c.rwLock.Lock() + defer c.rwLock.Unlock() + + span := trace.SpanFromContextSafe(ctx) + + span.Debugf("lock volume: args vid[%d]", vid) + err = c.client.LockVolume(ctx, &cmapi.LockVolumeArgs{Vid: vid}) + span.Debugf("lock volume ret: err[%+v]", err) + return +} + +// UnlockVolume unlock volume +func (c *clustermgrClient) UnlockVolume(ctx context.Context, vid proto.Vid) (err error) { + c.rwLock.Lock() + defer c.rwLock.Unlock() + + span := trace.SpanFromContextSafe(ctx) + + span.Debugf("unlock volume: args vid[%d]", vid) + err = c.client.UnlockVolume(ctx, &cmapi.UnlockVolumeArgs{Vid: vid}) + span.Debugf("unlock volume ret: err[%+v]", err) + if rpc.DetectStatusCode(err) == errcode.CodeUnlockNotAllow { + span.Infof("unlock volume failed but deem lock success: err[%+v], code[%d]", err, rpc.DetectStatusCode(err)) + return nil + } + + return +} + +// UpdateVolume update volume +func (c *clustermgrClient) UpdateVolume(ctx context.Context, newVuid, oldVuid proto.Vuid, newDiskID proto.DiskID) (err error) { + c.rwLock.Lock() + defer c.rwLock.Unlock() + + span := trace.SpanFromContextSafe(ctx) + + span.Infof("update volume: args new vuid[%d], old vuid[%d], new disk_id[%d]", newVuid, oldVuid, newDiskID) + err = c.client.UpdateVolume(ctx, &cmapi.UpdateVolumeArgs{NewVuid: newVuid, OldVuid: oldVuid, NewDiskID: newDiskID}) + span.Infof("update volume ret: err %+v", err) + return +} + +// AllocVolumeUnit alloc volume unit +func (c *clustermgrClient) AllocVolumeUnit(ctx context.Context, vuid proto.Vuid) (*AllocVunitInfo, error) { + c.rwLock.Lock() + defer c.rwLock.Unlock() + + span := trace.SpanFromContextSafe(ctx) + + span.Debugf("alloc volume unit: args vuid[%d]", vuid) + ret := &AllocVunitInfo{} + info, err := c.client.AllocVolumeUnit(ctx, &cmapi.AllocVolumeUnitArgs{Vuid: vuid}) + if err != nil { + span.Errorf("alloc volume unit failed: err[%+v]", err) + return nil, err + } + span.Debugf("alloc volume unit ret: unit[%+v]", *info) + + diskInfo, err := c.client.DiskInfo(ctx, info.DiskID) + if err != nil { + return nil, err + } + span.Debugf("get disk info ret: disk[%+v]", diskInfo) + + ret.set(info, diskInfo.Host) + return ret, err +} + +// ReleaseVolumeUnit release volume unit +func (c *clustermgrClient) ReleaseVolumeUnit(ctx context.Context, vuid proto.Vuid, diskID proto.DiskID) (err error) { + c.rwLock.Lock() + defer c.rwLock.Unlock() + + span := trace.SpanFromContextSafe(ctx) + + span.Debugf("release volume unit: args vuid[%d], disk_id[%d]", vuid, diskID) + err = c.client.ReleaseVolumeUnit(ctx, &cmapi.ReleaseVolumeUnitArgs{Vuid: vuid, DiskID: diskID}) + span.Debugf("release volume unit ret: err[%+v]", err) + + return +} + +// ListDiskVolumeUnits list disk volume units +func (c *clustermgrClient) ListDiskVolumeUnits(ctx context.Context, diskID proto.DiskID) (rets []*VunitInfoSimple, err error) { + c.rwLock.RLock() + defer c.rwLock.RUnlock() + + span := trace.SpanFromContextSafe(ctx) + + span.Debugf("list disk volume units: args disk_id[%d]", diskID) + infos, err := c.client.ListVolumeUnit(ctx, &cmapi.ListVolumeUnitArgs{DiskID: diskID}) + if err != nil { + span.Errorf("list disk volume units failed: err[%+v]", err) + return nil, err + } + + for idx, info := range infos { + span.Debugf("list disk volume units ret: idx[%d], info[%+v]", idx, *info) + } + + diskInfo, err := c.client.DiskInfo(ctx, diskID) + if err != nil { + span.Errorf("get disk info failed: err[%+v]", err) + return nil, err + } + span.Debugf("get disk info ret: disk[%+v]", *diskInfo) + + for _, info := range infos { + ele := VunitInfoSimple{} + ele.set(info, diskInfo.Host) + rets = append(rets, &ele) + } + return rets, nil +} + +// ListVolume list volume +func (c *clustermgrClient) ListVolume(ctx context.Context, marker proto.Vid, count int) (rets []*VolumeInfoSimple, nextVid proto.Vid, err error) { + c.rwLock.RLock() + defer c.rwLock.RUnlock() + + vols, err := c.client.ListVolume(ctx, &cmapi.ListVolumeArgs{Marker: marker, Count: count}) + if err != nil { + return + } + for index := range vols.Volumes { + ret := &VolumeInfoSimple{} + ret.set(vols.Volumes[index]) + rets = append(rets, ret) + } + nextVid = vols.Marker + return +} + +// ListClusterDisks list all disks +func (c *clustermgrClient) ListClusterDisks(ctx context.Context) (disks []*DiskInfoSimple, err error) { + c.rwLock.RLock() + defer c.rwLock.RUnlock() + return c.listAllDisks(ctx, proto.DiskStatusNormal) +} + +// ListBrokenDisks list all broken disks +func (c *clustermgrClient) ListBrokenDisks(ctx context.Context, count int) (disks []*DiskInfoSimple, err error) { + c.rwLock.RLock() + defer c.rwLock.RUnlock() + return c.listDisks(ctx, proto.DiskStatusBroken, count) +} + +// ListRepairingDisks list repairing disks +func (c *clustermgrClient) ListRepairingDisks(ctx context.Context) (disks []*DiskInfoSimple, err error) { + c.rwLock.RLock() + defer c.rwLock.RUnlock() + return c.listAllDisks(ctx, proto.DiskStatusRepairing) +} + +func (c *clustermgrClient) listAllDisks(ctx context.Context, status proto.DiskStatus) (disks []*DiskInfoSimple, err error) { + span := trace.SpanFromContextSafe(ctx) + marker := defaultListDiskMarker + for { + args := &cmapi.ListOptionArgs{ + Status: status, + Count: defaultListDiskNum, + Marker: marker, + } + selectDisks, selectMarker, err := c.listDisk(ctx, args) + if err != nil { + span.Errorf("list disk failed: err[%+v]", err) + return nil, err + } + + marker = selectMarker + disks = append(disks, selectDisks...) + if marker == defaultListDiskMarker { + break + } + } + return +} + +func (c *clustermgrClient) listDisks(ctx context.Context, status proto.DiskStatus, count int) (disks []*DiskInfoSimple, err error) { + span := trace.SpanFromContextSafe(ctx) + + marker := defaultListDiskMarker + needDiskCount := count + for { + args := &cmapi.ListOptionArgs{ + Status: status, + Count: needDiskCount, + Marker: marker, + } + selectDisks, selectMarker, err := c.listDisk(ctx, args) + if err != nil { + span.Errorf("list disk failed: err[%+v]", err) + return nil, err + } + + marker = selectMarker + disks = append(disks, selectDisks...) + needDiskCount -= len(disks) + if marker == defaultListDiskMarker || needDiskCount <= 0 { + break + } + } + return +} + +func (c *clustermgrClient) listDisk(ctx context.Context, args *cmapi.ListOptionArgs) (disks []*DiskInfoSimple, marker proto.DiskID, err error) { + span := trace.SpanFromContextSafe(ctx) + + span.Debugf("list disk: args[%+v]", *args) + infos, err := c.client.ListDisk(ctx, args) + if err != nil { + span.Errorf("list disk failed: err[%+v]", err) + return nil, defaultListDiskMarker, err + } + marker = infos.Marker + for _, info := range infos.Disks { + span.Debugf("list disk ret: disk[%+v]", *info) + ele := DiskInfoSimple{} + ele.set(info) + disks = append(disks, &ele) + } + return +} + +// ListDropDisks list drop disks, may contain {DiskStatusNormal,DiskStatusReadOnly,DiskStatusBroken,DiskStatusRepairing,DiskStatusRepaired} disks +func (c *clustermgrClient) ListDropDisks(ctx context.Context) (disks []*DiskInfoSimple, err error) { + c.rwLock.RLock() + defer c.rwLock.RUnlock() + + span := trace.SpanFromContextSafe(ctx) + + infos, err := c.client.ListDroppingDisk(ctx) + if err != nil { + span.Errorf("list drop disks failed: err[%+v]", err) + return nil, err + } + span.Infof("list drop disks: len[%d]", len(infos)) + for _, info := range infos { + span.Debugf("list drop disks ret: disk[%+v]", *info) + disk := DiskInfoSimple{} + disk.set(info) + span.Infof("disk status: [%s]", disk.Status.String()) + if disk.IsHealth() { + disks = append(disks, &disk) + } + } + return disks, nil +} + +// SetDiskRepairing set disk repairing +func (c *clustermgrClient) SetDiskRepairing(ctx context.Context, diskID proto.DiskID) (err error) { + c.rwLock.Lock() + defer c.rwLock.Unlock() + + span := trace.SpanFromContextSafe(ctx) + + span.Debugf("set disk repairing: args disk_id[%d], status[%s]", diskID, proto.DiskStatusRepairing.String()) + err = c.setDiskStatus(ctx, diskID, proto.DiskStatusRepairing) + span.Debugf("set disk repairing ret: err[%+v]", err) + return +} + +// SetDiskRepaired set disk repaired +func (c *clustermgrClient) SetDiskRepaired(ctx context.Context, diskID proto.DiskID) (err error) { + c.rwLock.Lock() + defer c.rwLock.Unlock() + + span := trace.SpanFromContextSafe(ctx) + + span.Debugf("set disk repaired: args disk_id[%d], disk status[%s]", diskID, proto.DiskStatusRepaired.String()) + err = c.setDiskStatus(ctx, diskID, proto.DiskStatusRepaired) + span.Debugf("set disk repaired ret: err[%+v]", err) + return +} + +// SetDiskDropped set disk dropped +func (c *clustermgrClient) SetDiskDropped(ctx context.Context, diskID proto.DiskID) (err error) { + c.rwLock.Lock() + defer c.rwLock.Unlock() + span := trace.SpanFromContextSafe(ctx) + + info, err := c.client.DiskInfo(ctx, diskID) + if err != nil { + span.Errorf("get disk info failed: disk_id[%d], err[%+v]", diskID, err) + return err + } + + disk := &DiskInfoSimple{} + disk.set(info) + if disk.IsDropped() { + return nil + } + + if !disk.CanDropped() { + return errcode.ErrCanNotDropped + } + + span.Debugf("set disk dropped: args disk_id[%d], status[%s]", diskID, proto.DiskStatusDropped.String()) + err = c.client.DroppedDisk(ctx, diskID) + span.Debugf("set disk dropped ret: err[%+v]", err) + return +} + +func (c *clustermgrClient) setDiskStatus(ctx context.Context, diskID proto.DiskID, status proto.DiskStatus) (err error) { + return c.client.SetDisk(ctx, diskID, status) +} + +// GetDiskInfo returns disk info +func (c *clustermgrClient) GetDiskInfo(ctx context.Context, diskID proto.DiskID) (ret *DiskInfoSimple, err error) { + c.rwLock.RLock() + defer c.rwLock.RUnlock() + + span := trace.SpanFromContextSafe(ctx) + + span.Debugf("get disk info: args disk_id[%d]", diskID) + info, err := c.client.DiskInfo(ctx, diskID) + if err != nil { + span.Errorf("get disk info failed: err[%+v]", err) + return nil, err + } + span.Debugf("get disk info ret: disk[%+v]", *info) + ret = &DiskInfoSimple{} + ret.set(info) + return ret, nil +} + +func (c *clustermgrClient) Register(ctx context.Context, info RegisterInfo) error { + node := cmapi.ServiceNode{ + ClusterID: info.ClusterID, + Name: info.Name, + Host: info.Host, + Idc: info.Idc, + } + return c.client.RegisterService(ctx, node, info.HeartbeatIntervalS, info.HeartbeatTicks, info.ExpiresTicks) +} + +func (c *clustermgrClient) GetService(ctx context.Context, name string, clusterID proto.ClusterID) (hosts []string, err error) { + svrInfos, err := c.client.GetService(ctx, cmapi.GetServiceArgs{Name: name}) + if err != nil { + return nil, err + } + for _, s := range svrInfos.Nodes { + if clusterID == proto.ClusterID(s.ClusterID) { + hosts = append(hosts, s.Host) + } + } + return +} diff --git a/blobstore/scheduler/client/clustermgr_mock_test.go b/blobstore/scheduler/client/clustermgr_mock_test.go new file mode 100644 index 000000000..5bae67823 --- /dev/null +++ b/blobstore/scheduler/client/clustermgr_mock_test.go @@ -0,0 +1,271 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cubefs/cubefs/blobstore/scheduler/client (interfaces: IClusterManager) + +// Package client is a generated GoMock package. +package client + +import ( + context "context" + reflect "reflect" + + blobnode "github.com/cubefs/cubefs/blobstore/api/blobnode" + clustermgr "github.com/cubefs/cubefs/blobstore/api/clustermgr" + proto "github.com/cubefs/cubefs/blobstore/common/proto" + gomock "github.com/golang/mock/gomock" +) + +// MockClusterManager is a mock of IClusterManager interface. +type MockClusterManager struct { + ctrl *gomock.Controller + recorder *MockClusterManagerMockRecorder +} + +// MockClusterManagerMockRecorder is the mock recorder for MockClusterManager. +type MockClusterManagerMockRecorder struct { + mock *MockClusterManager +} + +// NewMockClusterManager creates a new mock instance. +func NewMockClusterManager(ctrl *gomock.Controller) *MockClusterManager { + mock := &MockClusterManager{ctrl: ctrl} + mock.recorder = &MockClusterManagerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockClusterManager) EXPECT() *MockClusterManagerMockRecorder { + return m.recorder +} + +// AllocVolumeUnit mocks base method. +func (m *MockClusterManager) AllocVolumeUnit(arg0 context.Context, arg1 *clustermgr.AllocVolumeUnitArgs) (*clustermgr.AllocVolumeUnit, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AllocVolumeUnit", arg0, arg1) + ret0, _ := ret[0].(*clustermgr.AllocVolumeUnit) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AllocVolumeUnit indicates an expected call of AllocVolumeUnit. +func (mr *MockClusterManagerMockRecorder) AllocVolumeUnit(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocVolumeUnit", reflect.TypeOf((*MockClusterManager)(nil).AllocVolumeUnit), arg0, arg1) +} + +// DiskInfo mocks base method. +func (m *MockClusterManager) DiskInfo(arg0 context.Context, arg1 proto.DiskID) (*blobnode.DiskInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DiskInfo", arg0, arg1) + ret0, _ := ret[0].(*blobnode.DiskInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DiskInfo indicates an expected call of DiskInfo. +func (mr *MockClusterManagerMockRecorder) DiskInfo(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiskInfo", reflect.TypeOf((*MockClusterManager)(nil).DiskInfo), arg0, arg1) +} + +// DroppedDisk mocks base method. +func (m *MockClusterManager) DroppedDisk(arg0 context.Context, arg1 proto.DiskID) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DroppedDisk", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// DroppedDisk indicates an expected call of DroppedDisk. +func (mr *MockClusterManagerMockRecorder) DroppedDisk(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DroppedDisk", reflect.TypeOf((*MockClusterManager)(nil).DroppedDisk), arg0, arg1) +} + +// GetConfig mocks base method. +func (m *MockClusterManager) GetConfig(arg0 context.Context, arg1 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetConfig", arg0, arg1) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetConfig indicates an expected call of GetConfig. +func (mr *MockClusterManagerMockRecorder) GetConfig(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfig", reflect.TypeOf((*MockClusterManager)(nil).GetConfig), arg0, arg1) +} + +// GetService mocks base method. +func (m *MockClusterManager) GetService(arg0 context.Context, arg1 clustermgr.GetServiceArgs) (clustermgr.ServiceInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetService", arg0, arg1) + ret0, _ := ret[0].(clustermgr.ServiceInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetService indicates an expected call of GetService. +func (mr *MockClusterManagerMockRecorder) GetService(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetService", reflect.TypeOf((*MockClusterManager)(nil).GetService), arg0, arg1) +} + +// GetVolumeInfo mocks base method. +func (m *MockClusterManager) GetVolumeInfo(arg0 context.Context, arg1 *clustermgr.GetVolumeArgs) (*clustermgr.VolumeInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVolumeInfo", arg0, arg1) + ret0, _ := ret[0].(*clustermgr.VolumeInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetVolumeInfo indicates an expected call of GetVolumeInfo. +func (mr *MockClusterManagerMockRecorder) GetVolumeInfo(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVolumeInfo", reflect.TypeOf((*MockClusterManager)(nil).GetVolumeInfo), arg0, arg1) +} + +// ListDisk mocks base method. +func (m *MockClusterManager) ListDisk(arg0 context.Context, arg1 *clustermgr.ListOptionArgs) (clustermgr.ListDiskRet, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListDisk", arg0, arg1) + ret0, _ := ret[0].(clustermgr.ListDiskRet) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListDisk indicates an expected call of ListDisk. +func (mr *MockClusterManagerMockRecorder) ListDisk(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDisk", reflect.TypeOf((*MockClusterManager)(nil).ListDisk), arg0, arg1) +} + +// ListDroppingDisk mocks base method. +func (m *MockClusterManager) ListDroppingDisk(arg0 context.Context) ([]*blobnode.DiskInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListDroppingDisk", arg0) + ret0, _ := ret[0].([]*blobnode.DiskInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListDroppingDisk indicates an expected call of ListDroppingDisk. +func (mr *MockClusterManagerMockRecorder) ListDroppingDisk(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDroppingDisk", reflect.TypeOf((*MockClusterManager)(nil).ListDroppingDisk), arg0) +} + +// ListVolume mocks base method. +func (m *MockClusterManager) ListVolume(arg0 context.Context, arg1 *clustermgr.ListVolumeArgs) (clustermgr.ListVolumes, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListVolume", arg0, arg1) + ret0, _ := ret[0].(clustermgr.ListVolumes) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListVolume indicates an expected call of ListVolume. +func (mr *MockClusterManagerMockRecorder) ListVolume(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListVolume", reflect.TypeOf((*MockClusterManager)(nil).ListVolume), arg0, arg1) +} + +// ListVolumeUnit mocks base method. +func (m *MockClusterManager) ListVolumeUnit(arg0 context.Context, arg1 *clustermgr.ListVolumeUnitArgs) ([]*clustermgr.VolumeUnitInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListVolumeUnit", arg0, arg1) + ret0, _ := ret[0].([]*clustermgr.VolumeUnitInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListVolumeUnit indicates an expected call of ListVolumeUnit. +func (mr *MockClusterManagerMockRecorder) ListVolumeUnit(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListVolumeUnit", reflect.TypeOf((*MockClusterManager)(nil).ListVolumeUnit), arg0, arg1) +} + +// LockVolume mocks base method. +func (m *MockClusterManager) LockVolume(arg0 context.Context, arg1 *clustermgr.LockVolumeArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LockVolume", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// LockVolume indicates an expected call of LockVolume. +func (mr *MockClusterManagerMockRecorder) LockVolume(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LockVolume", reflect.TypeOf((*MockClusterManager)(nil).LockVolume), arg0, arg1) +} + +// RegisterService mocks base method. +func (m *MockClusterManager) RegisterService(arg0 context.Context, arg1 clustermgr.ServiceNode, arg2, arg3, arg4 uint32) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RegisterService", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].(error) + return ret0 +} + +// RegisterService indicates an expected call of RegisterService. +func (mr *MockClusterManagerMockRecorder) RegisterService(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterService", reflect.TypeOf((*MockClusterManager)(nil).RegisterService), arg0, arg1, arg2, arg3, arg4) +} + +// ReleaseVolumeUnit mocks base method. +func (m *MockClusterManager) ReleaseVolumeUnit(arg0 context.Context, arg1 *clustermgr.ReleaseVolumeUnitArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReleaseVolumeUnit", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// ReleaseVolumeUnit indicates an expected call of ReleaseVolumeUnit. +func (mr *MockClusterManagerMockRecorder) ReleaseVolumeUnit(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReleaseVolumeUnit", reflect.TypeOf((*MockClusterManager)(nil).ReleaseVolumeUnit), arg0, arg1) +} + +// SetDisk mocks base method. +func (m *MockClusterManager) SetDisk(arg0 context.Context, arg1 proto.DiskID, arg2 proto.DiskStatus) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetDisk", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetDisk indicates an expected call of SetDisk. +func (mr *MockClusterManagerMockRecorder) SetDisk(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDisk", reflect.TypeOf((*MockClusterManager)(nil).SetDisk), arg0, arg1, arg2) +} + +// UnlockVolume mocks base method. +func (m *MockClusterManager) UnlockVolume(arg0 context.Context, arg1 *clustermgr.UnlockVolumeArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UnlockVolume", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// UnlockVolume indicates an expected call of UnlockVolume. +func (mr *MockClusterManagerMockRecorder) UnlockVolume(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnlockVolume", reflect.TypeOf((*MockClusterManager)(nil).UnlockVolume), arg0, arg1) +} + +// UpdateVolume mocks base method. +func (m *MockClusterManager) UpdateVolume(arg0 context.Context, arg1 *clustermgr.UpdateVolumeArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateVolume", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateVolume indicates an expected call of UpdateVolume. +func (mr *MockClusterManagerMockRecorder) UpdateVolume(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateVolume", reflect.TypeOf((*MockClusterManager)(nil).UpdateVolume), arg0, arg1) +} diff --git a/blobstore/scheduler/client/clustermgr_test.go b/blobstore/scheduler/client/clustermgr_test.go new file mode 100644 index 000000000..b53f61ad2 --- /dev/null +++ b/blobstore/scheduler/client/clustermgr_test.go @@ -0,0 +1,305 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package client + +//go:generate mockgen -destination=./clustermgr_mock_test.go -package=client -mock_names IClusterManager=MockClusterManager github.com/cubefs/cubefs/blobstore/scheduler/client IClusterManager + +import ( + "context" + "errors" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/api/blobnode" + cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/common/codemode" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/taskswitch" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +var ( + defaultVolumeListMarker = proto.Vid(0) + defaultDiskListMarker = proto.DiskID(0) +) + +func init() { + log.SetOutputLevel(log.Lfatal) +} + +func MockGenVolInfo(vid proto.Vid, mode codemode.CodeMode, status proto.VolumeStatus) *cmapi.VolumeInfo { + cmInfo := mode.Tactic() + vunitCnt := cmInfo.M + cmInfo.N + cmInfo.L + host := "127.0.0.0:xxx" + locations := make([]cmapi.Unit, vunitCnt) + var idx uint8 + for i := 0; i < vunitCnt; i++ { + locations[i].Vuid, _ = proto.NewVuid(vid, idx, 1) + locations[i].Host = host + locations[i].DiskID = proto.DiskID(locations[i].Vuid) + idx++ + } + + return &cmapi.VolumeInfo{ + Units: locations, + VolumeInfoBase: cmapi.VolumeInfoBase{ + Vid: vid, + CodeMode: mode, + Status: status, + }, + } +} + +func TestClustermgrClient(t *testing.T) { + cli := NewClusterMgrClient(&cmapi.Config{}).(*clustermgrClient) + mockCli := NewMockClusterManager(gomock.NewController(t)) + cli.client = mockCli + + ctx := context.Background() + any := gomock.Any() + errMock := errors.New("fake error") + { + // get config + cli.client.(*MockClusterManager).EXPECT().GetConfig(any, any).Return("", errMock) + _, err := cli.GetConfig(ctx, "config") + require.True(t, errors.Is(err, errMock)) + + cli.client.(*MockClusterManager).EXPECT().GetConfig(any, any).Return(taskswitch.SwitchOpen, nil) + enable, err := cli.GetConfig(ctx, "config") + require.NoError(t, err) + require.Equal(t, "true", enable) + } + { + // get volume info + cli.client.(*MockClusterManager).EXPECT().GetVolumeInfo(any, any).Return(nil, errMock) + _, err := cli.GetVolumeInfo(ctx, proto.Vid(1)) + require.True(t, errors.Is(err, errMock)) + + volume := MockGenVolInfo(10, codemode.EC6P6, proto.VolumeStatusIdle) + volume2 := MockGenVolInfo(10, codemode.EC6P6, proto.VolumeStatusActive) + cli.client.(*MockClusterManager).EXPECT().GetVolumeInfo(any, any).Return(volume, nil) + cli.client.(*MockClusterManager).EXPECT().GetVolumeInfo(any, any).Return(volume2, nil) + vol, err := cli.GetVolumeInfo(ctx, proto.Vid(1)) + require.NoError(t, err) + vol2, err := cli.GetVolumeInfo(ctx, proto.Vid(2)) + require.NoError(t, err) + require.Equal(t, vol.Vid, volume.Vid) + require.Equal(t, vol.CodeMode, volume.CodeMode) + require.Equal(t, vol.Status, volume.Status) + require.Equal(t, len(vol.VunitLocations), len(volume.Units)) + require.True(t, vol.IsIdle()) + require.True(t, vol2.IsActive()) + require.True(t, vol.EqualWith(vol)) + require.False(t, vol.EqualWith(vol2)) + } + { + // lock volume + cli.client.(*MockClusterManager).EXPECT().LockVolume(any, any).Return(nil) + err := cli.LockVolume(ctx, proto.Vid(1)) + require.NoError(t, err) + } + { + // unlock volume + cli.client.(*MockClusterManager).EXPECT().UnlockVolume(any, any).Return(nil) + err := cli.UnlockVolume(ctx, proto.Vid(1)) + require.NoError(t, err) + + cli.client.(*MockClusterManager).EXPECT().UnlockVolume(any, any).Return(errcode.ErrUnlockNotAllow) + err = cli.UnlockVolume(ctx, proto.Vid(1)) + require.NoError(t, err) + + cli.client.(*MockClusterManager).EXPECT().UnlockVolume(any, any).Return(errMock) + err = cli.UnlockVolume(ctx, proto.Vid(1)) + require.True(t, errors.Is(err, errMock)) + } + { + // update volume + cli.client.(*MockClusterManager).EXPECT().UpdateVolume(any, any).Return(nil) + err := cli.UpdateVolume(ctx, proto.Vuid(2), proto.Vuid(1), proto.DiskID(1)) + require.NoError(t, err) + } + { + // update volume + cli.client.(*MockClusterManager).EXPECT().AllocVolumeUnit(any, any).Return(nil, errMock) + _, err := cli.AllocVolumeUnit(ctx, proto.Vuid(2)) + require.True(t, errors.Is(err, errMock)) + + unit := &cmapi.AllocVolumeUnit{Vuid: proto.Vuid(3), DiskID: proto.DiskID(2)} + cli.client.(*MockClusterManager).EXPECT().AllocVolumeUnit(any, any).Return(unit, nil) + cli.client.(*MockClusterManager).EXPECT().DiskInfo(any, any).Return(nil, errMock) + _, err = cli.AllocVolumeUnit(ctx, proto.Vuid(2)) + require.True(t, errors.Is(err, errMock)) + + cli.client.(*MockClusterManager).EXPECT().AllocVolumeUnit(any, any).Return(unit, nil) + cli.client.(*MockClusterManager).EXPECT().DiskInfo(any, any).Return(&blobnode.DiskInfo{Host: "127.0.0.1:xxx"}, nil) + allocUnit, err := cli.AllocVolumeUnit(ctx, proto.Vuid(2)) + require.NoError(t, err) + require.Equal(t, unit.Vuid, allocUnit.Location().Vuid) + } + { + // release volume unit + cli.client.(*MockClusterManager).EXPECT().ReleaseVolumeUnit(any, any).Return(nil) + err := cli.ReleaseVolumeUnit(ctx, proto.Vuid(2), proto.DiskID(1)) + require.NoError(t, err) + } + { + // list disk volume units + cli.client.(*MockClusterManager).EXPECT().ListVolumeUnit(any, any).Return(nil, errMock) + _, err := cli.ListDiskVolumeUnits(ctx, proto.DiskID(1)) + require.True(t, errors.Is(err, errMock)) + + unit := &cmapi.VolumeUnitInfo{Vuid: proto.Vuid(3), DiskID: proto.DiskID(2)} + cli.client.(*MockClusterManager).EXPECT().ListVolumeUnit(any, any).Return([]*cmapi.VolumeUnitInfo{unit}, nil) + cli.client.(*MockClusterManager).EXPECT().DiskInfo(any, any).Return(nil, errMock) + _, err = cli.ListDiskVolumeUnits(ctx, proto.DiskID(1)) + require.True(t, errors.Is(err, errMock)) + + cli.client.(*MockClusterManager).EXPECT().ListVolumeUnit(any, any).Return([]*cmapi.VolumeUnitInfo{unit}, nil) + cli.client.(*MockClusterManager).EXPECT().DiskInfo(any, any).Return(&blobnode.DiskInfo{Host: "127.0.0.1:xxx"}, nil) + units, err := cli.ListDiskVolumeUnits(ctx, proto.DiskID(1)) + require.NoError(t, err) + require.Equal(t, 1, len(units)) + } + { + // list volume + cli.client.(*MockClusterManager).EXPECT().ListVolume(any, any).Return(cmapi.ListVolumes{}, errMock) + _, _, err := cli.ListVolume(ctx, defaultVolumeListMarker, 10) + require.True(t, errors.Is(err, errMock)) + + cli.client.(*MockClusterManager).EXPECT().ListVolume(any, any).Return(cmapi.ListVolumes{}, nil) + rets, _, err := cli.ListVolume(ctx, defaultVolumeListMarker, 10) + require.NoError(t, err) + require.Equal(t, 0, len(rets)) + + volume := MockGenVolInfo(10, codemode.EC6P6, proto.VolumeStatusIdle) + cli.client.(*MockClusterManager).EXPECT().ListVolume(any, any).Return(cmapi.ListVolumes{Volumes: []*cmapi.VolumeInfo{volume}, Marker: defaultVolumeListMarker}, nil) + rets, marker, err := cli.ListVolume(ctx, defaultVolumeListMarker, 10) + require.NoError(t, err) + require.Equal(t, 1, len(rets)) + require.Equal(t, marker, defaultVolumeListMarker) + } + { + // list cluster disk + cli.client.(*MockClusterManager).EXPECT().ListDisk(any, any).Return(cmapi.ListDiskRet{}, errMock) + _, err := cli.ListClusterDisks(ctx) + require.True(t, errors.Is(err, errMock)) + } + { + // list broken disk + cli.client.(*MockClusterManager).EXPECT().ListDisk(any, any).Return(cmapi.ListDiskRet{}, errMock) + _, err := cli.ListBrokenDisks(ctx, 1) + require.True(t, errors.Is(err, errMock)) + } + { + // list repair disk + cli.client.(*MockClusterManager).EXPECT().ListDisk(any, any).Return(cmapi.ListDiskRet{}, errMock) + _, err := cli.ListRepairingDisks(ctx) + require.True(t, errors.Is(err, errMock)) + } + { + // list all disk + cli.client.(*MockClusterManager).EXPECT().ListDisk(any, any).Return(cmapi.ListDiskRet{}, errMock) + _, err := cli.listAllDisks(ctx, proto.DiskStatusNormal) + require.True(t, errors.Is(err, errMock)) + + disk1 := &blobnode.DiskInfo{Host: "127.0.0.1:xxx", Status: proto.DiskStatusNormal} + cli.client.(*MockClusterManager).EXPECT().ListDisk(any, any).Return(cmapi.ListDiskRet{Disks: []*blobnode.DiskInfo{disk1}, Marker: defaultDiskListMarker}, nil) + disks, err := cli.listAllDisks(ctx, proto.DiskStatusNormal) + require.NoError(t, err) + require.Equal(t, 1, len(disks)) + } + { + // list disks + cli.client.(*MockClusterManager).EXPECT().ListDisk(any, any).Return(cmapi.ListDiskRet{}, errMock) + _, err := cli.listDisks(ctx, proto.DiskStatusNormal, 1) + require.True(t, errors.Is(err, errMock)) + + disk1 := &blobnode.DiskInfo{Host: "127.0.0.1:xxx", Status: proto.DiskStatusNormal} + disk2 := &blobnode.DiskInfo{Host: "127.0.0.1:xxx", Status: proto.DiskStatusNormal} + cli.client.(*MockClusterManager).EXPECT().ListDisk(any, any).Return(cmapi.ListDiskRet{Disks: []*blobnode.DiskInfo{disk1}, Marker: proto.DiskID(2)}, nil) + cli.client.(*MockClusterManager).EXPECT().ListDisk(any, any).Return(cmapi.ListDiskRet{Disks: []*blobnode.DiskInfo{disk2}, Marker: defaultDiskListMarker}, nil) + disks, err := cli.listDisks(ctx, proto.DiskStatusNormal, 2) + require.NoError(t, err) + require.Equal(t, 2, len(disks)) + } + { + // list drop disk + cli.client.(*MockClusterManager).EXPECT().ListDroppingDisk(any).Return(nil, errMock) + _, err := cli.ListDropDisks(ctx) + require.True(t, errors.Is(err, errMock)) + + disk1 := &blobnode.DiskInfo{Host: "127.0.0.1:xxx", Status: proto.DiskStatusNormal} + cli.client.(*MockClusterManager).EXPECT().ListDroppingDisk(any).Return([]*blobnode.DiskInfo{disk1}, nil) + disks, err := cli.ListDropDisks(ctx) + require.NoError(t, err) + require.Equal(t, 1, len(disks)) + } + { + // set disk repair + cli.client.(*MockClusterManager).EXPECT().SetDisk(any, any, any).Return(nil) + err := cli.SetDiskRepairing(ctx, proto.DiskID(1)) + require.NoError(t, err) + } + { + // set disk repaired + cli.client.(*MockClusterManager).EXPECT().SetDisk(any, any, any).Return(nil) + err := cli.SetDiskRepaired(ctx, proto.DiskID(1)) + require.NoError(t, err) + } + { + // set disk repaired + cli.client.(*MockClusterManager).EXPECT().DiskInfo(any, any).Return(nil, errMock) + err := cli.SetDiskDropped(ctx, proto.DiskID(1)) + require.True(t, errors.Is(err, errMock)) + + disk1 := &blobnode.DiskInfo{Host: "127.0.0.1:xxx", Status: proto.DiskStatusDropped} + cli.client.(*MockClusterManager).EXPECT().DiskInfo(any, any).Return(disk1, nil) + err = cli.SetDiskDropped(ctx, proto.DiskID(1)) + require.NoError(t, err) + + disk2 := &blobnode.DiskInfo{Host: "127.0.0.1:xxx", Status: proto.DiskStatusRepairing} + cli.client.(*MockClusterManager).EXPECT().DiskInfo(any, any).Return(disk2, nil) + err = cli.SetDiskDropped(ctx, proto.DiskID(1)) + require.True(t, errors.Is(err, errcode.ErrCanNotDropped)) + + disk3 := &blobnode.DiskInfo{Host: "127.0.0.1:xxx", Status: proto.DiskStatusNormal} + cli.client.(*MockClusterManager).EXPECT().DiskInfo(any, any).Return(disk3, nil) + cli.client.(*MockClusterManager).EXPECT().DroppedDisk(any, any).Return(nil) + err = cli.SetDiskDropped(ctx, proto.DiskID(1)) + require.NoError(t, err) + } + { + // get disk info + cli.client.(*MockClusterManager).EXPECT().DiskInfo(any, any).Return(nil, errMock) + _, err := cli.GetDiskInfo(ctx, proto.DiskID(1)) + require.True(t, errors.Is(err, errMock)) + + disk1 := &blobnode.DiskInfo{Host: "127.0.0.1:xxx", Status: proto.DiskStatusDropped} + cli.client.(*MockClusterManager).EXPECT().DiskInfo(any, any).Return(disk1, nil) + disk, err := cli.GetDiskInfo(ctx, proto.DiskID(1)) + require.NoError(t, err) + require.Equal(t, disk.Status, disk1.Status) + require.False(t, disk.IsBroken()) + } + { + // register service + cli.client.(*MockClusterManager).EXPECT().RegisterService(any, any, any, any, any).Return(nil) + err := cli.Register(ctx, RegisterInfo{}) + require.NoError(t, err) + } +} diff --git a/blobstore/scheduler/client/proxy.go b/blobstore/scheduler/client/proxy.go new file mode 100644 index 000000000..13e768f57 --- /dev/null +++ b/blobstore/scheduler/client/proxy.go @@ -0,0 +1,58 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package client + +import ( + "context" + + cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr" + api "github.com/cubefs/cubefs/blobstore/api/proxy" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/trace" +) + +// ProxyAPI define the interface of proxy used by scheduler +type ProxyAPI interface { + SendShardRepairMsg(ctx context.Context, vid proto.Vid, bid proto.BlobID, badIdx []uint8) error +} + +// proxyClient proxy client +type proxyClient struct { + client api.LbMsgSender + clusterID proto.ClusterID +} + +// NewProxyClient returns proxy client +func NewProxyClient(cfg *api.LbConfig, clusterMgr *cmapi.Client, clusterID proto.ClusterID) ProxyAPI { + return &proxyClient{client: api.NewMQLbClient(cfg, clusterMgr, clusterID), clusterID: clusterID} +} + +// SendShardRepairMsg send shard repair message +func (c *proxyClient) SendShardRepairMsg(ctx context.Context, vid proto.Vid, bid proto.BlobID, badIdx []uint8) error { + pSpan := trace.SpanFromContextSafe(ctx) + span, ctx := trace.StartSpanFromContextWithTraceID(context.Background(), "SendShardRepairMsg", pSpan.TraceID()) + span.Debugf("send shard repair msg vid %d bid %d badIdx %+v", vid, bid, badIdx) + + err := c.client.SendShardRepairMsg(ctx, &api.ShardRepairArgs{ + ClusterID: c.clusterID, + Bid: bid, + Vid: vid, + BadIdxes: badIdx, + Reason: "inspect", + }) + + span.Debugf("send shard repair msg ret err %+v", err) + return err +} diff --git a/blobstore/scheduler/client/proxy_test.go b/blobstore/scheduler/client/proxy_test.go new file mode 100644 index 000000000..254f65bc0 --- /dev/null +++ b/blobstore/scheduler/client/proxy_test.go @@ -0,0 +1,36 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package client + +import ( + "context" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/testing/mocks" +) + +func TestMQProxy(t *testing.T) { + mqcli := mocks.NewMockProxyLbRpcClient(gomock.NewController(t)) + mqcli.EXPECT().SendShardRepairMsg(gomock.Any(), gomock.Any()).Return(nil) + cli := &proxyClient{ + client: mqcli, + clusterID: 1, + } + err := cli.SendShardRepairMsg(context.Background(), 0, 0, []uint8{0}) + require.NoError(t, err) +} diff --git a/blobstore/scheduler/client/volume_update.go b/blobstore/scheduler/client/volume_update.go new file mode 100644 index 000000000..353d7db9d --- /dev/null +++ b/blobstore/scheduler/client/volume_update.go @@ -0,0 +1,34 @@ +package client + +import ( + "context" + + api "github.com/cubefs/cubefs/blobstore/api/scheduler" + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +// IVolumeUpdater update volume cache +type IVolumeUpdater interface { + UpdateFollowerVolumeCache(ctx context.Context, host string, vid proto.Vid) (err error) + UpdateLeaderVolumeCache(ctx context.Context, vid proto.Vid) (err error) +} + +type volumeUpdater struct { + LeaderHost string + Client api.IVolumeUpdater +} + +func NewVolumeUpdater(cfg *api.Config, host string) IVolumeUpdater { + return &volumeUpdater{ + LeaderHost: host, + Client: api.NewVolumeUpdater(cfg), + } +} + +func (v *volumeUpdater) UpdateLeaderVolumeCache(ctx context.Context, vid proto.Vid) (err error) { + return v.Client.UpdateVol(ctx, v.LeaderHost, vid) +} + +func (v *volumeUpdater) UpdateFollowerVolumeCache(ctx context.Context, host string, vid proto.Vid) (err error) { + return v.Client.UpdateVol(ctx, host, vid) +} diff --git a/blobstore/scheduler/client/volume_update_test.go b/blobstore/scheduler/client/volume_update_test.go new file mode 100644 index 000000000..5df1ecafe --- /dev/null +++ b/blobstore/scheduler/client/volume_update_test.go @@ -0,0 +1,46 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package client + +import ( + "context" + "errors" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + api "github.com/cubefs/cubefs/blobstore/api/scheduler" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/testing/mocks" +) + +func TestVolumeUpdate(t *testing.T) { + any := gomock.Any() + errMock := errors.New("fake error") + cli := NewVolumeUpdater(&api.Config{}, "127.0.0.1:xxx").(*volumeUpdater) + + ctx := context.Background() + schedulerCli := mocks.NewMockIScheduler(gomock.NewController(t)) + schedulerCli.EXPECT().UpdateVol(any, any, any).Return(nil) + cli.Client = schedulerCli + + err := cli.UpdateLeaderVolumeCache(ctx, proto.Vid(1)) + require.NoError(t, err) + + schedulerCli.EXPECT().UpdateVol(any, any, any).Return(errMock) + err = cli.UpdateFollowerVolumeCache(ctx, "127.0.0.1:xxx", proto.Vid(1)) + require.True(t, errors.Is(err, errMock)) +} diff --git a/blobstore/scheduler/client_mock_test.go b/blobstore/scheduler/client_mock_test.go new file mode 100644 index 000000000..3d3a841da --- /dev/null +++ b/blobstore/scheduler/client_mock_test.go @@ -0,0 +1,468 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cubefs/cubefs/blobstore/scheduler/client (interfaces: ClusterMgrAPI,BlobnodeAPI,IVolumeUpdater,ProxyAPI) + +// Package scheduler is a generated GoMock package. +package scheduler + +import ( + context "context" + reflect "reflect" + + proto "github.com/cubefs/cubefs/blobstore/common/proto" + client "github.com/cubefs/cubefs/blobstore/scheduler/client" + gomock "github.com/golang/mock/gomock" +) + +// MockClusterMgrAPI is a mock of ClusterMgrAPI interface. +type MockClusterMgrAPI struct { + ctrl *gomock.Controller + recorder *MockClusterMgrAPIMockRecorder +} + +// MockClusterMgrAPIMockRecorder is the mock recorder for MockClusterMgrAPI. +type MockClusterMgrAPIMockRecorder struct { + mock *MockClusterMgrAPI +} + +// NewMockClusterMgrAPI creates a new mock instance. +func NewMockClusterMgrAPI(ctrl *gomock.Controller) *MockClusterMgrAPI { + mock := &MockClusterMgrAPI{ctrl: ctrl} + mock.recorder = &MockClusterMgrAPIMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockClusterMgrAPI) EXPECT() *MockClusterMgrAPIMockRecorder { + return m.recorder +} + +// AllocVolumeUnit mocks base method. +func (m *MockClusterMgrAPI) AllocVolumeUnit(arg0 context.Context, arg1 proto.Vuid) (*client.AllocVunitInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AllocVolumeUnit", arg0, arg1) + ret0, _ := ret[0].(*client.AllocVunitInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AllocVolumeUnit indicates an expected call of AllocVolumeUnit. +func (mr *MockClusterMgrAPIMockRecorder) AllocVolumeUnit(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocVolumeUnit", reflect.TypeOf((*MockClusterMgrAPI)(nil).AllocVolumeUnit), arg0, arg1) +} + +// GetConfig mocks base method. +func (m *MockClusterMgrAPI) GetConfig(arg0 context.Context, arg1 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetConfig", arg0, arg1) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetConfig indicates an expected call of GetConfig. +func (mr *MockClusterMgrAPIMockRecorder) GetConfig(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfig", reflect.TypeOf((*MockClusterMgrAPI)(nil).GetConfig), arg0, arg1) +} + +// GetDiskInfo mocks base method. +func (m *MockClusterMgrAPI) GetDiskInfo(arg0 context.Context, arg1 proto.DiskID) (*client.DiskInfoSimple, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetDiskInfo", arg0, arg1) + ret0, _ := ret[0].(*client.DiskInfoSimple) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetDiskInfo indicates an expected call of GetDiskInfo. +func (mr *MockClusterMgrAPIMockRecorder) GetDiskInfo(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDiskInfo", reflect.TypeOf((*MockClusterMgrAPI)(nil).GetDiskInfo), arg0, arg1) +} + +// GetService mocks base method. +func (m *MockClusterMgrAPI) GetService(arg0 context.Context, arg1 string, arg2 proto.ClusterID) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetService", arg0, arg1, arg2) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetService indicates an expected call of GetService. +func (mr *MockClusterMgrAPIMockRecorder) GetService(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetService", reflect.TypeOf((*MockClusterMgrAPI)(nil).GetService), arg0, arg1, arg2) +} + +// GetVolumeInfo mocks base method. +func (m *MockClusterMgrAPI) GetVolumeInfo(arg0 context.Context, arg1 proto.Vid) (*client.VolumeInfoSimple, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVolumeInfo", arg0, arg1) + ret0, _ := ret[0].(*client.VolumeInfoSimple) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetVolumeInfo indicates an expected call of GetVolumeInfo. +func (mr *MockClusterMgrAPIMockRecorder) GetVolumeInfo(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVolumeInfo", reflect.TypeOf((*MockClusterMgrAPI)(nil).GetVolumeInfo), arg0, arg1) +} + +// ListBrokenDisks mocks base method. +func (m *MockClusterMgrAPI) ListBrokenDisks(arg0 context.Context, arg1 int) ([]*client.DiskInfoSimple, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListBrokenDisks", arg0, arg1) + ret0, _ := ret[0].([]*client.DiskInfoSimple) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListBrokenDisks indicates an expected call of ListBrokenDisks. +func (mr *MockClusterMgrAPIMockRecorder) ListBrokenDisks(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListBrokenDisks", reflect.TypeOf((*MockClusterMgrAPI)(nil).ListBrokenDisks), arg0, arg1) +} + +// ListClusterDisks mocks base method. +func (m *MockClusterMgrAPI) ListClusterDisks(arg0 context.Context) ([]*client.DiskInfoSimple, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListClusterDisks", arg0) + ret0, _ := ret[0].([]*client.DiskInfoSimple) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListClusterDisks indicates an expected call of ListClusterDisks. +func (mr *MockClusterMgrAPIMockRecorder) ListClusterDisks(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListClusterDisks", reflect.TypeOf((*MockClusterMgrAPI)(nil).ListClusterDisks), arg0) +} + +// ListDiskVolumeUnits mocks base method. +func (m *MockClusterMgrAPI) ListDiskVolumeUnits(arg0 context.Context, arg1 proto.DiskID) ([]*client.VunitInfoSimple, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListDiskVolumeUnits", arg0, arg1) + ret0, _ := ret[0].([]*client.VunitInfoSimple) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListDiskVolumeUnits indicates an expected call of ListDiskVolumeUnits. +func (mr *MockClusterMgrAPIMockRecorder) ListDiskVolumeUnits(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDiskVolumeUnits", reflect.TypeOf((*MockClusterMgrAPI)(nil).ListDiskVolumeUnits), arg0, arg1) +} + +// ListDropDisks mocks base method. +func (m *MockClusterMgrAPI) ListDropDisks(arg0 context.Context) ([]*client.DiskInfoSimple, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListDropDisks", arg0) + ret0, _ := ret[0].([]*client.DiskInfoSimple) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListDropDisks indicates an expected call of ListDropDisks. +func (mr *MockClusterMgrAPIMockRecorder) ListDropDisks(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDropDisks", reflect.TypeOf((*MockClusterMgrAPI)(nil).ListDropDisks), arg0) +} + +// ListRepairingDisks mocks base method. +func (m *MockClusterMgrAPI) ListRepairingDisks(arg0 context.Context) ([]*client.DiskInfoSimple, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListRepairingDisks", arg0) + ret0, _ := ret[0].([]*client.DiskInfoSimple) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRepairingDisks indicates an expected call of ListRepairingDisks. +func (mr *MockClusterMgrAPIMockRecorder) ListRepairingDisks(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRepairingDisks", reflect.TypeOf((*MockClusterMgrAPI)(nil).ListRepairingDisks), arg0) +} + +// ListVolume mocks base method. +func (m *MockClusterMgrAPI) ListVolume(arg0 context.Context, arg1 proto.Vid, arg2 int) ([]*client.VolumeInfoSimple, proto.Vid, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListVolume", arg0, arg1, arg2) + ret0, _ := ret[0].([]*client.VolumeInfoSimple) + ret1, _ := ret[1].(proto.Vid) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// ListVolume indicates an expected call of ListVolume. +func (mr *MockClusterMgrAPIMockRecorder) ListVolume(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListVolume", reflect.TypeOf((*MockClusterMgrAPI)(nil).ListVolume), arg0, arg1, arg2) +} + +// LockVolume mocks base method. +func (m *MockClusterMgrAPI) LockVolume(arg0 context.Context, arg1 proto.Vid) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LockVolume", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// LockVolume indicates an expected call of LockVolume. +func (mr *MockClusterMgrAPIMockRecorder) LockVolume(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LockVolume", reflect.TypeOf((*MockClusterMgrAPI)(nil).LockVolume), arg0, arg1) +} + +// Register mocks base method. +func (m *MockClusterMgrAPI) Register(arg0 context.Context, arg1 client.RegisterInfo) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Register", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Register indicates an expected call of Register. +func (mr *MockClusterMgrAPIMockRecorder) Register(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Register", reflect.TypeOf((*MockClusterMgrAPI)(nil).Register), arg0, arg1) +} + +// ReleaseVolumeUnit mocks base method. +func (m *MockClusterMgrAPI) ReleaseVolumeUnit(arg0 context.Context, arg1 proto.Vuid, arg2 proto.DiskID) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReleaseVolumeUnit", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// ReleaseVolumeUnit indicates an expected call of ReleaseVolumeUnit. +func (mr *MockClusterMgrAPIMockRecorder) ReleaseVolumeUnit(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReleaseVolumeUnit", reflect.TypeOf((*MockClusterMgrAPI)(nil).ReleaseVolumeUnit), arg0, arg1, arg2) +} + +// SetDiskDropped mocks base method. +func (m *MockClusterMgrAPI) SetDiskDropped(arg0 context.Context, arg1 proto.DiskID) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetDiskDropped", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetDiskDropped indicates an expected call of SetDiskDropped. +func (mr *MockClusterMgrAPIMockRecorder) SetDiskDropped(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDiskDropped", reflect.TypeOf((*MockClusterMgrAPI)(nil).SetDiskDropped), arg0, arg1) +} + +// SetDiskRepaired mocks base method. +func (m *MockClusterMgrAPI) SetDiskRepaired(arg0 context.Context, arg1 proto.DiskID) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetDiskRepaired", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetDiskRepaired indicates an expected call of SetDiskRepaired. +func (mr *MockClusterMgrAPIMockRecorder) SetDiskRepaired(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDiskRepaired", reflect.TypeOf((*MockClusterMgrAPI)(nil).SetDiskRepaired), arg0, arg1) +} + +// SetDiskRepairing mocks base method. +func (m *MockClusterMgrAPI) SetDiskRepairing(arg0 context.Context, arg1 proto.DiskID) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetDiskRepairing", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetDiskRepairing indicates an expected call of SetDiskRepairing. +func (mr *MockClusterMgrAPIMockRecorder) SetDiskRepairing(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDiskRepairing", reflect.TypeOf((*MockClusterMgrAPI)(nil).SetDiskRepairing), arg0, arg1) +} + +// UnlockVolume mocks base method. +func (m *MockClusterMgrAPI) UnlockVolume(arg0 context.Context, arg1 proto.Vid) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UnlockVolume", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// UnlockVolume indicates an expected call of UnlockVolume. +func (mr *MockClusterMgrAPIMockRecorder) UnlockVolume(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnlockVolume", reflect.TypeOf((*MockClusterMgrAPI)(nil).UnlockVolume), arg0, arg1) +} + +// UpdateVolume mocks base method. +func (m *MockClusterMgrAPI) UpdateVolume(arg0 context.Context, arg1, arg2 proto.Vuid, arg3 proto.DiskID) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateVolume", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateVolume indicates an expected call of UpdateVolume. +func (mr *MockClusterMgrAPIMockRecorder) UpdateVolume(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateVolume", reflect.TypeOf((*MockClusterMgrAPI)(nil).UpdateVolume), arg0, arg1, arg2, arg3) +} + +// MockBlobnodeAPI is a mock of BlobnodeAPI interface. +type MockBlobnodeAPI struct { + ctrl *gomock.Controller + recorder *MockBlobnodeAPIMockRecorder +} + +// MockBlobnodeAPIMockRecorder is the mock recorder for MockBlobnodeAPI. +type MockBlobnodeAPIMockRecorder struct { + mock *MockBlobnodeAPI +} + +// NewMockBlobnodeAPI creates a new mock instance. +func NewMockBlobnodeAPI(ctrl *gomock.Controller) *MockBlobnodeAPI { + mock := &MockBlobnodeAPI{ctrl: ctrl} + mock.recorder = &MockBlobnodeAPIMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockBlobnodeAPI) EXPECT() *MockBlobnodeAPIMockRecorder { + return m.recorder +} + +// Delete mocks base method. +func (m *MockBlobnodeAPI) Delete(arg0 context.Context, arg1 proto.VunitLocation, arg2 proto.BlobID) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockBlobnodeAPIMockRecorder) Delete(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockBlobnodeAPI)(nil).Delete), arg0, arg1, arg2) +} + +// MarkDelete mocks base method. +func (m *MockBlobnodeAPI) MarkDelete(arg0 context.Context, arg1 proto.VunitLocation, arg2 proto.BlobID) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MarkDelete", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// MarkDelete indicates an expected call of MarkDelete. +func (mr *MockBlobnodeAPIMockRecorder) MarkDelete(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkDelete", reflect.TypeOf((*MockBlobnodeAPI)(nil).MarkDelete), arg0, arg1, arg2) +} + +// RepairShard mocks base method. +func (m *MockBlobnodeAPI) RepairShard(arg0 context.Context, arg1 string, arg2 proto.ShardRepairTask) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RepairShard", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// RepairShard indicates an expected call of RepairShard. +func (mr *MockBlobnodeAPIMockRecorder) RepairShard(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RepairShard", reflect.TypeOf((*MockBlobnodeAPI)(nil).RepairShard), arg0, arg1, arg2) +} + +// MockVolumeUpdater is a mock of IVolumeUpdater interface. +type MockVolumeUpdater struct { + ctrl *gomock.Controller + recorder *MockVolumeUpdaterMockRecorder +} + +// MockVolumeUpdaterMockRecorder is the mock recorder for MockVolumeUpdater. +type MockVolumeUpdaterMockRecorder struct { + mock *MockVolumeUpdater +} + +// NewMockVolumeUpdater creates a new mock instance. +func NewMockVolumeUpdater(ctrl *gomock.Controller) *MockVolumeUpdater { + mock := &MockVolumeUpdater{ctrl: ctrl} + mock.recorder = &MockVolumeUpdaterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockVolumeUpdater) EXPECT() *MockVolumeUpdaterMockRecorder { + return m.recorder +} + +// UpdateFollowerVolumeCache mocks base method. +func (m *MockVolumeUpdater) UpdateFollowerVolumeCache(arg0 context.Context, arg1 string, arg2 proto.Vid) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateFollowerVolumeCache", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateFollowerVolumeCache indicates an expected call of UpdateFollowerVolumeCache. +func (mr *MockVolumeUpdaterMockRecorder) UpdateFollowerVolumeCache(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateFollowerVolumeCache", reflect.TypeOf((*MockVolumeUpdater)(nil).UpdateFollowerVolumeCache), arg0, arg1, arg2) +} + +// UpdateLeaderVolumeCache mocks base method. +func (m *MockVolumeUpdater) UpdateLeaderVolumeCache(arg0 context.Context, arg1 proto.Vid) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateLeaderVolumeCache", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateLeaderVolumeCache indicates an expected call of UpdateLeaderVolumeCache. +func (mr *MockVolumeUpdaterMockRecorder) UpdateLeaderVolumeCache(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateLeaderVolumeCache", reflect.TypeOf((*MockVolumeUpdater)(nil).UpdateLeaderVolumeCache), arg0, arg1) +} + +// MockMqProxyAPI is a mock of ProxyAPI interface. +type MockMqProxyAPI struct { + ctrl *gomock.Controller + recorder *MockMqProxyAPIMockRecorder +} + +// MockMqProxyAPIMockRecorder is the mock recorder for MockMqProxyAPI. +type MockMqProxyAPIMockRecorder struct { + mock *MockMqProxyAPI +} + +// NewMockMqProxyAPI creates a new mock instance. +func NewMockMqProxyAPI(ctrl *gomock.Controller) *MockMqProxyAPI { + mock := &MockMqProxyAPI{ctrl: ctrl} + mock.recorder = &MockMqProxyAPIMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockMqProxyAPI) EXPECT() *MockMqProxyAPIMockRecorder { + return m.recorder +} + +// SendShardRepairMsg mocks base method. +func (m *MockMqProxyAPI) SendShardRepairMsg(arg0 context.Context, arg1 proto.Vid, arg2 proto.BlobID, arg3 []byte) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendShardRepairMsg", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendShardRepairMsg indicates an expected call of SendShardRepairMsg. +func (mr *MockMqProxyAPIMockRecorder) SendShardRepairMsg(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendShardRepairMsg", reflect.TypeOf((*MockMqProxyAPI)(nil).SendShardRepairMsg), arg0, arg1, arg2, arg3) +} diff --git a/blobstore/scheduler/cluster_topology.go b/blobstore/scheduler/cluster_topology.go new file mode 100644 index 000000000..9586b44eb --- /dev/null +++ b/blobstore/scheduler/cluster_topology.go @@ -0,0 +1,238 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "sort" + "time" + + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/scheduler/base" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/util/closer" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +// IClusterTopology define the interface og cluster topology +type IClusterTopology interface { + GetIDCs() map[string]*IDC + GetIDCDisks(idc string) (disks []*client.DiskInfoSimple) + closer.Closer +} + +type clusterTopoConf struct { + ClusterID proto.ClusterID + UpdateInterval time.Duration + FreeChunkCounterBuckets []float64 +} + +// ClusterTopology cluster topology +type ClusterTopology struct { + clusterID proto.ClusterID + idcMap map[string]*IDC + diskMap map[string][]*client.DiskInfoSimple + FreeChunkCnt int64 + MaxChunkCnt int64 +} + +// IDC idc info +type IDC struct { + name string + rackMap map[string]*Rack + FreeChunkCnt int64 + MaxChunkCnt int64 +} + +// Rack rack info +type Rack struct { + name string + diskMap map[string]*Host + FreeChunkCnt int64 + MaxChunkCnt int64 +} + +// Host host info +type Host struct { + host string // ip+port + disks []*client.DiskInfoSimple // disk list + FreeChunkCnt int64 // host free chunk count + MaxChunkCnt int64 // total chunk count +} + +// ClusterTopologyMgr cluster topology manager +type ClusterTopologyMgr struct { + closer.Closer + updateInterval time.Duration + clusterID proto.ClusterID + + clusterMgrCli client.ClusterMgrAPI + + clusterTopo *ClusterTopology + taskStatsMgr *base.ClusterTopologyStatsMgr +} + +// NewClusterTopologyMgr returns cluster topology manager +func NewClusterTopologyMgr(topologyClient client.ClusterMgrAPI, conf *clusterTopoConf) IClusterTopology { + mgr := &ClusterTopologyMgr{ + Closer: closer.New(), + updateInterval: conf.UpdateInterval, + clusterID: conf.ClusterID, + + clusterMgrCli: topologyClient, + + clusterTopo: &ClusterTopology{ + idcMap: make(map[string]*IDC), + diskMap: make(map[string][]*client.DiskInfoSimple), + }, + taskStatsMgr: base.NewClusterTopologyStatisticsMgr(conf.ClusterID, conf.FreeChunkCounterBuckets), + } + go mgr.loopUpdate() + return mgr +} + +func (m *ClusterTopologyMgr) loopUpdate() { + t := time.NewTicker(m.updateInterval) + defer t.Stop() + + for { + select { + case <-t.C: + m.updateClusterTopology() + case <-m.Closer.Done(): + return + } + } +} + +func (m *ClusterTopologyMgr) updateClusterTopology() { + span, ctx := trace.StartSpanFromContext(context.Background(), "updateClusterTopology") + + disks, err := m.clusterMgrCli.ListClusterDisks(ctx) + if err != nil { + span.Errorf("update cluster topology failed: err[%+v]", err) + } + + m.buildClusterTopo(disks, m.clusterID) +} + +// GetIDCs returns IDCs +func (m *ClusterTopologyMgr) GetIDCs() map[string]*IDC { + return m.clusterTopo.idcMap +} + +// GetIDCDisks returns disks with IDC +func (m *ClusterTopologyMgr) GetIDCDisks(idc string) (disks []*client.DiskInfoSimple) { + return m.clusterTopo.diskMap[idc] +} + +// ReportFreeChunkCnt report free chunk cnt +func (m *ClusterTopologyMgr) ReportFreeChunkCnt(disk *client.DiskInfoSimple) { + m.taskStatsMgr.ReportFreeChunk(disk) +} + +func (m *ClusterTopologyMgr) buildClusterTopo(disks []*client.DiskInfoSimple, clusterID proto.ClusterID) { + cluster := &ClusterTopology{ + clusterID: clusterID, + idcMap: make(map[string]*IDC), + diskMap: make(map[string][]*client.DiskInfoSimple), + } + + for i := range disks { + if cluster.clusterID != disks[i].ClusterID { + log.Errorf("the disk does not belong to this cluster: cluster_id[%d], disk[%+v]", cluster.clusterID, disks[i]) + continue + } + cluster.addDisk(disks[i]) + m.ReportFreeChunkCnt(disks[i]) + } + + for idc := range cluster.diskMap { + sortDiskByFreeChunkCnt(cluster.diskMap[idc]) + } + m.clusterTopo = cluster +} + +func (cluster *ClusterTopology) addDisk(disk *client.DiskInfoSimple) { + cluster.addDiskToCluster(disk) + cluster.addDiskToDiskMap(disk) + cluster.addDiskToIdc(disk) + cluster.addDiskToRack(disk) + cluster.addDiskToHost(disk) +} + +func (cluster *ClusterTopology) addDiskToCluster(disk *client.DiskInfoSimple) { + // statistics cluster chunk info + cluster.FreeChunkCnt += disk.FreeChunkCnt + cluster.MaxChunkCnt += disk.MaxChunkCnt +} + +func (cluster *ClusterTopology) addDiskToDiskMap(disk *client.DiskInfoSimple) { + if _, ok := cluster.diskMap[disk.Idc]; !ok { + var disks []*client.DiskInfoSimple + cluster.diskMap[disk.Idc] = disks + } + cluster.diskMap[disk.Idc] = append(cluster.diskMap[disk.Idc], disk) +} + +func (cluster *ClusterTopology) addDiskToIdc(disk *client.DiskInfoSimple) { + idcName := disk.Idc + if _, ok := cluster.idcMap[idcName]; !ok { + cluster.idcMap[idcName] = &IDC{ + name: idcName, + rackMap: make(map[string]*Rack), + } + } + // statistics idc chunk info + cluster.idcMap[idcName].FreeChunkCnt += disk.FreeChunkCnt + cluster.idcMap[idcName].MaxChunkCnt += disk.MaxChunkCnt +} + +func (cluster *ClusterTopology) addDiskToRack(disk *client.DiskInfoSimple) { + idc := cluster.idcMap[disk.Idc] + rackName := disk.Rack + if _, ok := idc.rackMap[rackName]; !ok { + idc.rackMap[rackName] = &Rack{ + name: rackName, + diskMap: make(map[string]*Host), + } + } + // statistics rack chunk info + idc.rackMap[rackName].FreeChunkCnt += disk.FreeChunkCnt + idc.rackMap[rackName].MaxChunkCnt += disk.MaxChunkCnt +} + +func (cluster *ClusterTopology) addDiskToHost(disk *client.DiskInfoSimple) { + rack := cluster.idcMap[disk.Idc].rackMap[disk.Rack] + if _, ok := rack.diskMap[disk.Host]; !ok { + var disks []*client.DiskInfoSimple + rack.diskMap[disk.Host] = &Host{ + host: disk.Host, + disks: disks, + } + } + rack.diskMap[disk.Host].disks = append(rack.diskMap[disk.Host].disks, disk) + + // statistics host chunk info + rack.diskMap[disk.Host].FreeChunkCnt += disk.FreeChunkCnt + rack.diskMap[disk.Host].MaxChunkCnt += disk.MaxChunkCnt +} + +func sortDiskByFreeChunkCnt(disks []*client.DiskInfoSimple) { + sort.Slice(disks, func(i, j int) bool { + return disks[i].FreeChunkCnt < disks[j].FreeChunkCnt + }) +} diff --git a/blobstore/scheduler/cluster_topology_test.go b/blobstore/scheduler/cluster_topology_test.go new file mode 100644 index 000000000..c29ef759c --- /dev/null +++ b/blobstore/scheduler/cluster_topology_test.go @@ -0,0 +1,114 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/scheduler/base" + "github.com/cubefs/cubefs/blobstore/scheduler/client" +) + +var ( + topoDisk1 = &client.DiskInfoSimple{ + ClusterID: 1, + Idc: "z0", + Rack: "rack1", + Host: "127.0.0.1:8000", + DiskID: 1, + FreeChunkCnt: 10, + MaxChunkCnt: 700, + } + topoDisk2 = &client.DiskInfoSimple{ + ClusterID: 1, + Idc: "z0", + Rack: "rack1", + Host: "127.0.0.2:8000", + DiskID: 2, + FreeChunkCnt: 100, + MaxChunkCnt: 700, + } + topoDisk3 = &client.DiskInfoSimple{ + ClusterID: 1, + Idc: "z1", + Rack: "rack1", + Host: "127.0.0.3:8000", + DiskID: 3, + FreeChunkCnt: 20, + MaxChunkCnt: 700, + } + topoDisk4 = &client.DiskInfoSimple{ + ClusterID: 1, + Idc: "z1", + Rack: "rack2", + Host: "127.0.0.4:8000", + DiskID: 4, + FreeChunkCnt: 5, + MaxChunkCnt: 700, + } + topoDisk5 = &client.DiskInfoSimple{ + ClusterID: 1, + Idc: "z2", + Rack: "rack2", + Host: "127.0.0.4:8000", + DiskID: 5, + FreeChunkCnt: 200, + MaxChunkCnt: 700, + } + topoDisk6 = &client.DiskInfoSimple{ + ClusterID: 123, + Idc: "z2", + Rack: "rack2", + Host: "127.0.0.4:8000", + DiskID: 5, + FreeChunkCnt: 200, + MaxChunkCnt: 700, + } + + topoDisks = []*client.DiskInfoSimple{topoDisk1, topoDisk2, topoDisk3, topoDisk4, topoDisk5, topoDisk6} +) + +func TestNewClusterTopologyMgr(t *testing.T) { + clusterTopMgr := &ClusterTopologyMgr{ + taskStatsMgr: base.NewClusterTopologyStatisticsMgr(1, []float64{}), + } + clusterTopMgr.buildClusterTopo(topoDisks, 1) + require.Equal(t, 3, len(clusterTopMgr.GetIDCs())) + disks := clusterTopMgr.GetIDCDisks("z0") + require.Equal(t, 2, len(disks)) + disks = clusterTopMgr.GetIDCDisks("z1") + require.Equal(t, 2, len(disks)) + disks = clusterTopMgr.GetIDCDisks("z2") + require.Equal(t, 1, len(disks)) + disks = clusterTopMgr.GetIDCDisks("z3") + require.True(t, disks == nil) + + ctr := gomock.NewController(t) + clusterMgrCli := NewMockClusterMgrAPI(ctr) + clusterMgrCli.EXPECT().ListClusterDisks(any).AnyTimes().Return(nil, errMock) + conf := &clusterTopoConf{ + ClusterID: 1, + UpdateInterval: 1 * time.Microsecond, + } + mgr := NewClusterTopologyMgr(clusterMgrCli, conf) + defer mgr.Close() + + // wait topology update + time.Sleep(2 * time.Microsecond) +} diff --git a/blobstore/scheduler/config.go b/blobstore/scheduler/config.go new file mode 100644 index 000000000..549be329f --- /dev/null +++ b/blobstore/scheduler/config.go @@ -0,0 +1,306 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "github.com/cubefs/cubefs/blobstore/api/blobnode" + "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/api/proxy" + "github.com/cubefs/cubefs/blobstore/cmd" + "github.com/cubefs/cubefs/blobstore/common/mongoutil" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/scheduler/db" + "github.com/cubefs/cubefs/blobstore/util/defaulter" +) + +const ( + defaultTopologyUpdateIntervalMin = 1 + defaultVolumeCacheUpdateIntervalS = 10 + defaultArchiveDelayMin = 5 + defaultArchiveIntervalMin = 5 + defaultRetryHostsCnt = 1 + defaultClientTimeoutMs = int64(1000) + defaultHostSyncIntervalMs = int64(1000) + defaultMongoTimeoutMs = int64(3000) + + defaultBalanceDiskCntLimit = 100 + defaultMaxDiskFreeChunkCnt = int64(1024) + defaultMinDiskFreeChunkCnt = int64(20) + + defaultInspectIntervalS = 1 + defaultListVolIntervalMs = 10 + defaultListVolStep = 100 + defaultInspectBatch = 1000 + defaultInspectTimeoutMs = 10000 + + defaultTaskPoolSize = 10 + defaultHandleBatchCnt = 100 + defaultFailMsgConsumeIntervalMs = int64(10000) + defaultDeleteLogChunkSize = uint(29) + defaultDeleteDelayH = int64(72) + defaultDeleteNoDelay = int64(0) + + defaultTickInterval = uint32(1) + defaultHeartbeatTicks = uint32(30) + defaultExpiresTicks = uint32(60) + + defaultDatabase = "scheduler" + defaultBalanceTable = "balance_tbl" + defaultDiskDropTable = "disk_drop_tbl" + defaultRepairTable = "repair_tbl" + defaultInspectCheckPointTable = "inspect_checkpoint_tbl" + defaultManualMigrateTable = "manual_migrate_tbl" + defaultArchiveTasksTable = "archive_tasks_tbl" + defaultKafkaOffsetTable = "kafka_offset_tbl" + defaultOrphanedShardTable = "orphaned_shard_tbl" + + defaultShardRepairNormalTopic = "shard_repair" + defaultShardRepairPriorityTopic = "shard_repair_prior" + defaultShardRepairFailedTopic = "shard_repair_failed" + + defaultBlobDeleteNormalTopic = "blob_delete" + defaultBlobDeleteFailedTopic = "blob_delete_failed" +) + +var defaultWriteConfig = mongoutil.DefaultWriteConfig + +// Config service config +type Config struct { + cmd.Config + + ClusterID proto.ClusterID `json:"cluster_id"` + Services Services `json:"services"` + + Database db.Config `json:"database"` + TopologyUpdateIntervalMin int `json:"topology_update_interval_min"` + VolumeCacheUpdateIntervalS int `json:"volume_cache_update_interval_s"` + FreeChunkCounterBuckets []float64 `json:"free_chunk_counter_buckets"` + + ClusterMgr clustermgr.Config `json:"clustermgr"` + Proxy proxy.LbConfig `json:"proxy"` + Blobnode blobnode.Config `json:"blobnode"` + Scheduler rpc.Config `json:"scheduler"` + + Balance BalanceMgrConfig `json:"balance"` + DiskDrop DiskDropMgrConfig `json:"disk_drop"` + DiskRepair DiskRepairMgrCfg `json:"disk_repair"` + VolumeInspect VolumeInspectMgrCfg `json:"volume_inspect"` + Archive ArchiveStoreConfig `json:"archive"` + + Kafka KafkaConfig `json:"kafka"` + ShardRepair ShardRepairConfig `json:"shard_repair"` + BlobDelete BlobDeleteConfig `json:"blob_delete"` + + ServiceRegister ServiceRegisterConfig `json:"service_register"` +} + +// ServiceRegisterConfig is service register info +type ServiceRegisterConfig struct { + TickInterval uint32 `json:"tick_interval"` + HeartbeatTicks uint32 `json:"heartbeat_ticks"` + ExpiresTicks uint32 `json:"expires_ticks"` + Idc string `json:"idc"` + Host string `json:"host"` +} + +// ShardRepairKafkaConfig is kafka config of shard repair +type ShardRepairKafkaConfig struct { + BrokerList []string `json:"-"` + FailMsgSenderTimeoutMs int64 `json:"-"` + Normal TopicConfig `json:"normal"` + Failed TopicConfig `json:"failed"` + Priority TopicConfig `json:"priority"` +} + +// BlobDeleteKafkaConfig is kafka config of blob delete +type BlobDeleteKafkaConfig struct { + BrokerList []string `json:"-"` + FailMsgSenderTimeoutMs int64 `json:"-"` + Normal TopicConfig `json:"normal"` + Failed TopicConfig `json:"failed"` +} + +// KafkaConfig kafka config +type KafkaConfig struct { + BrokerList []string `json:"broker_list"` + FailMsgSenderTimeoutMs int64 `json:"fail_msg_sender_timeout_ms"` + ShardRepair ShardRepairKafkaConfig `json:"shard_repair"` + BlobDelete BlobDeleteKafkaConfig `json:"blob_delete"` +} + +// TopicConfig topic config +type TopicConfig struct { + Topic string `json:"topic"` + Partitions []int32 `json:"partitions"` +} + +type Services struct { + Leader uint64 `json:"leader"` + NodeID uint64 `json:"node_id"` + Members map[uint64]string `json:"members"` +} + +func (c *Config) IsLeader() bool { + return c.Services.Leader == c.Services.NodeID +} + +func (c *Config) Leader() string { + return c.Services.Members[c.Services.Leader] +} + +func (c *Config) Follower() []string { + var followers []string + for k, v := range c.Services.Members { + if k != c.Services.Leader { + followers = append(followers, v) + } + } + return followers +} + +func (c *Config) fixServices() error { + if len(c.Services.Members) < 1 { + return errInvalidMembers + } + if _, ok := c.Services.Members[c.Services.Leader]; !ok { + return errInvalidLeader + } + if _, ok := c.Services.Members[c.Services.NodeID]; !ok { + return errInvalidNodeID + } + return nil +} + +func (c *Config) fixConfig() (err error) { + if c.ClusterID == 0 { + return errIllegalClusterID + } + if err := c.fixServices(); err != nil { + return err + } + defaulter.LessOrEqual(&c.TopologyUpdateIntervalMin, defaultTopologyUpdateIntervalMin) + defaulter.LessOrEqual(&c.VolumeCacheUpdateIntervalS, defaultVolumeCacheUpdateIntervalS) + c.fixClientConfig() + c.fixDataBaseConfig() + c.fixKafkaConfig() + c.fixBalanceConfig() + c.fixDiskDropConfig() + c.fixRepairConfig() + c.fixInspectConfig() + c.fixShardRepairConfig() + c.fixBlobDeleteConfig() + c.fixArchiveStoreConfig() + c.fixRegisterConfig() + return nil +} + +func (c *Config) fixClientConfig() { + defaulter.LessOrEqual(&c.Proxy.ClientTimeoutMs, defaultClientTimeoutMs) + defaulter.LessOrEqual(&c.Proxy.HostSyncIntervalMs, defaultHostSyncIntervalMs) + defaulter.LessOrEqual(&c.Proxy.RetryHostsCnt, defaultRetryHostsCnt) + defaulter.LessOrEqual(&c.Blobnode.ClientTimeoutMs, defaultClientTimeoutMs) + defaulter.LessOrEqual(&c.Scheduler.ClientTimeoutMs, defaultClientTimeoutMs) +} + +func (c *Config) fixDataBaseConfig() { + if c.Database.Mongo.WriteConcern == nil { + c.Database.Mongo.WriteConcern = &defaultWriteConfig + } + defaulter.LessOrEqual(&c.Database.Mongo.TimeoutMs, defaultMongoTimeoutMs) + defaulter.Empty(&c.Database.DBName, defaultDatabase) + defaulter.Empty(&c.Database.BalanceTable, defaultBalanceTable) + defaulter.Empty(&c.Database.DiskDropTable, defaultDiskDropTable) + defaulter.Empty(&c.Database.RepairTable, defaultRepairTable) + defaulter.Empty(&c.Database.InspectCheckPointTable, defaultInspectCheckPointTable) + defaulter.Empty(&c.Database.ManualMigrateTable, defaultManualMigrateTable) + defaulter.Empty(&c.Database.ArchiveTasksTable, defaultArchiveTasksTable) + defaulter.Empty(&c.Database.KafkaOffsetTable, defaultKafkaOffsetTable) + defaulter.Empty(&c.Database.OrphanShardTable, defaultOrphanedShardTable) +} + +func (c *Config) fixKafkaConfig() { + defaulter.Empty(&c.Kafka.BlobDelete.Normal.Topic, defaultBlobDeleteNormalTopic) + defaulter.Empty(&c.Kafka.BlobDelete.Failed.Topic, defaultBlobDeleteFailedTopic) + defaulter.Empty(&c.Kafka.ShardRepair.Normal.Topic, defaultShardRepairNormalTopic) + defaulter.Empty(&c.Kafka.ShardRepair.Priority.Topic, defaultShardRepairPriorityTopic) + defaulter.Empty(&c.Kafka.ShardRepair.Failed.Topic, defaultShardRepairFailedTopic) + defaulter.LessOrEqual(&c.Kafka.FailMsgSenderTimeoutMs, defaultClientTimeoutMs) + c.BlobDelete.Kafka.FailMsgSenderTimeoutMs = c.Kafka.FailMsgSenderTimeoutMs + c.ShardRepair.Kafka.FailMsgSenderTimeoutMs = c.Kafka.FailMsgSenderTimeoutMs + c.Kafka.ShardRepair.BrokerList = c.Kafka.BrokerList + c.Kafka.BlobDelete.BrokerList = c.Kafka.BrokerList +} + +func (c *Config) fixRepairConfig() { + c.DiskRepair.ClusterID = c.ClusterID + c.DiskRepair.CheckAndFix() +} + +func (c *Config) fixBalanceConfig() { + c.Balance.ClusterID = c.ClusterID + defaulter.LessOrEqual(&c.Balance.BalanceDiskCntLimit, defaultBalanceDiskCntLimit) + defaulter.LessOrEqual(&c.Balance.MaxDiskFreeChunkCnt, defaultMaxDiskFreeChunkCnt) + defaulter.LessOrEqual(&c.Balance.MinDiskFreeChunkCnt, defaultMinDiskFreeChunkCnt) + c.Balance.CheckAndFix() +} + +func (c *Config) fixDiskDropConfig() { + c.DiskDrop.ClusterID = c.ClusterID + c.DiskDrop.CheckAndFix() +} + +func (c *Config) fixInspectConfig() { + defaulter.LessOrEqual(&c.VolumeInspect.TimeoutMs, defaultInspectTimeoutMs) + defaulter.LessOrEqual(&c.VolumeInspect.ListVolStep, defaultListVolStep) + defaulter.LessOrEqual(&c.VolumeInspect.ListVolIntervalMs, defaultListVolIntervalMs) + defaulter.LessOrEqual(&c.VolumeInspect.InspectBatch, defaultInspectBatch) + if c.VolumeInspect.InspectBatch < c.VolumeInspect.ListVolStep { + c.VolumeInspect.InspectBatch = c.VolumeInspect.ListVolStep + } + defaulter.LessOrEqual(&c.VolumeInspect.InspectIntervalS, defaultInspectIntervalS) +} + +func (c *Config) fixShardRepairConfig() { + c.ShardRepair.ClusterID = c.ClusterID + defaulter.LessOrEqual(&c.ShardRepair.TaskPoolSize, defaultTaskPoolSize) + defaulter.LessOrEqual(&c.ShardRepair.NormalHandleBatchCnt, defaultHandleBatchCnt) + defaulter.LessOrEqual(&c.ShardRepair.FailHandleBatchCnt, defaultHandleBatchCnt) + defaulter.LessOrEqual(&c.ShardRepair.FailMsgConsumeIntervalMs, defaultFailMsgConsumeIntervalMs) + c.ShardRepair.Kafka = c.Kafka.ShardRepair +} + +func (c *Config) fixBlobDeleteConfig() { + c.BlobDelete.ClusterID = c.ClusterID + defaulter.LessOrEqual(&c.BlobDelete.TaskPoolSize, defaultTaskPoolSize) + defaulter.LessOrEqual(&c.BlobDelete.NormalHandleBatchCnt, defaultHandleBatchCnt) + defaulter.LessOrEqual(&c.BlobDelete.FailHandleBatchCnt, defaultHandleBatchCnt) + defaulter.LessOrEqual(&c.BlobDelete.FailMsgConsumeIntervalMs, defaultFailMsgConsumeIntervalMs) + defaulter.LessOrEqual(&c.BlobDelete.DeleteLog.ChunkBits, defaultDeleteLogChunkSize) + defaulter.Equal(&c.BlobDelete.SafeDelayTimeH, defaultDeleteDelayH) + defaulter.Less(&c.BlobDelete.SafeDelayTimeH, defaultDeleteNoDelay) + c.BlobDelete.Kafka = c.Kafka.BlobDelete +} + +func (c *Config) fixArchiveStoreConfig() { + defaulter.LessOrEqual(&c.Archive.ArchiveDelayMin, defaultArchiveDelayMin) + defaulter.LessOrEqual(&c.Archive.ArchiveIntervalMin, defaultArchiveIntervalMin) +} + +func (c *Config) fixRegisterConfig() { + defaulter.LessOrEqual(&c.ServiceRegister.TickInterval, defaultTickInterval) + defaulter.LessOrEqual(&c.ServiceRegister.HeartbeatTicks, defaultHeartbeatTicks) + defaulter.LessOrEqual(&c.ServiceRegister.ExpiresTicks, defaultExpiresTicks) +} diff --git a/blobstore/scheduler/config_test.go b/blobstore/scheduler/config_test.go new file mode 100644 index 000000000..9637be133 --- /dev/null +++ b/blobstore/scheduler/config_test.go @@ -0,0 +1,55 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestConfigCheckAndFix(t *testing.T) { + cfg := &Config{} + err := cfg.fixConfig() + require.Error(t, err, errIllegalClusterID.Error()) + + cfg.ClusterID = 1 + err = cfg.fixConfig() + require.Error(t, err, errInvalidMembers) + + cfg.Services.Members = map[uint64]string{1: "127.0.0.1:9800"} + err = cfg.fixConfig() + require.Error(t, err, errInvalidLeader) + + cfg.Services.Leader = 1 + err = cfg.fixConfig() + require.Error(t, err, errInvalidNodeID) + + cfg.Services.NodeID = 1 + err = cfg.fixConfig() + require.NoError(t, err) + require.True(t, cfg.IsLeader()) + require.Equal(t, "127.0.0.1:9800", cfg.Leader()) + require.Nil(t, cfg.Follower()) + require.Equal(t, defaultDeleteDelayH, cfg.BlobDelete.SafeDelayTimeH) + cfg.Services.Members[2] = "127.0.0.1:9880" + require.Equal(t, "127.0.0.1:9880", cfg.Follower()[0]) + + cfg.Services.NodeID = 1 + cfg.BlobDelete.SafeDelayTimeH = -1 + err = cfg.fixConfig() + require.NoError(t, err) + require.Equal(t, defaultDeleteNoDelay, cfg.BlobDelete.SafeDelayTimeH) +} diff --git a/blobstore/scheduler/db/database.go b/blobstore/scheduler/db/database.go new file mode 100644 index 000000000..8741f706e --- /dev/null +++ b/blobstore/scheduler/db/database.go @@ -0,0 +1,127 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package db + +import ( + "context" + "fmt" + "strings" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/x/bsonx" + + "github.com/cubefs/cubefs/blobstore/common/mongoutil" + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +const ( + // deleteMark for mark delete + deleteMark = "delete_mark" +) + +// Config mongo config +type Config struct { + Mongo mongoutil.Config `json:"mongo"` + DBName string `json:"db_name"` + + BalanceTable string `json:"balance_table"` + DiskDropTable string `json:"disk_drop_table"` + ManualMigrateTable string `json:"manual_migrate_table"` + RepairTable string `json:"repair_table"` + InspectCheckPointTable string `json:"inspect_checkpoint_table"` + OrphanShardTable string `json:"orphaned_shard_table"` + KafkaOffsetTable string `json:"kafka_offset_table"` + + ArchiveTasksTable string `json:"archive_tasks_table"` +} + +// Database used for database operate +type Database struct { + DB *mongo.Database + + BalanceTable IMigrateTaskTable + DiskDropTable IMigrateTaskTable + ManualMigrateTable IMigrateTaskTable + RepairTaskTable IRepairTaskTable + + KafkaOffsetTable IKafkaOffsetTable + OrphanShardTable IOrphanShardTable + + InspectCheckPointTable IInspectCheckPointTable + + ArchiveTable IArchiveTable +} + +// OpenDatabase open database +func OpenDatabase(conf *Config) (tables *Database, err error) { + client, err := mongoutil.GetClient(conf.Mongo) + if err != nil { + return nil, err + } + db := client.Database(conf.DBName) + tables = &Database{DB: db} + + if tables.BalanceTable, err = openMigrateTbl( + mustCreateCollection(db, conf.BalanceTable), + proto.BalanceTaskType); err != nil { + return nil, err + } + if tables.DiskDropTable, err = openMigrateTbl( + mustCreateCollection(db, conf.DiskDropTable), + proto.DiskDropTaskType); err != nil { + return nil, err + } + if tables.ManualMigrateTable, err = openMigrateTbl( + mustCreateCollection(db, conf.ManualMigrateTable), + proto.ManualMigrateType); err != nil { + return nil, err + } + if tables.RepairTaskTable, err = OpenRepairTaskTbl( + mustCreateCollection(db, conf.RepairTable), + proto.RepairTaskType); err != nil { + return nil, err + } + if tables.InspectCheckPointTable, err = OpenInspectCheckPointTbl( + mustCreateCollection(db, conf.InspectCheckPointTable)); err != nil { + return nil, err + } + if tables.ArchiveTable, err = openArchiveTbl( + mustCreateCollection(db, conf.ArchiveTasksTable)); err != nil { + return nil, err + } + tables.KafkaOffsetTable = openKafkaOffsetTable(mustCreateCollection(db, conf.KafkaOffsetTable)) + tables.OrphanShardTable = openOrphanedShardTable(mustCreateCollection(db, conf.OrphanShardTable)) + + return +} + +func mustCreateCollection(db *mongo.Database, collName string) *mongo.Collection { + err := db.RunCommand(context.Background(), bsonx.Doc{{Key: "create", Value: bsonx.String(collName)}}).Err() + if err == nil || strings.Contains(err.Error(), "already exists") { + return db.Collection(collName) + } + panic(fmt.Sprintf("create collection error: %v", err)) +} + +func deleteBson() bson.M { + return bson.M{"$set": bson.M{deleteMark: true, "del_time": time.Now().Unix()}} +} + +func inDelayTime(delTime int64, delayMin int) bool { + now := time.Now() + return now.Sub(time.Unix(delTime, 0)) <= time.Duration(delayMin)*time.Minute +} diff --git a/blobstore/scheduler/db/disk_repair_table.go b/blobstore/scheduler/db/disk_repair_table.go new file mode 100644 index 000000000..22c7dd085 --- /dev/null +++ b/blobstore/scheduler/db/disk_repair_table.go @@ -0,0 +1,151 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package db + +import ( + "context" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/trace" +) + +// IRepairTaskTable define the interface of db used by disk repair +type IRepairTaskTable interface { + Insert(ctx context.Context, t *proto.VolRepairTask) error + Update(ctx context.Context, t *proto.VolRepairTask) error + Find(ctx context.Context, taskID string) (task *proto.VolRepairTask, err error) + FindByDiskID(ctx context.Context, diskID proto.DiskID) (tasks []*proto.VolRepairTask, err error) + FindAll(ctx context.Context) (tasks []*proto.VolRepairTask, err error) + MarkDeleteByDiskID(ctx context.Context, diskID proto.DiskID) error + + IRecordSrcTbl +} + +// RepairTaskTable disk repair task table +type RepairTaskTbl struct { + coll *mongo.Collection + name string +} + +// OpenRepairTaskTbl open disk repair task table +func OpenRepairTaskTbl(coll *mongo.Collection, name string) (IRepairTaskTable, error) { + tbl := &RepairTaskTbl{ + coll: coll, + name: name, + } + return tbl, nil +} + +// Insert insert task +func (tbl *RepairTaskTbl) Insert(ctx context.Context, t *proto.VolRepairTask) error { + t.Ctime = time.Now().String() + t.MTime = t.Ctime + trace.SpanFromContextSafe(ctx).Debugf("insert task %+v", t) + _, err := tbl.coll.InsertOne(ctx, t) + return err +} + +// Update update task +func (tbl *RepairTaskTbl) Update(ctx context.Context, t *proto.VolRepairTask) error { + span := trace.SpanFromContextSafe(ctx) + span.Debugf("update repair task tbl task %+v", *t) + + t.MTime = time.Now().String() + return tbl.coll.FindOneAndReplace(ctx, bson.M{"_id": t.TaskID}, t).Err() +} + +// Find find task by taskID +func (tbl *RepairTaskTbl) Find(ctx context.Context, taskID string) (task *proto.VolRepairTask, err error) { + err = tbl.coll.FindOne(ctx, bson.M{"_id": taskID, deleteMark: bson.M{"$ne": true}}).Decode(&task) + return +} + +// FindByDiskID find task by diskID +func (tbl *RepairTaskTbl) FindByDiskID(ctx context.Context, diskID proto.DiskID) (tasks []*proto.VolRepairTask, err error) { + cursor, err := tbl.coll.Find(ctx, bson.M{"repair_disk_id": diskID, deleteMark: bson.M{"$ne": true}}) + if err != nil { + return nil, err + } + err = cursor.All(ctx, &tasks) + return tasks, err +} + +// FindAll return all tasks +func (tbl *RepairTaskTbl) FindAll(ctx context.Context) (tasks []*proto.VolRepairTask, err error) { + cursor, err := tbl.coll.Find(ctx, bson.M{deleteMark: bson.M{"$ne": true}}) + if err != nil { + return nil, err + } + err = cursor.All(ctx, &tasks) + return tasks, err +} + +// MarkDeleteByDiskID mark delete task by diskID +func (tbl *RepairTaskTbl) MarkDeleteByDiskID(ctx context.Context, diskID proto.DiskID) error { + span := trace.SpanFromContextSafe(ctx) + span.Debugf("mark delete by disk_id %d", diskID) + + _, err := tbl.coll.UpdateMany(ctx, bson.M{"repair_disk_id": diskID}, deleteBson()) + return err +} + +// QueryMarkDeleteTasks find mark delete tasks +func (tbl *RepairTaskTbl) QueryMarkDeleteTasks(ctx context.Context, delayMin int) (records []*proto.ArchiveRecord, err error) { + span := trace.SpanFromContextSafe(ctx) + + type VolRepairTaskEx struct { + proto.VolRepairTask `bson:",inline"` + DelTime int64 `bson:"del_time"` + } + var tasks []*VolRepairTaskEx + cursor, err := tbl.coll.Find(ctx, bson.M{deleteMark: true}) + if err != nil { + return nil, err + } + err = cursor.All(ctx, &tasks) + if err != nil { + return nil, err + } + + for _, task := range tasks { + if inDelayTime(task.DelTime, delayMin) { + span.Debugf("task_id %s is in delay time", task.TaskID) + continue + } + + r := &proto.ArchiveRecord{ + TaskID: task.TaskID, + TaskType: tbl.Name(), + Content: task, + } + records = append(records, r) + } + return records, nil +} + +// RemoveMarkDelete remove mark delete task by taskID +func (tbl *RepairTaskTbl) RemoveMarkDelete(ctx context.Context, taskID string) error { + _, err := tbl.coll.DeleteOne(ctx, bson.M{"_id": taskID, deleteMark: true}) + return err +} + +// Name return repair table name +func (tbl *RepairTaskTbl) Name() string { + return tbl.name +} diff --git a/blobstore/scheduler/db/inspect_table.go b/blobstore/scheduler/db/inspect_table.go new file mode 100644 index 000000000..7a9da88b2 --- /dev/null +++ b/blobstore/scheduler/db/inspect_table.go @@ -0,0 +1,67 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package db + +import ( + "context" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/trace" +) + +// record the current checkpoint of inspect which save min vid in last batch volumes +// service will start inspect worker from checkpoint when service start +const inspectID = "inspect_checkpoint" + +// IInspectCheckPointTable define the interface of db used by inspect +type IInspectCheckPointTable interface { + GetCheckPoint(ctx context.Context) (ck *proto.InspectCheckPoint, err error) + SaveCheckPoint(ctx context.Context, startVid proto.Vid) error +} + +// InspectCheckPointTable inspect check point table +type InspectCheckPointTbl struct { + coll *mongo.Collection +} + +// OpenInspectCheckPointTbl returns inspect check point table +func OpenInspectCheckPointTbl(coll *mongo.Collection) (IInspectCheckPointTable, error) { + return &InspectCheckPointTbl{ + coll: coll, + }, nil +} + +// GetCheckPoint returns check point +func (tbl *InspectCheckPointTbl) GetCheckPoint(ctx context.Context) (ck *proto.InspectCheckPoint, err error) { + err = tbl.coll.FindOne(ctx, bson.M{}).Decode(&ck) + return ck, err +} + +// SaveCheckPoint save check point +func (tbl *InspectCheckPointTbl) SaveCheckPoint(ctx context.Context, startVid proto.Vid) error { + ck := proto.InspectCheckPoint{ + Id: inspectID, + StartVid: startVid, + Ctime: time.Now().String(), + } + trace.SpanFromContextSafe(ctx).Infof("save checkpoint %+v", ck) + _, err := tbl.coll.ReplaceOne(ctx, bson.M{"_id": inspectID}, ck, options.Replace().SetUpsert(true)) + return err +} diff --git a/blobstore/scheduler/db/kafka_offset_table.go b/blobstore/scheduler/db/kafka_offset_table.go new file mode 100644 index 000000000..8872ef86e --- /dev/null +++ b/blobstore/scheduler/db/kafka_offset_table.go @@ -0,0 +1,62 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package db + +import ( + "context" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// IKafkaOffsetTable define interface of kafka offset table use by delete or repair message consume. +type IKafkaOffsetTable interface { + Set(topic string, partition int32, offset int64) error + Get(topic string, partition int32) (int64, error) +} + +type kafkaOffset struct { + Topic string `bson:"topic"` + Partition int32 `bson:"partition"` + Offset int64 `bson:"offset"` +} + +type kafkaOffsetTable struct { + coll *mongo.Collection +} + +func openKafkaOffsetTable(coll *mongo.Collection) IKafkaOffsetTable { + return &kafkaOffsetTable{coll: coll} +} + +func (t *kafkaOffsetTable) Set(topic string, partition int32, off int64) error { + info := kafkaOffset{Topic: topic, Partition: partition, Offset: off} + selector := bson.M{"topic": topic, "partition": partition} + + update := bson.M{ + "$set": info, + } + opts := options.Update().SetUpsert(true) + _, err := t.coll.UpdateOne(context.Background(), selector, update, opts) + return err +} + +func (t *kafkaOffsetTable) Get(topic string, partition int32) (int64, error) { + infos := kafkaOffset{} + selector := bson.M{"topic": topic, "partition": partition} + err := t.coll.FindOne(context.Background(), &selector).Decode(&infos) + return infos.Offset, err +} diff --git a/blobstore/scheduler/db/migrate_task_table.go b/blobstore/scheduler/db/migrate_task_table.go new file mode 100644 index 000000000..d420104ff --- /dev/null +++ b/blobstore/scheduler/db/migrate_task_table.go @@ -0,0 +1,197 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package db + +import ( + "context" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/x/bsonx" + + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/trace" +) + +// IMigrateTaskTable define the interface of db use by migrate +type IMigrateTaskTable interface { + Insert(ctx context.Context, task *proto.MigrateTask) error + Delete(ctx context.Context, taskID string) error + MarkDeleteByDiskID(ctx context.Context, diskID proto.DiskID) error + MarkDeleteByStates(ctx context.Context, states []proto.MigrateState) error + Update(ctx context.Context, oldState proto.MigrateState, task *proto.MigrateTask) error + Find(ctx context.Context, taskID string) (task *proto.MigrateTask, err error) + FindByDiskID(ctx context.Context, diskID proto.DiskID) (tasks []*proto.MigrateTask, err error) + FindAll(ctx context.Context) (tasks []*proto.MigrateTask, err error) + + IRecordSrcTbl +} + +// MigrateTaskTbl migrate table +type MigrateTaskTbl struct { + coll *mongo.Collection + name string +} + +// openMigrateTbl open migrate tables +func openMigrateTbl(coll *mongo.Collection, name string) (IMigrateTaskTable, error) { + tbl := &MigrateTaskTbl{ + coll: coll, + name: name, + } + err := tbl.createIndex() + return tbl, err +} + +func (tbl *MigrateTaskTbl) createIndex() error { + ctx := context.Background() + opts := options.CreateIndexes().SetMaxTime(10 * time.Second) + mods := []mongo.IndexModel{ + { + Keys: bsonx.Doc{{Key: "state", Value: bsonx.Int32(-1)}}, + Options: options.Index().SetName("_state_").SetBackground(true), + }, + { + Keys: bsonx.Doc{{Key: deleteMark, Value: bsonx.Int32(-1)}}, + Options: options.Index().SetName("_delete_mark_").SetBackground(true), + }, + { + Keys: bsonx.Doc{{Key: "source_disk_id", Value: bsonx.Int32(-1)}}, + Options: options.Index().SetName("_source_disk_id_").SetBackground(true), + }, + } + + _, err := tbl.coll.Indexes().CreateMany(ctx, mods, opts) + return err +} + +// Insert insert task to db +func (tbl *MigrateTaskTbl) Insert(ctx context.Context, task *proto.MigrateTask) error { + span := trace.SpanFromContextSafe(ctx) + span.Debugf("DB:insert task, taskId: %s, %+v", task.TaskID, task) + + task.Ctime = time.Now().String() + task.MTime = task.Ctime + _, err := tbl.coll.InsertOne(ctx, task) + return err +} + +// Update update task +func (tbl *MigrateTaskTbl) Update(ctx context.Context, oldState proto.MigrateState, task *proto.MigrateTask) error { + span := trace.SpanFromContextSafe(ctx) + span.Debugf("DB:update task, taskId: %s,state: %d", task.TaskID, task.State) + + task.MTime = time.Now().String() + + states := []proto.MigrateState{oldState, task.State} + return tbl.coll.FindOneAndReplace(ctx, bson.M{"_id": task.TaskID, "state": bson.M{"$in": states}}, task).Err() +} + +// Delete delete task +func (tbl *MigrateTaskTbl) Delete(ctx context.Context, taskID string) error { + span := trace.SpanFromContextSafe(ctx) + span.Debugf("DB:delete task, taskID: %s", taskID) + + _, err := tbl.coll.UpdateOne(ctx, bson.M{"_id": taskID}, deleteBson()) + return err +} + +// MarkDeleteByDiskID mark delete task by diskID +func (tbl *MigrateTaskTbl) MarkDeleteByDiskID(ctx context.Context, diskID proto.DiskID) error { + span := trace.SpanFromContextSafe(ctx) + span.Debugf("delete db task by diskID %d", diskID) + + _, err := tbl.coll.UpdateMany(ctx, bson.M{"source_disk_id": diskID}, deleteBson()) + return err +} + +// MarkDeleteByStates mark delete task by status +func (tbl *MigrateTaskTbl) MarkDeleteByStates(ctx context.Context, states []proto.MigrateState) error { + _, err := tbl.coll.UpdateMany(ctx, bson.M{"state": bson.M{"$in": states}, deleteMark: bson.M{"$ne": true}}, deleteBson()) + return err +} + +// FindAll returns all un mark delete task +func (tbl *MigrateTaskTbl) FindAll(ctx context.Context) (tasks []*proto.MigrateTask, err error) { + cursor, err := tbl.coll.Find(ctx, bson.M{deleteMark: bson.M{"$ne": true}}) + if err != nil { + return nil, err + } + err = cursor.All(ctx, &tasks) + return tasks, err +} + +// Find find task by taskID +func (tbl *MigrateTaskTbl) Find(ctx context.Context, taskID string) (task *proto.MigrateTask, err error) { + err = tbl.coll.FindOne(ctx, bson.M{"_id": taskID, deleteMark: bson.M{"$ne": true}}).Decode(&task) + return +} + +// FindByDiskID find task by diskID +func (tbl *MigrateTaskTbl) FindByDiskID(ctx context.Context, diskID proto.DiskID) (tasks []*proto.MigrateTask, err error) { + cursor, err := tbl.coll.Find(ctx, bson.M{"source_disk_id": diskID, deleteMark: bson.M{"$ne": true}}) + if err != nil { + return nil, err + } + err = cursor.All(ctx, &tasks) + return tasks, err +} + +// QueryMarkDeleteTasks find mark delete task for archive +func (tbl *MigrateTaskTbl) QueryMarkDeleteTasks(ctx context.Context, delayMin int) (records []*proto.ArchiveRecord, err error) { + span := trace.SpanFromContextSafe(ctx) + + type MigrateTaskEx struct { + proto.MigrateTask `bson:",inline"` + DelTime int64 `bson:"del_time"` + } + var tasks []*MigrateTaskEx + cursor, err := tbl.coll.Find(ctx, bson.M{deleteMark: true}) + if err != nil { + return nil, err + } + err = cursor.All(ctx, &tasks) + if err != nil { + return nil, err + } + + for _, task := range tasks { + if inDelayTime(task.DelTime, delayMin) { + span.Debugf("task_id %s is in delay time", task.TaskID) + continue + } + + r := &proto.ArchiveRecord{ + TaskID: task.TaskID, + TaskType: tbl.Name(), + Content: task, + } + records = append(records, r) + } + return records, nil +} + +// RemoveMarkDelete remove mark delete task +func (tbl *MigrateTaskTbl) RemoveMarkDelete(ctx context.Context, taskID string) error { + _, err := tbl.coll.DeleteOne(ctx, bson.M{"_id": taskID, deleteMark: true}) + return err +} + +// Name return table name +func (tbl *MigrateTaskTbl) Name() string { + return tbl.name +} diff --git a/blobstore/scheduler/db/orphan_shard_table.go b/blobstore/scheduler/db/orphan_shard_table.go new file mode 100644 index 000000000..aa59851bd --- /dev/null +++ b/blobstore/scheduler/db/orphan_shard_table.go @@ -0,0 +1,48 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package db + +import ( + "context" + + "go.mongodb.org/mongo-driver/mongo" + + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +// IOrphanShardTable define the interface to save orphan shard record. +type IOrphanShardTable interface { + Save(shard OrphanShard) error +} + +// OrphanShard orphan shard identification. +type OrphanShard struct { + ClusterID proto.ClusterID `bson:"cluster_id"` + Vid proto.Vid `bson:"vid"` + Bid proto.BlobID `bson:"bid"` +} + +type orphanShardTable struct { + coll *mongo.Collection +} + +func openOrphanedShardTable(coll *mongo.Collection) IOrphanShardTable { + return &orphanShardTable{coll: coll} +} + +func (t *orphanShardTable) Save(shard OrphanShard) error { + _, err := t.coll.InsertOne(context.Background(), shard) + return err +} diff --git a/blobstore/scheduler/db/task_archive_table.go b/blobstore/scheduler/db/task_archive_table.go new file mode 100644 index 000000000..9619fe731 --- /dev/null +++ b/blobstore/scheduler/db/task_archive_table.go @@ -0,0 +1,67 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package db + +import ( + "context" + "time" + + "github.com/globalsign/mgo/bson" + "go.mongodb.org/mongo-driver/mongo" + + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/trace" +) + +//duties:transfer the deleted records to the archive table + +// IArchiveTable define the interface of db use by archive +type IArchiveTable interface { + Insert(ctx context.Context, record *proto.ArchiveRecord) error + FindTask(ctx context.Context, taskID string) (record *proto.ArchiveRecord, err error) +} + +// IRecordSrcTbl define the interface of source record table used by archive +type IRecordSrcTbl interface { + QueryMarkDeleteTasks(ctx context.Context, delayMin int) (records []*proto.ArchiveRecord, err error) + RemoveMarkDelete(ctx context.Context, taskID string) error + Name() string +} + +type archiveTbl struct { + coll *mongo.Collection +} + +func openArchiveTbl(coll *mongo.Collection) (IArchiveTable, error) { + return &archiveTbl{ + coll: coll, + }, nil +} + +// Insert insert record +func (tbl *archiveTbl) Insert(ctx context.Context, record *proto.ArchiveRecord) error { + span := trace.SpanFromContextSafe(ctx) + span.Debugf("archiveTbl:insert task %s", record.TaskID) + + record.ArchiveTime = time.Now().String() + _, err := tbl.coll.InsertOne(ctx, record) + return err +} + +// FindTask find task by taskID +func (tbl *archiveTbl) FindTask(ctx context.Context, taskID string) (record *proto.ArchiveRecord, err error) { + err = tbl.coll.FindOne(ctx, bson.M{"_id": taskID}).Decode(&record) + return +} diff --git a/blobstore/scheduler/db_mock_test.go b/blobstore/scheduler/db_mock_test.go new file mode 100644 index 000000000..ca017c47b --- /dev/null +++ b/blobstore/scheduler/db_mock_test.go @@ -0,0 +1,541 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cubefs/cubefs/blobstore/scheduler/db (interfaces: IKafkaOffsetTable,IOrphanShardTable,IArchiveTable,IMigrateTaskTable,IRepairTaskTable,IInspectCheckPointTable) + +// Package scheduler is a generated GoMock package. +package scheduler + +import ( + context "context" + reflect "reflect" + + proto "github.com/cubefs/cubefs/blobstore/common/proto" + db "github.com/cubefs/cubefs/blobstore/scheduler/db" + gomock "github.com/golang/mock/gomock" +) + +// MockKafkaOffsetTable is a mock of IKafkaOffsetTable interface. +type MockKafkaOffsetTable struct { + ctrl *gomock.Controller + recorder *MockKafkaOffsetTableMockRecorder +} + +// MockKafkaOffsetTableMockRecorder is the mock recorder for MockKafkaOffsetTable. +type MockKafkaOffsetTableMockRecorder struct { + mock *MockKafkaOffsetTable +} + +// NewMockKafkaOffsetTable creates a new mock instance. +func NewMockKafkaOffsetTable(ctrl *gomock.Controller) *MockKafkaOffsetTable { + mock := &MockKafkaOffsetTable{ctrl: ctrl} + mock.recorder = &MockKafkaOffsetTableMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockKafkaOffsetTable) EXPECT() *MockKafkaOffsetTableMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockKafkaOffsetTable) Get(arg0 string, arg1 int32) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", arg0, arg1) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockKafkaOffsetTableMockRecorder) Get(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockKafkaOffsetTable)(nil).Get), arg0, arg1) +} + +// Set mocks base method. +func (m *MockKafkaOffsetTable) Set(arg0 string, arg1 int32, arg2 int64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Set", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// Set indicates an expected call of Set. +func (mr *MockKafkaOffsetTableMockRecorder) Set(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockKafkaOffsetTable)(nil).Set), arg0, arg1, arg2) +} + +// MockOrphanShardTable is a mock of IOrphanShardTable interface. +type MockOrphanShardTable struct { + ctrl *gomock.Controller + recorder *MockOrphanShardTableMockRecorder +} + +// MockOrphanShardTableMockRecorder is the mock recorder for MockOrphanShardTable. +type MockOrphanShardTableMockRecorder struct { + mock *MockOrphanShardTable +} + +// NewMockOrphanShardTable creates a new mock instance. +func NewMockOrphanShardTable(ctrl *gomock.Controller) *MockOrphanShardTable { + mock := &MockOrphanShardTable{ctrl: ctrl} + mock.recorder = &MockOrphanShardTableMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockOrphanShardTable) EXPECT() *MockOrphanShardTableMockRecorder { + return m.recorder +} + +// Save mocks base method. +func (m *MockOrphanShardTable) Save(arg0 db.OrphanShard) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Save", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Save indicates an expected call of Save. +func (mr *MockOrphanShardTableMockRecorder) Save(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Save", reflect.TypeOf((*MockOrphanShardTable)(nil).Save), arg0) +} + +// MockArchiveTable is a mock of IArchiveTable interface. +type MockArchiveTable struct { + ctrl *gomock.Controller + recorder *MockArchiveTableMockRecorder +} + +// MockArchiveTableMockRecorder is the mock recorder for MockArchiveTable. +type MockArchiveTableMockRecorder struct { + mock *MockArchiveTable +} + +// NewMockArchiveTable creates a new mock instance. +func NewMockArchiveTable(ctrl *gomock.Controller) *MockArchiveTable { + mock := &MockArchiveTable{ctrl: ctrl} + mock.recorder = &MockArchiveTableMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockArchiveTable) EXPECT() *MockArchiveTableMockRecorder { + return m.recorder +} + +// FindTask mocks base method. +func (m *MockArchiveTable) FindTask(arg0 context.Context, arg1 string) (*proto.ArchiveRecord, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FindTask", arg0, arg1) + ret0, _ := ret[0].(*proto.ArchiveRecord) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FindTask indicates an expected call of FindTask. +func (mr *MockArchiveTableMockRecorder) FindTask(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindTask", reflect.TypeOf((*MockArchiveTable)(nil).FindTask), arg0, arg1) +} + +// Insert mocks base method. +func (m *MockArchiveTable) Insert(arg0 context.Context, arg1 *proto.ArchiveRecord) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Insert", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Insert indicates an expected call of Insert. +func (mr *MockArchiveTableMockRecorder) Insert(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Insert", reflect.TypeOf((*MockArchiveTable)(nil).Insert), arg0, arg1) +} + +// MockMigrateTaskTable is a mock of IMigrateTaskTable interface. +type MockMigrateTaskTable struct { + ctrl *gomock.Controller + recorder *MockMigrateTaskTableMockRecorder +} + +// MockMigrateTaskTableMockRecorder is the mock recorder for MockMigrateTaskTable. +type MockMigrateTaskTableMockRecorder struct { + mock *MockMigrateTaskTable +} + +// NewMockMigrateTaskTable creates a new mock instance. +func NewMockMigrateTaskTable(ctrl *gomock.Controller) *MockMigrateTaskTable { + mock := &MockMigrateTaskTable{ctrl: ctrl} + mock.recorder = &MockMigrateTaskTableMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockMigrateTaskTable) EXPECT() *MockMigrateTaskTableMockRecorder { + return m.recorder +} + +// Delete mocks base method. +func (m *MockMigrateTaskTable) Delete(arg0 context.Context, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockMigrateTaskTableMockRecorder) Delete(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockMigrateTaskTable)(nil).Delete), arg0, arg1) +} + +// Find mocks base method. +func (m *MockMigrateTaskTable) Find(arg0 context.Context, arg1 string) (*proto.MigrateTask, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Find", arg0, arg1) + ret0, _ := ret[0].(*proto.MigrateTask) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Find indicates an expected call of Find. +func (mr *MockMigrateTaskTableMockRecorder) Find(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Find", reflect.TypeOf((*MockMigrateTaskTable)(nil).Find), arg0, arg1) +} + +// FindAll mocks base method. +func (m *MockMigrateTaskTable) FindAll(arg0 context.Context) ([]*proto.MigrateTask, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FindAll", arg0) + ret0, _ := ret[0].([]*proto.MigrateTask) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FindAll indicates an expected call of FindAll. +func (mr *MockMigrateTaskTableMockRecorder) FindAll(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindAll", reflect.TypeOf((*MockMigrateTaskTable)(nil).FindAll), arg0) +} + +// FindByDiskID mocks base method. +func (m *MockMigrateTaskTable) FindByDiskID(arg0 context.Context, arg1 proto.DiskID) ([]*proto.MigrateTask, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FindByDiskID", arg0, arg1) + ret0, _ := ret[0].([]*proto.MigrateTask) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FindByDiskID indicates an expected call of FindByDiskID. +func (mr *MockMigrateTaskTableMockRecorder) FindByDiskID(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindByDiskID", reflect.TypeOf((*MockMigrateTaskTable)(nil).FindByDiskID), arg0, arg1) +} + +// Insert mocks base method. +func (m *MockMigrateTaskTable) Insert(arg0 context.Context, arg1 *proto.MigrateTask) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Insert", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Insert indicates an expected call of Insert. +func (mr *MockMigrateTaskTableMockRecorder) Insert(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Insert", reflect.TypeOf((*MockMigrateTaskTable)(nil).Insert), arg0, arg1) +} + +// MarkDeleteByDiskID mocks base method. +func (m *MockMigrateTaskTable) MarkDeleteByDiskID(arg0 context.Context, arg1 proto.DiskID) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MarkDeleteByDiskID", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// MarkDeleteByDiskID indicates an expected call of MarkDeleteByDiskID. +func (mr *MockMigrateTaskTableMockRecorder) MarkDeleteByDiskID(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkDeleteByDiskID", reflect.TypeOf((*MockMigrateTaskTable)(nil).MarkDeleteByDiskID), arg0, arg1) +} + +// MarkDeleteByStates mocks base method. +func (m *MockMigrateTaskTable) MarkDeleteByStates(arg0 context.Context, arg1 []proto.MigrateState) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MarkDeleteByStates", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// MarkDeleteByStates indicates an expected call of MarkDeleteByStates. +func (mr *MockMigrateTaskTableMockRecorder) MarkDeleteByStates(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkDeleteByStates", reflect.TypeOf((*MockMigrateTaskTable)(nil).MarkDeleteByStates), arg0, arg1) +} + +// Name mocks base method. +func (m *MockMigrateTaskTable) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name. +func (mr *MockMigrateTaskTableMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockMigrateTaskTable)(nil).Name)) +} + +// QueryMarkDeleteTasks mocks base method. +func (m *MockMigrateTaskTable) QueryMarkDeleteTasks(arg0 context.Context, arg1 int) ([]*proto.ArchiveRecord, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "QueryMarkDeleteTasks", arg0, arg1) + ret0, _ := ret[0].([]*proto.ArchiveRecord) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// QueryMarkDeleteTasks indicates an expected call of QueryMarkDeleteTasks. +func (mr *MockMigrateTaskTableMockRecorder) QueryMarkDeleteTasks(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueryMarkDeleteTasks", reflect.TypeOf((*MockMigrateTaskTable)(nil).QueryMarkDeleteTasks), arg0, arg1) +} + +// RemoveMarkDelete mocks base method. +func (m *MockMigrateTaskTable) RemoveMarkDelete(arg0 context.Context, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveMarkDelete", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// RemoveMarkDelete indicates an expected call of RemoveMarkDelete. +func (mr *MockMigrateTaskTableMockRecorder) RemoveMarkDelete(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveMarkDelete", reflect.TypeOf((*MockMigrateTaskTable)(nil).RemoveMarkDelete), arg0, arg1) +} + +// Update mocks base method. +func (m *MockMigrateTaskTable) Update(arg0 context.Context, arg1 proto.MigrateState, arg2 *proto.MigrateTask) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Update", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// Update indicates an expected call of Update. +func (mr *MockMigrateTaskTableMockRecorder) Update(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockMigrateTaskTable)(nil).Update), arg0, arg1, arg2) +} + +// MockRepairTaskTable is a mock of IRepairTaskTable interface. +type MockRepairTaskTable struct { + ctrl *gomock.Controller + recorder *MockRepairTaskTableMockRecorder +} + +// MockRepairTaskTableMockRecorder is the mock recorder for MockRepairTaskTable. +type MockRepairTaskTableMockRecorder struct { + mock *MockRepairTaskTable +} + +// NewMockRepairTaskTable creates a new mock instance. +func NewMockRepairTaskTable(ctrl *gomock.Controller) *MockRepairTaskTable { + mock := &MockRepairTaskTable{ctrl: ctrl} + mock.recorder = &MockRepairTaskTableMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRepairTaskTable) EXPECT() *MockRepairTaskTableMockRecorder { + return m.recorder +} + +// Find mocks base method. +func (m *MockRepairTaskTable) Find(arg0 context.Context, arg1 string) (*proto.VolRepairTask, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Find", arg0, arg1) + ret0, _ := ret[0].(*proto.VolRepairTask) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Find indicates an expected call of Find. +func (mr *MockRepairTaskTableMockRecorder) Find(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Find", reflect.TypeOf((*MockRepairTaskTable)(nil).Find), arg0, arg1) +} + +// FindAll mocks base method. +func (m *MockRepairTaskTable) FindAll(arg0 context.Context) ([]*proto.VolRepairTask, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FindAll", arg0) + ret0, _ := ret[0].([]*proto.VolRepairTask) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FindAll indicates an expected call of FindAll. +func (mr *MockRepairTaskTableMockRecorder) FindAll(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindAll", reflect.TypeOf((*MockRepairTaskTable)(nil).FindAll), arg0) +} + +// FindByDiskID mocks base method. +func (m *MockRepairTaskTable) FindByDiskID(arg0 context.Context, arg1 proto.DiskID) ([]*proto.VolRepairTask, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FindByDiskID", arg0, arg1) + ret0, _ := ret[0].([]*proto.VolRepairTask) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FindByDiskID indicates an expected call of FindByDiskID. +func (mr *MockRepairTaskTableMockRecorder) FindByDiskID(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindByDiskID", reflect.TypeOf((*MockRepairTaskTable)(nil).FindByDiskID), arg0, arg1) +} + +// Insert mocks base method. +func (m *MockRepairTaskTable) Insert(arg0 context.Context, arg1 *proto.VolRepairTask) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Insert", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Insert indicates an expected call of Insert. +func (mr *MockRepairTaskTableMockRecorder) Insert(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Insert", reflect.TypeOf((*MockRepairTaskTable)(nil).Insert), arg0, arg1) +} + +// MarkDeleteByDiskID mocks base method. +func (m *MockRepairTaskTable) MarkDeleteByDiskID(arg0 context.Context, arg1 proto.DiskID) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MarkDeleteByDiskID", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// MarkDeleteByDiskID indicates an expected call of MarkDeleteByDiskID. +func (mr *MockRepairTaskTableMockRecorder) MarkDeleteByDiskID(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkDeleteByDiskID", reflect.TypeOf((*MockRepairTaskTable)(nil).MarkDeleteByDiskID), arg0, arg1) +} + +// Name mocks base method. +func (m *MockRepairTaskTable) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name. +func (mr *MockRepairTaskTableMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockRepairTaskTable)(nil).Name)) +} + +// QueryMarkDeleteTasks mocks base method. +func (m *MockRepairTaskTable) QueryMarkDeleteTasks(arg0 context.Context, arg1 int) ([]*proto.ArchiveRecord, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "QueryMarkDeleteTasks", arg0, arg1) + ret0, _ := ret[0].([]*proto.ArchiveRecord) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// QueryMarkDeleteTasks indicates an expected call of QueryMarkDeleteTasks. +func (mr *MockRepairTaskTableMockRecorder) QueryMarkDeleteTasks(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueryMarkDeleteTasks", reflect.TypeOf((*MockRepairTaskTable)(nil).QueryMarkDeleteTasks), arg0, arg1) +} + +// RemoveMarkDelete mocks base method. +func (m *MockRepairTaskTable) RemoveMarkDelete(arg0 context.Context, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveMarkDelete", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// RemoveMarkDelete indicates an expected call of RemoveMarkDelete. +func (mr *MockRepairTaskTableMockRecorder) RemoveMarkDelete(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveMarkDelete", reflect.TypeOf((*MockRepairTaskTable)(nil).RemoveMarkDelete), arg0, arg1) +} + +// Update mocks base method. +func (m *MockRepairTaskTable) Update(arg0 context.Context, arg1 *proto.VolRepairTask) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Update", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Update indicates an expected call of Update. +func (mr *MockRepairTaskTableMockRecorder) Update(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockRepairTaskTable)(nil).Update), arg0, arg1) +} + +// MockInspectCheckPointTable is a mock of IInspectCheckPointTable interface. +type MockInspectCheckPointTable struct { + ctrl *gomock.Controller + recorder *MockInspectCheckPointTableMockRecorder +} + +// MockInspectCheckPointTableMockRecorder is the mock recorder for MockInspectCheckPointTable. +type MockInspectCheckPointTableMockRecorder struct { + mock *MockInspectCheckPointTable +} + +// NewMockInspectCheckPointTable creates a new mock instance. +func NewMockInspectCheckPointTable(ctrl *gomock.Controller) *MockInspectCheckPointTable { + mock := &MockInspectCheckPointTable{ctrl: ctrl} + mock.recorder = &MockInspectCheckPointTableMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockInspectCheckPointTable) EXPECT() *MockInspectCheckPointTableMockRecorder { + return m.recorder +} + +// GetCheckPoint mocks base method. +func (m *MockInspectCheckPointTable) GetCheckPoint(arg0 context.Context) (*proto.InspectCheckPoint, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCheckPoint", arg0) + ret0, _ := ret[0].(*proto.InspectCheckPoint) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCheckPoint indicates an expected call of GetCheckPoint. +func (mr *MockInspectCheckPointTableMockRecorder) GetCheckPoint(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCheckPoint", reflect.TypeOf((*MockInspectCheckPointTable)(nil).GetCheckPoint), arg0) +} + +// SaveCheckPoint mocks base method. +func (m *MockInspectCheckPointTable) SaveCheckPoint(arg0 context.Context, arg1 proto.Vid) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SaveCheckPoint", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// SaveCheckPoint indicates an expected call of SaveCheckPoint. +func (mr *MockInspectCheckPointTableMockRecorder) SaveCheckPoint(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveCheckPoint", reflect.TypeOf((*MockInspectCheckPointTable)(nil).SaveCheckPoint), arg0, arg1) +} diff --git a/blobstore/scheduler/disk_droper.go b/blobstore/scheduler/disk_droper.go new file mode 100644 index 000000000..81f83695d --- /dev/null +++ b/blobstore/scheduler/disk_droper.go @@ -0,0 +1,380 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "sync" + "time" + + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/taskswitch" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/scheduler/base" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/scheduler/db" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +// IDiskDroper define the interface of disk drop manager +type IDiskDroper interface { + Migrator + Progress(ctx context.Context) (repairingDiskID proto.DiskID, total, repaired int) +} + +// DiskDropMgrConfig disk drop manager config +type DiskDropMgrConfig struct { + MigrateConfig +} + +// DiskDropMgr disk drop manager +type DiskDropMgr struct { + IMigrater + + mu sync.Mutex + dropDisk *client.DiskInfoSimple + droppingDiskID proto.DiskID + clusterMgrCli client.ClusterMgrAPI + hasRevised bool + cfg *DiskDropMgrConfig +} + +// NewDiskDropMgr returns disk drop manager +func NewDiskDropMgr( + clusterMgrCli client.ClusterMgrAPI, + volumeUpdater client.IVolumeUpdater, + taskSwitch taskswitch.ISwitcher, + taskTbl db.IMigrateTaskTable, + conf *DiskDropMgrConfig) *DiskDropMgr { + mgr := &DiskDropMgr{ + clusterMgrCli: clusterMgrCli, + cfg: conf, + } + mgr.IMigrater = NewMigrateMgr(clusterMgrCli, volumeUpdater, taskSwitch, taskTbl, + &conf.MigrateConfig, proto.DiskDropTaskType, conf.ClusterID) + return mgr +} + +// Load load disk drop task from database +func (mgr *DiskDropMgr) Load() (err error) { + ctx := context.Background() + allTasks, err := mgr.IMigrater.FindAll(ctx) + if err != nil { + return err + } + if len(allTasks) == 0 { + log.Infof("no drop tasks in db") + return + } + + droppingDiskID := allTasks[0].SrcMigDiskID() + tasks, err := mgr.IMigrater.FindByDiskID(ctx, droppingDiskID) + if err != nil { + return err + } + if len(allTasks) != len(tasks) { + panic("can not allow many disk dropping") + } + + mgr.setDroppingDiskID(droppingDiskID) + + return mgr.IMigrater.Load() +} + +// Run run disk drop task +func (mgr *DiskDropMgr) Run() { + go mgr.collectTaskLoop() + mgr.IMigrater.Run() + go mgr.checkDroppedAndClearLoop() +} + +// collectTaskLoop collect disk drop task loop +func (mgr *DiskDropMgr) collectTaskLoop() { + t := time.NewTicker(time.Duration(mgr.cfg.CollectTaskIntervalS) * time.Second) + defer t.Stop() + + for { + select { + case <-t.C: + mgr.IMigrater.WaitEnable() + mgr.collectTask() + case <-mgr.IMigrater.Done(): + return + } + } +} + +func (mgr *DiskDropMgr) collectTask() { + span, ctx := trace.StartSpanFromContext(context.Background(), "disk_drop.collectTask") + defer span.Finish() + + if !mgr.hasRevised && mgr.hasDroppingDisk() { + err := mgr.reviseDropTask(ctx, mgr.getDroppingDiskID()) + if err == nil { + span.Infof("drop collect revise tasks success") + mgr.hasRevised = true + mgr.setDroppingDiskID(mgr.getDroppingDiskID()) + return + } + span.Errorf("drop collect revise task failed: err[%+v]", err) + return + } + + if mgr.hasDroppingDisk() { + return + } + + dropDisk, err := mgr.acquireDropDisk(ctx) + if err != nil { + span.Info("acquire drop disk failed: err[%+v]", err) + return + } + + if dropDisk == nil { + return + } + + err = mgr.genDiskDropTasks(ctx, dropDisk.DiskID, dropDisk.Idc) + if err != nil { + span.Errorf("drop collect drop task failed: err[%+v]", err) + return + } + + mgr.setDroppingDiskID(dropDisk.DiskID) +} + +func (mgr *DiskDropMgr) reviseDropTask(ctx context.Context, diskID proto.DiskID) error { + span := trace.SpanFromContextSafe(ctx) + + diskInfo, err := mgr.clusterMgrCli.GetDiskInfo(ctx, diskID) + if err != nil { + span.Errorf("get disk info failed: err[%+v]", err) + return err + } + + err = mgr.genDiskDropTasks(ctx, diskInfo.DiskID, diskInfo.Idc) + if err != nil { + span.Errorf("gen disk drop tasks failed: err[%+v]", err) + return err + } + return nil +} + +func (mgr *DiskDropMgr) genDiskDropTasks(ctx context.Context, diskID proto.DiskID, diskIdc string) error { + span := trace.SpanFromContextSafe(ctx) + + span.Infof("start generate disk drop tasks: disk_id[%d], disk_idc[%s]", diskID, diskIdc) + + vuidsDb, err := mgr.dropVuidsFromDb(ctx, diskID) + if err != nil { + span.Errorf("get drop vuids from db failed: err[%+v]", err) + return err + } + span.Infof("drop vuids from db success: len[%d]", len(vuidsDb)) + + vuidsCm, err := mgr.dropVuidsFromCm(ctx, diskID) + if err != nil { + span.Errorf("get drop vuid from clustermgr failed: err[%+v]", err) + return err + } + span.Infof("drop vuids from clustermgr success: len[%d]", len(vuidsCm)) + + remain := base.Subtraction(vuidsCm, vuidsDb) + span.Infof("should gen tasks: remain len[%d]", len(remain)) + for _, vuid := range remain { + mgr.initOneTask(ctx, vuid, diskID, diskIdc) + span.Infof("init drop task success: vuid[%d]", vuid) + } + return nil +} + +func (mgr *DiskDropMgr) dropVuidsFromDb(ctx context.Context, diskID proto.DiskID) (drops []proto.Vuid, err error) { + tasks, err := mgr.IMigrater.FindByDiskID(ctx, diskID) + if err != nil { + return nil, err + } + + for _, t := range tasks { + drops = append(drops, t.SourceVuid) + } + return drops, nil +} + +func (mgr *DiskDropMgr) dropVuidsFromCm(ctx context.Context, diskID proto.DiskID) (drops []proto.Vuid, err error) { + vunits, err := mgr.clusterMgrCli.ListDiskVolumeUnits(ctx, diskID) + if err != nil { + return nil, err + } + + for _, vunit := range vunits { + drops = append(drops, vunit.Vuid) + } + return drops, nil +} + +func (mgr *DiskDropMgr) initOneTask(ctx context.Context, src proto.Vuid, dropDiskID proto.DiskID, diskIDC string) { + vid := src.Vid() + t := proto.MigrateTask{ + TaskID: mgr.genUniqTaskID(vid), + State: proto.MigrateStateInited, + SourceDiskID: dropDiskID, + SourceIdc: diskIDC, + SourceVuid: src, + } + mgr.IMigrater.AddTask(ctx, &t) +} + +func (mgr *DiskDropMgr) acquireDropDisk(ctx context.Context) (*client.DiskInfoSimple, error) { + // it will retry when break in collectTask, + // should make sure acquire same disk + if mgr.dropDisk != nil { + return mgr.dropDisk, nil + } + + dropDisks, err := mgr.clusterMgrCli.ListDropDisks(ctx) + if err != nil { + return nil, err + } + if len(dropDisks) == 0 { + return nil, nil + } + + mgr.dropDisk = dropDisks[0] + return mgr.dropDisk, nil +} + +func (mgr *DiskDropMgr) checkDroppedAndClearLoop() { + t := time.NewTicker(time.Duration(mgr.cfg.CheckTaskIntervalS) * time.Second) + defer t.Stop() + + for { + select { + case <-t.C: + mgr.IMigrater.WaitEnable() + mgr.checkDroppedAndClear() + case <-mgr.IMigrater.Done(): + return + } + } +} + +func (mgr *DiskDropMgr) checkDroppedAndClear() { + diskID := mgr.getDroppingDiskID() + + span, ctx := trace.StartSpanFromContext(context.Background(), "disk_drop.checkDroppedAndClear") + defer span.Finish() + + if !mgr.hasDroppingDisk() { + return + } + if mgr.checkDropped(ctx, diskID) { + err := mgr.clusterMgrCli.SetDiskDropped(ctx, diskID) + if err != nil { + span.Errorf("set disk dropped failed: err[%+v]", err) + return + } + span.Infof("start clear dropped disk: disk_id[%d]", diskID) + mgr.clearTasksByDiskID(ctx, diskID) + mgr.emptyDroppingDiskID() + } +} + +func (mgr *DiskDropMgr) checkDropped(ctx context.Context, diskID proto.DiskID) bool { + span := trace.SpanFromContextSafe(ctx) + span.Infof("check dropped: disk_id[%d]", diskID) + + tasks, err := mgr.IMigrater.FindByDiskID(ctx, diskID) + if err != nil { + span.Errorf("find all tasks failed: disk_id[%d], err[%+v]", diskID, err) + return false + } + for _, task := range tasks { + if !task.Finished() { + return false + } + } + span.Infof("disk drop has finished: disk_id[%d], task len[%d]", diskID, len(tasks)) + + vunitInfos, err := mgr.clusterMgrCli.ListDiskVolumeUnits(ctx, diskID) + if err != nil { + span.Errorf("list disk volume units failed: disk_id[%s], err[%+v]", diskID, err) + return false + } + if len(vunitInfos) != 0 { + // it may be occur when migration done and repair tasks generate concurrent, list volume units may not return the migrate unit + span.Warnf("clustermgr has some volume unit not repair and revise again: disk_id[%d], volume units len[%d]", diskID, len(vunitInfos)) + if err = mgr.reviseDropTask(ctx, diskID); err != nil { + span.Errorf("revise repair task failed: err[%+v]", err) + } + return false + } + return true +} + +func (mgr *DiskDropMgr) clearTasksByDiskID(ctx context.Context, diskID proto.DiskID) { + mgr.IMigrater.ClearTasksByDiskID(ctx, diskID) +} + +func (mgr *DiskDropMgr) setDroppingDiskID(diskID proto.DiskID) { + mgr.mu.Lock() + defer mgr.mu.Unlock() + mgr.droppingDiskID = diskID +} + +func (mgr *DiskDropMgr) emptyDroppingDiskID() { + mgr.mu.Lock() + defer mgr.mu.Unlock() + mgr.droppingDiskID = base.EmptyDiskID + mgr.dropDisk = nil +} + +func (mgr *DiskDropMgr) getDroppingDiskID() proto.DiskID { + mgr.mu.Lock() + defer mgr.mu.Unlock() + return mgr.droppingDiskID +} + +func (mgr *DiskDropMgr) hasDroppingDisk() bool { + mgr.mu.Lock() + defer mgr.mu.Unlock() + return mgr.droppingDiskID != base.EmptyDiskID +} + +func (mgr *DiskDropMgr) genUniqTaskID(vid proto.Vid) string { + return base.GenTaskID("disk_drop", vid) +} + +// Progress returns disk drop progress +func (mgr *DiskDropMgr) Progress(ctx context.Context) (dropDiskID proto.DiskID, total, dropped int) { + span := trace.SpanFromContextSafe(ctx) + + dropDiskID = mgr.getDroppingDiskID() + if dropDiskID == base.EmptyDiskID { + return base.EmptyDiskID, 0, 0 + } + + allTasks, err := mgr.IMigrater.FindByDiskID(ctx, dropDiskID) + if err != nil { + span.Errorf("find all task failed: err[%+v]", err) + return dropDiskID, 0, 0 + } + total = len(allTasks) + for _, task := range allTasks { + if task.Finished() { + dropped++ + } + } + return dropDiskID, total, dropped +} diff --git a/blobstore/scheduler/disk_droper_test.go b/blobstore/scheduler/disk_droper_test.go new file mode 100644 index 000000000..369d3b16f --- /dev/null +++ b/blobstore/scheduler/disk_droper_test.go @@ -0,0 +1,364 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + api "github.com/cubefs/cubefs/blobstore/api/scheduler" + "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/scheduler/base" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/testing/mocks" + "github.com/cubefs/cubefs/blobstore/util/closer" +) + +func newDiskDroper(t *testing.T) *DiskDropMgr { + ctr := gomock.NewController(t) + clusterMgr := NewMockClusterMgrAPI(ctr) + volumeUpdater := NewMockVolumeUpdater(ctr) + taskSwitch := mocks.NewMockSwitcher(ctr) + migrateTable := NewMockMigrateTaskTable(ctr) + conf := &DiskDropMgrConfig{} + c := closer.New() + + migrater := NewMockMigrater(ctr) + migrater.EXPECT().StatQueueTaskCnt().AnyTimes().Return(0, 0, 0) + migrater.EXPECT().Close().AnyTimes().DoAndReturn(c.Close) + migrater.EXPECT().Done().AnyTimes().Return(c.Done()) + mgr := NewDiskDropMgr(clusterMgr, volumeUpdater, taskSwitch, migrateTable, conf) + mgr.IMigrater = migrater + return mgr +} + +func TestDiskDropLoad(t *testing.T) { + { + mgr := newDiskDroper(t) + mgr.IMigrater.(*MockMigrater).EXPECT().FindAll(any).Return(nil, errMock) + err := mgr.Load() + require.True(t, errors.Is(err, errMock)) + } + { + mgr := newDiskDroper(t) + mgr.IMigrater.(*MockMigrater).EXPECT().FindAll(any).Return(nil, nil) + err := mgr.Load() + require.NoError(t, err) + } + { + mgr := newDiskDroper(t) + mgr.IMigrater.(*MockMigrater).EXPECT().FindAll(any).Return([]*proto.MigrateTask{{SourceDiskID: proto.DiskID(1)}, {SourceDiskID: proto.DiskID(2)}}, nil) + mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return(nil, errMock) + err := mgr.Load() + require.True(t, errors.Is(err, errMock)) + } + { + mgr := newDiskDroper(t) + mgr.IMigrater.(*MockMigrater).EXPECT().FindAll(any).Return([]*proto.MigrateTask{{SourceDiskID: proto.DiskID(1)}, {SourceDiskID: proto.DiskID(2)}}, nil) + mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return([]*proto.MigrateTask{{SourceDiskID: proto.DiskID(1)}}, nil) + require.Panics(t, func() { + mgr.Load() + }) + } + { + mgr := newDiskDroper(t) + mgr.IMigrater.(*MockMigrater).EXPECT().FindAll(any).Return([]*proto.MigrateTask{{SourceDiskID: proto.DiskID(1)}}, nil) + mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return([]*proto.MigrateTask{{SourceDiskID: proto.DiskID(1)}}, nil) + mgr.IMigrater.(*MockMigrater).EXPECT().Load().Return(nil) + err := mgr.Load() + require.NoError(t, err) + } +} + +func TestDiskDropRun(t *testing.T) { + mgr := newDiskDroper(t) + defer mgr.Close() + + mgr.IMigrater.(*MockMigrater).EXPECT().WaitEnable().AnyTimes().Return() + mgr.IMigrater.(*MockMigrater).EXPECT().Enabled().AnyTimes().Return(true) + mgr.IMigrater.(*MockMigrater).EXPECT().Run().Return() + mgr.IMigrater.(*MockMigrater).EXPECT().ClearTasksByStates(any, any).AnyTimes().Return() + mgr.IMigrater.(*MockMigrater).EXPECT().GetMigratingDiskNum().AnyTimes().Return(1) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDropDisks(any).AnyTimes().Return(nil, errMock) + mgr.cfg.CollectTaskIntervalS = 1 + mgr.cfg.CheckTaskIntervalS = 1 + require.True(t, mgr.Enabled()) + mgr.Run() + + time.Sleep(1 * time.Second) +} + +func TestDiskDropCollectTask(t *testing.T) { + { + // reviseDropTask failed + mgr := newDiskDroper(t) + mgr.hasRevised = false + mgr.droppingDiskID = proto.DiskID(1) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(nil, errMock) + mgr.collectTask() + } + { + // genDiskDropTasks failed + mgr := newDiskDroper(t) + mgr.hasRevised = false + mgr.droppingDiskID = proto.DiskID(1) + + // find in db failed + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(&client.DiskInfoSimple{DiskID: mgr.droppingDiskID}, nil) + mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return(nil, errMock) + mgr.collectTask() + + // find in cm failed + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(&client.DiskInfoSimple{DiskID: mgr.droppingDiskID}, nil) + mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return(nil, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, errMock) + mgr.collectTask() + + // genDiskDropTasks success + volume := MockGenVolInfo(10005, codemode.EC6P6, proto.VolumeStatusIdle) + var units []*client.VunitInfoSimple + for _, unit := range volume.VunitLocations { + ele := client.VunitInfoSimple{ + Vuid: unit.Vuid, + DiskID: unit.DiskID, + } + units = append(units, &ele) + } + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(&client.DiskInfoSimple{DiskID: mgr.droppingDiskID}, nil) + mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return(nil, nil) + mgr.IMigrater.(*MockMigrater).EXPECT().AddTask(any, any).AnyTimes().Return() + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil) + mgr.collectTask() + } + { + mgr := newDiskDroper(t) + mgr.hasRevised = true + mgr.droppingDiskID = proto.DiskID(1) + mgr.collectTask() + } + { + // acquireDropDisk + mgr := newDiskDroper(t) + mgr.hasRevised = true + + // list drop disk failed + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDropDisks(any).Return(nil, errMock) + mgr.collectTask() + + // no drop disk + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDropDisks(any).Return(nil, nil) + mgr.collectTask() + } + { + mgr := newDiskDroper(t) + mgr.hasRevised = true + disk1 := &client.DiskInfoSimple{ + ClusterID: 1, + Idc: "z0", + Rack: "rack1", + Host: "127.0.0.1:8000", + Status: proto.DiskStatusNormal, + DiskID: 1, + FreeChunkCnt: 10, + MaxChunkCnt: 700, + } + volume := MockGenVolInfo(10005, codemode.EC6P6, proto.VolumeStatusIdle) + var units []*client.VunitInfoSimple + for _, unit := range volume.VunitLocations { + ele := client.VunitInfoSimple{ + Vuid: unit.Vuid, + DiskID: unit.DiskID, + } + units = append(units, &ele) + } + mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return(nil, nil) + mgr.IMigrater.(*MockMigrater).EXPECT().AddTask(any, any).AnyTimes().Return() + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDropDisks(any).Return([]*client.DiskInfoSimple{disk1}, nil) + mgr.collectTask() + } + { + mgr := newDiskDroper(t) + mgr.hasRevised = true + disk1 := &client.DiskInfoSimple{ + ClusterID: 1, + Idc: "z0", + Rack: "rack1", + Host: "127.0.0.1:8000", + Status: proto.DiskStatusNormal, + DiskID: 1, + FreeChunkCnt: 10, + MaxChunkCnt: 700, + } + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDropDisks(any).Return([]*client.DiskInfoSimple{disk1}, nil) + mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return(nil, errMock) + mgr.collectTask() + } +} + +func TestDiskDropCheckDroppedAndClear(t *testing.T) { + { + mgr := newDiskDroper(t) + mgr.checkDroppedAndClear() + } + { + // check dropped return false + mgr := newDiskDroper(t) + mgr.droppingDiskID = proto.DiskID(1) + + mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return(nil, errMock) + mgr.checkDroppedAndClear() + require.True(t, mgr.hasDroppingDisk()) + + mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return(nil, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, errMock) + mgr.checkDroppedAndClear() + require.True(t, mgr.hasDroppingDisk()) + + task1 := &proto.MigrateTask{State: proto.MigrateStatePrepared} + mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return([]*proto.MigrateTask{task1}, nil) + mgr.checkDroppedAndClear() + require.True(t, mgr.hasDroppingDisk()) + + task2 := &proto.MigrateTask{State: proto.MigrateStateFinished} + task3 := &proto.MigrateTask{State: proto.MigrateStateFinishedInAdvance} + mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return([]*proto.MigrateTask{task1, task2, task3}, nil) + mgr.checkDroppedAndClear() + require.True(t, mgr.hasDroppingDisk()) + } + { + // check dropped return true + mgr := newDiskDroper(t) + mgr.droppingDiskID = proto.DiskID(1) + task1 := &proto.MigrateTask{State: proto.MigrateStateFinished} + task2 := &proto.MigrateTask{State: proto.MigrateStateFinishedInAdvance} + mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return([]*proto.MigrateTask{task1, task2}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().SetDiskDropped(any, any).Return(errMock) + mgr.checkDroppedAndClear() + require.True(t, mgr.hasDroppingDisk()) + + volume := MockGenVolInfo(10, codemode.EC6P6, proto.VolumeStatusIdle) + var units []*client.VunitInfoSimple + for _, unit := range volume.VunitLocations { + ele := client.VunitInfoSimple{ + Vuid: unit.Vuid, + DiskID: unit.DiskID, + } + units = append(units, &ele) + } + mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return([]*proto.MigrateTask{task1, task2}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(nil, errMock) + mgr.checkDroppedAndClear() + require.True(t, mgr.hasDroppingDisk()) + + mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return([]*proto.MigrateTask{task1, task2}, nil) + mgr.IMigrater.(*MockMigrater).EXPECT().ClearTasksByDiskID(any, any).Return() + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().SetDiskDropped(any, any).Return(nil) + mgr.checkDroppedAndClear() + require.False(t, mgr.hasDroppingDisk()) + } +} + +func TestDiskDropAcquireTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + mgr := newDiskDroper(t) + mgr.IMigrater.(*MockMigrater).EXPECT().AcquireTask(any, any).Return(&proto.MigrateTask{}, nil) + _, err := mgr.AcquireTask(ctx, idc) + require.NoError(t, err) +} + +func TestDiskDropCancelTask(t *testing.T) { + ctx := context.Background() + mgr := newDiskDroper(t) + mgr.IMigrater.(*MockMigrater).EXPECT().CancelTask(any, any).Return(nil) + err := mgr.CancelTask(ctx, &api.CancelTaskArgs{}) + require.NoError(t, err) +} + +func TestDiskDropReclaimTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + mgr := newDiskDroper(t) + mgr.IMigrater.(*MockMigrater).EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil) + t1 := mockGenMigrateTask(idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap) + err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{}) + require.NoError(t, err) +} + +func TestDiskDropCompleteTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + mgr := newDiskDroper(t) + mgr.IMigrater.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(nil) + t1 := mockGenMigrateTask(idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap) + err := mgr.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: idc, TaskId: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + require.NoError(t, err) + + mgr.IMigrater.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(errMock) + err = mgr.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: idc, TaskId: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + require.True(t, errors.Is(err, errMock)) +} + +func TestDiskDropRenewalTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + mgr := newDiskDroper(t) + mgr.IMigrater.(*MockMigrater).EXPECT().RenewalTask(any, any, any).Return(nil) + err := mgr.RenewalTask(ctx, idc, "") + require.NoError(t, err) + + mgr.IMigrater.(*MockMigrater).EXPECT().RenewalTask(any, any, any).Return(errMock) + err = mgr.RenewalTask(ctx, idc, "") + require.True(t, errors.Is(err, errMock)) +} + +func TestDiskDropProgress(t *testing.T) { + ctx := context.Background() + { + mgr := newDiskDroper(t) + diskID, _, _ := mgr.Progress(ctx) + require.Equal(t, base.EmptyDiskID, diskID) + } + { + mgr := newDiskDroper(t) + mgr.droppingDiskID = proto.DiskID(1) + mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return(nil, errMock) + diskID, _, _ := mgr.Progress(ctx) + require.Equal(t, proto.DiskID(1), diskID) + } + { + mgr := newDiskDroper(t) + diskID := proto.DiskID(1) + mgr.droppingDiskID = diskID + task1 := &proto.MigrateTask{State: proto.MigrateStatePrepared, SourceDiskID: diskID} + task2 := &proto.MigrateTask{State: proto.MigrateStateFinished, SourceDiskID: diskID} + task3 := &proto.MigrateTask{State: proto.MigrateStateFinishedInAdvance, SourceDiskID: diskID} + mgr.IMigrater.(*MockMigrater).EXPECT().FindByDiskID(any, any).Return([]*proto.MigrateTask{task1, task2, task3}, nil) + doingDisk, tatal, dropped := mgr.Progress(ctx) + require.Equal(t, diskID, doingDisk) + require.Equal(t, 3, tatal) + require.Equal(t, 2, dropped) + } +} diff --git a/blobstore/scheduler/disk_repairer.go b/blobstore/scheduler/disk_repairer.go new file mode 100644 index 000000000..9e8299256 --- /dev/null +++ b/blobstore/scheduler/disk_repairer.go @@ -0,0 +1,848 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "fmt" + "sync" + "time" + + api "github.com/cubefs/cubefs/blobstore/api/scheduler" + "github.com/cubefs/cubefs/blobstore/common/counter" + "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/taskswitch" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/scheduler/base" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/scheduler/db" + "github.com/cubefs/cubefs/blobstore/util/closer" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +// IDiskRepairer define the interface of disk repair manager +type IDiskRepairer interface { + AcquireTask(ctx context.Context, idc string) (task *proto.VolRepairTask, err error) + CancelTask(ctx context.Context, args *api.CancelTaskArgs) error + CompleteTask(ctx context.Context, args *api.CompleteTaskArgs) error + ReclaimTask(ctx context.Context, idc, taskID string, + src []proto.VunitLocation, oldDst proto.VunitLocation, newDst *client.AllocVunitInfo) error + RenewalTask(ctx context.Context, idc, taskID string) error + QueryTask(ctx context.Context, taskID string) (*api.RepairTaskDetail, error) + ReportWorkerTaskStats(st *api.TaskReportArgs) + StatQueueTaskCnt() (inited, prepared, completed int) + Stats() api.MigrateTasksStat + Progress(ctx context.Context) (repairingDiskID proto.DiskID, total, repaired int) + Enabled() bool + Load() error + Run() + closer.Closer +} + +const ( + prepareIntervalS = 1 + finishIntervalS = 5 +) + +// DiskRepairMgrCfg repair manager config +type DiskRepairMgrCfg struct { + ClusterID proto.ClusterID `json:"cluster_id"` + base.TaskCommonConfig +} + +// DiskRepairMgr repair task manager +type DiskRepairMgr struct { + closer.Closer + repairingDiskID proto.DiskID // only supports repair one disk at the same time temporarily + brokenDisk *client.DiskInfoSimple + + mu sync.Mutex + + taskTbl db.IRepairTaskTable + + prepareQueue *base.TaskQueue + workQueue *base.WorkerTaskQueue + finishQueue *base.TaskQueue + + clusterMgrCli client.ClusterMgrAPI + + taskSwitch taskswitch.ISwitcher + + // for stats + finishTaskCounter counter.Counter + taskStatsMgr *base.TaskStatsMgr + + hasRevised bool + cfg *DiskRepairMgrCfg +} + +// NewRepairMgr returns repair manager +func NewRepairMgr(cfg *DiskRepairMgrCfg, taskSwitch taskswitch.ISwitcher, + taskTbl db.IRepairTaskTable, cmCli client.ClusterMgrAPI) *DiskRepairMgr { + mgr := &DiskRepairMgr{ + Closer: closer.New(), + taskTbl: taskTbl, + prepareQueue: base.NewTaskQueue(time.Duration(cfg.PrepareQueueRetryDelayS) * time.Second), + workQueue: base.NewWorkerTaskQueue(time.Duration(cfg.CancelPunishDurationS) * time.Second), + finishQueue: base.NewTaskQueue(time.Duration(cfg.FinishQueueRetryDelayS) * time.Second), + + clusterMgrCli: cmCli, + taskSwitch: taskSwitch, + cfg: cfg, + + hasRevised: false, + } + mgr.taskStatsMgr = base.NewTaskStatsMgrAndRun(cfg.ClusterID, proto.RepairTaskType, mgr) + return mgr +} + +// Load load repair task from database +func (mgr *DiskRepairMgr) Load() error { + ctx := context.Background() + + allTasks, err := mgr.taskTbl.FindAll(ctx) + if err != nil { + return err + } + log.Infof("repair load tasks: len[%d]", len(allTasks)) + + if len(allTasks) == 0 { + return nil + } + + repairingDiskID := allTasks[0].RepairDiskID + tasks, err := mgr.taskTbl.FindByDiskID(ctx, repairingDiskID) + if err != nil { + return err + } + if len(allTasks) != len(tasks) { + panic("can not allow many disk repairing") + } + + mgr.setRepairingDiskID(repairingDiskID) + + for _, t := range tasks { + if t.Running() { + err = base.VolTaskLockerInst().TryLock(ctx, t.Vid()) + if err != nil { + log.Panicf("repair task conflict: task[%+v], err[%+v]", + t, err.Error()) + } + } + + log.Infof("load task success: task_id[%s], state[%d]", t.TaskID, t.State) + switch t.State { + case proto.RepairStateInited: + mgr.prepareQueue.PushTask(t.TaskID, t) + case proto.RepairStatePrepared: + mgr.workQueue.AddPreparedTask(t.BrokenDiskIDC, t.TaskID, t) + case proto.RepairStateWorkCompleted: + mgr.finishQueue.PushTask(t.TaskID, t) + case proto.RepairStateFinished, proto.RepairStateFinishedInAdvance: + continue + default: + panic("unexpect repair state") + } + } + + return nil +} + +// Run run repair task includes collect/prepare/finish/check phase +func (mgr *DiskRepairMgr) Run() { + go mgr.collectTaskLoop() + go mgr.prepareTaskLoop() + go mgr.finishTaskLoop() + go mgr.checkRepairedAndClearLoop() +} + +func (mgr *DiskRepairMgr) Enabled() bool { + return mgr.taskSwitch.Enabled() +} + +func (mgr *DiskRepairMgr) collectTaskLoop() { + t := time.NewTicker(time.Duration(mgr.cfg.CollectTaskIntervalS) * time.Second) + defer t.Stop() + + for { + select { + case <-t.C: + mgr.taskSwitch.WaitEnable() + mgr.collectTask() + case <-mgr.Closer.Done(): + return + } + } +} + +func (mgr *DiskRepairMgr) collectTask() { + span, ctx := trace.StartSpanFromContext(context.Background(), "disk_repair.collectTask") + defer span.Finish() + + // revise repair tasks to make sure data consistency when services start + if !mgr.hasRevised && mgr.hasRepairingDisk() { + span.Infof("first collect task will revise repair task") + err := mgr.reviseRepairTask(ctx, mgr.getRepairingDiskID()) + if err == nil { + span.Infof("firstCollectTask finished") + mgr.hasRevised = true + } + return + } + + if mgr.hasRepairingDisk() { + span.Infof("disk is repairing and skip collect task: disk_id[%d]", mgr.getRepairingDiskID()) + return + } + + brokenDisk, err := mgr.acquireBrokenDisk(ctx) + if err != nil { + span.Errorf("acquire broken disk failed: err[%+v]", err) + return + } + if brokenDisk == nil { + return + } + + err = mgr.genDiskRepairTasks(ctx, brokenDisk.DiskID, brokenDisk.Idc) + if err != nil { + span.Errorf("generate disk repair tasks failed: err[%+v]", err) + return + } + + base.InsistOn(ctx, "set disk diskId %d repairing failed", func() error { + return mgr.clusterMgrCli.SetDiskRepairing(ctx, brokenDisk.DiskID) + }) + + mgr.setRepairingDiskID(brokenDisk.DiskID) +} + +func (mgr *DiskRepairMgr) reviseRepairTask(ctx context.Context, diskID proto.DiskID) error { + span := trace.SpanFromContextSafe(ctx) + + diskInfo, err := mgr.clusterMgrCli.GetDiskInfo(ctx, diskID) + if err != nil { + span.Errorf("get disk info failed: err[%+v]", err) + return err + } + + if err = mgr.genDiskRepairTasks(ctx, diskID, diskInfo.Idc); err != nil { + span.Errorf("generate disk repair tasks failed: err[%+v]", err) + return err + } + + if diskInfo.IsBroken() { + execMsg := fmt.Sprintf("set disk diskId %d repairing", mgr.getRepairingDiskID()) + base.InsistOn(ctx, execMsg, func() error { + return mgr.clusterMgrCli.SetDiskRepairing(ctx, diskID) + }) + } + return nil +} + +func (mgr *DiskRepairMgr) genDiskRepairTasks(ctx context.Context, diskID proto.DiskID, diskIdc string) error { + span := trace.SpanFromContextSafe(ctx) + span.Infof("start generate disk repair tasks: disk_id[%d], disk_idc[%s]", diskID, diskIdc) + + vuidsDb, err := mgr.badVuidsFromDb(ctx, diskID) + if err != nil { + span.Errorf("get bad vuids from db failed: err[%+v]", err) + return err + } + span.Infof("bad vuids from db: len[%d]", len(vuidsDb)) + + vuidsCm, err := mgr.badVuidsFromClusterMgr(ctx, diskID) + if err != nil { + span.Errorf("get bad vuid from clustermgr failed: err[%+v]", err) + return err + } + span.Infof("bad vuid from clustermgr: len[%d]", len(vuidsCm)) + + remain := base.Subtraction(vuidsCm, vuidsDb) + span.Infof("should gen tasks remain: len[%d]", len(remain)) + for _, vuid := range remain { + mgr.initOneTask(ctx, vuid, diskID, diskIdc) + } + return nil +} + +func (mgr *DiskRepairMgr) badVuidsFromDb(ctx context.Context, diskID proto.DiskID) (bads []proto.Vuid, err error) { + tasks, err := mgr.taskTbl.FindByDiskID(ctx, diskID) + if err != nil { + return nil, err + } + + for _, t := range tasks { + bads = append(bads, t.RepairVuid()) + } + return bads, nil +} + +func (mgr *DiskRepairMgr) badVuidsFromClusterMgr(ctx context.Context, diskID proto.DiskID) (bads []proto.Vuid, err error) { + vunits, err := mgr.clusterMgrCli.ListDiskVolumeUnits(ctx, diskID) + if err != nil { + return nil, err + } + + for _, vunit := range vunits { + bads = append(bads, vunit.Vuid) + } + return bads, nil +} + +func (mgr *DiskRepairMgr) initOneTask(ctx context.Context, badVuid proto.Vuid, brokenDiskID proto.DiskID, brokenDiskIdc string) { + span := trace.SpanFromContextSafe(ctx) + + vid := badVuid.Vid() + t := proto.VolRepairTask{ + TaskID: mgr.genUniqTaskID(vid), + State: proto.RepairStateInited, + RepairDiskID: brokenDiskID, + + BadVuid: badVuid, + BadIdx: badVuid.Index(), + + BrokenDiskIDC: brokenDiskIdc, + TriggerBy: proto.BrokenDiskTrigger, + } + base.InsistOn(ctx, "repair init one task insert task to tbl", func() error { + return mgr.taskTbl.Insert(ctx, &t) + }) + + mgr.prepareQueue.PushTask(t.TaskID, &t) + span.Infof("init repair task success %+v", t) +} + +func (mgr *DiskRepairMgr) genUniqTaskID(vid proto.Vid) string { + return base.GenTaskID("disk-repair", vid) +} + +func (mgr *DiskRepairMgr) acquireBrokenDisk(ctx context.Context) (*client.DiskInfoSimple, error) { + // can not assume request cm to acquire broken disk is the same disk + // because break in generate tasks(eg. generate task return an error), + // and reentry(not because of starting of service) need the same disk + // cache last broken disk acquired from cm + if mgr.brokenDisk != nil { + return mgr.brokenDisk, nil + } + + brokenDisks, err := mgr.clusterMgrCli.ListBrokenDisks(ctx, 1) + if err != nil { + return nil, err + } + if len(brokenDisks) == 0 { + return nil, nil + } + + mgr.brokenDisk = brokenDisks[0] + return mgr.brokenDisk, nil +} + +func (mgr *DiskRepairMgr) prepareTaskLoop() { + for { + mgr.taskSwitch.WaitEnable() + todo, doing := mgr.workQueue.StatsTasks() + if !mgr.hasRepairingDisk() || todo+doing >= mgr.cfg.WorkQueueSize { + time.Sleep(1 * time.Second) + continue + } + + err := mgr.popTaskAndPrepare() + if err == base.ErrNoTaskInQueue { + time.Sleep(time.Duration(prepareIntervalS) * time.Second) + } + } +} + +func (mgr *DiskRepairMgr) popTaskAndPrepare() error { + _, task, exist := mgr.prepareQueue.PopTask() + if !exist { + return base.ErrNoTaskInQueue + } + + var err error + span, ctx := trace.StartSpanFromContext(context.Background(), "disk_repair.popTaskAndPrepare") + defer span.Finish() + + defer func() { + if err != nil { + span.Errorf("prepare task failed and retry task: task_id[%s], err[%+v]", task.(*proto.VolRepairTask).TaskID, err) + mgr.prepareQueue.RetryTask(task.(*proto.VolRepairTask).TaskID) + } + }() + + //why:avoid to change task in queue + t := task.(*proto.VolRepairTask).Copy() + span.Infof("pop task: task_id[%s], task[%+v]", t.TaskID, t) + // whether vid has another running task + err = base.VolTaskLockerInst().TryLock(ctx, t.Vid()) + if err != nil { + span.Warnf("tryLock failed: vid[%d]", t.Vid()) + return base.ErrVolNotOnlyOneTask + } + defer func() { + if err != nil { + span.Errorf("prepare task failed: task_id[%s], err[%+v]", t.TaskID, err) + base.VolTaskLockerInst().Unlock(ctx, t.Vid()) + } + }() + + err = mgr.prepareTask(t) + if err != nil { + span.Errorf("prepare task failed: task_id[%s], err[%+v]", t.TaskID, err) + return err + } + + span.Infof("prepare task success: task_id[%s]", t.TaskID) + return nil +} + +func (mgr *DiskRepairMgr) prepareTask(t *proto.VolRepairTask) error { + span, ctx := trace.StartSpanFromContext( + context.Background(), + "DiskRepairMgr.prepareTask") + defer span.Finish() + + span.Infof("start prepare repair task: task_id[%s], task[%+v]", t.TaskID, t) + + volInfo, err := mgr.clusterMgrCli.GetVolumeInfo(ctx, t.Vid()) + if err != nil { + span.Errorf("prepare task get volume info failed: err[%+v]", err) + return err + } + + // 1.check necessity of generating current task + badVuid := t.RepairVuid() + if volInfo.VunitLocations[t.BadIdx].Vuid != badVuid { + span.Infof("repair task finish in advance: task_id[%s]", t.TaskID) + mgr.finishTaskInAdvance(ctx, t) + return nil + } + + // 2.generate src and destination for task & task persist + allocDstVunit, err := base.AllocVunitSafe(ctx, mgr.clusterMgrCli, badVuid, t.Sources) + if err != nil { + span.Errorf("repair alloc volume unit failed: err[%+v]", err) + return err + } + + t.CodeMode = volInfo.CodeMode + t.Sources = volInfo.VunitLocations + t.Destination = allocDstVunit.Location() + t.State = proto.RepairStatePrepared + base.InsistOn(ctx, "repair prepare task update task tbl", func() error { + return mgr.taskTbl.Update(ctx, t) + }) + + mgr.sendToWorkQueue(t) + return nil +} + +func (mgr *DiskRepairMgr) sendToWorkQueue(t *proto.VolRepairTask) { + mgr.workQueue.AddPreparedTask(t.BrokenDiskIDC, t.TaskID, t) + mgr.prepareQueue.RemoveTask(t.TaskID) +} + +func (mgr *DiskRepairMgr) finishTaskInAdvance(ctx context.Context, t *proto.VolRepairTask) { + t.State = proto.RepairStateFinishedInAdvance + base.InsistOn(ctx, "repair finish task in advance update task tbl", func() error { + return mgr.taskTbl.Update(ctx, t) + }) + + mgr.finishTaskCounter.Add() + mgr.prepareQueue.RemoveTask(t.TaskID) + base.VolTaskLockerInst().Unlock(ctx, t.Vid()) +} + +func (mgr *DiskRepairMgr) finishTaskLoop() { + for { + mgr.taskSwitch.WaitEnable() + err := mgr.popTaskAndFinish() + if err == base.ErrNoTaskInQueue { + time.Sleep(time.Duration(finishIntervalS) * time.Second) + } + } +} + +func (mgr *DiskRepairMgr) popTaskAndFinish() error { + _, task, exist := mgr.finishQueue.PopTask() + if !exist { + return base.ErrNoTaskInQueue + } + + span, ctx := trace.StartSpanFromContext(context.Background(), "disk_repair.popTaskAndFinish") + defer span.Finish() + + t := task.(*proto.VolRepairTask).Copy() + err := mgr.finishTask(ctx, t) + if err != nil { + span.Errorf("finish task failed: err[%+v]", err) + return err + } + + span.Infof("finish task success: task_id[%s]", t.TaskID) + return nil +} + +func (mgr *DiskRepairMgr) finishTask(ctx context.Context, task *proto.VolRepairTask) (retErr error) { + span := trace.SpanFromContextSafe(ctx) + + defer func() { + if retErr != nil { + mgr.finishQueue.RetryTask(task.TaskID) + } + }() + + if task.State != proto.RepairStateWorkCompleted { + span.Panicf("task state not expect: task_id[%s], expect state[%d], actual state[%d]", proto.RepairStateWorkCompleted, task.State) + } + // complete stage can not make sure to save task info to db, + // finish stage make sure to save task info to db + // execute update volume mapping relation when can not save task with completed state is dangerous + // because if process restart will reload task and redo by worker + // worker will write data to chunk which is online + base.InsistOn(ctx, "repair finish task update task state completed", func() error { + return mgr.taskTbl.Update(ctx, task) + }) + + newVuid := task.Destination.Vuid + oldVuid := task.RepairVuid() + err := mgr.clusterMgrCli.UpdateVolume(ctx, newVuid, oldVuid, task.NewDiskId()) + if err != nil { + span.Errorf("update volume failed: err[%+v]", err) + return mgr.handleUpdateVolMappingFail(ctx, task, err) + } + + task.State = proto.RepairStateFinished + base.InsistOn(ctx, "repair finish task update task state finished", func() error { + return mgr.taskTbl.Update(ctx, task) + }) + + mgr.finishTaskCounter.Add() + // 1.remove task in memory + // 2.release lock of volume task + mgr.finishQueue.RemoveTask(task.TaskID) + base.VolTaskLockerInst().Unlock(ctx, task.Vid()) + + return nil +} + +func (mgr *DiskRepairMgr) handleUpdateVolMappingFail(ctx context.Context, task *proto.VolRepairTask, err error) error { + span := trace.SpanFromContextSafe(ctx) + span.Infof("handle update vol mapping failed: task_id[%s], state[%d], dest vuid[%d]", task.TaskID, task.State, task.Destination.Vuid) + + code := rpc.DetectStatusCode(err) + if code == errors.CodeOldVuidNotMatch { + span.Panicf("change volume unit relationship got unexpected err") + } + + if base.ShouldAllocAndRedo(code) { + span.Infof("realloc vunit and redo: task_id[%s]", task.TaskID) + + newVunit, err := base.AllocVunitSafe(ctx, mgr.clusterMgrCli, task.BadVuid, task.Sources) + if err != nil { + span.Errorf("realloc failed: vuid[%d], err[%+v]", task.BadVuid, err) + return err + } + task.SetDest(newVunit.Location()) + task.State = proto.RepairStatePrepared + task.WorkerRedoCnt++ + + base.InsistOn(ctx, "repair redo task update task tbl", func() error { + return mgr.taskTbl.Update(ctx, task) + }) + + mgr.finishQueue.RemoveTask(task.TaskID) + mgr.workQueue.AddPreparedTask(task.BrokenDiskIDC, task.TaskID, task) + span.Infof("task redo again: task_id[%v]", task.TaskID) + return nil + } + + return err +} + +func (mgr *DiskRepairMgr) checkRepairedAndClearLoop() { + t := time.NewTicker(time.Duration(mgr.cfg.CheckTaskIntervalS) * time.Second) + defer t.Stop() + + for { + select { + case <-t.C: + mgr.taskSwitch.WaitEnable() + mgr.checkRepairedAndClear() + case <-mgr.Closer.Done(): + return + } + } +} + +func (mgr *DiskRepairMgr) checkRepairedAndClear() { + diskID := mgr.getRepairingDiskID() + span, ctx := trace.StartSpanFromContext(context.Background(), "disk_repair.checkRepairedAndClear") + defer span.Finish() + + if !mgr.hasRepairingDisk() { + return + } + + span.Infof("check repaired: disk_id[%d]", diskID) + if mgr.checkRepaired(ctx, diskID) { + err := mgr.clusterMgrCli.SetDiskRepaired(ctx, diskID) + if err != nil { + return + } + span.Infof("disk repaired will start clear: disk_id[%d]", diskID) + mgr.clearTasksByDiskID(diskID) + mgr.emptyRepairingDiskID() + } +} + +func (mgr *DiskRepairMgr) checkRepaired(ctx context.Context, diskID proto.DiskID) bool { + span := trace.SpanFromContextSafe(ctx) + span.Infof("check repaired: disk_id[%d]", diskID) + + tasks, err := mgr.taskTbl.FindByDiskID(ctx, diskID) + if err != nil { + span.Errorf("check repaired and find task failed: disk_iD[%d], err[%+v]", diskID, err) + return false + } + for _, task := range tasks { + if !task.Finished() { + return false + } + } + + span.Infof("disk repair has finished: disk_id[%d], task len[%d]", diskID, len(tasks)) + + vunitInfos, err := mgr.clusterMgrCli.ListDiskVolumeUnits(ctx, diskID) + if err != nil { + span.Errorf("check repaired list disk volume units failed: disk_id[%s], err[%+v]", diskID, err) + return false + } + if len(vunitInfos) != 0 { + // it may be occur when migration done and repair tasks generate concurrent, list volume units may not return the migrate unit + span.Warnf("clustermgr has some volume unit not repair and revise again: disk_id[%d], volume units len[%d]", diskID, len(vunitInfos)) + if err = mgr.reviseRepairTask(ctx, diskID); err != nil { + span.Errorf("revise repair task failed: err[%+v]", err) + } + return false + } + return true +} + +func (mgr *DiskRepairMgr) clearTasksByDiskID(diskID proto.DiskID) { + span, ctx := trace.StartSpanFromContext(context.Background(), "disk_repair.clearTasksByDiskID") + defer span.Finish() + + base.InsistOn(ctx, "repair clear task by diskID", func() error { + return mgr.taskTbl.MarkDeleteByDiskID(ctx, diskID) + }) +} + +func (mgr *DiskRepairMgr) setRepairingDiskID(diskID proto.DiskID) { + mgr.mu.Lock() + defer mgr.mu.Unlock() + mgr.repairingDiskID = diskID + mgr.brokenDisk = nil +} + +func (mgr *DiskRepairMgr) emptyRepairingDiskID() { + mgr.mu.Lock() + defer mgr.mu.Unlock() + mgr.repairingDiskID = base.EmptyDiskID +} + +func (mgr *DiskRepairMgr) getRepairingDiskID() proto.DiskID { + mgr.mu.Lock() + defer mgr.mu.Unlock() + return mgr.repairingDiskID +} + +func (mgr *DiskRepairMgr) hasRepairingDisk() bool { + mgr.mu.Lock() + defer mgr.mu.Unlock() + return mgr.repairingDiskID != base.EmptyDiskID +} + +// AcquireTask acquire repair task +func (mgr *DiskRepairMgr) AcquireTask(ctx context.Context, idc string) (*proto.VolRepairTask, error) { + if !mgr.taskSwitch.Enabled() { + return nil, proto.ErrTaskPaused + } + + _, task, _ := mgr.workQueue.Acquire(idc) + if task != nil { + t := task.(*proto.VolRepairTask) + return t, nil + } + return nil, proto.ErrTaskEmpty +} + +// CancelTask cancel repair task +func (mgr *DiskRepairMgr) CancelTask(ctx context.Context, args *api.CancelTaskArgs) error { + span := trace.SpanFromContextSafe(ctx) + + err := mgr.workQueue.Cancel(args.IDC, args.TaskId, args.Src, args.Dest) + if err != nil { + span.Errorf("cancel repair failed: task_id[%s], err[%+v]", args.TaskId, err) + } + + mgr.taskStatsMgr.CancelTask() + + return err +} + +// ReclaimTask reclaim repair task +func (mgr *DiskRepairMgr) ReclaimTask(ctx context.Context, + idc, taskID string, + src []proto.VunitLocation, + oldDst proto.VunitLocation, + newDst *client.AllocVunitInfo) error { + span := trace.SpanFromContextSafe(ctx) + + err := mgr.workQueue.Reclaim(idc, taskID, src, oldDst, newDst.Location(), newDst.DiskID) + if err != nil { + // task has finished,because only complete will remove task from queue + span.Errorf("reclaim repair task failed: task_id[%s], err[%+v]", taskID, err) + return err + } + + task, err := mgr.workQueue.Query(idc, taskID) + if err != nil { + span.Errorf("found task in workQueue failed: idc[%s], task_id[%s], err[%+v]", idc, taskID, err) + return err + } + + err = mgr.taskTbl.Update(ctx, task.(*proto.VolRepairTask)) + if err != nil { + span.Warnf("update reclaim task failed: task_id[%s], err[%+v]", taskID, err) + } + + mgr.taskStatsMgr.ReclaimTask() + return nil +} + +// CompleteTask complete repair task +func (mgr *DiskRepairMgr) CompleteTask(ctx context.Context, args *api.CompleteTaskArgs) error { + span := trace.SpanFromContextSafe(ctx) + + completeTask, err := mgr.workQueue.Complete(args.IDC, args.TaskId, args.Src, args.Dest) + if err != nil { + span.Errorf("complete repair task failed: task_id[%s], err[%+v]", args.TaskId, err) + return err + } + + t := completeTask.(*proto.VolRepairTask) + t.State = proto.RepairStateWorkCompleted + + mgr.finishQueue.PushTask(args.TaskId, t) + // as complete func is face to svr api, so can not loop save task + // to db until success, it will make saving task info to be difficult, + // that delay saving task info in finish stage is a simply way + return nil +} + +// RenewalTask renewal repair task +func (mgr *DiskRepairMgr) RenewalTask(ctx context.Context, idc, taskID string) error { + if !mgr.taskSwitch.Enabled() { + // renewal task stopping will touch off worker to stop task + return proto.ErrTaskPaused + } + + span := trace.SpanFromContextSafe(ctx) + err := mgr.workQueue.Renewal(idc, taskID) + if err != nil { + span.Warnf("renewal repair task failed: task_id[%s], err[%+v]", taskID, err) + } + + return err +} + +// ReportWorkerTaskStats reports task stats +func (mgr *DiskRepairMgr) ReportWorkerTaskStats(st *api.TaskReportArgs) { + mgr.taskStatsMgr.ReportWorkerTaskStats(st.TaskId, st.TaskStats, st.IncreaseDataSizeByte, st.IncreaseShardCnt) +} + +// QueryTask return task statistics +func (mgr *DiskRepairMgr) QueryTask(ctx context.Context, taskID string) (*api.RepairTaskDetail, error) { + detail := &api.RepairTaskDetail{} + taskInfo, err := mgr.taskTbl.Find(ctx, taskID) + if err != nil { + return detail, err + } + detail.TaskInfo = *taskInfo + + detailRunInfo, err := mgr.taskStatsMgr.QueryTaskDetail(taskID) + if err != nil { + return detail, nil + } + detail.RunStats = detailRunInfo.Statistics + return detail, nil +} + +// StatQueueTaskCnt returns task queue stats +func (mgr *DiskRepairMgr) StatQueueTaskCnt() (inited, prepared, completed int) { + todo, doing := mgr.prepareQueue.StatsTasks() + inited = todo + doing + + todo, doing = mgr.workQueue.StatsTasks() + prepared = todo + doing + + todo, doing = mgr.finishQueue.StatsTasks() + completed = todo + doing + return +} + +// Stats returns task stats +func (mgr *DiskRepairMgr) Stats() api.MigrateTasksStat { + preparing, workerDoing, finishing := mgr.StatQueueTaskCnt() + finishedCnt := mgr.finishTaskCounter.Show() + increaseDataSize, increaseShardCnt := mgr.taskStatsMgr.Counters() + return api.MigrateTasksStat{ + PreparingCnt: preparing, + WorkerDoingCnt: workerDoing, + FinishingCnt: finishing, + StatsPerMin: api.PerMinStats{ + FinishedCnt: fmt.Sprint(finishedCnt), + DataAmountByte: base.DataMountFormat(increaseDataSize), + ShardCnt: fmt.Sprint(increaseShardCnt), + }, + } +} + +// Progress repair manager progress +func (mgr *DiskRepairMgr) Progress(ctx context.Context) (repairingDiskID proto.DiskID, total, repaired int) { + span := trace.SpanFromContextSafe(ctx) + repairingDiskID = mgr.getRepairingDiskID() + if repairingDiskID == base.EmptyDiskID { + return base.EmptyDiskID, 0, 0 + } + + allTasks, err := mgr.taskTbl.FindByDiskID(ctx, repairingDiskID) + if err != nil { + span.Errorf("find all task failed: err[%+v]", err) + return repairingDiskID, 0, 0 + } + total = len(allTasks) + for _, task := range allTasks { + if task.Finished() { + repaired++ + } + } + + return repairingDiskID, total, repaired +} diff --git a/blobstore/scheduler/disk_repairer_test.go b/blobstore/scheduler/disk_repairer_test.go new file mode 100644 index 000000000..5d8ff8475 --- /dev/null +++ b/blobstore/scheduler/disk_repairer_test.go @@ -0,0 +1,721 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + api "github.com/cubefs/cubefs/blobstore/api/scheduler" + "github.com/cubefs/cubefs/blobstore/common/codemode" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/scheduler/base" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/testing/mocks" +) + +func newMockVolInfoMap() map[proto.Vid]*client.VolumeInfoSimple { + return map[proto.Vid]*client.VolumeInfoSimple{ + 1: MockGenVolInfo(1, codemode.EC6P6, proto.VolumeStatusIdle), + 2: MockGenVolInfo(2, codemode.EC6P10L2, proto.VolumeStatusIdle), + 3: MockGenVolInfo(3, codemode.EC6P10L2, proto.VolumeStatusActive), + 4: MockGenVolInfo(4, codemode.EC6P6, proto.VolumeStatusLock), + 5: MockGenVolInfo(5, codemode.EC6P6, proto.VolumeStatusLock), + + 6: MockGenVolInfo(6, codemode.EC6P6, proto.VolumeStatusLock), + 7: MockGenVolInfo(7, codemode.EC6P6, proto.VolumeStatusLock), + } +} + +func newDiskRepairer(t *testing.T) *DiskRepairMgr { + ctr := gomock.NewController(t) + clusterMgr := NewMockClusterMgrAPI(ctr) + taskSwitch := mocks.NewMockSwitcher(ctr) + repairTable := NewMockRepairTaskTable(ctr) + conf := &DiskRepairMgrCfg{ + TaskCommonConfig: base.TaskCommonConfig{ + CollectTaskIntervalS: 1, + CheckTaskIntervalS: 1, + }, + } + return NewRepairMgr(conf, taskSwitch, repairTable, clusterMgr) +} + +func TestDiskRepairerLoad(t *testing.T) { + { + mgr := newDiskRepairer(t) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindAll(any).Return(nil, errMock) + err := mgr.Load() + require.True(t, errors.Is(err, errMock)) + } + { + mgr := newDiskRepairer(t) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindAll(any).Return(nil, nil) + err := mgr.Load() + require.NoError(t, err) + } + { + mgr := newDiskRepairer(t) + t1 := mockGenVolRepairTask(1, proto.RepairStateInited, 1, newMockVolInfoMap()) + t2 := mockGenVolRepairTask(2, proto.RepairStatePrepared, 2, newMockVolInfoMap()) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindAll(any).Return([]*proto.VolRepairTask{t1, t2}, nil) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return(nil, errMock) + err := mgr.Load() + require.True(t, errors.Is(err, errMock)) + } + { + mgr := newDiskRepairer(t) + t1 := mockGenVolRepairTask(1, proto.RepairStateInited, 1, newMockVolInfoMap()) + t2 := mockGenVolRepairTask(2, proto.RepairStatePrepared, 2, newMockVolInfoMap()) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindAll(any).Return([]*proto.VolRepairTask{t1, t2}, nil) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1}, nil) + require.Panics(t, func() { + mgr.Load() + }) + } + { + mgr := newDiskRepairer(t) + t1 := mockGenVolRepairTask(1, proto.RepairStateInited, 1, newMockVolInfoMap()) + t2 := mockGenVolRepairTask(2, proto.RepairStatePrepared, 1, newMockVolInfoMap()) + t3 := mockGenVolRepairTask(3, proto.RepairStateFinishedInAdvance, 1, newMockVolInfoMap()) + t4 := mockGenVolRepairTask(4, proto.RepairStateWorkCompleted, 1, newMockVolInfoMap()) + t5 := mockGenVolRepairTask(5, proto.RepairStateFinished, 1, newMockVolInfoMap()) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindAll(any).Return([]*proto.VolRepairTask{t1, t2, t3, t4, t5}, nil) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1, t2, t3, t4, t5}, nil) + err := mgr.Load() + require.NoError(t, err) + } + { + mgr := newDiskRepairer(t) + t1 := mockGenVolRepairTask(1, proto.RepairStateInited, 1, newMockVolInfoMap()) + t2 := mockGenVolRepairTask(2, proto.RepairStatePrepared, 2, newMockVolInfoMap()) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindAll(any).Return([]*proto.VolRepairTask{t1, t2}, nil) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1, t2}, nil) + require.Panics(t, func() { + mgr.Load() + }) + } + { + mgr := newDiskRepairer(t) + t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap()) + t2 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap()) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindAll(any).Return([]*proto.VolRepairTask{t1, t2}, nil) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1, t2}, nil) + require.Panics(t, func() { + mgr.Load() + }) + } + { + mgr := newDiskRepairer(t) + t1 := mockGenVolRepairTask(1, proto.RepairState(111), 1, newMockVolInfoMap()) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindAll(any).Return([]*proto.VolRepairTask{t1}, nil) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1}, nil) + require.Panics(t, func() { + mgr.Load() + }) + } +} + +func TestDiskRepairerRun(t *testing.T) { + mgr := newDiskRepairer(t) + defer mgr.Close() + + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().WaitEnable().AnyTimes().Return() + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().AnyTimes().Return(true) + + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).AnyTimes().Return(nil, errMock) + require.True(t, mgr.Enabled()) + mgr.hasRevised = true + mgr.repairingDiskID = proto.DiskID(1) + + mgr.Run() + time.Sleep(1 * time.Second) +} + +func TestDiskRepairerCollectTask(t *testing.T) { + { + mgr := newDiskRepairer(t) + mgr.hasRevised = false + mgr.repairingDiskID = proto.DiskID(1) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(nil, errMock) + mgr.collectTask() + } + { + mgr := newDiskRepairer(t) + mgr.hasRevised = false + mgr.repairingDiskID = proto.DiskID(1) + // genDiskRepairTasks failed + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(&client.DiskInfoSimple{DiskID: mgr.repairingDiskID, Status: proto.DiskStatusBroken}, nil) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return(nil, errMock) + mgr.collectTask() + + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return(nil, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(&client.DiskInfoSimple{DiskID: mgr.repairingDiskID, Status: proto.DiskStatusBroken}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, errMock) + mgr.collectTask() + + // gen task success + volume := MockGenVolInfo(10, codemode.EC6P6, proto.VolumeStatusIdle) + var units []*client.VunitInfoSimple + for _, unit := range volume.VunitLocations { + ele := client.VunitInfoSimple{ + Vuid: unit.Vuid, + DiskID: unit.DiskID, + } + units = append(units, &ele) + } + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return(nil, nil) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Insert(any, any).AnyTimes().Return(nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(&client.DiskInfoSimple{DiskID: mgr.repairingDiskID, Status: proto.DiskStatusBroken}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().SetDiskRepairing(any, any).Return(nil) + mgr.collectTask() + todo, doing := mgr.prepareQueue.StatsTasks() + require.Equal(t, 12, todo+doing) + require.Equal(t, true, mgr.hasRevised) + } + { + mgr := newDiskRepairer(t) + mgr.hasRevised = true + mgr.repairingDiskID = proto.DiskID(0) + + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListBrokenDisks(any, any).Return(nil, errMock) + mgr.collectTask() + + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListBrokenDisks(any, any).Return(nil, nil) + mgr.collectTask() + + disk1 := &client.DiskInfoSimple{ + ClusterID: 1, + Idc: "z0", + Rack: "rack1", + Host: "127.0.0.1:8000", + Status: proto.DiskStatusBroken, + DiskID: proto.DiskID(1), + FreeChunkCnt: 10, + MaxChunkCnt: 700, + } + + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListBrokenDisks(any, any).Return([]*client.DiskInfoSimple{disk1}, nil) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return(nil, errMock) + mgr.collectTask() + + } + { + mgr := newDiskRepairer(t) + mgr.hasRevised = true + mgr.repairingDiskID = proto.DiskID(0) + + disk1 := &client.DiskInfoSimple{ + ClusterID: 1, + Idc: "z0", + Rack: "rack1", + Host: "127.0.0.1:8000", + Status: proto.DiskStatusBroken, + DiskID: proto.DiskID(1), + FreeChunkCnt: 10, + MaxChunkCnt: 700, + } + + volume := MockGenVolInfo(10, codemode.EC6P6, proto.VolumeStatusIdle) + var units []*client.VunitInfoSimple + for _, unit := range volume.VunitLocations { + ele := client.VunitInfoSimple{ + Vuid: unit.Vuid, + DiskID: unit.DiskID, + } + units = append(units, &ele) + } + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListBrokenDisks(any, any).Return([]*client.DiskInfoSimple{disk1}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().SetDiskRepairing(any, any).Return(nil) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return(nil, nil) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Insert(any, any).AnyTimes().Return(nil) + mgr.collectTask() + todo, doing := mgr.prepareQueue.StatsTasks() + require.Equal(t, disk1.DiskID, mgr.repairingDiskID) + require.Equal(t, 12, todo+doing) + } + { + mgr := newDiskRepairer(t) + mgr.hasRevised = true + mgr.repairingDiskID = proto.DiskID(0) + + disk1 := &client.DiskInfoSimple{ + ClusterID: 1, + Idc: "z0", + Rack: "rack1", + Host: "127.0.0.1:8000", + Status: proto.DiskStatusBroken, + DiskID: proto.DiskID(1), + FreeChunkCnt: 10, + MaxChunkCnt: 700, + } + + volume := MockGenVolInfo(10, codemode.EC6P6, proto.VolumeStatusIdle) + var units []*client.VunitInfoSimple + for _, unit := range volume.VunitLocations { + ele := client.VunitInfoSimple{ + Vuid: unit.Vuid, + DiskID: unit.DiskID, + } + units = append(units, &ele) + } + t1 := &proto.VolRepairTask{ + TaskID: base.GenTaskID("disk-repair", volume.Vid), + BadVuid: units[0].Vuid, + } + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListBrokenDisks(any, any).Return([]*client.DiskInfoSimple{disk1}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().SetDiskRepairing(any, any).Return(nil) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1}, nil) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Insert(any, any).AnyTimes().Return(nil) + mgr.collectTask() + todo, doing := mgr.prepareQueue.StatsTasks() + require.Equal(t, disk1.DiskID, mgr.repairingDiskID) + require.Equal(t, 11, todo+doing) + } +} + +func TestDiskRepairerPopTaskAndPrepare(t *testing.T) { + { + mgr := newDiskRepairer(t) + err := mgr.popTaskAndPrepare() + require.True(t, errors.Is(err, base.ErrNoTaskInQueue)) + } + { + mgr := newDiskRepairer(t) + mgr.hasRevised = true + mgr.repairingDiskID = proto.DiskID(0) + + disk1 := &client.DiskInfoSimple{ + ClusterID: 1, + Idc: "z0", + Rack: "rack1", + Host: "127.0.0.1:8000", + Status: proto.DiskStatusBroken, + DiskID: proto.DiskID(1), + FreeChunkCnt: 10, + MaxChunkCnt: 700, + } + + volume := MockGenVolInfo(10, codemode.EC6P6, proto.VolumeStatusIdle) + var units []*client.VunitInfoSimple + for _, unit := range volume.VunitLocations { + ele := client.VunitInfoSimple{ + Vuid: unit.Vuid, + DiskID: unit.DiskID, + } + units = append(units, &ele) + } + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListBrokenDisks(any, any).Return([]*client.DiskInfoSimple{disk1}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().SetDiskRepairing(any, any).Return(nil) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return(nil, nil) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Insert(any, any).AnyTimes().Return(nil) + mgr.collectTask() + todo, doing := mgr.prepareQueue.StatsTasks() + require.Equal(t, disk1.DiskID, mgr.repairingDiskID) + require.Equal(t, 12, todo+doing) + + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(nil, errMock) + err := mgr.popTaskAndPrepare() + require.True(t, errors.Is(err, errMock)) + + // finish in advance + volume.VunitLocations[0].Vuid = volume.VunitLocations[0].Vuid + 1 + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(volume, nil) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Update(any, any).Return(nil) + err = mgr.popTaskAndPrepare() + todo, doing = mgr.prepareQueue.StatsTasks() + + require.NoError(t, err) + require.Equal(t, 11, todo+doing) + + // alloc volume unit failed + volume.VunitLocations[0].Vuid = volume.VunitLocations[0].Vuid - 1 + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(any, any).Return(nil, errMock) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(volume, nil) + err = mgr.popTaskAndPrepare() + require.True(t, errors.Is(err, errMock)) + + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(any, any).DoAndReturn(func(ctx context.Context, vuid proto.Vuid) (*client.AllocVunitInfo, error) { + vid := vuid.Vid() + idx := vuid.Index() + epoch := vuid.Epoch() + epoch++ + newVuid, _ := proto.NewVuid(vid, idx, epoch) + return &client.AllocVunitInfo{ + VunitLocation: proto.VunitLocation{Vuid: newVuid}, + }, nil + }) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(volume, nil) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Update(any, any).Return(nil) + err = mgr.popTaskAndPrepare() + require.NoError(t, err) + + todo, doing = mgr.prepareQueue.StatsTasks() + require.Equal(t, 10, todo+doing) + todo, doing = mgr.workQueue.StatsTasks() + require.Equal(t, 1, todo+doing) + } +} + +func TestDiskRepairerPopTaskAndFinish(t *testing.T) { + { + mgr := newDiskRepairer(t) + err := mgr.popTaskAndFinish() + require.True(t, errors.Is(err, base.ErrNoTaskInQueue)) + } + { + mgr := newDiskRepairer(t) + t1 := mockGenVolRepairTask(1, proto.RepairStateFinished, 1, newMockVolInfoMap()) + mgr.finishQueue.PushTask(t1.TaskID, t1) + require.Panics(t, func() { + mgr.popTaskAndFinish() + }) + } + { + mgr := newDiskRepairer(t) + t1 := mockGenVolRepairTask(1, proto.RepairStateWorkCompleted, 1, newMockVolInfoMap()) + mgr.finishQueue.PushTask(t1.TaskID, t1) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Update(any, any).Return(nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateVolume(any, any, any, any).Return(errMock) + err := mgr.popTaskAndFinish() + require.True(t, errors.Is(err, errMock)) + } + { + mgr := newDiskRepairer(t) + t1 := mockGenVolRepairTask(1, proto.RepairStateWorkCompleted, 1, newMockVolInfoMap()) + mgr.finishQueue.PushTask(t1.TaskID, t1) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Update(any, any).Return(nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateVolume(any, any, any, any).Return(errcode.ErrOldVuidNotMatch) + require.Panics(t, func() { + mgr.popTaskAndFinish() + }) + } + { + mgr := newDiskRepairer(t) + t1 := mockGenVolRepairTask(1, proto.RepairStateWorkCompleted, 1, newMockVolInfoMap()) + mgr.finishQueue.PushTask(t1.TaskID, t1) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Update(any, any).Return(nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateVolume(any, any, any, any).Return(errcode.ErrNewVuidNotMatch) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(any, any).Return(nil, errMock) + err := mgr.popTaskAndFinish() + require.True(t, errors.Is(err, errMock)) + } + { + mgr := newDiskRepairer(t) + t1 := mockGenVolRepairTask(1, proto.RepairStateWorkCompleted, 1, newMockVolInfoMap()) + mgr.finishQueue.PushTask(t1.TaskID, t1) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Update(any, any).Times(2).Return(nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateVolume(any, any, any, any).Return(errcode.ErrNewVuidNotMatch) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(any, any).DoAndReturn(func(ctx context.Context, vuid proto.Vuid) (*client.AllocVunitInfo, error) { + vid := vuid.Vid() + idx := vuid.Index() + epoch := vuid.Epoch() + epoch++ + newVuid, _ := proto.NewVuid(vid, idx, epoch) + return &client.AllocVunitInfo{ + VunitLocation: proto.VunitLocation{Vuid: newVuid}, + }, nil + }) + err := mgr.popTaskAndFinish() + require.NoError(t, err) + todo, doing := mgr.finishQueue.StatsTasks() + require.Equal(t, 0, todo+doing) + todo, doing = mgr.workQueue.StatsTasks() + require.Equal(t, 1, todo+doing) + } + { + mgr := newDiskRepairer(t) + t1 := mockGenVolRepairTask(1, proto.RepairStateWorkCompleted, 1, newMockVolInfoMap()) + mgr.finishQueue.PushTask(t1.TaskID, t1) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Update(any, any).Times(2).Return(nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateVolume(any, any, any, any).Return(errcode.ErrStatChunkFailed) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(any, any).DoAndReturn(func(ctx context.Context, vuid proto.Vuid) (*client.AllocVunitInfo, error) { + vid := vuid.Vid() + idx := vuid.Index() + epoch := vuid.Epoch() + epoch++ + newVuid, _ := proto.NewVuid(vid, idx, epoch) + return &client.AllocVunitInfo{ + VunitLocation: proto.VunitLocation{Vuid: newVuid}, + }, nil + }) + err := mgr.popTaskAndFinish() + require.NoError(t, err) + todo, doing := mgr.finishQueue.StatsTasks() + require.Equal(t, 0, todo+doing) + todo, doing = mgr.workQueue.StatsTasks() + require.Equal(t, 1, todo+doing) + } + { + mgr := newDiskRepairer(t) + t1 := mockGenVolRepairTask(1, proto.RepairStateWorkCompleted, 1, newMockVolInfoMap()) + mgr.finishQueue.PushTask(t1.TaskID, t1) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Update(any, any).Times(2).Return(nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateVolume(any, any, any, any).Return(nil) + err := mgr.popTaskAndFinish() + require.NoError(t, err) + todo, doing := mgr.finishQueue.StatsTasks() + require.Equal(t, 0, todo+doing) + } +} + +func TestDiskRepairerCheckRepairedAndClear(t *testing.T) { + { + mgr := newDiskRepairer(t) + mgr.checkRepairedAndClear() + } + { + mgr := newDiskRepairer(t) + mgr.repairingDiskID = proto.DiskID(1) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return(nil, errMock) + mgr.checkRepairedAndClear() + } + { + mgr := newDiskRepairer(t) + mgr.repairingDiskID = proto.DiskID(1) + volume := MockGenVolInfo(10, codemode.EC6P6, proto.VolumeStatusIdle) + var units []*client.VunitInfoSimple + for _, unit := range volume.VunitLocations { + ele := client.VunitInfoSimple{ + Vuid: unit.Vuid, + DiskID: unit.DiskID, + } + units = append(units, &ele) + } + t1 := &proto.VolRepairTask{ + TaskID: base.GenTaskID("disk-repair", volume.Vid), + BadVuid: units[0].Vuid, + } + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1}, nil) + mgr.checkRepairedAndClear() + + t1.State = proto.RepairStateFinished + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, errMock) + mgr.checkRepairedAndClear() + + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(nil, errMock) + mgr.checkRepairedAndClear() + + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().SetDiskRepaired(any, any).Return(errMock) + mgr.checkRepairedAndClear() + + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1}, nil) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().MarkDeleteByDiskID(any, any).Return(nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().SetDiskRepaired(any, any).Return(nil) + mgr.checkRepairedAndClear() + require.False(t, mgr.hasRepairingDisk()) + } +} + +func TestDiskRepairerAcquireTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + { + mgr := newDiskRepairer(t) + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(false) + _, err := mgr.AcquireTask(ctx, idc) + require.True(t, errors.Is(err, proto.ErrTaskPaused)) + } + { + mgr := newDiskRepairer(t) + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(true) + _, err := mgr.AcquireTask(ctx, idc) + require.True(t, errors.Is(err, proto.ErrTaskEmpty)) + } + { + mgr := newDiskRepairer(t) + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(true) + t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap()) + mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1) + _, err := mgr.AcquireTask(ctx, idc) + require.NoError(t, err) + } +} + +func TestDiskRepairerCancelTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + { + mgr := newDiskRepairer(t) + err := mgr.CancelTask(ctx, &api.CancelTaskArgs{}) + require.Error(t, err) + } + { + mgr := newDiskRepairer(t) + t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap()) + mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1) + + err := mgr.CancelTask(ctx, &api.CancelTaskArgs{}) + require.Error(t, err) + } +} + +func TestDiskRepairerReclaimTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + { + mgr := newDiskRepairer(t) + t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap()) + err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{}) + require.Error(t, err) + } + { + mgr := newDiskRepairer(t) + t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap()) + mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Update(any, any).Return(errMock) + err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{}) + require.NoError(t, err) + } + { + mgr := newDiskRepairer(t) + t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap()) + mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Update(any, any).Return(nil) + err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{}) + require.NoError(t, err) + } +} + +func TestDiskRepairerCompleteTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + { + mgr := newDiskRepairer(t) + t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap()) + err := mgr.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: idc, TaskId: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + require.Error(t, err) + } + { + mgr := newDiskRepairer(t) + t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap()) + mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1) + err := mgr.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: idc, TaskId: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + require.NoError(t, err) + todo, doing := mgr.finishQueue.StatsTasks() + require.Equal(t, 1, todo+doing) + todo, doing = mgr.workQueue.StatsTasks() + require.Equal(t, 0, todo+doing) + } +} + +func TestDiskRepairerRenewalTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + { + mgr := newDiskRepairer(t) + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(false) + err := mgr.RenewalTask(ctx, idc, "") + require.True(t, errors.Is(err, proto.ErrTaskPaused)) + } + { + mgr := newDiskRepairer(t) + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(true) + err := mgr.RenewalTask(ctx, idc, "") + require.Error(t, err) + } + { + mgr := newDiskRepairer(t) + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(true) + t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap()) + mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1) + err := mgr.RenewalTask(ctx, idc, t1.TaskID) + require.NoError(t, err) + } +} + +func TestDiskRepairerStats(t *testing.T) { + mgr := newDiskRepairer(t) + mgr.Stats() +} + +func TestDiskRepairerStatQueueTaskCnt(t *testing.T) { + mgr := newDiskRepairer(t) + inited, prepared, completed := mgr.StatQueueTaskCnt() + require.Equal(t, 0, inited) + require.Equal(t, 0, prepared) + require.Equal(t, 0, completed) +} + +func TestDiskRepairerQueryTask(t *testing.T) { + ctx := context.Background() + taskID := "task" + { + mgr := newDiskRepairer(t) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Find(any, any).Return(nil, errMock) + _, err := mgr.QueryTask(ctx, taskID) + require.True(t, errors.Is(err, errMock)) + } + { + mgr := newDiskRepairer(t) + t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap()) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().Find(any, any).Return(t1, nil) + _, err := mgr.QueryTask(ctx, taskID) + require.NoError(t, err) + } +} + +func TestDiskRepairerReportWorkerTaskStats(t *testing.T) { + mgr := newDiskRepairer(t) + mgr.ReportWorkerTaskStats(&api.TaskReportArgs{ + TaskId: "task", + IncreaseDataSizeByte: 1, + IncreaseShardCnt: 1, + }) +} + +func TestDiskRepairerProgress(t *testing.T) { + ctx := context.Background() + { + mgr := newDiskRepairer(t) + diskID, _, _ := mgr.Progress(ctx) + require.Equal(t, base.EmptyDiskID, diskID) + } + { + mgr := newDiskRepairer(t) + mgr.repairingDiskID = proto.DiskID(1) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return(nil, errMock) + diskID, _, _ := mgr.Progress(ctx) + require.Equal(t, proto.DiskID(1), diskID) + } + { + mgr := newDiskRepairer(t) + mgr.repairingDiskID = proto.DiskID(1) + t1 := mockGenVolRepairTask(1, proto.RepairStatePrepared, 1, newMockVolInfoMap()) + t2 := mockGenVolRepairTask(2, proto.RepairStateFinished, 1, newMockVolInfoMap()) + t3 := mockGenVolRepairTask(3, proto.RepairStateFinishedInAdvance, 1, newMockVolInfoMap()) + mgr.taskTbl.(*MockRepairTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.VolRepairTask{t1, t2, t3}, nil) + diskID, total, repaired := mgr.Progress(ctx) + require.Equal(t, proto.DiskID(1), diskID) + require.Equal(t, 3, total) + require.Equal(t, 2, repaired) + } +} diff --git a/blobstore/scheduler/manual_migrater.go b/blobstore/scheduler/manual_migrater.go new file mode 100644 index 000000000..80a88e8ca --- /dev/null +++ b/blobstore/scheduler/manual_migrater.go @@ -0,0 +1,95 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/taskswitch" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/scheduler/base" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/scheduler/db" +) + +// IManualMigrater interface of manual migrater +type IManualMigrater interface { + Migrator + AddManualTask(ctx context.Context, vuid proto.Vuid, forbiddenDirectDownload bool) (err error) +} + +// ManualMigrateMgr manual migrate manager +type ManualMigrateMgr struct { + IMigrater + + clusterMgrCli client.ClusterMgrAPI +} + +// NewManualMigrateMgr returns manual migrate manager +func NewManualMigrateMgr(clusterMgrCli client.ClusterMgrAPI, volumeUpdater client.IVolumeUpdater, + taskTbl db.IMigrateTaskTable, clusterID proto.ClusterID) *ManualMigrateMgr { + mgr := &ManualMigrateMgr{ + clusterMgrCli: clusterMgrCli, + } + cfg := defaultMigrateConfig(clusterID) + + mgr.IMigrater = NewMigrateMgr(clusterMgrCli, volumeUpdater, taskswitch.NewEnabledTaskSwitch(), taskTbl, + &cfg, proto.ManualMigrateType, clusterID) + mgr.IMigrater.SetLockFailHandleFunc(mgr.IMigrater.FinishTaskInAdvanceWhenLockFail) + return mgr +} + +// AddManualTask add manual migrate task +func (mgr *ManualMigrateMgr) AddManualTask(ctx context.Context, vuid proto.Vuid, forbiddenDirectDownload bool) (err error) { + span := trace.SpanFromContextSafe(ctx) + + volume, err := mgr.clusterMgrCli.GetVolumeInfo(ctx, vuid.Vid()) + if err != nil { + span.Errorf("get volume failed: vid[%d], err[%+v]", vuid.Vid(), err) + return err + } + diskID := volume.VunitLocations[vuid.Index()].DiskID + disk, err := mgr.clusterMgrCli.GetDiskInfo(ctx, diskID) + if err != nil { + span.Errorf("get disk info failed: disk_id[%d], err[%+v]", err) + return err + } + + task := &proto.MigrateTask{ + TaskID: mgr.genUniqTaskID(vuid.Vid()), + State: proto.MigrateStateInited, + SourceIdc: disk.Idc, + SourceDiskID: disk.DiskID, + SourceVuid: vuid, + ForbiddenDirectDownload: forbiddenDirectDownload, + } + mgr.IMigrater.AddTask(ctx, task) + + span.Debugf("add manual migrate task success: task_info[%+v]", task) + return nil +} + +func (mgr *ManualMigrateMgr) genUniqTaskID(vid proto.Vid) string { + return base.GenTaskID("manual_migrate", vid) +} + +func defaultMigrateConfig(clusterID proto.ClusterID) MigrateConfig { + cfg := MigrateConfig{ + ClusterID: clusterID, + } + cfg.CheckAndFix() + return cfg +} diff --git a/blobstore/scheduler/manual_migrater_test.go b/blobstore/scheduler/manual_migrater_test.go new file mode 100644 index 000000000..3cb5db0bb --- /dev/null +++ b/blobstore/scheduler/manual_migrater_test.go @@ -0,0 +1,134 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "errors" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + api "github.com/cubefs/cubefs/blobstore/api/scheduler" + "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/scheduler/client" +) + +func newManualMigrater(t *testing.T) *ManualMigrateMgr { + ctr := gomock.NewController(t) + clusterMgr := NewMockClusterMgrAPI(ctr) + volumeUpdater := NewMockVolumeUpdater(ctr) + migrateTable := NewMockMigrateTaskTable(ctr) + migrater := NewMockMigrater(ctr) + mgr := NewManualMigrateMgr(clusterMgr, volumeUpdater, migrateTable, proto.ClusterID(1)) + mgr.IMigrater = migrater + return mgr +} + +func TestManualMigrateLoad(t *testing.T) { + mgr := newManualMigrater(t) + mgr.IMigrater.(*MockMigrater).EXPECT().Load().Return(nil) + err := mgr.Load() + require.NoError(t, err) +} + +func TestManualMigrateRun(t *testing.T) { + mgr := newManualMigrater(t) + mgr.IMigrater.(*MockMigrater).EXPECT().Run().Return() + mgr.Run() +} + +func TestManualMigrateAddTask(t *testing.T) { + ctx := context.Background() + { + mgr := newManualMigrater(t) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(nil, errMock) + err := mgr.AddManualTask(ctx, proto.Vuid(1), false) + require.True(t, errors.Is(err, errMock)) + } + { + mgr := newManualMigrater(t) + volume := MockGenVolInfo(10001, codemode.EC6P6, proto.VolumeStatusIdle) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(volume, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(nil, errMock) + err := mgr.AddManualTask(ctx, proto.Vuid(1), false) + require.True(t, errors.Is(err, errMock)) + } + { + mgr := newManualMigrater(t) + volume := MockGenVolInfo(10001, codemode.EC6P6, proto.VolumeStatusIdle) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(volume, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(&client.DiskInfoSimple{}, nil) + mgr.IMigrater.(*MockMigrater).EXPECT().AddTask(any, any).Return() + err := mgr.AddManualTask(ctx, proto.Vuid(1), false) + require.NoError(t, err) + } +} + +func TestManualMigrateAcquireTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + mgr := newManualMigrater(t) + mgr.IMigrater.(*MockMigrater).EXPECT().AcquireTask(any, any).Return(&proto.MigrateTask{}, nil) + _, err := mgr.AcquireTask(ctx, idc) + require.NoError(t, err) +} + +func TestManualMigrateCancelTask(t *testing.T) { + ctx := context.Background() + mgr := newManualMigrater(t) + mgr.IMigrater.(*MockMigrater).EXPECT().CancelTask(any, any).Return(nil) + err := mgr.CancelTask(ctx, &api.CancelTaskArgs{}) + require.NoError(t, err) +} + +func TestManualMigrateReclaimTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + mgr := newManualMigrater(t) + mgr.IMigrater.(*MockMigrater).EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil) + t1 := mockGenMigrateTask(idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap) + err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{}) + require.NoError(t, err) +} + +func TestManualMigrateCompleteTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + mgr := newManualMigrater(t) + mgr.IMigrater.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(nil) + t1 := mockGenMigrateTask(idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap) + err := mgr.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: idc, TaskId: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + require.NoError(t, err) + + mgr.IMigrater.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(errMock) + err = mgr.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: idc, TaskId: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + require.True(t, errors.Is(err, errMock)) +} + +func TestManualMigrateRenewalTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + mgr := newManualMigrater(t) + mgr.IMigrater.(*MockMigrater).EXPECT().RenewalTask(any, any, any).Return(nil) + err := mgr.RenewalTask(ctx, idc, "") + require.NoError(t, err) + + mgr.IMigrater.(*MockMigrater).EXPECT().RenewalTask(any, any, any).Return(errMock) + err = mgr.RenewalTask(ctx, idc, "") + require.True(t, errors.Is(err, errMock)) +} diff --git a/blobstore/scheduler/migrate.go b/blobstore/scheduler/migrate.go new file mode 100644 index 000000000..ef2994e09 --- /dev/null +++ b/blobstore/scheduler/migrate.go @@ -0,0 +1,718 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "fmt" + "sync" + "time" + + api "github.com/cubefs/cubefs/blobstore/api/scheduler" + "github.com/cubefs/cubefs/blobstore/common/counter" + "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/taskswitch" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/scheduler/base" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/scheduler/db" + "github.com/cubefs/cubefs/blobstore/util/closer" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +const ( + prepareMigrateTaskIntervalS = 1 + finishMigrateTaskIntervalS = 1 + prepareTaskPauseS = 2 +) + +// MMigrater merged interfaces for mocking. +type MMigrater interface { + IMigrater + IBalancer + IDiskDroper + IManualMigrater +} + +// Migrator base interface of migrate, balancer, disk_droper, manual_migrater. +type Migrator interface { + AcquireTask(ctx context.Context, idc string) (*proto.MigrateTask, error) + CancelTask(ctx context.Context, args *api.CancelTaskArgs) error + CompleteTask(ctx context.Context, args *api.CompleteTaskArgs) error + ReclaimTask(ctx context.Context, idc, taskID string, + src []proto.VunitLocation, oldDst proto.VunitLocation, newDst *client.AllocVunitInfo) error + RenewalTask(ctx context.Context, idc, taskID string) error + QueryTask(ctx context.Context, taskID string) (*api.MigrateTaskDetail, error) + // status + ReportWorkerTaskStats(st *api.TaskReportArgs) + StatQueueTaskCnt() (inited, prepared, completed int) + Stats() api.MigrateTasksStat + // control + taskswitch.ISwitcher + closer.Closer + Load() error + Run() +} + +// IMigrater interface of common migrator +type IMigrater interface { + Migrator + // inner interface + SetLockFailHandleFunc(lockFailHandleFunc func(ctx context.Context, task *proto.MigrateTask)) + AddTask(ctx context.Context, task *proto.MigrateTask) + GetMigratingDiskNum() int + IsMigratingDisk(diskID proto.DiskID) bool + ClearTasksByStates(ctx context.Context, states []proto.MigrateState) + FindAll(ctx context.Context) (tasks []*proto.MigrateTask, err error) + FindTask(ctx context.Context, taskID string) (*proto.MigrateTask, error) + FindByDiskID(ctx context.Context, diskID proto.DiskID) (tasks []*proto.MigrateTask, err error) + FinishTaskInAdvanceWhenLockFail(ctx context.Context, task *proto.MigrateTask) + ClearTasksByDiskID(ctx context.Context, diskID proto.DiskID) +} + +// MigratingVuids record migrating vuid info +type MigratingVuids map[proto.Vuid]string + +type diskMigratingVuids struct { + vuids map[proto.DiskID]MigratingVuids + lock sync.RWMutex +} + +func newDiskMigratingVuids() *diskMigratingVuids { + return &diskMigratingVuids{ + vuids: make(map[proto.DiskID]MigratingVuids), + } +} + +func (m *diskMigratingVuids) addMigratingVuid(diskID proto.DiskID, vuid proto.Vuid, taskID string) { + m.lock.Lock() + if m.vuids[diskID] == nil { + m.vuids[diskID] = make(MigratingVuids) + } + m.vuids[diskID][vuid] = taskID + m.lock.Unlock() +} + +func (m *diskMigratingVuids) deleteMigratingVuid(diskID proto.DiskID, vuid proto.Vuid) { + m.lock.Lock() + delete(m.vuids[diskID], vuid) + if len(m.vuids[diskID]) == 0 { + delete(m.vuids, diskID) + } + m.lock.Unlock() +} + +func (m *diskMigratingVuids) getCurrMigratingDisksCnt() int { + m.lock.RLock() + defer m.lock.RUnlock() + return len(m.vuids) +} + +func (m *diskMigratingVuids) isMigratingDisk(diskID proto.DiskID) (ok bool) { + m.lock.RLock() + defer m.lock.RUnlock() + _, ok = m.vuids[diskID] + return +} + +// MigrateConfig migrate config +type MigrateConfig struct { + ClusterID proto.ClusterID + base.TaskCommonConfig +} + +// MigrateMgr migrate manager +type MigrateMgr struct { + closer.Closer + + taskType string + diskMigratingVuids *diskMigratingVuids + + taskTbl db.IMigrateTaskTable + + clusterMgrCli client.ClusterMgrAPI + volumeUpdater client.IVolumeUpdater + + taskSwitch taskswitch.ISwitcher + + prepareQueue *base.TaskQueue // store inited task + workQueue *base.WorkerTaskQueue // store prepared task + finishQueue *base.TaskQueue // store completed task + + finishTaskCounter counter.Counter + taskStatsMgr *base.TaskStatsMgr + + cfg *MigrateConfig + + // handle func when lock volume fail + lockFailHandleFunc func(ctx context.Context, task *proto.MigrateTask) +} + +// NewMigrateMgr returns migrate manager +func NewMigrateMgr( + clusterMgrCli client.ClusterMgrAPI, + volumeUpdater client.IVolumeUpdater, + taskSwitch taskswitch.ISwitcher, + taskTbl db.IMigrateTaskTable, + conf *MigrateConfig, + taskType string, + clusterID proto.ClusterID, +) *MigrateMgr { + mgr := &MigrateMgr{ + taskType: taskType, + diskMigratingVuids: newDiskMigratingVuids(), + + taskTbl: taskTbl, + + taskSwitch: taskSwitch, + + clusterMgrCli: clusterMgrCli, + volumeUpdater: volumeUpdater, + + prepareQueue: base.NewTaskQueue(time.Duration(conf.PrepareQueueRetryDelayS) * time.Second), + workQueue: base.NewWorkerTaskQueue(time.Duration(conf.CancelPunishDurationS) * time.Second), + finishQueue: base.NewTaskQueue(time.Duration(conf.FinishQueueRetryDelayS) * time.Second), + + cfg: conf, + + Closer: closer.New(), + } + + mgr.taskStatsMgr = base.NewTaskStatsMgrAndRun(clusterID, taskType, mgr) + return mgr +} + +// SetLockFailHandleFunc set lock failed func +func (mgr *MigrateMgr) SetLockFailHandleFunc(lockFailHandleFunc func(ctx context.Context, task *proto.MigrateTask)) { + mgr.lockFailHandleFunc = lockFailHandleFunc +} + +// Load load migrate task from database +func (mgr *MigrateMgr) Load() (err error) { + log.Infof("start load migrate task: task_type[%s]", mgr.taskType) + ctx := context.Background() + + // load task from db + tasks, err := mgr.taskTbl.FindAll(ctx) + if err != nil { + log.Errorf("find all tasks failed: err[%+v]", err) + return + } + log.Infof("load task success: task_type[%s], tasks len[%d]", mgr.taskType, len(tasks)) + + for i := range tasks { + if tasks[i].Running() { + err = base.VolTaskLockerInst().TryLock(ctx, tasks[i].SourceVuid.Vid()) + if err != nil { + log.Panicf("migrate task conflict: vid[%d], task[%+v], err[%+v]", + tasks[i].SourceVuid.Vid(), tasks[i], err.Error()) + } + } + + if !tasks[i].Finished() { + mgr.diskMigratingVuids.addMigratingVuid(tasks[i].SourceDiskID, tasks[i].SourceVuid, tasks[i].TaskID) + } + + log.Infof("load task success: task_type[%s], task_id[%s], state[%d]", mgr.taskType, tasks[i].TaskID, tasks[i].State) + switch tasks[i].State { + case proto.MigrateStateInited: + mgr.prepareQueue.PushTask(tasks[i].TaskID, tasks[i]) + case proto.MigrateStatePrepared: + mgr.workQueue.AddPreparedTask(tasks[i].SourceIdc, tasks[i].TaskID, tasks[i]) + case proto.MigrateStateWorkCompleted: + mgr.finishQueue.PushTask(tasks[i].TaskID, tasks[i]) + case proto.MigrateStateFinished, proto.MigrateStateFinishedInAdvance: + continue + default: + log.Panicf("unexpect migrate state: task[%+v]", tasks[i]) + } + } + return +} + +// Run run migrate task do prepare and finish task phase +func (mgr *MigrateMgr) Run() { + go mgr.prepareTaskLoop() + go mgr.finishTaskLoop() +} + +func (mgr *MigrateMgr) prepareTaskLoop() { + for { + mgr.taskSwitch.WaitEnable() + todo, doing := mgr.workQueue.StatsTasks() + if todo+doing >= mgr.cfg.WorkQueueSize { + time.Sleep(time.Duration(prepareTaskPauseS) * time.Second) + continue + } + err := mgr.prepareTask() + if err == base.ErrNoTaskInQueue { + log.Debugf("no task in prepare queue and sleep: sleep second[%d]", prepareMigrateTaskIntervalS) + time.Sleep(time.Duration(prepareMigrateTaskIntervalS) * time.Second) + } + } +} + +func (mgr *MigrateMgr) prepareTask() (err error) { + _, task, exist := mgr.prepareQueue.PopTask() + if !exist { + return base.ErrNoTaskInQueue + } + + span, ctx := trace.StartSpanFromContext(context.Background(), "migrate.prepareTask") + defer span.Finish() + + defer func() { + if err != nil { + mgr.prepareQueue.RetryTask(task.(*proto.MigrateTask).TaskID) + } + }() + + migTask := task.(*proto.MigrateTask).Copy() + + span.Infof("prepare task phase: task_id[%s], state[%+v]", migTask.TaskID, migTask.State) + + err = base.VolTaskLockerInst().TryLock(ctx, migTask.SourceVuid.Vid()) + if err != nil { + span.Warnf("lock volume failed: volume_id[%v], err[%+v]", migTask.SourceVuid.Vid(), err) + return base.ErrVolNotOnlyOneTask + } + defer func() { + if err != nil { + base.VolTaskLockerInst().Unlock(ctx, task.(*proto.MigrateTask).SourceVuid.Vid()) + } + }() + + volInfo, err := mgr.clusterMgrCli.GetVolumeInfo(ctx, migTask.SourceVuid.Vid()) + if err != nil { + span.Errorf("prepare task failed: err[%v]", err) + return err + } + + // check necessity of generating current task + if migTask.SourceVuid != volInfo.VunitLocations[migTask.SourceVuid.Index()].Vuid { + span.Infof("the source unit has been moved and finish task immediately: task_id[%s], task source vuid[%v], current vuid[%v]", + migTask.TaskID, migTask.SourceVuid, volInfo.VunitLocations[migTask.SourceVuid.Index()].Vuid) + + // volume may be locked, try unlock the volume + // for example + // 1. lock volume success + // 2. alloc chunk failed and VolTaskLockerInst().Unlock + // 3. this volume maybe execute other tasks, such as disk repair + // 4. then enter this branch and volume status is locked + err := mgr.clusterMgrCli.UnlockVolume(ctx, migTask.SourceVuid.Vid()) + if err != nil { + span.Errorf("before finish in advance try unlock volume failed: vid[%d], err[%+v]", + migTask.SourceVuid.Vid(), err) + return err + } + + mgr.finishTaskInAdvance(ctx, migTask, "volume has migrated") + return nil + } + + // lock volume + err = mgr.clusterMgrCli.LockVolume(ctx, migTask.SourceVuid.Vid()) + if err != nil { + if rpc.DetectStatusCode(err) == errors.CodeLockNotAllow && mgr.lockFailHandleFunc != nil { + mgr.lockFailHandleFunc(ctx, migTask) + return nil + } + span.Errorf("lock volume failed: volume_id[%v], err[%+v]", migTask.SourceVuid.Vid(), err) + return err + } + + // alloc volume unit + ret, err := base.AllocVunitSafe(ctx, mgr.clusterMgrCli, migTask.SourceVuid, migTask.Sources) + if err != nil { + span.Errorf("alloc volume unit failed: err[%+v]", err) + return + } + + migTask.CodeMode = volInfo.CodeMode + migTask.Sources = volInfo.VunitLocations + migTask.SetDest(ret.Location()) + migTask.State = proto.MigrateStatePrepared + + // update db + base.InsistOn(ctx, "migrate prepare task update task tbl", func() error { + return mgr.taskTbl.Update(ctx, proto.MigrateStateInited, migTask) + }) + + // send task to worker queue and remove task in prepareQueue + mgr.workQueue.AddPreparedTask(migTask.SourceIdc, migTask.TaskID, migTask) + mgr.prepareQueue.RemoveTask(migTask.TaskID) + + span.Infof("prepare task success: task_id[%s], state[%v]", migTask.TaskID, migTask.State) + return +} + +func (mgr *MigrateMgr) finishTaskLoop() { + for { + mgr.taskSwitch.WaitEnable() + err := mgr.finishTask() + if err == base.ErrNoTaskInQueue { + log.Debugf("no task in finish queue and sleep: sleep second[%d]", finishMigrateTaskIntervalS) + time.Sleep(time.Duration(finishMigrateTaskIntervalS) * time.Second) + } + } +} + +func (mgr *MigrateMgr) finishTask() (err error) { + _, task, exist := mgr.finishQueue.PopTask() + if !exist { + return base.ErrNoTaskInQueue + } + + span, ctx := trace.StartSpanFromContext(context.Background(), "migrate.finishTask") + defer span.Finish() + + defer func() { + if err != nil { + mgr.finishQueue.RetryTask(task.(*proto.MigrateTask).TaskID) + } + }() + + migTask := task.(*proto.MigrateTask).Copy() + span.Infof("finish task phase: task_id[%s], state[%v]", migTask.TaskID, migTask.State) + + if migTask.State != proto.MigrateStateWorkCompleted { + span.Panicf("unexpect task state: task_id[%s], expect state[%d], actual state[%d]", proto.MigrateStateWorkCompleted, migTask.State) + } + + // because competed task did not persisted to the database, so in finish phase need to do it + // the task maybe update more than once, which is allowed + base.InsistOn(ctx, "migrate finish task update task tbl to state completed ", func() error { + return mgr.taskTbl.Update(ctx, proto.MigrateStatePrepared, migTask) + }) + + // update volume mapping relationship + err = mgr.clusterMgrCli.UpdateVolume(ctx, migTask.Destination.Vuid, migTask.SourceVuid, migTask.DestinationDiskId()) + if err != nil { + span.Errorf("change volume unit relationship failed: old vuid[%d], new vuid[%d], new diskId[%d], err[%+v]", + migTask.SourceVuid, + migTask.Destination.Vuid, + migTask.DestinationDiskId(), + err) + return mgr.handleUpdateVolMappingFail(ctx, migTask, err) + } + + err = mgr.clusterMgrCli.ReleaseVolumeUnit(ctx, migTask.SourceVuid, migTask.SourceDiskID) + if err != nil { + span.Errorf("release volume unit failed: err[%+v]", err) + // 1. CodeVuidNotFound means the volume unit dose not exist and ignore it + // 2. CodeDiskBroken need ignore it + // 3. Other err, all scheduler need to be notified to update the volume mapping relationship + // to avoid affecting the deletion process due to caching the old mapping relationship. + // If the update is successful, continue with the following process, and return err it fails. + httpCode := rpc.DetectStatusCode(err) + if httpCode != errors.CodeVuidNotFound && httpCode != errors.CodeDiskBroken { + err = mgr.updateVolumeCache(ctx, migTask) + if err != nil { + return base.ErrUpdateVolumeCache + } + } + err = nil + } + + err = mgr.clusterMgrCli.UnlockVolume(ctx, migTask.SourceVuid.Vid()) + if err != nil { + span.Errorf("unlock volume failed: err[%+v]", err) + return + } + // update db + migTask.State = proto.MigrateStateFinished + base.InsistOn(ctx, "migrate finish task update task tbl", func() error { + return mgr.taskTbl.Update(ctx, proto.MigrateStateWorkCompleted, migTask) + }) + + mgr.finishQueue.RemoveTask(migTask.TaskID) + + base.VolTaskLockerInst().Unlock(ctx, migTask.SourceVuid.Vid()) + mgr.diskMigratingVuids.deleteMigratingVuid(migTask.SourceDiskID, migTask.SourceVuid) + + mgr.finishTaskCounter.Add() + + span.Infof("finish task phase success: task_id[%s], state[%v]", migTask.TaskID, migTask.State) + return +} + +func (mgr *MigrateMgr) updateVolumeCache(ctx context.Context, task *proto.MigrateTask) (err error) { + span := trace.SpanFromContextSafe(ctx) + span.Infof("update volume cache: vid[%d], task_id[%s]", task.SourceVuid.Vid(), task.TaskID) + return mgr.volumeUpdater.UpdateLeaderVolumeCache(ctx, task.SourceVuid.Vid()) +} + +// AddTask adds migrate task +func (mgr *MigrateMgr) AddTask(ctx context.Context, task *proto.MigrateTask) { + // add task to db + base.InsistOn(ctx, "migrate add task insert task to tbl", func() error { + return mgr.taskTbl.Insert(ctx, task) + }) + + // add task to prepare queue + mgr.prepareQueue.PushTask(task.TaskID, task) + + mgr.diskMigratingVuids.addMigratingVuid(task.SourceDiskID, task.SourceVuid, task.TaskID) +} + +// FinishTaskInAdvanceWhenLockFail finish migrate task in advance when lock volume failed +func (mgr *MigrateMgr) FinishTaskInAdvanceWhenLockFail(ctx context.Context, task *proto.MigrateTask) { + mgr.finishTaskInAdvance(ctx, task, "lock volume fail") +} + +func (mgr *MigrateMgr) finishTaskInAdvance(ctx context.Context, task *proto.MigrateTask, reason string) { + span := trace.SpanFromContextSafe(ctx) + span.Infof("finish task in advance: task_id[%s], task[%+v]", task.TaskID, task) + + task.State = proto.MigrateStateFinishedInAdvance + task.FinishAdvanceReason = reason + + base.InsistOn(ctx, "migrate finish task in advance update tbl", func() error { + return mgr.taskTbl.Update(ctx, proto.MigrateStateInited, task) + }) + + mgr.finishTaskCounter.Add() + mgr.prepareQueue.RemoveTask(task.TaskID) + base.VolTaskLockerInst().Unlock(ctx, task.SourceVuid.Vid()) +} + +func (mgr *MigrateMgr) handleUpdateVolMappingFail(ctx context.Context, task *proto.MigrateTask, err error) error { + span := trace.SpanFromContextSafe(ctx) + span.Infof("handle update vol mapping failed: task_id[%s], state[%d], dest vuid[%d]", task.TaskID, task.State, task.Destination.Vuid) + + code := rpc.DetectStatusCode(err) + if code == errors.CodeOldVuidNotMatch { + span.Panicf("change volume unit relationship failed: old vuid not match") + } + + if base.ShouldAllocAndRedo(code) { + span.Infof("realloc vunit and redo: task_id[%s]", task.TaskID) + newVunit, err := base.AllocVunitSafe(ctx, mgr.clusterMgrCli, task.SourceVuid, task.Sources) + if err != nil { + span.Errorf("realloc failed: vuid[%d], err[%+v]", task.SourceVuid, err) + return err + } + task.SetDest(newVunit.Location()) + task.State = proto.MigrateStatePrepared + task.WorkerRedoCnt++ + + base.InsistOn(ctx, "migrate redo task update task tbl", func() error { + return mgr.taskTbl.Update(ctx, proto.MigrateStateWorkCompleted, task) + }) + + mgr.finishQueue.RemoveTask(task.TaskID) + mgr.workQueue.AddPreparedTask(task.SourceIdc, task.TaskID, task) + span.Infof("task %+v redo again", task) + + return nil + } + + return err +} + +// StatQueueTaskCnt returns queue task count +func (mgr *MigrateMgr) StatQueueTaskCnt() (inited, prepared, completed int) { + todo, doing := mgr.prepareQueue.StatsTasks() + inited = todo + doing + + todo, doing = mgr.workQueue.StatsTasks() + prepared = todo + doing + + todo, doing = mgr.finishQueue.StatsTasks() + completed = todo + doing + return +} + +// Stats implement migrator +func (mgr *MigrateMgr) Stats() api.MigrateTasksStat { + preparing, workerDoing, finishing := mgr.StatQueueTaskCnt() + finishedCnt := mgr.finishTaskCounter.Show() + increaseDataSize, increaseShardCnt := mgr.taskStatsMgr.Counters() + return api.MigrateTasksStat{ + PreparingCnt: preparing, + WorkerDoingCnt: workerDoing, + FinishingCnt: finishing, + StatsPerMin: api.PerMinStats{ + FinishedCnt: fmt.Sprint(finishedCnt), + DataAmountByte: base.DataMountFormat(increaseDataSize), + ShardCnt: fmt.Sprint(increaseShardCnt), + }, + } +} + +// AcquireTask acquire migrate task +func (mgr *MigrateMgr) AcquireTask(ctx context.Context, idc string) (task *proto.MigrateTask, err error) { + span := trace.SpanFromContextSafe(ctx) + + if !mgr.taskSwitch.Enabled() { + return nil, proto.ErrTaskPaused + } + + _, migTask, _ := mgr.workQueue.Acquire(idc) + if migTask != nil { + task = migTask.(*proto.MigrateTask) + span.Infof("acquire %s taskId: %s", mgr.taskType, task.TaskID) + return task, nil + } + return nil, proto.ErrTaskEmpty +} + +// CancelTask cancel migrate task +func (mgr *MigrateMgr) CancelTask(ctx context.Context, args *api.CancelTaskArgs) (err error) { + mgr.taskStatsMgr.CancelTask() + + err = mgr.workQueue.Cancel(args.IDC, args.TaskId, args.Src, args.Dest) + if err != nil { + span := trace.SpanFromContextSafe(ctx) + span.Errorf("cancel migrate failed: task_type[%s], task_id[%s], err[%+v]", mgr.taskType, args.TaskId, err) + } + return +} + +// ReclaimTask reclaim migrate task +func (mgr *MigrateMgr) ReclaimTask(ctx context.Context, idc, taskID string, + src []proto.VunitLocation, oldDst proto.VunitLocation, newDst *client.AllocVunitInfo) (err error) { + mgr.taskStatsMgr.ReclaimTask() + + span := trace.SpanFromContextSafe(ctx) + err = mgr.workQueue.Reclaim(idc, taskID, src, oldDst, newDst.Location(), newDst.DiskID) + if err != nil { + span.Errorf("reclaim migrate task failed: task_type:[%s],task_id[%s], err[%+v]", mgr.taskType, taskID, err) + return err + } + + task, err := mgr.workQueue.Query(idc, taskID) + if err != nil { + span.Errorf("found task in workQueue failed: idc[%s], task_id[%s], err[%+v]", idc, taskID, err) + return err + } + + err = mgr.taskTbl.Update(ctx, proto.MigrateStatePrepared, task.(*proto.MigrateTask)) + if err != nil { + span.Errorf("update reclaim task failed: task_id[%s], err[%+v]", taskID, err) + } + return +} + +// CompleteTask complete migrate task +func (mgr *MigrateMgr) CompleteTask(ctx context.Context, args *api.CompleteTaskArgs) (err error) { + span := trace.SpanFromContextSafe(ctx) + + completeTask, err := mgr.workQueue.Complete(args.IDC, args.TaskId, args.Src, args.Dest) + if err != nil { + span.Errorf("complete migrate task failed: task_id[%s], err[%+v]", args.TaskId, err) + return err + } + + t := completeTask.(*proto.MigrateTask) + t.State = proto.MigrateStateWorkCompleted + + err = mgr.taskTbl.Update(ctx, proto.MigrateStatePrepared, t) + if err != nil { + // there is no impact if we failed to update task state in db, + // because we will do it in finishTask again, so assume complete success + span.Errorf("complete migrate task into db failed: task_id[%s], err[%+v]", t.TaskID, err) + err = nil + } + mgr.finishQueue.PushTask(args.TaskId, t) + return +} + +// RenewalTask renewal migrate task +func (mgr *MigrateMgr) RenewalTask(ctx context.Context, idc, taskID string) (err error) { + if !mgr.taskSwitch.Enabled() { + return proto.ErrTaskPaused + } + + err = mgr.workQueue.Renewal(idc, taskID) + if err != nil { + span := trace.SpanFromContextSafe(ctx) + span.Warnf("renewal migrate task failed: task_type[%s], task_id[%s], err[%+v]", mgr.taskType, taskID, err) + } + return +} + +// IsMigratingDisk returns true if disk is migrating +func (mgr *MigrateMgr) IsMigratingDisk(diskID proto.DiskID) bool { + return mgr.diskMigratingVuids.isMigratingDisk(diskID) +} + +// GetMigratingDiskNum returns migrating disk count +func (mgr *MigrateMgr) GetMigratingDiskNum() int { + return mgr.diskMigratingVuids.getCurrMigratingDisksCnt() +} + +// FindAll returns all migrate task +func (mgr *MigrateMgr) FindAll(ctx context.Context) (tasks []*proto.MigrateTask, err error) { + return mgr.taskTbl.FindAll(ctx) +} + +// ClearTasksByDiskID clear migrate task by diskID +func (mgr *MigrateMgr) ClearTasksByDiskID(ctx context.Context, diskID proto.DiskID) { + base.InsistOn(ctx, "migrate clear task by diskId", func() error { + return mgr.taskTbl.MarkDeleteByDiskID(ctx, diskID) + }) +} + +// ClearTasksByStates clear migrate task and set migrateState to deleteMark +func (mgr *MigrateMgr) ClearTasksByStates(ctx context.Context, states []proto.MigrateState) { + base.InsistOn(ctx, "migrate clear tasks by states", func() error { + return mgr.taskTbl.MarkDeleteByStates(ctx, states) + }) +} + +// FindTask returns task in db +func (mgr *MigrateMgr) FindTask(ctx context.Context, taskID string) (*proto.MigrateTask, error) { + return mgr.taskTbl.Find(ctx, taskID) +} + +// FindByDiskID return all task by diskID +func (mgr *MigrateMgr) FindByDiskID(ctx context.Context, diskID proto.DiskID) (tasks []*proto.MigrateTask, err error) { + return mgr.taskTbl.FindByDiskID(ctx, diskID) +} + +// QueryTask implement migrator +func (mgr *MigrateMgr) QueryTask(ctx context.Context, taskID string) (*api.MigrateTaskDetail, error) { + detail := &api.MigrateTaskDetail{} + taskInfo, err := mgr.FindTask(ctx, taskID) + if err != nil { + return detail, err + } + detail.TaskInfo = *taskInfo + + detailRunInfo, err := mgr.taskStatsMgr.QueryTaskDetail(taskID) + if err != nil { + return detail, nil + } + detail.RunStats = detailRunInfo.Statistics + return detail, nil +} + +// ReportWorkerTaskStats implement migrator +func (mgr *MigrateMgr) ReportWorkerTaskStats(st *api.TaskReportArgs) { + mgr.taskStatsMgr.ReportWorkerTaskStats(st.TaskId, st.TaskStats, st.IncreaseDataSizeByte, st.IncreaseShardCnt) +} + +// Enabled returns enable or not. +func (mgr *MigrateMgr) Enabled() bool { + return mgr.taskSwitch.Enabled() +} + +// WaitEnable block to wait enable. +func (mgr *MigrateMgr) WaitEnable() { + mgr.taskSwitch.WaitEnable() +} diff --git a/blobstore/scheduler/migrate_test.go b/blobstore/scheduler/migrate_test.go new file mode 100644 index 000000000..6cda7aa6b --- /dev/null +++ b/blobstore/scheduler/migrate_test.go @@ -0,0 +1,551 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + api "github.com/cubefs/cubefs/blobstore/api/scheduler" + "github.com/cubefs/cubefs/blobstore/common/codemode" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/scheduler/base" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/testing/mocks" +) + +var MockMigrateVolInfoMap = map[proto.Vid]*client.VolumeInfoSimple{ + 100: MockGenVolInfo(100, codemode.EC6P6, proto.VolumeStatusIdle), + 101: MockGenVolInfo(101, codemode.EC6P10L2, proto.VolumeStatusIdle), + 102: MockGenVolInfo(102, codemode.EC6P10L2, proto.VolumeStatusActive), + 103: MockGenVolInfo(103, codemode.EC6P6, proto.VolumeStatusLock), + 104: MockGenVolInfo(104, codemode.EC6P6, proto.VolumeStatusLock), + 105: MockGenVolInfo(105, codemode.EC6P6, proto.VolumeStatusActive), + + 300: MockGenVolInfo(300, codemode.EC6P6, proto.VolumeStatusIdle), + 301: MockGenVolInfo(301, codemode.EC6P10L2, proto.VolumeStatusIdle), + 302: MockGenVolInfo(302, codemode.EC6P10L2, proto.VolumeStatusActive), + + 400: MockGenVolInfo(400, codemode.EC6P6, proto.VolumeStatusIdle), + 401: MockGenVolInfo(401, codemode.EC6P10L2, proto.VolumeStatusIdle), + 402: MockGenVolInfo(402, codemode.EC6P10L2, proto.VolumeStatusActive), +} + +func newMigrateMgr(t *testing.T) *MigrateMgr { + ctr := gomock.NewController(t) + clusterMgr := NewMockClusterMgrAPI(ctr) + taskSwitch := mocks.NewMockSwitcher(ctr) + + migrateTbl := NewMockMigrateTaskTable(ctr) + volumeUpdater := NewMockVolumeUpdater(ctr) + conf := &MigrateConfig{ + TaskCommonConfig: base.TaskCommonConfig{ + PrepareQueueRetryDelayS: 0, + FinishQueueRetryDelayS: 0, + CancelPunishDurationS: 0, + WorkQueueSize: 3, + }, + } + mgr := NewMigrateMgr(clusterMgr, volumeUpdater, taskSwitch, migrateTbl, conf, proto.BalanceTaskType, 0) + mgr.SetLockFailHandleFunc(mgr.FinishTaskInAdvanceWhenLockFail) + return mgr +} + +func TestMigrateMigrateLoad(t *testing.T) { + mgr := newMigrateMgr(t) + + { + // load failed + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().FindAll(any).Return(nil, errMock) + err := mgr.Load() + require.True(t, errors.Is(err, errMock)) + } + { + t1 := mockGenMigrateTask("z0", 4, 100, proto.MigrateStateInited, MockMigrateVolInfoMap) + t2 := mockGenMigrateTask("z0", 5, 101, proto.MigrateStatePrepared, MockMigrateVolInfoMap) + t3 := mockGenMigrateTask("z1", 6, 102, proto.MigrateStateWorkCompleted, MockMigrateVolInfoMap) + t4 := mockGenMigrateTask("z2", 7, 103, proto.MigrateStateFinishedInAdvance, MockMigrateVolInfoMap) + t5 := mockGenMigrateTask("z2", 8, 104, proto.MigrateStateFinished, MockMigrateVolInfoMap) + t6 := mockGenMigrateTask("z0", 4, 105, proto.MigrateStateInited, MockMigrateVolInfoMap) + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().FindAll(any).Return([]*proto.MigrateTask{t1, t2, t3, t4, t5, t6}, nil) + err := mgr.Load() + require.NoError(t, err) + } + { + t2 := mockGenMigrateTask("z0", 5, 101, proto.MigrateStatePrepared, MockMigrateVolInfoMap) + t3 := mockGenMigrateTask("z1", 6, 101, proto.MigrateStateWorkCompleted, MockMigrateVolInfoMap) + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().FindAll(any).Return([]*proto.MigrateTask{t2, t3}, nil) + // panic + require.Panics(t, func() { + mgr.Load() + }) + t4 := mockGenMigrateTask("z2", 7, 103, 100, MockMigrateVolInfoMap) + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().FindAll(any).Return([]*proto.MigrateTask{t4}, nil) + require.Panics(t, func() { + mgr.Load() + }) + } +} + +func TestPrepareMigrateTask(t *testing.T) { + ctx := context.Background() + { + // no task + mgr := newMigrateMgr(t) + err := mgr.prepareTask() + require.True(t, errors.Is(err, base.ErrNoTaskInQueue)) + } + { + // one task and finish in advance + mgr := newMigrateMgr(t) + t1 := mockGenMigrateTask("z0", 4, 100, proto.MigrateStateInited, MockMigrateVolInfoMap) + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Insert(any, any).Return(nil) + mgr.AddTask(ctx, t1) + + // lock failed and send task to queue + err := base.VolTaskLockerInst().TryLock(ctx, 100) + require.NoError(t, err) + err = mgr.prepareTask() + require.True(t, errors.Is(err, base.ErrVolNotOnlyOneTask)) + base.VolTaskLockerInst().Unlock(ctx, 100) + + // get volume info failed + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(nil, errMock) + err = mgr.prepareTask() + require.True(t, errors.Is(err, errMock)) + + // finish task in advance because source chunk has moved + // unlock failed + volume := MockMigrateVolInfoMap[100] + volume.VunitLocations[int(t1.SourceVuid.Index())].Vuid = volume.VunitLocations[int(t1.SourceVuid.Index())].Vuid + 1 + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(volume, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UnlockVolume(any, any).Return(errMock) + err = mgr.prepareTask() + require.True(t, errors.Is(err, errMock)) + // unlock success + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(volume, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UnlockVolume(any, any).Return(nil) + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Update(any, any, any).Return(nil) + err = mgr.prepareTask() + require.NoError(t, err) + } + { + // one task and finish in advance because other migrate task is doing on this volume + mgr := newMigrateMgr(t) + t1 := mockGenMigrateTask("z0", 4, 100, proto.MigrateStateInited, MockMigrateVolInfoMap) + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Insert(any, any).Return(nil) + mgr.AddTask(ctx, t1) + + // lock cm volume failed + volume := MockMigrateVolInfoMap[100] + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(volume, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().LockVolume(any, any).Return(errMock) + err := mgr.prepareTask() + require.True(t, errors.Is(err, errMock)) + + // lock failed and call lockFailHandleFunc + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(volume, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().LockVolume(any, any).Return(errcode.ErrLockNotAllow) + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Update(any, any, any).Return(nil) + err = mgr.prepareTask() + require.NoError(t, err) + } + { + // one task and normal finish + mgr := newMigrateMgr(t) + t1 := mockGenMigrateTask("z0", 4, 100, proto.MigrateStateInited, MockMigrateVolInfoMap) + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Insert(any, any).Return(nil) + mgr.AddTask(ctx, t1) + + // lock cm volume failed + volume := MockMigrateVolInfoMap[100] + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(volume, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().LockVolume(any, any).Return(nil) + + // alloc volume failed + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(any, any).Return(nil, errMock) + err := mgr.prepareTask() + require.True(t, errors.Is(err, errMock)) + + // alloc success + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(volume, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().LockVolume(any, any).Return(nil) + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Update(any, any, any).Return(nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(any, any).DoAndReturn( + func(ctx context.Context, vuid proto.Vuid) (*client.AllocVunitInfo, error) { + vid := vuid.Vid() + idx := vuid.Index() + epoch := vuid.Epoch() + epoch++ + newVuid, _ := proto.NewVuid(vid, idx, epoch) + return &client.AllocVunitInfo{ + VunitLocation: proto.VunitLocation{Vuid: newVuid}, + }, nil + }) + err = mgr.prepareTask() + require.NoError(t, err) + } +} + +func TestFinishMigrateTask(t *testing.T) { + { + // no task + mgr := newMigrateMgr(t) + err := mgr.finishTask() + require.True(t, errors.Is(err, base.ErrNoTaskInQueue)) + } + { + // panic :status not eql proto.MigrateStateWorkCompleted + mgr := newMigrateMgr(t) + t1 := mockGenMigrateTask("z0", 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap) + mgr.finishQueue.PushTask(t1.TaskID, t1) + require.Panics(t, func() { + mgr.finishTask() + }) + } + { + { + // one task and redo success finally + mgr := newMigrateMgr(t) + t1 := mockGenMigrateTask("z0", 4, 100, proto.MigrateStateWorkCompleted, MockMigrateVolInfoMap) + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Update(any, any, any).Return(nil) + mgr.finishQueue.PushTask(t1.TaskID, t1) + + // update relationship failed + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateVolume(any, any, any, any).Return(errMock) + err := mgr.finishTask() + require.True(t, errors.Is(err, errMock)) + + // update relationship failed and need redo + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateVolume(any, any, any, any).Return(errcode.ErrNewVuidNotMatch) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(any, any).Return(nil, errMock) + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Update(any, any, any).Return(nil) + // alloc failed + err = mgr.finishTask() + require.True(t, errors.Is(err, errMock)) + + // panic + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Update(any, any, any).Return(nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateVolume(any, any, any, any).Return(errcode.ErrOldVuidNotMatch) + require.Panics(t, func() { + mgr.finishTask() + }) + + // redo success + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateVolume(any, any, any, any).Return(errcode.ErrNewVuidNotMatch) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(any, any).DoAndReturn( + func(ctx context.Context, vuid proto.Vuid) (*client.AllocVunitInfo, error) { + vid := vuid.Vid() + idx := vuid.Index() + epoch := vuid.Epoch() + epoch++ + newVuid, _ := proto.NewVuid(vid, idx, epoch) + return &client.AllocVunitInfo{ + VunitLocation: proto.VunitLocation{Vuid: newVuid}, + }, nil + }) + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Update(any, any, any).Return(nil) + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Update(any, any, any).Return(nil) + err = mgr.finishTask() + require.NoError(t, err) + } + { + // one task and success normal + mgr := newMigrateMgr(t) + t1 := mockGenMigrateTask("z0", 4, 100, proto.MigrateStateWorkCompleted, MockMigrateVolInfoMap) + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Update(any, any, any).Return(nil) + mgr.finishQueue.PushTask(t1.TaskID, t1) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateVolume(any, any, any, any).Return(nil) + // release failed and update volume cache failed + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ReleaseVolumeUnit(any, any, any).Return(errMock) + mgr.volumeUpdater.(*MockVolumeUpdater).EXPECT().UpdateLeaderVolumeCache(any, any).Return(errMock) + err := mgr.finishTask() + require.True(t, errors.Is(err, base.ErrUpdateVolumeCache)) + + // release failed and update volume cache success + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Update(any, any, any).Return(nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateVolume(any, any, any, any).Return(nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ReleaseVolumeUnit(any, any, any).Return(errMock) + mgr.volumeUpdater.(*MockVolumeUpdater).EXPECT().UpdateLeaderVolumeCache(any, any).Return(nil) + // unlock volume failed + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UnlockVolume(any, any).Return(errMock) + err = mgr.finishTask() + require.True(t, errors.Is(err, errMock)) + + // update volume success + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Update(any, any, any).Return(nil) + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Update(any, any, any).Return(nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateVolume(any, any, any, any).Return(nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ReleaseVolumeUnit(any, any, any).Return(nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UnlockVolume(any, any).Return(nil) + err = mgr.finishTask() + require.NoError(t, err) + } + } +} + +func TestAcquireMigrateTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + { + // task switch is close + mgr := newMigrateMgr(t) + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(false) + _, err := mgr.AcquireTask(ctx, idc) + require.True(t, errors.Is(err, proto.ErrTaskPaused)) + } + { + // no task in queue + mgr := newMigrateMgr(t) + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(true) + _, err := mgr.AcquireTask(ctx, idc) + require.True(t, errors.Is(err, proto.ErrTaskEmpty)) + } + { + // one task in queue + mgr := newMigrateMgr(t) + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(true) + t1 := mockGenMigrateTask(idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap) + mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1) + task, err := mgr.AcquireTask(ctx, idc) + require.NoError(t, err) + require.Equal(t, t1.TaskID, task.TaskID) + } +} + +func TestCancelMigrateTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + { + mgr := newMigrateMgr(t) + err := mgr.CancelTask(ctx, &api.CancelTaskArgs{IDC: idc}) + require.Error(t, err) + } + { + mgr := newMigrateMgr(t) + t1 := mockGenMigrateTask(idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap) + mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1) + + // no such task + err := mgr.CancelTask(ctx, &api.CancelTaskArgs{IDC: idc}) + require.Error(t, err) + + err = mgr.CancelTask(ctx, &api.CancelTaskArgs{IDC: idc, TaskId: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + require.NoError(t, err) + } +} + +func TestReclaimMigrateTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + { + // no task + mgr := newMigrateMgr(t) + err := mgr.ReclaimTask(ctx, idc, "", nil, proto.VunitLocation{}, &client.AllocVunitInfo{}) + require.Error(t, err) + } + { + mgr := newMigrateMgr(t) + t1 := mockGenMigrateTask(idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap) + mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1) + + // update failed + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Update(any, any, any).Return(errMock) + err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{}) + require.True(t, errors.Is(err, errMock)) + + // update success + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Update(any, any, any).Return(nil) + err = mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{}) + require.NoError(t, err) + } +} + +func TestCompleteMigrateTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + { + // no task + mgr := newMigrateMgr(t) + err := mgr.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: idc}) + require.Error(t, err) + } + { + mgr := newMigrateMgr(t) + t1 := mockGenMigrateTask(idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap) + mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1) + + // update failed + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Update(any, any, any).Return(errMock) + err := mgr.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: idc, TaskId: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + require.NoError(t, err) + + // no task in queue + err = mgr.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: idc, TaskId: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + require.Error(t, err) + + // update success + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Update(any, any, any).Return(nil) + t2 := mockGenMigrateTask(idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap) + mgr.workQueue.AddPreparedTask(idc, t2.TaskID, t2) + err = mgr.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: idc, TaskId: t2.TaskID, Src: t2.Sources, Dest: t2.Destination}) + require.NoError(t, err) + } +} + +func TestRenewalMigrateTask(t *testing.T) { + ctx := context.Background() + idc := "z0" + { + // task switch is close + mgr := newMigrateMgr(t) + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(false) + err := mgr.RenewalTask(ctx, idc, "") + require.True(t, errors.Is(err, proto.ErrTaskPaused)) + } + { + // no task + mgr := newMigrateMgr(t) + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(true) + err := mgr.RenewalTask(ctx, idc, "") + require.Error(t, err) + } + { + mgr := newMigrateMgr(t) + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(true) + t1 := mockGenMigrateTask(idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap) + mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1) + err := mgr.RenewalTask(ctx, idc, t1.TaskID) + require.NoError(t, err) + } +} + +func TestAddMigrateTask(t *testing.T) { + ctx := context.Background() + mgr := newMigrateMgr(t) + t1 := mockGenMigrateTask("z0", 4, 100, proto.MigrateStateInited, MockMigrateVolInfoMap) + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Insert(any, any).Return(nil) + mgr.AddTask(ctx, t1) + require.True(t, mgr.IsMigratingDisk(proto.DiskID(4))) + require.False(t, mgr.IsMigratingDisk(proto.DiskID(5))) + require.Equal(t, 1, mgr.GetMigratingDiskNum()) + + inited, prepared, completed := mgr.StatQueueTaskCnt() + require.Equal(t, 1, inited) + require.Equal(t, 0, prepared) + require.Equal(t, 0, completed) + + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().FindAll(any).Return([]*proto.MigrateTask{t1}, nil) + tasks, err := mgr.FindAll(ctx) + require.NoError(t, err) + require.Equal(t, 1, len(tasks)) + + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Find(any, any).Return(t1, nil) + task, err := mgr.FindTask(ctx, t1.TaskID) + require.NoError(t, err) + require.Equal(t, t1.TaskID, task.TaskID) + + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().FindByDiskID(any, any).Return([]*proto.MigrateTask{}, nil) + _, err = mgr.FindByDiskID(ctx, proto.DiskID(1)) + require.NoError(t, err) +} + +func TestClearMigrateTasks(t *testing.T) { + ctx := context.Background() + mgr := newMigrateMgr(t) + + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().MarkDeleteByDiskID(any, any).Return(nil) + mgr.ClearTasksByDiskID(ctx, proto.DiskID(1)) + + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().MarkDeleteByStates(any, any).Return(nil) + mgr.ClearTasksByStates(ctx, []proto.MigrateState{proto.MigrateStateWorkCompleted}) +} + +func TestMigrateRun(t *testing.T) { + mgr := newMigrateMgr(t) + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().WaitEnable().AnyTimes().Return() + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().AnyTimes().Return(true) + mgr.Run() + + // wait to run + time.Sleep(2 * time.Millisecond) +} + +func TestMigrateQueryTask(t *testing.T) { + ctx := context.Background() + taskID := "task_id" + mgr := newMigrateMgr(t) + + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Find(any, any).Return(nil, errMock) + _, err := mgr.QueryTask(ctx, taskID) + require.ErrorIs(t, errMock, err) + + mgr.taskTbl.(*MockMigrateTaskTable).EXPECT().Find(any, any).Return(&proto.MigrateTask{}, nil) + _, err = mgr.QueryTask(ctx, taskID) + require.NoError(t, err) +} + +func TestMigrateReportWorkerTaskStats(t *testing.T) { + mgr := newMigrateMgr(t) + mgr.ReportWorkerTaskStats(&api.TaskReportArgs{ + TaskId: "task_id", + IncreaseDataSizeByte: 1, + IncreaseShardCnt: 1, + }) +} + +func TestMigrateStatQueueTaskCnt(t *testing.T) { + mgr := newMigrateMgr(t) + inited, prepared, completed := mgr.StatQueueTaskCnt() + require.Equal(t, 0, inited) + require.Equal(t, 0, prepared) + require.Equal(t, 0, completed) +} + +func TestMigrateStats(t *testing.T) { + mgr := newMigrateMgr(t) + mgr.Stats() +} + +func TestMigrateAction(t *testing.T) { + mgr := newMigrateMgr(t) + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().WaitEnable().Return() + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(true) + + mgr.WaitEnable() + require.True(t, mgr.Enabled()) + + select { + case <-mgr.Done(): + require.Fail(t, "cannot be there") + default: + } + + mgr.Close() + + select { + case <-mgr.Done(): + default: + require.Fail(t, "cannot be there") + } + + mgr.Close() +} diff --git a/blobstore/scheduler/scheduler_mock_test.go b/blobstore/scheduler/scheduler_mock_test.go new file mode 100644 index 000000000..c31e045b2 --- /dev/null +++ b/blobstore/scheduler/scheduler_mock_test.go @@ -0,0 +1,1058 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cubefs/cubefs/blobstore/scheduler (interfaces: ITaskRunner,IVolumeCache,MMigrater,IDiskRepairer,IVolumeInspector,IClusterTopology,IArchiver) + +// Package scheduler is a generated GoMock package. +package scheduler + +import ( + context "context" + reflect "reflect" + + scheduler "github.com/cubefs/cubefs/blobstore/api/scheduler" + proto "github.com/cubefs/cubefs/blobstore/common/proto" + client "github.com/cubefs/cubefs/blobstore/scheduler/client" + db "github.com/cubefs/cubefs/blobstore/scheduler/db" + gomock "github.com/golang/mock/gomock" +) + +// MockTaskRunner is a mock of ITaskRunner interface. +type MockTaskRunner struct { + ctrl *gomock.Controller + recorder *MockTaskRunnerMockRecorder +} + +// MockTaskRunnerMockRecorder is the mock recorder for MockTaskRunner. +type MockTaskRunnerMockRecorder struct { + mock *MockTaskRunner +} + +// NewMockTaskRunner creates a new mock instance. +func NewMockTaskRunner(ctrl *gomock.Controller) *MockTaskRunner { + mock := &MockTaskRunner{ctrl: ctrl} + mock.recorder = &MockTaskRunnerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockTaskRunner) EXPECT() *MockTaskRunnerMockRecorder { + return m.recorder +} + +// Enabled mocks base method. +func (m *MockTaskRunner) Enabled() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Enabled") + ret0, _ := ret[0].(bool) + return ret0 +} + +// Enabled indicates an expected call of Enabled. +func (mr *MockTaskRunnerMockRecorder) Enabled() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Enabled", reflect.TypeOf((*MockTaskRunner)(nil).Enabled)) +} + +// GetErrorStats mocks base method. +func (m *MockTaskRunner) GetErrorStats() ([]string, uint64) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetErrorStats") + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(uint64) + return ret0, ret1 +} + +// GetErrorStats indicates an expected call of GetErrorStats. +func (mr *MockTaskRunnerMockRecorder) GetErrorStats() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetErrorStats", reflect.TypeOf((*MockTaskRunner)(nil).GetErrorStats)) +} + +// GetTaskStats mocks base method. +func (m *MockTaskRunner) GetTaskStats() ([20]int, [20]int) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTaskStats") + ret0, _ := ret[0].([20]int) + ret1, _ := ret[1].([20]int) + return ret0, ret1 +} + +// GetTaskStats indicates an expected call of GetTaskStats. +func (mr *MockTaskRunnerMockRecorder) GetTaskStats() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTaskStats", reflect.TypeOf((*MockTaskRunner)(nil).GetTaskStats)) +} + +// RunTask mocks base method. +func (m *MockTaskRunner) RunTask() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "RunTask") +} + +// RunTask indicates an expected call of RunTask. +func (mr *MockTaskRunnerMockRecorder) RunTask() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RunTask", reflect.TypeOf((*MockTaskRunner)(nil).RunTask)) +} + +// MockVolumeCache is a mock of IVolumeCache interface. +type MockVolumeCache struct { + ctrl *gomock.Controller + recorder *MockVolumeCacheMockRecorder +} + +// MockVolumeCacheMockRecorder is the mock recorder for MockVolumeCache. +type MockVolumeCacheMockRecorder struct { + mock *MockVolumeCache +} + +// NewMockVolumeCache creates a new mock instance. +func NewMockVolumeCache(ctrl *gomock.Controller) *MockVolumeCache { + mock := &MockVolumeCache{ctrl: ctrl} + mock.recorder = &MockVolumeCacheMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockVolumeCache) EXPECT() *MockVolumeCacheMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockVolumeCache) Get(arg0 proto.Vid) (*client.VolumeInfoSimple, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", arg0) + ret0, _ := ret[0].(*client.VolumeInfoSimple) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockVolumeCacheMockRecorder) Get(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockVolumeCache)(nil).Get), arg0) +} + +// Load mocks base method. +func (m *MockVolumeCache) Load() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Load") + ret0, _ := ret[0].(error) + return ret0 +} + +// Load indicates an expected call of Load. +func (mr *MockVolumeCacheMockRecorder) Load() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Load", reflect.TypeOf((*MockVolumeCache)(nil).Load)) +} + +// Update mocks base method. +func (m *MockVolumeCache) Update(arg0 proto.Vid) (*client.VolumeInfoSimple, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Update", arg0) + ret0, _ := ret[0].(*client.VolumeInfoSimple) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Update indicates an expected call of Update. +func (mr *MockVolumeCacheMockRecorder) Update(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockVolumeCache)(nil).Update), arg0) +} + +// MockMigrater is a mock of MMigrater interface. +type MockMigrater struct { + ctrl *gomock.Controller + recorder *MockMigraterMockRecorder +} + +// MockMigraterMockRecorder is the mock recorder for MockMigrater. +type MockMigraterMockRecorder struct { + mock *MockMigrater +} + +// NewMockMigrater creates a new mock instance. +func NewMockMigrater(ctrl *gomock.Controller) *MockMigrater { + mock := &MockMigrater{ctrl: ctrl} + mock.recorder = &MockMigraterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockMigrater) EXPECT() *MockMigraterMockRecorder { + return m.recorder +} + +// AcquireTask mocks base method. +func (m *MockMigrater) AcquireTask(arg0 context.Context, arg1 string) (*proto.MigrateTask, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AcquireTask", arg0, arg1) + ret0, _ := ret[0].(*proto.MigrateTask) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AcquireTask indicates an expected call of AcquireTask. +func (mr *MockMigraterMockRecorder) AcquireTask(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AcquireTask", reflect.TypeOf((*MockMigrater)(nil).AcquireTask), arg0, arg1) +} + +// AddManualTask mocks base method. +func (m *MockMigrater) AddManualTask(arg0 context.Context, arg1 proto.Vuid, arg2 bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddManualTask", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddManualTask indicates an expected call of AddManualTask. +func (mr *MockMigraterMockRecorder) AddManualTask(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddManualTask", reflect.TypeOf((*MockMigrater)(nil).AddManualTask), arg0, arg1, arg2) +} + +// AddTask mocks base method. +func (m *MockMigrater) AddTask(arg0 context.Context, arg1 *proto.MigrateTask) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "AddTask", arg0, arg1) +} + +// AddTask indicates an expected call of AddTask. +func (mr *MockMigraterMockRecorder) AddTask(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddTask", reflect.TypeOf((*MockMigrater)(nil).AddTask), arg0, arg1) +} + +// CancelTask mocks base method. +func (m *MockMigrater) CancelTask(arg0 context.Context, arg1 *scheduler.CancelTaskArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CancelTask", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// CancelTask indicates an expected call of CancelTask. +func (mr *MockMigraterMockRecorder) CancelTask(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CancelTask", reflect.TypeOf((*MockMigrater)(nil).CancelTask), arg0, arg1) +} + +// ClearTasksByDiskID mocks base method. +func (m *MockMigrater) ClearTasksByDiskID(arg0 context.Context, arg1 proto.DiskID) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "ClearTasksByDiskID", arg0, arg1) +} + +// ClearTasksByDiskID indicates an expected call of ClearTasksByDiskID. +func (mr *MockMigraterMockRecorder) ClearTasksByDiskID(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClearTasksByDiskID", reflect.TypeOf((*MockMigrater)(nil).ClearTasksByDiskID), arg0, arg1) +} + +// ClearTasksByStates mocks base method. +func (m *MockMigrater) ClearTasksByStates(arg0 context.Context, arg1 []proto.MigrateState) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "ClearTasksByStates", arg0, arg1) +} + +// ClearTasksByStates indicates an expected call of ClearTasksByStates. +func (mr *MockMigraterMockRecorder) ClearTasksByStates(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClearTasksByStates", reflect.TypeOf((*MockMigrater)(nil).ClearTasksByStates), arg0, arg1) +} + +// Close mocks base method. +func (m *MockMigrater) Close() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Close") +} + +// Close indicates an expected call of Close. +func (mr *MockMigraterMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockMigrater)(nil).Close)) +} + +// CompleteTask mocks base method. +func (m *MockMigrater) CompleteTask(arg0 context.Context, arg1 *scheduler.CompleteTaskArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CompleteTask", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// CompleteTask indicates an expected call of CompleteTask. +func (mr *MockMigraterMockRecorder) CompleteTask(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompleteTask", reflect.TypeOf((*MockMigrater)(nil).CompleteTask), arg0, arg1) +} + +// Done mocks base method. +func (m *MockMigrater) Done() <-chan struct{} { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Done") + ret0, _ := ret[0].(<-chan struct{}) + return ret0 +} + +// Done indicates an expected call of Done. +func (mr *MockMigraterMockRecorder) Done() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Done", reflect.TypeOf((*MockMigrater)(nil).Done)) +} + +// Enabled mocks base method. +func (m *MockMigrater) Enabled() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Enabled") + ret0, _ := ret[0].(bool) + return ret0 +} + +// Enabled indicates an expected call of Enabled. +func (mr *MockMigraterMockRecorder) Enabled() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Enabled", reflect.TypeOf((*MockMigrater)(nil).Enabled)) +} + +// FindAll mocks base method. +func (m *MockMigrater) FindAll(arg0 context.Context) ([]*proto.MigrateTask, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FindAll", arg0) + ret0, _ := ret[0].([]*proto.MigrateTask) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FindAll indicates an expected call of FindAll. +func (mr *MockMigraterMockRecorder) FindAll(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindAll", reflect.TypeOf((*MockMigrater)(nil).FindAll), arg0) +} + +// FindByDiskID mocks base method. +func (m *MockMigrater) FindByDiskID(arg0 context.Context, arg1 proto.DiskID) ([]*proto.MigrateTask, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FindByDiskID", arg0, arg1) + ret0, _ := ret[0].([]*proto.MigrateTask) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FindByDiskID indicates an expected call of FindByDiskID. +func (mr *MockMigraterMockRecorder) FindByDiskID(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindByDiskID", reflect.TypeOf((*MockMigrater)(nil).FindByDiskID), arg0, arg1) +} + +// FindTask mocks base method. +func (m *MockMigrater) FindTask(arg0 context.Context, arg1 string) (*proto.MigrateTask, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FindTask", arg0, arg1) + ret0, _ := ret[0].(*proto.MigrateTask) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FindTask indicates an expected call of FindTask. +func (mr *MockMigraterMockRecorder) FindTask(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindTask", reflect.TypeOf((*MockMigrater)(nil).FindTask), arg0, arg1) +} + +// FinishTaskInAdvanceWhenLockFail mocks base method. +func (m *MockMigrater) FinishTaskInAdvanceWhenLockFail(arg0 context.Context, arg1 *proto.MigrateTask) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "FinishTaskInAdvanceWhenLockFail", arg0, arg1) +} + +// FinishTaskInAdvanceWhenLockFail indicates an expected call of FinishTaskInAdvanceWhenLockFail. +func (mr *MockMigraterMockRecorder) FinishTaskInAdvanceWhenLockFail(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FinishTaskInAdvanceWhenLockFail", reflect.TypeOf((*MockMigrater)(nil).FinishTaskInAdvanceWhenLockFail), arg0, arg1) +} + +// GetMigratingDiskNum mocks base method. +func (m *MockMigrater) GetMigratingDiskNum() int { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMigratingDiskNum") + ret0, _ := ret[0].(int) + return ret0 +} + +// GetMigratingDiskNum indicates an expected call of GetMigratingDiskNum. +func (mr *MockMigraterMockRecorder) GetMigratingDiskNum() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMigratingDiskNum", reflect.TypeOf((*MockMigrater)(nil).GetMigratingDiskNum)) +} + +// IsMigratingDisk mocks base method. +func (m *MockMigrater) IsMigratingDisk(arg0 proto.DiskID) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsMigratingDisk", arg0) + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsMigratingDisk indicates an expected call of IsMigratingDisk. +func (mr *MockMigraterMockRecorder) IsMigratingDisk(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsMigratingDisk", reflect.TypeOf((*MockMigrater)(nil).IsMigratingDisk), arg0) +} + +// Load mocks base method. +func (m *MockMigrater) Load() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Load") + ret0, _ := ret[0].(error) + return ret0 +} + +// Load indicates an expected call of Load. +func (mr *MockMigraterMockRecorder) Load() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Load", reflect.TypeOf((*MockMigrater)(nil).Load)) +} + +// Progress mocks base method. +func (m *MockMigrater) Progress(arg0 context.Context) (proto.DiskID, int, int) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Progress", arg0) + ret0, _ := ret[0].(proto.DiskID) + ret1, _ := ret[1].(int) + ret2, _ := ret[2].(int) + return ret0, ret1, ret2 +} + +// Progress indicates an expected call of Progress. +func (mr *MockMigraterMockRecorder) Progress(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Progress", reflect.TypeOf((*MockMigrater)(nil).Progress), arg0) +} + +// QueryTask mocks base method. +func (m *MockMigrater) QueryTask(arg0 context.Context, arg1 string) (*scheduler.MigrateTaskDetail, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "QueryTask", arg0, arg1) + ret0, _ := ret[0].(*scheduler.MigrateTaskDetail) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// QueryTask indicates an expected call of QueryTask. +func (mr *MockMigraterMockRecorder) QueryTask(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueryTask", reflect.TypeOf((*MockMigrater)(nil).QueryTask), arg0, arg1) +} + +// ReclaimTask mocks base method. +func (m *MockMigrater) ReclaimTask(arg0 context.Context, arg1, arg2 string, arg3 []proto.VunitLocation, arg4 proto.VunitLocation, arg5 *client.AllocVunitInfo) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReclaimTask", arg0, arg1, arg2, arg3, arg4, arg5) + ret0, _ := ret[0].(error) + return ret0 +} + +// ReclaimTask indicates an expected call of ReclaimTask. +func (mr *MockMigraterMockRecorder) ReclaimTask(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReclaimTask", reflect.TypeOf((*MockMigrater)(nil).ReclaimTask), arg0, arg1, arg2, arg3, arg4, arg5) +} + +// RenewalTask mocks base method. +func (m *MockMigrater) RenewalTask(arg0 context.Context, arg1, arg2 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RenewalTask", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// RenewalTask indicates an expected call of RenewalTask. +func (mr *MockMigraterMockRecorder) RenewalTask(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RenewalTask", reflect.TypeOf((*MockMigrater)(nil).RenewalTask), arg0, arg1, arg2) +} + +// ReportWorkerTaskStats mocks base method. +func (m *MockMigrater) ReportWorkerTaskStats(arg0 *scheduler.TaskReportArgs) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "ReportWorkerTaskStats", arg0) +} + +// ReportWorkerTaskStats indicates an expected call of ReportWorkerTaskStats. +func (mr *MockMigraterMockRecorder) ReportWorkerTaskStats(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportWorkerTaskStats", reflect.TypeOf((*MockMigrater)(nil).ReportWorkerTaskStats), arg0) +} + +// Run mocks base method. +func (m *MockMigrater) Run() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Run") +} + +// Run indicates an expected call of Run. +func (mr *MockMigraterMockRecorder) Run() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockMigrater)(nil).Run)) +} + +// SetLockFailHandleFunc mocks base method. +func (m *MockMigrater) SetLockFailHandleFunc(arg0 func(context.Context, *proto.MigrateTask)) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetLockFailHandleFunc", arg0) +} + +// SetLockFailHandleFunc indicates an expected call of SetLockFailHandleFunc. +func (mr *MockMigraterMockRecorder) SetLockFailHandleFunc(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLockFailHandleFunc", reflect.TypeOf((*MockMigrater)(nil).SetLockFailHandleFunc), arg0) +} + +// StatQueueTaskCnt mocks base method. +func (m *MockMigrater) StatQueueTaskCnt() (int, int, int) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StatQueueTaskCnt") + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(int) + ret2, _ := ret[2].(int) + return ret0, ret1, ret2 +} + +// StatQueueTaskCnt indicates an expected call of StatQueueTaskCnt. +func (mr *MockMigraterMockRecorder) StatQueueTaskCnt() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StatQueueTaskCnt", reflect.TypeOf((*MockMigrater)(nil).StatQueueTaskCnt)) +} + +// Stats mocks base method. +func (m *MockMigrater) Stats() scheduler.MigrateTasksStat { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Stats") + ret0, _ := ret[0].(scheduler.MigrateTasksStat) + return ret0 +} + +// Stats indicates an expected call of Stats. +func (mr *MockMigraterMockRecorder) Stats() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stats", reflect.TypeOf((*MockMigrater)(nil).Stats)) +} + +// WaitEnable mocks base method. +func (m *MockMigrater) WaitEnable() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "WaitEnable") +} + +// WaitEnable indicates an expected call of WaitEnable. +func (mr *MockMigraterMockRecorder) WaitEnable() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitEnable", reflect.TypeOf((*MockMigrater)(nil).WaitEnable)) +} + +// MockDiskRepairer is a mock of IDiskRepairer interface. +type MockDiskRepairer struct { + ctrl *gomock.Controller + recorder *MockDiskRepairerMockRecorder +} + +// MockDiskRepairerMockRecorder is the mock recorder for MockDiskRepairer. +type MockDiskRepairerMockRecorder struct { + mock *MockDiskRepairer +} + +// NewMockDiskRepairer creates a new mock instance. +func NewMockDiskRepairer(ctrl *gomock.Controller) *MockDiskRepairer { + mock := &MockDiskRepairer{ctrl: ctrl} + mock.recorder = &MockDiskRepairerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDiskRepairer) EXPECT() *MockDiskRepairerMockRecorder { + return m.recorder +} + +// AcquireTask mocks base method. +func (m *MockDiskRepairer) AcquireTask(arg0 context.Context, arg1 string) (*proto.VolRepairTask, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AcquireTask", arg0, arg1) + ret0, _ := ret[0].(*proto.VolRepairTask) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AcquireTask indicates an expected call of AcquireTask. +func (mr *MockDiskRepairerMockRecorder) AcquireTask(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AcquireTask", reflect.TypeOf((*MockDiskRepairer)(nil).AcquireTask), arg0, arg1) +} + +// CancelTask mocks base method. +func (m *MockDiskRepairer) CancelTask(arg0 context.Context, arg1 *scheduler.CancelTaskArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CancelTask", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// CancelTask indicates an expected call of CancelTask. +func (mr *MockDiskRepairerMockRecorder) CancelTask(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CancelTask", reflect.TypeOf((*MockDiskRepairer)(nil).CancelTask), arg0, arg1) +} + +// Close mocks base method. +func (m *MockDiskRepairer) Close() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Close") +} + +// Close indicates an expected call of Close. +func (mr *MockDiskRepairerMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockDiskRepairer)(nil).Close)) +} + +// CompleteTask mocks base method. +func (m *MockDiskRepairer) CompleteTask(arg0 context.Context, arg1 *scheduler.CompleteTaskArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CompleteTask", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// CompleteTask indicates an expected call of CompleteTask. +func (mr *MockDiskRepairerMockRecorder) CompleteTask(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompleteTask", reflect.TypeOf((*MockDiskRepairer)(nil).CompleteTask), arg0, arg1) +} + +// Done mocks base method. +func (m *MockDiskRepairer) Done() <-chan struct{} { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Done") + ret0, _ := ret[0].(<-chan struct{}) + return ret0 +} + +// Done indicates an expected call of Done. +func (mr *MockDiskRepairerMockRecorder) Done() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Done", reflect.TypeOf((*MockDiskRepairer)(nil).Done)) +} + +// Enabled mocks base method. +func (m *MockDiskRepairer) Enabled() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Enabled") + ret0, _ := ret[0].(bool) + return ret0 +} + +// Enabled indicates an expected call of Enabled. +func (mr *MockDiskRepairerMockRecorder) Enabled() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Enabled", reflect.TypeOf((*MockDiskRepairer)(nil).Enabled)) +} + +// Load mocks base method. +func (m *MockDiskRepairer) Load() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Load") + ret0, _ := ret[0].(error) + return ret0 +} + +// Load indicates an expected call of Load. +func (mr *MockDiskRepairerMockRecorder) Load() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Load", reflect.TypeOf((*MockDiskRepairer)(nil).Load)) +} + +// Progress mocks base method. +func (m *MockDiskRepairer) Progress(arg0 context.Context) (proto.DiskID, int, int) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Progress", arg0) + ret0, _ := ret[0].(proto.DiskID) + ret1, _ := ret[1].(int) + ret2, _ := ret[2].(int) + return ret0, ret1, ret2 +} + +// Progress indicates an expected call of Progress. +func (mr *MockDiskRepairerMockRecorder) Progress(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Progress", reflect.TypeOf((*MockDiskRepairer)(nil).Progress), arg0) +} + +// QueryTask mocks base method. +func (m *MockDiskRepairer) QueryTask(arg0 context.Context, arg1 string) (*scheduler.RepairTaskDetail, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "QueryTask", arg0, arg1) + ret0, _ := ret[0].(*scheduler.RepairTaskDetail) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// QueryTask indicates an expected call of QueryTask. +func (mr *MockDiskRepairerMockRecorder) QueryTask(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueryTask", reflect.TypeOf((*MockDiskRepairer)(nil).QueryTask), arg0, arg1) +} + +// ReclaimTask mocks base method. +func (m *MockDiskRepairer) ReclaimTask(arg0 context.Context, arg1, arg2 string, arg3 []proto.VunitLocation, arg4 proto.VunitLocation, arg5 *client.AllocVunitInfo) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReclaimTask", arg0, arg1, arg2, arg3, arg4, arg5) + ret0, _ := ret[0].(error) + return ret0 +} + +// ReclaimTask indicates an expected call of ReclaimTask. +func (mr *MockDiskRepairerMockRecorder) ReclaimTask(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReclaimTask", reflect.TypeOf((*MockDiskRepairer)(nil).ReclaimTask), arg0, arg1, arg2, arg3, arg4, arg5) +} + +// RenewalTask mocks base method. +func (m *MockDiskRepairer) RenewalTask(arg0 context.Context, arg1, arg2 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RenewalTask", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// RenewalTask indicates an expected call of RenewalTask. +func (mr *MockDiskRepairerMockRecorder) RenewalTask(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RenewalTask", reflect.TypeOf((*MockDiskRepairer)(nil).RenewalTask), arg0, arg1, arg2) +} + +// ReportWorkerTaskStats mocks base method. +func (m *MockDiskRepairer) ReportWorkerTaskStats(arg0 *scheduler.TaskReportArgs) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "ReportWorkerTaskStats", arg0) +} + +// ReportWorkerTaskStats indicates an expected call of ReportWorkerTaskStats. +func (mr *MockDiskRepairerMockRecorder) ReportWorkerTaskStats(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportWorkerTaskStats", reflect.TypeOf((*MockDiskRepairer)(nil).ReportWorkerTaskStats), arg0) +} + +// Run mocks base method. +func (m *MockDiskRepairer) Run() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Run") +} + +// Run indicates an expected call of Run. +func (mr *MockDiskRepairerMockRecorder) Run() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockDiskRepairer)(nil).Run)) +} + +// StatQueueTaskCnt mocks base method. +func (m *MockDiskRepairer) StatQueueTaskCnt() (int, int, int) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StatQueueTaskCnt") + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(int) + ret2, _ := ret[2].(int) + return ret0, ret1, ret2 +} + +// StatQueueTaskCnt indicates an expected call of StatQueueTaskCnt. +func (mr *MockDiskRepairerMockRecorder) StatQueueTaskCnt() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StatQueueTaskCnt", reflect.TypeOf((*MockDiskRepairer)(nil).StatQueueTaskCnt)) +} + +// Stats mocks base method. +func (m *MockDiskRepairer) Stats() scheduler.MigrateTasksStat { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Stats") + ret0, _ := ret[0].(scheduler.MigrateTasksStat) + return ret0 +} + +// Stats indicates an expected call of Stats. +func (mr *MockDiskRepairerMockRecorder) Stats() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stats", reflect.TypeOf((*MockDiskRepairer)(nil).Stats)) +} + +// MockVolumeInspector is a mock of IVolumeInspector interface. +type MockVolumeInspector struct { + ctrl *gomock.Controller + recorder *MockVolumeInspectorMockRecorder +} + +// MockVolumeInspectorMockRecorder is the mock recorder for MockVolumeInspector. +type MockVolumeInspectorMockRecorder struct { + mock *MockVolumeInspector +} + +// NewMockVolumeInspector creates a new mock instance. +func NewMockVolumeInspector(ctrl *gomock.Controller) *MockVolumeInspector { + mock := &MockVolumeInspector{ctrl: ctrl} + mock.recorder = &MockVolumeInspectorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockVolumeInspector) EXPECT() *MockVolumeInspectorMockRecorder { + return m.recorder +} + +// AcquireInspect mocks base method. +func (m *MockVolumeInspector) AcquireInspect(arg0 context.Context) (*proto.InspectTask, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AcquireInspect", arg0) + ret0, _ := ret[0].(*proto.InspectTask) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AcquireInspect indicates an expected call of AcquireInspect. +func (mr *MockVolumeInspectorMockRecorder) AcquireInspect(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AcquireInspect", reflect.TypeOf((*MockVolumeInspector)(nil).AcquireInspect), arg0) +} + +// Close mocks base method. +func (m *MockVolumeInspector) Close() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Close") +} + +// Close indicates an expected call of Close. +func (mr *MockVolumeInspectorMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockVolumeInspector)(nil).Close)) +} + +// CompleteInspect mocks base method. +func (m *MockVolumeInspector) CompleteInspect(arg0 context.Context, arg1 *proto.InspectRet) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "CompleteInspect", arg0, arg1) +} + +// CompleteInspect indicates an expected call of CompleteInspect. +func (mr *MockVolumeInspectorMockRecorder) CompleteInspect(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompleteInspect", reflect.TypeOf((*MockVolumeInspector)(nil).CompleteInspect), arg0, arg1) +} + +// Done mocks base method. +func (m *MockVolumeInspector) Done() <-chan struct{} { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Done") + ret0, _ := ret[0].(<-chan struct{}) + return ret0 +} + +// Done indicates an expected call of Done. +func (mr *MockVolumeInspectorMockRecorder) Done() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Done", reflect.TypeOf((*MockVolumeInspector)(nil).Done)) +} + +// Enabled mocks base method. +func (m *MockVolumeInspector) Enabled() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Enabled") + ret0, _ := ret[0].(bool) + return ret0 +} + +// Enabled indicates an expected call of Enabled. +func (mr *MockVolumeInspectorMockRecorder) Enabled() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Enabled", reflect.TypeOf((*MockVolumeInspector)(nil).Enabled)) +} + +// GetTaskStats mocks base method. +func (m *MockVolumeInspector) GetTaskStats() ([20]int, [20]int) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTaskStats") + ret0, _ := ret[0].([20]int) + ret1, _ := ret[1].([20]int) + return ret0, ret1 +} + +// GetTaskStats indicates an expected call of GetTaskStats. +func (mr *MockVolumeInspectorMockRecorder) GetTaskStats() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTaskStats", reflect.TypeOf((*MockVolumeInspector)(nil).GetTaskStats)) +} + +// Run mocks base method. +func (m *MockVolumeInspector) Run() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Run") +} + +// Run indicates an expected call of Run. +func (mr *MockVolumeInspectorMockRecorder) Run() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockVolumeInspector)(nil).Run)) +} + +// MockClusterTopology is a mock of IClusterTopology interface. +type MockClusterTopology struct { + ctrl *gomock.Controller + recorder *MockClusterTopologyMockRecorder +} + +// MockClusterTopologyMockRecorder is the mock recorder for MockClusterTopology. +type MockClusterTopologyMockRecorder struct { + mock *MockClusterTopology +} + +// NewMockClusterTopology creates a new mock instance. +func NewMockClusterTopology(ctrl *gomock.Controller) *MockClusterTopology { + mock := &MockClusterTopology{ctrl: ctrl} + mock.recorder = &MockClusterTopologyMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockClusterTopology) EXPECT() *MockClusterTopologyMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockClusterTopology) Close() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Close") +} + +// Close indicates an expected call of Close. +func (mr *MockClusterTopologyMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockClusterTopology)(nil).Close)) +} + +// Done mocks base method. +func (m *MockClusterTopology) Done() <-chan struct{} { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Done") + ret0, _ := ret[0].(<-chan struct{}) + return ret0 +} + +// Done indicates an expected call of Done. +func (mr *MockClusterTopologyMockRecorder) Done() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Done", reflect.TypeOf((*MockClusterTopology)(nil).Done)) +} + +// GetIDCDisks mocks base method. +func (m *MockClusterTopology) GetIDCDisks(arg0 string) []*client.DiskInfoSimple { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetIDCDisks", arg0) + ret0, _ := ret[0].([]*client.DiskInfoSimple) + return ret0 +} + +// GetIDCDisks indicates an expected call of GetIDCDisks. +func (mr *MockClusterTopologyMockRecorder) GetIDCDisks(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIDCDisks", reflect.TypeOf((*MockClusterTopology)(nil).GetIDCDisks), arg0) +} + +// GetIDCs mocks base method. +func (m *MockClusterTopology) GetIDCs() map[string]*IDC { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetIDCs") + ret0, _ := ret[0].(map[string]*IDC) + return ret0 +} + +// GetIDCs indicates an expected call of GetIDCs. +func (mr *MockClusterTopologyMockRecorder) GetIDCs() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIDCs", reflect.TypeOf((*MockClusterTopology)(nil).GetIDCs)) +} + +// MockArchiver is a mock of IArchiver interface. +type MockArchiver struct { + ctrl *gomock.Controller + recorder *MockArchiverMockRecorder +} + +// MockArchiverMockRecorder is the mock recorder for MockArchiver. +type MockArchiverMockRecorder struct { + mock *MockArchiver +} + +// NewMockArchiver creates a new mock instance. +func NewMockArchiver(ctrl *gomock.Controller) *MockArchiver { + mock := &MockArchiver{ctrl: ctrl} + mock.recorder = &MockArchiverMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockArchiver) EXPECT() *MockArchiverMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockArchiver) Close() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Close") +} + +// Close indicates an expected call of Close. +func (mr *MockArchiverMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockArchiver)(nil).Close)) +} + +// Done mocks base method. +func (m *MockArchiver) Done() <-chan struct{} { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Done") + ret0, _ := ret[0].(<-chan struct{}) + return ret0 +} + +// Done indicates an expected call of Done. +func (mr *MockArchiverMockRecorder) Done() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Done", reflect.TypeOf((*MockArchiver)(nil).Done)) +} + +// RegisterTables mocks base method. +func (m *MockArchiver) RegisterTables(arg0 ...db.IRecordSrcTbl) { + m.ctrl.T.Helper() + varargs := []interface{}{} + for _, a := range arg0 { + varargs = append(varargs, a) + } + m.ctrl.Call(m, "RegisterTables", varargs...) +} + +// RegisterTables indicates an expected call of RegisterTables. +func (mr *MockArchiverMockRecorder) RegisterTables(arg0 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterTables", reflect.TypeOf((*MockArchiver)(nil).RegisterTables), arg0...) +} + +// Run mocks base method. +func (m *MockArchiver) Run() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Run") +} + +// Run indicates an expected call of Run. +func (mr *MockArchiverMockRecorder) Run() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockArchiver)(nil).Run)) +} diff --git a/blobstore/scheduler/scheduler_test.go b/blobstore/scheduler/scheduler_test.go new file mode 100644 index 000000000..01f1b4b7b --- /dev/null +++ b/blobstore/scheduler/scheduler_test.go @@ -0,0 +1,149 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "errors" + "testing" + "time" + + "github.com/Shopify/sarama" + "github.com/golang/mock/gomock" + + "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/scheduler/base" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +// github.com/cubefs/cubefs/blobstore/scheduler/... module scheduler interfaces +//go:generate mockgen -destination=./client_mock_test.go -package=scheduler -mock_names ClusterMgrAPI=MockClusterMgrAPI,BlobnodeAPI=MockBlobnodeAPI,IVolumeUpdater=MockVolumeUpdater,ProxyAPI=MockMqProxyAPI github.com/cubefs/cubefs/blobstore/scheduler/client ClusterMgrAPI,BlobnodeAPI,IVolumeUpdater,ProxyAPI +//go:generate mockgen -destination=./base_mock_test.go -package=scheduler -mock_names IConsumer=MockConsumer,IProducer=MockProducer github.com/cubefs/cubefs/blobstore/scheduler/base IConsumer,IProducer +//go:generate mockgen -destination=./db_mock_test.go -package=scheduler -mock_names IKafkaOffsetTable=MockKafkaOffsetTable,IOrphanShardTable=MockOrphanShardTable,IArchiveTable=MockArchiveTable,IMigrateTaskTable=MockMigrateTaskTable,IRepairTaskTable=MockRepairTaskTable,IInspectCheckPointTable=MockInspectCheckPointTable github.com/cubefs/cubefs/blobstore/scheduler/db IKafkaOffsetTable,IOrphanShardTable,IArchiveTable,IMigrateTaskTable,IRepairTaskTable,IInspectCheckPointTable +//go:generate mockgen -destination=./scheduler_mock_test.go -package=scheduler -mock_names ITaskRunner=MockTaskRunner,IVolumeCache=MockVolumeCache,MMigrater=MockMigrater,IDiskRepairer=MockDiskRepairer,IVolumeInspector=MockVolumeInspector,IClusterTopology=MockClusterTopology,IArchiver=MockArchiver github.com/cubefs/cubefs/blobstore/scheduler ITaskRunner,IVolumeCache,MMigrater,IDiskRepairer,IVolumeInspector,IClusterTopology,IArchiver + +const ( + testTopic = "test_topic" +) + +var ( + any = gomock.Any() + errMock = errors.New("fake error") +) + +func init() { + log.SetOutputLevel(log.Lfatal) +} + +func NewBroker(t *testing.T) *sarama.MockBroker { + mockFetchResponse := sarama.NewMockFetchResponse(t, 1) + mockFetchResponse.SetVersion(1) + var msg sarama.ByteEncoder = []byte("FOO") + for i := 0; i < 1000; i++ { + mockFetchResponse.SetMessage(testTopic, 0, int64(i), msg) + } + + broker0 := sarama.NewMockBrokerAddr(t, 0, "127.0.0.1:0") + broker0.SetHandlerByMap(map[string]sarama.MockResponse{ + "MetadataRequest": sarama.NewMockMetadataResponse(t). + SetBroker(broker0.Addr(), broker0.BrokerID()). + SetLeader(testTopic, 0, broker0.BrokerID()), + "OffsetRequest": sarama.NewMockOffsetResponse(t). + SetOffset(testTopic, 0, sarama.OffsetOldest, 0). + SetOffset(testTopic, 0, sarama.OffsetNewest, 2345), + "FetchRequest": mockFetchResponse, + }) + return broker0 +} + +func mockGenMigrateTask(idc string, diskID proto.DiskID, vid proto.Vid, state proto.MigrateState, volInfoMap map[proto.Vid]*client.VolumeInfoSimple) (task *proto.MigrateTask) { + srcs := volInfoMap[vid].VunitLocations + + codeMode := volInfoMap[vid].CodeMode + vunitInfo := MockAlloc(volInfoMap[vid].VunitLocations[0].Vuid) + task = &proto.MigrateTask{ + TaskID: base.GenTaskID("balance", vid), + State: state, + SourceIdc: idc, + SourceDiskID: diskID, + SourceVuid: volInfoMap[vid].VunitLocations[0].Vuid, + Sources: srcs, + CodeMode: codeMode, + + Destination: vunitInfo.Location(), + Ctime: time.Now().String(), + MTime: time.Now().String(), + } + return task +} + +func MockGenVolInfo(vid proto.Vid, cm codemode.CodeMode, status proto.VolumeStatus) *client.VolumeInfoSimple { + vol := client.VolumeInfoSimple{} + cmInfo := cm.Tactic() + vunitCnt := cmInfo.M + cmInfo.N + cmInfo.L + host := "127.0.0.0:xxx" + locations := make([]proto.VunitLocation, vunitCnt) + var idx uint8 + for i := 0; i < vunitCnt; i++ { + locations[i].Vuid, _ = proto.NewVuid(vid, idx, 1) + locations[i].Host = host + locations[i].DiskID = proto.DiskID(locations[i].Vuid) + idx++ + } + vol.Status = status + vol.VunitLocations = locations + vol.Vid = vid + vol.CodeMode = cm + return &vol +} + +func mockGenVolRepairTask(vid proto.Vid, state proto.RepairState, diskID proto.DiskID, volInfoMap map[proto.Vid]*client.VolumeInfoSimple) *proto.VolRepairTask { + vunitLocations := volInfoMap[vid].VunitLocations + codeMode := volInfoMap[vid].CodeMode + dst := MockAlloc(volInfoMap[vid].VunitLocations[0].Vuid).Location() + task := proto.VolRepairTask{ + TaskID: base.GenTaskID("disk-repair", vid), + State: state, + RepairDiskID: diskID, + CodeMode: codeMode, + Sources: vunitLocations, + BadVuid: vunitLocations[0].Vuid, + BadIdx: 0, + BrokenDiskIDC: "z0", + } + + if state == proto.RepairStatePrepared || + state == proto.RepairStateWorkCompleted || + state == proto.RepairStateFinished { + task.Destination = dst + } + return &task +} + +func MockAlloc(vuid proto.Vuid) *client.AllocVunitInfo { + vid := vuid.Vid() + idx := vuid.Index() + epoch := vuid.Epoch() + epoch++ + newVuid, _ := proto.NewVuid(vid, idx, epoch) + return &client.AllocVunitInfo{ + VunitLocation: proto.VunitLocation{ + Vuid: newVuid, + DiskID: proto.DiskID(newVuid), + Host: "127.0.0.0:xxx", + }, + } +} diff --git a/blobstore/scheduler/service.go b/blobstore/scheduler/service.go new file mode 100644 index 000000000..fb02f84ff --- /dev/null +++ b/blobstore/scheduler/service.go @@ -0,0 +1,477 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "fmt" + "net/http" + + api "github.com/cubefs/cubefs/blobstore/api/scheduler" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/scheduler/base" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/util/task" +) + +// Service rpc service +type Service struct { + ClusterID proto.ClusterID + leader bool + leaderHost string + followerHosts []string + + balanceMgr IBalancer + diskDropMgr IDiskDroper + diskRepairMgr IDiskRepairer + manualMigMgr IManualMigrater + inspectMgr IVolumeInspector + archiveMgr IArchiver + + shardRepairMgr ITaskRunner + blobDeleteMgr ITaskRunner + volCache IVolumeCache + volumeUpdater client.IVolumeUpdater + + clusterMgrCli client.ClusterMgrAPI +} + +// HTTPTaskAcquire acquire task +func (svr *Service) HTTPTaskAcquire(c *rpc.Context) { + ctx := c.Request.Context() + + args := new(api.AcquireArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + manualMigTask, err := svr.manualMigMgr.AcquireTask(ctx, args.IDC) + if err == nil { + ret := &api.WorkerTask{ + TaskType: proto.ManualMigrateType, + ManualMigrate: manualMigTask, + } + c.RespondJSON(ret) + return + } + + repairTask, err := svr.diskRepairMgr.AcquireTask(ctx, args.IDC) + if err == nil { + ret := &api.WorkerTask{ + TaskType: proto.RepairTaskType, + Repair: repairTask, + } + c.RespondJSON(ret) + return + } + + diskDropTask, err := svr.diskDropMgr.AcquireTask(ctx, args.IDC) + if err == nil { + ret := &api.WorkerTask{ + TaskType: proto.DiskDropTaskType, + DiskDrop: diskDropTask, + } + c.RespondJSON(ret) + return + } + + balanceTask, err := svr.balanceMgr.AcquireTask(ctx, args.IDC) + if err == nil { + ret := &api.WorkerTask{ + TaskType: proto.BalanceTaskType, + Balance: balanceTask, + } + c.RespondJSON(ret) + return + } + + c.RespondError(errcode.ErrNothingTodo) +} + +// HTTPTaskReclaim reclaim task +func (svr *Service) HTTPTaskReclaim(c *rpc.Context) { + ctx := c.Request.Context() + + args := new(api.ReclaimTaskArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + if !proto.ValidTaskType(args.TaskType) { + c.RespondError(rpc.NewError(http.StatusBadRequest, "illegal_type", errcode.ErrIllegalTaskType)) + return + } + + newDst, err := base.AllocVunitSafe(ctx, svr.clusterMgrCli, args.Dest.Vuid, args.Src) + if err != nil { + c.RespondError(err) + return + } + + switch args.TaskType { + case proto.RepairTaskType: + err = svr.diskRepairMgr.ReclaimTask(ctx, args.IDC, args.TaskId, args.Src, args.Dest, newDst) + case proto.BalanceTaskType: + err = svr.balanceMgr.ReclaimTask(ctx, args.IDC, args.TaskId, args.Src, args.Dest, newDst) + case proto.DiskDropTaskType: + err = svr.diskDropMgr.ReclaimTask(ctx, args.IDC, args.TaskId, args.Src, args.Dest, newDst) + case proto.ManualMigrateType: + err = svr.manualMigMgr.ReclaimTask(ctx, args.IDC, args.TaskId, args.Src, args.Dest, newDst) + } + + c.RespondError(err) +} + +// HTTPTaskCancel cancel task +func (svr *Service) HTTPTaskCancel(c *rpc.Context) { + ctx := c.Request.Context() + + args := new(api.CancelTaskArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + if !proto.ValidTaskType(args.TaskType) { + c.RespondError(rpc.NewError(http.StatusBadRequest, "illegal_type", errcode.ErrIllegalTaskType)) + return + } + + var err error + switch args.TaskType { + case proto.RepairTaskType: + err = svr.diskRepairMgr.CancelTask(ctx, args) + case proto.BalanceTaskType: + err = svr.balanceMgr.CancelTask(ctx, args) + case proto.DiskDropTaskType: + err = svr.diskDropMgr.CancelTask(ctx, args) + case proto.ManualMigrateType: + err = svr.manualMigMgr.CancelTask(ctx, args) + } + + c.RespondError(err) +} + +// HTTPTaskComplete complete task +func (svr *Service) HTTPTaskComplete(c *rpc.Context) { + ctx := c.Request.Context() + + args := new(api.CompleteTaskArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + if !proto.ValidTaskType(args.TaskType) { + c.RespondError(rpc.NewError(http.StatusBadRequest, "illegal_type", errcode.ErrIllegalTaskType)) + return + } + + var err error + switch args.TaskType { + case proto.RepairTaskType: + err = svr.diskRepairMgr.CompleteTask(ctx, args) + case proto.BalanceTaskType: + err = svr.balanceMgr.CompleteTask(ctx, args) + case proto.DiskDropTaskType: + err = svr.diskDropMgr.CompleteTask(ctx, args) + case proto.ManualMigrateType: + err = svr.manualMigMgr.CompleteTask(ctx, args) + } + + c.RespondError(err) +} + +// HTTPInspectAcquire acquire inspect task +func (svr *Service) HTTPInspectAcquire(c *rpc.Context) { + ctx := c.Request.Context() + + task, _ := svr.inspectMgr.AcquireInspect(ctx) + if task != nil { + c.RespondJSON(api.WorkerInspectTask{Task: task}) + return + } + + c.RespondError(errcode.ErrNothingTodo) +} + +// HTTPInspectComplete complete inspect task +func (svr *Service) HTTPInspectComplete(c *rpc.Context) { + ctx := c.Request.Context() + + args := new(api.CompleteInspectArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + svr.inspectMgr.CompleteInspect(ctx, args.InspectRet) + c.Respond() +} + +// HTTPTaskRenewal renewal task +func (svr *Service) HTTPTaskRenewal(c *rpc.Context) { + ctx := c.Request.Context() + + args := new(api.TaskRenewalArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + idc := args.IDC + ret := &api.TaskRenewalRet{ + Repair: make(map[string]string), + Balance: make(map[string]string), + DiskDrop: make(map[string]string), + ManualMigrate: make(map[string]string), + } + + for taskID := range args.Repair { + err := svr.diskRepairMgr.RenewalTask(ctx, idc, taskID) + ret.Repair[taskID] = getErrMsg(err) + } + + for taskID := range args.Balance { + err := svr.balanceMgr.RenewalTask(ctx, idc, taskID) + ret.Balance[taskID] = getErrMsg(err) + } + + for taskID := range args.DiskDrop { + err := svr.diskDropMgr.RenewalTask(ctx, idc, taskID) + ret.DiskDrop[taskID] = getErrMsg(err) + } + + for taskID := range args.ManualMigrate { + err := svr.manualMigMgr.RenewalTask(ctx, idc, taskID) + ret.ManualMigrate[taskID] = getErrMsg(err) + } + + c.RespondJSON(ret) +} + +func getErrMsg(err error) string { + if err == nil { + return "" + } + return err.Error() +} + +// HTTPTaskReport reports task stats +func (svr *Service) HTTPTaskReport(c *rpc.Context) { + args := new(api.TaskReportArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + switch args.TaskType { + case proto.RepairTaskType: + svr.diskRepairMgr.ReportWorkerTaskStats(args) + case proto.BalanceTaskType: + svr.balanceMgr.ReportWorkerTaskStats(args) + case proto.DiskDropTaskType: + svr.diskDropMgr.ReportWorkerTaskStats(args) + case proto.ManualMigrateType: + svr.manualMigMgr.ReportWorkerTaskStats(args) + } + + c.Respond() +} + +func respondTaskDetail(c *rpc.Context, mgr interface { + QueryTask(context.Context, string) (*api.MigrateTaskDetail, error) +}) { + args := new(api.TaskStatArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + detail, err := mgr.QueryTask(c.Request.Context(), args.TaskId) + if err != nil { + c.RespondError(rpc.NewError(http.StatusNotFound, "NotFound", err)) + return + } + c.RespondJSON(detail) +} + +// HTTPBalanceTaskDetail returns balance task detail stats +func (svr *Service) HTTPBalanceTaskDetail(c *rpc.Context) { + respondTaskDetail(c, svr.balanceMgr) +} + +// HTTPDropTaskDetail returns disk drop task detail stats +func (svr *Service) HTTPDropTaskDetail(c *rpc.Context) { + respondTaskDetail(c, svr.diskDropMgr) +} + +// HTTPManualMigrateTaskDetail returns manual migrate task detail stats +func (svr *Service) HTTPManualMigrateTaskDetail(c *rpc.Context) { + respondTaskDetail(c, svr.manualMigMgr) +} + +// HTTPRepairTaskDetail returns repair task detail stats +func (svr *Service) HTTPRepairTaskDetail(c *rpc.Context) { + args := new(api.TaskStatArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + detail, err := svr.diskRepairMgr.QueryTask(c.Request.Context(), args.TaskId) + if err != nil { + c.RespondError(rpc.NewError(http.StatusNotFound, "NotFound", err)) + return + } + c.RespondJSON(detail) +} + +// HTTPStats returns service stats +func (svr *Service) HTTPStats(c *rpc.Context) { + ctx := c.Request.Context() + taskStats := api.TasksStat{} + + // delete stats + deleteSuccessCounter, deleteFailedCounter := svr.blobDeleteMgr.GetTaskStats() + delErrStats, delTotalErrCnt := svr.blobDeleteMgr.GetErrorStats() + taskStats.BlobDelete = &api.RunnerStat{ + Enable: svr.blobDeleteMgr.Enabled(), + SuccessPerMin: fmt.Sprint(deleteSuccessCounter), + FailedPerMin: fmt.Sprint(deleteFailedCounter), + TotalErrCnt: delTotalErrCnt, + ErrStats: delErrStats, + } + + // stats shard repair tasks + repairSuccessCounter, repairFailedCounter := svr.shardRepairMgr.GetTaskStats() + repairErrStats, repairTotalErrCnt := svr.shardRepairMgr.GetErrorStats() + taskStats.ShardRepair = &api.RunnerStat{ + Enable: svr.shardRepairMgr.Enabled(), + SuccessPerMin: fmt.Sprint(repairSuccessCounter), + FailedPerMin: fmt.Sprint(repairFailedCounter), + TotalErrCnt: repairTotalErrCnt, + ErrStats: repairErrStats, + } + + if !svr.leader { + c.RespondJSON(taskStats) + return + } + + // stats repair tasks + repairDiskID, totalTasksCnt, repairedTasksCnt := svr.diskRepairMgr.Progress(ctx) + taskStats.DiskRepair = &api.DiskRepairTasksStat{ + Enable: svr.diskRepairMgr.Enabled(), + RepairingDiskID: repairDiskID, + TotalTasksCnt: totalTasksCnt, + RepairedTasksCnt: repairedTasksCnt, + MigrateTasksStat: svr.diskRepairMgr.Stats(), + } + + // stats drop tasks + dropDiskID, totalTasksCnt, droppedTasksCnt := svr.diskDropMgr.Progress(ctx) + taskStats.DiskDrop = &api.DiskDropTasksStat{ + Enable: svr.diskDropMgr.Enabled(), + DroppingDiskID: dropDiskID, + TotalTasksCnt: totalTasksCnt, + DroppedTasksCnt: droppedTasksCnt, + MigrateTasksStat: svr.diskDropMgr.Stats(), + } + + // stats balance tasks + taskStats.Balance = &api.BalanceTasksStat{ + Enable: svr.balanceMgr.Enabled(), + MigrateTasksStat: svr.balanceMgr.Stats(), + } + + // stats manual migrate tasks + taskStats.ManualMigrate = &api.ManualMigrateTasksStat{ + MigrateTasksStat: svr.manualMigMgr.Stats(), + } + + // stats inspect tasks + finished, timeout := svr.inspectMgr.GetTaskStats() + taskStats.VolumeInspect = &api.VolumeInspectTasksStat{ + Enable: svr.inspectMgr.Enabled(), + FinishedPerMin: fmt.Sprint(finished), + TimeOutPerMin: fmt.Sprint(timeout), + } + + c.RespondJSON(taskStats) +} + +// HTTPManualMigrateTaskAdd adds manual migrate task +func (svr *Service) HTTPManualMigrateTaskAdd(c *rpc.Context) { + ctx := c.Request.Context() + + args := new(api.AddManualMigrateArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + if !args.Valid() { + c.RespondError(errcode.ErrIllegalArguments) + return + } + + err := svr.manualMigMgr.AddManualTask(ctx, args.Vuid, !args.DirectDownload) + c.RespondError(rpc.Error2HTTPError(err)) +} + +// HTTPUpdateVolume updates volume cache +func (svr *Service) HTTPUpdateVolume(c *rpc.Context) { + args := new(api.UpdateVolumeArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + ctx := c.Request.Context() + span := trace.SpanFromContextSafe(ctx) + + // update local cache of volume + _, err := svr.volCache.Update(args.Vid) + if err != nil { + span.Errorf("local volume cache update failed: vid[%d], err[%+v]", args.Vid, err) + c.RespondError(err) + return + } + + if !svr.leader { + c.Respond() + return + } + + tasks := make([]func() error, 0, len(svr.followerHosts)) + for _, host := range svr.followerHosts { + host := host + tasks = append(tasks, func() error { + return svr.volumeUpdater.UpdateFollowerVolumeCache(ctx, host, args.Vid) + }) + } + + span.Debug("to update follower volume cache") + if err := task.Run(ctx, tasks...); err != nil { + span.Errorf("notify follower to update cache err[%+v]", err) + c.RespondError(err) + return + } + c.Respond() +} diff --git a/blobstore/scheduler/service_test.go b/blobstore/scheduler/service_test.go new file mode 100644 index 000000000..a54834936 --- /dev/null +++ b/blobstore/scheduler/service_test.go @@ -0,0 +1,307 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "net/http/httptest" + "sync" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr" + api "github.com/cubefs/cubefs/blobstore/api/scheduler" + "github.com/cubefs/cubefs/blobstore/common/counter" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/testing/mocks" +) + +var ( + schedulerServer *httptest.Server + once sync.Once +) + +func runMockService(s *Service) string { + once.Do(func() { + schedulerServer = httptest.NewServer(NewHandler(s)) + }) + return schedulerServer.URL +} + +func newMockService(t *testing.T) *Service { + ctr := gomock.NewController(t) + + clusterMgrCli := NewMockClusterMgrAPI(ctr) + blobDeleteMgr := NewMockTaskRunner(ctr) + shardRepairMgr := NewMockTaskRunner(ctr) + diskDropMgr := NewMockMigrater(ctr) + diskRepairMgr := NewMockDiskRepairer(ctr) + manualMgr := NewMockMigrater(ctr) + balanceMgr := NewMockMigrater(ctr) + inspectorMgr := NewMockVolumeInspector(ctr) + archiver := NewMockArchiver(ctr) + volumeCache := NewMockVolumeCache(ctr) + + // return balance task + manualMgr.EXPECT().AcquireTask(any, any).Return(nil, errMock) + diskRepairMgr.EXPECT().AcquireTask(any, any).Return(nil, errMock) + diskDropMgr.EXPECT().AcquireTask(any, any).Return(nil, errMock) + balanceMgr.EXPECT().AcquireTask(any, any).Return(&proto.MigrateTask{}, nil) + // return disk drop task + manualMgr.EXPECT().AcquireTask(any, any).Return(nil, errMock) + diskRepairMgr.EXPECT().AcquireTask(any, any).Return(nil, errMock) + diskDropMgr.EXPECT().AcquireTask(any, any).Return(&proto.MigrateTask{}, nil) + // return disk repair task + manualMgr.EXPECT().AcquireTask(any, any).Return(nil, errMock) + diskRepairMgr.EXPECT().AcquireTask(any, any).Return(&proto.VolRepairTask{}, nil) + // return manual migrate task + manualMgr.EXPECT().AcquireTask(any, any).Return(&proto.MigrateTask{}, nil) + + // reclaim repair task + diskRepairMgr.EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil) + clusterMgrCli.EXPECT().AllocVolumeUnit(any, any).Return(&client.AllocVunitInfo{}, nil) + // reclaim balance task + balanceMgr.EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil) + clusterMgrCli.EXPECT().AllocVolumeUnit(any, any).Return(&client.AllocVunitInfo{}, nil) + // reclaim disk drop task + diskDropMgr.EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil) + clusterMgrCli.EXPECT().AllocVolumeUnit(any, any).Return(&client.AllocVunitInfo{}, nil) + // reclaim manual migrate task + manualMgr.EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil) + clusterMgrCli.EXPECT().AllocVolumeUnit(any, any).Return(&client.AllocVunitInfo{}, nil) + + // cancel repair task + diskRepairMgr.EXPECT().CancelTask(any, any).Return(nil) + // cancel balance task + balanceMgr.EXPECT().CancelTask(any, any).Return(nil) + // cancel disk drop task + diskDropMgr.EXPECT().CancelTask(any, any).Return(nil) + // cancel manual migrate task + manualMgr.EXPECT().CancelTask(any, any).Return(nil) + + // complete repair task + diskRepairMgr.EXPECT().CompleteTask(any, any).Return(nil) + // complete balance task + balanceMgr.EXPECT().CompleteTask(any, any).Return(nil) + // complete disk drop task + diskDropMgr.EXPECT().CompleteTask(any, any).Return(nil) + // complete manual migrate task + manualMgr.EXPECT().CompleteTask(any, any).Return(nil) + + // renewal repair task + diskRepairMgr.EXPECT().RenewalTask(any, any, any).Times(3).Return(nil) + // renewal balance task + balanceMgr.EXPECT().RenewalTask(any, any, any).Times(3).Return(nil) + // renewal disk drop task + diskDropMgr.EXPECT().RenewalTask(any, any, any).Times(3).Return(nil) + // renewal manual migrate task + manualMgr.EXPECT().RenewalTask(any, any, any).Times(3).Return(nil) + + // report repair task + diskRepairMgr.EXPECT().ReportWorkerTaskStats(any).Return() + // report balance task + balanceMgr.EXPECT().ReportWorkerTaskStats(any).Return() + // report disk drop task + diskDropMgr.EXPECT().ReportWorkerTaskStats(any).Return() + // report manual migrate task + manualMgr.EXPECT().ReportWorkerTaskStats(any).Return() + + // add manual migrate task + manualMgr.EXPECT().AddManualTask(any, any, any).Return(nil) + + // acquire inspect task + inspectorMgr.EXPECT().AcquireInspect(any).Return(&proto.InspectTask{}, nil) + + // complete inspect task + inspectorMgr.EXPECT().CompleteInspect(any, any).Return() + + // volume update + volumeCache.EXPECT().Update(any).Return(&client.VolumeInfoSimple{}, nil) + volumeCache.EXPECT().Update(any).Return(nil, errMock) + + // stats + blobDeleteMgr.EXPECT().GetErrorStats().Return([]string{}, uint64(0)) + blobDeleteMgr.EXPECT().GetTaskStats().Return([counter.SLOT]int{}, [counter.SLOT]int{}) + blobDeleteMgr.EXPECT().Enabled().Return(true) + shardRepairMgr.EXPECT().GetErrorStats().Return([]string{}, uint64(0)) + shardRepairMgr.EXPECT().GetTaskStats().Return([counter.SLOT]int{}, [counter.SLOT]int{}) + shardRepairMgr.EXPECT().Enabled().Return(true) + diskRepairMgr.EXPECT().Stats().Return(api.MigrateTasksStat{}) + diskRepairMgr.EXPECT().Progress(any).Return(proto.DiskID(1), 0, 0) + diskRepairMgr.EXPECT().Enabled().Return(true) + diskDropMgr.EXPECT().Stats().Return(api.MigrateTasksStat{}) + diskDropMgr.EXPECT().Progress(any).Return(proto.DiskID(1), 0, 0) + diskDropMgr.EXPECT().Enabled().Return(true) + balanceMgr.EXPECT().Stats().Return(api.MigrateTasksStat{}) + balanceMgr.EXPECT().Enabled().Return(true) + manualMgr.EXPECT().Stats().Return(api.MigrateTasksStat{}) + inspectorMgr.EXPECT().GetTaskStats().Return([counter.SLOT]int{}, [counter.SLOT]int{}) + inspectorMgr.EXPECT().Enabled().Return(true) + + // task detail + balanceMgr.EXPECT().QueryTask(any, any).Return(nil, nil) + diskDropMgr.EXPECT().QueryTask(any, any).Return(nil, nil) + diskRepairMgr.EXPECT().QueryTask(any, any).Return(nil, nil) + manualMgr.EXPECT().QueryTask(any, any).Return(nil, nil) + balanceMgr.EXPECT().QueryTask(any, any).Return(nil, errMock) + diskDropMgr.EXPECT().QueryTask(any, any).Return(nil, errMock) + diskRepairMgr.EXPECT().QueryTask(any, any).Return(nil, errMock) + manualMgr.EXPECT().QueryTask(any, any).Return(nil, errMock) + + service := &Service{ + ClusterID: 1, + leader: true, + leaderHost: localHost + ":9800", + balanceMgr: balanceMgr, + diskDropMgr: diskDropMgr, + manualMigMgr: manualMgr, + diskRepairMgr: diskRepairMgr, + inspectMgr: inspectorMgr, + archiveMgr: archiver, + + shardRepairMgr: shardRepairMgr, + blobDeleteMgr: blobDeleteMgr, + volCache: volumeCache, + + clusterMgrCli: clusterMgrCli, + } + return service +} + +func TestServiceAPI(t *testing.T) { + runMockService(newMockService(t)) + ctr := gomock.NewController(t) + clusterMgrCli := mocks.NewMockClientAPI(ctr) + clusterMgrCli.EXPECT().GetService(any, any).AnyTimes().Return(cmapi.ServiceInfo{Nodes: []cmapi.ServiceNode{{ClusterID: 1, Host: schedulerServer.URL}}}, nil) + + ctx := context.Background() + schedulerCli := api.New(&api.Config{}, clusterMgrCli, proto.ClusterID(1)) + { + // acquire task + task, err := schedulerCli.AcquireTask(ctx, &api.AcquireArgs{IDC: "z0"}) + require.NoError(t, err) + require.Equal(t, proto.BalanceTaskType, task.TaskType) + + task, err = schedulerCli.AcquireTask(ctx, &api.AcquireArgs{IDC: "z0"}) + require.NoError(t, err) + require.Equal(t, proto.DiskDropTaskType, task.TaskType) + + task, err = schedulerCli.AcquireTask(ctx, &api.AcquireArgs{IDC: "z0"}) + require.NoError(t, err) + require.Equal(t, proto.RepairTaskType, task.TaskType) + + task, err = schedulerCli.AcquireTask(ctx, &api.AcquireArgs{IDC: "z0"}) + require.NoError(t, err) + require.Equal(t, proto.ManualMigrateType, task.TaskType) + + // reclaim task + err = schedulerCli.ReclaimTask(ctx, &api.ReclaimTaskArgs{IDC: "z0", TaskType: proto.RepairTaskType}) + require.NoError(t, err) + err = schedulerCli.ReclaimTask(ctx, &api.ReclaimTaskArgs{IDC: "z0", TaskType: proto.BalanceTaskType}) + require.NoError(t, err) + err = schedulerCli.ReclaimTask(ctx, &api.ReclaimTaskArgs{IDC: "z0", TaskType: proto.DiskDropTaskType}) + require.NoError(t, err) + err = schedulerCli.ReclaimTask(ctx, &api.ReclaimTaskArgs{IDC: "z0", TaskType: proto.ManualMigrateType}) + require.NoError(t, err) + err = schedulerCli.ReclaimTask(ctx, &api.ReclaimTaskArgs{IDC: "z0", TaskType: "task"}) + require.Error(t, err) + + // cancel task + err = schedulerCli.CancelTask(ctx, &api.CancelTaskArgs{IDC: "z0", TaskType: proto.RepairTaskType}) + require.NoError(t, err) + err = schedulerCli.CancelTask(ctx, &api.CancelTaskArgs{IDC: "z0", TaskType: proto.BalanceTaskType}) + require.NoError(t, err) + err = schedulerCli.CancelTask(ctx, &api.CancelTaskArgs{IDC: "z0", TaskType: proto.DiskDropTaskType}) + require.NoError(t, err) + err = schedulerCli.CancelTask(ctx, &api.CancelTaskArgs{IDC: "z0", TaskType: proto.ManualMigrateType}) + require.NoError(t, err) + err = schedulerCli.CancelTask(ctx, &api.CancelTaskArgs{IDC: "z0", TaskType: "task"}) + require.Error(t, err) + + // complete task + err = schedulerCli.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: "z0", TaskType: proto.RepairTaskType}) + require.NoError(t, err) + err = schedulerCli.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: "z0", TaskType: proto.BalanceTaskType}) + require.NoError(t, err) + err = schedulerCli.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: "z0", TaskType: proto.DiskDropTaskType}) + require.NoError(t, err) + err = schedulerCli.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: "z0", TaskType: proto.ManualMigrateType}) + require.NoError(t, err) + err = schedulerCli.CompleteTask(ctx, &api.CompleteTaskArgs{IDC: "z0", TaskType: "task"}) + require.Error(t, err) + + // renewal task + tasks := map[string]struct{}{"task1": {}, "task2": {}, "task3": {}} + _, err = schedulerCli.RenewalTask(ctx, &api.TaskRenewalArgs{IDC: "z0", Repair: tasks, Balance: tasks, DiskDrop: tasks, ManualMigrate: tasks}) + require.NoError(t, err) + + // report task + err = schedulerCli.ReportTask(ctx, &api.TaskReportArgs{TaskType: proto.RepairTaskType}) + require.NoError(t, err) + err = schedulerCli.ReportTask(ctx, &api.TaskReportArgs{TaskType: proto.BalanceTaskType}) + require.NoError(t, err) + err = schedulerCli.ReportTask(ctx, &api.TaskReportArgs{TaskType: proto.DiskDropTaskType}) + require.NoError(t, err) + err = schedulerCli.ReportTask(ctx, &api.TaskReportArgs{TaskType: proto.ManualMigrateType}) + require.NoError(t, err) + + // add manual migrate task + err = schedulerCli.AddManualMigrateTask(ctx, &api.AddManualMigrateArgs{}) + require.Equal(t, 400, rpc.DetectStatusCode(err)) + err = schedulerCli.AddManualMigrateTask(ctx, &api.AddManualMigrateArgs{Vuid: proto.Vuid(24726512599042)}) + require.NoError(t, err) + + // acquire inspect task + _, err = schedulerCli.AcquireInspectTask(ctx) + require.NoError(t, err) + + // complete inspect task + err = schedulerCli.CompleteInspect(ctx, &api.CompleteInspectArgs{}) + require.NoError(t, err) + + // volume update + err = schedulerCli.UpdateVol(ctx, schedulerServer.URL, proto.Vid(1)) + require.NoError(t, err) + err = schedulerCli.UpdateVol(ctx, schedulerServer.URL, proto.Vid(1)) + require.Error(t, err) + + // stats + _, err = schedulerCli.Stats(ctx, schedulerServer.URL) + require.NoError(t, err) + + // task detail + _, err = schedulerCli.BalanceTaskDetail(ctx, &api.TaskStatArgs{}) + require.NoError(t, err) + _, err = schedulerCli.DiskRepairTaskDetail(ctx, &api.TaskStatArgs{}) + require.NoError(t, err) + _, err = schedulerCli.DiskDropTaskDetail(ctx, &api.TaskStatArgs{}) + require.NoError(t, err) + _, err = schedulerCli.ManualMigrateTaskDetail(ctx, &api.TaskStatArgs{}) + require.NoError(t, err) + + _, err = schedulerCli.BalanceTaskDetail(ctx, &api.TaskStatArgs{}) + require.Error(t, err) + _, err = schedulerCli.DiskRepairTaskDetail(ctx, &api.TaskStatArgs{}) + require.Error(t, err) + _, err = schedulerCli.DiskDropTaskDetail(ctx, &api.TaskStatArgs{}) + require.Error(t, err) + _, err = schedulerCli.ManualMigrateTaskDetail(ctx, &api.TaskStatArgs{}) + require.Error(t, err) + } +} diff --git a/blobstore/scheduler/shard_repairer.go b/blobstore/scheduler/shard_repairer.go new file mode 100644 index 000000000..1e094947e --- /dev/null +++ b/blobstore/scheduler/shard_repairer.go @@ -0,0 +1,453 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/Shopify/sarama" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/trace" + + "github.com/prometheus/client_golang/prometheus" + "golang.org/x/sync/singleflight" + + "github.com/cubefs/cubefs/blobstore/common/counter" + "github.com/cubefs/cubefs/blobstore/common/kafka" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/taskswitch" + "github.com/cubefs/cubefs/blobstore/scheduler/base" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/scheduler/db" + "github.com/cubefs/cubefs/blobstore/util/selector" + "github.com/cubefs/cubefs/blobstore/util/taskpool" +) + +type shardRepairStatus int + +// shard repair status +const ( + ShardRepairDone = shardRepairStatus(iota) + ShardRepairFailed + ShardRepairUnexpect + ShardRepairOrphan +) + +// shard repair name +const ( + ShardRepair = "shard_repair" + priorityConsumer = 1 + normalConsumer = 0 +) + +// ErrBlobnodeServiceUnavailable worker service unavailable +var ErrBlobnodeServiceUnavailable = errors.New("blobnode service unavailable") + +// ShardRepairConfig shard repair config +type ShardRepairConfig struct { + ClusterID proto.ClusterID + IDC string + + TaskPoolSize int `json:"task_pool_size"` + NormalHandleBatchCnt int `json:"normal_handle_batch_cnt"` + + FailHandleBatchCnt int `json:"fail_handle_batch_cnt"` + FailMsgConsumeIntervalMs int64 `json:"fail_msg_consume_interval_ms"` + + Kafka ShardRepairKafkaConfig `json:"-"` +} + +func (cfg *ShardRepairConfig) priorityConsumerConfigs() (consumers []base.PriorityConsumerConfig) { + consumers = append(consumers, base.PriorityConsumerConfig{ + KafkaConfig: base.KafkaConfig{ + BrokerList: cfg.Kafka.BrokerList, + Topic: cfg.Kafka.Priority.Topic, + Partitions: cfg.Kafka.Priority.Partitions, + }, + Priority: priorityConsumer, + }, base.PriorityConsumerConfig{ + KafkaConfig: base.KafkaConfig{ + BrokerList: cfg.Kafka.BrokerList, + Topic: cfg.Kafka.Normal.Topic, + Partitions: cfg.Kafka.Normal.Partitions, + }, + Priority: normalConsumer, + }) + return +} + +func (cfg *ShardRepairConfig) failedConsumerConfig() *base.KafkaConfig { + return &base.KafkaConfig{ + Topic: cfg.Kafka.Failed.Topic, + Partitions: cfg.Kafka.Failed.Partitions, + BrokerList: cfg.Kafka.BrokerList, + } +} + +func (cfg *ShardRepairConfig) failedProducerConfig() *kafka.ProducerCfg { + return &kafka.ProducerCfg{ + BrokerList: cfg.Kafka.BrokerList, + Topic: cfg.Kafka.Failed.Topic, + TimeoutMs: cfg.Kafka.FailMsgSenderTimeoutMs, + } +} + +// ShardRepairMgr shard repair manager +type ShardRepairMgr struct { + taskPool taskpool.TaskPool + taskSwitch *taskswitch.TaskSwitch + volCache IVolumeCache + + normalPriorConsumers base.IConsumer + + failTopicConsumers []base.IConsumer + failMsgConsumeIntervalMs time.Duration + failMsgSender base.IProducer + + normalHandleBatchCnt int + failHandlerBatchCnt int + + blobnodeCli client.BlobnodeAPI + blobnodeSelector selector.Selector + + orphanShardTable db.IOrphanShardTable + + repairSuccessCounter prometheus.Counter + repairSuccessCounterMin *counter.Counter + repairFailedCounter prometheus.Counter + repairFailedCounterMin *counter.Counter + errStatsDistribution *base.ErrorStats + + group singleflight.Group +} + +// NewShardRepairMgr returns shard repair manager +func NewShardRepairMgr( + cfg *ShardRepairConfig, + vc IVolumeCache, + switchMgr *taskswitch.SwitchMgr, + offAccessor db.IKafkaOffsetTable, + orphanShardTbl db.IOrphanShardTable, + blobnodeCli client.BlobnodeAPI, + serviceGetter client.ClusterMgrAPI, +) (*ShardRepairMgr, error) { + priorConsumers, err := base.NewPriorityConsumer(cfg.priorityConsumerConfigs(), offAccessor) + if err != nil { + return nil, err + } + + failTopicConsumers, err := base.NewKafkaPartitionConsumers(cfg.failedConsumerConfig(), offAccessor) + if err != nil { + return nil, err + } + + taskSwitch, err := switchMgr.AddSwitch(taskswitch.ShardRepairSwitchName) + if err != nil { + return nil, err + } + + workerSelector := selector.NewSelectorWithGetter(10*1000, func() (hosts []string, err error) { + return serviceGetter.GetService(context.Background(), proto.ServiceNameBlobNode, cfg.ClusterID) + }) + failMsgSender, err := base.NewMsgSender(cfg.failedProducerConfig()) + if err != nil { + return nil, err + } + + return &ShardRepairMgr{ + blobnodeCli: blobnodeCli, + taskPool: taskpool.New(cfg.TaskPoolSize, cfg.TaskPoolSize), + taskSwitch: taskSwitch, + volCache: vc, + blobnodeSelector: workerSelector, + + normalPriorConsumers: priorConsumers, + + failTopicConsumers: failTopicConsumers, + failMsgSender: failMsgSender, + failMsgConsumeIntervalMs: time.Duration(cfg.FailMsgConsumeIntervalMs) * time.Millisecond, + + normalHandleBatchCnt: cfg.NormalHandleBatchCnt, + failHandlerBatchCnt: cfg.FailHandleBatchCnt, + + orphanShardTable: orphanShardTbl, + + repairSuccessCounter: base.NewCounter(cfg.ClusterID, ShardRepair, base.KindSuccess), + repairFailedCounter: base.NewCounter(cfg.ClusterID, ShardRepair, base.KindFailed), + errStatsDistribution: base.NewErrorStats(), + repairSuccessCounterMin: &counter.Counter{}, + repairFailedCounterMin: &counter.Counter{}, + }, nil +} + +// Enabled returns true if shard repair task is enabled, otherwise returns false +func (s *ShardRepairMgr) Enabled() bool { + return s.taskSwitch.Enabled() +} + +// RunTask run shard repair task +func (s *ShardRepairMgr) RunTask() { + go func() { + for { + s.taskSwitch.WaitEnable() + s.consumerAndRepair(s.normalPriorConsumers, s.normalHandleBatchCnt) + } + }() + + failPtConsumeBatchCnt := s.failHandlerBatchCnt / len(s.failTopicConsumers) + for _, c := range s.failTopicConsumers { + c := c + go func() { + for { + s.taskSwitch.WaitEnable() + s.consumerAndRepair(c, failPtConsumeBatchCnt) + time.Sleep(s.failMsgConsumeIntervalMs) + } + }() + } +} + +type shardRepairRet struct { + status shardRepairStatus + err error + repairMsg *proto.ShardRepairMsg +} + +func (s *ShardRepairMgr) consumerAndRepair(consumer base.IConsumer, batchCnt int) { + span, ctx := trace.StartSpanFromContext(context.Background(), "consumerAndRepair") + defer span.Finish() + + if batchCnt <= 0 { + batchCnt = 1 + } + msgs := consumer.ConsumeMessages(ctx, batchCnt) + + s.handleMsgBatch(ctx, msgs) + + base.InsistOn(ctx, "repairer consumer.CommitOffset", func() error { + return consumer.CommitOffset(ctx) + }) +} + +func (s *ShardRepairMgr) handleMsgBatch(ctx context.Context, msgs []*sarama.ConsumerMessage) { + span := trace.SpanFromContextSafe(ctx) + ctx = trace.ContextWithSpan(ctx, span) + + span.Infof("handle repair msg: len[%d]", len(msgs)) + + finishCh := make(chan shardRepairRet, len(msgs)) + for _, m := range msgs { + func(msg *sarama.ConsumerMessage) { + s.taskPool.Run(func() { + s.handleOneMsg(ctx, msg, finishCh) + }) + }(m) + } + + for i := 0; i < len(msgs); i++ { + ret := <-finishCh + switch ret.status { + case ShardRepairDone: + span.Debugf("repair success: vid[%d], bid[%d], trace_id[%s]", ret.repairMsg.Vid, ret.repairMsg.Bid, ret.repairMsg.ReqId) + s.repairSuccessCounter.Inc() + s.repairSuccessCounterMin.Add() + + case ShardRepairFailed: + span.Warnf("repair failed and send msg to fail queue: vid[%d], bid[%d], reqid[%s], retry[%d], err[%+v]", + ret.repairMsg.Vid, ret.repairMsg.Bid, ret.repairMsg.ReqId, ret.repairMsg.Retry, ret.err) + s.repairFailedCounter.Inc() + s.repairFailedCounterMin.Add() + s.errStatsDistribution.AddFail(ret.err) + + base.InsistOn(ctx, "repairer send2FailQueue", func() error { + return s.send2FailQueue(ctx, *ret.repairMsg) + }) + case ShardRepairUnexpect, ShardRepairOrphan: + s.repairFailedCounter.Inc() + s.repairFailedCounterMin.Add() + s.errStatsDistribution.AddFail(ret.err) + span.Warnf("unexpected result: msg[%+v], err[%+v]", ret.repairMsg, ret.err) + } + } +} + +func (s *ShardRepairMgr) handleOneMsg(ctx context.Context, msg *sarama.ConsumerMessage, finishCh chan<- shardRepairRet) { + var repairMsg proto.ShardRepairMsg + err := json.Unmarshal(msg.Value, &repairMsg) + if err != nil { + finishCh <- shardRepairRet{ + status: ShardRepairUnexpect, + err: err, + repairMsg: nil, + } + return + } + + if !repairMsg.IsValid() { + finishCh <- shardRepairRet{ + status: ShardRepairUnexpect, + err: proto.ErrInvalidMsg, + repairMsg: nil, + } + return + } + pSpan := trace.SpanFromContextSafe(ctx) + pSpan.Debugf("handle one repair msg: msg[%+v]", repairMsg) + _, tmpCtx := trace.StartSpanFromContextWithTraceID(context.Background(), "handleRepairMsg", repairMsg.ReqId) + jobKey := fmt.Sprintf("%d:%d:%s", repairMsg.Vid, repairMsg.Bid, repairMsg.BadIdx) + _, err, _ = s.group.Do(jobKey, func() (ret interface{}, e error) { + e = s.repairWithCheckVolConsistency(tmpCtx, repairMsg) + return + }) + + if isOrphanShard(err) { + finishCh <- shardRepairRet{ + status: ShardRepairOrphan, + err: err, + repairMsg: &repairMsg, + } + return + } + + if err != nil { + finishCh <- shardRepairRet{ + status: ShardRepairFailed, + err: err, + repairMsg: &repairMsg, + } + return + } + + finishCh <- shardRepairRet{ + status: ShardRepairDone, + repairMsg: &repairMsg, + } +} + +func (s *ShardRepairMgr) repairWithCheckVolConsistency(ctx context.Context, repairMsg proto.ShardRepairMsg) error { + return DoubleCheckedRun(ctx, s.volCache, repairMsg.Vid, func(info *client.VolumeInfoSimple) error { + return s.tryRepair(ctx, info, repairMsg) + }) +} + +func (s *ShardRepairMgr) tryRepair(ctx context.Context, volInfo *client.VolumeInfoSimple, repairMsg proto.ShardRepairMsg) error { + span := trace.SpanFromContextSafe(ctx) + + err := s.repairShard(ctx, volInfo, repairMsg) + if err == nil { + return nil + } + + if err == ErrBlobnodeServiceUnavailable { + return err + } + + newVol, err1 := s.volCache.Update(volInfo.Vid) + if err1 != nil || newVol.EqualWith(volInfo) { + // if update volInfo failed or volInfo not updated, don't need retry + span.Warnf("new volInfo is same or volCache.Update failed: vid[%d], vol cache update err[%+v], repair err[%+v]", + volInfo.Vid, err1, err) + return err + } + + if newVol.EqualWith(volInfo) { + span.Errorf("volInfo not updated: volInfo[%+v], newVolInfo[%+v]", volInfo, newVol) + } + return s.repairShard(ctx, newVol, repairMsg) +} + +func (s *ShardRepairMgr) repairShard(ctx context.Context, volInfo *client.VolumeInfoSimple, repairMsg proto.ShardRepairMsg) (err error) { + span := trace.SpanFromContextSafe(ctx) + + span.Infof("repair shard: msg[%+v], vol info[%+v]", repairMsg, volInfo) + + hosts := s.blobnodeSelector.GetRandomN(1) + if len(hosts) == 0 { + return ErrBlobnodeServiceUnavailable + } + workerHost := hosts[0] + + task := proto.ShardRepairTask{ + Bid: repairMsg.Bid, + CodeMode: volInfo.CodeMode, + Sources: volInfo.VunitLocations, + BadIdxs: repairMsg.BadIdx, + Reason: repairMsg.Reason, + } + + err = s.blobnodeCli.RepairShard(ctx, workerHost, task) + if err == nil { + return nil + } + + if isOrphanShard(err) { + s.saveOrphanShard(ctx, repairMsg) + } + + return err +} + +func (s *ShardRepairMgr) saveOrphanShard(ctx context.Context, repairMsg proto.ShardRepairMsg) { + span := trace.SpanFromContextSafe(ctx) + + shard := db.OrphanShard{ + ClusterID: repairMsg.ClusterID, + Vid: repairMsg.Vid, + Bid: repairMsg.Bid, + } + span.Infof("save orphan shard: [%+v]", shard) + + base.InsistOn(ctx, "save orphan shard", func() error { + return s.orphanShardTable.Save(shard) + }) +} + +func (s *ShardRepairMgr) send2FailQueue(ctx context.Context, msg proto.ShardRepairMsg) error { + span := trace.SpanFromContextSafe(ctx) + + msg.Retry++ + b, err := json.Marshal(msg) + if err != nil { + // just panic if marsh fail + span.Panicf("send to fail queue msg json.Marshal failed: msg[%+v], err[%+v]", msg, err) + } + + err = s.failMsgSender.SendMessage(b) + if err != nil { + return fmt.Errorf("send message: err[%w]", err) + } + + return nil +} + +func isOrphanShard(err error) bool { + return rpc.DetectStatusCode(err) == errcode.CodeOrphanShard +} + +// GetTaskStats returns task stats +func (s *ShardRepairMgr) GetTaskStats() (success [counter.SLOT]int, failed [counter.SLOT]int) { + return s.repairSuccessCounterMin.Show(), s.repairFailedCounterMin.Show() +} + +// GetErrorStats returns service error stats +func (s *ShardRepairMgr) GetErrorStats() (errStats []string, totalErrCnt uint64) { + statsResult, totalErrCnt := s.errStatsDistribution.Stats() + return base.FormatPrint(statsResult), totalErrCnt +} diff --git a/blobstore/scheduler/shard_repairer_test.go b/blobstore/scheduler/shard_repairer_test.go new file mode 100644 index 000000000..cf49087b0 --- /dev/null +++ b/blobstore/scheduler/shard_repairer_test.go @@ -0,0 +1,230 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "encoding/json" + "testing" + + "github.com/Shopify/sarama" + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/common/counter" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/taskswitch" + "github.com/cubefs/cubefs/blobstore/scheduler/base" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/testing/mocks" + "github.com/cubefs/cubefs/blobstore/util/taskpool" + "github.com/golang/mock/gomock" +) + +func newShardRepairMgr(t *testing.T) *ShardRepairMgr { + ctr := gomock.NewController(t) + + volCache := NewMockVolumeCache(ctr) + volCache.EXPECT().Get(any).AnyTimes().Return(&client.VolumeInfoSimple{}, nil) + volCache.EXPECT().Update(any).AnyTimes().Return(&client.VolumeInfoSimple{}, nil) + + selector := mocks.NewMockSelector(ctr) + selector.EXPECT().GetRandomN(any).AnyTimes().Return([]string{"http://127.0.0.1:9600"}) + + blobnode := NewMockBlobnodeAPI(ctr) + blobnode.EXPECT().RepairShard(any, any, any).AnyTimes().Return(nil) + + sender := NewMockProducer(ctr) + sender.EXPECT().SendMessage(any).AnyTimes().Return(nil) + + db := NewMockOrphanShardTable(ctr) + db.EXPECT().Save(any).AnyTimes().Return(nil) + + clusterMgrCli := NewMockClusterMgrAPI(ctr) + clusterMgrCli.EXPECT().GetConfig(any, any).AnyTimes().Return("", nil) + switchMgr := taskswitch.NewSwitchMgr(clusterMgrCli) + taskSwitch, _ := switchMgr.AddSwitch(taskswitch.BlobDeleteSwitchName) + + consumer := NewMockConsumer(ctr) + + return &ShardRepairMgr{ + volCache: volCache, + blobnodeSelector: selector, + blobnodeCli: blobnode, + failMsgSender: sender, + orphanShardTable: db, + taskSwitch: taskSwitch, + failTopicConsumers: []base.IConsumer{consumer}, + taskPool: taskpool.New(1, 1), + repairSuccessCounter: base.NewCounter(1, ShardRepair, base.KindSuccess), + repairFailedCounter: base.NewCounter(1, ShardRepair, base.KindFailed), + errStatsDistribution: base.NewErrorStats(), + repairSuccessCounterMin: &counter.Counter{}, + repairFailedCounterMin: &counter.Counter{}, + } +} + +func TestConsumerShardRepairMsg(t *testing.T) { + ctr := gomock.NewController(t) + service := newShardRepairMgr(t) + consumer := NewMockConsumer(ctr) + consumer.EXPECT().CommitOffset(any).AnyTimes().Return(nil) + { + // no messages + consumer.EXPECT().ConsumeMessages(any, any).DoAndReturn( + func(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) { + return []*sarama.ConsumerMessage{} + }, + ) + service.consumerAndRepair(consumer, 0) + } + { + // one message: message is invalid + consumer.EXPECT().ConsumeMessages(any, any).DoAndReturn( + func(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) { + msg := struct{}{} + msgByte, _ := json.Marshal(msg) + kafkaMgs := &sarama.ConsumerMessage{ + Value: msgByte, + } + return []*sarama.ConsumerMessage{kafkaMgs} + }, + ) + service.consumerAndRepair(consumer, 1) + } + { + // return one message and repair success + consumer.EXPECT().ConsumeMessages(any, any).DoAndReturn( + func(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) { + msg := proto.ShardRepairMsg{Bid: 1, Vid: 1, ReqId: "123456", BadIdx: []uint8{0, 1}} + msgByte, _ := json.Marshal(msg) + kafkaMgs := &sarama.ConsumerMessage{ + Value: msgByte, + } + return []*sarama.ConsumerMessage{kafkaMgs} + }, + ) + service.consumerAndRepair(consumer, 2) + } + { + // return one message and repair failed because worker err + consumer.EXPECT().ConsumeMessages(any, any).DoAndReturn( + func(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) { + msg := proto.ShardRepairMsg{Bid: 1, Vid: 1, ReqId: "123456", BadIdx: []uint8{0, 1}} + msgByte, _ := json.Marshal(msg) + kafkaMgs := &sarama.ConsumerMessage{ + Value: msgByte, + } + return []*sarama.ConsumerMessage{kafkaMgs} + }, + ) + oldBlobnode := service.blobnodeCli + blobnode := NewMockBlobnodeAPI(ctr) + blobnode.EXPECT().RepairShard(any, any, any).AnyTimes().Return(errMock) + service.blobnodeCli = blobnode + service.consumerAndRepair(consumer, 2) + service.blobnodeCli = oldBlobnode + } + { + // return one message and repair failed because worker err(should update volume map) + consumer.EXPECT().ConsumeMessages(any, any).DoAndReturn( + func(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) { + msg := proto.ShardRepairMsg{Bid: 1, Vid: 1, ReqId: "123456", BadIdx: []uint8{0, 1}} + msgByte, _ := json.Marshal(msg) + kafkaMgs := &sarama.ConsumerMessage{ + Value: msgByte, + } + return []*sarama.ConsumerMessage{kafkaMgs} + }, + ) + oldBlobnode := service.blobnodeCli + blobnode := NewMockBlobnodeAPI(ctr) + blobnode.EXPECT().RepairShard(any, any, any).AnyTimes().Return(errcode.ErrDestReplicaBad) + service.blobnodeCli = blobnode + service.consumerAndRepair(consumer, 2) + service.blobnodeCli = oldBlobnode + } + { + + // return one message and repair failed because worker return ErrOrphanShard err + consumer.EXPECT().ConsumeMessages(any, any).DoAndReturn( + func(ctx context.Context, msgCnt int) (msgs []*sarama.ConsumerMessage) { + msg := proto.ShardRepairMsg{Bid: 1, Vid: 1, ReqId: "123456", BadIdx: []uint8{0, 1}} + msgByte, _ := json.Marshal(msg) + kafkaMgs := &sarama.ConsumerMessage{ + Value: msgByte, + } + return []*sarama.ConsumerMessage{kafkaMgs} + }, + ) + oldBlobnode := service.blobnodeCli + blobnode := NewMockBlobnodeAPI(ctr) + blobnode.EXPECT().RepairShard(any, any, any).AnyTimes().Return(errcode.ErrOrphanShard) + service.blobnodeCli = blobnode + service.consumerAndRepair(consumer, 2) + service.blobnodeCli = oldBlobnode + } + { + // get stats + service.GetErrorStats() + service.GetTaskStats() + } + { + // run task + service.RunTask() + } +} + +func TestNewShardRepairMgr(t *testing.T) { + ctr := gomock.NewController(t) + + broker0 := NewBroker(t) + defer broker0.Close() + + cfg := &ShardRepairConfig{ + Kafka: ShardRepairKafkaConfig{ + BrokerList: []string{broker0.Addr()}, + Normal: TopicConfig{Topic: testTopic, Partitions: []int32{0}}, + Priority: TopicConfig{Topic: testTopic, Partitions: []int32{0}}, + Failed: TopicConfig{Topic: testTopic, Partitions: []int32{0}}, + }, + } + + volCache := NewMockVolumeCache(ctr) + volCache.EXPECT().Get(any).AnyTimes().Return(&client.VolumeInfoSimple{}, nil) + volCache.EXPECT().Update(any).AnyTimes().Return(&client.VolumeInfoSimple{}, nil) + + clusterMgrCli := NewMockClusterMgrAPI(ctr) + switchMgr := taskswitch.NewSwitchMgr(clusterMgrCli) + + accessor := NewMockKafkaOffsetTable(ctr) + accessor.EXPECT().Get(any, any).AnyTimes().Return(int64(0), nil) + accessor.EXPECT().Set(any, any, any).AnyTimes().Return(nil) + + orphanShardTable := NewMockOrphanShardTable(ctr) + orphanShardTable.EXPECT().Save(any).AnyTimes().Return(nil) + + blobnode := NewMockBlobnodeAPI(ctr) + blobnode.EXPECT().RepairShard(any, any, any).AnyTimes().Return(nil) + + clusterCli := NewMockClusterMgrAPI(ctr) + clusterCli.EXPECT().GetService(any, any, any).Return(nil, errMock) + + _, err := NewShardRepairMgr(cfg, volCache, switchMgr, accessor, orphanShardTable, blobnode, clusterCli) + require.NoError(t, err) + + _, err = NewShardRepairMgr(cfg, volCache, switchMgr, accessor, orphanShardTable, blobnode, clusterCli) + require.Error(t, err) +} diff --git a/blobstore/scheduler/startup.go b/blobstore/scheduler/startup.go new file mode 100644 index 000000000..71dfbb1d2 --- /dev/null +++ b/blobstore/scheduler/startup.go @@ -0,0 +1,387 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "errors" + "fmt" + "net/http" + httpproxy "net/http/httputil" + "net/url" + "time" + + cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr" + + api "github.com/cubefs/cubefs/blobstore/api/scheduler" + "github.com/cubefs/cubefs/blobstore/cmd" + "github.com/cubefs/cubefs/blobstore/common/config" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/taskswitch" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/scheduler/base" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/scheduler/db" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +const ( + localHost = "127.0.0.1" + scheme = "http://" +) + +var ( + errIllegalClusterID = errors.New("illegal cluster_id") + errInvalidMembers = errors.New("invalid members") + errInvalidLeader = errors.New("invalid leader") + errInvalidNodeID = errors.New("invalid node_id") + errInvalidMongo = errors.New("invalid mongo") +) + +var ( + service *Service + conf Config +) + +func init() { + mod := &cmd.Module{ + Name: proto.ServiceNameScheduler, + InitConfig: initConfig, + SetUp: setUp, + TearDown: tearDown, + } + cmd.RegisterModule(mod) +} + +func initConfig(args []string) (*cmd.Config, error) { + config.Init("f", "", "scheduler.conf") + + if err := config.Load(&conf); err != nil { + return nil, err + } + + return &conf.Config, nil +} + +func setUp() (*rpc.Router, []rpc.ProgressHandler) { + var err error + service, err = NewService(&conf) + if err != nil { + log.Panicf("new service failed, err: %v", err) + } + return NewHandler(service), []rpc.ProgressHandler{service} +} + +func tearDown() { + // close record file safety + service.Close() +} + +// NewService returns scheduler service +func NewService(conf *Config) (svr *Service, err error) { + if err := conf.fixConfig(); err != nil { + log.Errorf("service config check failed: err[%v]", err) + return nil, err + } + + svr = &Service{ + ClusterID: conf.ClusterID, + leader: conf.IsLeader(), + leaderHost: conf.Leader(), + followerHosts: conf.Follower(), + } + + // init db + database, err := db.OpenDatabase(&conf.Database) + if err != nil { + log.Errorf("open database failed: err[%+v]", err) + return nil, errInvalidMongo + } + + clusterMgrCli := client.NewClusterMgrClient(&conf.ClusterMgr) + + blobnodeCli := client.NewBlobnodeClient(&conf.Blobnode) + switchMgr := taskswitch.NewSwitchMgr(clusterMgrCli) + volumeUpdater := client.NewVolumeUpdater(&api.Config{Config: conf.Scheduler}, scheme+localHost+conf.BindAddr) + + vc := NewVolumeCache(clusterMgrCli, conf.VolumeCacheUpdateIntervalS) + conf.ShardRepair.Kafka = conf.Kafka.ShardRepair + conf.ShardRepair.Kafka.BrokerList = conf.Kafka.BrokerList + shardRepairMgr, err := NewShardRepairMgr(&conf.ShardRepair, vc, switchMgr, database.KafkaOffsetTable, database.OrphanShardTable, blobnodeCli, clusterMgrCli) + if err != nil { + log.Errorf("new shard repair mgr: cfg[%+v], err[%w]", conf.ShardRepair, err) + return nil, err + } + + conf.BlobDelete.Kafka = conf.Kafka.BlobDelete + conf.BlobDelete.Kafka.BrokerList = conf.Kafka.BrokerList + deleteMgr, err := NewBlobDeleteMgr(&conf.BlobDelete, vc, database.KafkaOffsetTable, blobnodeCli, switchMgr) + if err != nil { + log.Errorf("new blob delete mgr: cfg[%+v], err[%w]", conf.BlobDelete, err) + return nil, err + } + + svr.shardRepairMgr = shardRepairMgr + svr.blobDeleteMgr = deleteMgr + svr.volCache = vc + svr.volumeUpdater = volumeUpdater + svr.clusterMgrCli = clusterMgrCli + + if err = svr.register(conf.ServiceRegister); err != nil { + return nil, fmt.Errorf("service register: err:[%w]", err) + } + + go svr.RunTask() + + if !svr.leader { + return + } + + err = svr.runKafkaMonitor(conf.ClusterID, database.KafkaOffsetTable) + if err != nil { + log.Errorf("run kafka monitor failed: err[%w]", err) + return nil, err + } + + // init cluster topology + topoConf := &clusterTopoConf{ + ClusterID: conf.ClusterID, + UpdateInterval: time.Duration(conf.TopologyUpdateIntervalMin) * time.Minute, + FreeChunkCounterBuckets: conf.FreeChunkCounterBuckets, + } + topologyMgr := NewClusterTopologyMgr(clusterMgrCli, topoConf) + + // new balance manager + balanceTaskSwitch, err := switchMgr.AddSwitch(taskswitch.BalanceSwitchName) + if err != nil { + panic("unexpect add task switch fail") + } + balanceMgr := NewBalanceMgr(clusterMgrCli, volumeUpdater, balanceTaskSwitch, topologyMgr, database.BalanceTable, &conf.Balance) + + // new disk drop manager + diskDropTaskSwitch, err := switchMgr.AddSwitch(taskswitch.DiskDropSwitchName) + if err != nil { + panic("unexpect add task switch fail") + } + diskDropMgr := NewDiskDropMgr(clusterMgrCli, volumeUpdater, diskDropTaskSwitch, database.DiskDropTable, &conf.DiskDrop) + + // new manual migrate manager + manualMigMgr := NewManualMigrateMgr(clusterMgrCli, volumeUpdater, database.ManualMigrateTable, conf.ClusterID) + + // new disk repair manager + diskRepairTaskSwitch, err := switchMgr.AddSwitch(taskswitch.DiskRepairSwitchName) + if err != nil { + return nil, err + } + repairMgr := NewRepairMgr(&conf.DiskRepair, diskRepairTaskSwitch, database.RepairTaskTable, clusterMgrCli) + + mqProxy := client.NewProxyClient(&conf.Proxy, cmapi.New(&conf.ClusterMgr), conf.ClusterID) + inspectorTaskSwitch, err := switchMgr.AddSwitch(taskswitch.VolumeInspectSwitchName) + if err != nil { + return nil, err + } + inspectMgr := NewVolumeInspectMgr(database.InspectCheckPointTable, clusterMgrCli, mqProxy, inspectorTaskSwitch, &conf.VolumeInspect) + + // new archive store manager + archiveMgr := NewArchiveStoreMgr(database.ArchiveTable, conf.Archive) + archiveMgr.RegisterTables(database.BalanceTable, database.DiskDropTable, database.RepairTaskTable) + + svr.balanceMgr = balanceMgr + svr.diskDropMgr = diskDropMgr + svr.manualMigMgr = manualMigMgr + svr.diskRepairMgr = repairMgr + svr.inspectMgr = inspectMgr + svr.archiveMgr = archiveMgr + + err = svr.waitAndLoad() + if err != nil { + log.Errorf("load task from database failed: err[%+v]", err) + return nil, err + } + + go svr.Run() + return svr, nil +} + +func (svr *Service) waitAndLoad() error { + //why:service stop a task lease period to make sure all worker release task + //so there will not a task run on multiple worker + log.Infof("start waitAndLoad") + time.Sleep(proto.TaskLeaseExpiredS * time.Second) + return svr.load() +} + +func (svr *Service) load() (err error) { + if err = svr.diskRepairMgr.Load(); err != nil { + return + } + if err = svr.balanceMgr.Load(); err != nil { + return + } + if err = svr.diskDropMgr.Load(); err != nil { + return + } + if err = svr.manualMigMgr.Load(); err != nil { + return + } + + return +} + +func (svr *Service) register(cfg ServiceRegisterConfig) error { + info := client.RegisterInfo{ + ClusterID: uint64(svr.ClusterID), + Name: proto.ServiceNameScheduler, + Host: cfg.Host, + Idc: cfg.Idc, + HeartbeatIntervalS: cfg.TickInterval, + HeartbeatTicks: cfg.HeartbeatTicks, + ExpiresTicks: cfg.ExpiresTicks, + } + return svr.clusterMgrCli.Register(context.Background(), info) +} + +// Run run task +func (svr *Service) Run() { + svr.diskRepairMgr.Run() + svr.balanceMgr.Run() + svr.diskDropMgr.Run() + svr.manualMigMgr.Run() + svr.archiveMgr.Run() + svr.inspectMgr.Run() +} + +// RunTask run shard repair and blob delete tasks +func (svr *Service) RunTask() { + err := svr.LoadVolInfo() + if err != nil { + log.Panicf("load volume info failed: err[%+v]", err) + } + svr.shardRepairMgr.RunTask() + svr.blobDeleteMgr.RunTask() +} + +func (svr *Service) runKafkaMonitor(clusterID proto.ClusterID, access db.IKafkaOffsetTable) error { + // collect cfg + var topicCfgs []*base.KafkaConfig + topicCfgs = append(topicCfgs, conf.BlobDelete.normalConsumerConfig()) + topicCfgs = append(topicCfgs, conf.BlobDelete.failedConsumerConfig()) + topicCfgs = append(topicCfgs, conf.ShardRepair.failedConsumerConfig()) + for _, topicCfg := range conf.ShardRepair.priorityConsumerConfigs() { + topicCfgs = append(topicCfgs, &topicCfg.KafkaConfig) + } + + // start topic monitor + monitorIntervalS := 1 + for _, topicCfg := range topicCfgs { + m, err := base.NewKafkaTopicMonitor(clusterID, topicCfg, access, monitorIntervalS) + if err != nil { + log.Errorf("new kafka topic monitor topic failed: topic[%s], err[%+v]", topicCfg.Topic, err) + return err + } + go m.Run() + } + return nil +} + +// LoadVolInfo load volume info +func (svr *Service) LoadVolInfo() error { + return svr.volCache.Load() +} + +func (svr *Service) Handler(w http.ResponseWriter, req *http.Request, f func(http.ResponseWriter, *http.Request)) { + if svr.needForwardToLeader(req) { + svr.forwardToLeader(w, req) + return + } + f(w, req) +} + +func (svr *Service) needForwardToLeader(req *http.Request) bool { + if !svr.leader { + switch req.URL.Path { + case api.PathUpdateVolume, api.PathStats: + return false + default: + return true + } + } + return false +} + +// forwardToLeader will forward http request to raft leader +func (svr *Service) forwardToLeader(w http.ResponseWriter, req *http.Request) { + url, err := url.Parse(scheme + req.RequestURI) + if err != nil { + panic("parse leader host url failed: " + err.Error()) + } + url.Host = svr.leaderHost + + span := trace.SpanFromContextSafe(req.Context()) + span.Debugf("forward url: %v", url) + + proxy := httpproxy.ReverseProxy{ + Director: func(request *http.Request) { + request.URL = url + }, + } + + proxy.ServeHTTP(w, req) +} + +// Close close service safe +func (svr *Service) Close() { + if !svr.leader { + return + } + svr.balanceMgr.Close() + svr.diskRepairMgr.Close() + svr.diskDropMgr.Close() + svr.manualMigMgr.Close() + svr.inspectMgr.Close() + svr.archiveMgr.Close() +} + +// NewHandler returns app server handler +func NewHandler(service *Service) *rpc.Router { + rpc.RegisterArgsParser(&api.AcquireArgs{}, "json") + rpc.RegisterArgsParser(&api.TaskStatArgs{}, "json") + + // rpc http svr interface + rpc.GET(api.PathTaskAcquire, service.HTTPTaskAcquire, rpc.OptArgsQuery()) + rpc.POST(api.PathTaskReclaim, service.HTTPTaskReclaim, rpc.OptArgsBody()) + rpc.POST(api.PathTaskCancel, service.HTTPTaskCancel, rpc.OptArgsBody()) + rpc.POST(api.PathTaskComplete, service.HTTPTaskComplete, rpc.OptArgsBody()) + rpc.POST(api.PathManualMigrateTaskAdd, service.HTTPManualMigrateTaskAdd, rpc.OptArgsBody()) + + rpc.GET(api.PathInspectAcquire, service.HTTPInspectAcquire) + rpc.POST(api.PathInspectComplete, service.HTTPInspectComplete, rpc.OptArgsBody()) + + rpc.POST(api.PathTaskReport, service.HTTPTaskReport, rpc.OptArgsBody()) + rpc.POST(api.PathTaskRenewal, service.HTTPTaskRenewal, rpc.OptArgsBody()) + + // TODO: merge details into only one uri + rpc.POST(api.PathBalanceTaskDetail, service.HTTPBalanceTaskDetail, rpc.OptArgsBody()) + rpc.POST(api.PathRepairTaskDetail, service.HTTPRepairTaskDetail, rpc.OptArgsBody()) + rpc.POST(api.PathDropTaskDetail, service.HTTPDropTaskDetail, rpc.OptArgsBody()) + rpc.POST(api.PathManualMigrateTaskDetail, service.HTTPManualMigrateTaskDetail, rpc.OptArgsBody()) + rpc.GET(api.PathStats, service.HTTPStats, rpc.OptArgsQuery()) + rpc.GET(api.PathLeaderStats, service.HTTPStats, rpc.OptArgsQuery()) + + rpc.POST(api.PathUpdateVolume, service.HTTPUpdateVolume, rpc.OptArgsBody()) + + return rpc.DefaultRouter +} diff --git a/blobstore/scheduler/startup_test.go b/blobstore/scheduler/startup_test.go new file mode 100644 index 000000000..77dd1a1ee --- /dev/null +++ b/blobstore/scheduler/startup_test.go @@ -0,0 +1,274 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/golang/mock/gomock" + + cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr" + api "github.com/cubefs/cubefs/blobstore/api/scheduler" + "github.com/cubefs/cubefs/blobstore/common/counter" + "github.com/cubefs/cubefs/blobstore/common/mongoutil" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/scheduler/db" + "github.com/cubefs/cubefs/blobstore/testing/mocks" + "github.com/stretchr/testify/require" +) + +var ( + leaderServer *httptest.Server + followerServer *httptest.Server +) + +func runMockLeaderService(s *Service) string { + router := rpc.New() + router.Handle(http.MethodGet, api.PathStats, s.HTTPStats, rpc.OptArgsQuery()) + router.Handle(http.MethodGet, api.PathLeaderStats, s.HTTPStats, rpc.OptArgsQuery()) + router.Handle(http.MethodPost, api.PathUpdateVolume, s.HTTPUpdateVolume, rpc.OptArgsBody()) + router.Handle(http.MethodGet, api.PathTaskAcquire, s.HTTPTaskAcquire, rpc.OptArgsQuery()) + + leaderServer = httptest.NewServer(rpc.MiddlewareHandlerWith(router, s)) + return leaderServer.URL +} + +func runMockFollowerService(s *Service) string { + router := rpc.New() + router.Handle(http.MethodGet, api.PathStats, s.HTTPStats, rpc.OptArgsQuery()) + router.Handle(http.MethodPost, api.PathUpdateVolume, s.HTTPUpdateVolume, rpc.OptArgsBody()) + + followerServer = httptest.NewServer(rpc.MiddlewareHandlerWith(router, s)) + return followerServer.URL +} + +func TestNewService(t *testing.T) { + mongoURI := "" + testCases := []struct { + conf *Config + err error + }{ + { + conf: &Config{}, + err: errIllegalClusterID, + }, + { + conf: &Config{ClusterID: proto.ClusterID(1)}, + err: errInvalidMembers, + }, + { + conf: &Config{ + ClusterID: proto.ClusterID(1), + Services: Services{Members: map[uint64]string{1: "127.0.0.1:9800"}}, + }, + err: errInvalidLeader, + }, + { + conf: &Config{ + ClusterID: proto.ClusterID(1), + Services: Services{Leader: 1, Members: map[uint64]string{1: "127.0.0.1:9800"}}, + }, + err: errInvalidNodeID, + }, + { + conf: &Config{ + ClusterID: proto.ClusterID(1), + Services: Services{Leader: 1, NodeID: 2, Members: map[uint64]string{1: "127.0.0.1:9800", 2: "127.0.0.1:9880"}}, + }, + err: errInvalidMongo, + }, + { + conf: &Config{ + ClusterID: proto.ClusterID(1), + Database: db.Config{Mongo: mongoutil.Config{URI: mongoURI}}, + Services: Services{Leader: 1, NodeID: 2, Members: map[uint64]string{1: "127.0.0.1:9800", 2: "127.0.0.1:9880"}}, + }, + err: errInvalidMongo, + }, + } + for _, tc := range testCases { + _, err := NewService(tc.conf) + require.Equal(t, true, errors.Is(err, tc.err)) + } +} + +func newMockServiceWithOpts(ctr *gomock.Controller, isLeader bool) *Service { + clusterMgrCli := NewMockClusterMgrAPI(ctr) + blobDeleteMgr := NewMockTaskRunner(ctr) + shardRepairMgr := NewMockTaskRunner(ctr) + diskDropMgr := NewMockMigrater(ctr) + diskRepairMgr := NewMockDiskRepairer(ctr) + manualMgr := NewMockMigrater(ctr) + balanceMgr := NewMockMigrater(ctr) + inspecterMgr := NewMockVolumeInspector(ctr) + archiver := NewMockArchiver(ctr) + volumeCache := NewMockVolumeCache(ctr) + volumeUpdater := NewMockVolumeUpdater(ctr) + + balanceMgr.EXPECT().Close().AnyTimes().Return() + diskRepairMgr.EXPECT().Close().AnyTimes().Return() + diskDropMgr.EXPECT().Close().AnyTimes().Return() + manualMgr.EXPECT().Close().AnyTimes().Return() + archiver.EXPECT().Close().AnyTimes().Return() + inspecterMgr.EXPECT().Close().AnyTimes().Return() + + balanceMgr.EXPECT().Run().AnyTimes().Return() + diskDropMgr.EXPECT().Run().AnyTimes().Return() + diskRepairMgr.EXPECT().Run().AnyTimes().Return() + archiver.EXPECT().Run().AnyTimes().Return() + inspecterMgr.EXPECT().Run().AnyTimes().Return() + manualMgr.EXPECT().Run().AnyTimes().Return() + + volumeCache.EXPECT().Load().AnyTimes().Return(nil) + shardRepairMgr.EXPECT().RunTask().AnyTimes().Return() + blobDeleteMgr.EXPECT().RunTask().AnyTimes().Return() + + balanceMgr.EXPECT().Load().AnyTimes().Return(nil) + diskRepairMgr.EXPECT().Load().AnyTimes().Return(nil) + diskDropMgr.EXPECT().Load().AnyTimes().Return(nil) + manualMgr.EXPECT().Load().AnyTimes().Return(nil) + + blobDeleteMgr.EXPECT().GetErrorStats().AnyTimes().Return([]string{}, uint64(0)) + blobDeleteMgr.EXPECT().GetTaskStats().AnyTimes().Return([counter.SLOT]int{}, [counter.SLOT]int{}) + blobDeleteMgr.EXPECT().Enabled().AnyTimes().Return(true) + shardRepairMgr.EXPECT().GetErrorStats().AnyTimes().Return([]string{}, uint64(0)) + shardRepairMgr.EXPECT().GetTaskStats().AnyTimes().Return([counter.SLOT]int{}, [counter.SLOT]int{}) + shardRepairMgr.EXPECT().Enabled().AnyTimes().Return(true) + diskRepairMgr.EXPECT().Stats().AnyTimes().Return(api.MigrateTasksStat{}) + diskRepairMgr.EXPECT().Progress(any).AnyTimes().Return(proto.DiskID(1), 0, 0) + diskRepairMgr.EXPECT().Enabled().AnyTimes().Return(true) + diskDropMgr.EXPECT().Stats().AnyTimes().Return(api.MigrateTasksStat{}) + diskDropMgr.EXPECT().Progress(any).AnyTimes().Return(proto.DiskID(1), 0, 0) + diskDropMgr.EXPECT().Enabled().AnyTimes().Return(true) + balanceMgr.EXPECT().Stats().AnyTimes().Return(api.MigrateTasksStat{}) + balanceMgr.EXPECT().Enabled().AnyTimes().Return(true) + manualMgr.EXPECT().Stats().AnyTimes().Return(api.MigrateTasksStat{}) + inspecterMgr.EXPECT().GetTaskStats().AnyTimes().Return([counter.SLOT]int{}, [counter.SLOT]int{}) + inspecterMgr.EXPECT().Enabled().AnyTimes().Return(true) + + volumeUpdater.EXPECT().UpdateFollowerVolumeCache(any, any, any).AnyTimes().Return(nil) + volumeUpdater.EXPECT().UpdateLeaderVolumeCache(any, any).AnyTimes().Return(nil) + + manualMgr.EXPECT().AcquireTask(any, any).AnyTimes().Return(&proto.MigrateTask{}, nil) + + volumeCache.EXPECT().Update(any).AnyTimes().Return(&client.VolumeInfoSimple{}, nil) + + clusterMgrCli.EXPECT().GetConfig(any, any).AnyTimes().Return("", errMock) + service := &Service{ + ClusterID: 1, + leader: isLeader, + balanceMgr: balanceMgr, + diskDropMgr: diskDropMgr, + manualMigMgr: manualMgr, + diskRepairMgr: diskRepairMgr, + inspectMgr: inspecterMgr, + archiveMgr: archiver, + + shardRepairMgr: shardRepairMgr, + blobDeleteMgr: blobDeleteMgr, + volCache: volumeCache, + volumeUpdater: volumeUpdater, + + clusterMgrCli: clusterMgrCli, + } + return service +} + +func TestServer(t *testing.T) { + ctx := context.Background() + ctr := gomock.NewController(t) + + leaderServer := newMockServiceWithOpts(ctr, true) + defer leaderServer.Close() + + leaderHost := runMockLeaderService(leaderServer) + + followerServer := newMockServiceWithOpts(ctr, false) + defer followerServer.Close() + + followerHost := runMockFollowerService(followerServer) + + leaderServer.leaderHost = leaderHost[len(scheme):] + followerServer.leaderHost = leaderHost[len(scheme):] + leaderServer.followerHosts = []string{followerHost[len(scheme):]} + followerServer.followerHosts = []string{followerHost[len(scheme):]} + + go leaderServer.Run() + go followerServer.Run() + + go leaderServer.RunTask() + go followerServer.RunTask() + + kafkaOffset := NewMockKafkaOffsetTable(ctr) + kafkaOffset.EXPECT().Get(any, any).AnyTimes().Return(int64(0), nil) + kafkaOffset.EXPECT().Set(any, any, any).AnyTimes().Return(nil) + + err := leaderServer.runKafkaMonitor(proto.ClusterID(1), kafkaOffset) + require.Error(t, err) + + hosts := []string{leaderHost, followerHost} + for _, host := range hosts { + clusterMgrCli := mocks.NewMockClientAPI(ctr) + clusterMgrCli.EXPECT().GetService(any, any).AnyTimes().Return(cmapi.ServiceInfo{Nodes: []cmapi.ServiceNode{{ClusterID: 1, Host: host}}}, nil) + cli := api.New(&api.Config{}, clusterMgrCli, proto.ClusterID(1)) + _, err = cli.Stats(ctx, host) + require.NoError(t, err) + _, err = cli.LeaderStats(ctx) + require.NoError(t, err) + + err = cli.UpdateVol(ctx, leaderHost, proto.Vid(1)) + require.NoError(t, err) + + err = cli.UpdateVol(ctx, followerHost, proto.Vid(1)) + require.NoError(t, err) + + _, err = cli.AcquireTask(ctx, &api.AcquireArgs{IDC: "z0"}) + require.NoError(t, err) + } + + err = leaderServer.load() + require.NoError(t, err) +} + +func TestServiceSetup(t *testing.T) { + require.Panics(t, func() { + setUp() + }) +} + +func TestServiceInitConf(t *testing.T) { + _, err := initConfig([]string{}) + require.Error(t, err) +} + +func TestServiceClose(t *testing.T) { + ctr := gomock.NewController(t) + service = newMockServiceWithOpts(ctr, false) + tearDown() +} + +func TestServiceRegister(t *testing.T) { + ctr := gomock.NewController(t) + service = newMockServiceWithOpts(ctr, false) + service.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().Register(any, any).Return(nil) + err := service.register(ServiceRegisterConfig{}) + require.NoError(t, err) +} diff --git a/blobstore/scheduler/volume_cache.go b/blobstore/scheduler/volume_cache.go new file mode 100644 index 000000000..cb5603bda --- /dev/null +++ b/blobstore/scheduler/volume_cache.go @@ -0,0 +1,225 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "golang.org/x/sync/singleflight" + + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/util/log" + "github.com/cubefs/cubefs/blobstore/util/retry" +) + +// IVolumeCache define the interface used for volume cache manager +type IVolumeCache interface { + Update(vid proto.Vid) (*client.VolumeInfoSimple, error) + Get(vid proto.Vid) (*client.VolumeInfoSimple, error) + Load() error +} + +const ( + shardCount = 32 + + defaultMarker = proto.Vid(0) + defaultCount = 1000 +) + +// ErrFrequentlyUpdate frequently update +var ErrFrequentlyUpdate = errors.New("frequently update") + +var errVolumeMissmatch = errors.New("volume missmatch during running task") + +type ( + volumeTime struct { + time time.Time + volume client.VolumeInfoSimple + } + shardCacher struct { + sync.RWMutex + m map[proto.Vid]*volumeTime + } + volumeCacher struct { + interval time.Duration + cache [shardCount]*shardCacher + } +) + +func (vc *volumeCacher) getShard(vid proto.Vid) *shardCacher { + return vc.cache[uint(vid)%shardCount] +} + +func (vc *volumeCacher) Get(vid proto.Vid) (*client.VolumeInfoSimple, bool) { + shard := vc.getShard(vid) + shard.RLock() + val, exist := shard.m[vid] + shard.RUnlock() + if exist { + return &val.volume, true + } + return nil, false +} + +func (vc *volumeCacher) Set(vid proto.Vid, volume client.VolumeInfoSimple) { + shard := vc.getShard(vid) + shard.Lock() + shard.m[vid] = &volumeTime{ + time: time.Now(), + volume: volume, + } + shard.Unlock() +} + +func (vc *volumeCacher) Settable(vid proto.Vid) bool { + shard := vc.getShard(vid) + shard.RLock() + val, exist := shard.m[vid] + shard.RUnlock() + if !exist { + return true + } + return time.Now().After(val.time.Add(vc.interval)) +} + +func newVolumeCacher(interval time.Duration) *volumeCacher { + c := new(volumeCacher) + c.interval = interval + for idx := range c.cache { + m := make(map[proto.Vid]*volumeTime, 32) + c.cache[idx] = &shardCacher{m: m} + } + return c +} + +// VolumeCache volume cache +type VolumeCache struct { + clusterMgrCli client.ClusterMgrAPI + group singleflight.Group + cache *volumeCacher +} + +// NewVolumeCache returns volume cache manager. +func NewVolumeCache(client client.ClusterMgrAPI, updateIntervalS int) *VolumeCache { + return &VolumeCache{ + clusterMgrCli: client, + cache: newVolumeCacher(time.Duration(updateIntervalS) * time.Second), + } +} + +// Load list all volumes info memory cache. +func (c *VolumeCache) Load() error { + marker := defaultMarker + for { + log.Infof("to load volume marker[%d], count[%d]", marker, defaultCount) + + var ( + volInfos []*client.VolumeInfoSimple + nextMarker proto.Vid + err error + ) + if err = retry.Timed(3, 200).On(func() error { + volInfos, nextMarker, err = c.clusterMgrCli.ListVolume(context.Background(), marker, defaultCount) + return err + }); err != nil { + log.Errorf("list volume: marker[%d], count[%+v], code[%d], error[%v]", + marker, defaultCount, rpc.DetectStatusCode(err), err) + return err + } + + for _, v := range volInfos { + c.cache.Set(v.Vid, *v) + } + if len(volInfos) == 0 || nextMarker == defaultMarker { + break + } + + marker = nextMarker + } + return nil +} + +// Get returns this volume info. +func (c *VolumeCache) Get(vid proto.Vid) (*client.VolumeInfoSimple, error) { + if vol, ok := c.cache.Get(vid); ok { + return vol, nil + } + + vol, err := c.Update(vid) + if err != nil { + return nil, err + } + return vol, nil +} + +// Update this volume info cache. +func (c *VolumeCache) Update(vid proto.Vid) (*client.VolumeInfoSimple, error) { + if !c.cache.Settable(vid) { + return nil, ErrFrequentlyUpdate + } + + val, err, _ := c.group.Do(fmt.Sprintf("volume-update-%d", vid), func() (interface{}, error) { + vol, err := c.clusterMgrCli.GetVolumeInfo(context.Background(), vid) + if err != nil { + return nil, err + } + + c.cache.Set(vid, *vol) + return vol, nil + }) + if err != nil { + return nil, err + } + + return val.(*client.VolumeInfoSimple), nil +} + +// DoubleCheckedRun the scheduler updates volume mapping relation asynchronously, +// then some task(delete or repair) had started with old volume mapping. +// +// if delete on old relation, there will has garbage shard in new chunk. ==> garbage shard +// if repair on old relation, there still is missing shard in new chunk. ==> missing shard +func DoubleCheckedRun(ctx context.Context, c IVolumeCache, vid proto.Vid, task func(*client.VolumeInfoSimple) error) error { + span := trace.SpanFromContextSafe(ctx) + vol, err := c.Get(vid) + if err != nil { + return err + } + + for range [3]struct{}{} { + if err := task(vol); err != nil { + return err + } + + newVol, err := c.Get(vol.Vid) + if err != nil { + return err + } + if newVol.EqualWith(vol) { + return nil + } + + span.Warnf("volume changed from [%+v] to [%+v]", vol, newVol) + vol = newVol + } + return errVolumeMissmatch +} diff --git a/blobstore/scheduler/volume_cache_test.go b/blobstore/scheduler/volume_cache_test.go new file mode 100644 index 000000000..a0436f7f4 --- /dev/null +++ b/blobstore/scheduler/volume_cache_test.go @@ -0,0 +1,152 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/scheduler/client" +) + +func TestVolumeCache(t *testing.T) { + cmClient := NewMockClusterMgrAPI(gomock.NewController(t)) + cmClient.EXPECT().ListVolume(any, any, any).Times(2).DoAndReturn( + func(_ context.Context, marker proto.Vid, _ int) ([]*client.VolumeInfoSimple, proto.Vid, error) { + if marker == defaultMarker { + return []*client.VolumeInfoSimple{{Vid: 4}}, proto.Vid(10), nil + } + return []*client.VolumeInfoSimple{{Vid: 9}}, defaultMarker, nil + }, + ) + cmClient.EXPECT().GetVolumeInfo(any, any).DoAndReturn( + func(_ context.Context, vid proto.Vid) (*client.VolumeInfoSimple, error) { + return &client.VolumeInfoSimple{Vid: vid}, nil + }, + ) + + volCache := NewVolumeCache(cmClient, 10) + err := volCache.Load() + require.NoError(t, err) + + // no cache will update + _, err = volCache.Get(1) + require.NoError(t, err) + // return cache + _, err = volCache.Get(1) + require.NoError(t, err) + + // update ErrFrequentlyUpdate + _, err = volCache.Update(1) + require.ErrorIs(t, err, ErrFrequentlyUpdate) + + // list and get failed + cmClient.EXPECT().ListVolume(any, any, any).AnyTimes().Return(nil, proto.Vid(0), errMock) + cmClient.EXPECT().GetVolumeInfo(any, any).Return(&client.VolumeInfoSimple{}, errMock) + volCache = NewVolumeCache(cmClient, -1) + _, err = volCache.Get(1) + require.ErrorIs(t, err, errMock) + err = volCache.Load() + require.ErrorIs(t, err, errMock) +} + +func TestDoubleCheckedRun(t *testing.T) { + cmClient := NewMockClusterMgrAPI(gomock.NewController(t)) + { + cmClient.EXPECT().GetVolumeInfo(any, any).Return(&client.VolumeInfoSimple{}, nil) + c := NewVolumeCache(cmClient, -1) + err := DoubleCheckedRun(context.Background(), c, 1, func(*client.VolumeInfoSimple) error { return errMock }) + require.Equal(t, err, errMock) + } + { + cmClient.EXPECT().GetVolumeInfo(any, any).Return(nil, errcode.ErrVolumeNotExist) + c := NewVolumeCache(cmClient, -1) + err := DoubleCheckedRun(context.Background(), c, 1, func(*client.VolumeInfoSimple) error { return errMock }) + require.ErrorIs(t, err, errcode.ErrVolumeNotExist) + } + { + cmClient.EXPECT().GetVolumeInfo(any, any).Return(&client.VolumeInfoSimple{}, nil) + cmClient.EXPECT().GetVolumeInfo(any, any).Return(nil, errcode.ErrVolumeNotExist) + c := NewVolumeCache(cmClient, -1) + err := DoubleCheckedRun(context.Background(), c, 1, func(*client.VolumeInfoSimple) error { return nil }) + require.Equal(t, err, errcode.ErrVolumeNotExist) + } + { + cmClient.EXPECT().GetVolumeInfo(any, any).Return( + &client.VolumeInfoSimple{Vid: 1, VunitLocations: []proto.VunitLocation{{Vuid: 1}}}, nil) + cmClient.EXPECT().GetVolumeInfo(any, any).Return( + &client.VolumeInfoSimple{Vid: 1, VunitLocations: []proto.VunitLocation{{Vuid: 2}}}, nil) + cmClient.EXPECT().GetVolumeInfo(any, any).Return( + &client.VolumeInfoSimple{Vid: 1, VunitLocations: []proto.VunitLocation{{Vuid: 2}}}, nil) + + c := NewVolumeCache(cmClient, -1) + err := DoubleCheckedRun(context.Background(), c, 1, func(volInfo *client.VolumeInfoSimple) error { + c.Update(1) + return nil + }) + require.NoError(t, err) + } + { + cmClient.EXPECT().GetVolumeInfo(any, any).Return( + &client.VolumeInfoSimple{Vid: 1, VunitLocations: []proto.VunitLocation{{Vuid: 1}}}, nil) + cmClient.EXPECT().GetVolumeInfo(any, any).Return( + &client.VolumeInfoSimple{Vid: 1, VunitLocations: []proto.VunitLocation{{Vuid: 2}}}, nil) + cmClient.EXPECT().GetVolumeInfo(any, any).Return( + &client.VolumeInfoSimple{Vid: 1, VunitLocations: []proto.VunitLocation{{Vuid: 3}}}, nil) + cmClient.EXPECT().GetVolumeInfo(any, any).Return( + &client.VolumeInfoSimple{Vid: 1, VunitLocations: []proto.VunitLocation{{Vuid: 4}}}, nil) + + c := NewVolumeCache(cmClient, -1) + err := DoubleCheckedRun(context.Background(), c, 1, func(volInfo *client.VolumeInfoSimple) error { + c.Update(1) + return nil + }) + require.Equal(t, err, errVolumeMissmatch) + } +} + +func BenchmarkVolumeCache(b *testing.B) { + for _, cs := range []struct { + Name string + Items int + }{ + {"tiny", 1 << 5}, + {"norm", 1 << 16}, + {"huge", 1 << 20}, + } { + items := cs.Items + b.Run(cs.Name, func(b *testing.B) { + vols := make([]*client.VolumeInfoSimple, items) + for idx := range vols { + vols[idx] = &client.VolumeInfoSimple{Vid: proto.Vid(idx)} + } + cmCli := NewMockClusterMgrAPI(gomock.NewController(b)) + cmCli.EXPECT().ListVolume(any, any, any).Return(vols, defaultMarker, nil) + + cacher := NewVolumeCache(cmCli, -1) + require.NoError(b, cacher.Load()) + + b.ResetTimer() + for ii := 0; ii < b.N; ii++ { + cacher.Get(proto.Vid(ii % items)) + } + }) + } +} diff --git a/blobstore/scheduler/volume_inspector.go b/blobstore/scheduler/volume_inspector.go new file mode 100644 index 000000000..05823c02b --- /dev/null +++ b/blobstore/scheduler/volume_inspector.go @@ -0,0 +1,575 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "errors" + "fmt" + "sort" + "sync" + "time" + + "github.com/cubefs/cubefs/blobstore/common/counter" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/taskswitch" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/scheduler/base" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/scheduler/db" + "github.com/cubefs/cubefs/blobstore/util/closer" + "github.com/cubefs/cubefs/blobstore/util/log" + "github.com/cubefs/cubefs/blobstore/util/retry" +) + +// IVolumeInspector define the interface of volume inspect manager +type IVolumeInspector interface { + AcquireInspect(ctx context.Context) (*proto.InspectTask, error) + CompleteInspect(ctx context.Context, ret *proto.InspectRet) + GetTaskStats() (finished, timeout [counter.SLOT]int) + Enabled() bool + Run() + closer.Closer +} + +const ( + defaultPrepareFailSleepS = 10 + zeroVid = proto.Vid(0) + defaultDuplicateCnt = 10000000 +) + +// manager of volumes inspect +// batch execution in steps(Non-persistent) +// step1.gen inspect task +// step2.worker execute inspect task +// step3.collect inspect missed shards info and notice mq proxy +var ( + errTaskHasAcquired = errors.New("task has been acquired") + errForbiddenAcquire = errors.New("forbidden acquire task") +) + +// IVolsGetter define the interface of clustermgr used by inspect +type IVolsGetter interface { + ListVolume(ctx context.Context, vid proto.Vid, count int) ([]*client.VolumeInfoSimple, proto.Vid, error) + GetVolumeInfo(ctx context.Context, Vid proto.Vid) (ret *client.VolumeInfoSimple, err error) +} + +// IRepairShardSender define the shard repair interface used by inspect +type IRepairShardSender interface { + SendShardRepairMsg(ctx context.Context, vid proto.Vid, bid proto.BlobID, badIdx []uint8) error +} + +type inspectTaskInfo struct { + t *proto.InspectTask + ret *proto.InspectRet + acquireTime *time.Time +} + +func (t *inspectTaskInfo) tryAcquire() error { + if t.acquired() { + return errTaskHasAcquired + } + now := time.Now() + t.acquireTime = &now + return nil +} + +func (t *inspectTaskInfo) complete(ret *proto.InspectRet) { + t.ret = ret +} + +func (t *inspectTaskInfo) running(timeoutMs time.Duration) bool { + if t.completed() || t.timeout(timeoutMs) { + return false + } + return true +} + +func (t *inspectTaskInfo) timeout(timeoutMs time.Duration) bool { + if t.acquired() && !t.completed() { + deadline := t.acquireTime.Add(timeoutMs * time.Millisecond) + return time.Now().After(deadline) + } + return false +} + +func (t *inspectTaskInfo) hasMissedShard() bool { + return t.completed() && len(t.ret.MissedShards) != 0 +} + +func (t *inspectTaskInfo) acquired() bool { + return t.acquireTime != nil +} + +func (t *inspectTaskInfo) completed() bool { + return t.ret != nil +} + +type badShardDeduplicator struct { + l sync.Mutex + badShards map[string]struct{} + shardsCntLimit int +} + +func newBadShardDeduplicator(shardsCntLimit int) *badShardDeduplicator { + return &badShardDeduplicator{ + badShards: make(map[string]struct{}), + shardsCntLimit: shardsCntLimit, + } +} + +func (d *badShardDeduplicator) add(vid proto.Vid, bid proto.BlobID, badIdxs []uint8) { + d.l.Lock() + defer d.l.Unlock() + + if len(d.badShards) >= d.shardsCntLimit { + d.badShards = make(map[string]struct{}) + } + key := d.key(vid, bid, badIdxs) + d.badShards[key] = struct{}{} +} + +func (d *badShardDeduplicator) reduplicate(vid proto.Vid, bid proto.BlobID, badIdxs []uint8) bool { + d.l.Lock() + defer d.l.Unlock() + + key := d.key(vid, bid, badIdxs) + if _, ok := d.badShards[key]; ok { + return true + } + return false +} + +func (d *badShardDeduplicator) key(vid proto.Vid, bid proto.BlobID, badIdxs []uint8) string { + sortBads(badIdxs) + return fmt.Sprintf("%d-%d-%v", vid, bid, badIdxs) +} + +// VolumeInspectMgrCfg inspect task manager config +type VolumeInspectMgrCfg struct { + InspectIntervalS int `json:"inspect_interval_s"` + InspectBatch int `json:"inspect_batch"` + + // iops of list volume info + ListVolStep int `json:"list_vol_step"` + ListVolIntervalMs int `json:"list_vol_interval_ms"` + + // timeout of inspect + TimeoutMs int `json:"timeout_ms"` +} + +// VolumeInspectMgr inspect task manager +type VolumeInspectMgr struct { + closer.Closer + tasks map[string]*inspectTaskInfo + tasksL sync.Mutex + + acquireEnable bool + acquireEnableL sync.Mutex + + // start vid in current batch + startVid proto.Vid + // start vid in next batch + nextVid proto.Vid + + firstPrepare bool + + taskSwitch taskswitch.ISwitcher + tbl db.IInspectCheckPointTable + volsGetter IVolsGetter + + repairShardSender IRepairShardSender + sendDeduplicator *badShardDeduplicator + + completeTaskCounter counter.Counter + timeoutCounter counter.Counter + + cfg *VolumeInspectMgrCfg +} + +// NewVolumeInspectMgr returns inspect task manager +func NewVolumeInspectMgr( + tbl db.IInspectCheckPointTable, + volsGetter IVolsGetter, + repairShardSender IRepairShardSender, + taskSwitch taskswitch.ISwitcher, cfg *VolumeInspectMgrCfg) *VolumeInspectMgr { + return &VolumeInspectMgr{ + Closer: closer.New(), + tasks: make(map[string]*inspectTaskInfo), + acquireEnable: false, + firstPrepare: true, + taskSwitch: taskSwitch, + tbl: tbl, + volsGetter: volsGetter, + repairShardSender: repairShardSender, + sendDeduplicator: newBadShardDeduplicator(defaultDuplicateCnt), + cfg: cfg, + } +} + +// Enabled returns true if task switch status +func (mgr *VolumeInspectMgr) Enabled() bool { + return mgr.taskSwitch.Enabled() +} + +// Run run inspect task manager +func (mgr *VolumeInspectMgr) Run() { + go mgr.run() +} + +func (mgr *VolumeInspectMgr) run() { + t := time.NewTicker(time.Duration(mgr.cfg.InspectIntervalS) * time.Second) + defer t.Stop() + + for { + select { + case <-t.C: + mgr.taskSwitch.WaitEnable() + mgr.inspectRun() + case <-mgr.Closer.Done(): + return + } + } +} + +func (mgr *VolumeInspectMgr) inspectRun() { + span, ctx := trace.StartSpanFromContext(context.Background(), "inspector.run") + defer span.Finish() + + mgr.prepare(ctx) + mgr.waitCompleted(ctx) + mgr.finish(ctx) +} + +func (mgr *VolumeInspectMgr) enableAcquire(enable bool) { + mgr.acquireEnableL.Lock() + defer mgr.acquireEnableL.Unlock() + mgr.acquireEnable = enable +} + +func (mgr *VolumeInspectMgr) canAcquire() bool { + mgr.acquireEnableL.Lock() + defer mgr.acquireEnableL.Unlock() + return mgr.acquireEnable +} + +func (mgr *VolumeInspectMgr) getStartVid(ctx context.Context) proto.Vid { + if mgr.firstPrepare { + mgr.firstPrepare = false + ck, err := mgr.tbl.GetCheckPoint(ctx) + if err == nil { + return ck.StartVid + } + log.Warnf("firstPrepare get check point failed: err[%+v]", err) + return zeroVid + } + + if !mgr.allVolVisited() { + return mgr.nextVid + } + return zeroVid +} + +func (mgr *VolumeInspectMgr) prepare(ctx context.Context) { + span := trace.SpanFromContextSafe(ctx) + + var ( + err error + vols []*client.VolumeInfoSimple + nextVid proto.Vid + volCnt int + ) + + mgr.startVid = mgr.getStartVid(ctx) + startVid := mgr.startVid + span.Infof("start prepare inspect task: start vid[%d]", startVid) + + for volCnt < mgr.cfg.InspectBatch { + remainCnt := mgr.cfg.InspectBatch - volCnt + listStep := mgr.cfg.ListVolStep + if remainCnt <= mgr.cfg.ListVolStep { + listStep = remainCnt + } + + span.Debugf("prepare inspect task: start vid[%d], list step[%d]", startVid, listStep) + vols, nextVid, err = mgr.volsGetter.ListVolume(ctx, startVid, listStep) + if err != nil { + span.Errorf("list volume failed: err[%+v]", err) + time.Sleep(defaultPrepareFailSleepS * time.Second) + continue + } + + if len(vols) == 0 { + break + } + + for _, vol := range vols { + if vol.IsActive() { + span.Infof("volume is active and skip: vid[%d]", vol.Vid) + continue + } + + taskID := mgr.genTaskID(vol) + mgr.tasks[taskID] = &inspectTaskInfo{ + t: mgr.genInspectTask(taskID, vol), + ret: nil, + acquireTime: nil, + } + span.Debugf("prepare inspect task: vid[%d], task_id[%s]", vol.Vid, taskID) + volCnt++ + } + + startVid = nextVid + time.Sleep(time.Duration(mgr.cfg.ListVolIntervalMs) * time.Millisecond) + } + + mgr.nextVid = nextVid + + span.Infof("prepare finished: next vid[%d], task count[%d]", nextVid, len(mgr.tasks)) +} + +// AcquireInspect acquire inspect task +func (mgr *VolumeInspectMgr) AcquireInspect(ctx context.Context) (*proto.InspectTask, error) { + if !mgr.canAcquire() { + return nil, errForbiddenAcquire + } + + if !mgr.taskSwitch.Enabled() { + return nil, proto.ErrTaskPaused + } + + mgr.tasksL.Lock() + defer mgr.tasksL.Unlock() + + for _, task := range mgr.tasks { + if task.tryAcquire() == nil { + return task.t, nil + } + } + + return nil, proto.ErrTaskEmpty +} + +// CompleteInspect complete inspect task +func (mgr *VolumeInspectMgr) CompleteInspect(ctx context.Context, ret *proto.InspectRet) { + span := trace.SpanFromContextSafe(ctx) + + if !mgr.canAcquire() { + return + } + + mgr.tasksL.Lock() + defer mgr.tasksL.Unlock() + + taskID := ret.TaskID + if _, ok := mgr.tasks[taskID]; !ok { + span.Warnf("inspect task not found: task_id[%s]", taskID) + return + } + + mgr.tasks[taskID].complete(ret) + mgr.completeTaskCounter.Add() + + span.Debugf("inspect complete: task_id[%s]", taskID) +} + +func (mgr *VolumeInspectMgr) waitCompleted(ctx context.Context) { + span := trace.SpanFromContextSafe(ctx) + span.Infof("start wait completed...") + + mgr.enableAcquire(true) + defer mgr.enableAcquire(false) + + t := time.NewTicker(time.Duration(mgr.cfg.TimeoutMs) * time.Millisecond) + defer t.Stop() + + for { + select { + case <-t.C: + span.Debugf("check all task completed") + if mgr.allTaskCompleted() { + return + } + case <-mgr.Closer.Done(): + return + } + } +} + +func (mgr *VolumeInspectMgr) allTaskCompleted() bool { + mgr.tasksL.Lock() + defer mgr.tasksL.Unlock() + + for _, task := range mgr.tasks { + if task.running(time.Duration(mgr.cfg.TimeoutMs)) { + return false + } + } + return true +} + +func (mgr *VolumeInspectMgr) finish(ctx context.Context) { + span := trace.SpanFromContextSafe(ctx) + span.Infof("start finish inspect tasks...") + + mgr.tasksL.Lock() + defer mgr.tasksL.Unlock() + + // collect missed bids + var missedShards [][]*proto.MissedShard + for _, task := range mgr.tasks { + if task.hasMissedShard() { + missedShards = append(missedShards, task.ret.MissedShards) + continue + } + } + + // clear & stats tasks + for taskID, task := range mgr.tasks { + span.Debugf("check task and clear: task_id[%s]", taskID) + if task.completed() { + span.Debugf("inspect task is completed: task_id[%s]", taskID) + } + + if task.timeout(time.Duration(mgr.cfg.TimeoutMs)) { + span.Debugf("inspect timeout: task_id[%s]", taskID) + mgr.timeoutCounter.Add() + } + delete(mgr.tasks, taskID) + } + + // post repair shard msg + for _, volMissedShards := range missedShards { + vid := volMissedShards[0].Vuid.Vid() + + volInfo, err := mgr.volsGetter.GetVolumeInfo(ctx, vid) + if err != nil { + span.Errorf("get volume info failed: err[%+v]", err) + continue + } + + if volInfo.IsActive() { + span.Infof("volume is active and will skip: vid[%d]", volInfo.Vid) + continue + } + + bidsBads, err := mgr.collectVolInspectBads(ctx, volMissedShards) + if err != nil { + span.Errorf("collect volume inspect bads failed: vid[%d], err[%+v]", vid, err) + continue + } + + for bid, bads := range bidsBads { + span.Infof("inspect missed: vid[%d], bid[%d], shards[%+v]", vid, bid, bads) + base.InsistOn(ctx, "send shard repair msg failed", func() error { + return mgr.trySendShardRepairMsg(ctx, vid, bid, bads) + }) + } + } + + err := retry.Timed(3, 200).On(func() error { + return mgr.tbl.SaveCheckPoint(ctx, mgr.nextVid) + }) + if err != nil { + span.Warnf("save checkpoint failed: err[%+v]", err) + } +} + +func (mgr *VolumeInspectMgr) collectVolInspectBads( + ctx context.Context, + volMissedShards []*proto.MissedShard) (bidsMissed map[proto.BlobID][]uint8, err error, +) { + span := trace.SpanFromContextSafe(ctx) + if len(volMissedShards) == 0 { + return + } + vid := volMissedShards[0].Vuid.Vid() + for _, missedShard := range volMissedShards { + if missedShard.Vuid.Vid() != vid { + span.Errorf("all missed shard vid should be same: missed vid[%d], vid[%d]", missedShard.Vuid.Vid(), vid) + err = errors.New("unexpect:vid not same") + return + } + } + + bidMissedVuid := make(map[proto.BlobID]map[proto.Vuid]struct{}) + for _, missedShard := range volMissedShards { + bid := missedShard.Bid + vuid := missedShard.Vuid + _, ok := bidMissedVuid[bid] + if !ok { + bidMissedVuid[bid] = make(map[proto.Vuid]struct{}) + } + bidMissedVuid[bid][vuid] = struct{}{} + } + + bidsMissed = make(map[proto.BlobID][]uint8) + for bid, missVuids := range bidMissedVuid { + var bads []uint8 + for vuid := range missVuids { + bads = append(bads, vuid.Index()) + } + + sortBads(bads) + bidsMissed[bid] = bads + } + return +} + +func (mgr *VolumeInspectMgr) trySendShardRepairMsg(ctx context.Context, vid proto.Vid, bid proto.BlobID, badIdxs []uint8) error { + span := trace.SpanFromContextSafe(ctx) + if mgr.sendDeduplicator.reduplicate(vid, bid, badIdxs) { + span.Infof("volume has send shard repair msg: vid[%d], bid[%d], bad idxs[%+v]", vid, bid, badIdxs) + return nil + } + + err := mgr.repairShardSender.SendShardRepairMsg(ctx, vid, bid, badIdxs) + if err != nil { + return err + } + span.Infof("send shard repair msg success: vid[%d], bid[%d], bad idxs[%+v]", vid, bid, badIdxs) + + mgr.sendDeduplicator.add(vid, bid, badIdxs) + return nil +} + +func (mgr *VolumeInspectMgr) allVolVisited() bool { + return mgr.startVid == mgr.nextVid +} + +func (mgr *VolumeInspectMgr) genTaskID(vol *client.VolumeInfoSimple) string { + return base.GenTaskID("inspect", vol.Vid) +} + +func (mgr *VolumeInspectMgr) genInspectTask(taskID string, vol *client.VolumeInfoSimple) *proto.InspectTask { + return &proto.InspectTask{ + TaskId: taskID, + Mode: vol.CodeMode, + Replicas: vol.VunitLocations, + } +} + +// GetTaskStats return task stats +func (mgr *VolumeInspectMgr) GetTaskStats() (finished, timeout [counter.SLOT]int) { + finished = mgr.completeTaskCounter.Show() + timeout = mgr.timeoutCounter.Show() + return +} + +func sortBads(bads []uint8) { + sort.Slice(bads, func(i, j int) bool { + return bads[i] < bads[j] + }) +} diff --git a/blobstore/scheduler/volume_inspector_test.go b/blobstore/scheduler/volume_inspector_test.go new file mode 100644 index 000000000..16a8f9d03 --- /dev/null +++ b/blobstore/scheduler/volume_inspector_test.go @@ -0,0 +1,317 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/testing/mocks" +) + +func genMockFailShards(vid proto.Vid, bids []proto.BlobID) []*proto.MissedShard { + vuid, _ := proto.NewVuid(vid, 1, 1) + var FailShards []*proto.MissedShard + for _, bid := range bids { + FailShards = append(FailShards, &proto.MissedShard{Vuid: vuid, Bid: bid}) + } + return FailShards +} + +func TestTaskTimeout(t *testing.T) { + task := inspectTaskInfo{} + require.NoError(t, task.tryAcquire()) + require.Equal(t, false, task.timeout(5)) + time.Sleep(10 * time.Millisecond) + require.Equal(t, true, task.timeout(5)) + require.Equal(t, false, task.timeout(10000)) +} + +func TestBadShardDeduplicator(t *testing.T) { + d := newBadShardDeduplicator(3) + require.Equal(t, false, d.reduplicate(1, 1, []uint8{1, 2})) + + d.add(1, 1, []uint8{1, 2}) + require.Equal(t, true, d.reduplicate(1, 1, []uint8{1, 2})) + + require.Equal(t, false, d.reduplicate(2, 1, []uint8{1, 2})) + d.add(2, 1, []uint8{1, 2}) + + require.Equal(t, false, d.reduplicate(2, 1, []uint8{1, 2, 3})) + d.add(2, 1, []uint8{1, 2, 3}) + + d.add(2, 2, []uint8{1, 2, 3}) + require.Equal(t, false, d.reduplicate(2, 1, []uint8{1, 2, 3})) +} + +func newInspector(t *testing.T) *VolumeInspectMgr { + ctr := gomock.NewController(t) + clusterMgr := NewMockClusterMgrAPI(ctr) + taskSwitch := mocks.NewMockSwitcher(ctr) + inspectTable := NewMockInspectCheckPointTable(ctr) + shardRepairSender := NewMockMqProxyAPI(ctr) + conf := &VolumeInspectMgrCfg{InspectIntervalS: defaultInspectIntervalS, TimeoutMs: 1} + return NewVolumeInspectMgr(inspectTable, clusterMgr, shardRepairSender, taskSwitch, conf) +} + +func TestInspectorRun(t *testing.T) { + mgr := newInspector(t) + + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().WaitEnable().AnyTimes().Return() + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().AnyTimes().Return(true) + mgr.tbl.(*MockInspectCheckPointTable).EXPECT().GetCheckPoint(any).AnyTimes().Return(nil, errMock) + mgr.tbl.(*MockInspectCheckPointTable).EXPECT().SaveCheckPoint(any, any).AnyTimes().Return(errMock) + + require.True(t, mgr.Enabled()) + go mgr.Run() + + time.Sleep(defaultInspectIntervalS * time.Second) + mgr.Close() +} + +func TestInspectorPrepare(t *testing.T) { + ctx := context.Background() + { + mgr := newInspector(t) + mgr.cfg.InspectBatch = 2 + mgr.cfg.ListVolStep = 2 + + mgr.tbl.(*MockInspectCheckPointTable).EXPECT().GetCheckPoint(any).AnyTimes().Return(&proto.InspectCheckPoint{}, nil) + mgr.volsGetter.(*MockClusterMgrAPI).EXPECT().ListVolume(any, any, any).Return(nil, proto.Vid(0), nil) + mgr.prepare(ctx) + } + { + mgr := newInspector(t) + mgr.cfg.InspectBatch = 2 + mgr.cfg.ListVolStep = 2 + + volume1 := MockGenVolInfo(100012, codemode.EC6P6, proto.VolumeStatusIdle) + volume2 := MockGenVolInfo(100012, codemode.EC6P6, proto.VolumeStatusActive) + mgr.tbl.(*MockInspectCheckPointTable).EXPECT().GetCheckPoint(any).AnyTimes().Return(&proto.InspectCheckPoint{}, nil) + mgr.volsGetter.(*MockClusterMgrAPI).EXPECT().ListVolume(any, any, any).Return([]*client.VolumeInfoSimple{volume1}, proto.Vid(0), nil) + mgr.volsGetter.(*MockClusterMgrAPI).EXPECT().ListVolume(any, any, any).Return([]*client.VolumeInfoSimple{volume2}, proto.Vid(0), nil) + mgr.volsGetter.(*MockClusterMgrAPI).EXPECT().ListVolume(any, any, any).Return(nil, proto.Vid(0), nil) + + mgr.prepare(ctx) + require.Equal(t, 1, len(mgr.tasks)) + } + { + mgr := newInspector(t) + mgr.firstPrepare = false + mgr.cfg.InspectBatch = 2 + mgr.cfg.ListVolStep = 2 + + volume1 := MockGenVolInfo(100012, codemode.EC6P6, proto.VolumeStatusIdle) + volume2 := MockGenVolInfo(100012, codemode.EC6P6, proto.VolumeStatusActive) + mgr.tbl.(*MockInspectCheckPointTable).EXPECT().GetCheckPoint(any).AnyTimes().Return(&proto.InspectCheckPoint{}, nil) + mgr.volsGetter.(*MockClusterMgrAPI).EXPECT().ListVolume(any, any, any).Return([]*client.VolumeInfoSimple{volume1}, proto.Vid(0), nil) + mgr.volsGetter.(*MockClusterMgrAPI).EXPECT().ListVolume(any, any, any).Return([]*client.VolumeInfoSimple{volume2}, proto.Vid(0), nil) + mgr.volsGetter.(*MockClusterMgrAPI).EXPECT().ListVolume(any, any, any).Return(nil, proto.Vid(0), nil) + + mgr.prepare(ctx) + require.Equal(t, 1, len(mgr.tasks)) + } +} + +func TestInspectorWaitCompleted(t *testing.T) { + ctx := context.Background() + { + mgr := newInspector(t) + + mgr.cfg.InspectBatch = 1 + mgr.cfg.ListVolStep = 1 + + volume := MockGenVolInfo(100012, codemode.EC6P6, proto.VolumeStatusIdle) + mgr.tbl.(*MockInspectCheckPointTable).EXPECT().GetCheckPoint(any).AnyTimes().Return(&proto.InspectCheckPoint{}, nil) + mgr.volsGetter.(*MockClusterMgrAPI).EXPECT().ListVolume(any, any, any).Return([]*client.VolumeInfoSimple{volume}, proto.Vid(0), nil) + + mgr.prepare(ctx) + require.Equal(t, 1, len(mgr.tasks)) + + for _, task := range mgr.tasks { + task.ret = &proto.InspectRet{} + } + mgr.waitCompleted(ctx) + } +} + +func TestInspectorFinish(t *testing.T) { + ctx := context.Background() + { + mgr := newInspector(t) + + mgr.cfg.InspectBatch = 1 + mgr.cfg.ListVolStep = 1 + + volume := MockGenVolInfo(100012, codemode.EC6P6, proto.VolumeStatusIdle) + mgr.tbl.(*MockInspectCheckPointTable).EXPECT().GetCheckPoint(any).AnyTimes().Return(&proto.InspectCheckPoint{}, nil) + mgr.volsGetter.(*MockClusterMgrAPI).EXPECT().ListVolume(any, any, any).Return([]*client.VolumeInfoSimple{volume}, proto.Vid(0), nil) + + mgr.prepare(ctx) + require.Equal(t, 1, len(mgr.tasks)) + + for _, task := range mgr.tasks { + task.ret = &proto.InspectRet{MissedShards: genMockFailShards(100012, []proto.BlobID{3, 4})} + } + mgr.volsGetter.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(nil, errMock) + mgr.tbl.(*MockInspectCheckPointTable).EXPECT().SaveCheckPoint(any, any).Return(nil) + mgr.finish(ctx) + require.Equal(t, 0, len(mgr.tasks)) + } + { + mgr := newInspector(t) + + mgr.cfg.InspectBatch = 1 + mgr.cfg.ListVolStep = 1 + + volume := MockGenVolInfo(100012, codemode.EC6P6, proto.VolumeStatusIdle) + mgr.tbl.(*MockInspectCheckPointTable).EXPECT().GetCheckPoint(any).AnyTimes().Return(&proto.InspectCheckPoint{}, nil) + mgr.volsGetter.(*MockClusterMgrAPI).EXPECT().ListVolume(any, any, any).Return([]*client.VolumeInfoSimple{volume}, proto.Vid(0), nil) + + mgr.prepare(ctx) + require.Equal(t, 1, len(mgr.tasks)) + + for _, task := range mgr.tasks { + task.ret = &proto.InspectRet{MissedShards: genMockFailShards(100012, []proto.BlobID{3, 4})} + } + volume = MockGenVolInfo(100012, codemode.EC6P6, proto.VolumeStatusActive) + mgr.volsGetter.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(volume, nil) + mgr.tbl.(*MockInspectCheckPointTable).EXPECT().SaveCheckPoint(any, any).Return(nil) + mgr.finish(ctx) + require.Equal(t, 0, len(mgr.tasks)) + } + { + mgr := newInspector(t) + + mgr.cfg.InspectBatch = 1 + mgr.cfg.ListVolStep = 1 + + volume := MockGenVolInfo(100012, codemode.EC6P6, proto.VolumeStatusIdle) + mgr.tbl.(*MockInspectCheckPointTable).EXPECT().GetCheckPoint(any).AnyTimes().Return(&proto.InspectCheckPoint{}, nil) + mgr.volsGetter.(*MockClusterMgrAPI).EXPECT().ListVolume(any, any, any).Return([]*client.VolumeInfoSimple{volume}, proto.Vid(0), nil) + + mgr.prepare(ctx) + require.Equal(t, 1, len(mgr.tasks)) + + for _, task := range mgr.tasks { + task.ret = &proto.InspectRet{MissedShards: genMockFailShards(100012, []proto.BlobID{3, 4})} + } + mgr.volsGetter.(*MockClusterMgrAPI).EXPECT().GetVolumeInfo(any, any).Return(volume, nil) + mgr.repairShardSender.(*MockMqProxyAPI).EXPECT().SendShardRepairMsg(any, any, any, any).Return(errMock) + mgr.repairShardSender.(*MockMqProxyAPI).EXPECT().SendShardRepairMsg(any, any, any, any).AnyTimes().Return(nil) + mgr.tbl.(*MockInspectCheckPointTable).EXPECT().SaveCheckPoint(any, any).Return(nil) + + mgr.finish(ctx) + require.Equal(t, 0, len(mgr.tasks)) + } +} + +func TestInspectorAcquire(t *testing.T) { + ctx := context.Background() + { + mgr := newInspector(t) + mgr.enableAcquire(false) + _, err := mgr.AcquireInspect(ctx) + require.True(t, errors.Is(err, errForbiddenAcquire)) + } + { + mgr := newInspector(t) + mgr.enableAcquire(true) + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(false) + _, err := mgr.AcquireInspect(ctx) + require.True(t, errors.Is(err, proto.ErrTaskPaused)) + } + { + mgr := newInspector(t) + mgr.enableAcquire(true) + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(true) + _, err := mgr.AcquireInspect(ctx) + require.True(t, errors.Is(err, proto.ErrTaskEmpty)) + } + { + mgr := newInspector(t) + mgr.enableAcquire(true) + mgr.taskSwitch.(*mocks.MockSwitcher).EXPECT().Enabled().Return(true) + + mgr.cfg.InspectBatch = 1 + mgr.cfg.ListVolStep = 1 + + volume := MockGenVolInfo(100012, codemode.EC6P6, proto.VolumeStatusIdle) + mgr.tbl.(*MockInspectCheckPointTable).EXPECT().GetCheckPoint(any).AnyTimes().Return(&proto.InspectCheckPoint{}, nil) + mgr.volsGetter.(*MockClusterMgrAPI).EXPECT().ListVolume(any, any, any).Return([]*client.VolumeInfoSimple{volume}, proto.Vid(0), nil) + + mgr.prepare(ctx) + require.Equal(t, 1, len(mgr.tasks)) + + var taskID string + for k := range mgr.tasks { + taskID = k + } + + task, err := mgr.AcquireInspect(ctx) + require.NoError(t, err) + require.Equal(t, mgr.tasks[taskID].t.TaskId, task.TaskId) + } +} + +func TestInspectorComplete(t *testing.T) { + ctx := context.Background() + { + mgr := newInspector(t) + mgr.enableAcquire(false) + + mgr.CompleteInspect(ctx, &proto.InspectRet{}) + } + { + mgr := newInspector(t) + mgr.enableAcquire(true) + + mgr.CompleteInspect(ctx, &proto.InspectRet{}) + } + { + mgr := newInspector(t) + mgr.enableAcquire(true) + + mgr.cfg.InspectBatch = 1 + mgr.cfg.ListVolStep = 1 + + volume := MockGenVolInfo(100012, codemode.EC6P6, proto.VolumeStatusIdle) + mgr.tbl.(*MockInspectCheckPointTable).EXPECT().GetCheckPoint(any).AnyTimes().Return(&proto.InspectCheckPoint{}, nil) + mgr.volsGetter.(*MockClusterMgrAPI).EXPECT().ListVolume(any, any, any).Return([]*client.VolumeInfoSimple{volume}, proto.Vid(0), nil) + + mgr.prepare(ctx) + require.Equal(t, 1, len(mgr.tasks)) + + var taskID string + for k := range mgr.tasks { + taskID = k + } + + mgr.CompleteInspect(ctx, &proto.InspectRet{TaskID: taskID}) + } +} + +func TestInspectorGetTaskStats(t *testing.T) { + mgr := newInspector(t) + mgr.GetTaskStats() +}