From 19e3ec09e4da7d662438abef6c1932f4a908c979 Mon Sep 17 00:00:00 2001 From: JasonHu520 Date: Wed, 24 Jul 2024 19:55:31 +0800 Subject: [PATCH] feat(scheduler): add shard node task 1. Update the scheduler and worker interaction protocols to support new task types with: #22417838 Signed-off-by: JasonHu520 --- blobstore/api/scheduler/client.go | 10 +- blobstore/api/scheduler/task.go | 94 ++++++- blobstore/blobnode/client/blobnode_cli.go | 55 ++++ blobstore/blobnode/task_runner.go | 59 ++-- blobstore/blobnode/task_runner_mgr.go | 41 ++- blobstore/blobnode/task_runner_migrate.go | 12 +- blobstore/blobnode/task_runner_test.go | 12 +- blobstore/blobnode/task_shardnode_migrate.go | 144 ++++++++++ blobstore/blobnode/worker_service.go | 63 ++++- blobstore/blobnode/worker_service_test.go | 10 +- blobstore/common/proto/const.go | 1 + blobstore/common/proto/scheduler.go | 219 ++++++++++++++- blobstore/common/proto/scheduler_test.go | 17 ++ blobstore/scheduler/balancer_test.go | 16 +- blobstore/scheduler/base/queue.go | 169 +++++++++++- blobstore/scheduler/base/queue_test.go | 16 +- blobstore/scheduler/client/clustermgr.go | 90 +++++-- blobstore/scheduler/client/clustermgr_test.go | 16 +- blobstore/scheduler/client_mock_test.go | 20 +- blobstore/scheduler/disk_droper.go | 7 +- blobstore/scheduler/disk_droper_test.go | 30 ++- blobstore/scheduler/disk_repairer.go | 171 ++++++++---- blobstore/scheduler/disk_repairer_test.go | 132 +++++---- blobstore/scheduler/manual_migrater_test.go | 14 +- blobstore/scheduler/migrate.go | 254 +++++++++++++----- blobstore/scheduler/migrate_test.go | 92 ++++--- blobstore/scheduler/scheduler_mock_test.go | 34 ++- blobstore/scheduler/service.go | 67 ++--- blobstore/scheduler/service_test.go | 64 ++--- blobstore/scheduler/shard_migrate.go | 223 +++++++++++++++ blobstore/scheduler/startup.go | 2 +- blobstore/scheduler/startup_test.go | 8 +- blobstore/testing/mocks/api_scheduler.go | 12 +- 33 files changed, 1734 insertions(+), 440 deletions(-) create mode 100644 blobstore/blobnode/task_shardnode_migrate.go create mode 100644 blobstore/scheduler/shard_migrate.go diff --git a/blobstore/api/scheduler/client.go b/blobstore/api/scheduler/client.go index 63a77f2e9..efc8efc0f 100644 --- a/blobstore/api/scheduler/client.go +++ b/blobstore/api/scheduler/client.go @@ -51,12 +51,12 @@ var errNoServiceAvailable = errors.New("no service available") // IMigrator scheduler migrate task. type IMigrator interface { - AcquireTask(ctx context.Context, args *AcquireArgs) (ret *proto.MigrateTask, err error) + AcquireTask(ctx context.Context, args *AcquireArgs) (ret *proto.Task, err error) RenewalTask(ctx context.Context, args *TaskRenewalArgs) (ret *TaskRenewalRet, err error) - ReportTask(ctx context.Context, args *TaskReportArgs) (err error) - ReclaimTask(ctx context.Context, args *OperateTaskArgs) (err error) - CancelTask(ctx context.Context, args *OperateTaskArgs) (err error) - CompleteTask(ctx context.Context, args *OperateTaskArgs) (err error) + ReportTask(ctx context.Context, args *TaskArgs) (err error) + ReclaimTask(ctx context.Context, args *TaskArgs) (err error) + CancelTask(ctx context.Context, args *TaskArgs) (err error) + CompleteTask(ctx context.Context, args *TaskArgs) (err error) } // IInspector volume inspect task. diff --git a/blobstore/api/scheduler/task.go b/blobstore/api/scheduler/task.go index 5ed01c123..c1efb0f50 100644 --- a/blobstore/api/scheduler/task.go +++ b/blobstore/api/scheduler/task.go @@ -16,6 +16,7 @@ package scheduler import ( "context" + "encoding/json" "fmt" "net/url" @@ -27,9 +28,22 @@ type AcquireArgs struct { IDC string `json:"idc"` } -func (c *client) AcquireTask(ctx context.Context, args *AcquireArgs) (ret *proto.MigrateTask, err error) { +type TaskArgs struct { + ModuleType proto.ModuleType `json:"module_type"` + TaskType proto.TaskType `json:"task_type"` + Data []byte `json:"data"` +} + +// TaskRet use for query task info +type TaskRet struct { + TaskType proto.TaskType `json:"task_type"` + Data []byte `json:"data"` +} + +func (c *client) AcquireTask(ctx context.Context, args *AcquireArgs) (ret *proto.Task, err error) { + ret = new(proto.Task) err = c.request(func(host string) error { - return c.GetWith(ctx, host+PathTaskAcquire+"?idc="+args.IDC, &ret) + return c.GetWith(ctx, host+PathTaskAcquire+"?idc="+args.IDC, ret) }) return } @@ -59,13 +73,33 @@ type TaskReportArgs struct { IncreaseShardCnt int `json:"increase_shard_cnt"` } -func (c *client) ReportTask(ctx context.Context, args *TaskReportArgs) (err error) { +func (t *TaskReportArgs) Unmarshal(data []byte) error { + return json.Unmarshal(data, t) +} + +func (t *TaskReportArgs) Marshal() (data []byte, err error) { + return json.Marshal(t) +} + +func (t *TaskReportArgs) TaskArgs() (*TaskArgs, error) { + ret := new(TaskArgs) + ret.ModuleType = proto.TypeBlobNode + ret.TaskType = t.TaskType + data, err := t.Marshal() + if err != nil { + return nil, err + } + ret.Data = data + return ret, nil +} + +func (c *client) ReportTask(ctx context.Context, args *TaskArgs) (err error) { return c.request(func(host string) error { return c.PostWith(ctx, host+PathTaskReport, nil, args) }) } -// OperateTaskArgs for task action. +// OperateTaskArgs for blobnode task action. type OperateTaskArgs struct { IDC string `json:"idc"` TaskID string `json:"task_id"` @@ -75,19 +109,39 @@ type OperateTaskArgs struct { Reason string `json:"reason"` } -func (c *client) ReclaimTask(ctx context.Context, args *OperateTaskArgs) (err error) { +func (t *OperateTaskArgs) Unmarshal(data []byte) error { + return json.Unmarshal(data, t) +} + +func (t *OperateTaskArgs) Marshal() (data []byte, err error) { + return json.Marshal(t) +} + +func (t *OperateTaskArgs) TaskArgs() (*TaskArgs, error) { + ret := new(TaskArgs) + ret.ModuleType = proto.TypeBlobNode + ret.TaskType = t.TaskType + data, err := t.Marshal() + if err != nil { + return nil, err + } + ret.Data = data + return ret, nil +} + +func (c *client) ReclaimTask(ctx context.Context, args *TaskArgs) (err error) { return c.request(func(host string) error { return c.PostWith(ctx, host+PathTaskReclaim, nil, args) }) } -func (c *client) CancelTask(ctx context.Context, args *OperateTaskArgs) (err error) { +func (c *client) CancelTask(ctx context.Context, args *TaskArgs) (err error) { return c.request(func(host string) error { return c.PostWith(ctx, host+PathTaskCancel, nil, args) }) } -func (c *client) CompleteTask(ctx context.Context, args *OperateTaskArgs) (err error) { +func (c *client) CompleteTask(ctx context.Context, args *TaskArgs) (err error) { return c.request(func(host string) error { return c.PostWith(ctx, host+PathTaskComplete, nil, args) }) @@ -154,6 +208,12 @@ type MigrateTasksStat struct { StatsPerMin PerMinStats `json:"stats_per_min"` } +type ShardTaskStat struct { + PreparingCnt int `json:"preparing_cnt"` + WorkerDoingCnt int `json:"worker_doing_cnt"` + FinishingCnt int `json:"finishing_cnt"` +} + type DiskDropTasksStat struct { Enable bool `json:"enable"` DroppingDisks []proto.DiskID `json:"dropping_disks"` @@ -272,3 +332,23 @@ func hostWithScheme(host string) string { } return "http://" + host } + +// ShardTaskArgs for shard node task action. +type ShardTaskArgs struct { + IDC string `json:"idc"` + TaskID string `json:"task_id"` + TaskType proto.TaskType `json:"task_type"` + Source proto.SunitLocation `json:"source"` + Dest proto.SunitLocation `json:"dest"` + Leader proto.SunitLocation `json:"leader"` + Learner bool `json:"learner"` + Reason string `json:"reason"` +} + +func (t *ShardTaskArgs) Unmarshal(data []byte) error { + return json.Unmarshal(data, t) +} + +func (t *ShardTaskArgs) Marshal() (data []byte, err error) { + return json.Marshal(t) +} diff --git a/blobstore/blobnode/client/blobnode_cli.go b/blobstore/blobnode/client/blobnode_cli.go index d6fa7b466..b739023ed 100644 --- a/blobstore/blobnode/client/blobnode_cli.go +++ b/blobstore/blobnode/client/blobnode_cli.go @@ -16,6 +16,7 @@ package client import ( "context" + "github.com/cubefs/cubefs/blobstore/common/sharding" "io" "strings" @@ -38,6 +39,60 @@ type IBlobNode interface { PutShard(ctx context.Context, location proto.VunitLocation, bid proto.BlobID, size int64, body io.Reader, ioType api.IOType) (err error) } +type ShardStatusRet struct { +} + +type ShardUnitInfo struct { + Suid proto.Suid `json:"suid"` + DiskID proto.DiskID `json:"disk_id"` + Learner bool `json:"learner"` +} + +type UpdateShardArgs struct { + ShardID proto.ShardID `json:"shard_id"` + DiskID proto.DiskID `json:"disk_id"` + Units []*ShardUnitInfo `json:"units"` +} + +type SubRange struct { + Min int64 `json:"min"` + Max int64 `json:"max"` +} + +type Range struct { + RangeType sharding.RangeType `json:"range_type"` + SubRanges []SubRange `json:"sub_ranges"` +} + +type AddShardArgs struct { + DiskID proto.DiskID `json:"disk_id"` + ShardID proto.ShardID `json:"shard_id"` + Range Range + Units []*ShardUnitInfo + Epoch uint64 +} + +// IShardNode ShardNode client +type IShardNode interface { + UpdateShard(ctx context.Context, args *UpdateShardArgs) error + GetShardStatus(ctx context.Context, id proto.ShardID) (error, *ShardStatusRet) + AdShard(ctx context.Context, args *AddShardArgs) error +} + +type ShardNodeClient struct { +} + +func (c *ShardNodeClient) UpdateShard(ctx context.Context, args *UpdateShardArgs) error { + return nil +} + +func (c *ShardNodeClient) GetShardStatus(ctx context.Context, id proto.ShardID) (error, *ShardStatusRet) { + return nil, nil +} +func (c *ShardNodeClient) AdShard(ctx context.Context, args *AddShardArgs) error { + return nil +} + // BlobNodeClient blobnode client type BlobNodeClient struct { cli api.StorageAPI diff --git a/blobstore/blobnode/task_runner.go b/blobstore/blobnode/task_runner.go index 0df6249f9..b731fade0 100644 --- a/blobstore/blobnode/task_runner.go +++ b/blobstore/blobnode/task_runner.go @@ -16,6 +16,7 @@ package blobnode import ( "context" + "encoding/json" "errors" "fmt" "sync" @@ -39,13 +40,13 @@ const ( var errKilled = errors.New("task killed") -// taskState task state -type taskState struct { +// TaskState task state +type TaskState struct { sync.Mutex state uint8 } -func (ts *taskState) set(st uint8) { +func (ts *TaskState) set(st uint8) { ts.Lock() defer ts.Unlock() if ts.state >= TaskSuccess { @@ -54,13 +55,13 @@ func (ts *taskState) set(st uint8) { ts.state = st } -func (ts *taskState) stopped() bool { +func (ts *TaskState) stopped() bool { ts.Lock() defer ts.Unlock() return ts.state >= TaskSuccess } -func (ts *taskState) alive() bool { +func (ts *TaskState) alive() bool { ts.Lock() defer ts.Unlock() return ts.state <= TaskRunning @@ -140,6 +141,15 @@ func genWorkError(err error, errType WokeErrorType) *WorkError { return &WorkError{errType: errType, err: err} } +type Runner interface { + Run() + Stop() + Stopped() bool + Alive() bool + TaskID() string + State() *TaskState +} + // ITaskWorker define interface used for task execution type ITaskWorker interface { // split tasklets accord by volume benchmark bids @@ -148,7 +158,7 @@ type ITaskWorker interface { ExecTasklet(ctx context.Context, t Tasklet) *WorkError // check whether the task is executed successfully when volume task finish Check(ctx context.Context) *WorkError - OperateArgs() scheduler.OperateTaskArgs + OperateArgs(reason string) *scheduler.TaskArgs TaskType() (taskType proto.TaskType) GetBenchmarkBids() []*ShardInfoSimple } @@ -174,7 +184,7 @@ type TaskRunner struct { idc string taskletRunConcurrency int - state taskState + state TaskState ctx context.Context cancel context.CancelFunc @@ -190,8 +200,7 @@ type TaskRunner struct { // NewTaskRunner return task runner func NewTaskRunner(ctx context.Context, taskID string, w ITaskWorker, idc string, - taskletRunConcurrency int, taskCounter *taskCounter, schedulerCli scheduler.IMigrator, -) *TaskRunner { + taskletRunConcurrency int, taskCounter *taskCounter, schedulerCli scheduler.IMigrator) Runner { span, ctx := trace.StartSpanFromContext(ctx, "taskRunner") ctx, cancel := context.WithCancel(ctx) @@ -289,6 +298,14 @@ func (r *TaskRunner) execTaskletWrap(ctx context.Context, t Tasklet) { } } +func (r *TaskRunner) TaskID() string { + return r.taskID +} + +func (r *TaskRunner) State() *TaskState { + return &r.state +} + // Stop stops task func (r *TaskRunner) Stop() { r.state.set(TaskStopping) @@ -314,11 +331,9 @@ func (r *TaskRunner) cancelOrReclaim(retErr *WorkError) { defer r.state.set(TaskStopped) if ShouldReclaim(retErr) { - args := r.w.OperateArgs() - args.IDC = r.idc - args.Reason = retErr.Error() + args := r.w.OperateArgs(retErr.Error()) span.Infof("reclaim task: taskID[%s], err[%s]", r.taskID, retErr.String()) - if err := r.schedulerCli.ReclaimTask(r.newCtx(), &args); err != nil { + if err := r.schedulerCli.ReclaimTask(r.newCtx(), args); err != nil { span.Errorf("reclaim task failed: taskID[%s], args[%+v], code[%d], err[%+v]", r.taskID, args, rpc.DetectStatusCode(err), err) } @@ -328,10 +343,8 @@ func (r *TaskRunner) cancelOrReclaim(retErr *WorkError) { span.Infof("cancel task: taskID[%s], err[%+v]", r.taskID, retErr) - args := r.w.OperateArgs() - args.IDC = r.idc - args.Reason = retErr.Error() - if err := r.schedulerCli.CancelTask(r.newCtx(), &args); err != nil { + args := r.w.OperateArgs(retErr.Error()) + if err := r.schedulerCli.CancelTask(r.newCtx(), args); err != nil { span.Errorf("cancel failed: taskID[%s], args[%+v], code[%d], err[%+v]", r.taskID, args, rpc.DetectStatusCode(err), err) } @@ -342,9 +355,8 @@ func (r *TaskRunner) completeTask() { defer r.state.set(TaskSuccess) r.span.Infof("complete task: taskID[%s]", r.taskID) - args := r.w.OperateArgs() - args.IDC = r.idc - if err := r.schedulerCli.CompleteTask(r.newCtx(), &args); err != nil { + args := r.w.OperateArgs("") + if err := r.schedulerCli.CompleteTask(r.newCtx(), args); err != nil { r.span.Errorf("complete failed: taskID[%s], args[%+v], code[%d], err[%+v]", r.taskID, args, rpc.DetectStatusCode(err), err) } @@ -360,7 +372,12 @@ func (r *TaskRunner) statsAndReportTask(increaseDataSize, increaseShardCnt uint6 IncreaseDataSizeByte: int(increaseDataSize), IncreaseShardCnt: int(increaseShardCnt), } - err := r.schedulerCli.ReportTask(r.newCtx(), &reportArgs) + data, _ := json.Marshal(reportArgs) + + err := r.schedulerCli.ReportTask(r.newCtx(), &scheduler.TaskArgs{ + TaskType: reportArgs.TaskType, + ModuleType: proto.TypeBlobNode, + Data: data}) if err != nil { r.span.Errorf("report task failed: taskID[%s], code[%d], err[%+v]", r.taskID, rpc.DetectStatusCode(err), err) } diff --git a/blobstore/blobnode/task_runner_mgr.go b/blobstore/blobnode/task_runner_mgr.go index 46287a355..0ee8b9760 100644 --- a/blobstore/blobnode/task_runner_mgr.go +++ b/blobstore/blobnode/task_runner_mgr.go @@ -24,6 +24,7 @@ import ( "github.com/cubefs/cubefs/blobstore/api/blobnode" "github.com/cubefs/cubefs/blobstore/api/scheduler" "github.com/cubefs/cubefs/blobstore/blobnode/base" + "github.com/cubefs/cubefs/blobstore/blobnode/client" "github.com/cubefs/cubefs/blobstore/common/counter" "github.com/cubefs/cubefs/blobstore/common/proto" "github.com/cubefs/cubefs/blobstore/common/trace" @@ -59,10 +60,11 @@ func NewTaskRunnerMgr(idc string, meter WorkerConfigMeter, genWorker WorkerGener ) *TaskRunnerMgr { return &TaskRunnerMgr{ typeMgr: map[proto.TaskType]mapTaskRunner{ - proto.TaskTypeBalance: make(mapTaskRunner), - proto.TaskTypeDiskDrop: make(mapTaskRunner), - proto.TaskTypeDiskRepair: make(mapTaskRunner), - proto.TaskTypeManualMigrate: make(mapTaskRunner), + proto.TaskTypeBalance: make(mapTaskRunner), + proto.TaskTypeDiskDrop: make(mapTaskRunner), + proto.TaskTypeDiskRepair: make(mapTaskRunner), + proto.TaskTypeManualMigrate: make(mapTaskRunner), + proto.TaskTypeShardDiskRepair: make(mapTaskRunner), }, idc: idc, @@ -122,7 +124,7 @@ func (tm *TaskRunnerMgr) renewalTask() { } } -// AddTask add migrate task. +// AddTask add blobBode migrate task. func (tm *TaskRunnerMgr) AddTask(ctx context.Context, task MigrateTaskEx) error { tm.mu.Lock() defer tm.mu.Unlock() @@ -148,6 +150,25 @@ func (tm *TaskRunnerMgr) AddTask(ctx context.Context, task MigrateTaskEx) error return nil } +// AddShardTask add shardNode migrate task. +func (tm *TaskRunnerMgr) AddShardTask(ctx context.Context, shardClient client.IShardNode, task *proto.ShardMigrateTask) error { + tm.mu.Lock() + defer tm.mu.Unlock() + + mgr, ok := tm.typeMgr[task.TaskType] + if !ok { + return fmt.Errorf("invalid task type: %s", task.TaskType) + } + w := &ShardWorker{shardNodeCli: shardClient, t: task} + runner := NewShardNodeTaskRunner(ctx, task.TaskID, w, task.SourceIDC, &tm.taskCounter, tm.schedulerCli) + if err := mgr.addTask(task.TaskID, runner); err != nil { + return err + } + + go runner.Run() + return nil +} + // GetAliveTasks returns all alive migrate task. func (tm *TaskRunnerMgr) GetAliveTasks() map[proto.TaskType][]string { tm.mu.Lock() @@ -194,22 +215,22 @@ func (tm *TaskRunnerMgr) TaskStats() blobnode.WorkerStats { } } -type mapTaskRunner map[string]*TaskRunner +type mapTaskRunner map[string]Runner func (m mapTaskRunner) eliminateStopped() mapTaskRunner { newTasks := make(mapTaskRunner, len(m)) for taskID, task := range m { if task.Stopped() { - log.Infof("remove stopped task: taskID[%s], state[%d]", task.taskID, task.state.state) + log.Infof("remove stopped task: taskID[%s], state[%d]", task.TaskID(), task.State().state) continue } - log.Debugf("remain task: taskID[%s], state[%d]", task.taskID, task.state.state) + log.Debugf("remain task: taskID[%s], state[%d]", task.TaskID(), task.State().state) newTasks[taskID] = task } return newTasks } -func (m mapTaskRunner) addTask(taskID string, runner *TaskRunner) error { +func (m mapTaskRunner) addTask(taskID string, runner Runner) error { if r, ok := m[taskID]; ok { if !r.Stopped() { log.Warnf("task is running shouldn't add again: taskID[%s]", taskID) @@ -232,7 +253,7 @@ func (m mapTaskRunner) getAliveTasks() []string { alive := make([]string, 0, 16) for _, r := range m { if r.Alive() { - alive = append(alive, r.taskID) + alive = append(alive, r.TaskID()) } } return alive diff --git a/blobstore/blobnode/task_runner_migrate.go b/blobstore/blobnode/task_runner_migrate.go index 8974082ea..2cb4eef77 100644 --- a/blobstore/blobnode/task_runner_migrate.go +++ b/blobstore/blobnode/task_runner_migrate.go @@ -129,13 +129,21 @@ func (w *MigrateWorker) GetBenchmarkBids() []*ShardInfoSimple { } // OperateArgs args for cancel, complete, reclaim. -func (w *MigrateWorker) OperateArgs() scheduler.OperateTaskArgs { - return scheduler.OperateTaskArgs{ +func (w *MigrateWorker) OperateArgs(reason string) *scheduler.TaskArgs { + operateTaskArgs := &scheduler.OperateTaskArgs{ TaskID: w.t.TaskID, TaskType: w.t.TaskType, Src: w.t.Sources, Dest: w.t.Destination, + Reason: reason, + IDC: w.t.SourceIDC, } + data, _ := operateTaskArgs.Marshal() + ret := new(scheduler.TaskArgs) + ret.Data = data + ret.ModuleType = proto.TypeBlobNode + ret.TaskType = w.t.TaskType + return ret } // TaskType returns task type diff --git a/blobstore/blobnode/task_runner_test.go b/blobstore/blobnode/task_runner_test.go index 916ff4ad1..b1dc71ff5 100644 --- a/blobstore/blobnode/task_runner_test.go +++ b/blobstore/blobnode/task_runner_test.go @@ -70,8 +70,8 @@ func (w *mockWorker) Check(ctx context.Context) *WorkError { return OtherError(w.checkRetErr) } -func (w *mockWorker) OperateArgs() scheduler.OperateTaskArgs { - return scheduler.OperateTaskArgs{TaskID: "test_mock_task", TaskType: w.TaskType()} +func (w *mockWorker) OperateArgs(reason string) *scheduler.TaskArgs { + return &scheduler.TaskArgs{ModuleType: proto.TypeShardNode, TaskType: w.TaskType(), Data: nil} } func (w *mockWorker) TaskType() proto.TaskType { @@ -95,19 +95,19 @@ func newMockSchedulerCli(t *testing.T, stats *mockStats) scheduler.IMigrator { cli := mocks.NewMockIScheduler(C(t)) cli.EXPECT().ReportTask(A, A).AnyTimes().Return(nil) cli.EXPECT().CancelTask(A, A).AnyTimes().DoAndReturn( - func(context.Context, *scheduler.OperateTaskArgs) error { + func(context.Context, *scheduler.TaskArgs) error { stats.step = "Cancel" stats.wg.Done() return errors.New("nothing") }) cli.EXPECT().CompleteTask(A, A).AnyTimes().DoAndReturn( - func(context.Context, *scheduler.OperateTaskArgs) error { + func(context.Context, *scheduler.TaskArgs) error { stats.step = "Complete" stats.wg.Done() return errors.New("nothing") }) cli.EXPECT().ReclaimTask(A, A).AnyTimes().DoAndReturn( - func(context.Context, *scheduler.OperateTaskArgs) error { + func(context.Context, *scheduler.TaskArgs) error { stats.step = "Reclaim" stats.wg.Done() return errors.New("nothing") @@ -197,7 +197,7 @@ func TestTaskRunner(t *testing.T) { } func TestTaskState(t *testing.T) { - s := taskState{} + s := TaskState{} s.set(TaskRunning) require.Equal(t, TaskRunning, s.state) s.set(TaskRunning) diff --git a/blobstore/blobnode/task_shardnode_migrate.go b/blobstore/blobnode/task_shardnode_migrate.go new file mode 100644 index 000000000..959221445 --- /dev/null +++ b/blobstore/blobnode/task_shardnode_migrate.go @@ -0,0 +1,144 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package blobnode + +import ( + "context" + "sync" + + "github.com/cubefs/cubefs/blobstore/api/scheduler" + "github.com/cubefs/cubefs/blobstore/blobnode/client" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/trace" +) + +// ShardWorker used to manager shard migrate task +type ShardWorker struct { + t *proto.ShardMigrateTask + shardNodeCli client.IShardNode +} + +func (s *ShardWorker) Do(ctx context.Context) { + +} + +func (s *ShardWorker) OperateArgs(reason string) *scheduler.TaskArgs { + shardTaskArgs := &scheduler.ShardTaskArgs{ + TaskType: s.t.TaskType, + IDC: s.t.SourceIDC, + TaskID: s.t.TaskID, + Source: s.t.Source, + Dest: s.t.Destination, + Leader: s.t.Leader, + Learner: s.t.Learner, + Reason: reason, + } + data, _ := shardTaskArgs.Marshal() + ret := new(scheduler.TaskArgs) + ret.Data = data + ret.TaskType = shardTaskArgs.TaskType + ret.ModuleType = proto.TypeShardNode + return ret +} + +// IShardWorker define for shard node task executor +type IShardWorker interface { + Do(ctx context.Context) + OperateArgs(reason string) *scheduler.TaskArgs +} + +// ShardNodeTaskRunner used to manage task +type ShardNodeTaskRunner struct { + taskID string + w IShardWorker + idc string + state *TaskState + + ctx context.Context + cancel context.CancelFunc + span trace.Span + + stopMu sync.Mutex + stopReason *WorkError + + schedulerCli scheduler.IMigrator + stats proto.TaskProgress // task progress statics + taskCounter *taskCounter +} + +// NewShardNodeTaskRunner return task runner +func NewShardNodeTaskRunner(ctx context.Context, taskID string, w IShardWorker, idc string, + taskCounter *taskCounter, schedulerCli scheduler.IMigrator) Runner { + span, ctx := trace.StartSpanFromContext(ctx, "taskRunner") + ctx, cancel := context.WithCancel(ctx) + + task := ShardNodeTaskRunner{ + taskID: taskID, + w: w, + idc: idc, + ctx: ctx, + cancel: cancel, + span: span, + schedulerCli: schedulerCli, + stats: proto.NewTaskProgress(), + taskCounter: taskCounter, + } + task.state.set(TaskInit) + return &task +} + +func (s *ShardNodeTaskRunner) Run() { + span := s.span + span.Infof("start run task: taskID[%s]", s.taskID) + + s.state.set(TaskRunning) + +} + +func (s *ShardNodeTaskRunner) Stop() { + s.state.set(TaskStopping) + s.stopWithFail(OtherError(errKilled)) +} +func (s *ShardNodeTaskRunner) Stopped() bool { + return s.state.stopped() +} +func (s *ShardNodeTaskRunner) Alive() bool { + return s.state.alive() +} +func (s *ShardNodeTaskRunner) TaskID() string { + return s.taskID +} +func (s *ShardNodeTaskRunner) State() *TaskState { + return s.state +} + +func (s *ShardNodeTaskRunner) stopWithFail(fail *WorkError) { + s.span.Infof("stop task: taskID[%s], err_type[%d], err[%+v]", s.taskID, fail.errType, fail.err) + s.stopMu.Lock() + if s.stopReason == nil { + s.stopReason = fail + } + s.stopMu.Unlock() + s.cancel() +} + +func (s *ShardNodeTaskRunner) cancelOrReclaim(retErr *WorkError) { + defer s.state.set(TaskStopped) + +} + +func (s *ShardNodeTaskRunner) complete() { + defer s.state.set(TaskSuccess) +} diff --git a/blobstore/blobnode/worker_service.go b/blobstore/blobnode/worker_service.go index 1eb9638d0..47a1cd06e 100644 --- a/blobstore/blobnode/worker_service.go +++ b/blobstore/blobnode/worker_service.go @@ -101,6 +101,7 @@ type WorkerService struct { schedulerCli scheduler.IScheduler blobNodeCli client.IBlobNode + shardNodeCli client.IShardNode } func (cfg *WorkerConfig) checkAndFix() { @@ -257,21 +258,14 @@ func (s *WorkerService) acquireTask() { } return } - - if !t.IsValid() { - span.Errorf("task is illegal: task type[%s], task[%+v]", t.TaskType, t) - return + switch t.ModuleType { + case proto.TypeBlobNode: + s.addBlobNodeTask(ctx, t) + case proto.TypeShardNode: + s.addShardNodeTask(ctx, t) + default: + span.Errorf(" task not support module[%d]", t.ModuleType) } - err = s.taskRunnerMgr.AddTask(ctx, MigrateTaskEx{ - taskInfo: t, - downloadShardConcurrency: s.DownloadShardConcurrency, - blobNodeCli: s.blobNodeCli, - }) - if err != nil { - span.Errorf("add task failed: taskID[%s], err[%v]", t.TaskID, err) - return - } - span.Infof("acquire task success: task_type[%s], taskID[%s]", t.TaskType, t.TaskID) } // acquire inspect task @@ -300,3 +294,44 @@ func (s *WorkerService) acquireInspectTask() { span.Infof("acquire inspect task success: taskID[%s] task[%+v]", t.TaskID, t) } + +func (s *WorkerService) addBlobNodeTask(ctx context.Context, t *proto.Task) { + span := trace.SpanFromContextSafe(ctx) + task := &proto.MigrateTask{} + if err := task.Unmarshal(t.Data); err != nil { + span.Errorf("task decode failed: task type[%s], task[%+v]", t.TaskType, t) + return + } + + if !task.IsValid() { + span.Errorf("task is illegal: task type[%s], task[%+v]", task.TaskType, task) + return + } + if err := s.taskRunnerMgr.AddTask(ctx, MigrateTaskEx{ + taskInfo: task, + downloadShardConcurrency: s.DownloadShardConcurrency, + blobNodeCli: s.blobNodeCli, + }); err != nil { + span.Errorf("add task failed: taskID[%s], err[%v]", task.TaskID, err) + return + } + span.Infof("acquire task success: task_type[%s], taskID[%s]", task.TaskType, task.TaskID) +} + +func (s *WorkerService) addShardNodeTask(ctx context.Context, t *proto.Task) { + span := trace.SpanFromContextSafe(ctx) + task := &proto.ShardMigrateTask{} + if err := task.Unmarshal(t.Data); err != nil { + span.Errorf("task decode failed: task type[%s], task[%+v]", t.TaskType, t) + return + } + if !task.IsValid() { + span.Errorf("task is illegal: task type[%s], task[%+v]", task.TaskType, task) + return + } + if err := s.taskRunnerMgr.AddShardTask(ctx, s.shardNodeCli, task); err != nil { + span.Errorf("add task failed: taskID[%s], err[%v]", task.TaskID, err) + return + } + span.Infof("acquire task success: task_type[%s], taskID[%s]", task.TaskType, task.TaskID) +} diff --git a/blobstore/blobnode/worker_service_test.go b/blobstore/blobnode/worker_service_test.go index 62fccfeb0..85798628f 100644 --- a/blobstore/blobnode/worker_service_test.go +++ b/blobstore/blobnode/worker_service_test.go @@ -16,6 +16,7 @@ package blobnode import ( "context" + "encoding/json" "fmt" "io" "net/http/httptest" @@ -76,7 +77,7 @@ type mockScheCli struct { inspectCnt int } -func (m *mockScheCli) AcquireTask(ctx context.Context, args *scheduler.AcquireArgs) (ret *proto.MigrateTask, err error) { +func (m *mockScheCli) AcquireTask(ctx context.Context, args *scheduler.AcquireArgs) (ret *proto.Task, err error) { m.migrateID++ mode := codemode.EC6P10L2 srcReplicas := genMockVol(1, mode) @@ -108,7 +109,12 @@ func (m *mockScheCli) AcquireTask(ctx context.Context, args *scheduler.AcquireAr default: // do nothing } - return &task, nil + data, err := json.Marshal(task) + if err != nil { + return nil, err + } + ret = &proto.Task{ModuleType: proto.TypeBlobNode, TaskType: task.TaskType, Data: data} + return } func (m *mockScheCli) AcquireInspectTask(ctx context.Context) (ret *proto.VolumeInspectTask, err error) { diff --git a/blobstore/common/proto/const.go b/blobstore/common/proto/const.go index 51314659e..53efc356c 100644 --- a/blobstore/common/proto/const.go +++ b/blobstore/common/proto/const.go @@ -182,6 +182,7 @@ const ( InvalidCrc32 = uint32(0) InvalidVid = Vid(0) InvalidVuid = Vuid(0) + InvalidSuid = Suid(0) InvalidSpaceID = SpaceID(0) ) diff --git a/blobstore/common/proto/scheduler.go b/blobstore/common/proto/scheduler.go index 2abe4953e..5c561d158 100644 --- a/blobstore/common/proto/scheduler.go +++ b/blobstore/common/proto/scheduler.go @@ -15,9 +15,14 @@ package proto import ( + "bytes" + "encoding/binary" + "encoding/json" + "io" "sync" "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/cubefs/cubefs/blobstore/common/rpc" "github.com/cubefs/cubefs/blobstore/util/errors" ) @@ -43,12 +48,18 @@ const ( TaskTypeVolumeInspect TaskType = "volume_inspect" TaskTypeShardRepair TaskType = "shard_repair" TaskTypeBlobDelete TaskType = "blob_delete" + + TaskTypeShardInspect TaskType = "shard_inspect" + TaskTypeShardDiskRepair TaskType = "shard_disk_repair" + TaskTypeShardMigrate TaskType = "shard_migrate" + TaskTypeShardDiskDrop TaskType = "shard_disk_drop" ) func (t TaskType) Valid() bool { switch t { case TaskTypeDiskRepair, TaskTypeBalance, TaskTypeDiskDrop, TaskTypeManualMigrate, - TaskTypeVolumeInspect, TaskTypeShardRepair, TaskTypeBlobDelete: + TaskTypeVolumeInspect, TaskTypeShardRepair, TaskTypeBlobDelete, + TaskTypeShardInspect, TaskTypeShardDiskRepair, TaskTypeShardMigrate, TaskTypeShardDiskDrop: return true default: return false @@ -59,13 +70,14 @@ func (t TaskType) String() string { return string(t) } +// VunitLocation volume or shard location type VunitLocation struct { Vuid Vuid `json:"vuid" bson:"vuid"` Host string `json:"host" bson:"host"` DiskID DiskID `json:"disk_id" bson:"disk_id"` } -// for task check +// CheckVunitLocations for task check func CheckVunitLocations(locations []VunitLocation) bool { if len(locations) == 0 { return false @@ -79,6 +91,22 @@ func CheckVunitLocations(locations []VunitLocation) bool { return true } +// SunitLocation shard location +type SunitLocation struct { + // todo for proto suid + Suid Suid `json:"suid"` + Host string `json:"host"` + DiskID DiskID `json:"disk_id"` +} + +// CheckSunitLocation CheckSunitLocations for shard task check +func CheckSunitLocation(location SunitLocation) bool { + if location.Suid == InvalidSuid || location.Host == "" || location.DiskID == InvalidDiskID { + return false + } + return true +} + type MigrateState uint8 const ( @@ -89,6 +117,7 @@ const ( MigrateStateFinishedInAdvance ) +// MigrateTask for blobnode task type MigrateTask struct { TaskID string `json:"task_id"` // task id TaskType TaskType `json:"task_type"` // task type @@ -113,6 +142,27 @@ type MigrateTask struct { WorkerRedoCnt uint8 `json:"worker_redo_cnt"` // worker redo task count } +func (t *MigrateTask) Unmarshal(data []byte) error { + return json.Unmarshal(data, t) +} + +func (t *MigrateTask) Marshal() (data []byte, err error) { + return json.Marshal(t) +} + +func (t *MigrateTask) Task() (*Task, error) { + ret := new(Task) + ret.ModuleType = TypeBlobNode + ret.TaskType = t.TaskType + ret.TaskID = t.TaskID + data, err := t.Marshal() + if err != nil { + return nil, err + } + ret.Data = data + return ret, nil +} + func (t *MigrateTask) Vid() Vid { return t.SourceVuid.Vid() } @@ -156,6 +206,74 @@ func (t *MigrateTask) IsValid() bool { CheckVunitLocations([]VunitLocation{t.Destination}) } +type ShardTaskState uint8 + +const ( + ShardTaskStateInited ShardTaskState = iota + 1 + ShardTaskStatePrepared + ShardTaskStateWorkCompleted + ShardTaskStateFinished + ShardTaskStateFinishedInAdvance +) + +// ShardMigrateTask for shard node task +type ShardMigrateTask struct { + TaskID string `json:"task_id"` // task id + TaskType TaskType `json:"task_type"` // task type + State ShardTaskState `json:"state"` // task state + + SourceIDC string `json:"source_idc"` // source idc + + Ctime string `json:"ctime"` // create time + MTime string `json:"mtime"` // modify time + + Source SunitLocation `json:"source"` // old shard location + Leader SunitLocation `json:"leader"` // shard leader location + Destination SunitLocation `json:"destination"` // new shard location + Learner bool `json:"learner"` +} + +func (s *ShardMigrateTask) Unmarshal(data []byte) error { + return json.Unmarshal(data, s) +} + +func (s *ShardMigrateTask) Marshal() (data []byte, err error) { + return json.Marshal(s) +} + +func (s *ShardMigrateTask) Task() (*Task, error) { + ret := new(Task) + ret.TaskID = s.TaskID + ret.ModuleType = TypeShardNode + ret.TaskType = s.TaskType + data, err := s.Marshal() + if err != nil { + return nil, err + } + ret.Data = data + return ret, err +} + +func (s *ShardMigrateTask) GetSource() SunitLocation { + return s.Source +} + +func (s *ShardMigrateTask) GetLeader() SunitLocation { + return s.Leader +} + +func (s *ShardMigrateTask) GetDestination() SunitLocation { + return s.Destination +} + +func (s *ShardMigrateTask) SetDestination(dest SunitLocation) { + s.Destination = dest +} + +func (s *ShardMigrateTask) IsValid() bool { + return CheckSunitLocation(s.Source) && CheckSunitLocation(s.Destination) +} + type VolumeInspectCheckPoint struct { StartVid Vid `json:"start_vid"` // min vid in current batch volumes Ctime string `json:"ctime"` @@ -259,3 +377,100 @@ func (p *taskProgress) Done() TaskStatistics { p.mu.Unlock() return st } + +type ModuleType uint8 + +const ( + TypeMin ModuleType = iota + TypeBlobNode + TypeShardNode + TypeMax +) + +type Task struct { + ModuleType ModuleType `json:"module_type"` + TaskType TaskType `json:"task_type"` + TaskID string `json:"task_id"` + Data []byte `json:"data"` +} + +func (mt ModuleType) IsValid() bool { + return mt > TypeMin && mt < TypeMax +} + +func (t *Task) Marshal() ([]byte, string, error) { + buffer := bytes.NewBuffer(nil) + if err := binary.Write(buffer, binary.BigEndian, t.ModuleType); err != nil { + return nil, "", err + } + taskType := []byte(t.TaskType) + numTaskType := int32(len(taskType)) + if err := binary.Write(buffer, binary.BigEndian, numTaskType); err != nil { + return nil, "", err + } + if _, err := buffer.Write(taskType); err != nil { + return nil, "", err + } + taskID := []byte(t.TaskID) + numTaskID := int32(len(taskID)) + if err := binary.Write(buffer, binary.BigEndian, numTaskID); err != nil { + return nil, "", err + } + if _, err := buffer.Write(taskID); err != nil { + return nil, "", err + } + + numData := int32(len(t.Data)) + if err := binary.Write(buffer, binary.BigEndian, numData); err != nil { + return nil, "", err + } + if _, err := buffer.Write(t.Data); err != nil { + return nil, "", err + } + return buffer.Bytes(), rpc.MIMEStream, nil +} + +func (t *Task) UnmarshalFrom(body io.Reader) (err error) { + if err = binary.Read(body, binary.BigEndian, &t.ModuleType); err != nil { + return + } + + numTaskType := int32(0) + if err = binary.Read(body, binary.BigEndian, &numTaskType); err != nil { + return + } + taskType := make([]byte, numTaskType) + if _, err = io.ReadFull(body, taskType); err != nil { + return err + } + + numTaskID := int32(0) + if err = binary.Read(body, binary.BigEndian, &numTaskID); err != nil { + return + } + taskID := make([]byte, numTaskID) + if _, err = io.ReadFull(body, taskID); err != nil { + return err + } + + numData := int32(0) + if err = binary.Read(body, binary.BigEndian, &numData); err != nil { + return + } + data := make([]byte, numData) + if _, err = io.ReadFull(body, data); err != nil { + return err + } + t.Data = data + t.TaskType = TaskType(taskType) + t.TaskID = string(taskID) + return nil +} + +func (t *Task) Unmarshal(data []byte) error { + if len(data) == 0 { + return nil + } + reader := bytes.NewReader(data) + return t.UnmarshalFrom(reader) +} diff --git a/blobstore/common/proto/scheduler_test.go b/blobstore/common/proto/scheduler_test.go index 26e6588dd..febf38f55 100644 --- a/blobstore/common/proto/scheduler_test.go +++ b/blobstore/common/proto/scheduler_test.go @@ -15,6 +15,7 @@ package proto_test import ( + "bytes" "testing" "github.com/stretchr/testify/require" @@ -107,3 +108,19 @@ func TestSchedulerTaskProgress(t *testing.T) { require.True(t, tp.Done().Progress >= 100) } } + +func TestTask(t *testing.T) { + task := &proto.Task{ + ModuleType: proto.TypeShardNode, + TaskType: proto.TaskTypeShardInspect, + Data: []byte("test_data"), + TaskID: "test_id", + } + data, _, err := task.Marshal() + require.Nil(t, err) + reader := bytes.NewReader(data) + newTask := &proto.Task{} + err = newTask.UnmarshalFrom(reader) + require.NoError(t, err) + require.EqualValues(t, task, newTask, "") +} diff --git a/blobstore/scheduler/balancer_test.go b/blobstore/scheduler/balancer_test.go index 2274c3793..26e0e4862 100644 --- a/blobstore/scheduler/balancer_test.go +++ b/blobstore/scheduler/balancer_test.go @@ -169,7 +169,7 @@ func TestBalanceAcquireTask(t *testing.T) { ctx := context.Background() idc := "z0" mgr := newBalancer(t) - mgr.IMigrator.(*MockMigrater).EXPECT().AcquireTask(any, any).Return(proto.MigrateTask{TaskType: proto.TaskTypeBalance}, nil) + mgr.IMigrator.(*MockMigrater).EXPECT().AcquireTask(any, any).Return(&proto.Task{TaskType: proto.TaskTypeBalance}, nil) _, err := mgr.AcquireTask(ctx, idc) require.NoError(t, err) } @@ -178,7 +178,7 @@ func TestBalanceCancelTask(t *testing.T) { ctx := context.Background() mgr := newBalancer(t) mgr.IMigrator.(*MockMigrater).EXPECT().CancelTask(any, any).Return(nil) - err := mgr.CancelTask(ctx, &api.OperateTaskArgs{}) + err := mgr.CancelTask(ctx, &api.TaskArgs{}) require.NoError(t, err) } @@ -186,9 +186,10 @@ func TestBalanceReclaimTask(t *testing.T) { ctx := context.Background() idc := "z0" mgr := newBalancer(t) - mgr.IMigrator.(*MockMigrater).EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil) + mgr.IMigrator.(*MockMigrater).EXPECT().ReclaimTask(any, any).Return(nil) t1 := mockGenMigrateTask(proto.TaskTypeBalance, idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap) - err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{}) + taskArgs := generateTaskArgs(t1, "") + err := mgr.ReclaimTask(ctx, taskArgs) require.NoError(t, err) } @@ -198,11 +199,12 @@ func TestBalanceCompleteTask(t *testing.T) { mgr := newBalancer(t) mgr.IMigrator.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(nil) t1 := mockGenMigrateTask(proto.TaskTypeBalance, idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap) - err := mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + taskArgs := generateTaskArgs(t1, "") + err := mgr.CompleteTask(ctx, taskArgs) require.NoError(t, err) mgr.IMigrator.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(errMock) - err = mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + err = mgr.CompleteTask(ctx, taskArgs) require.True(t, errors.Is(err, errMock)) } @@ -262,7 +264,7 @@ func TestBalanceCheckAndClearJunkTasks(t *testing.T) { mgr.IMigrator.(*MockMigrater).EXPECT().DeletedTasks().Return([]DeletedTask{ {DiskID: proto.DiskID(1), TaskID: xid.New().String(), DeletedTime: time.Now().Add(-junkMigrationTaskProtectionWindow)}, }) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetMigrateTask(any, any, any).Return(&proto.MigrateTask{}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetMigrateTask(any, any, any).Return(&proto.Task{}, nil) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().DeleteMigrateTask(any, any).Return(nil) mgr.IMigrator.(*MockMigrater).EXPECT().ClearDeletedTaskByID(any, any).Return() mgr.checkAndClearJunkTasks() diff --git a/blobstore/scheduler/base/queue.go b/blobstore/scheduler/base/queue.go index 151feb0e6..4f45f46a8 100644 --- a/blobstore/scheduler/base/queue.go +++ b/blobstore/scheduler/base/queue.go @@ -32,6 +32,7 @@ var ( ErrNoSuchMessageID = errors.New("no such message id") // ErrUnmatchedVuids unmatched task vuids ErrUnmatchedVuids = errors.New("unmatched task vuids") + ErrUnmatchedSuid = errors.New("unmatched task suid") errNoSuchIDCQueue = errors.New("no such idc queue") errExistingMessageID = errors.New("existing message id") ) @@ -197,6 +198,7 @@ type WorkerTask interface { GetSources() []proto.VunitLocation GetDestination() proto.VunitLocation SetDestination(dest proto.VunitLocation) + Task() (*proto.Task, error) } // TaskQueue task queue @@ -215,7 +217,7 @@ func NewTaskQueue(retryDelay time.Duration) *TaskQueue { } // PushTask push task to queue -func (q *TaskQueue) PushTask(taskID string, task WorkerTask) { +func (q *TaskQueue) PushTask(taskID string, task interface{}) { q.mu.Lock() defer q.mu.Unlock() err := q.queue.Push(taskID, task) @@ -225,12 +227,12 @@ func (q *TaskQueue) PushTask(taskID string, task WorkerTask) { } // PopTask return args： taskID, task, flag of task exist -func (q *TaskQueue) PopTask() (string, WorkerTask, bool) { +func (q *TaskQueue) PopTask() (string, interface{}, bool) { q.mu.Lock() defer q.mu.Unlock() taskID, task, exist := q.queue.Pop() if exist { - return taskID, task.(WorkerTask), true + return taskID, task, true } return "", nil, false } @@ -254,14 +256,14 @@ func (q *TaskQueue) RetryTask(taskID string) { } // Query find task by taskID -func (q *TaskQueue) Query(taskID string) (WorkerTask, bool) { +func (q *TaskQueue) Query(taskID string) (interface{}, bool) { q.mu.Lock() defer q.mu.Unlock() task, err := q.queue.Get(taskID) if err != nil { return nil, false } - return task.(WorkerTask), true + return task, true } // StatsTasks returns task stats @@ -271,7 +273,7 @@ func (q *TaskQueue) StatsTasks() (todo int, doing int) { return q.queue.Stats() } -// WorkerTaskQueue task queue for worker +// WorkerTaskQueue task queue for blobnode task type WorkerTaskQueue struct { mu sync.Mutex idcQueues map[string]*Queue @@ -462,3 +464,158 @@ func vunitSliceEqual(a, b []proto.VunitLocation) bool { } return true } + +// ShardTask define shard task interface +type ShardTask interface { + GetSource() proto.SunitLocation + GetLeader() proto.SunitLocation + GetDestination() proto.SunitLocation + SetDestination(dest proto.SunitLocation) + Task() (*proto.Task, error) +} + +// ShardTaskQueue task queue for shard task +type ShardTaskQueue struct { + mu sync.Mutex + idcQueues map[string]*Queue + + cancelPunishDuration time.Duration // task cancel will punish a period of time to avoid frequent failure retry + leaseExpiredS time.Duration +} + +// AddPreparedTask add prepared task +func (q *ShardTaskQueue) AddPreparedTask(idc, taskID string, wtask ShardTask) { + q.mu.Lock() + defer q.mu.Unlock() + + idcQueue, ok := q.idcQueues[idc] + if !ok { + idcQueue = NewQueue(q.leaseExpiredS) + q.idcQueues[idc] = idcQueue + } + err := idcQueue.Push(taskID, wtask) + if err != nil { + panic("unexpect add prepared task fail:" + err.Error()) + } +} + +// Acquire acquire task by idc +func (q *ShardTaskQueue) Acquire(idc string) (taskID string, wtask ShardTask, exist bool) { + q.mu.Lock() + defer q.mu.Unlock() + + idcQueue, ok := q.idcQueues[idc] + if !ok { + return "", nil, false + } + + taskID, task, exist := idcQueue.Pop() + if exist { + return taskID, task.(ShardTask), exist + } + return "", nil, false +} + +// Cancel cancel task +func (q *ShardTaskQueue) Cancel(idc, taskID string, src, dst proto.SunitLocation) error { + q.mu.Lock() + defer q.mu.Unlock() + + idcQueue, ok := q.idcQueues[idc] + if !ok { + return errNoSuchIDCQueue + } + + task, err := idcQueue.Get(taskID) + if err != nil { + return err + } + err = checkShardTaskValid(task.(ShardTask), src, dst) + if err != nil { + return err + } + + return idcQueue.Requeue(taskID, q.cancelPunishDuration) +} + +// Reclaim reclaim task +func (q *ShardTaskQueue) Reclaim(idc, taskID string, src, oldDest, newDest proto.SunitLocation, newDiskID proto.DiskID) error { + q.mu.Lock() + defer q.mu.Unlock() + + idcQueue, ok := q.idcQueues[idc] + if !ok { + return errNoSuchIDCQueue + } + + task, err := idcQueue.Get(taskID) + if err != nil { + return err + } + wtask := task.(ShardTask) + err = checkShardTaskValid(wtask, src, oldDest) + if err != nil { + return err + } + wtask.SetDestination(newDest) + return idcQueue.Requeue(taskID, 0) +} + +// Renewal renewal task +func (q *ShardTaskQueue) Renewal(idc, taskID string) error { + q.mu.Lock() + defer q.mu.Unlock() + idcQueue, ok := q.idcQueues[idc] + if !ok { + return errNoSuchIDCQueue + } + return idcQueue.Requeue(taskID, q.leaseExpiredS) +} + +// Complete task +func (q *ShardTaskQueue) Complete(idc, taskID string, src, dst proto.SunitLocation) (ShardTask, error) { + q.mu.Lock() + defer q.mu.Unlock() + + idcQueue, ok := q.idcQueues[idc] + if !ok { + return nil, errNoSuchIDCQueue + } + + task, err := idcQueue.Get(taskID) + if err != nil { + return nil, err + } + + t := task.(ShardTask) + err = checkShardTaskValid(t, src, dst) + if err != nil { + return nil, err + } + + err = idcQueue.Remove(taskID) + if err != nil { + panic(fmt.Sprintf("task %s remove form queue fail err %v", taskID, err)) + } + + return t, err +} + +// StatsTasks returns task stats +func (q *ShardTaskQueue) StatsTasks() (todo int, doing int) { + q.mu.Lock() + defer q.mu.Unlock() + for _, queue := range q.idcQueues { + todoTmp, doingTmp := queue.Stats() + todo += todoTmp + doing += doingTmp + } + return todo, doing +} + +func checkShardTaskValid(task ShardTask, src proto.SunitLocation, dst proto.SunitLocation) error { + if task.GetSource() != src || task.GetDestination() != dst { + return ErrUnmatchedVuids + } + return nil +} diff --git a/blobstore/scheduler/base/queue_test.go b/blobstore/scheduler/base/queue_test.go index cb7bc25f7..33989c055 100644 --- a/blobstore/scheduler/base/queue_test.go +++ b/blobstore/scheduler/base/queue_test.go @@ -120,6 +120,10 @@ func (t *mockWorkerTask) SetDestination(dstVuid proto.VunitLocation) { t.dst = dstVuid } +func (t *mockWorkerTask) Task() (*proto.Task, error) { + return nil, nil +} + func TestTaskQueue(t *testing.T) { // test Push taskID1 := "task_id1" @@ -133,10 +137,12 @@ func TestTaskQueue(t *testing.T) { // test PopTask id, wt, exist := q.PopTask() + task, ok := wt.(WorkerTask) + require.True(t, ok) require.Equal(t, true, exist) require.Equal(t, id, taskID1) - require.Equal(t, task1.GetSources(), wt.GetSources()) - require.Equal(t, task1.GetDestination(), wt.GetDestination()) + require.Equal(t, task1.GetSources(), task.GetSources()) + require.Equal(t, task1.GetDestination(), task.GetDestination()) _, _, exist = q.PopTask() require.Equal(t, false, exist) @@ -144,10 +150,12 @@ func TestTaskQueue(t *testing.T) { q.RetryTask(taskID1) time.Sleep(100 * time.Millisecond) id, wt, exist = q.PopTask() + task, ok = wt.(WorkerTask) + require.True(t, ok) require.Equal(t, true, exist) require.Equal(t, id, taskID1) - require.Equal(t, vunits([]proto.Vuid{1, 2, 3}), wt.GetSources()) - require.Equal(t, vunit(4), wt.GetDestination()) + require.Equal(t, vunits([]proto.Vuid{1, 2, 3}), task.GetSources()) + require.Equal(t, vunit(4), task.GetDestination()) // test Stats taskID2 := "task_id2" diff --git a/blobstore/scheduler/client/clustermgr.go b/blobstore/scheduler/client/clustermgr.go index 90c0954ef..bfc7e1087 100644 --- a/blobstore/scheduler/client/clustermgr.go +++ b/blobstore/scheduler/client/clustermgr.go @@ -49,6 +49,15 @@ type ClusterMgrVolumeAPI interface { ListVolume(ctx context.Context, marker proto.Vid, count int) (volInfo []*VolumeInfoSimple, retVid proto.Vid, err error) } +type ClusterMgrShardAPI interface { + GetShardInfo(ctx context.Context, Sid proto.Vid) (ret *ShardInfoSimple, err error) + UpdateShard(ctx context.Context, newVuid, oldVuid proto.Suid, newDiskID proto.DiskID) (err error) + AllocShardUnit(ctx context.Context, vuid proto.Vuid) (ret *AllocVunitInfo, err error) + ReleaseShardUnit(ctx context.Context, vuid proto.Vuid, diskID proto.DiskID) (err error) + ListDiskShardUnits(ctx context.Context, diskID proto.DiskID) (ret []*VunitInfoSimple, err error) + ListShard(ctx context.Context, marker proto.Vid, count int) (volInfo []*VolumeInfoSimple, retVid proto.Vid, err error) +} + type ClusterMgrDiskAPI interface { ListClusterDisks(ctx context.Context) (disks []*DiskInfoSimple, err error) ListBrokenDisks(ctx context.Context) (disks []*DiskInfoSimple, err error) @@ -60,19 +69,30 @@ type ClusterMgrDiskAPI interface { GetDiskInfo(ctx context.Context, diskID proto.DiskID) (ret *DiskInfoSimple, err error) } +type ClusterShardNodeAPI interface { + ListClusterDisks(ctx context.Context) (disks []*ShardNodeDiskInfo, err error) + ListBrokenDisks(ctx context.Context) (disks []*ShardNodeDiskInfo, err error) + ListRepairingDisks(ctx context.Context) (disks []*ShardNodeDiskInfo, err error) + ListDropDisks(ctx context.Context) (disks []*ShardNodeDiskInfo, err error) + SetDiskRepairing(ctx context.Context, diskID proto.DiskID) (err error) + SetDiskRepaired(ctx context.Context, diskID proto.DiskID) (err error) + SetDiskDropped(ctx context.Context, diskID proto.DiskID) (err error) + GetDiskInfo(ctx context.Context, diskID proto.DiskID) (ret *ShardNodeDiskInfo, err error) +} + type ClusterMgrServiceAPI interface { Register(ctx context.Context, info RegisterInfo) error GetService(ctx context.Context, name string, clusterID proto.ClusterID) (hosts []string, err error) } type ClusterMgrTaskAPI interface { - UpdateMigrateTask(ctx context.Context, value *proto.MigrateTask) (err error) - AddMigrateTask(ctx context.Context, value *proto.MigrateTask) (err error) - GetMigrateTask(ctx context.Context, taskType proto.TaskType, key string) (task *proto.MigrateTask, err error) + UpdateMigrateTask(ctx context.Context, value *proto.Task) (err error) + AddMigrateTask(ctx context.Context, value *proto.Task) (err error) + ListMigrateTasks(ctx context.Context, taskType proto.TaskType, args *cmapi.ListKvOpts) (tasks []*proto.Task, marker string, err error) + ListAllMigrateTasks(ctx context.Context, taskType proto.TaskType) (tasks []*proto.Task, err error) + ListAllMigrateTasksByDiskID(ctx context.Context, taskType proto.TaskType, diskID proto.DiskID) (tasks []*proto.Task, err error) + GetMigrateTask(ctx context.Context, taskType proto.TaskType, key string) (task *proto.Task, err error) DeleteMigrateTask(ctx context.Context, key string) (err error) - ListMigrateTasks(ctx context.Context, taskType proto.TaskType, args *cmapi.ListKvOpts) (tasks []*proto.MigrateTask, marker string, err error) - ListAllMigrateTasks(ctx context.Context, taskType proto.TaskType) (tasks []*proto.MigrateTask, err error) - ListAllMigrateTasksByDiskID(ctx context.Context, taskType proto.TaskType, diskID proto.DiskID) (tasks []*proto.MigrateTask, err error) AddMigratingDisk(ctx context.Context, value *MigratingDiskMeta) (err error) DeleteMigratingDisk(ctx context.Context, taskType proto.TaskType, diskID proto.DiskID) (err error) GetMigratingDisk(ctx context.Context, taskType proto.TaskType, diskID proto.DiskID) (meta *MigratingDiskMeta, err error) @@ -285,7 +305,7 @@ func (vunit *VunitInfoSimple) set(info *cmapi.VolumeUnitInfo, host string) { vunit.Used = info.Used } -// DiskInfoSimple disk simple info +// DiskInfoSimple disk simple info for blobnode type DiskInfoSimple struct { ClusterID proto.ClusterID `json:"cluster_id"` DiskID proto.DiskID `json:"disk_id"` @@ -299,6 +319,19 @@ type DiskInfoSimple struct { FreeChunkCnt int64 `json:"free_chunk_cnt"` } +// ShardNodeDiskInfo diskInfo for shard node +type ShardNodeDiskInfo struct { + ClusterID proto.ClusterID `json:"cluster_id"` + DiskID proto.DiskID `json:"disk_id"` + Idc string `json:"idc"` + Rack string `json:"rack"` + Host string `json:"host"` + Status proto.DiskStatus `json:"status"` + Readonly bool `json:"readonly"` + UsedShardCnt int64 `json:"used_shard_cnt"` + FreeShardCnt int64 `json:"free_shard_cnt"` +} + // IsHealth return true if disk is health func (disk *DiskInfoSimple) IsHealth() bool { return disk.Status == proto.DiskStatusNormal @@ -774,16 +807,12 @@ func (c *clustermgrClient) GetService(ctx context.Context, name string, clusterI } // AddMigrateTask adds migrate task -func (c *clustermgrClient) AddMigrateTask(ctx context.Context, value *proto.MigrateTask) (err error) { - value.Ctime = time.Now().String() - value.MTime = value.Ctime - +func (c *clustermgrClient) AddMigrateTask(ctx context.Context, value *proto.Task) (err error) { return c.setTask(ctx, value.TaskID, value) } // UpdateMigrateTask updates migrate task -func (c *clustermgrClient) UpdateMigrateTask(ctx context.Context, value *proto.MigrateTask) (err error) { - value.MTime = time.Now().String() +func (c *clustermgrClient) UpdateMigrateTask(ctx context.Context, value *proto.Task) (err error) { return c.setTask(ctx, value.TaskID, value) } @@ -796,12 +825,14 @@ func (c *clustermgrClient) setTask(ctx context.Context, key string, value interf } // GetMigrateTask returns migrate task -func (c *clustermgrClient) GetMigrateTask(ctx context.Context, taskType proto.TaskType, key string) (task *proto.MigrateTask, err error) { +func (c *clustermgrClient) GetMigrateTask(ctx context.Context, taskType proto.TaskType, key string) (task *proto.Task, err error) { + // todo compatible with old task, or delete old task val, err := c.client.GetKV(ctx, key) if err != nil { return nil, err } - err = json.Unmarshal(val.Value, &task) + task = new(proto.Task) + err = task.Unmarshal(val.Value) if err != nil { return nil, err } @@ -817,21 +848,21 @@ func (c *clustermgrClient) DeleteMigrateTask(ctx context.Context, key string) (e } // ListAllMigrateTasksByDiskID returns all migrate task with disk_id -func (c *clustermgrClient) ListAllMigrateTasksByDiskID(ctx context.Context, taskType proto.TaskType, diskID proto.DiskID) (tasks []*proto.MigrateTask, err error) { +func (c *clustermgrClient) ListAllMigrateTasksByDiskID(ctx context.Context, taskType proto.TaskType, diskID proto.DiskID) (tasks []*proto.Task, err error) { return c.listAllMigrateTasks(ctx, GenMigrateTaskPrefixByDiskID(taskType, diskID), taskType) } // ListAllMigrateTasks returns all migrate task -func (c *clustermgrClient) ListAllMigrateTasks(ctx context.Context, taskType proto.TaskType) (tasks []*proto.MigrateTask, err error) { +func (c *clustermgrClient) ListAllMigrateTasks(ctx context.Context, taskType proto.TaskType) (tasks []*proto.Task, err error) { return c.listAllMigrateTasks(ctx, GenMigrateTaskPrefix(taskType), taskType) } // ListMigrateTasks returns migrate task base on page size -func (c *clustermgrClient) ListMigrateTasks(ctx context.Context, taskType proto.TaskType, args *cmapi.ListKvOpts) (tasks []*proto.MigrateTask, marker string, err error) { +func (c *clustermgrClient) ListMigrateTasks(ctx context.Context, taskType proto.TaskType, args *cmapi.ListKvOpts) (tasks []*proto.Task, marker string, err error) { return c.listMigrateTasks(ctx, taskType, args) } -func (c *clustermgrClient) listMigrateTasks(ctx context.Context, taskType proto.TaskType, args *cmapi.ListKvOpts) (tasks []*proto.MigrateTask, marker string, err error) { +func (c *clustermgrClient) listMigrateTasks(ctx context.Context, taskType proto.TaskType, args *cmapi.ListKvOpts) (tasks []*proto.Task, marker string, err error) { span := trace.SpanFromContextSafe(ctx) ret, err := c.client.ListKV(ctx, args) if err != nil { @@ -839,7 +870,7 @@ func (c *clustermgrClient) listMigrateTasks(ctx context.Context, taskType proto. return nil, marker, err } for _, v := range ret.Kvs { - var task *proto.MigrateTask + var task *proto.Task err = json.Unmarshal(v.Value, &task) if err != nil { span.Errorf("unmarshal task failed: err[%+v]", err) @@ -855,7 +886,7 @@ func (c *clustermgrClient) listMigrateTasks(ctx context.Context, taskType proto. return } -func (c *clustermgrClient) listAllMigrateTasks(ctx context.Context, prefix string, taskType proto.TaskType) (tasks []*proto.MigrateTask, err error) { +func (c *clustermgrClient) listAllMigrateTasks(ctx context.Context, prefix string, taskType proto.TaskType) (tasks []*proto.Task, err error) { marker := defaultListTaskMarker for { args := &cmapi.ListKvOpts{ @@ -990,3 +1021,20 @@ func (c *clustermgrClient) SetConsumeOffset(taskType proto.TaskType, topic strin } return c.client.SetKV(context.Background(), genConsumerOffsetKey(taskType, topic, partition), consumeOffsetBytes) } + +// ShardInfoSimple shard info used by scheduler +type ShardInfoSimple struct { + Sid proto.ShardID `json:"sid"` + ApplyIndex uint64 `json:"apply_index"` + Leader proto.NodeID `json:"leader"` + Status proto.ShardStatus `json:"status"` + SunitLocations []proto.SunitLocation `json:"sunit_locations"` +} + +type UpdateShardArgs struct { + NewSuid proto.Suid `json:"new_suid"` + OldSuid proto.Suid `json:"old_suid"` + OldIsLearner bool `json:"old_is_learner"` + NewIsLearner bool `json:"new_is_learner"` + NewDiskID bool `json:"new_disk_id"` +} diff --git a/blobstore/scheduler/client/clustermgr_test.go b/blobstore/scheduler/client/clustermgr_test.go index 8b39d572f..b61191f54 100644 --- a/blobstore/scheduler/client/clustermgr_test.go +++ b/blobstore/scheduler/client/clustermgr_test.go @@ -307,19 +307,21 @@ func TestClustermgrClient(t *testing.T) { { // add migrate task cli.client.(*MockClusterManager).EXPECT().SetKV(any, any, any).Return(nil) - err := cli.AddMigrateTask(ctx, &proto.MigrateTask{TaskID: GenMigrateTaskID(proto.TaskTypeDiskRepair, proto.DiskID(1), proto.Vid(1))}) + task, _ := (&proto.MigrateTask{TaskID: GenMigrateTaskID(proto.TaskTypeDiskRepair, proto.DiskID(1), proto.Vid(1))}).Task() + err := cli.AddMigrateTask(ctx, task) require.NoError(t, err) } { // update migrate task cli.client.(*MockClusterManager).EXPECT().SetKV(any, any, any).Return(nil) - err := cli.UpdateMigrateTask(ctx, &proto.MigrateTask{TaskID: GenMigrateTaskID(proto.TaskTypeDiskRepair, proto.DiskID(1), proto.Vid(1))}) + task, _ := (&proto.MigrateTask{TaskID: GenMigrateTaskID(proto.TaskTypeDiskRepair, proto.DiskID(1), proto.Vid(1))}).Task() + err := cli.UpdateMigrateTask(ctx, task) require.NoError(t, err) } { // get migrate task - task1 := &proto.MigrateTask{TaskID: GenMigrateTaskID(proto.TaskTypeDiskRepair, proto.DiskID(1), proto.Vid(1))} - taskBytes, _ := json.Marshal(task1) + task1, _ := (&proto.MigrateTask{TaskID: GenMigrateTaskID(proto.TaskTypeDiskRepair, proto.DiskID(1), proto.Vid(1))}).Task() + taskBytes, _, _ := task1.Marshal() cli.client.(*MockClusterManager).EXPECT().SetKV(any, any, any).Return(nil) cli.client.(*MockClusterManager).EXPECT().GetKV(any, any).Return(cmapi.GetKvRet{Value: taskBytes}, nil) err := cli.AddMigrateTask(ctx, task1) @@ -328,12 +330,6 @@ func TestClustermgrClient(t *testing.T) { require.NoError(t, err) require.Equal(t, task1.TaskID, task2.TaskID) - // unmarshal failed - taskBytes = append(taskBytes, []byte("mock")...) - cli.client.(*MockClusterManager).EXPECT().GetKV(any, any).Return(cmapi.GetKvRet{Value: taskBytes}, nil) - _, err = cli.GetMigrateTask(ctx, task1.TaskType, task1.TaskID) - require.Error(t, err) - // clustermgr return err cli.client.(*MockClusterManager).EXPECT().GetKV(any, any).Return(cmapi.GetKvRet{}, errMock) _, err = cli.GetMigrateTask(ctx, task1.TaskType, task1.TaskID) diff --git a/blobstore/scheduler/client_mock_test.go b/blobstore/scheduler/client_mock_test.go index cf510c4c8..50943d1e0 100644 --- a/blobstore/scheduler/client_mock_test.go +++ b/blobstore/scheduler/client_mock_test.go @@ -38,7 +38,7 @@ func (m *MockClusterMgrAPI) EXPECT() *MockClusterMgrAPIMockRecorder { } // AddMigrateTask mocks base method. -func (m *MockClusterMgrAPI) AddMigrateTask(arg0 context.Context, arg1 *proto.MigrateTask) error { +func (m *MockClusterMgrAPI) AddMigrateTask(arg0 context.Context, arg1 *proto.Task) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "AddMigrateTask", arg0, arg1) ret0, _ := ret[0].(error) @@ -154,10 +154,10 @@ func (mr *MockClusterMgrAPIMockRecorder) GetDiskInfo(arg0, arg1 interface{}) *go } // GetMigrateTask mocks base method. -func (m *MockClusterMgrAPI) GetMigrateTask(arg0 context.Context, arg1 proto.TaskType, arg2 string) (*proto.MigrateTask, error) { +func (m *MockClusterMgrAPI) GetMigrateTask(arg0 context.Context, arg1 proto.TaskType, arg2 string) (*proto.Task, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetMigrateTask", arg0, arg1, arg2) - ret0, _ := ret[0].(*proto.MigrateTask) + ret0, _ := ret[0].(*proto.Task) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -229,10 +229,10 @@ func (mr *MockClusterMgrAPIMockRecorder) GetVolumeInspectCheckPoint(arg0 interfa } // ListAllMigrateTasks mocks base method. -func (m *MockClusterMgrAPI) ListAllMigrateTasks(arg0 context.Context, arg1 proto.TaskType) ([]*proto.MigrateTask, error) { +func (m *MockClusterMgrAPI) ListAllMigrateTasks(arg0 context.Context, arg1 proto.TaskType) ([]*proto.Task, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListAllMigrateTasks", arg0, arg1) - ret0, _ := ret[0].([]*proto.MigrateTask) + ret0, _ := ret[0].([]*proto.Task) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -244,10 +244,10 @@ func (mr *MockClusterMgrAPIMockRecorder) ListAllMigrateTasks(arg0, arg1 interfac } // ListAllMigrateTasksByDiskID mocks base method. -func (m *MockClusterMgrAPI) ListAllMigrateTasksByDiskID(arg0 context.Context, arg1 proto.TaskType, arg2 proto.DiskID) ([]*proto.MigrateTask, error) { +func (m *MockClusterMgrAPI) ListAllMigrateTasksByDiskID(arg0 context.Context, arg1 proto.TaskType, arg2 proto.DiskID) ([]*proto.Task, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListAllMigrateTasksByDiskID", arg0, arg1, arg2) - ret0, _ := ret[0].([]*proto.MigrateTask) + ret0, _ := ret[0].([]*proto.Task) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -319,10 +319,10 @@ func (mr *MockClusterMgrAPIMockRecorder) ListDropDisks(arg0 interface{}) *gomock } // ListMigrateTasks mocks base method. -func (m *MockClusterMgrAPI) ListMigrateTasks(arg0 context.Context, arg1 proto.TaskType, arg2 *clustermgr.ListKvOpts) ([]*proto.MigrateTask, string, error) { +func (m *MockClusterMgrAPI) ListMigrateTasks(arg0 context.Context, arg1 proto.TaskType, arg2 *clustermgr.ListKvOpts) ([]*proto.Task, string, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListMigrateTasks", arg0, arg1, arg2) - ret0, _ := ret[0].([]*proto.MigrateTask) + ret0, _ := ret[0].([]*proto.Task) ret1, _ := ret[1].(string) ret2, _ := ret[2].(error) return ret0, ret1, ret2 @@ -521,7 +521,7 @@ func (mr *MockClusterMgrAPIMockRecorder) UnlockVolume(arg0, arg1 interface{}) *g } // UpdateMigrateTask mocks base method. -func (m *MockClusterMgrAPI) UpdateMigrateTask(arg0 context.Context, arg1 *proto.MigrateTask) error { +func (m *MockClusterMgrAPI) UpdateMigrateTask(arg0 context.Context, arg1 *proto.Task) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "UpdateMigrateTask", arg0, arg1) ret0, _ := ret[0].(error) diff --git a/blobstore/scheduler/disk_droper.go b/blobstore/scheduler/disk_droper.go index cdabd41ba..19cee0278 100644 --- a/blobstore/scheduler/disk_droper.go +++ b/blobstore/scheduler/disk_droper.go @@ -204,7 +204,12 @@ func (mgr *DiskDropMgr) Load() (err error) { return err } for _, task := range tasks { - migrated, err := mgr.isMigrated(task.SourceVuid) + t := &proto.MigrateTask{} + err = t.Unmarshal(task.Data) + if err != nil { + return err + } + migrated, err := mgr.isMigrated(t.SourceVuid) if err != nil { return err } diff --git a/blobstore/scheduler/disk_droper_test.go b/blobstore/scheduler/disk_droper_test.go index d0caf4100..edb932b69 100644 --- a/blobstore/scheduler/disk_droper_test.go +++ b/blobstore/scheduler/disk_droper_test.go @@ -56,6 +56,8 @@ func newDiskDroper(t *testing.T) *DiskDropMgr { func TestDiskDropLoad(t *testing.T) { t1 := mockGenMigrateTask(proto.TaskTypeDiskDrop, "z0", 1, 110, proto.MigrateStateInited, MockMigrateVolInfoMap) + task, err := t1.Task() + require.NoError(t, err) // loopGenerateTask success volume := MockGenVolInfo(110, codemode.EC6P6, proto.VolumeStatusIdle) { @@ -68,7 +70,7 @@ func TestDiskDropLoad(t *testing.T) { { mgr := newDiskDroper(t) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDropDisks(any).Times(1).Return(nil, nil) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Times(1).Return([]*proto.MigrateTask{t1}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Times(1).Return([]*proto.Task{task}, nil) vuid := volume.VunitLocations[0].Vuid epoch := vuid.Epoch() volume.VunitLocations[0].Vuid = proto.EncodeVuid(vuid.VuidPrefix(), epoch+1) @@ -283,7 +285,7 @@ func TestDiskDropCheckAndClearJunkTasks(t *testing.T) { // has junk task and clear mgr.droppedDisks.add(disk1.DiskID, time.Now().Add(-junkMigrationTaskProtectionWindow)) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.MigrateTask{{TaskID: "test"}}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.Task{{TaskID: "test"}}, nil) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().DeleteMigrateTask(any, any).Return(nil) mgr.checkAndClearJunkTasks() require.Equal(t, 0, mgr.droppedDisks.size()) @@ -294,7 +296,7 @@ func TestDiskDropAcquireTask(t *testing.T) { ctx := context.Background() idc := "z0" mgr := newDiskDroper(t) - mgr.IMigrator.(*MockMigrater).EXPECT().AcquireTask(any, any).Return(proto.MigrateTask{TaskType: proto.TaskTypeDiskDrop}, nil) + mgr.IMigrator.(*MockMigrater).EXPECT().AcquireTask(any, any).Return(&proto.Task{TaskType: proto.TaskTypeDiskDrop}, nil) _, err := mgr.AcquireTask(ctx, idc) require.NoError(t, err) } @@ -303,7 +305,7 @@ func TestDiskDropCancelTask(t *testing.T) { ctx := context.Background() mgr := newDiskDroper(t) mgr.IMigrator.(*MockMigrater).EXPECT().CancelTask(any, any).Return(nil) - err := mgr.CancelTask(ctx, &api.OperateTaskArgs{}) + err := mgr.CancelTask(ctx, &api.TaskArgs{}) require.NoError(t, err) } @@ -311,9 +313,11 @@ func TestDiskDropReclaimTask(t *testing.T) { ctx := context.Background() idc := "z0" mgr := newDiskDroper(t) - mgr.IMigrator.(*MockMigrater).EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil) + mgr.IMigrator.(*MockMigrater).EXPECT().ReclaimTask(any, any).Return(nil) t1 := mockGenMigrateTask(proto.TaskTypeDiskDrop, idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap) - err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{}) + + taskArgs := generateTaskArgs(t1, "") + err := mgr.ReclaimTask(ctx, taskArgs) require.NoError(t, err) } @@ -323,11 +327,13 @@ func TestDiskDropCompleteTask(t *testing.T) { mgr := newDiskDroper(t) mgr.IMigrator.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(nil) t1 := mockGenMigrateTask(proto.TaskTypeDiskDrop, idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap) - err := mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + taskArgs := generateTaskArgs(t1, "") + err := mgr.CompleteTask(ctx, taskArgs) require.NoError(t, err) mgr.IMigrator.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(errMock) - err = mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + + err = mgr.CompleteTask(ctx, taskArgs) require.True(t, errors.Is(err, errMock)) } @@ -418,11 +424,11 @@ func TestDiskDropDiskProgress(t *testing.T) { mgr := newDiskDroper(t) mgr.allDisks.add(&dropDisk{DiskInfoSimple: testDisk1, undoneTaskCnt: 3}) - task1 := &proto.MigrateTask{State: proto.MigrateStatePrepared, SourceDiskID: testDisk1.DiskID} - task2 := &proto.MigrateTask{State: proto.MigrateStateInited, SourceDiskID: testDisk1.DiskID} - task3 := &proto.MigrateTask{State: proto.MigrateStateWorkCompleted, SourceDiskID: testDisk1.DiskID} + task1, _ := (&proto.MigrateTask{State: proto.MigrateStatePrepared, SourceDiskID: testDisk1.DiskID}).Task() + task2, _ := (&proto.MigrateTask{State: proto.MigrateStateInited, SourceDiskID: testDisk1.DiskID}).Task() + task3, _ := (&proto.MigrateTask{State: proto.MigrateStateWorkCompleted, SourceDiskID: testDisk1.DiskID}).Task() - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.MigrateTask{task1, task2, task3}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.Task{task1, task2, task3}, nil) stats, err := mgr.DiskProgress(ctx, testDisk1.DiskID) require.NoError(t, err) require.Equal(t, int(testDisk1.UsedChunkCnt), stats.TotalTasksCnt) diff --git a/blobstore/scheduler/disk_repairer.go b/blobstore/scheduler/disk_repairer.go index d1ab208d6..c52b39de4 100644 --- a/blobstore/scheduler/disk_repairer.go +++ b/blobstore/scheduler/disk_repairer.go @@ -16,6 +16,7 @@ package scheduler import ( "context" + "encoding/json" "errors" "fmt" "time" @@ -102,26 +103,31 @@ func (mgr *DiskRepairMgr) Load() error { var junkTasks []*proto.MigrateTask for _, t := range tasks { - if _, ok := mgr.repairingDisks.get(t.SourceDiskID); !ok { - junkTasks = append(junkTasks, t) + task := &proto.MigrateTask{} + err = task.Unmarshal(t.Data) + if err != nil { + return err + } + if _, ok := mgr.repairingDisks.get(task.SourceDiskID); !ok { + junkTasks = append(junkTasks, task) continue } - if t.Running() { - err = base.VolTaskLockerInst().TryLock(ctx, t.Vid()) + if task.Running() { + err = base.VolTaskLockerInst().TryLock(ctx, task.Vid()) if err != nil { return fmt.Errorf("repair task conflict: task[%+v], err[%+v]", t, err.Error()) } } - span.Infof("load task success: task_id[%s], state[%d]", t.TaskID, t.State) - switch t.State { + span.Infof("load task success: task_id[%s], state[%d]", t.TaskID, task.State) + switch task.State { case proto.MigrateStateInited: - mgr.prepareQueue.PushTask(t.TaskID, t) + mgr.prepareQueue.PushTask(t.TaskID, task) case proto.MigrateStatePrepared: - mgr.workQueue.AddPreparedTask(t.SourceIDC, t.TaskID, t) + mgr.workQueue.AddPreparedTask(task.SourceIDC, t.TaskID, task) case proto.MigrateStateWorkCompleted: - mgr.finishQueue.PushTask(t.TaskID, t) + mgr.finishQueue.PushTask(t.TaskID, task) case proto.MigrateStateFinished, proto.MigrateStateFinishedInAdvance: return fmt.Errorf("task should be deleted from db: task[%+v]", t) default: @@ -321,8 +327,13 @@ func (mgr *DiskRepairMgr) listMigratingVuid(ctx context.Context, diskID proto.Di return nil, err } + task := &proto.MigrateTask{} for _, t := range tasks { - bads = append(bads, t.SourceVuid) + err = task.Unmarshal(t.Data) + if err != nil { + return nil, err + } + bads = append(bads, task.SourceVuid) } return bads, nil } @@ -350,9 +361,16 @@ func (mgr *DiskRepairMgr) initOneTask(ctx context.Context, badVuid proto.Vuid, b SourceVuid: badVuid, SourceIDC: brokenDiskIdc, ForbiddenDirectDownload: true, + Ctime: time.Now().String(), } + t.MTime = t.Ctime + base.InsistOn(ctx, "repair init one task insert task to tbl", func() error { - return mgr.clusterMgrCli.AddMigrateTask(ctx, &t) + task, err := t.Task() + if err != nil { + return err + } + return mgr.clusterMgrCli.AddMigrateTask(ctx, task) }) mgr.prepareQueue.PushTask(t.TaskID, &t) @@ -452,7 +470,11 @@ func (mgr *DiskRepairMgr) prepareTask(t *proto.MigrateTask) error { t.Destination = allocDstVunit.Location() t.State = proto.MigrateStatePrepared base.InsistOn(ctx, "repair prepare task update task tbl", func() error { - return mgr.clusterMgrCli.UpdateMigrateTask(ctx, t) + task, err := t.Task() + if err != nil { + return err + } + return mgr.clusterMgrCli.UpdateMigrateTask(ctx, task) }) mgr.sendToWorkQueue(t) @@ -528,7 +550,11 @@ func (mgr *DiskRepairMgr) finishTask(ctx context.Context, task *proto.MigrateTas // because if process restart will reload task and redo by worker // worker will write data to chunk which is online base.InsistOn(ctx, "repair finish task update task state completed", func() error { - return mgr.clusterMgrCli.UpdateMigrateTask(ctx, task) + t, err := task.Task() + if err != nil { + return err + } + return mgr.clusterMgrCli.UpdateMigrateTask(ctx, t) }) newVuid := task.Destination.Vuid @@ -583,7 +609,11 @@ func (mgr *DiskRepairMgr) handleUpdateVolMappingFail(ctx context.Context, task * task.WorkerRedoCnt++ base.InsistOn(ctx, "repair redo task update task tbl", func() error { - return mgr.clusterMgrCli.UpdateMigrateTask(ctx, task) + t, err := task.Task() + if err != nil { + return err + } + return mgr.clusterMgrCli.UpdateMigrateTask(ctx, t) }) mgr.finishQueue.RemoveTask(task.TaskID) @@ -665,7 +695,7 @@ func (mgr *DiskRepairMgr) checkDiskRepaired(ctx context.Context, diskID proto.Di return len(tasks) == 0 && len(vunitInfos) == 0 } -func (mgr *DiskRepairMgr) clearJunkTasks(ctx context.Context, diskID proto.DiskID, tasks []*proto.MigrateTask) { +func (mgr *DiskRepairMgr) clearJunkTasks(ctx context.Context, diskID proto.DiskID, tasks []*proto.Task) { span := trace.SpanFromContextSafe(ctx) for _, task := range tasks { if !mgr.deletedTasks.exits(diskID, task.TaskID) { @@ -728,26 +758,38 @@ func (mgr *DiskRepairMgr) checkAndClearJunkTasks() { } // AcquireTask acquire repair task -func (mgr *DiskRepairMgr) AcquireTask(ctx context.Context, idc string) (task proto.MigrateTask, err error) { +func (mgr *DiskRepairMgr) AcquireTask(ctx context.Context, idc string) (task *proto.Task, err error) { if !mgr.taskSwitch.Enabled() { return task, proto.ErrTaskPaused } - _, repairTask, _ := mgr.workQueue.Acquire(idc) if repairTask != nil { - task = *repairTask.(*proto.MigrateTask) + t := *repairTask.(*proto.MigrateTask) + data, err := t.Marshal() + if err != nil { + return task, err + } + task = &proto.Task{} + task.Data = data + task.ModuleType = proto.TypeBlobNode return task, nil } return task, proto.ErrTaskEmpty } // CancelTask cancel repair task -func (mgr *DiskRepairMgr) CancelTask(ctx context.Context, args *api.OperateTaskArgs) error { +func (mgr *DiskRepairMgr) CancelTask(ctx context.Context, args *api.TaskArgs) error { span := trace.SpanFromContextSafe(ctx) - err := mgr.workQueue.Cancel(args.IDC, args.TaskID, args.Src, args.Dest) + arg := &api.OperateTaskArgs{} + err := arg.Unmarshal(args.Data) if err != nil { - span.Errorf("cancel repair failed: task_id[%s], err[%+v]", args.TaskID, err) + return err + } + + err = mgr.workQueue.Cancel(arg.IDC, arg.TaskID, arg.Src, arg.Dest) + if err != nil { + span.Errorf("cancel repair failed: task_id[%s], err[%+v]", arg.TaskID, err) } mgr.taskStatsMgr.CancelTask() @@ -756,30 +798,36 @@ func (mgr *DiskRepairMgr) CancelTask(ctx context.Context, args *api.OperateTaskA } // ReclaimTask reclaim repair task -func (mgr *DiskRepairMgr) ReclaimTask(ctx context.Context, - idc, taskID string, - src []proto.VunitLocation, - oldDst proto.VunitLocation, - newDst *client.AllocVunitInfo, -) error { +func (mgr *DiskRepairMgr) ReclaimTask(ctx context.Context, args *api.TaskArgs) error { span := trace.SpanFromContextSafe(ctx) - err := mgr.workQueue.Reclaim(idc, taskID, src, oldDst, newDst.Location(), newDst.DiskID) + arg := &api.OperateTaskArgs{} + err := arg.Unmarshal(args.Data) if err != nil { - // task has finished,because only complete will remove task from queue - span.Errorf("reclaim repair task failed: task_id[%s], err[%+v]", taskID, err) return err } - task, err := mgr.workQueue.Query(idc, taskID) + newDst, err := base.AllocVunitSafe(ctx, mgr.clusterMgrCli, arg.Dest.Vuid, arg.Src) if err != nil { - span.Errorf("found task in workQueue failed: idc[%s], task_id[%s], err[%+v]", idc, taskID, err) + span.Errorf("alloc volume unit from clustermgr failed, err: %s", err) return err } - err = mgr.clusterMgrCli.UpdateMigrateTask(ctx, task.(*proto.MigrateTask)) + err = mgr.workQueue.Reclaim(arg.IDC, arg.TaskID, arg.Src, arg.Dest, newDst.VunitLocation, newDst.DiskID) if err != nil { - span.Warnf("update reclaim task failed: task_id[%s], err[%+v]", taskID, err) + span.Errorf("reclaim task in workQueue failed: idc[%s], task_id[%s], err[%+v]", arg.IDC, arg.TaskID, err) + return err + } + + task, err := mgr.workQueue.Query(arg.IDC, arg.TaskID) + if err != nil { + span.Errorf("found task in workQueue failed: idc[%s], task_id[%s], err[%+v]", arg.IDC, arg.TaskID, err) + return err + } + t, _ := task.Task() + err = mgr.clusterMgrCli.UpdateMigrateTask(ctx, t) + if err != nil { + span.Warnf("update reclaim task failed: task_id[%s], err[%+v]", arg.TaskID, err) } mgr.taskStatsMgr.ReclaimTask() @@ -787,19 +835,25 @@ func (mgr *DiskRepairMgr) ReclaimTask(ctx context.Context, } // CompleteTask complete repair task -func (mgr *DiskRepairMgr) CompleteTask(ctx context.Context, args *api.OperateTaskArgs) error { +func (mgr *DiskRepairMgr) CompleteTask(ctx context.Context, args *api.TaskArgs) error { span := trace.SpanFromContextSafe(ctx) - completeTask, err := mgr.workQueue.Complete(args.IDC, args.TaskID, args.Src, args.Dest) + arg := &api.OperateTaskArgs{} + err := arg.Unmarshal(args.Data) if err != nil { - span.Errorf("complete repair task failed: task_id[%s], err[%+v]", args.TaskID, err) + return err + } + + completeTask, err := mgr.workQueue.Complete(arg.IDC, arg.TaskID, arg.Src, arg.Dest) + if err != nil { + span.Errorf("complete repair task failed: task_id[%s], err[%+v]", arg.TaskID, err) return err } t := completeTask.(*proto.MigrateTask) t.State = proto.MigrateStateWorkCompleted - mgr.finishQueue.PushTask(args.TaskID, t) + mgr.finishQueue.PushTask(arg.TaskID, t) // as complete func is face to svr api, so can not loop save task // to db until success, it will make saving task info to be difficult, // that delay saving task info in finish stage is a simply way @@ -818,30 +872,51 @@ func (mgr *DiskRepairMgr) RenewalTask(ctx context.Context, idc, taskID string) e if err != nil { span.Warnf("renewal repair task failed: task_id[%s], err[%+v]", taskID, err) } - return err } +func (mgr *DiskRepairMgr) ReportTask(ctx context.Context, args *api.TaskArgs) (err error) { + arg := &api.TaskReportArgs{} + err = arg.Unmarshal(args.Data) + if err != nil { + return err + } + if !client.ValidMigrateTask(arg.TaskType, arg.TaskID) { + return errcode.ErrIllegalArguments + } + mgr.ReportWorkerTaskStats(arg) + return nil +} + // ReportWorkerTaskStats reports task stats func (mgr *DiskRepairMgr) ReportWorkerTaskStats(st *api.TaskReportArgs) { mgr.taskStatsMgr.ReportWorkerTaskStats(st.TaskID, st.TaskStats, st.IncreaseDataSizeByte, st.IncreaseShardCnt) } // QueryTask return task statistics -func (mgr *DiskRepairMgr) QueryTask(ctx context.Context, taskID string) (*api.MigrateTaskDetail, error) { - detail := &api.MigrateTaskDetail{} - taskInfo, err := mgr.clusterMgrCli.GetMigrateTask(ctx, proto.TaskTypeDiskRepair, taskID) +func (mgr *DiskRepairMgr) QueryTask(ctx context.Context, taskID string) (*api.TaskRet, error) { + migTask := &api.MigrateTaskDetail{} + task, err := mgr.clusterMgrCli.GetMigrateTask(ctx, proto.TaskTypeDiskRepair, taskID) if err != nil { - return detail, err + return nil, err } - detail.Task = *taskInfo - + migrateTask := &proto.MigrateTask{} + err = migrateTask.Unmarshal(task.Data) + if err != nil { + return nil, err + } + migTask.Task = *migrateTask detailRunInfo, err := mgr.taskStatsMgr.QueryTaskDetail(taskID) if err != nil { - return detail, nil + return nil, nil } - detail.Stat = detailRunInfo.Statistics - return detail, nil + migTask.Stat = detailRunInfo.Statistics + + data, err := json.Marshal(migTask) + if err != nil { + return nil, err + } + return &api.TaskRet{TaskType: proto.TaskTypeDiskRepair, Data: data}, nil } // StatQueueTaskCnt returns task queue stats diff --git a/blobstore/scheduler/disk_repairer_test.go b/blobstore/scheduler/disk_repairer_test.go index 44d72617e..ef39a9d4d 100644 --- a/blobstore/scheduler/disk_repairer_test.go +++ b/blobstore/scheduler/disk_repairer_test.go @@ -60,7 +60,29 @@ func newDiskRepairer(t *testing.T) *DiskRepairMgr { return NewDiskRepairMgr(clusterMgr, taskSwitch, taskLogger, conf) } +func generateTaskArgs(task *proto.MigrateTask, reason string) *api.TaskArgs { + ret := new(api.TaskArgs) + args := api.OperateTaskArgs{ + IDC: task.SourceIDC, + TaskType: task.TaskType, + Src: task.Sources, + TaskID: task.TaskID, + Reason: reason, + Dest: task.Destination, + } + data, _ := args.Marshal() + ret.Data = data + ret.TaskType = task.TaskType + ret.ModuleType = proto.TypeBlobNode + return ret +} + func TestDiskRepairerLoad(t *testing.T) { + task, err := (&proto.MigrateTask{ + SourceDiskID: testDisk1.DiskID, + TaskID: client.GenMigrateTaskID(proto.TaskTypeDiskRepair, testDisk1.DiskID, proto.Vid(1)), + }).Task() + require.NoError(t, err) { mgr := newDiskRepairer(t) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListMigratingDisks(any, any).Return(nil, errMock) @@ -84,10 +106,7 @@ func TestDiskRepairerLoad(t *testing.T) { { mgr := newDiskRepairer(t) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListMigratingDisks(any, any).Return(nil, nil) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{{ - SourceDiskID: testDisk1.DiskID, - TaskID: client.GenMigrateTaskID(proto.TaskTypeDiskRepair, testDisk1.DiskID, proto.Vid(1)), - }}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{task}, nil) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return( &client.DiskInfoSimple{DiskID: testDisk1.DiskID, Status: proto.DiskStatusRepaired}, nil) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().DeleteMigrateTask(any, any).Return(nil) @@ -97,10 +116,7 @@ func TestDiskRepairerLoad(t *testing.T) { { mgr := newDiskRepairer(t) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListMigratingDisks(any, any).Return(nil, nil) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{{ - SourceDiskID: testDisk1.DiskID, - TaskID: client.GenMigrateTaskID(proto.TaskTypeDiskRepair, testDisk1.DiskID, proto.Vid(1)), - }}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{task}, nil) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return( &client.DiskInfoSimple{DiskID: testDisk1.DiskID, Status: proto.DiskStatusNormal}, nil) err := mgr.Load() @@ -109,10 +125,7 @@ func TestDiskRepairerLoad(t *testing.T) { { mgr := newDiskRepairer(t) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListMigratingDisks(any, any).Return(nil, nil) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{{ - SourceDiskID: testDisk1.DiskID, - TaskID: client.GenMigrateTaskID(proto.TaskTypeDiskRepair, testDisk1.DiskID, proto.Vid(1)), - }}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{task}, nil) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(nil, errMock) err := mgr.Load() require.Error(t, err) @@ -135,42 +148,46 @@ func TestDiskRepairerLoad(t *testing.T) { { mgr := newDiskRepairer(t) t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStateFinishedInAdvance, newMockVolInfoMap()) + task, err = t1.Task() + require.NoError(t, err) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListMigratingDisks(any, any).Return([]*client.MigratingDiskMeta{{Disk: &client.DiskInfoSimple{DiskID: proto.DiskID(1)}}}, nil) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t1}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{task}, nil) err := mgr.Load() require.Error(t, err) t2 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStateFinished, newMockVolInfoMap()) + task, err = t2.Task() + require.NoError(t, err) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListMigratingDisks(any, any).Return([]*client.MigratingDiskMeta{{Disk: &client.DiskInfoSimple{DiskID: proto.DiskID(1)}}}, nil) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t2}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{task}, nil) err = mgr.Load() require.Error(t, err) } { mgr := newDiskRepairer(t) - t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStateInited, newMockVolInfoMap()) - t2 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 2, proto.MigrateStatePrepared, newMockVolInfoMap()) - t3 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 4, proto.MigrateStateWorkCompleted, newMockVolInfoMap()) + t1, _ := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStateInited, newMockVolInfoMap()).Task() + t2, _ := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 2, proto.MigrateStatePrepared, newMockVolInfoMap()).Task() + t3, _ := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 4, proto.MigrateStateWorkCompleted, newMockVolInfoMap()).Task() mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListMigratingDisks(any, any).Return([]*client.MigratingDiskMeta{{Disk: &client.DiskInfoSimple{DiskID: proto.DiskID(1)}}}, nil) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t1, t2, t3}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{t1, t2, t3}, nil) err := mgr.Load() require.NoError(t, err) } { // same volume task prepared mgr := newDiskRepairer(t) - t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap()) - t2 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap()) + t1, _ := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap()).Task() + t2, _ := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap()).Task() mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListMigratingDisks(any, any).Return([]*client.MigratingDiskMeta{{Disk: &client.DiskInfoSimple{DiskID: proto.DiskID(1)}}}, nil) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t1, t2}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{t1, t2}, nil) err := mgr.Load() require.Error(t, err) } { mgr := newDiskRepairer(t) - t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateState(111), newMockVolInfoMap()) + t1, _ := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateState(111), newMockVolInfoMap()).Task() mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListMigratingDisks(any, any).Return([]*client.MigratingDiskMeta{{Disk: &client.DiskInfoSimple{DiskID: proto.DiskID(1)}}}, nil) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t1}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{t1}, nil) err := mgr.Load() require.Error(t, err) } @@ -306,15 +323,15 @@ func TestDiskRepairerCollectTask(t *testing.T) { } units = append(units, &ele) } - t1 := &proto.MigrateTask{ + t1, _ := (&proto.MigrateTask{ TaskID: client.GenMigrateTaskID(proto.TaskTypeDiskRepair, proto.DiskID(1), volume.Vid), TaskType: proto.TaskTypeDiskRepair, SourceVuid: units[0].Vuid, - } + }).Task() mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListBrokenDisks(any).Return([]*client.DiskInfoSimple{testDisk1}, nil) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().SetDiskRepairing(any, any).Return(nil) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.MigrateTask{t1}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.Task{t1}, nil) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AddMigratingDisk(any, any).Return(nil) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AddMigrateTask(any, any).AnyTimes().Return(nil) mgr.collectTask() @@ -564,12 +581,13 @@ func TestDiskRepairerCheckRepairedAndClear(t *testing.T) { } units = append(units, &ele) } - t1 := &proto.MigrateTask{ + task := &proto.MigrateTask{ TaskID: client.GenMigrateTaskID(proto.TaskTypeDiskRepair, proto.DiskID(1), volume.Vid), TaskType: proto.TaskTypeDiskRepair, SourceVuid: units[0].Vuid, } - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Times(3).Return([]*proto.MigrateTask{t1}, nil) + t1, _ := task.Task() + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Times(3).Return([]*proto.Task{t1}, nil) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil) mgr.checkRepairedAndClear() require.True(t, mgr.repairingDisks.size() > 0) @@ -579,11 +597,11 @@ func TestDiskRepairerCheckRepairedAndClear(t *testing.T) { require.True(t, mgr.repairingDisks.size() > 0) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, nil) - mgr.deletedTasks.add(t1.SourceDiskID, t1.TaskID) + mgr.deletedTasks.add(task.SourceDiskID, t1.TaskID) mgr.checkRepairedAndClear() require.True(t, mgr.repairingDisks.size() > 0) - t1.State = proto.MigrateStateFinished + // t1.State = proto.MigrateStateFinished mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Times(4).Return(nil, nil) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, errMock) mgr.checkRepairedAndClear() @@ -654,7 +672,7 @@ func TestDiskRepairerCheckAndClearJunkTasks(t *testing.T) { // has junk task and clear mgr.repairedDisks.add(disk1.DiskID, time.Now().Add(-junkMigrationTaskProtectionWindow)) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.MigrateTask{{TaskID: "test"}}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.Task{{TaskID: "test"}}, nil) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().DeleteMigrateTask(any, any).Return(nil) mgr.checkAndClearJunkTasks() require.Equal(t, 0, mgr.repairedDisks.size()) @@ -691,7 +709,7 @@ func TestDiskRepairerCancelTask(t *testing.T) { idc := "z0" { mgr := newDiskRepairer(t) - err := mgr.CancelTask(ctx, &api.OperateTaskArgs{}) + err := mgr.CancelTask(ctx, &api.TaskArgs{}) require.Error(t, err) } { @@ -699,7 +717,7 @@ func TestDiskRepairerCancelTask(t *testing.T) { t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap()) mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1) - err := mgr.CancelTask(ctx, &api.OperateTaskArgs{}) + err := mgr.CancelTask(ctx, &api.TaskArgs{}) require.Error(t, err) } } @@ -710,23 +728,33 @@ func TestDiskRepairerReclaimTask(t *testing.T) { { mgr := newDiskRepairer(t) t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap()) - err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{}) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(gomock.Any(), gomock.Any()).Return(nil, errMock) + taskArgs := generateTaskArgs(t1, "") + err := mgr.ReclaimTask(ctx, taskArgs) require.Error(t, err) } { mgr := newDiskRepairer(t) t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap()) + location := t1.Destination + location.Vuid += 1 mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(gomock.Any(), gomock.Any()).Return(&client.AllocVunitInfo{VunitLocation: location}, nil) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateMigrateTask(any, any).Return(errMock) - err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{}) + args := generateTaskArgs(t1, "") + err := mgr.ReclaimTask(ctx, args) require.NoError(t, err) } { mgr := newDiskRepairer(t) t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap()) + location := t1.Destination + location.Vuid += 1 mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(gomock.Any(), gomock.Any()).Return(&client.AllocVunitInfo{VunitLocation: location}, nil) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateMigrateTask(any, any).Return(nil) - err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{}) + args := generateTaskArgs(t1, "") + err := mgr.ReclaimTask(ctx, args) require.NoError(t, err) } } @@ -737,14 +765,16 @@ func TestDiskRepairerCompleteTask(t *testing.T) { { mgr := newDiskRepairer(t) t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap()) - err := mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + args := generateTaskArgs(t1, "") + err := mgr.CompleteTask(ctx, args) require.Error(t, err) } { mgr := newDiskRepairer(t) t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap()) mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1) - err := mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + args := generateTaskArgs(t1, "") + err := mgr.CompleteTask(ctx, args) require.NoError(t, err) todo, doing := mgr.finishQueue.StatsTasks() require.Equal(t, 1, todo+doing) @@ -802,7 +832,7 @@ func TestDiskRepairerQueryTask(t *testing.T) { } { mgr := newDiskRepairer(t) - t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap()) + t1, _ := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap()).Task() mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetMigrateTask(any, any, any).Return(t1, nil) _, err := mgr.QueryTask(ctx, taskID) require.NoError(t, err) @@ -835,9 +865,9 @@ func TestDiskRepairerProgress(t *testing.T) { { mgr := newDiskRepairer(t) mgr.repairingDisks.add(testDisk1.DiskID, testDisk1) - t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap()) - t2 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 2, proto.MigrateStateInited, newMockVolInfoMap()) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.MigrateTask{t1, t2}, nil) + t1, _ := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap()).Task() + t2, _ := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 2, proto.MigrateStateInited, newMockVolInfoMap()).Task() + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.Task{t1, t2}, nil) disks, total, repaired := mgr.Progress(ctx) require.Equal(t, 1, len(disks)) require.Equal(t, testDisk1.DiskID, disks[0]) @@ -848,11 +878,11 @@ func TestDiskRepairerProgress(t *testing.T) { mgr := newDiskRepairer(t) mgr.repairingDisks.add(testDisk1.DiskID, testDisk1) mgr.repairingDisks.add(testDisk2.DiskID, testDisk2) - task1 := &proto.MigrateTask{State: proto.MigrateStatePrepared, SourceDiskID: testDisk1.DiskID} - task2 := &proto.MigrateTask{State: proto.MigrateStateInited, SourceDiskID: testDisk1.DiskID} - task3 := &proto.MigrateTask{State: proto.MigrateStateWorkCompleted, SourceDiskID: testDisk1.DiskID} - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.MigrateTask{task1, task2, task3}, nil) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.MigrateTask{task1, task2}, nil) + task1, _ := (&proto.MigrateTask{State: proto.MigrateStatePrepared, SourceDiskID: testDisk1.DiskID}).Task() + task2, _ := (&proto.MigrateTask{State: proto.MigrateStateInited, SourceDiskID: testDisk1.DiskID}).Task() + task3, _ := (&proto.MigrateTask{State: proto.MigrateStateWorkCompleted, SourceDiskID: testDisk1.DiskID}).Task() + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.Task{task1, task2, task3}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.Task{task1, task2}, nil) disks, tatal, repaired := mgr.Progress(ctx) require.Equal(t, 2, len(disks)) require.Equal(t, testDisk1.UsedChunkCnt+testDisk2.UsedChunkCnt, int64(tatal)) @@ -878,11 +908,11 @@ func TestDiskRepairerDiskProgress(t *testing.T) { mgr := newDiskRepairer(t) mgr.repairingDisks.add(testDisk1.DiskID, testDisk1) - task1 := &proto.MigrateTask{State: proto.MigrateStatePrepared, SourceDiskID: testDisk1.DiskID} - task2 := &proto.MigrateTask{State: proto.MigrateStateInited, SourceDiskID: testDisk1.DiskID} - task3 := &proto.MigrateTask{State: proto.MigrateStateWorkCompleted, SourceDiskID: testDisk1.DiskID} + task1, _ := (&proto.MigrateTask{State: proto.MigrateStatePrepared, SourceDiskID: testDisk1.DiskID}).Task() + task2, _ := (&proto.MigrateTask{State: proto.MigrateStateInited, SourceDiskID: testDisk1.DiskID}).Task() + task3, _ := (&proto.MigrateTask{State: proto.MigrateStateWorkCompleted, SourceDiskID: testDisk1.DiskID}).Task() - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.MigrateTask{task1, task2, task3}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.Task{task1, task2, task3}, nil) stats, err := mgr.DiskProgress(ctx, testDisk1.DiskID) require.NoError(t, err) require.Equal(t, int(testDisk1.UsedChunkCnt), stats.TotalTasksCnt) diff --git a/blobstore/scheduler/manual_migrater_test.go b/blobstore/scheduler/manual_migrater_test.go index 735bd2772..6bbedd61d 100644 --- a/blobstore/scheduler/manual_migrater_test.go +++ b/blobstore/scheduler/manual_migrater_test.go @@ -84,7 +84,7 @@ func TestManualMigrateAcquireTask(t *testing.T) { ctx := context.Background() idc := "z0" mgr := newManualMigrater(t) - mgr.IMigrator.(*MockMigrater).EXPECT().AcquireTask(any, any).Return(proto.MigrateTask{TaskType: proto.TaskTypeManualMigrate}, nil) + mgr.IMigrator.(*MockMigrater).EXPECT().AcquireTask(any, any).Return(&proto.Task{TaskType: proto.TaskTypeManualMigrate}, nil) _, err := mgr.AcquireTask(ctx, idc) require.NoError(t, err) } @@ -93,7 +93,7 @@ func TestManualMigrateCancelTask(t *testing.T) { ctx := context.Background() mgr := newManualMigrater(t) mgr.IMigrator.(*MockMigrater).EXPECT().CancelTask(any, any).Return(nil) - err := mgr.CancelTask(ctx, &api.OperateTaskArgs{}) + err := mgr.CancelTask(ctx, &api.TaskArgs{}) require.NoError(t, err) } @@ -101,9 +101,10 @@ func TestManualMigrateReclaimTask(t *testing.T) { ctx := context.Background() idc := "z0" mgr := newManualMigrater(t) - mgr.IMigrator.(*MockMigrater).EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil) + mgr.IMigrator.(*MockMigrater).EXPECT().ReclaimTask(any, any).Return(nil) t1 := mockGenMigrateTask(proto.TaskTypeManualMigrate, idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap) - err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{}) + taskArgs := generateTaskArgs(t1, "") + err := mgr.ReclaimTask(ctx, taskArgs) require.NoError(t, err) } @@ -113,11 +114,12 @@ func TestManualMigrateCompleteTask(t *testing.T) { mgr := newManualMigrater(t) mgr.IMigrator.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(nil) t1 := mockGenMigrateTask(proto.TaskTypeManualMigrate, idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap) - err := mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + taskArgs := generateTaskArgs(t1, "") + err := mgr.CompleteTask(ctx, taskArgs) require.NoError(t, err) mgr.IMigrator.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(errMock) - err = mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + err = mgr.CompleteTask(ctx, taskArgs) require.True(t, errors.Is(err, errMock)) } diff --git a/blobstore/scheduler/migrate.go b/blobstore/scheduler/migrate.go index 733d83a47..bf0ffa962 100644 --- a/blobstore/scheduler/migrate.go +++ b/blobstore/scheduler/migrate.go @@ -16,6 +16,7 @@ package scheduler import ( "context" + "encoding/json" "errors" "fmt" "sync" @@ -49,19 +50,30 @@ type MMigrator interface { IManualMigrator } -// Migrator base interface of migrate, balancer, disk_droper, manual_migrater. -type Migrator interface { - AcquireTask(ctx context.Context, idc string) (proto.MigrateTask, error) - CancelTask(ctx context.Context, args *api.OperateTaskArgs) error - CompleteTask(ctx context.Context, args *api.OperateTaskArgs) error - ReclaimTask(ctx context.Context, idc, taskID string, - src []proto.VunitLocation, oldDst proto.VunitLocation, newDst *client.AllocVunitInfo) error +var ( + _ BaseMigrator = (*DiskRepairMgr)(nil) + _ BaseMigrator = (*MigrateMgr)(nil) +) + +// BaseMigrator base interface for shard and blobnode task. +type BaseMigrator interface { + AcquireTask(ctx context.Context, idc string) (*proto.Task, error) RenewalTask(ctx context.Context, idc, taskID string) error - QueryTask(ctx context.Context, taskID string) (*api.MigrateTaskDetail, error) + CancelTask(ctx context.Context, args *api.TaskArgs) error + CompleteTask(ctx context.Context, args *api.TaskArgs) error + ReclaimTask(ctx context.Context, args *api.TaskArgs) error + ReportTask(ctx context.Context, args *api.TaskArgs) (err error) + QueryTask(ctx context.Context, taskID string) (*api.TaskRet, error) +} + +// Migrator interface of blobnode migrate, balancer, disk_dropper, manual_migrator. +type Migrator interface { + BaseMigrator // status ReportWorkerTaskStats(st *api.TaskReportArgs) StatQueueTaskCnt() (inited, prepared, completed int) Stats() api.MigrateTasksStat + // control taskswitch.ISwitcher closer.Closer @@ -426,33 +438,38 @@ func (mgr *MigrateMgr) Load() (err error) { } var junkTasks []*proto.MigrateTask for i := range tasks { - if mgr.isJunkTask(disks, tasks[i]) { - junkTasks = append(junkTasks, tasks[i]) + task := &proto.MigrateTask{} + err = task.Unmarshal(tasks[i].Data) + if err != nil { + return err + } + if mgr.isJunkTask(disks, task) { + junkTasks = append(junkTasks, task) continue } - if tasks[i].Running() { - err = base.VolTaskLockerInst().TryLock(ctx, tasks[i].SourceVuid.Vid()) + if task.Running() { + err = base.VolTaskLockerInst().TryLock(ctx, task.SourceVuid.Vid()) if err != nil { return fmt.Errorf("migrate task conflict: vid[%d], task[%+v], err[%+v]", - tasks[i].SourceVuid.Vid(), tasks[i], err) + task.SourceVuid.Vid(), tasks[i], err) } } - mgr.loadTaskCallback(tasks[i].SourceDiskID) - mgr.addMigratingVuid(tasks[i].SourceDiskID, tasks[i].SourceVuid, tasks[i].TaskID) + mgr.loadTaskCallback(task.SourceDiskID) + mgr.addMigratingVuid(task.SourceDiskID, task.SourceVuid, task.TaskID) - span.Infof("load task success: task_type[%s], task_id[%s], state[%d]", mgr.taskType, tasks[i].TaskID, tasks[i].State) - switch tasks[i].State { + span.Infof("load task success: task_type[%s], task_id[%s], state[%d]", mgr.taskType, task.TaskID, task.State) + switch task.State { case proto.MigrateStateInited: - mgr.prepareQueue.PushTask(tasks[i].TaskID, tasks[i]) + mgr.prepareQueue.PushTask(task.TaskID, task) case proto.MigrateStatePrepared: - mgr.workQueue.AddPreparedTask(tasks[i].SourceIDC, tasks[i].TaskID, tasks[i]) + mgr.workQueue.AddPreparedTask(task.SourceIDC, task.TaskID, task) case proto.MigrateStateWorkCompleted: - mgr.finishQueue.PushTask(tasks[i].TaskID, tasks[i]) + mgr.finishQueue.PushTask(task.TaskID, task) case proto.MigrateStateFinished, proto.MigrateStateFinishedInAdvance: - return fmt.Errorf("task should be deleted from db: task[%+v]", tasks[i]) + return fmt.Errorf("task should be deleted from db: task[%+v]", task) default: - return fmt.Errorf("unexpect migrate state: task[%+v]", tasks[i]) + return fmt.Errorf("unexpect migrate state: task[%+v]", task) } } return mgr.clearJunkTasksCallBack(ctx, junkTasks) @@ -591,7 +608,11 @@ func (mgr *MigrateMgr) prepareTask() (err error) { // update db base.InsistOn(ctx, "migrate prepare task update task tbl", func() error { - return mgr.clusterMgrCli.UpdateMigrateTask(ctx, migTask) + task, err := migTask.Task() + if err != nil { + return err + } + return mgr.clusterMgrCli.UpdateMigrateTask(ctx, task) }) // send task to worker queue and remove task in prepareQueue @@ -636,7 +657,11 @@ func (mgr *MigrateMgr) finishTask() (err error) { // because competed task did not persisted to the database, so in finish phase need to do it // the task maybe update more than once, which is allowed base.InsistOn(ctx, "migrate finish task update task tbl to state completed ", func() error { - return mgr.clusterMgrCli.UpdateMigrateTask(ctx, migrateTask) + task, err := migrateTask.Task() + if err != nil { + return err + } + return mgr.clusterMgrCli.UpdateMigrateTask(ctx, task) }) // update volume mapping relationship @@ -658,7 +683,6 @@ func (mgr *MigrateMgr) finishTask() (err error) { err = mgr.handleUpdateVolMappingFail(ctx, migrateTask, err) return } - span.Warnf("task_id[%s] update volume failed but updated by tiomeout request", migrateTask.TaskID) } err = mgr.clusterMgrCli.ReleaseVolumeUnit(ctx, migrateTask.SourceVuid, migrateTask.SourceDiskID) @@ -720,7 +744,11 @@ func (mgr *MigrateMgr) updateVolumeCache(ctx context.Context, task *proto.Migrat func (mgr *MigrateMgr) AddTask(ctx context.Context, task *proto.MigrateTask) { // add task to db base.InsistOn(ctx, "migrate add task insert task to tbl", func() error { - return mgr.clusterMgrCli.AddMigrateTask(ctx, task) + t, err := task.Task() + if err != nil { + return err + } + return mgr.clusterMgrCli.AddMigrateTask(ctx, t) }) // add task to prepare queue @@ -778,7 +806,11 @@ func (mgr *MigrateMgr) handleUpdateVolMappingFail(ctx context.Context, task *pro task.WorkerRedoCnt++ base.InsistOn(ctx, "migrate redo task update task tbl", func() error { - return mgr.clusterMgrCli.UpdateMigrateTask(ctx, task) + t, err := task.Task() + if err != nil { + return err + } + return mgr.clusterMgrCli.UpdateMigrateTask(ctx, t) }) _ = mgr.finishQueue.RemoveTask(task.TaskID) @@ -874,80 +906,128 @@ func (mgr *MigrateMgr) Stats() api.MigrateTasksStat { } // AcquireTask acquire migrate task -func (mgr *MigrateMgr) AcquireTask(ctx context.Context, idc string) (task proto.MigrateTask, err error) { +func (mgr *MigrateMgr) AcquireTask(ctx context.Context, idc string) (task *proto.Task, err error) { span := trace.SpanFromContextSafe(ctx) - + task = &proto.Task{} + task.ModuleType = proto.TypeBlobNode if !mgr.taskSwitch.Enabled() { return task, proto.ErrTaskPaused } _, migTask, _ := mgr.workQueue.Acquire(idc) if migTask != nil { - task = *migTask.(*proto.MigrateTask) - span.Infof("acquire %s taskId: %s", mgr.taskType, task.TaskID) + t := *migTask.(*proto.MigrateTask) + data, err := t.Marshal() + if err != nil { + return task, err + } + task.TaskID = t.TaskID + task.Data = data + span.Infof("acquire %s taskId: %s", mgr.taskType, t.TaskID) return task, nil } return task, proto.ErrTaskEmpty } // CancelTask cancel migrate task -func (mgr *MigrateMgr) CancelTask(ctx context.Context, args *api.OperateTaskArgs) (err error) { +func (mgr *MigrateMgr) CancelTask(ctx context.Context, args *api.TaskArgs) (err error) { mgr.taskStatsMgr.CancelTask() - err = mgr.workQueue.Cancel(args.IDC, args.TaskID, args.Src, args.Dest) + arg := &api.OperateTaskArgs{} + err = arg.Unmarshal(args.Data) + if err != nil { + return err + } + + if !client.ValidMigrateTask(args.TaskType, arg.TaskID) { + return errcode.ErrIllegalArguments + } + + err = mgr.workQueue.Cancel(arg.IDC, arg.TaskID, arg.Src, arg.Dest) if err != nil { span := trace.SpanFromContextSafe(ctx) - span.Errorf("cancel migrate failed: task_type[%s], task_id[%s], err[%+v]", mgr.taskType, args.TaskID, err) + span.Errorf("cancel migrate failed: task_type[%s], task_id[%s], err[%+v]", mgr.taskType, arg.TaskID, err) } return } // ReclaimTask reclaim migrate task -func (mgr *MigrateMgr) ReclaimTask(ctx context.Context, idc, taskID string, - src []proto.VunitLocation, oldDst proto.VunitLocation, newDst *client.AllocVunitInfo, -) (err error) { +func (mgr *MigrateMgr) ReclaimTask(ctx context.Context, args *api.TaskArgs) (err error) { mgr.taskStatsMgr.ReclaimTask() - span := trace.SpanFromContextSafe(ctx) - err = mgr.workQueue.Reclaim(idc, taskID, src, oldDst, newDst.Location(), newDst.DiskID) + + arg := &api.OperateTaskArgs{} + err = arg.Unmarshal(args.Data) if err != nil { - span.Errorf("reclaim migrate task failed: task_type:[%s],task_id[%s], err[%+v]", mgr.taskType, taskID, err) return err } - task, err := mgr.workQueue.Query(idc, taskID) + if !client.ValidMigrateTask(args.TaskType, arg.TaskID) { + return errcode.ErrIllegalArguments + } + + newDst, err := base.AllocVunitSafe(ctx, mgr.clusterMgrCli, arg.Dest.Vuid, arg.Src) if err != nil { - span.Errorf("found task in workQueue failed: idc[%s], task_id[%s], err[%+v]", idc, taskID, err) + span.Errorf("alloc volume unit from clustermgr failed, err: %s", err) return err } - err = mgr.clusterMgrCli.UpdateMigrateTask(ctx, task.(*proto.MigrateTask)) + + err = mgr.workQueue.Reclaim(arg.IDC, arg.TaskID, arg.Src, arg.Dest, newDst.Location(), newDst.DiskID) if err != nil { - span.Errorf("update reclaim task failed: task_id[%s], err[%+v]", taskID, err) + span.Errorf("reclaim migrate task failed: task_type:[%s],task_id[%s], err[%+v]", mgr.taskType, arg.TaskID, err) + return err + } + + task, err := mgr.workQueue.Query(arg.IDC, arg.TaskID) + if err != nil { + span.Errorf("found task in workQueue failed: idc[%s], task_id[%s], err[%+v]", arg.IDC, arg.TaskID, err) + return err + } + t, err := task.Task() + if err != nil { + return err + } + err = mgr.clusterMgrCli.UpdateMigrateTask(ctx, t) + + if err != nil { + span.Errorf("update reclaim task failed: task_id[%s], err[%+v]", arg.TaskID, err) } return } // CompleteTask complete migrate task -func (mgr *MigrateMgr) CompleteTask(ctx context.Context, args *api.OperateTaskArgs) (err error) { +func (mgr *MigrateMgr) CompleteTask(ctx context.Context, args *api.TaskArgs) (err error) { span := trace.SpanFromContextSafe(ctx) - completeTask, err := mgr.workQueue.Complete(args.IDC, args.TaskID, args.Src, args.Dest) + arg := &api.OperateTaskArgs{} + err = arg.Unmarshal(args.Data) if err != nil { - span.Errorf("complete migrate task failed: task_id[%s], err[%+v]", args.TaskID, err) + return err + } + if !client.ValidMigrateTask(args.TaskType, arg.TaskID) { + return errcode.ErrIllegalArguments + } + + completeTask, err := mgr.workQueue.Complete(arg.IDC, arg.TaskID, arg.Src, arg.Dest) + if err != nil { + span.Errorf("complete migrate task failed: task_id[%s], err[%+v]", arg.TaskID, err) return err } t := completeTask.(*proto.MigrateTask) t.State = proto.MigrateStateWorkCompleted - - err = mgr.clusterMgrCli.UpdateMigrateTask(ctx, t) + task, err := t.Task() + if err != nil { + return err + } + err = mgr.clusterMgrCli.UpdateMigrateTask(ctx, task) if err != nil { // there is no impact if we failed to update task state in db, // because we will do it in finishTask again, so assume complete success span.Errorf("complete migrate task into db failed: task_id[%s], err[%+v]", t.TaskID, err) err = nil } - mgr.finishQueue.PushTask(args.TaskID, t) + mgr.finishQueue.PushTask(arg.TaskID, t) return } @@ -977,34 +1057,71 @@ func (mgr *MigrateMgr) GetMigratingDiskNum() int { // ListAllTask returns all migrate task func (mgr *MigrateMgr) ListAllTask(ctx context.Context) (tasks []*proto.MigrateTask, err error) { - return mgr.clusterMgrCli.ListAllMigrateTasks(ctx, mgr.taskType) + ts, err := mgr.clusterMgrCli.ListAllMigrateTasks(ctx, mgr.taskType) + if err != nil { + return nil, err + } + for _, t := range ts { + task := &proto.MigrateTask{} + err = task.Unmarshal(t.Data) + if err != nil { + return nil, err + } + tasks = append(tasks, task) + } + return } // ListAllTaskByDiskID return all task by diskID func (mgr *MigrateMgr) ListAllTaskByDiskID(ctx context.Context, diskID proto.DiskID) (tasks []*proto.MigrateTask, err error) { - return mgr.clusterMgrCli.ListAllMigrateTasksByDiskID(ctx, mgr.taskType, diskID) + ts, err := mgr.clusterMgrCli.ListAllMigrateTasksByDiskID(ctx, mgr.taskType, diskID) + if err != nil { + return nil, err + } + for _, t := range ts { + task := &proto.MigrateTask{} + err = task.Unmarshal(t.Data) + if err != nil { + return nil, err + } + tasks = append(tasks, task) + } + return } // GetTask returns task in db func (mgr *MigrateMgr) GetTask(ctx context.Context, taskID string) (*proto.MigrateTask, error) { - return mgr.clusterMgrCli.GetMigrateTask(ctx, mgr.taskType, taskID) + task, err := mgr.clusterMgrCli.GetMigrateTask(ctx, mgr.taskType, taskID) + if err != nil { + return nil, err + } + ret := &proto.MigrateTask{} + err = ret.Unmarshal(task.Data) + if err != nil { + return nil, err + } + return ret, nil } // QueryTask implement migrator -func (mgr *MigrateMgr) QueryTask(ctx context.Context, taskID string) (*api.MigrateTaskDetail, error) { +func (mgr *MigrateMgr) QueryTask(ctx context.Context, taskID string) (*api.TaskRet, error) { detail := &api.MigrateTaskDetail{} - taskInfo, err := mgr.GetTask(ctx, taskID) + task, err := mgr.GetTask(ctx, taskID) if err != nil { - return detail, err + return nil, err } - detail.Task = *taskInfo + detail.Task = *task detailRunInfo, err := mgr.taskStatsMgr.QueryTaskDetail(taskID) - if err != nil { - return detail, nil + if err == nil { + detail.Stat = detailRunInfo.Statistics } - detail.Stat = detailRunInfo.Statistics - return detail, nil + data, err := json.Marshal(detail) + if err != nil { + return nil, err + } + + return &api.TaskRet{TaskType: task.TaskType, Data: data}, nil } // ReportWorkerTaskStats implement migrator @@ -1033,3 +1150,18 @@ func checkMigrateConf(conf *MigrateConfig) { conf.loadTaskCallback = defaultDiskTaskLimitFunc } } + +func (mgr *MigrateMgr) ReportTask(ctx context.Context, args *api.TaskArgs) (err error) { + arg := &api.TaskReportArgs{} + err = arg.Unmarshal(args.Data) + if err != nil { + return err + } + + if !client.ValidMigrateTask(arg.TaskType, arg.TaskID) { + return errcode.ErrIllegalArguments + } + + mgr.ReportWorkerTaskStats(arg) + return nil +} diff --git a/blobstore/scheduler/migrate_test.go b/blobstore/scheduler/migrate_test.go index ee5edb678..ec7d6ac23 100644 --- a/blobstore/scheduler/migrate_test.go +++ b/blobstore/scheduler/migrate_test.go @@ -86,34 +86,34 @@ func TestMigrateMigrateLoad(t *testing.T) { require.True(t, errors.Is(err, errMock)) } { - t1 := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 4, 100, proto.MigrateStateInited, MockMigrateVolInfoMap) - t2 := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 5, 101, proto.MigrateStatePrepared, MockMigrateVolInfoMap) - t3 := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z1", 6, 102, proto.MigrateStateWorkCompleted, MockMigrateVolInfoMap) - t4 := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 4, 105, proto.MigrateStateInited, MockMigrateVolInfoMap) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t1, t2, t3, t4}, nil) + t1, _ := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 4, 100, proto.MigrateStateInited, MockMigrateVolInfoMap).Task() + t2, _ := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 5, 101, proto.MigrateStatePrepared, MockMigrateVolInfoMap).Task() + t3, _ := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z1", 6, 102, proto.MigrateStateWorkCompleted, MockMigrateVolInfoMap).Task() + t4, _ := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 4, 105, proto.MigrateStateInited, MockMigrateVolInfoMap).Task() + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{t1, t2, t3, t4}, nil) err := mgr.Load() require.NoError(t, err) } { - t1 := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z2", 8, 104, proto.MigrateStateFinished, MockMigrateVolInfoMap) - t2 := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 4, 105, proto.MigrateStateFinishedInAdvance, MockMigrateVolInfoMap) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t1}, nil) + t1, _ := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z2", 8, 104, proto.MigrateStateFinished, MockMigrateVolInfoMap).Task() + t2, _ := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 4, 105, proto.MigrateStateFinishedInAdvance, MockMigrateVolInfoMap).Task() + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{t1}, nil) err := mgr.Load() require.Error(t, err) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t2}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{t2}, nil) err = mgr.Load() require.Error(t, err) } { - t2 := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 5, 101, proto.MigrateStatePrepared, MockMigrateVolInfoMap) - t3 := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z1", 6, 101, proto.MigrateStateWorkCompleted, MockMigrateVolInfoMap) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t2, t3}, nil) + t2, _ := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 5, 101, proto.MigrateStatePrepared, MockMigrateVolInfoMap).Task() + t3, _ := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z1", 6, 101, proto.MigrateStateWorkCompleted, MockMigrateVolInfoMap).Task() + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{t2, t3}, nil) err := mgr.Load() require.Error(t, err) - t4 := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z2", 7, 103, 100, MockMigrateVolInfoMap) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t4}, nil) + t4, _ := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z2", 7, 103, 100, MockMigrateVolInfoMap).Task() + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{t4}, nil) err = mgr.Load() require.Error(t, err) } @@ -121,12 +121,12 @@ func TestMigrateMigrateLoad(t *testing.T) { mgr := newMigrateMgr(t) mgr.taskType = proto.TaskTypeDiskDrop - t1 := mockGenMigrateTask(proto.TaskTypeDiskDrop, "z0", 1, 110, proto.MigrateStateInited, MockMigrateVolInfoMap) - t2 := mockGenMigrateTask(proto.TaskTypeDiskDrop, "z0", 2, 111, proto.MigrateStatePrepared, MockMigrateVolInfoMap) - t3 := mockGenMigrateTask(proto.TaskTypeDiskDrop, "z1", 3, 112, proto.MigrateStateWorkCompleted, MockMigrateVolInfoMap) - t4 := mockGenMigrateTask(proto.TaskTypeDiskDrop, "z0", 4, 113, proto.MigrateStateInited, MockMigrateVolInfoMap) + t1, _ := mockGenMigrateTask(proto.TaskTypeDiskDrop, "z0", 1, 110, proto.MigrateStateInited, MockMigrateVolInfoMap).Task() + t2, _ := mockGenMigrateTask(proto.TaskTypeDiskDrop, "z0", 2, 111, proto.MigrateStatePrepared, MockMigrateVolInfoMap).Task() + t3, _ := mockGenMigrateTask(proto.TaskTypeDiskDrop, "z1", 3, 112, proto.MigrateStateWorkCompleted, MockMigrateVolInfoMap).Task() + t4, _ := mockGenMigrateTask(proto.TaskTypeDiskDrop, "z0", 4, 113, proto.MigrateStateInited, MockMigrateVolInfoMap).Task() - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t1, t2, t3, t4}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{t1, t2, t3, t4}, nil) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDropDisks(any).Return([]*client.DiskInfoSimple{testDisk1, testDisk2}, nil) err := mgr.Load() require.NoError(t, err) @@ -390,7 +390,8 @@ func TestCancelMigrateTask(t *testing.T) { idc := "z0" { mgr := newMigrateMgr(t) - err := mgr.CancelTask(ctx, &api.OperateTaskArgs{IDC: idc}) + + err := mgr.CancelTask(ctx, &api.TaskArgs{}) require.Error(t, err) } { @@ -399,10 +400,10 @@ func TestCancelMigrateTask(t *testing.T) { mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1) // no such task - err := mgr.CancelTask(ctx, &api.OperateTaskArgs{IDC: idc}) + err := mgr.CancelTask(ctx, &api.TaskArgs{}) require.Error(t, err) - - err = mgr.CancelTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + taskArgs := generateTaskArgs(t1, "") + err = mgr.CancelTask(ctx, taskArgs) require.NoError(t, err) } } @@ -413,22 +414,33 @@ func TestReclaimMigrateTask(t *testing.T) { { // no task mgr := newMigrateMgr(t) - err := mgr.ReclaimTask(ctx, idc, "", nil, proto.VunitLocation{}, &client.AllocVunitInfo{}) + err := mgr.ReclaimTask(ctx, &api.TaskArgs{}) require.Error(t, err) } { mgr := newMigrateMgr(t) t1 := mockGenMigrateTask(proto.TaskTypeManualMigrate, idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap) + location := t1.Destination + location.Vuid += 1 mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1) // update failed + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(gomock.Any(), gomock.Any()).Return(&client.AllocVunitInfo{VunitLocation: location}, nil) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateMigrateTask(any, any).Return(errMock) - err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{}) + taskArgs := generateTaskArgs(t1, "") + err := mgr.ReclaimTask(ctx, taskArgs) require.True(t, errors.Is(err, errMock)) // update success + task, err := mgr.workQueue.Query(t1.SourceIDC, t1.TaskID) + require.NoError(t, err) + t1 = task.(*proto.MigrateTask) + taskArgs = generateTaskArgs(t1, "") + location = t1.Sources[0] + location.Vuid += 2 + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(gomock.Any(), gomock.Any()).Return(&client.AllocVunitInfo{VunitLocation: location}, nil) mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateMigrateTask(any, any).Return(nil) - err = mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{}) + err = mgr.ReclaimTask(ctx, taskArgs) require.NoError(t, err) } } @@ -439,7 +451,7 @@ func TestCompleteMigrateTask(t *testing.T) { { // no task mgr := newMigrateMgr(t) - err := mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc}) + err := mgr.CompleteTask(ctx, &api.TaskArgs{}) require.Error(t, err) } { @@ -449,18 +461,20 @@ func TestCompleteMigrateTask(t *testing.T) { // update failed mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateMigrateTask(any, any).Return(errMock) - err := mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + taskArgs := generateTaskArgs(t1, "") + err := mgr.CompleteTask(ctx, taskArgs) require.NoError(t, err) // no task in queue - err = mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination}) + err = mgr.CompleteTask(ctx, taskArgs) require.Error(t, err) // update success mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateMigrateTask(any, any).Return(nil) t2 := mockGenMigrateTask(proto.TaskTypeManualMigrate, idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap) mgr.workQueue.AddPreparedTask(idc, t2.TaskID, t2) - err = mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t2.TaskID, Src: t2.Sources, Dest: t2.Destination}) + args := generateTaskArgs(t2, "") + err = mgr.CompleteTask(ctx, args) require.NoError(t, err) } } @@ -522,18 +536,18 @@ func TestAddMigrateTask(t *testing.T) { require.Equal(t, 1, inited) require.Equal(t, 0, prepared) require.Equal(t, 0, completed) - - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t1}, nil) + task, err := t1.Task() + require.NoError(t, err) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{task}, nil) tasks, err := mgr.ListAllTask(ctx) require.NoError(t, err) require.Equal(t, 1, len(tasks)) - - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetMigrateTask(any, any, any).Return(t1, nil) - task, err := mgr.GetTask(ctx, t1.TaskID) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetMigrateTask(any, any, any).Return(task, nil) + tsk, err := mgr.GetTask(ctx, t1.TaskID) require.NoError(t, err) - require.Equal(t, t1.TaskID, task.TaskID) + require.Equal(t, t1.TaskID, tsk.TaskID) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.MigrateTask{}, nil) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.Task{}, nil) _, err = mgr.ListAllTaskByDiskID(ctx, proto.DiskID(1)) require.NoError(t, err) } @@ -607,7 +621,9 @@ func TestMigrateQueryTask(t *testing.T) { _, err := mgr.QueryTask(ctx, taskID) require.ErrorIs(t, errMock, err) - mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetMigrateTask(any, any, any).Return(&proto.MigrateTask{}, nil) + t1, err := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 4, 100, proto.MigrateStateInited, MockMigrateVolInfoMap).Task() + require.NoError(t, err) + mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetMigrateTask(any, any, any).Return(t1, nil) _, err = mgr.QueryTask(ctx, taskID) require.NoError(t, err) } diff --git a/blobstore/scheduler/scheduler_mock_test.go b/blobstore/scheduler/scheduler_mock_test.go index 6c95c3ab8..1e54f6cdc 100644 --- a/blobstore/scheduler/scheduler_mock_test.go +++ b/blobstore/scheduler/scheduler_mock_test.go @@ -196,10 +196,10 @@ func (m *MockMigrater) EXPECT() *MockMigraterMockRecorder { } // AcquireTask mocks base method. -func (m *MockMigrater) AcquireTask(arg0 context.Context, arg1 string) (proto.MigrateTask, error) { +func (m *MockMigrater) AcquireTask(arg0 context.Context, arg1 string) (*proto.Task, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "AcquireTask", arg0, arg1) - ret0, _ := ret[0].(proto.MigrateTask) + ret0, _ := ret[0].(*proto.Task) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -237,7 +237,7 @@ func (mr *MockMigraterMockRecorder) AddTask(arg0, arg1 interface{}) *gomock.Call } // CancelTask mocks base method. -func (m *MockMigrater) CancelTask(arg0 context.Context, arg1 *scheduler.OperateTaskArgs) error { +func (m *MockMigrater) CancelTask(arg0 context.Context, arg1 *scheduler.TaskArgs) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CancelTask", arg0, arg1) ret0, _ := ret[0].(error) @@ -287,7 +287,7 @@ func (mr *MockMigraterMockRecorder) Close() *gomock.Call { } // CompleteTask mocks base method. -func (m *MockMigrater) CompleteTask(arg0 context.Context, arg1 *scheduler.OperateTaskArgs) error { +func (m *MockMigrater) CompleteTask(arg0 context.Context, arg1 *scheduler.TaskArgs) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CompleteTask", arg0, arg1) ret0, _ := ret[0].(error) @@ -475,10 +475,10 @@ func (mr *MockMigraterMockRecorder) Progress(arg0 interface{}) *gomock.Call { } // QueryTask mocks base method. -func (m *MockMigrater) QueryTask(arg0 context.Context, arg1 string) (*scheduler.MigrateTaskDetail, error) { +func (m *MockMigrater) QueryTask(arg0 context.Context, arg1 string) (*scheduler.TaskRet, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "QueryTask", arg0, arg1) - ret0, _ := ret[0].(*scheduler.MigrateTaskDetail) + ret0, _ := ret[0].(*scheduler.TaskRet) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -490,17 +490,17 @@ func (mr *MockMigraterMockRecorder) QueryTask(arg0, arg1 interface{}) *gomock.Ca } // ReclaimTask mocks base method. -func (m *MockMigrater) ReclaimTask(arg0 context.Context, arg1, arg2 string, arg3 []proto.VunitLocation, arg4 proto.VunitLocation, arg5 *client.AllocVunitInfo) error { +func (m *MockMigrater) ReclaimTask(arg0 context.Context, arg1 *scheduler.TaskArgs) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ReclaimTask", arg0, arg1, arg2, arg3, arg4, arg5) + ret := m.ctrl.Call(m, "ReclaimTask", arg0, arg1) ret0, _ := ret[0].(error) return ret0 } // ReclaimTask indicates an expected call of ReclaimTask. -func (mr *MockMigraterMockRecorder) ReclaimTask(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call { +func (mr *MockMigraterMockRecorder) ReclaimTask(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReclaimTask", reflect.TypeOf((*MockMigrater)(nil).ReclaimTask), arg0, arg1, arg2, arg3, arg4, arg5) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReclaimTask", reflect.TypeOf((*MockMigrater)(nil).ReclaimTask), arg0, arg1) } // RenewalTask mocks base method. @@ -517,6 +517,20 @@ func (mr *MockMigraterMockRecorder) RenewalTask(arg0, arg1, arg2 interface{}) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RenewalTask", reflect.TypeOf((*MockMigrater)(nil).RenewalTask), arg0, arg1, arg2) } +// ReportTask mocks base method. +func (m *MockMigrater) ReportTask(arg0 context.Context, arg1 *scheduler.TaskArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReportTask", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// ReportTask indicates an expected call of ReportTask. +func (mr *MockMigraterMockRecorder) ReportTask(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportTask", reflect.TypeOf((*MockMigrater)(nil).ReportTask), arg0, arg1) +} + // ReportWorkerTaskStats mocks base method. func (m *MockMigrater) ReportWorkerTaskStats(arg0 *scheduler.TaskReportArgs) { m.ctrl.T.Helper() diff --git a/blobstore/scheduler/service.go b/blobstore/scheduler/service.go index ea4605119..d1c244966 100644 --- a/blobstore/scheduler/service.go +++ b/blobstore/scheduler/service.go @@ -53,7 +53,7 @@ type Service struct { clusterMgrCli client.ClusterMgrAPI } -func (svr *Service) mgrByType(typ proto.TaskType) (Migrator, error) { +func (svr *Service) mgrByType(typ proto.TaskType) (BaseMigrator, error) { switch typ { case proto.TaskTypeDiskRepair: return svr.diskRepairMgr, nil @@ -63,6 +63,15 @@ func (svr *Service) mgrByType(typ proto.TaskType) (Migrator, error) { return svr.diskDropMgr, nil case proto.TaskTypeManualMigrate: return svr.manualMigMgr, nil + case proto.TaskTypeShardDiskRepair: + // todo + return nil, nil + case proto.TaskTypeShardInspect: + return nil, errIllegalTaskType + case proto.TaskTypeShardMigrate: + return nil, errIllegalTaskType + case proto.TaskTypeShardDiskDrop: + return nil, errIllegalTaskType default: return nil, errIllegalTaskType } @@ -89,7 +98,7 @@ func (svr *Service) HTTPTaskAcquire(c *rpc.Context) { // acquire task ordered: returns disk repair task first and other random ctx := c.Request.Context() - migrators := []Migrator{svr.diskRepairMgr, svr.manualMigMgr, svr.diskDropMgr, svr.balanceMgr} + migrators := []BaseMigrator{svr.diskRepairMgr, svr.manualMigMgr, svr.diskDropMgr, svr.balanceMgr} shuffledMigrators := migrators[1:] rand.Shuffle(len(shuffledMigrators), func(i, j int) { shuffledMigrators[i], shuffledMigrators[j] = shuffledMigrators[j], shuffledMigrators[i] @@ -105,15 +114,12 @@ func (svr *Service) HTTPTaskAcquire(c *rpc.Context) { // HTTPTaskReclaim reclaim task func (svr *Service) HTTPTaskReclaim(c *rpc.Context) { - args := new(api.OperateTaskArgs) + args := new(api.TaskArgs) if err := c.ParseArgs(args); err != nil { c.RespondError(err) return } - if !client.ValidMigrateTask(args.TaskType, args.TaskID) { - c.RespondError(errcode.ErrIllegalArguments) - return - } + ctx := c.Request.Context() reclaimer, err := svr.mgrByType(args.TaskType) if err != nil { @@ -121,30 +127,16 @@ func (svr *Service) HTTPTaskReclaim(c *rpc.Context) { return } - if int(args.Dest.Vuid.Index()) >= len(args.Src) || args.Dest.Vuid.Index() != args.Src[args.Dest.Vuid.Index()].Vuid.Index() { - c.RespondError(errcode.ErrIllegalArguments) - return - } - - newDst, err := base.AllocVunitSafe(ctx, svr.clusterMgrCli, args.Src[args.Dest.Vuid.Index()].Vuid, args.Src) - if err != nil { - c.RespondError(err) - return - } - c.RespondError(reclaimer.ReclaimTask(ctx, args.IDC, args.TaskID, args.Src, args.Dest, newDst)) + c.RespondError(reclaimer.ReclaimTask(ctx, args)) } // HTTPTaskCancel cancel task func (svr *Service) HTTPTaskCancel(c *rpc.Context) { - args := new(api.OperateTaskArgs) + args := new(api.TaskArgs) if err := c.ParseArgs(args); err != nil { c.RespondError(err) return } - if !client.ValidMigrateTask(args.TaskType, args.TaskID) { - c.RespondError(errcode.ErrIllegalArguments) - return - } ctx := c.Request.Context() canceler, err := svr.mgrByType(args.TaskType) if err != nil { @@ -156,15 +148,11 @@ func (svr *Service) HTTPTaskCancel(c *rpc.Context) { // HTTPTaskComplete complete task func (svr *Service) HTTPTaskComplete(c *rpc.Context) { - args := new(api.OperateTaskArgs) + args := new(api.TaskArgs) if err := c.ParseArgs(args); err != nil { c.RespondError(err) return } - if !client.ValidMigrateTask(args.TaskType, args.TaskID) { - c.RespondError(errcode.ErrIllegalArguments) - return - } ctx := c.Request.Context() completer, err := svr.mgrByType(args.TaskType) if err != nil { @@ -178,9 +166,9 @@ func (svr *Service) HTTPTaskComplete(c *rpc.Context) { func (svr *Service) HTTPInspectAcquire(c *rpc.Context) { ctx := c.Request.Context() - task, _ := svr.inspectMgr.AcquireInspect(ctx) - if task != nil { - c.RespondJSON(task) + inspectTask, _ := svr.inspectMgr.AcquireInspect(ctx) + if inspectTask != nil { + c.RespondJSON(inspectTask) return } c.RespondError(errcode.ErrNothingTodo) @@ -236,26 +224,25 @@ func (svr *Service) HTTPTaskRenewal(c *rpc.Context) { // HTTPTaskReport reports task stats func (svr *Service) HTTPTaskReport(c *rpc.Context) { - args := new(api.TaskReportArgs) + args := new(api.TaskArgs) if err := c.ParseArgs(args); err != nil { c.RespondError(err) return } - if !client.ValidMigrateTask(args.TaskType, args.TaskID) { - c.RespondError(errcode.ErrIllegalArguments) - return - } reporter, err := svr.mgrByType(args.TaskType) if err != nil { c.RespondError(err) return } - reporter.ReportWorkerTaskStats(args) + if err := reporter.ReportTask(c.Request.Context(), args); err != nil { + c.RespondError(err) + return + } c.Respond() } -// HTTPMigrateTaskDetail returns migrate task detail. -func (svr *Service) HTTPMigrateTaskDetail(c *rpc.Context) { +// HTTPTaskDetail returns migrate task detail. +func (svr *Service) HTTPTaskDetail(c *rpc.Context) { args := new(api.MigrateTaskDetailArgs) if err := c.ParseArgs(args); err != nil { c.RespondError(err) @@ -276,7 +263,7 @@ func (svr *Service) HTTPMigrateTaskDetail(c *rpc.Context) { c.RespondError(rpc.NewError(http.StatusNotFound, "NotFound", err)) return } - c.RespondJSON(detail) + c.RespondWith(http.StatusOK, rpc.MIMEJSON, detail.Data) } // HTTPDiskMigratingStats returns disk migrating stats diff --git a/blobstore/scheduler/service_test.go b/blobstore/scheduler/service_test.go index 3f2491232..a0805978e 100644 --- a/blobstore/scheduler/service_test.go +++ b/blobstore/scheduler/service_test.go @@ -58,20 +58,16 @@ func newMockService(t *testing.T) *Service { clusterTopology := NewMockClusterTopology(ctr) // return disk repair task - diskRepairMgr.EXPECT().AcquireTask(any, any).Return(proto.MigrateTask{TaskType: proto.TaskTypeDiskRepair}, nil) + diskRepairMgr.EXPECT().AcquireTask(any, any).Return(&proto.Task{TaskType: proto.TaskTypeDiskRepair}, nil) // reclaim repair task - diskRepairMgr.EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil) - clusterMgrCli.EXPECT().AllocVolumeUnit(any, any).Return(&client.AllocVunitInfo{}, nil) + diskRepairMgr.EXPECT().ReclaimTask(any, any).Return(nil) // reclaim balance task - balanceMgr.EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil) - clusterMgrCli.EXPECT().AllocVolumeUnit(any, any).Return(&client.AllocVunitInfo{}, nil) + balanceMgr.EXPECT().ReclaimTask(any, any).Return(nil) // reclaim disk drop task - diskDropMgr.EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil) - clusterMgrCli.EXPECT().AllocVolumeUnit(any, any).Return(&client.AllocVunitInfo{}, nil) + diskDropMgr.EXPECT().ReclaimTask(any, any).Return(nil) // reclaim manual migrate task - manualMgr.EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil) - clusterMgrCli.EXPECT().AllocVolumeUnit(any, any).Return(&client.AllocVunitInfo{}, nil) + manualMgr.EXPECT().ReclaimTask(any, any).Return(nil) // cancel repair task diskRepairMgr.EXPECT().CancelTask(any, any).Return(nil) @@ -101,13 +97,13 @@ func newMockService(t *testing.T) *Service { manualMgr.EXPECT().RenewalTask(any, any, any).Times(3).Return(nil) // report repair task - diskRepairMgr.EXPECT().ReportWorkerTaskStats(any).Return() + diskRepairMgr.EXPECT().ReportTask(any, any).Return(nil) // report balance task - balanceMgr.EXPECT().ReportWorkerTaskStats(any).Return() + balanceMgr.EXPECT().ReportTask(any, any).Return(nil) // report disk drop task - diskDropMgr.EXPECT().ReportWorkerTaskStats(any).Return() + diskDropMgr.EXPECT().ReportTask(any, any).Return(nil) // report manual migrate task - manualMgr.EXPECT().ReportWorkerTaskStats(any).Return() + manualMgr.EXPECT().ReportTask(any, any).Return(nil) // add manual migrate task manualMgr.EXPECT().AddManualTask(any, any, any).Return(nil) @@ -142,10 +138,10 @@ func newMockService(t *testing.T) *Service { inspectorMgr.EXPECT().Enabled().Return(true) // task detail - balanceMgr.EXPECT().QueryTask(any, any).Return(nil, nil) - diskDropMgr.EXPECT().QueryTask(any, any).Return(nil, nil) - diskRepairMgr.EXPECT().QueryTask(any, any).Return(nil, nil) - manualMgr.EXPECT().QueryTask(any, any).Return(nil, nil) + balanceMgr.EXPECT().QueryTask(any, any).Return(&api.TaskRet{TaskType: proto.TaskTypeBalance}, nil) + diskDropMgr.EXPECT().QueryTask(any, any).Return(&api.TaskRet{TaskType: proto.TaskTypeShardDiskDrop}, nil) + diskRepairMgr.EXPECT().QueryTask(any, any).Return(&api.TaskRet{TaskType: proto.TaskTypeDiskRepair}, nil) + manualMgr.EXPECT().QueryTask(any, any).Return(&api.TaskRet{TaskType: proto.TaskTypeManualMigrate}, nil) balanceMgr.EXPECT().QueryTask(any, any).Return(nil, errMock) diskDropMgr.EXPECT().QueryTask(any, any).Return(nil, errMock) diskRepairMgr.EXPECT().QueryTask(any, any).Return(nil, errMock) @@ -198,18 +194,23 @@ func TestServiceAPI(t *testing.T) { require.Equal(t, proto.TaskTypeDiskRepair, task.TaskType) for _, taskType := range taskTypes { - require.NoError(t, cli.ReclaimTask(ctx, &api.OperateTaskArgs{ - IDC: idc, TaskType: taskType, TaskID: client.GenMigrateTaskID(taskType, diskID, volumeID), - Src: []proto.VunitLocation{{Vuid: 1, Host: "127.0.0.1:xx", DiskID: 1}}, - })) - require.NoError(t, cli.CancelTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskType: taskType, TaskID: client.GenMigrateTaskID(taskType, diskID, volumeID)})) - require.NoError(t, cli.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskType: taskType, TaskID: client.GenMigrateTaskID(taskType, diskID, volumeID)})) - require.NoError(t, cli.ReportTask(ctx, &api.TaskReportArgs{TaskType: taskType, TaskID: client.GenMigrateTaskID(taskType, diskID, volumeID)})) + taskArgs, err := (&api.OperateTaskArgs{ + IDC: idc, TaskType: taskType, + TaskID: client.GenMigrateTaskID(taskType, diskID, volumeID), + }).TaskArgs() + require.NoError(t, err) + require.NoError(t, cli.ReclaimTask(ctx, taskArgs)) + require.NoError(t, cli.CancelTask(ctx, taskArgs)) + require.NoError(t, cli.CompleteTask(ctx, taskArgs)) + args, err := (&api.TaskReportArgs{TaskType: taskType, TaskID: client.GenMigrateTaskID(taskType, diskID, volumeID)}).TaskArgs() + require.NoError(t, err) + require.NoError(t, cli.ReportTask(ctx, args)) } - - require.Error(t, cli.ReclaimTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskType: "task"})) - require.Error(t, cli.CancelTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskType: "task"})) - require.Error(t, cli.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskType: "task"})) + taskArgs, err := (&api.OperateTaskArgs{IDC: idc, TaskType: "task"}).TaskArgs() + require.NoError(t, err) + require.Error(t, cli.ReclaimTask(ctx, taskArgs)) + require.Error(t, cli.CancelTask(ctx, taskArgs)) + require.Error(t, cli.CompleteTask(ctx, taskArgs)) // renewal task _, err = cli.RenewalTask(ctx, &api.TaskRenewalArgs{ @@ -288,11 +289,4 @@ func TestServiceAPI(t *testing.T) { require.Equal(t, int(testDisk1.UsedChunkCnt), stats.TotalTasksCnt) require.Equal(t, 1, stats.MigratedTasksCnt) } - - // reclaim failed - err = cli.ReclaimTask(ctx, &api.OperateTaskArgs{ - IDC: idc, TaskType: proto.TaskTypeDiskRepair, - TaskID: client.GenMigrateTaskID(proto.TaskTypeDiskRepair, diskID, volumeID), - }) - require.Error(t, err) } diff --git a/blobstore/scheduler/shard_migrate.go b/blobstore/scheduler/shard_migrate.go new file mode 100644 index 000000000..ab9d46859 --- /dev/null +++ b/blobstore/scheduler/shard_migrate.go @@ -0,0 +1,223 @@ +// Copyright 2024 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package scheduler + +import ( + "context" + "errors" + "time" + + api "github.com/cubefs/cubefs/blobstore/api/scheduler" + "github.com/cubefs/cubefs/blobstore/common/counter" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/taskswitch" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/scheduler/base" + "github.com/cubefs/cubefs/blobstore/scheduler/client" + "github.com/cubefs/cubefs/blobstore/util/closer" +) + +// ShardMigrateConfig migrate config +type ShardMigrateConfig struct { + ClusterID proto.ClusterID `json:"-"` // fill in config.go + base.TaskCommonConfig +} + +type ShardMigrator interface { + BaseMigrator + + Stats() api.ShardTaskStat + + taskswitch.ISwitcher + closer.Closer + Load() error + Run() +} + +type ShardMigrateMgr struct { + closer.Closer + + taskType proto.TaskType + taskSwitch taskswitch.ISwitcher + + prepareQueue *base.TaskQueue // store inited task + workQueue *base.ShardTaskQueue // store prepared task + finishQueue *base.TaskQueue // store completed task + + clusterMgrCli client.ClusterMgrAPI + + finishTaskCounter counter.Counter + taskStatsMgr *base.TaskStatsMgr + + cfg *ShardMigrateConfig +} + +func (mgr *ShardMigrateMgr) AcquireTask(ctx context.Context, idc string) (task *proto.Task, err error) { + span := trace.SpanFromContextSafe(ctx) + task = new(proto.Task) + task.ModuleType = proto.TypeShardNode + if !mgr.taskSwitch.Enabled() { + return task, proto.ErrTaskPaused + } + _, migTask, _ := mgr.workQueue.Acquire(idc) + if migTask != nil { + t := *migTask.(*proto.ShardMigrateTask) + data, err := t.Marshal() + if err != nil { + return task, err + } + task.Data = data + span.Infof("acquire %s taskId: %s", mgr.taskType, t.TaskID) + return task, nil + } + return task, proto.ErrTaskEmpty +} + +func (mgr *ShardMigrateMgr) CancelTask(ctx context.Context, args *api.TaskArgs) error { + mgr.taskStatsMgr.CancelTask() + + arg := &api.ShardTaskArgs{} + err := arg.Unmarshal(args.Data) + if err != nil { + return err + } + if !client.ValidMigrateTask(args.TaskType, arg.TaskID) { + return errcode.ErrIllegalArguments + } + + err = mgr.workQueue.Cancel(arg.IDC, arg.TaskID, arg.Source, arg.Dest) + if err != nil { + span := trace.SpanFromContextSafe(ctx) + span.Errorf("cancel shard migrate failed: task_type[%s], task_id[%s], err[%+v]", mgr.taskType, arg.TaskID, err) + } + return nil +} + +func (mgr *ShardMigrateMgr) CompleteTask(ctx context.Context, args *api.TaskArgs) error { + span := trace.SpanFromContextSafe(ctx) + + arg := &api.ShardTaskArgs{} + err := arg.Unmarshal(args.Data) + if err != nil { + return err + } + if !client.ValidMigrateTask(args.TaskType, arg.TaskID) { + return errcode.ErrIllegalArguments + } + + completeTask, err := mgr.workQueue.Complete(arg.IDC, arg.TaskID, arg.Source, arg.Dest) + if err != nil { + span.Errorf("complete migrate task failed: task_id[%s], err[%+v]", arg.TaskID, err) + return err + } + + t := completeTask.(*proto.ShardMigrateTask) + t.State = proto.ShardTaskStateWorkCompleted + task, err := t.Task() + if err != nil { + return err + } + err = mgr.clusterMgrCli.UpdateMigrateTask(ctx, task) + if err != nil { + // there is no impact if we failed to update task state in db, + // because we will do it in finishTask again, so assume complete success + span.Errorf("complete migrate task into db failed: task_id[%s], err[%+v]", t.TaskID, err) + err = nil + } + mgr.finishQueue.PushTask(arg.TaskID, t) + return nil +} + +func (mgr *ShardMigrateMgr) ReclaimTask(ctx context.Context, args *api.TaskArgs) error { + return nil +} + +func (mgr *ShardMigrateMgr) RenewalTask(ctx context.Context, idc, taskID string) error { + return nil +} + +func (mgr *ShardMigrateMgr) QueryTask(ctx context.Context, taskID string) (*api.TaskRet, error) { + return nil, nil +} + +func (mgr *ShardMigrateMgr) ReportTask(ctx context.Context, args *api.TaskArgs) (err error) { + return nil +} + +func (mgr *ShardMigrateMgr) Stats() api.ShardTaskStat { + return api.ShardTaskStat{} +} + +// Enabled task switch +func (mgr *ShardMigrateMgr) Enabled() bool { + return mgr.taskSwitch.Enabled() +} + +func (mgr *ShardMigrateMgr) WaitEnable() { + mgr.taskSwitch.WaitEnable() +} + +func (mgr *ShardMigrateMgr) Close() { + mgr.Closer.Close() +} + +// Done returns a channel that's closed when object was closed. +func (mgr *ShardMigrateMgr) Done() <-chan struct{} { + return mgr.Closer.Done() +} + +func (mgr *ShardMigrateMgr) Load() error { + return nil +} + +func (mgr *ShardMigrateMgr) Run() { + go mgr.prepareTaskLoop() + go mgr.finishTaskLoop() +} + +func (mgr *ShardMigrateMgr) prepareTaskLoop() { + go func() { + for { + mgr.taskSwitch.WaitEnable() + todo, doing := mgr.workQueue.StatsTasks() + if todo+doing >= mgr.cfg.WorkQueueSize { + time.Sleep(prepareTaskPause) + continue + } + err := mgr.prepareTask() + if errors.Is(err, base.ErrNoTaskInQueue) { + time.Sleep(prepareMigrateTaskInterval) + } + } + }() +} + +func (mgr *ShardMigrateMgr) finishTaskLoop() { + for { + err := mgr.finishTask() + if errors.Is(err, base.ErrNoTaskInQueue) { + time.Sleep(finishMigrateTaskInterval) + } + } +} + +func (mgr *ShardMigrateMgr) prepareTask() error { + return nil +} + +func (mgr *ShardMigrateMgr) finishTask() error { + return nil +} diff --git a/blobstore/scheduler/startup.go b/blobstore/scheduler/startup.go index 89462033f..912035f61 100644 --- a/blobstore/scheduler/startup.go +++ b/blobstore/scheduler/startup.go @@ -372,7 +372,7 @@ func NewHandler(service *Service) *rpc.Router { rpc.POST(api.PathTaskReport, service.HTTPTaskReport, rpc.OptArgsBody()) rpc.POST(api.PathTaskRenewal, service.HTTPTaskRenewal, rpc.OptArgsBody()) - rpc.GET(api.PathTaskDetailURI, service.HTTPMigrateTaskDetail, rpc.OptArgsURI()) + rpc.GET(api.PathTaskDetailURI, service.HTTPTaskDetail, rpc.OptArgsURI()) rpc.GET(api.PathStats, service.HTTPStats, rpc.OptArgsQuery()) rpc.GET(api.PathStatsLeader, service.HTTPStats, rpc.OptArgsQuery()) rpc.GET(api.PathStatsDiskMigrating, service.HTTPDiskMigratingStats, rpc.OptArgsQuery()) diff --git a/blobstore/scheduler/startup_test.go b/blobstore/scheduler/startup_test.go index 225d93d8e..eb8867ab5 100644 --- a/blobstore/scheduler/startup_test.go +++ b/blobstore/scheduler/startup_test.go @@ -148,10 +148,10 @@ func newMockServiceWithOpts(ctr *gomock.Controller, isLeader bool) *Service { volumeUpdater.EXPECT().UpdateFollowerVolumeCache(any, any, any).AnyTimes().Return(nil) volumeUpdater.EXPECT().UpdateLeaderVolumeCache(any, any).AnyTimes().Return(nil) - manualMgr.EXPECT().AcquireTask(any, any).AnyTimes().Return(proto.MigrateTask{TaskType: proto.TaskTypeManualMigrate}, nil) - diskRepairMgr.EXPECT().AcquireTask(any, any).AnyTimes().Return(proto.MigrateTask{}, errMock) - diskDropMgr.EXPECT().AcquireTask(any, any).AnyTimes().Return(proto.MigrateTask{}, errMock) - balanceMgr.EXPECT().AcquireTask(any, any).AnyTimes().Return(proto.MigrateTask{}, errMock) + manualMgr.EXPECT().AcquireTask(any, any).AnyTimes().Return(&proto.Task{TaskType: proto.TaskTypeManualMigrate}, nil) + diskRepairMgr.EXPECT().AcquireTask(any, any).AnyTimes().Return(&proto.Task{}, errMock) + diskDropMgr.EXPECT().AcquireTask(any, any).AnyTimes().Return(&proto.Task{}, errMock) + balanceMgr.EXPECT().AcquireTask(any, any).AnyTimes().Return(&proto.Task{}, errMock) clusterTopology.EXPECT().UpdateVolume(any).AnyTimes().Return(&client.VolumeInfoSimple{}, nil) clusterMgrCli.EXPECT().GetConfig(any, any).AnyTimes().Return("", errMock) diff --git a/blobstore/testing/mocks/api_scheduler.go b/blobstore/testing/mocks/api_scheduler.go index 0e9352f0e..ecaef9c65 100644 --- a/blobstore/testing/mocks/api_scheduler.go +++ b/blobstore/testing/mocks/api_scheduler.go @@ -52,10 +52,10 @@ func (mr *MockISchedulerMockRecorder) AcquireInspectTask(arg0 interface{}) *gomo } // AcquireTask mocks base method. -func (m *MockIScheduler) AcquireTask(arg0 context.Context, arg1 *scheduler.AcquireArgs) (*proto.MigrateTask, error) { +func (m *MockIScheduler) AcquireTask(arg0 context.Context, arg1 *scheduler.AcquireArgs) (*proto.Task, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "AcquireTask", arg0, arg1) - ret0, _ := ret[0].(*proto.MigrateTask) + ret0, _ := ret[0].(*proto.Task) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -81,7 +81,7 @@ func (mr *MockISchedulerMockRecorder) AddManualMigrateTask(arg0, arg1 interface{ } // CancelTask mocks base method. -func (m *MockIScheduler) CancelTask(arg0 context.Context, arg1 *scheduler.OperateTaskArgs) error { +func (m *MockIScheduler) CancelTask(arg0 context.Context, arg1 *scheduler.TaskArgs) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CancelTask", arg0, arg1) ret0, _ := ret[0].(error) @@ -109,7 +109,7 @@ func (mr *MockISchedulerMockRecorder) CompleteInspectTask(arg0, arg1 interface{} } // CompleteTask mocks base method. -func (m *MockIScheduler) CompleteTask(arg0 context.Context, arg1 *scheduler.OperateTaskArgs) error { +func (m *MockIScheduler) CompleteTask(arg0 context.Context, arg1 *scheduler.TaskArgs) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CompleteTask", arg0, arg1) ret0, _ := ret[0].(error) @@ -168,7 +168,7 @@ func (mr *MockISchedulerMockRecorder) LeaderStats(arg0 interface{}) *gomock.Call } // ReclaimTask mocks base method. -func (m *MockIScheduler) ReclaimTask(arg0 context.Context, arg1 *scheduler.OperateTaskArgs) error { +func (m *MockIScheduler) ReclaimTask(arg0 context.Context, arg1 *scheduler.TaskArgs) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ReclaimTask", arg0, arg1) ret0, _ := ret[0].(error) @@ -197,7 +197,7 @@ func (mr *MockISchedulerMockRecorder) RenewalTask(arg0, arg1 interface{}) *gomoc } // ReportTask mocks base method. -func (m *MockIScheduler) ReportTask(arg0 context.Context, arg1 *scheduler.TaskReportArgs) error { +func (m *MockIScheduler) ReportTask(arg0 context.Context, arg1 *scheduler.TaskArgs) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ReportTask", arg0, arg1) ret0, _ := ret[0].(error)