gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 02:00:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(scheduler): add shard node task

Browse Source 
1. Update the scheduler and worker interaction protocols to support new
task types

with: #22417838

Signed-off-by: JasonHu520 <huzongchao@oppo.com>
This commit is contained in:
JasonHu520 2024-07-24 19:55:31 +08:00 committed by slasher
parent afaef8a55c
commit 19e3ec09e4
33 changed files with 1734 additions and 440 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
blobstore/api/scheduler/client.go
View File

@ -51,12 +51,12 @@ var errNoServiceAvailable = errors.New("no service available")
// IMigrator scheduler migrate task.
type IMigrator interface {
AcquireTask(ctx context.Context, args *AcquireArgs) (ret *proto.MigrateTask, err error)
AcquireTask(ctx context.Context, args *AcquireArgs) (ret *proto.Task, err error)
RenewalTask(ctx context.Context, args *TaskRenewalArgs) (ret *TaskRenewalRet, err error)
ReportTask(ctx context.Context, args *TaskReportArgs) (err error)
ReclaimTask(ctx context.Context, args *OperateTaskArgs) (err error)
CancelTask(ctx context.Context, args *OperateTaskArgs) (err error)
CompleteTask(ctx context.Context, args *OperateTaskArgs) (err error)
ReportTask(ctx context.Context, args *TaskArgs) (err error)
ReclaimTask(ctx context.Context, args *TaskArgs) (err error)
CancelTask(ctx context.Context, args *TaskArgs) (err error)
CompleteTask(ctx context.Context, args *TaskArgs) (err error)
}
// IInspector volume inspect task.

94
blobstore/api/scheduler/task.go
View File

@ -16,6 +16,7 @@ package scheduler
import (
"context"
"encoding/json"
"fmt"
"net/url"
@ -27,9 +28,22 @@ type AcquireArgs struct {
IDC string `json:"idc"`
}
func (c *client) AcquireTask(ctx context.Context, args *AcquireArgs) (ret *proto.MigrateTask, err error) {
type TaskArgs struct {
ModuleType proto.ModuleType `json:"module_type"`
TaskType proto.TaskType `json:"task_type"`
Data []byte `json:"data"`
}
// TaskRet use for query task info
type TaskRet struct {
TaskType proto.TaskType `json:"task_type"`
Data []byte `json:"data"`
}
func (c *client) AcquireTask(ctx context.Context, args *AcquireArgs) (ret *proto.Task, err error) {
ret = new(proto.Task)
err = c.request(func(host string) error {
return c.GetWith(ctx, host+PathTaskAcquire+"?idc="+args.IDC, &ret)
return c.GetWith(ctx, host+PathTaskAcquire+"?idc="+args.IDC, ret)
})
return
}
@ -59,13 +73,33 @@ type TaskReportArgs struct {
IncreaseShardCnt int `json:"increase_shard_cnt"`
}
func (c *client) ReportTask(ctx context.Context, args *TaskReportArgs) (err error) {
func (t *TaskReportArgs) Unmarshal(data []byte) error {
return json.Unmarshal(data, t)
}
func (t *TaskReportArgs) Marshal() (data []byte, err error) {
return json.Marshal(t)
}
func (t *TaskReportArgs) TaskArgs() (*TaskArgs, error) {
ret := new(TaskArgs)
ret.ModuleType = proto.TypeBlobNode
ret.TaskType = t.TaskType
data, err := t.Marshal()
if err != nil {
return nil, err
}
ret.Data = data
return ret, nil
}
func (c *client) ReportTask(ctx context.Context, args *TaskArgs) (err error) {
return c.request(func(host string) error {
return c.PostWith(ctx, host+PathTaskReport, nil, args)
})
}
// OperateTaskArgs for task action.
// OperateTaskArgs for blobnode task action.
type OperateTaskArgs struct {
IDC string `json:"idc"`
TaskID string `json:"task_id"`
@ -75,19 +109,39 @@ type OperateTaskArgs struct {
Reason string `json:"reason"`
}
func (c *client) ReclaimTask(ctx context.Context, args *OperateTaskArgs) (err error) {
func (t *OperateTaskArgs) Unmarshal(data []byte) error {
return json.Unmarshal(data, t)
}
func (t *OperateTaskArgs) Marshal() (data []byte, err error) {
return json.Marshal(t)
}
func (t *OperateTaskArgs) TaskArgs() (*TaskArgs, error) {
ret := new(TaskArgs)
ret.ModuleType = proto.TypeBlobNode
ret.TaskType = t.TaskType
data, err := t.Marshal()
if err != nil {
return nil, err
}
ret.Data = data
return ret, nil
}
func (c *client) ReclaimTask(ctx context.Context, args *TaskArgs) (err error) {
return c.request(func(host string) error {
return c.PostWith(ctx, host+PathTaskReclaim, nil, args)
})
}
func (c *client) CancelTask(ctx context.Context, args *OperateTaskArgs) (err error) {
func (c *client) CancelTask(ctx context.Context, args *TaskArgs) (err error) {
return c.request(func(host string) error {
return c.PostWith(ctx, host+PathTaskCancel, nil, args)
})
}
func (c *client) CompleteTask(ctx context.Context, args *OperateTaskArgs) (err error) {
func (c *client) CompleteTask(ctx context.Context, args *TaskArgs) (err error) {
return c.request(func(host string) error {
return c.PostWith(ctx, host+PathTaskComplete, nil, args)
})
@ -154,6 +208,12 @@ type MigrateTasksStat struct {
StatsPerMin PerMinStats `json:"stats_per_min"`
}
type ShardTaskStat struct {
PreparingCnt int `json:"preparing_cnt"`
WorkerDoingCnt int `json:"worker_doing_cnt"`
FinishingCnt int `json:"finishing_cnt"`
}
type DiskDropTasksStat struct {
Enable bool `json:"enable"`
DroppingDisks []proto.DiskID `json:"dropping_disks"`
@ -272,3 +332,23 @@ func hostWithScheme(host string) string {
}
return "http://" + host
}
// ShardTaskArgs for shard node task action.
type ShardTaskArgs struct {
IDC string `json:"idc"`
TaskID string `json:"task_id"`
TaskType proto.TaskType `json:"task_type"`
Source proto.SunitLocation `json:"source"`
Dest proto.SunitLocation `json:"dest"`
Leader proto.SunitLocation `json:"leader"`
Learner bool `json:"learner"`
Reason string `json:"reason"`
}
func (t *ShardTaskArgs) Unmarshal(data []byte) error {
return json.Unmarshal(data, t)
}
func (t *ShardTaskArgs) Marshal() (data []byte, err error) {
return json.Marshal(t)
}

55
blobstore/blobnode/client/blobnode_cli.go
View File

@ -16,6 +16,7 @@ package client
import (
"context"
"github.com/cubefs/cubefs/blobstore/common/sharding"
"io"
"strings"
@ -38,6 +39,60 @@ type IBlobNode interface {
PutShard(ctx context.Context, location proto.VunitLocation, bid proto.BlobID, size int64, body io.Reader, ioType api.IOType) (err error)
}
type ShardStatusRet struct {
}
type ShardUnitInfo struct {
Suid proto.Suid `json:"suid"`
DiskID proto.DiskID `json:"disk_id"`
Learner bool `json:"learner"`
}
type UpdateShardArgs struct {
ShardID proto.ShardID `json:"shard_id"`
DiskID proto.DiskID `json:"disk_id"`
Units []*ShardUnitInfo `json:"units"`
}
type SubRange struct {
Min int64 `json:"min"`
Max int64 `json:"max"`
}
type Range struct {
RangeType sharding.RangeType `json:"range_type"`
SubRanges []SubRange `json:"sub_ranges"`
}
type AddShardArgs struct {
DiskID proto.DiskID `json:"disk_id"`
ShardID proto.ShardID `json:"shard_id"`
Range Range
Units []*ShardUnitInfo
Epoch uint64
}
// IShardNode ShardNode client
type IShardNode interface {
UpdateShard(ctx context.Context, args *UpdateShardArgs) error
GetShardStatus(ctx context.Context, id proto.ShardID) (error, *ShardStatusRet)
AdShard(ctx context.Context, args *AddShardArgs) error
}
type ShardNodeClient struct {
}
func (c *ShardNodeClient) UpdateShard(ctx context.Context, args *UpdateShardArgs) error {
return nil
}
func (c *ShardNodeClient) GetShardStatus(ctx context.Context, id proto.ShardID) (error, *ShardStatusRet) {
return nil, nil
}
func (c *ShardNodeClient) AdShard(ctx context.Context, args *AddShardArgs) error {
return nil
}
// BlobNodeClient blobnode client
type BlobNodeClient struct {
cli api.StorageAPI

59
blobstore/blobnode/task_runner.go
View File

@ -16,6 +16,7 @@ package blobnode
import (
"context"
"encoding/json"
"errors"
"fmt"
"sync"
@ -39,13 +40,13 @@ const (
var errKilled = errors.New("task killed")
// taskState task state
type taskState struct {
// TaskState task state
type TaskState struct {
sync.Mutex
state uint8
}
func (ts *taskState) set(st uint8) {
func (ts *TaskState) set(st uint8) {
ts.Lock()
defer ts.Unlock()
if ts.state >= TaskSuccess {
@ -54,13 +55,13 @@ func (ts *taskState) set(st uint8) {
ts.state = st
}
func (ts *taskState) stopped() bool {
func (ts *TaskState) stopped() bool {
ts.Lock()
defer ts.Unlock()
return ts.state >= TaskSuccess
}
func (ts *taskState) alive() bool {
func (ts *TaskState) alive() bool {
ts.Lock()
defer ts.Unlock()
return ts.state <= TaskRunning
@ -140,6 +141,15 @@ func genWorkError(err error, errType WokeErrorType) *WorkError {
return &WorkError{errType: errType, err: err}
}
type Runner interface {
Run()
Stop()
Stopped() bool
Alive() bool
TaskID() string
State() *TaskState
}
// ITaskWorker define interface used for task execution
type ITaskWorker interface {
// split tasklets accord by volume benchmark bids
@ -148,7 +158,7 @@ type ITaskWorker interface {
ExecTasklet(ctx context.Context, t Tasklet) *WorkError
// check whether the task is executed successfully when volume task finish
Check(ctx context.Context) *WorkError
OperateArgs() scheduler.OperateTaskArgs
OperateArgs(reason string) *scheduler.TaskArgs
TaskType() (taskType proto.TaskType)
GetBenchmarkBids() []*ShardInfoSimple
}
@ -174,7 +184,7 @@ type TaskRunner struct {
idc string
taskletRunConcurrency int
state taskState
state TaskState
ctx context.Context
cancel context.CancelFunc
@ -190,8 +200,7 @@ type TaskRunner struct {
// NewTaskRunner return task runner
func NewTaskRunner(ctx context.Context, taskID string, w ITaskWorker, idc string,
taskletRunConcurrency int, taskCounter *taskCounter, schedulerCli scheduler.IMigrator,
) *TaskRunner {
taskletRunConcurrency int, taskCounter *taskCounter, schedulerCli scheduler.IMigrator) Runner {
span, ctx := trace.StartSpanFromContext(ctx, "taskRunner")
ctx, cancel := context.WithCancel(ctx)
@ -289,6 +298,14 @@ func (r *TaskRunner) execTaskletWrap(ctx context.Context, t Tasklet) {
}
}
func (r *TaskRunner) TaskID() string {
return r.taskID
}
func (r *TaskRunner) State() *TaskState {
return &r.state
}
// Stop stops task
func (r *TaskRunner) Stop() {
r.state.set(TaskStopping)
@ -314,11 +331,9 @@ func (r *TaskRunner) cancelOrReclaim(retErr *WorkError) {
defer r.state.set(TaskStopped)
if ShouldReclaim(retErr) {
args := r.w.OperateArgs()
args.IDC = r.idc
args.Reason = retErr.Error()
args := r.w.OperateArgs(retErr.Error())
span.Infof("reclaim task: taskID[%s], err[%s]", r.taskID, retErr.String())
if err := r.schedulerCli.ReclaimTask(r.newCtx(), &args); err != nil {
if err := r.schedulerCli.ReclaimTask(r.newCtx(), args); err != nil {
span.Errorf("reclaim task failed: taskID[%s], args[%+v], code[%d], err[%+v]",
r.taskID, args, rpc.DetectStatusCode(err), err)
}
@ -328,10 +343,8 @@ func (r *TaskRunner) cancelOrReclaim(retErr *WorkError) {
span.Infof("cancel task: taskID[%s], err[%+v]", r.taskID, retErr)
args := r.w.OperateArgs()
args.IDC = r.idc
args.Reason = retErr.Error()
if err := r.schedulerCli.CancelTask(r.newCtx(), &args); err != nil {
args := r.w.OperateArgs(retErr.Error())
if err := r.schedulerCli.CancelTask(r.newCtx(), args); err != nil {
span.Errorf("cancel failed: taskID[%s], args[%+v], code[%d], err[%+v]",
r.taskID, args, rpc.DetectStatusCode(err), err)
}
@ -342,9 +355,8 @@ func (r *TaskRunner) completeTask() {
defer r.state.set(TaskSuccess)
r.span.Infof("complete task: taskID[%s]", r.taskID)
args := r.w.OperateArgs()
args.IDC = r.idc
if err := r.schedulerCli.CompleteTask(r.newCtx(), &args); err != nil {
args := r.w.OperateArgs("")
if err := r.schedulerCli.CompleteTask(r.newCtx(), args); err != nil {
r.span.Errorf("complete failed: taskID[%s], args[%+v], code[%d], err[%+v]",
r.taskID, args, rpc.DetectStatusCode(err), err)
}
@ -360,7 +372,12 @@ func (r *TaskRunner) statsAndReportTask(increaseDataSize, increaseShardCnt uint6
IncreaseDataSizeByte: int(increaseDataSize),
IncreaseShardCnt: int(increaseShardCnt),
}
err := r.schedulerCli.ReportTask(r.newCtx(), &reportArgs)
data, _ := json.Marshal(reportArgs)
err := r.schedulerCli.ReportTask(r.newCtx(), &scheduler.TaskArgs{
TaskType: reportArgs.TaskType,
ModuleType: proto.TypeBlobNode,
Data: data})
if err != nil {
r.span.Errorf("report task failed: taskID[%s], code[%d], err[%+v]", r.taskID, rpc.DetectStatusCode(err), err)
}

41
blobstore/blobnode/task_runner_mgr.go
View File

@ -24,6 +24,7 @@ import (
"github.com/cubefs/cubefs/blobstore/api/blobnode"
"github.com/cubefs/cubefs/blobstore/api/scheduler"
"github.com/cubefs/cubefs/blobstore/blobnode/base"
"github.com/cubefs/cubefs/blobstore/blobnode/client"
"github.com/cubefs/cubefs/blobstore/common/counter"
"github.com/cubefs/cubefs/blobstore/common/proto"
"github.com/cubefs/cubefs/blobstore/common/trace"
@ -59,10 +60,11 @@ func NewTaskRunnerMgr(idc string, meter WorkerConfigMeter, genWorker WorkerGener
) *TaskRunnerMgr {
return &TaskRunnerMgr{
typeMgr: map[proto.TaskType]mapTaskRunner{
proto.TaskTypeBalance: make(mapTaskRunner),
proto.TaskTypeDiskDrop: make(mapTaskRunner),
proto.TaskTypeDiskRepair: make(mapTaskRunner),
proto.TaskTypeManualMigrate: make(mapTaskRunner),
proto.TaskTypeBalance: make(mapTaskRunner),
proto.TaskTypeDiskDrop: make(mapTaskRunner),
proto.TaskTypeDiskRepair: make(mapTaskRunner),
proto.TaskTypeManualMigrate: make(mapTaskRunner),
proto.TaskTypeShardDiskRepair: make(mapTaskRunner),
},
idc: idc,
@ -122,7 +124,7 @@ func (tm *TaskRunnerMgr) renewalTask() {
}
}
// AddTask add migrate task.
// AddTask add blobBode migrate task.
func (tm *TaskRunnerMgr) AddTask(ctx context.Context, task MigrateTaskEx) error {
tm.mu.Lock()
defer tm.mu.Unlock()
@ -148,6 +150,25 @@ func (tm *TaskRunnerMgr) AddTask(ctx context.Context, task MigrateTaskEx) error
return nil
}
// AddShardTask add shardNode migrate task.
func (tm *TaskRunnerMgr) AddShardTask(ctx context.Context, shardClient client.IShardNode, task *proto.ShardMigrateTask) error {
tm.mu.Lock()
defer tm.mu.Unlock()
mgr, ok := tm.typeMgr[task.TaskType]
if !ok {
return fmt.Errorf("invalid task type: %s", task.TaskType)
}
w := &ShardWorker{shardNodeCli: shardClient, t: task}
runner := NewShardNodeTaskRunner(ctx, task.TaskID, w, task.SourceIDC, &tm.taskCounter, tm.schedulerCli)
if err := mgr.addTask(task.TaskID, runner); err != nil {
return err
}
go runner.Run()
return nil
}
// GetAliveTasks returns all alive migrate task.
func (tm *TaskRunnerMgr) GetAliveTasks() map[proto.TaskType][]string {
tm.mu.Lock()
@ -194,22 +215,22 @@ func (tm *TaskRunnerMgr) TaskStats() blobnode.WorkerStats {
}
}
type mapTaskRunner map[string]*TaskRunner
type mapTaskRunner map[string]Runner
func (m mapTaskRunner) eliminateStopped() mapTaskRunner {
newTasks := make(mapTaskRunner, len(m))
for taskID, task := range m {
if task.Stopped() {
log.Infof("remove stopped task: taskID[%s], state[%d]", task.taskID, task.state.state)
log.Infof("remove stopped task: taskID[%s], state[%d]", task.TaskID(), task.State().state)
continue
}
log.Debugf("remain task: taskID[%s], state[%d]", task.taskID, task.state.state)
log.Debugf("remain task: taskID[%s], state[%d]", task.TaskID(), task.State().state)
newTasks[taskID] = task
}
return newTasks
}
func (m mapTaskRunner) addTask(taskID string, runner *TaskRunner) error {
func (m mapTaskRunner) addTask(taskID string, runner Runner) error {
if r, ok := m[taskID]; ok {
if !r.Stopped() {
log.Warnf("task is running shouldn't add again: taskID[%s]", taskID)
@ -232,7 +253,7 @@ func (m mapTaskRunner) getAliveTasks() []string {
alive := make([]string, 0, 16)
for _, r := range m {
if r.Alive() {
alive = append(alive, r.taskID)
alive = append(alive, r.TaskID())
}
}
return alive

12
blobstore/blobnode/task_runner_migrate.go
View File

@ -129,13 +129,21 @@ func (w *MigrateWorker) GetBenchmarkBids() []*ShardInfoSimple {
}
// OperateArgs args for cancel, complete, reclaim.
func (w *MigrateWorker) OperateArgs() scheduler.OperateTaskArgs {
return scheduler.OperateTaskArgs{
func (w *MigrateWorker) OperateArgs(reason string) *scheduler.TaskArgs {
operateTaskArgs := &scheduler.OperateTaskArgs{
TaskID: w.t.TaskID,
TaskType: w.t.TaskType,
Src: w.t.Sources,
Dest: w.t.Destination,
Reason: reason,
IDC: w.t.SourceIDC,
}
data, _ := operateTaskArgs.Marshal()
ret := new(scheduler.TaskArgs)
ret.Data = data
ret.ModuleType = proto.TypeBlobNode
ret.TaskType = w.t.TaskType
return ret
}
// TaskType returns task type

12
blobstore/blobnode/task_runner_test.go
View File

@ -70,8 +70,8 @@ func (w *mockWorker) Check(ctx context.Context) *WorkError {
return OtherError(w.checkRetErr)
}
func (w *mockWorker) OperateArgs() scheduler.OperateTaskArgs {
return scheduler.OperateTaskArgs{TaskID: "test_mock_task", TaskType: w.TaskType()}
func (w *mockWorker) OperateArgs(reason string) *scheduler.TaskArgs {
return &scheduler.TaskArgs{ModuleType: proto.TypeShardNode, TaskType: w.TaskType(), Data: nil}
}
func (w *mockWorker) TaskType() proto.TaskType {
@ -95,19 +95,19 @@ func newMockSchedulerCli(t *testing.T, stats *mockStats) scheduler.IMigrator {
cli := mocks.NewMockIScheduler(C(t))
cli.EXPECT().ReportTask(A, A).AnyTimes().Return(nil)
cli.EXPECT().CancelTask(A, A).AnyTimes().DoAndReturn(
func(context.Context, *scheduler.OperateTaskArgs) error {
func(context.Context, *scheduler.TaskArgs) error {
stats.step = "Cancel"
stats.wg.Done()
return errors.New("nothing")
})
cli.EXPECT().CompleteTask(A, A).AnyTimes().DoAndReturn(
func(context.Context, *scheduler.OperateTaskArgs) error {
func(context.Context, *scheduler.TaskArgs) error {
stats.step = "Complete"
stats.wg.Done()
return errors.New("nothing")
})
cli.EXPECT().ReclaimTask(A, A).AnyTimes().DoAndReturn(
func(context.Context, *scheduler.OperateTaskArgs) error {
func(context.Context, *scheduler.TaskArgs) error {
stats.step = "Reclaim"
stats.wg.Done()
return errors.New("nothing")
@ -197,7 +197,7 @@ func TestTaskRunner(t *testing.T) {
}
func TestTaskState(t *testing.T) {
s := taskState{}
s := TaskState{}
s.set(TaskRunning)
require.Equal(t, TaskRunning, s.state)
s.set(TaskRunning)

144
blobstore/blobnode/task_shardnode_migrate.go Normal file
View File

@ -0,0 +1,144 @@
// Copyright 2022 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package blobnode
import (
"context"
"sync"
"github.com/cubefs/cubefs/blobstore/api/scheduler"
"github.com/cubefs/cubefs/blobstore/blobnode/client"
"github.com/cubefs/cubefs/blobstore/common/proto"
"github.com/cubefs/cubefs/blobstore/common/trace"
)
// ShardWorker used to manager shard migrate task
type ShardWorker struct {
t *proto.ShardMigrateTask
shardNodeCli client.IShardNode
}
func (s *ShardWorker) Do(ctx context.Context) {
}
func (s *ShardWorker) OperateArgs(reason string) *scheduler.TaskArgs {
shardTaskArgs := &scheduler.ShardTaskArgs{
TaskType: s.t.TaskType,
IDC: s.t.SourceIDC,
TaskID: s.t.TaskID,
Source: s.t.Source,
Dest: s.t.Destination,
Leader: s.t.Leader,
Learner: s.t.Learner,
Reason: reason,
}
data, _ := shardTaskArgs.Marshal()
ret := new(scheduler.TaskArgs)
ret.Data = data
ret.TaskType = shardTaskArgs.TaskType
ret.ModuleType = proto.TypeShardNode
return ret
}
// IShardWorker define for shard node task executor
type IShardWorker interface {
Do(ctx context.Context)
OperateArgs(reason string) *scheduler.TaskArgs
}
// ShardNodeTaskRunner used to manage task
type ShardNodeTaskRunner struct {
taskID string
w IShardWorker
idc string
state *TaskState
ctx context.Context
cancel context.CancelFunc
span trace.Span
stopMu sync.Mutex
stopReason *WorkError
schedulerCli scheduler.IMigrator
stats proto.TaskProgress // task progress statics
taskCounter *taskCounter
}
// NewShardNodeTaskRunner return task runner
func NewShardNodeTaskRunner(ctx context.Context, taskID string, w IShardWorker, idc string,
taskCounter *taskCounter, schedulerCli scheduler.IMigrator) Runner {
span, ctx := trace.StartSpanFromContext(ctx, "taskRunner")
ctx, cancel := context.WithCancel(ctx)
task := ShardNodeTaskRunner{
taskID: taskID,
w: w,
idc: idc,
ctx: ctx,
cancel: cancel,
span: span,
schedulerCli: schedulerCli,
stats: proto.NewTaskProgress(),
taskCounter: taskCounter,
}
task.state.set(TaskInit)
return &task
}
func (s *ShardNodeTaskRunner) Run() {
span := s.span
span.Infof("start run task: taskID[%s]", s.taskID)
s.state.set(TaskRunning)
}
func (s *ShardNodeTaskRunner) Stop() {
s.state.set(TaskStopping)
s.stopWithFail(OtherError(errKilled))
}
func (s *ShardNodeTaskRunner) Stopped() bool {
return s.state.stopped()
}
func (s *ShardNodeTaskRunner) Alive() bool {
return s.state.alive()
}
func (s *ShardNodeTaskRunner) TaskID() string {
return s.taskID
}
func (s *ShardNodeTaskRunner) State() *TaskState {
return s.state
}
func (s *ShardNodeTaskRunner) stopWithFail(fail *WorkError) {
s.span.Infof("stop task: taskID[%s], err_type[%d], err[%+v]", s.taskID, fail.errType, fail.err)
s.stopMu.Lock()
if s.stopReason == nil {
s.stopReason = fail
}
s.stopMu.Unlock()
s.cancel()
}
func (s *ShardNodeTaskRunner) cancelOrReclaim(retErr *WorkError) {
defer s.state.set(TaskStopped)
}
func (s *ShardNodeTaskRunner) complete() {
defer s.state.set(TaskSuccess)
}

63
blobstore/blobnode/worker_service.go
View File

@ -101,6 +101,7 @@ type WorkerService struct {
schedulerCli scheduler.IScheduler
blobNodeCli client.IBlobNode
shardNodeCli client.IShardNode
}
func (cfg *WorkerConfig) checkAndFix() {
@ -257,21 +258,14 @@ func (s *WorkerService) acquireTask() {
}
return
}
if !t.IsValid() {
span.Errorf("task is illegal: task type[%s], task[%+v]", t.TaskType, t)
return
switch t.ModuleType {
case proto.TypeBlobNode:
s.addBlobNodeTask(ctx, t)
case proto.TypeShardNode:
s.addShardNodeTask(ctx, t)
default:
span.Errorf(" task not support module[%d]", t.ModuleType)
}
err = s.taskRunnerMgr.AddTask(ctx, MigrateTaskEx{
taskInfo: t,
downloadShardConcurrency: s.DownloadShardConcurrency,
blobNodeCli: s.blobNodeCli,
})
if err != nil {
span.Errorf("add task failed: taskID[%s], err[%v]", t.TaskID, err)
return
}
span.Infof("acquire task success: task_type[%s], taskID[%s]", t.TaskType, t.TaskID)
}
// acquire inspect task
@ -300,3 +294,44 @@ func (s *WorkerService) acquireInspectTask() {
span.Infof("acquire inspect task success: taskID[%s] task[%+v]", t.TaskID, t)
}
func (s *WorkerService) addBlobNodeTask(ctx context.Context, t *proto.Task) {
span := trace.SpanFromContextSafe(ctx)
task := &proto.MigrateTask{}
if err := task.Unmarshal(t.Data); err != nil {
span.Errorf("task decode failed: task type[%s], task[%+v]", t.TaskType, t)
return
}
if !task.IsValid() {
span.Errorf("task is illegal: task type[%s], task[%+v]", task.TaskType, task)
return
}
if err := s.taskRunnerMgr.AddTask(ctx, MigrateTaskEx{
taskInfo: task,
downloadShardConcurrency: s.DownloadShardConcurrency,
blobNodeCli: s.blobNodeCli,
}); err != nil {
span.Errorf("add task failed: taskID[%s], err[%v]", task.TaskID, err)
return
}
span.Infof("acquire task success: task_type[%s], taskID[%s]", task.TaskType, task.TaskID)
}
func (s *WorkerService) addShardNodeTask(ctx context.Context, t *proto.Task) {
span := trace.SpanFromContextSafe(ctx)
task := &proto.ShardMigrateTask{}
if err := task.Unmarshal(t.Data); err != nil {
span.Errorf("task decode failed: task type[%s], task[%+v]", t.TaskType, t)
return
}
if !task.IsValid() {
span.Errorf("task is illegal: task type[%s], task[%+v]", task.TaskType, task)
return
}
if err := s.taskRunnerMgr.AddShardTask(ctx, s.shardNodeCli, task); err != nil {
span.Errorf("add task failed: taskID[%s], err[%v]", task.TaskID, err)
return
}
span.Infof("acquire task success: task_type[%s], taskID[%s]", task.TaskType, task.TaskID)
}

10
blobstore/blobnode/worker_service_test.go
View File

@ -16,6 +16,7 @@ package blobnode
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http/httptest"
@ -76,7 +77,7 @@ type mockScheCli struct {
inspectCnt int
}
func (m *mockScheCli) AcquireTask(ctx context.Context, args *scheduler.AcquireArgs) (ret *proto.MigrateTask, err error) {
func (m *mockScheCli) AcquireTask(ctx context.Context, args *scheduler.AcquireArgs) (ret *proto.Task, err error) {
m.migrateID++
mode := codemode.EC6P10L2
srcReplicas := genMockVol(1, mode)
@ -108,7 +109,12 @@ func (m *mockScheCli) AcquireTask(ctx context.Context, args *scheduler.AcquireAr
default:
// do nothing
}
return &task, nil
data, err := json.Marshal(task)
if err != nil {
return nil, err
}
ret = &proto.Task{ModuleType: proto.TypeBlobNode, TaskType: task.TaskType, Data: data}
return
}
func (m *mockScheCli) AcquireInspectTask(ctx context.Context) (ret *proto.VolumeInspectTask, err error) {

1
blobstore/common/proto/const.go
View File

@ -182,6 +182,7 @@ const (
InvalidCrc32 = uint32(0)
InvalidVid = Vid(0)
InvalidVuid = Vuid(0)
InvalidSuid = Suid(0)
InvalidSpaceID = SpaceID(0)
)

219
blobstore/common/proto/scheduler.go
View File

@ -15,9 +15,14 @@
package proto
import (
"bytes"
"encoding/binary"
"encoding/json"
"io"
"sync"
"github.com/cubefs/cubefs/blobstore/common/codemode"
"github.com/cubefs/cubefs/blobstore/common/rpc"
"github.com/cubefs/cubefs/blobstore/util/errors"
)
@ -43,12 +48,18 @@ const (
TaskTypeVolumeInspect TaskType = "volume_inspect"
TaskTypeShardRepair TaskType = "shard_repair"
TaskTypeBlobDelete TaskType = "blob_delete"
TaskTypeShardInspect TaskType = "shard_inspect"
TaskTypeShardDiskRepair TaskType = "shard_disk_repair"
TaskTypeShardMigrate TaskType = "shard_migrate"
TaskTypeShardDiskDrop TaskType = "shard_disk_drop"
)
func (t TaskType) Valid() bool {
switch t {
case TaskTypeDiskRepair, TaskTypeBalance, TaskTypeDiskDrop, TaskTypeManualMigrate,
TaskTypeVolumeInspect, TaskTypeShardRepair, TaskTypeBlobDelete:
TaskTypeVolumeInspect, TaskTypeShardRepair, TaskTypeBlobDelete,
TaskTypeShardInspect, TaskTypeShardDiskRepair, TaskTypeShardMigrate, TaskTypeShardDiskDrop:
return true
default:
return false
@ -59,13 +70,14 @@ func (t TaskType) String() string {
return string(t)
}
// VunitLocation volume or shard location
type VunitLocation struct {
Vuid Vuid `json:"vuid" bson:"vuid"`
Host string `json:"host" bson:"host"`
DiskID DiskID `json:"disk_id" bson:"disk_id"`
}
// for task check
// CheckVunitLocations for task check
func CheckVunitLocations(locations []VunitLocation) bool {
if len(locations) == 0 {
return false
@ -79,6 +91,22 @@ func CheckVunitLocations(locations []VunitLocation) bool {
return true
}
// SunitLocation shard location
type SunitLocation struct {
// todo for proto suid
Suid Suid `json:"suid"`
Host string `json:"host"`
DiskID DiskID `json:"disk_id"`
}
// CheckSunitLocation CheckSunitLocations for shard task check
func CheckSunitLocation(location SunitLocation) bool {
if location.Suid == InvalidSuid || location.Host == "" || location.DiskID == InvalidDiskID {
return false
}
return true
}
type MigrateState uint8
const (
@ -89,6 +117,7 @@ const (
MigrateStateFinishedInAdvance
)
// MigrateTask for blobnode task
type MigrateTask struct {
TaskID string `json:"task_id"` // task id
TaskType TaskType `json:"task_type"` // task type
@ -113,6 +142,27 @@ type MigrateTask struct {
WorkerRedoCnt uint8 `json:"worker_redo_cnt"` // worker redo task count
}
func (t *MigrateTask) Unmarshal(data []byte) error {
return json.Unmarshal(data, t)
}
func (t *MigrateTask) Marshal() (data []byte, err error) {
return json.Marshal(t)
}
func (t *MigrateTask) Task() (*Task, error) {
ret := new(Task)
ret.ModuleType = TypeBlobNode
ret.TaskType = t.TaskType
ret.TaskID = t.TaskID
data, err := t.Marshal()
if err != nil {
return nil, err
}
ret.Data = data
return ret, nil
}
func (t *MigrateTask) Vid() Vid {
return t.SourceVuid.Vid()
}
@ -156,6 +206,74 @@ func (t *MigrateTask) IsValid() bool {
CheckVunitLocations([]VunitLocation{t.Destination})
}
type ShardTaskState uint8
const (
ShardTaskStateInited ShardTaskState = iota + 1
ShardTaskStatePrepared
ShardTaskStateWorkCompleted
ShardTaskStateFinished
ShardTaskStateFinishedInAdvance
)
// ShardMigrateTask for shard node task
type ShardMigrateTask struct {
TaskID string `json:"task_id"` // task id
TaskType TaskType `json:"task_type"` // task type
State ShardTaskState `json:"state"` // task state
SourceIDC string `json:"source_idc"` // source idc
Ctime string `json:"ctime"` // create time
MTime string `json:"mtime"` // modify time
Source SunitLocation `json:"source"` // old shard location
Leader SunitLocation `json:"leader"` // shard leader location
Destination SunitLocation `json:"destination"` // new shard location
Learner bool `json:"learner"`
}
func (s *ShardMigrateTask) Unmarshal(data []byte) error {
return json.Unmarshal(data, s)
}
func (s *ShardMigrateTask) Marshal() (data []byte, err error) {
return json.Marshal(s)
}
func (s *ShardMigrateTask) Task() (*Task, error) {
ret := new(Task)
ret.TaskID = s.TaskID
ret.ModuleType = TypeShardNode
ret.TaskType = s.TaskType
data, err := s.Marshal()
if err != nil {
return nil, err
}
ret.Data = data
return ret, err
}
func (s *ShardMigrateTask) GetSource() SunitLocation {
return s.Source
}
func (s *ShardMigrateTask) GetLeader() SunitLocation {
return s.Leader
}
func (s *ShardMigrateTask) GetDestination() SunitLocation {
return s.Destination
}
func (s *ShardMigrateTask) SetDestination(dest SunitLocation) {
s.Destination = dest
}
func (s *ShardMigrateTask) IsValid() bool {
return CheckSunitLocation(s.Source) && CheckSunitLocation(s.Destination)
}
type VolumeInspectCheckPoint struct {
StartVid Vid `json:"start_vid"` // min vid in current batch volumes
Ctime string `json:"ctime"`
@ -259,3 +377,100 @@ func (p *taskProgress) Done() TaskStatistics {
p.mu.Unlock()
return st
}
type ModuleType uint8
const (
TypeMin ModuleType = iota
TypeBlobNode
TypeShardNode
TypeMax
)
type Task struct {
ModuleType ModuleType `json:"module_type"`
TaskType TaskType `json:"task_type"`
TaskID string `json:"task_id"`
Data []byte `json:"data"`
}
func (mt ModuleType) IsValid() bool {
return mt > TypeMin && mt < TypeMax
}
func (t *Task) Marshal() ([]byte, string, error) {
buffer := bytes.NewBuffer(nil)
if err := binary.Write(buffer, binary.BigEndian, t.ModuleType); err != nil {
return nil, "", err
}
taskType := []byte(t.TaskType)
numTaskType := int32(len(taskType))
if err := binary.Write(buffer, binary.BigEndian, numTaskType); err != nil {
return nil, "", err
}
if _, err := buffer.Write(taskType); err != nil {
return nil, "", err
}
taskID := []byte(t.TaskID)
numTaskID := int32(len(taskID))
if err := binary.Write(buffer, binary.BigEndian, numTaskID); err != nil {
return nil, "", err
}
if _, err := buffer.Write(taskID); err != nil {
return nil, "", err
}
numData := int32(len(t.Data))
if err := binary.Write(buffer, binary.BigEndian, numData); err != nil {
return nil, "", err
}
if _, err := buffer.Write(t.Data); err != nil {
return nil, "", err
}
return buffer.Bytes(), rpc.MIMEStream, nil
}
func (t *Task) UnmarshalFrom(body io.Reader) (err error) {
if err = binary.Read(body, binary.BigEndian, &t.ModuleType); err != nil {
return
}
numTaskType := int32(0)
if err = binary.Read(body, binary.BigEndian, &numTaskType); err != nil {
return
}
taskType := make([]byte, numTaskType)
if _, err = io.ReadFull(body, taskType); err != nil {
return err
}
numTaskID := int32(0)
if err = binary.Read(body, binary.BigEndian, &numTaskID); err != nil {
return
}
taskID := make([]byte, numTaskID)
if _, err = io.ReadFull(body, taskID); err != nil {
return err
}
numData := int32(0)
if err = binary.Read(body, binary.BigEndian, &numData); err != nil {
return
}
data := make([]byte, numData)
if _, err = io.ReadFull(body, data); err != nil {
return err
}
t.Data = data
t.TaskType = TaskType(taskType)
t.TaskID = string(taskID)
return nil
}
func (t *Task) Unmarshal(data []byte) error {
if len(data) == 0 {
return nil
}
reader := bytes.NewReader(data)
return t.UnmarshalFrom(reader)
}

17
blobstore/common/proto/scheduler_test.go
View File

@ -15,6 +15,7 @@
package proto_test
import (
"bytes"
"testing"
"github.com/stretchr/testify/require"
@ -107,3 +108,19 @@ func TestSchedulerTaskProgress(t *testing.T) {
require.True(t, tp.Done().Progress >= 100)
}
}
func TestTask(t *testing.T) {
task := &proto.Task{
ModuleType: proto.TypeShardNode,
TaskType: proto.TaskTypeShardInspect,
Data: []byte("test_data"),
TaskID: "test_id",
}
data, _, err := task.Marshal()
require.Nil(t, err)
reader := bytes.NewReader(data)
newTask := &proto.Task{}
err = newTask.UnmarshalFrom(reader)
require.NoError(t, err)
require.EqualValues(t, task, newTask, "")
}

16
blobstore/scheduler/balancer_test.go
View File

@ -169,7 +169,7 @@ func TestBalanceAcquireTask(t *testing.T) {
ctx := context.Background()
idc := "z0"
mgr := newBalancer(t)
mgr.IMigrator.(*MockMigrater).EXPECT().AcquireTask(any, any).Return(proto.MigrateTask{TaskType: proto.TaskTypeBalance}, nil)
mgr.IMigrator.(*MockMigrater).EXPECT().AcquireTask(any, any).Return(&proto.Task{TaskType: proto.TaskTypeBalance}, nil)
_, err := mgr.AcquireTask(ctx, idc)
require.NoError(t, err)
}
@ -178,7 +178,7 @@ func TestBalanceCancelTask(t *testing.T) {
ctx := context.Background()
mgr := newBalancer(t)
mgr.IMigrator.(*MockMigrater).EXPECT().CancelTask(any, any).Return(nil)
err := mgr.CancelTask(ctx, &api.OperateTaskArgs{})
err := mgr.CancelTask(ctx, &api.TaskArgs{})
require.NoError(t, err)
}
@ -186,9 +186,10 @@ func TestBalanceReclaimTask(t *testing.T) {
ctx := context.Background()
idc := "z0"
mgr := newBalancer(t)
mgr.IMigrator.(*MockMigrater).EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil)
mgr.IMigrator.(*MockMigrater).EXPECT().ReclaimTask(any, any).Return(nil)
t1 := mockGenMigrateTask(proto.TaskTypeBalance, idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap)
err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{})
taskArgs := generateTaskArgs(t1, "")
err := mgr.ReclaimTask(ctx, taskArgs)
require.NoError(t, err)
}
@ -198,11 +199,12 @@ func TestBalanceCompleteTask(t *testing.T) {
mgr := newBalancer(t)
mgr.IMigrator.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(nil)
t1 := mockGenMigrateTask(proto.TaskTypeBalance, idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap)
err := mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination})
taskArgs := generateTaskArgs(t1, "")
err := mgr.CompleteTask(ctx, taskArgs)
require.NoError(t, err)
mgr.IMigrator.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(errMock)
err = mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination})
err = mgr.CompleteTask(ctx, taskArgs)
require.True(t, errors.Is(err, errMock))
}
@ -262,7 +264,7 @@ func TestBalanceCheckAndClearJunkTasks(t *testing.T) {
mgr.IMigrator.(*MockMigrater).EXPECT().DeletedTasks().Return([]DeletedTask{
{DiskID: proto.DiskID(1), TaskID: xid.New().String(), DeletedTime: time.Now().Add(-junkMigrationTaskProtectionWindow)},
})
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetMigrateTask(any, any, any).Return(&proto.MigrateTask{}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetMigrateTask(any, any, any).Return(&proto.Task{}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().DeleteMigrateTask(any, any).Return(nil)
mgr.IMigrator.(*MockMigrater).EXPECT().ClearDeletedTaskByID(any, any).Return()
mgr.checkAndClearJunkTasks()

169
blobstore/scheduler/base/queue.go
View File

@ -32,6 +32,7 @@ var (
ErrNoSuchMessageID = errors.New("no such message id")
// ErrUnmatchedVuids unmatched task vuids
ErrUnmatchedVuids = errors.New("unmatched task vuids")
ErrUnmatchedSuid = errors.New("unmatched task suid")
errNoSuchIDCQueue = errors.New("no such idc queue")
errExistingMessageID = errors.New("existing message id")
)
@ -197,6 +198,7 @@ type WorkerTask interface {
GetSources() []proto.VunitLocation
GetDestination() proto.VunitLocation
SetDestination(dest proto.VunitLocation)
Task() (*proto.Task, error)
}
// TaskQueue task queue
@ -215,7 +217,7 @@ func NewTaskQueue(retryDelay time.Duration) *TaskQueue {
}
// PushTask push task to queue
func (q *TaskQueue) PushTask(taskID string, task WorkerTask) {
func (q *TaskQueue) PushTask(taskID string, task interface{}) {
q.mu.Lock()
defer q.mu.Unlock()
err := q.queue.Push(taskID, task)
@ -225,12 +227,12 @@ func (q *TaskQueue) PushTask(taskID string, task WorkerTask) {
}
// PopTask return args： taskID, task, flag of task exist
func (q *TaskQueue) PopTask() (string, WorkerTask, bool) {
func (q *TaskQueue) PopTask() (string, interface{}, bool) {
q.mu.Lock()
defer q.mu.Unlock()
taskID, task, exist := q.queue.Pop()
if exist {
return taskID, task.(WorkerTask), true
return taskID, task, true
}
return "", nil, false
}
@ -254,14 +256,14 @@ func (q *TaskQueue) RetryTask(taskID string) {
}
// Query find task by taskID
func (q *TaskQueue) Query(taskID string) (WorkerTask, bool) {
func (q *TaskQueue) Query(taskID string) (interface{}, bool) {
q.mu.Lock()
defer q.mu.Unlock()
task, err := q.queue.Get(taskID)
if err != nil {
return nil, false
}
return task.(WorkerTask), true
return task, true
}
// StatsTasks returns task stats
@ -271,7 +273,7 @@ func (q *TaskQueue) StatsTasks() (todo int, doing int) {
return q.queue.Stats()
}
// WorkerTaskQueue task queue for worker
// WorkerTaskQueue task queue for blobnode task
type WorkerTaskQueue struct {
mu sync.Mutex
idcQueues map[string]*Queue
@ -462,3 +464,158 @@ func vunitSliceEqual(a, b []proto.VunitLocation) bool {
}
return true
}
// ShardTask define shard task interface
type ShardTask interface {
GetSource() proto.SunitLocation
GetLeader() proto.SunitLocation
GetDestination() proto.SunitLocation
SetDestination(dest proto.SunitLocation)
Task() (*proto.Task, error)
}
// ShardTaskQueue task queue for shard task
type ShardTaskQueue struct {
mu sync.Mutex
idcQueues map[string]*Queue
cancelPunishDuration time.Duration // task cancel will punish a period of time to avoid frequent failure retry
leaseExpiredS time.Duration
}
// AddPreparedTask add prepared task
func (q *ShardTaskQueue) AddPreparedTask(idc, taskID string, wtask ShardTask) {
q.mu.Lock()
defer q.mu.Unlock()
idcQueue, ok := q.idcQueues[idc]
if !ok {
idcQueue = NewQueue(q.leaseExpiredS)
q.idcQueues[idc] = idcQueue
}
err := idcQueue.Push(taskID, wtask)
if err != nil {
panic("unexpect add prepared task fail:" + err.Error())
}
}
// Acquire acquire task by idc
func (q *ShardTaskQueue) Acquire(idc string) (taskID string, wtask ShardTask, exist bool) {
q.mu.Lock()
defer q.mu.Unlock()
idcQueue, ok := q.idcQueues[idc]
if !ok {
return "", nil, false
}
taskID, task, exist := idcQueue.Pop()
if exist {
return taskID, task.(ShardTask), exist
}
return "", nil, false
}
// Cancel cancel task
func (q *ShardTaskQueue) Cancel(idc, taskID string, src, dst proto.SunitLocation) error {
q.mu.Lock()
defer q.mu.Unlock()
idcQueue, ok := q.idcQueues[idc]
if !ok {
return errNoSuchIDCQueue
}
task, err := idcQueue.Get(taskID)
if err != nil {
return err
}
err = checkShardTaskValid(task.(ShardTask), src, dst)
if err != nil {
return err
}
return idcQueue.Requeue(taskID, q.cancelPunishDuration)
}
// Reclaim reclaim task
func (q *ShardTaskQueue) Reclaim(idc, taskID string, src, oldDest, newDest proto.SunitLocation, newDiskID proto.DiskID) error {
q.mu.Lock()
defer q.mu.Unlock()
idcQueue, ok := q.idcQueues[idc]
if !ok {
return errNoSuchIDCQueue
}
task, err := idcQueue.Get(taskID)
if err != nil {
return err
}
wtask := task.(ShardTask)
err = checkShardTaskValid(wtask, src, oldDest)
if err != nil {
return err
}
wtask.SetDestination(newDest)
return idcQueue.Requeue(taskID, 0)
}
// Renewal renewal task
func (q *ShardTaskQueue) Renewal(idc, taskID string) error {
q.mu.Lock()
defer q.mu.Unlock()
idcQueue, ok := q.idcQueues[idc]
if !ok {
return errNoSuchIDCQueue
}
return idcQueue.Requeue(taskID, q.leaseExpiredS)
}
// Complete task
func (q *ShardTaskQueue) Complete(idc, taskID string, src, dst proto.SunitLocation) (ShardTask, error) {
q.mu.Lock()
defer q.mu.Unlock()
idcQueue, ok := q.idcQueues[idc]
if !ok {
return nil, errNoSuchIDCQueue
}
task, err := idcQueue.Get(taskID)
if err != nil {
return nil, err
}
t := task.(ShardTask)
err = checkShardTaskValid(t, src, dst)
if err != nil {
return nil, err
}
err = idcQueue.Remove(taskID)
if err != nil {
panic(fmt.Sprintf("task %s remove form queue fail err %v", taskID, err))
}
return t, err
}
// StatsTasks returns task stats
func (q *ShardTaskQueue) StatsTasks() (todo int, doing int) {
q.mu.Lock()
defer q.mu.Unlock()
for _, queue := range q.idcQueues {
todoTmp, doingTmp := queue.Stats()
todo += todoTmp
doing += doingTmp
}
return todo, doing
}
func checkShardTaskValid(task ShardTask, src proto.SunitLocation, dst proto.SunitLocation) error {
if task.GetSource() != src || task.GetDestination() != dst {
return ErrUnmatchedVuids
}
return nil
}

16
blobstore/scheduler/base/queue_test.go
View File

@ -120,6 +120,10 @@ func (t *mockWorkerTask) SetDestination(dstVuid proto.VunitLocation) {
t.dst = dstVuid
}
func (t *mockWorkerTask) Task() (*proto.Task, error) {
return nil, nil
}
func TestTaskQueue(t *testing.T) {
// test Push
taskID1 := "task_id1"
@ -133,10 +137,12 @@ func TestTaskQueue(t *testing.T) {
// test PopTask
id, wt, exist := q.PopTask()
task, ok := wt.(WorkerTask)
require.True(t, ok)
require.Equal(t, true, exist)
require.Equal(t, id, taskID1)
require.Equal(t, task1.GetSources(), wt.GetSources())
require.Equal(t, task1.GetDestination(), wt.GetDestination())
require.Equal(t, task1.GetSources(), task.GetSources())
require.Equal(t, task1.GetDestination(), task.GetDestination())
_, _, exist = q.PopTask()
require.Equal(t, false, exist)
@ -144,10 +150,12 @@ func TestTaskQueue(t *testing.T) {
q.RetryTask(taskID1)
time.Sleep(100 * time.Millisecond)
id, wt, exist = q.PopTask()
task, ok = wt.(WorkerTask)
require.True(t, ok)
require.Equal(t, true, exist)
require.Equal(t, id, taskID1)
require.Equal(t, vunits([]proto.Vuid{1, 2, 3}), wt.GetSources())
require.Equal(t, vunit(4), wt.GetDestination())
require.Equal(t, vunits([]proto.Vuid{1, 2, 3}), task.GetSources())
require.Equal(t, vunit(4), task.GetDestination())
// test Stats
taskID2 := "task_id2"

90
blobstore/scheduler/client/clustermgr.go
View File

@ -49,6 +49,15 @@ type ClusterMgrVolumeAPI interface {
ListVolume(ctx context.Context, marker proto.Vid, count int) (volInfo []*VolumeInfoSimple, retVid proto.Vid, err error)
}
type ClusterMgrShardAPI interface {
GetShardInfo(ctx context.Context, Sid proto.Vid) (ret *ShardInfoSimple, err error)
UpdateShard(ctx context.Context, newVuid, oldVuid proto.Suid, newDiskID proto.DiskID) (err error)
AllocShardUnit(ctx context.Context, vuid proto.Vuid) (ret *AllocVunitInfo, err error)
ReleaseShardUnit(ctx context.Context, vuid proto.Vuid, diskID proto.DiskID) (err error)
ListDiskShardUnits(ctx context.Context, diskID proto.DiskID) (ret []*VunitInfoSimple, err error)
ListShard(ctx context.Context, marker proto.Vid, count int) (volInfo []*VolumeInfoSimple, retVid proto.Vid, err error)
}
type ClusterMgrDiskAPI interface {
ListClusterDisks(ctx context.Context) (disks []*DiskInfoSimple, err error)
ListBrokenDisks(ctx context.Context) (disks []*DiskInfoSimple, err error)
@ -60,19 +69,30 @@ type ClusterMgrDiskAPI interface {
GetDiskInfo(ctx context.Context, diskID proto.DiskID) (ret *DiskInfoSimple, err error)
}
type ClusterShardNodeAPI interface {
ListClusterDisks(ctx context.Context) (disks []*ShardNodeDiskInfo, err error)
ListBrokenDisks(ctx context.Context) (disks []*ShardNodeDiskInfo, err error)
ListRepairingDisks(ctx context.Context) (disks []*ShardNodeDiskInfo, err error)
ListDropDisks(ctx context.Context) (disks []*ShardNodeDiskInfo, err error)
SetDiskRepairing(ctx context.Context, diskID proto.DiskID) (err error)
SetDiskRepaired(ctx context.Context, diskID proto.DiskID) (err error)
SetDiskDropped(ctx context.Context, diskID proto.DiskID) (err error)
GetDiskInfo(ctx context.Context, diskID proto.DiskID) (ret *ShardNodeDiskInfo, err error)
}
type ClusterMgrServiceAPI interface {
Register(ctx context.Context, info RegisterInfo) error
GetService(ctx context.Context, name string, clusterID proto.ClusterID) (hosts []string, err error)
}
type ClusterMgrTaskAPI interface {
UpdateMigrateTask(ctx context.Context, value *proto.MigrateTask) (err error)
AddMigrateTask(ctx context.Context, value *proto.MigrateTask) (err error)
GetMigrateTask(ctx context.Context, taskType proto.TaskType, key string) (task *proto.MigrateTask, err error)
UpdateMigrateTask(ctx context.Context, value *proto.Task) (err error)
AddMigrateTask(ctx context.Context, value *proto.Task) (err error)
ListMigrateTasks(ctx context.Context, taskType proto.TaskType, args *cmapi.ListKvOpts) (tasks []*proto.Task, marker string, err error)
ListAllMigrateTasks(ctx context.Context, taskType proto.TaskType) (tasks []*proto.Task, err error)
ListAllMigrateTasksByDiskID(ctx context.Context, taskType proto.TaskType, diskID proto.DiskID) (tasks []*proto.Task, err error)
GetMigrateTask(ctx context.Context, taskType proto.TaskType, key string) (task *proto.Task, err error)
DeleteMigrateTask(ctx context.Context, key string) (err error)
ListMigrateTasks(ctx context.Context, taskType proto.TaskType, args *cmapi.ListKvOpts) (tasks []*proto.MigrateTask, marker string, err error)
ListAllMigrateTasks(ctx context.Context, taskType proto.TaskType) (tasks []*proto.MigrateTask, err error)
ListAllMigrateTasksByDiskID(ctx context.Context, taskType proto.TaskType, diskID proto.DiskID) (tasks []*proto.MigrateTask, err error)
AddMigratingDisk(ctx context.Context, value *MigratingDiskMeta) (err error)
DeleteMigratingDisk(ctx context.Context, taskType proto.TaskType, diskID proto.DiskID) (err error)
GetMigratingDisk(ctx context.Context, taskType proto.TaskType, diskID proto.DiskID) (meta *MigratingDiskMeta, err error)
@ -285,7 +305,7 @@ func (vunit *VunitInfoSimple) set(info *cmapi.VolumeUnitInfo, host string) {
vunit.Used = info.Used
}
// DiskInfoSimple disk simple info
// DiskInfoSimple disk simple info for blobnode
type DiskInfoSimple struct {
ClusterID proto.ClusterID `json:"cluster_id"`
DiskID proto.DiskID `json:"disk_id"`
@ -299,6 +319,19 @@ type DiskInfoSimple struct {
FreeChunkCnt int64 `json:"free_chunk_cnt"`
}
// ShardNodeDiskInfo diskInfo for shard node
type ShardNodeDiskInfo struct {
ClusterID proto.ClusterID `json:"cluster_id"`
DiskID proto.DiskID `json:"disk_id"`
Idc string `json:"idc"`
Rack string `json:"rack"`
Host string `json:"host"`
Status proto.DiskStatus `json:"status"`
Readonly bool `json:"readonly"`
UsedShardCnt int64 `json:"used_shard_cnt"`
FreeShardCnt int64 `json:"free_shard_cnt"`
}
// IsHealth return true if disk is health
func (disk *DiskInfoSimple) IsHealth() bool {
return disk.Status == proto.DiskStatusNormal
@ -774,16 +807,12 @@ func (c *clustermgrClient) GetService(ctx context.Context, name string, clusterI
}
// AddMigrateTask adds migrate task
func (c *clustermgrClient) AddMigrateTask(ctx context.Context, value *proto.MigrateTask) (err error) {
value.Ctime = time.Now().String()
value.MTime = value.Ctime
func (c *clustermgrClient) AddMigrateTask(ctx context.Context, value *proto.Task) (err error) {
return c.setTask(ctx, value.TaskID, value)
}
// UpdateMigrateTask updates migrate task
func (c *clustermgrClient) UpdateMigrateTask(ctx context.Context, value *proto.MigrateTask) (err error) {
value.MTime = time.Now().String()
func (c *clustermgrClient) UpdateMigrateTask(ctx context.Context, value *proto.Task) (err error) {
return c.setTask(ctx, value.TaskID, value)
}
@ -796,12 +825,14 @@ func (c *clustermgrClient) setTask(ctx context.Context, key string, value interf
}
// GetMigrateTask returns migrate task
func (c *clustermgrClient) GetMigrateTask(ctx context.Context, taskType proto.TaskType, key string) (task *proto.MigrateTask, err error) {
func (c *clustermgrClient) GetMigrateTask(ctx context.Context, taskType proto.TaskType, key string) (task *proto.Task, err error) {
// todo compatible with old task, or delete old task
val, err := c.client.GetKV(ctx, key)
if err != nil {
return nil, err
}
err = json.Unmarshal(val.Value, &task)
task = new(proto.Task)
err = task.Unmarshal(val.Value)
if err != nil {
return nil, err
}
@ -817,21 +848,21 @@ func (c *clustermgrClient) DeleteMigrateTask(ctx context.Context, key string) (e
}
// ListAllMigrateTasksByDiskID returns all migrate task with disk_id
func (c *clustermgrClient) ListAllMigrateTasksByDiskID(ctx context.Context, taskType proto.TaskType, diskID proto.DiskID) (tasks []*proto.MigrateTask, err error) {
func (c *clustermgrClient) ListAllMigrateTasksByDiskID(ctx context.Context, taskType proto.TaskType, diskID proto.DiskID) (tasks []*proto.Task, err error) {
return c.listAllMigrateTasks(ctx, GenMigrateTaskPrefixByDiskID(taskType, diskID), taskType)
}
// ListAllMigrateTasks returns all migrate task
func (c *clustermgrClient) ListAllMigrateTasks(ctx context.Context, taskType proto.TaskType) (tasks []*proto.MigrateTask, err error) {
func (c *clustermgrClient) ListAllMigrateTasks(ctx context.Context, taskType proto.TaskType) (tasks []*proto.Task, err error) {
return c.listAllMigrateTasks(ctx, GenMigrateTaskPrefix(taskType), taskType)
}
// ListMigrateTasks returns migrate task base on page size
func (c *clustermgrClient) ListMigrateTasks(ctx context.Context, taskType proto.TaskType, args *cmapi.ListKvOpts) (tasks []*proto.MigrateTask, marker string, err error) {
func (c *clustermgrClient) ListMigrateTasks(ctx context.Context, taskType proto.TaskType, args *cmapi.ListKvOpts) (tasks []*proto.Task, marker string, err error) {
return c.listMigrateTasks(ctx, taskType, args)
}
func (c *clustermgrClient) listMigrateTasks(ctx context.Context, taskType proto.TaskType, args *cmapi.ListKvOpts) (tasks []*proto.MigrateTask, marker string, err error) {
func (c *clustermgrClient) listMigrateTasks(ctx context.Context, taskType proto.TaskType, args *cmapi.ListKvOpts) (tasks []*proto.Task, marker string, err error) {
span := trace.SpanFromContextSafe(ctx)
ret, err := c.client.ListKV(ctx, args)
if err != nil {
@ -839,7 +870,7 @@ func (c *clustermgrClient) listMigrateTasks(ctx context.Context, taskType proto.
return nil, marker, err
}
for _, v := range ret.Kvs {
var task *proto.MigrateTask
var task *proto.Task
err = json.Unmarshal(v.Value, &task)
if err != nil {
span.Errorf("unmarshal task failed: err[%+v]", err)
@ -855,7 +886,7 @@ func (c *clustermgrClient) listMigrateTasks(ctx context.Context, taskType proto.
return
}
func (c *clustermgrClient) listAllMigrateTasks(ctx context.Context, prefix string, taskType proto.TaskType) (tasks []*proto.MigrateTask, err error) {
func (c *clustermgrClient) listAllMigrateTasks(ctx context.Context, prefix string, taskType proto.TaskType) (tasks []*proto.Task, err error) {
marker := defaultListTaskMarker
for {
args := &cmapi.ListKvOpts{
@ -990,3 +1021,20 @@ func (c *clustermgrClient) SetConsumeOffset(taskType proto.TaskType, topic strin
}
return c.client.SetKV(context.Background(), genConsumerOffsetKey(taskType, topic, partition), consumeOffsetBytes)
}
// ShardInfoSimple shard info used by scheduler
type ShardInfoSimple struct {
Sid proto.ShardID `json:"sid"`
ApplyIndex uint64 `json:"apply_index"`
Leader proto.NodeID `json:"leader"`
Status proto.ShardStatus `json:"status"`
SunitLocations []proto.SunitLocation `json:"sunit_locations"`
}
type UpdateShardArgs struct {
NewSuid proto.Suid `json:"new_suid"`
OldSuid proto.Suid `json:"old_suid"`
OldIsLearner bool `json:"old_is_learner"`
NewIsLearner bool `json:"new_is_learner"`
NewDiskID bool `json:"new_disk_id"`
}

16
blobstore/scheduler/client/clustermgr_test.go
View File

@ -307,19 +307,21 @@ func TestClustermgrClient(t *testing.T) {
{
// add migrate task
cli.client.(*MockClusterManager).EXPECT().SetKV(any, any, any).Return(nil)
err := cli.AddMigrateTask(ctx, &proto.MigrateTask{TaskID: GenMigrateTaskID(proto.TaskTypeDiskRepair, proto.DiskID(1), proto.Vid(1))})
task, _ := (&proto.MigrateTask{TaskID: GenMigrateTaskID(proto.TaskTypeDiskRepair, proto.DiskID(1), proto.Vid(1))}).Task()
err := cli.AddMigrateTask(ctx, task)
require.NoError(t, err)
}
{
// update migrate task
cli.client.(*MockClusterManager).EXPECT().SetKV(any, any, any).Return(nil)
err := cli.UpdateMigrateTask(ctx, &proto.MigrateTask{TaskID: GenMigrateTaskID(proto.TaskTypeDiskRepair, proto.DiskID(1), proto.Vid(1))})
task, _ := (&proto.MigrateTask{TaskID: GenMigrateTaskID(proto.TaskTypeDiskRepair, proto.DiskID(1), proto.Vid(1))}).Task()
err := cli.UpdateMigrateTask(ctx, task)
require.NoError(t, err)
}
{
// get migrate task
task1 := &proto.MigrateTask{TaskID: GenMigrateTaskID(proto.TaskTypeDiskRepair, proto.DiskID(1), proto.Vid(1))}
taskBytes, _ := json.Marshal(task1)
task1, _ := (&proto.MigrateTask{TaskID: GenMigrateTaskID(proto.TaskTypeDiskRepair, proto.DiskID(1), proto.Vid(1))}).Task()
taskBytes, _, _ := task1.Marshal()
cli.client.(*MockClusterManager).EXPECT().SetKV(any, any, any).Return(nil)
cli.client.(*MockClusterManager).EXPECT().GetKV(any, any).Return(cmapi.GetKvRet{Value: taskBytes}, nil)
err := cli.AddMigrateTask(ctx, task1)
@ -328,12 +330,6 @@ func TestClustermgrClient(t *testing.T) {
require.NoError(t, err)
require.Equal(t, task1.TaskID, task2.TaskID)
// unmarshal failed
taskBytes = append(taskBytes, []byte("mock")...)
cli.client.(*MockClusterManager).EXPECT().GetKV(any, any).Return(cmapi.GetKvRet{Value: taskBytes}, nil)
_, err = cli.GetMigrateTask(ctx, task1.TaskType, task1.TaskID)
require.Error(t, err)
// clustermgr return err
cli.client.(*MockClusterManager).EXPECT().GetKV(any, any).Return(cmapi.GetKvRet{}, errMock)
_, err = cli.GetMigrateTask(ctx, task1.TaskType, task1.TaskID)

20
blobstore/scheduler/client_mock_test.go
View File

@ -38,7 +38,7 @@ func (m *MockClusterMgrAPI) EXPECT() *MockClusterMgrAPIMockRecorder {
}
// AddMigrateTask mocks base method.
func (m *MockClusterMgrAPI) AddMigrateTask(arg0 context.Context, arg1 *proto.MigrateTask) error {
func (m *MockClusterMgrAPI) AddMigrateTask(arg0 context.Context, arg1 *proto.Task) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "AddMigrateTask", arg0, arg1)
ret0, _ := ret[0].(error)
@ -154,10 +154,10 @@ func (mr *MockClusterMgrAPIMockRecorder) GetDiskInfo(arg0, arg1 interface{}) *go
}
// GetMigrateTask mocks base method.
func (m *MockClusterMgrAPI) GetMigrateTask(arg0 context.Context, arg1 proto.TaskType, arg2 string) (*proto.MigrateTask, error) {
func (m *MockClusterMgrAPI) GetMigrateTask(arg0 context.Context, arg1 proto.TaskType, arg2 string) (*proto.Task, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetMigrateTask", arg0, arg1, arg2)
ret0, _ := ret[0].(*proto.MigrateTask)
ret0, _ := ret[0].(*proto.Task)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@ -229,10 +229,10 @@ func (mr *MockClusterMgrAPIMockRecorder) GetVolumeInspectCheckPoint(arg0 interfa
}
// ListAllMigrateTasks mocks base method.
func (m *MockClusterMgrAPI) ListAllMigrateTasks(arg0 context.Context, arg1 proto.TaskType) ([]*proto.MigrateTask, error) {
func (m *MockClusterMgrAPI) ListAllMigrateTasks(arg0 context.Context, arg1 proto.TaskType) ([]*proto.Task, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ListAllMigrateTasks", arg0, arg1)
ret0, _ := ret[0].([]*proto.MigrateTask)
ret0, _ := ret[0].([]*proto.Task)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@ -244,10 +244,10 @@ func (mr *MockClusterMgrAPIMockRecorder) ListAllMigrateTasks(arg0, arg1 interfac
}
// ListAllMigrateTasksByDiskID mocks base method.
func (m *MockClusterMgrAPI) ListAllMigrateTasksByDiskID(arg0 context.Context, arg1 proto.TaskType, arg2 proto.DiskID) ([]*proto.MigrateTask, error) {
func (m *MockClusterMgrAPI) ListAllMigrateTasksByDiskID(arg0 context.Context, arg1 proto.TaskType, arg2 proto.DiskID) ([]*proto.Task, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ListAllMigrateTasksByDiskID", arg0, arg1, arg2)
ret0, _ := ret[0].([]*proto.MigrateTask)
ret0, _ := ret[0].([]*proto.Task)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@ -319,10 +319,10 @@ func (mr *MockClusterMgrAPIMockRecorder) ListDropDisks(arg0 interface{}) *gomock
}
// ListMigrateTasks mocks base method.
func (m *MockClusterMgrAPI) ListMigrateTasks(arg0 context.Context, arg1 proto.TaskType, arg2 *clustermgr.ListKvOpts) ([]*proto.MigrateTask, string, error) {
func (m *MockClusterMgrAPI) ListMigrateTasks(arg0 context.Context, arg1 proto.TaskType, arg2 *clustermgr.ListKvOpts) ([]*proto.Task, string, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ListMigrateTasks", arg0, arg1, arg2)
ret0, _ := ret[0].([]*proto.MigrateTask)
ret0, _ := ret[0].([]*proto.Task)
ret1, _ := ret[1].(string)
ret2, _ := ret[2].(error)
return ret0, ret1, ret2
@ -521,7 +521,7 @@ func (mr *MockClusterMgrAPIMockRecorder) UnlockVolume(arg0, arg1 interface{}) *g
}
// UpdateMigrateTask mocks base method.
func (m *MockClusterMgrAPI) UpdateMigrateTask(arg0 context.Context, arg1 *proto.MigrateTask) error {
func (m *MockClusterMgrAPI) UpdateMigrateTask(arg0 context.Context, arg1 *proto.Task) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "UpdateMigrateTask", arg0, arg1)
ret0, _ := ret[0].(error)

7
blobstore/scheduler/disk_droper.go
View File

@ -204,7 +204,12 @@ func (mgr *DiskDropMgr) Load() (err error) {
return err
}
for _, task := range tasks {
migrated, err := mgr.isMigrated(task.SourceVuid)
t := &proto.MigrateTask{}
err = t.Unmarshal(task.Data)
if err != nil {
return err
}
migrated, err := mgr.isMigrated(t.SourceVuid)
if err != nil {
return err
}

30
blobstore/scheduler/disk_droper_test.go
View File

@ -56,6 +56,8 @@ func newDiskDroper(t *testing.T) *DiskDropMgr {
func TestDiskDropLoad(t *testing.T) {
t1 := mockGenMigrateTask(proto.TaskTypeDiskDrop, "z0", 1, 110, proto.MigrateStateInited, MockMigrateVolInfoMap)
task, err := t1.Task()
require.NoError(t, err)
// loopGenerateTask success
volume := MockGenVolInfo(110, codemode.EC6P6, proto.VolumeStatusIdle)
{
@ -68,7 +70,7 @@ func TestDiskDropLoad(t *testing.T) {
{
mgr := newDiskDroper(t)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDropDisks(any).Times(1).Return(nil, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Times(1).Return([]*proto.MigrateTask{t1}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Times(1).Return([]*proto.Task{task}, nil)
vuid := volume.VunitLocations[0].Vuid
epoch := vuid.Epoch()
volume.VunitLocations[0].Vuid = proto.EncodeVuid(vuid.VuidPrefix(), epoch+1)
@ -283,7 +285,7 @@ func TestDiskDropCheckAndClearJunkTasks(t *testing.T) {
// has junk task and clear
mgr.droppedDisks.add(disk1.DiskID, time.Now().Add(-junkMigrationTaskProtectionWindow))
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.MigrateTask{{TaskID: "test"}}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.Task{{TaskID: "test"}}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().DeleteMigrateTask(any, any).Return(nil)
mgr.checkAndClearJunkTasks()
require.Equal(t, 0, mgr.droppedDisks.size())
@ -294,7 +296,7 @@ func TestDiskDropAcquireTask(t *testing.T) {
ctx := context.Background()
idc := "z0"
mgr := newDiskDroper(t)
mgr.IMigrator.(*MockMigrater).EXPECT().AcquireTask(any, any).Return(proto.MigrateTask{TaskType: proto.TaskTypeDiskDrop}, nil)
mgr.IMigrator.(*MockMigrater).EXPECT().AcquireTask(any, any).Return(&proto.Task{TaskType: proto.TaskTypeDiskDrop}, nil)
_, err := mgr.AcquireTask(ctx, idc)
require.NoError(t, err)
}
@ -303,7 +305,7 @@ func TestDiskDropCancelTask(t *testing.T) {
ctx := context.Background()
mgr := newDiskDroper(t)
mgr.IMigrator.(*MockMigrater).EXPECT().CancelTask(any, any).Return(nil)
err := mgr.CancelTask(ctx, &api.OperateTaskArgs{})
err := mgr.CancelTask(ctx, &api.TaskArgs{})
require.NoError(t, err)
}
@ -311,9 +313,11 @@ func TestDiskDropReclaimTask(t *testing.T) {
ctx := context.Background()
idc := "z0"
mgr := newDiskDroper(t)
mgr.IMigrator.(*MockMigrater).EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil)
mgr.IMigrator.(*MockMigrater).EXPECT().ReclaimTask(any, any).Return(nil)
t1 := mockGenMigrateTask(proto.TaskTypeDiskDrop, idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap)
err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{})
taskArgs := generateTaskArgs(t1, "")
err := mgr.ReclaimTask(ctx, taskArgs)
require.NoError(t, err)
}
@ -323,11 +327,13 @@ func TestDiskDropCompleteTask(t *testing.T) {
mgr := newDiskDroper(t)
mgr.IMigrator.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(nil)
t1 := mockGenMigrateTask(proto.TaskTypeDiskDrop, idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap)
err := mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination})
taskArgs := generateTaskArgs(t1, "")
err := mgr.CompleteTask(ctx, taskArgs)
require.NoError(t, err)
mgr.IMigrator.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(errMock)
err = mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination})
err = mgr.CompleteTask(ctx, taskArgs)
require.True(t, errors.Is(err, errMock))
}
@ -418,11 +424,11 @@ func TestDiskDropDiskProgress(t *testing.T) {
mgr := newDiskDroper(t)
mgr.allDisks.add(&dropDisk{DiskInfoSimple: testDisk1, undoneTaskCnt: 3})
task1 := &proto.MigrateTask{State: proto.MigrateStatePrepared, SourceDiskID: testDisk1.DiskID}
task2 := &proto.MigrateTask{State: proto.MigrateStateInited, SourceDiskID: testDisk1.DiskID}
task3 := &proto.MigrateTask{State: proto.MigrateStateWorkCompleted, SourceDiskID: testDisk1.DiskID}
task1, _ := (&proto.MigrateTask{State: proto.MigrateStatePrepared, SourceDiskID: testDisk1.DiskID}).Task()
task2, _ := (&proto.MigrateTask{State: proto.MigrateStateInited, SourceDiskID: testDisk1.DiskID}).Task()
task3, _ := (&proto.MigrateTask{State: proto.MigrateStateWorkCompleted, SourceDiskID: testDisk1.DiskID}).Task()
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.MigrateTask{task1, task2, task3}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.Task{task1, task2, task3}, nil)
stats, err := mgr.DiskProgress(ctx, testDisk1.DiskID)
require.NoError(t, err)
require.Equal(t, int(testDisk1.UsedChunkCnt), stats.TotalTasksCnt)

171
blobstore/scheduler/disk_repairer.go
View File

@ -16,6 +16,7 @@ package scheduler
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
@ -102,26 +103,31 @@ func (mgr *DiskRepairMgr) Load() error {
var junkTasks []*proto.MigrateTask
for _, t := range tasks {
if _, ok := mgr.repairingDisks.get(t.SourceDiskID); !ok {
junkTasks = append(junkTasks, t)
task := &proto.MigrateTask{}
err = task.Unmarshal(t.Data)
if err != nil {
return err
}
if _, ok := mgr.repairingDisks.get(task.SourceDiskID); !ok {
junkTasks = append(junkTasks, task)
continue
}
if t.Running() {
err = base.VolTaskLockerInst().TryLock(ctx, t.Vid())
if task.Running() {
err = base.VolTaskLockerInst().TryLock(ctx, task.Vid())
if err != nil {
return fmt.Errorf("repair task conflict: task[%+v], err[%+v]",
t, err.Error())
}
}
span.Infof("load task success: task_id[%s], state[%d]", t.TaskID, t.State)
switch t.State {
span.Infof("load task success: task_id[%s], state[%d]", t.TaskID, task.State)
switch task.State {
case proto.MigrateStateInited:
mgr.prepareQueue.PushTask(t.TaskID, t)
mgr.prepareQueue.PushTask(t.TaskID, task)
case proto.MigrateStatePrepared:
mgr.workQueue.AddPreparedTask(t.SourceIDC, t.TaskID, t)
mgr.workQueue.AddPreparedTask(task.SourceIDC, t.TaskID, task)
case proto.MigrateStateWorkCompleted:
mgr.finishQueue.PushTask(t.TaskID, t)
mgr.finishQueue.PushTask(t.TaskID, task)
case proto.MigrateStateFinished, proto.MigrateStateFinishedInAdvance:
return fmt.Errorf("task should be deleted from db: task[%+v]", t)
default:
@ -321,8 +327,13 @@ func (mgr *DiskRepairMgr) listMigratingVuid(ctx context.Context, diskID proto.Di
return nil, err
}
task := &proto.MigrateTask{}
for _, t := range tasks {
bads = append(bads, t.SourceVuid)
err = task.Unmarshal(t.Data)
if err != nil {
return nil, err
}
bads = append(bads, task.SourceVuid)
}
return bads, nil
}
@ -350,9 +361,16 @@ func (mgr *DiskRepairMgr) initOneTask(ctx context.Context, badVuid proto.Vuid, b
SourceVuid: badVuid,
SourceIDC: brokenDiskIdc,
ForbiddenDirectDownload: true,
Ctime: time.Now().String(),
}
t.MTime = t.Ctime
base.InsistOn(ctx, "repair init one task insert task to tbl", func() error {
return mgr.clusterMgrCli.AddMigrateTask(ctx, &t)
task, err := t.Task()
if err != nil {
return err
}
return mgr.clusterMgrCli.AddMigrateTask(ctx, task)
})
mgr.prepareQueue.PushTask(t.TaskID, &t)
@ -452,7 +470,11 @@ func (mgr *DiskRepairMgr) prepareTask(t *proto.MigrateTask) error {
t.Destination = allocDstVunit.Location()
t.State = proto.MigrateStatePrepared
base.InsistOn(ctx, "repair prepare task update task tbl", func() error {
return mgr.clusterMgrCli.UpdateMigrateTask(ctx, t)
task, err := t.Task()
if err != nil {
return err
}
return mgr.clusterMgrCli.UpdateMigrateTask(ctx, task)
})
mgr.sendToWorkQueue(t)
@ -528,7 +550,11 @@ func (mgr *DiskRepairMgr) finishTask(ctx context.Context, task *proto.MigrateTas
// because if process restart will reload task and redo by worker
// worker will write data to chunk which is online
base.InsistOn(ctx, "repair finish task update task state completed", func() error {
return mgr.clusterMgrCli.UpdateMigrateTask(ctx, task)
t, err := task.Task()
if err != nil {
return err
}
return mgr.clusterMgrCli.UpdateMigrateTask(ctx, t)
})
newVuid := task.Destination.Vuid
@ -583,7 +609,11 @@ func (mgr *DiskRepairMgr) handleUpdateVolMappingFail(ctx context.Context, task *
task.WorkerRedoCnt++
base.InsistOn(ctx, "repair redo task update task tbl", func() error {
return mgr.clusterMgrCli.UpdateMigrateTask(ctx, task)
t, err := task.Task()
if err != nil {
return err
}
return mgr.clusterMgrCli.UpdateMigrateTask(ctx, t)
})
mgr.finishQueue.RemoveTask(task.TaskID)
@ -665,7 +695,7 @@ func (mgr *DiskRepairMgr) checkDiskRepaired(ctx context.Context, diskID proto.Di
return len(tasks) == 0 && len(vunitInfos) == 0
}
func (mgr *DiskRepairMgr) clearJunkTasks(ctx context.Context, diskID proto.DiskID, tasks []*proto.MigrateTask) {
func (mgr *DiskRepairMgr) clearJunkTasks(ctx context.Context, diskID proto.DiskID, tasks []*proto.Task) {
span := trace.SpanFromContextSafe(ctx)
for _, task := range tasks {
if !mgr.deletedTasks.exits(diskID, task.TaskID) {
@ -728,26 +758,38 @@ func (mgr *DiskRepairMgr) checkAndClearJunkTasks() {
}
// AcquireTask acquire repair task
func (mgr *DiskRepairMgr) AcquireTask(ctx context.Context, idc string) (task proto.MigrateTask, err error) {
func (mgr *DiskRepairMgr) AcquireTask(ctx context.Context, idc string) (task *proto.Task, err error) {
if !mgr.taskSwitch.Enabled() {
return task, proto.ErrTaskPaused
}
_, repairTask, _ := mgr.workQueue.Acquire(idc)
if repairTask != nil {
task = *repairTask.(*proto.MigrateTask)
t := *repairTask.(*proto.MigrateTask)
data, err := t.Marshal()
if err != nil {
return task, err
}
task = &proto.Task{}
task.Data = data
task.ModuleType = proto.TypeBlobNode
return task, nil
}
return task, proto.ErrTaskEmpty
}
// CancelTask cancel repair task
func (mgr *DiskRepairMgr) CancelTask(ctx context.Context, args *api.OperateTaskArgs) error {
func (mgr *DiskRepairMgr) CancelTask(ctx context.Context, args *api.TaskArgs) error {
span := trace.SpanFromContextSafe(ctx)
err := mgr.workQueue.Cancel(args.IDC, args.TaskID, args.Src, args.Dest)
arg := &api.OperateTaskArgs{}
err := arg.Unmarshal(args.Data)
if err != nil {
span.Errorf("cancel repair failed: task_id[%s], err[%+v]", args.TaskID, err)
return err
}
err = mgr.workQueue.Cancel(arg.IDC, arg.TaskID, arg.Src, arg.Dest)
if err != nil {
span.Errorf("cancel repair failed: task_id[%s], err[%+v]", arg.TaskID, err)
}
mgr.taskStatsMgr.CancelTask()
@ -756,30 +798,36 @@ func (mgr *DiskRepairMgr) CancelTask(ctx context.Context, args *api.OperateTaskA
}
// ReclaimTask reclaim repair task
func (mgr *DiskRepairMgr) ReclaimTask(ctx context.Context,
idc, taskID string,
src []proto.VunitLocation,
oldDst proto.VunitLocation,
newDst *client.AllocVunitInfo,
) error {
func (mgr *DiskRepairMgr) ReclaimTask(ctx context.Context, args *api.TaskArgs) error {
span := trace.SpanFromContextSafe(ctx)
err := mgr.workQueue.Reclaim(idc, taskID, src, oldDst, newDst.Location(), newDst.DiskID)
arg := &api.OperateTaskArgs{}
err := arg.Unmarshal(args.Data)
if err != nil {
// task has finished,because only complete will remove task from queue
span.Errorf("reclaim repair task failed: task_id[%s], err[%+v]", taskID, err)
return err
}
task, err := mgr.workQueue.Query(idc, taskID)
newDst, err := base.AllocVunitSafe(ctx, mgr.clusterMgrCli, arg.Dest.Vuid, arg.Src)
if err != nil {
span.Errorf("found task in workQueue failed: idc[%s], task_id[%s], err[%+v]", idc, taskID, err)
span.Errorf("alloc volume unit from clustermgr failed, err: %s", err)
return err
}
err = mgr.clusterMgrCli.UpdateMigrateTask(ctx, task.(*proto.MigrateTask))
err = mgr.workQueue.Reclaim(arg.IDC, arg.TaskID, arg.Src, arg.Dest, newDst.VunitLocation, newDst.DiskID)
if err != nil {
span.Warnf("update reclaim task failed: task_id[%s], err[%+v]", taskID, err)
span.Errorf("reclaim task in workQueue failed: idc[%s], task_id[%s], err[%+v]", arg.IDC, arg.TaskID, err)
return err
}
task, err := mgr.workQueue.Query(arg.IDC, arg.TaskID)
if err != nil {
span.Errorf("found task in workQueue failed: idc[%s], task_id[%s], err[%+v]", arg.IDC, arg.TaskID, err)
return err
}
t, _ := task.Task()
err = mgr.clusterMgrCli.UpdateMigrateTask(ctx, t)
if err != nil {
span.Warnf("update reclaim task failed: task_id[%s], err[%+v]", arg.TaskID, err)
}
mgr.taskStatsMgr.ReclaimTask()
@ -787,19 +835,25 @@ func (mgr *DiskRepairMgr) ReclaimTask(ctx context.Context,
}
// CompleteTask complete repair task
func (mgr *DiskRepairMgr) CompleteTask(ctx context.Context, args *api.OperateTaskArgs) error {
func (mgr *DiskRepairMgr) CompleteTask(ctx context.Context, args *api.TaskArgs) error {
span := trace.SpanFromContextSafe(ctx)
completeTask, err := mgr.workQueue.Complete(args.IDC, args.TaskID, args.Src, args.Dest)
arg := &api.OperateTaskArgs{}
err := arg.Unmarshal(args.Data)
if err != nil {
span.Errorf("complete repair task failed: task_id[%s], err[%+v]", args.TaskID, err)
return err
}
completeTask, err := mgr.workQueue.Complete(arg.IDC, arg.TaskID, arg.Src, arg.Dest)
if err != nil {
span.Errorf("complete repair task failed: task_id[%s], err[%+v]", arg.TaskID, err)
return err
}
t := completeTask.(*proto.MigrateTask)
t.State = proto.MigrateStateWorkCompleted
mgr.finishQueue.PushTask(args.TaskID, t)
mgr.finishQueue.PushTask(arg.TaskID, t)
// as complete func is face to svr api, so can not loop save task
// to db until success, it will make saving task info to be difficult,
// that delay saving task info in finish stage is a simply way
@ -818,30 +872,51 @@ func (mgr *DiskRepairMgr) RenewalTask(ctx context.Context, idc, taskID string) e
if err != nil {
span.Warnf("renewal repair task failed: task_id[%s], err[%+v]", taskID, err)
}
return err
}
func (mgr *DiskRepairMgr) ReportTask(ctx context.Context, args *api.TaskArgs) (err error) {
arg := &api.TaskReportArgs{}
err = arg.Unmarshal(args.Data)
if err != nil {
return err
}
if !client.ValidMigrateTask(arg.TaskType, arg.TaskID) {
return errcode.ErrIllegalArguments
}
mgr.ReportWorkerTaskStats(arg)
return nil
}
// ReportWorkerTaskStats reports task stats
func (mgr *DiskRepairMgr) ReportWorkerTaskStats(st *api.TaskReportArgs) {
mgr.taskStatsMgr.ReportWorkerTaskStats(st.TaskID, st.TaskStats, st.IncreaseDataSizeByte, st.IncreaseShardCnt)
}
// QueryTask return task statistics
func (mgr *DiskRepairMgr) QueryTask(ctx context.Context, taskID string) (*api.MigrateTaskDetail, error) {
detail := &api.MigrateTaskDetail{}
taskInfo, err := mgr.clusterMgrCli.GetMigrateTask(ctx, proto.TaskTypeDiskRepair, taskID)
func (mgr *DiskRepairMgr) QueryTask(ctx context.Context, taskID string) (*api.TaskRet, error) {
migTask := &api.MigrateTaskDetail{}
task, err := mgr.clusterMgrCli.GetMigrateTask(ctx, proto.TaskTypeDiskRepair, taskID)
if err != nil {
return detail, err
return nil, err
}
detail.Task = *taskInfo
migrateTask := &proto.MigrateTask{}
err = migrateTask.Unmarshal(task.Data)
if err != nil {
return nil, err
}
migTask.Task = *migrateTask
detailRunInfo, err := mgr.taskStatsMgr.QueryTaskDetail(taskID)
if err != nil {
return detail, nil
return nil, nil
}
detail.Stat = detailRunInfo.Statistics
return detail, nil
migTask.Stat = detailRunInfo.Statistics
data, err := json.Marshal(migTask)
if err != nil {
return nil, err
}
return &api.TaskRet{TaskType: proto.TaskTypeDiskRepair, Data: data}, nil
}
// StatQueueTaskCnt returns task queue stats

132
blobstore/scheduler/disk_repairer_test.go
View File

@ -60,7 +60,29 @@ func newDiskRepairer(t *testing.T) *DiskRepairMgr {
return NewDiskRepairMgr(clusterMgr, taskSwitch, taskLogger, conf)
}
func generateTaskArgs(task *proto.MigrateTask, reason string) *api.TaskArgs {
ret := new(api.TaskArgs)
args := api.OperateTaskArgs{
IDC: task.SourceIDC,
TaskType: task.TaskType,
Src: task.Sources,
TaskID: task.TaskID,
Reason: reason,
Dest: task.Destination,
}
data, _ := args.Marshal()
ret.Data = data
ret.TaskType = task.TaskType
ret.ModuleType = proto.TypeBlobNode
return ret
}
func TestDiskRepairerLoad(t *testing.T) {
task, err := (&proto.MigrateTask{
SourceDiskID: testDisk1.DiskID,
TaskID: client.GenMigrateTaskID(proto.TaskTypeDiskRepair, testDisk1.DiskID, proto.Vid(1)),
}).Task()
require.NoError(t, err)
{
mgr := newDiskRepairer(t)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListMigratingDisks(any, any).Return(nil, errMock)
@ -84,10 +106,7 @@ func TestDiskRepairerLoad(t *testing.T) {
{
mgr := newDiskRepairer(t)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListMigratingDisks(any, any).Return(nil, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{{
SourceDiskID: testDisk1.DiskID,
TaskID: client.GenMigrateTaskID(proto.TaskTypeDiskRepair, testDisk1.DiskID, proto.Vid(1)),
}}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{task}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(
&client.DiskInfoSimple{DiskID: testDisk1.DiskID, Status: proto.DiskStatusRepaired}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().DeleteMigrateTask(any, any).Return(nil)
@ -97,10 +116,7 @@ func TestDiskRepairerLoad(t *testing.T) {
{
mgr := newDiskRepairer(t)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListMigratingDisks(any, any).Return(nil, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{{
SourceDiskID: testDisk1.DiskID,
TaskID: client.GenMigrateTaskID(proto.TaskTypeDiskRepair, testDisk1.DiskID, proto.Vid(1)),
}}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{task}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(
&client.DiskInfoSimple{DiskID: testDisk1.DiskID, Status: proto.DiskStatusNormal}, nil)
err := mgr.Load()
@ -109,10 +125,7 @@ func TestDiskRepairerLoad(t *testing.T) {
{
mgr := newDiskRepairer(t)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListMigratingDisks(any, any).Return(nil, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{{
SourceDiskID: testDisk1.DiskID,
TaskID: client.GenMigrateTaskID(proto.TaskTypeDiskRepair, testDisk1.DiskID, proto.Vid(1)),
}}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{task}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetDiskInfo(any, any).Return(nil, errMock)
err := mgr.Load()
require.Error(t, err)
@ -135,42 +148,46 @@ func TestDiskRepairerLoad(t *testing.T) {
{
mgr := newDiskRepairer(t)
t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStateFinishedInAdvance, newMockVolInfoMap())
task, err = t1.Task()
require.NoError(t, err)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListMigratingDisks(any, any).Return([]*client.MigratingDiskMeta{{Disk: &client.DiskInfoSimple{DiskID: proto.DiskID(1)}}}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t1}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{task}, nil)
err := mgr.Load()
require.Error(t, err)
t2 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStateFinished, newMockVolInfoMap())
task, err = t2.Task()
require.NoError(t, err)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListMigratingDisks(any, any).Return([]*client.MigratingDiskMeta{{Disk: &client.DiskInfoSimple{DiskID: proto.DiskID(1)}}}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t2}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{task}, nil)
err = mgr.Load()
require.Error(t, err)
}
{
mgr := newDiskRepairer(t)
t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStateInited, newMockVolInfoMap())
t2 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 2, proto.MigrateStatePrepared, newMockVolInfoMap())
t3 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 4, proto.MigrateStateWorkCompleted, newMockVolInfoMap())
t1, _ := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStateInited, newMockVolInfoMap()).Task()
t2, _ := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 2, proto.MigrateStatePrepared, newMockVolInfoMap()).Task()
t3, _ := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 4, proto.MigrateStateWorkCompleted, newMockVolInfoMap()).Task()
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListMigratingDisks(any, any).Return([]*client.MigratingDiskMeta{{Disk: &client.DiskInfoSimple{DiskID: proto.DiskID(1)}}}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t1, t2, t3}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{t1, t2, t3}, nil)
err := mgr.Load()
require.NoError(t, err)
}
{
// same volume task prepared
mgr := newDiskRepairer(t)
t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap())
t2 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap())
t1, _ := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap()).Task()
t2, _ := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap()).Task()
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListMigratingDisks(any, any).Return([]*client.MigratingDiskMeta{{Disk: &client.DiskInfoSimple{DiskID: proto.DiskID(1)}}}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t1, t2}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{t1, t2}, nil)
err := mgr.Load()
require.Error(t, err)
}
{
mgr := newDiskRepairer(t)
t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateState(111), newMockVolInfoMap())
t1, _ := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateState(111), newMockVolInfoMap()).Task()
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListMigratingDisks(any, any).Return([]*client.MigratingDiskMeta{{Disk: &client.DiskInfoSimple{DiskID: proto.DiskID(1)}}}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t1}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{t1}, nil)
err := mgr.Load()
require.Error(t, err)
}
@ -306,15 +323,15 @@ func TestDiskRepairerCollectTask(t *testing.T) {
}
units = append(units, &ele)
}
t1 := &proto.MigrateTask{
t1, _ := (&proto.MigrateTask{
TaskID: client.GenMigrateTaskID(proto.TaskTypeDiskRepair, proto.DiskID(1), volume.Vid),
TaskType: proto.TaskTypeDiskRepair,
SourceVuid: units[0].Vuid,
}
}).Task()
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListBrokenDisks(any).Return([]*client.DiskInfoSimple{testDisk1}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().SetDiskRepairing(any, any).Return(nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.MigrateTask{t1}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.Task{t1}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AddMigratingDisk(any, any).Return(nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AddMigrateTask(any, any).AnyTimes().Return(nil)
mgr.collectTask()
@ -564,12 +581,13 @@ func TestDiskRepairerCheckRepairedAndClear(t *testing.T) {
}
units = append(units, &ele)
}
t1 := &proto.MigrateTask{
task := &proto.MigrateTask{
TaskID: client.GenMigrateTaskID(proto.TaskTypeDiskRepair, proto.DiskID(1), volume.Vid),
TaskType: proto.TaskTypeDiskRepair,
SourceVuid: units[0].Vuid,
}
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Times(3).Return([]*proto.MigrateTask{t1}, nil)
t1, _ := task.Task()
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Times(3).Return([]*proto.Task{t1}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(units, nil)
mgr.checkRepairedAndClear()
require.True(t, mgr.repairingDisks.size() > 0)
@ -579,11 +597,11 @@ func TestDiskRepairerCheckRepairedAndClear(t *testing.T) {
require.True(t, mgr.repairingDisks.size() > 0)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, nil)
mgr.deletedTasks.add(t1.SourceDiskID, t1.TaskID)
mgr.deletedTasks.add(task.SourceDiskID, t1.TaskID)
mgr.checkRepairedAndClear()
require.True(t, mgr.repairingDisks.size() > 0)
t1.State = proto.MigrateStateFinished
// t1.State = proto.MigrateStateFinished
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Times(4).Return(nil, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDiskVolumeUnits(any, any).Return(nil, errMock)
mgr.checkRepairedAndClear()
@ -654,7 +672,7 @@ func TestDiskRepairerCheckAndClearJunkTasks(t *testing.T) {
// has junk task and clear
mgr.repairedDisks.add(disk1.DiskID, time.Now().Add(-junkMigrationTaskProtectionWindow))
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.MigrateTask{{TaskID: "test"}}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.Task{{TaskID: "test"}}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().DeleteMigrateTask(any, any).Return(nil)
mgr.checkAndClearJunkTasks()
require.Equal(t, 0, mgr.repairedDisks.size())
@ -691,7 +709,7 @@ func TestDiskRepairerCancelTask(t *testing.T) {
idc := "z0"
{
mgr := newDiskRepairer(t)
err := mgr.CancelTask(ctx, &api.OperateTaskArgs{})
err := mgr.CancelTask(ctx, &api.TaskArgs{})
require.Error(t, err)
}
{
@ -699,7 +717,7 @@ func TestDiskRepairerCancelTask(t *testing.T) {
t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap())
mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1)
err := mgr.CancelTask(ctx, &api.OperateTaskArgs{})
err := mgr.CancelTask(ctx, &api.TaskArgs{})
require.Error(t, err)
}
}
@ -710,23 +728,33 @@ func TestDiskRepairerReclaimTask(t *testing.T) {
{
mgr := newDiskRepairer(t)
t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap())
err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{})
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(gomock.Any(), gomock.Any()).Return(nil, errMock)
taskArgs := generateTaskArgs(t1, "")
err := mgr.ReclaimTask(ctx, taskArgs)
require.Error(t, err)
}
{
mgr := newDiskRepairer(t)
t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap())
location := t1.Destination
location.Vuid += 1
mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(gomock.Any(), gomock.Any()).Return(&client.AllocVunitInfo{VunitLocation: location}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateMigrateTask(any, any).Return(errMock)
err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{})
args := generateTaskArgs(t1, "")
err := mgr.ReclaimTask(ctx, args)
require.NoError(t, err)
}
{
mgr := newDiskRepairer(t)
t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap())
location := t1.Destination
location.Vuid += 1
mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(gomock.Any(), gomock.Any()).Return(&client.AllocVunitInfo{VunitLocation: location}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateMigrateTask(any, any).Return(nil)
err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{})
args := generateTaskArgs(t1, "")
err := mgr.ReclaimTask(ctx, args)
require.NoError(t, err)
}
}
@ -737,14 +765,16 @@ func TestDiskRepairerCompleteTask(t *testing.T) {
{
mgr := newDiskRepairer(t)
t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap())
err := mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination})
args := generateTaskArgs(t1, "")
err := mgr.CompleteTask(ctx, args)
require.Error(t, err)
}
{
mgr := newDiskRepairer(t)
t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap())
mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1)
err := mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination})
args := generateTaskArgs(t1, "")
err := mgr.CompleteTask(ctx, args)
require.NoError(t, err)
todo, doing := mgr.finishQueue.StatsTasks()
require.Equal(t, 1, todo+doing)
@ -802,7 +832,7 @@ func TestDiskRepairerQueryTask(t *testing.T) {
}
{
mgr := newDiskRepairer(t)
t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap())
t1, _ := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap()).Task()
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetMigrateTask(any, any, any).Return(t1, nil)
_, err := mgr.QueryTask(ctx, taskID)
require.NoError(t, err)
@ -835,9 +865,9 @@ func TestDiskRepairerProgress(t *testing.T) {
{
mgr := newDiskRepairer(t)
mgr.repairingDisks.add(testDisk1.DiskID, testDisk1)
t1 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap())
t2 := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 2, proto.MigrateStateInited, newMockVolInfoMap())
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.MigrateTask{t1, t2}, nil)
t1, _ := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 1, proto.MigrateStatePrepared, newMockVolInfoMap()).Task()
t2, _ := mockGenMigrateTask(proto.TaskTypeDiskRepair, "z0", 1, 2, proto.MigrateStateInited, newMockVolInfoMap()).Task()
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.Task{t1, t2}, nil)
disks, total, repaired := mgr.Progress(ctx)
require.Equal(t, 1, len(disks))
require.Equal(t, testDisk1.DiskID, disks[0])
@ -848,11 +878,11 @@ func TestDiskRepairerProgress(t *testing.T) {
mgr := newDiskRepairer(t)
mgr.repairingDisks.add(testDisk1.DiskID, testDisk1)
mgr.repairingDisks.add(testDisk2.DiskID, testDisk2)
task1 := &proto.MigrateTask{State: proto.MigrateStatePrepared, SourceDiskID: testDisk1.DiskID}
task2 := &proto.MigrateTask{State: proto.MigrateStateInited, SourceDiskID: testDisk1.DiskID}
task3 := &proto.MigrateTask{State: proto.MigrateStateWorkCompleted, SourceDiskID: testDisk1.DiskID}
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.MigrateTask{task1, task2, task3}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.MigrateTask{task1, task2}, nil)
task1, _ := (&proto.MigrateTask{State: proto.MigrateStatePrepared, SourceDiskID: testDisk1.DiskID}).Task()
task2, _ := (&proto.MigrateTask{State: proto.MigrateStateInited, SourceDiskID: testDisk1.DiskID}).Task()
task3, _ := (&proto.MigrateTask{State: proto.MigrateStateWorkCompleted, SourceDiskID: testDisk1.DiskID}).Task()
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.Task{task1, task2, task3}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.Task{task1, task2}, nil)
disks, tatal, repaired := mgr.Progress(ctx)
require.Equal(t, 2, len(disks))
require.Equal(t, testDisk1.UsedChunkCnt+testDisk2.UsedChunkCnt, int64(tatal))
@ -878,11 +908,11 @@ func TestDiskRepairerDiskProgress(t *testing.T) {
mgr := newDiskRepairer(t)
mgr.repairingDisks.add(testDisk1.DiskID, testDisk1)
task1 := &proto.MigrateTask{State: proto.MigrateStatePrepared, SourceDiskID: testDisk1.DiskID}
task2 := &proto.MigrateTask{State: proto.MigrateStateInited, SourceDiskID: testDisk1.DiskID}
task3 := &proto.MigrateTask{State: proto.MigrateStateWorkCompleted, SourceDiskID: testDisk1.DiskID}
task1, _ := (&proto.MigrateTask{State: proto.MigrateStatePrepared, SourceDiskID: testDisk1.DiskID}).Task()
task2, _ := (&proto.MigrateTask{State: proto.MigrateStateInited, SourceDiskID: testDisk1.DiskID}).Task()
task3, _ := (&proto.MigrateTask{State: proto.MigrateStateWorkCompleted, SourceDiskID: testDisk1.DiskID}).Task()
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.MigrateTask{task1, task2, task3}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.Task{task1, task2, task3}, nil)
stats, err := mgr.DiskProgress(ctx, testDisk1.DiskID)
require.NoError(t, err)
require.Equal(t, int(testDisk1.UsedChunkCnt), stats.TotalTasksCnt)

14
blobstore/scheduler/manual_migrater_test.go
View File

@ -84,7 +84,7 @@ func TestManualMigrateAcquireTask(t *testing.T) {
ctx := context.Background()
idc := "z0"
mgr := newManualMigrater(t)
mgr.IMigrator.(*MockMigrater).EXPECT().AcquireTask(any, any).Return(proto.MigrateTask{TaskType: proto.TaskTypeManualMigrate}, nil)
mgr.IMigrator.(*MockMigrater).EXPECT().AcquireTask(any, any).Return(&proto.Task{TaskType: proto.TaskTypeManualMigrate}, nil)
_, err := mgr.AcquireTask(ctx, idc)
require.NoError(t, err)
}
@ -93,7 +93,7 @@ func TestManualMigrateCancelTask(t *testing.T) {
ctx := context.Background()
mgr := newManualMigrater(t)
mgr.IMigrator.(*MockMigrater).EXPECT().CancelTask(any, any).Return(nil)
err := mgr.CancelTask(ctx, &api.OperateTaskArgs{})
err := mgr.CancelTask(ctx, &api.TaskArgs{})
require.NoError(t, err)
}
@ -101,9 +101,10 @@ func TestManualMigrateReclaimTask(t *testing.T) {
ctx := context.Background()
idc := "z0"
mgr := newManualMigrater(t)
mgr.IMigrator.(*MockMigrater).EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil)
mgr.IMigrator.(*MockMigrater).EXPECT().ReclaimTask(any, any).Return(nil)
t1 := mockGenMigrateTask(proto.TaskTypeManualMigrate, idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap)
err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{})
taskArgs := generateTaskArgs(t1, "")
err := mgr.ReclaimTask(ctx, taskArgs)
require.NoError(t, err)
}
@ -113,11 +114,12 @@ func TestManualMigrateCompleteTask(t *testing.T) {
mgr := newManualMigrater(t)
mgr.IMigrator.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(nil)
t1 := mockGenMigrateTask(proto.TaskTypeManualMigrate, idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap)
err := mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination})
taskArgs := generateTaskArgs(t1, "")
err := mgr.CompleteTask(ctx, taskArgs)
require.NoError(t, err)
mgr.IMigrator.(*MockMigrater).EXPECT().CompleteTask(any, any).Return(errMock)
err = mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination})
err = mgr.CompleteTask(ctx, taskArgs)
require.True(t, errors.Is(err, errMock))
}

254
blobstore/scheduler/migrate.go
View File

@ -16,6 +16,7 @@ package scheduler
import (
"context"
"encoding/json"
"errors"
"fmt"
"sync"
@ -49,19 +50,30 @@ type MMigrator interface {
IManualMigrator
}
// Migrator base interface of migrate, balancer, disk_droper, manual_migrater.
type Migrator interface {
AcquireTask(ctx context.Context, idc string) (proto.MigrateTask, error)
CancelTask(ctx context.Context, args *api.OperateTaskArgs) error
CompleteTask(ctx context.Context, args *api.OperateTaskArgs) error
ReclaimTask(ctx context.Context, idc, taskID string,
src []proto.VunitLocation, oldDst proto.VunitLocation, newDst *client.AllocVunitInfo) error
var (
_ BaseMigrator = (*DiskRepairMgr)(nil)
_ BaseMigrator = (*MigrateMgr)(nil)
)
// BaseMigrator base interface for shard and blobnode task.
type BaseMigrator interface {
AcquireTask(ctx context.Context, idc string) (*proto.Task, error)
RenewalTask(ctx context.Context, idc, taskID string) error
QueryTask(ctx context.Context, taskID string) (*api.MigrateTaskDetail, error)
CancelTask(ctx context.Context, args *api.TaskArgs) error
CompleteTask(ctx context.Context, args *api.TaskArgs) error
ReclaimTask(ctx context.Context, args *api.TaskArgs) error
ReportTask(ctx context.Context, args *api.TaskArgs) (err error)
QueryTask(ctx context.Context, taskID string) (*api.TaskRet, error)
}
// Migrator interface of blobnode migrate, balancer, disk_dropper, manual_migrator.
type Migrator interface {
BaseMigrator
// status
ReportWorkerTaskStats(st *api.TaskReportArgs)
StatQueueTaskCnt() (inited, prepared, completed int)
Stats() api.MigrateTasksStat
// control
taskswitch.ISwitcher
closer.Closer
@ -426,33 +438,38 @@ func (mgr *MigrateMgr) Load() (err error) {
}
var junkTasks []*proto.MigrateTask
for i := range tasks {
if mgr.isJunkTask(disks, tasks[i]) {
junkTasks = append(junkTasks, tasks[i])
task := &proto.MigrateTask{}
err = task.Unmarshal(tasks[i].Data)
if err != nil {
return err
}
if mgr.isJunkTask(disks, task) {
junkTasks = append(junkTasks, task)
continue
}
if tasks[i].Running() {
err = base.VolTaskLockerInst().TryLock(ctx, tasks[i].SourceVuid.Vid())
if task.Running() {
err = base.VolTaskLockerInst().TryLock(ctx, task.SourceVuid.Vid())
if err != nil {
return fmt.Errorf("migrate task conflict: vid[%d], task[%+v], err[%+v]",
tasks[i].SourceVuid.Vid(), tasks[i], err)
task.SourceVuid.Vid(), tasks[i], err)
}
}
mgr.loadTaskCallback(tasks[i].SourceDiskID)
mgr.addMigratingVuid(tasks[i].SourceDiskID, tasks[i].SourceVuid, tasks[i].TaskID)
mgr.loadTaskCallback(task.SourceDiskID)
mgr.addMigratingVuid(task.SourceDiskID, task.SourceVuid, task.TaskID)
span.Infof("load task success: task_type[%s], task_id[%s], state[%d]", mgr.taskType, tasks[i].TaskID, tasks[i].State)
switch tasks[i].State {
span.Infof("load task success: task_type[%s], task_id[%s], state[%d]", mgr.taskType, task.TaskID, task.State)
switch task.State {
case proto.MigrateStateInited:
mgr.prepareQueue.PushTask(tasks[i].TaskID, tasks[i])
mgr.prepareQueue.PushTask(task.TaskID, task)
case proto.MigrateStatePrepared:
mgr.workQueue.AddPreparedTask(tasks[i].SourceIDC, tasks[i].TaskID, tasks[i])
mgr.workQueue.AddPreparedTask(task.SourceIDC, task.TaskID, task)
case proto.MigrateStateWorkCompleted:
mgr.finishQueue.PushTask(tasks[i].TaskID, tasks[i])
mgr.finishQueue.PushTask(task.TaskID, task)
case proto.MigrateStateFinished, proto.MigrateStateFinishedInAdvance:
return fmt.Errorf("task should be deleted from db: task[%+v]", tasks[i])
return fmt.Errorf("task should be deleted from db: task[%+v]", task)
default:
return fmt.Errorf("unexpect migrate state: task[%+v]", tasks[i])
return fmt.Errorf("unexpect migrate state: task[%+v]", task)
}
}
return mgr.clearJunkTasksCallBack(ctx, junkTasks)
@ -591,7 +608,11 @@ func (mgr *MigrateMgr) prepareTask() (err error) {
// update db
base.InsistOn(ctx, "migrate prepare task update task tbl", func() error {
return mgr.clusterMgrCli.UpdateMigrateTask(ctx, migTask)
task, err := migTask.Task()
if err != nil {
return err
}
return mgr.clusterMgrCli.UpdateMigrateTask(ctx, task)
})
// send task to worker queue and remove task in prepareQueue
@ -636,7 +657,11 @@ func (mgr *MigrateMgr) finishTask() (err error) {
// because competed task did not persisted to the database, so in finish phase need to do it
// the task maybe update more than once, which is allowed
base.InsistOn(ctx, "migrate finish task update task tbl to state completed ", func() error {
return mgr.clusterMgrCli.UpdateMigrateTask(ctx, migrateTask)
task, err := migrateTask.Task()
if err != nil {
return err
}
return mgr.clusterMgrCli.UpdateMigrateTask(ctx, task)
})
// update volume mapping relationship
@ -658,7 +683,6 @@ func (mgr *MigrateMgr) finishTask() (err error) {
err = mgr.handleUpdateVolMappingFail(ctx, migrateTask, err)
return
}
span.Warnf("task_id[%s] update volume failed but updated by tiomeout request", migrateTask.TaskID)
}
err = mgr.clusterMgrCli.ReleaseVolumeUnit(ctx, migrateTask.SourceVuid, migrateTask.SourceDiskID)
@ -720,7 +744,11 @@ func (mgr *MigrateMgr) updateVolumeCache(ctx context.Context, task *proto.Migrat
func (mgr *MigrateMgr) AddTask(ctx context.Context, task *proto.MigrateTask) {
// add task to db
base.InsistOn(ctx, "migrate add task insert task to tbl", func() error {
return mgr.clusterMgrCli.AddMigrateTask(ctx, task)
t, err := task.Task()
if err != nil {
return err
}
return mgr.clusterMgrCli.AddMigrateTask(ctx, t)
})
// add task to prepare queue
@ -778,7 +806,11 @@ func (mgr *MigrateMgr) handleUpdateVolMappingFail(ctx context.Context, task *pro
task.WorkerRedoCnt++
base.InsistOn(ctx, "migrate redo task update task tbl", func() error {
return mgr.clusterMgrCli.UpdateMigrateTask(ctx, task)
t, err := task.Task()
if err != nil {
return err
}
return mgr.clusterMgrCli.UpdateMigrateTask(ctx, t)
})
_ = mgr.finishQueue.RemoveTask(task.TaskID)
@ -874,80 +906,128 @@ func (mgr *MigrateMgr) Stats() api.MigrateTasksStat {
}
// AcquireTask acquire migrate task
func (mgr *MigrateMgr) AcquireTask(ctx context.Context, idc string) (task proto.MigrateTask, err error) {
func (mgr *MigrateMgr) AcquireTask(ctx context.Context, idc string) (task *proto.Task, err error) {
span := trace.SpanFromContextSafe(ctx)
task = &proto.Task{}
task.ModuleType = proto.TypeBlobNode
if !mgr.taskSwitch.Enabled() {
return task, proto.ErrTaskPaused
}
_, migTask, _ := mgr.workQueue.Acquire(idc)
if migTask != nil {
task = *migTask.(*proto.MigrateTask)
span.Infof("acquire %s taskId: %s", mgr.taskType, task.TaskID)
t := *migTask.(*proto.MigrateTask)
data, err := t.Marshal()
if err != nil {
return task, err
}
task.TaskID = t.TaskID
task.Data = data
span.Infof("acquire %s taskId: %s", mgr.taskType, t.TaskID)
return task, nil
}
return task, proto.ErrTaskEmpty
}
// CancelTask cancel migrate task
func (mgr *MigrateMgr) CancelTask(ctx context.Context, args *api.OperateTaskArgs) (err error) {
func (mgr *MigrateMgr) CancelTask(ctx context.Context, args *api.TaskArgs) (err error) {
mgr.taskStatsMgr.CancelTask()
err = mgr.workQueue.Cancel(args.IDC, args.TaskID, args.Src, args.Dest)
arg := &api.OperateTaskArgs{}
err = arg.Unmarshal(args.Data)
if err != nil {
return err
}
if !client.ValidMigrateTask(args.TaskType, arg.TaskID) {
return errcode.ErrIllegalArguments
}
err = mgr.workQueue.Cancel(arg.IDC, arg.TaskID, arg.Src, arg.Dest)
if err != nil {
span := trace.SpanFromContextSafe(ctx)
span.Errorf("cancel migrate failed: task_type[%s], task_id[%s], err[%+v]", mgr.taskType, args.TaskID, err)
span.Errorf("cancel migrate failed: task_type[%s], task_id[%s], err[%+v]", mgr.taskType, arg.TaskID, err)
}
return
}
// ReclaimTask reclaim migrate task
func (mgr *MigrateMgr) ReclaimTask(ctx context.Context, idc, taskID string,
src []proto.VunitLocation, oldDst proto.VunitLocation, newDst *client.AllocVunitInfo,
) (err error) {
func (mgr *MigrateMgr) ReclaimTask(ctx context.Context, args *api.TaskArgs) (err error) {
mgr.taskStatsMgr.ReclaimTask()
span := trace.SpanFromContextSafe(ctx)
err = mgr.workQueue.Reclaim(idc, taskID, src, oldDst, newDst.Location(), newDst.DiskID)
arg := &api.OperateTaskArgs{}
err = arg.Unmarshal(args.Data)
if err != nil {
span.Errorf("reclaim migrate task failed: task_type:[%s],task_id[%s], err[%+v]", mgr.taskType, taskID, err)
return err
}
task, err := mgr.workQueue.Query(idc, taskID)
if !client.ValidMigrateTask(args.TaskType, arg.TaskID) {
return errcode.ErrIllegalArguments
}
newDst, err := base.AllocVunitSafe(ctx, mgr.clusterMgrCli, arg.Dest.Vuid, arg.Src)
if err != nil {
span.Errorf("found task in workQueue failed: idc[%s], task_id[%s], err[%+v]", idc, taskID, err)
span.Errorf("alloc volume unit from clustermgr failed, err: %s", err)
return err
}
err = mgr.clusterMgrCli.UpdateMigrateTask(ctx, task.(*proto.MigrateTask))
err = mgr.workQueue.Reclaim(arg.IDC, arg.TaskID, arg.Src, arg.Dest, newDst.Location(), newDst.DiskID)
if err != nil {
span.Errorf("update reclaim task failed: task_id[%s], err[%+v]", taskID, err)
span.Errorf("reclaim migrate task failed: task_type:[%s],task_id[%s], err[%+v]", mgr.taskType, arg.TaskID, err)
return err
}
task, err := mgr.workQueue.Query(arg.IDC, arg.TaskID)
if err != nil {
span.Errorf("found task in workQueue failed: idc[%s], task_id[%s], err[%+v]", arg.IDC, arg.TaskID, err)
return err
}
t, err := task.Task()
if err != nil {
return err
}
err = mgr.clusterMgrCli.UpdateMigrateTask(ctx, t)
if err != nil {
span.Errorf("update reclaim task failed: task_id[%s], err[%+v]", arg.TaskID, err)
}
return
}
// CompleteTask complete migrate task
func (mgr *MigrateMgr) CompleteTask(ctx context.Context, args *api.OperateTaskArgs) (err error) {
func (mgr *MigrateMgr) CompleteTask(ctx context.Context, args *api.TaskArgs) (err error) {
span := trace.SpanFromContextSafe(ctx)
completeTask, err := mgr.workQueue.Complete(args.IDC, args.TaskID, args.Src, args.Dest)
arg := &api.OperateTaskArgs{}
err = arg.Unmarshal(args.Data)
if err != nil {
span.Errorf("complete migrate task failed: task_id[%s], err[%+v]", args.TaskID, err)
return err
}
if !client.ValidMigrateTask(args.TaskType, arg.TaskID) {
return errcode.ErrIllegalArguments
}
completeTask, err := mgr.workQueue.Complete(arg.IDC, arg.TaskID, arg.Src, arg.Dest)
if err != nil {
span.Errorf("complete migrate task failed: task_id[%s], err[%+v]", arg.TaskID, err)
return err
}
t := completeTask.(*proto.MigrateTask)
t.State = proto.MigrateStateWorkCompleted
err = mgr.clusterMgrCli.UpdateMigrateTask(ctx, t)
task, err := t.Task()
if err != nil {
return err
}
err = mgr.clusterMgrCli.UpdateMigrateTask(ctx, task)
if err != nil {
// there is no impact if we failed to update task state in db,
// because we will do it in finishTask again, so assume complete success
span.Errorf("complete migrate task into db failed: task_id[%s], err[%+v]", t.TaskID, err)
err = nil
}
mgr.finishQueue.PushTask(args.TaskID, t)
mgr.finishQueue.PushTask(arg.TaskID, t)
return
}
@ -977,34 +1057,71 @@ func (mgr *MigrateMgr) GetMigratingDiskNum() int {
// ListAllTask returns all migrate task
func (mgr *MigrateMgr) ListAllTask(ctx context.Context) (tasks []*proto.MigrateTask, err error) {
return mgr.clusterMgrCli.ListAllMigrateTasks(ctx, mgr.taskType)
ts, err := mgr.clusterMgrCli.ListAllMigrateTasks(ctx, mgr.taskType)
if err != nil {
return nil, err
}
for _, t := range ts {
task := &proto.MigrateTask{}
err = task.Unmarshal(t.Data)
if err != nil {
return nil, err
}
tasks = append(tasks, task)
}
return
}
// ListAllTaskByDiskID return all task by diskID
func (mgr *MigrateMgr) ListAllTaskByDiskID(ctx context.Context, diskID proto.DiskID) (tasks []*proto.MigrateTask, err error) {
return mgr.clusterMgrCli.ListAllMigrateTasksByDiskID(ctx, mgr.taskType, diskID)
ts, err := mgr.clusterMgrCli.ListAllMigrateTasksByDiskID(ctx, mgr.taskType, diskID)
if err != nil {
return nil, err
}
for _, t := range ts {
task := &proto.MigrateTask{}
err = task.Unmarshal(t.Data)
if err != nil {
return nil, err
}
tasks = append(tasks, task)
}
return
}
// GetTask returns task in db
func (mgr *MigrateMgr) GetTask(ctx context.Context, taskID string) (*proto.MigrateTask, error) {
return mgr.clusterMgrCli.GetMigrateTask(ctx, mgr.taskType, taskID)
task, err := mgr.clusterMgrCli.GetMigrateTask(ctx, mgr.taskType, taskID)
if err != nil {
return nil, err
}
ret := &proto.MigrateTask{}
err = ret.Unmarshal(task.Data)
if err != nil {
return nil, err
}
return ret, nil
}
// QueryTask implement migrator
func (mgr *MigrateMgr) QueryTask(ctx context.Context, taskID string) (*api.MigrateTaskDetail, error) {
func (mgr *MigrateMgr) QueryTask(ctx context.Context, taskID string) (*api.TaskRet, error) {
detail := &api.MigrateTaskDetail{}
taskInfo, err := mgr.GetTask(ctx, taskID)
task, err := mgr.GetTask(ctx, taskID)
if err != nil {
return detail, err
return nil, err
}
detail.Task = *taskInfo
detail.Task = *task
detailRunInfo, err := mgr.taskStatsMgr.QueryTaskDetail(taskID)
if err != nil {
return detail, nil
if err == nil {
detail.Stat = detailRunInfo.Statistics
}
detail.Stat = detailRunInfo.Statistics
return detail, nil
data, err := json.Marshal(detail)
if err != nil {
return nil, err
}
return &api.TaskRet{TaskType: task.TaskType, Data: data}, nil
}
// ReportWorkerTaskStats implement migrator
@ -1033,3 +1150,18 @@ func checkMigrateConf(conf *MigrateConfig) {
conf.loadTaskCallback = defaultDiskTaskLimitFunc
}
}
func (mgr *MigrateMgr) ReportTask(ctx context.Context, args *api.TaskArgs) (err error) {
arg := &api.TaskReportArgs{}
err = arg.Unmarshal(args.Data)
if err != nil {
return err
}
if !client.ValidMigrateTask(arg.TaskType, arg.TaskID) {
return errcode.ErrIllegalArguments
}
mgr.ReportWorkerTaskStats(arg)
return nil
}

92
blobstore/scheduler/migrate_test.go
View File

@ -86,34 +86,34 @@ func TestMigrateMigrateLoad(t *testing.T) {
require.True(t, errors.Is(err, errMock))
}
{
t1 := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 4, 100, proto.MigrateStateInited, MockMigrateVolInfoMap)
t2 := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 5, 101, proto.MigrateStatePrepared, MockMigrateVolInfoMap)
t3 := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z1", 6, 102, proto.MigrateStateWorkCompleted, MockMigrateVolInfoMap)
t4 := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 4, 105, proto.MigrateStateInited, MockMigrateVolInfoMap)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t1, t2, t3, t4}, nil)
t1, _ := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 4, 100, proto.MigrateStateInited, MockMigrateVolInfoMap).Task()
t2, _ := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 5, 101, proto.MigrateStatePrepared, MockMigrateVolInfoMap).Task()
t3, _ := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z1", 6, 102, proto.MigrateStateWorkCompleted, MockMigrateVolInfoMap).Task()
t4, _ := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 4, 105, proto.MigrateStateInited, MockMigrateVolInfoMap).Task()
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{t1, t2, t3, t4}, nil)
err := mgr.Load()
require.NoError(t, err)
}
{
t1 := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z2", 8, 104, proto.MigrateStateFinished, MockMigrateVolInfoMap)
t2 := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 4, 105, proto.MigrateStateFinishedInAdvance, MockMigrateVolInfoMap)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t1}, nil)
t1, _ := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z2", 8, 104, proto.MigrateStateFinished, MockMigrateVolInfoMap).Task()
t2, _ := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 4, 105, proto.MigrateStateFinishedInAdvance, MockMigrateVolInfoMap).Task()
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{t1}, nil)
err := mgr.Load()
require.Error(t, err)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t2}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{t2}, nil)
err = mgr.Load()
require.Error(t, err)
}
{
t2 := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 5, 101, proto.MigrateStatePrepared, MockMigrateVolInfoMap)
t3 := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z1", 6, 101, proto.MigrateStateWorkCompleted, MockMigrateVolInfoMap)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t2, t3}, nil)
t2, _ := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 5, 101, proto.MigrateStatePrepared, MockMigrateVolInfoMap).Task()
t3, _ := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z1", 6, 101, proto.MigrateStateWorkCompleted, MockMigrateVolInfoMap).Task()
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{t2, t3}, nil)
err := mgr.Load()
require.Error(t, err)
t4 := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z2", 7, 103, 100, MockMigrateVolInfoMap)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t4}, nil)
t4, _ := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z2", 7, 103, 100, MockMigrateVolInfoMap).Task()
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{t4}, nil)
err = mgr.Load()
require.Error(t, err)
}
@ -121,12 +121,12 @@ func TestMigrateMigrateLoad(t *testing.T) {
mgr := newMigrateMgr(t)
mgr.taskType = proto.TaskTypeDiskDrop
t1 := mockGenMigrateTask(proto.TaskTypeDiskDrop, "z0", 1, 110, proto.MigrateStateInited, MockMigrateVolInfoMap)
t2 := mockGenMigrateTask(proto.TaskTypeDiskDrop, "z0", 2, 111, proto.MigrateStatePrepared, MockMigrateVolInfoMap)
t3 := mockGenMigrateTask(proto.TaskTypeDiskDrop, "z1", 3, 112, proto.MigrateStateWorkCompleted, MockMigrateVolInfoMap)
t4 := mockGenMigrateTask(proto.TaskTypeDiskDrop, "z0", 4, 113, proto.MigrateStateInited, MockMigrateVolInfoMap)
t1, _ := mockGenMigrateTask(proto.TaskTypeDiskDrop, "z0", 1, 110, proto.MigrateStateInited, MockMigrateVolInfoMap).Task()
t2, _ := mockGenMigrateTask(proto.TaskTypeDiskDrop, "z0", 2, 111, proto.MigrateStatePrepared, MockMigrateVolInfoMap).Task()
t3, _ := mockGenMigrateTask(proto.TaskTypeDiskDrop, "z1", 3, 112, proto.MigrateStateWorkCompleted, MockMigrateVolInfoMap).Task()
t4, _ := mockGenMigrateTask(proto.TaskTypeDiskDrop, "z0", 4, 113, proto.MigrateStateInited, MockMigrateVolInfoMap).Task()
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t1, t2, t3, t4}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{t1, t2, t3, t4}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListDropDisks(any).Return([]*client.DiskInfoSimple{testDisk1, testDisk2}, nil)
err := mgr.Load()
require.NoError(t, err)
@ -390,7 +390,8 @@ func TestCancelMigrateTask(t *testing.T) {
idc := "z0"
{
mgr := newMigrateMgr(t)
err := mgr.CancelTask(ctx, &api.OperateTaskArgs{IDC: idc})
err := mgr.CancelTask(ctx, &api.TaskArgs{})
require.Error(t, err)
}
{
@ -399,10 +400,10 @@ func TestCancelMigrateTask(t *testing.T) {
mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1)
// no such task
err := mgr.CancelTask(ctx, &api.OperateTaskArgs{IDC: idc})
err := mgr.CancelTask(ctx, &api.TaskArgs{})
require.Error(t, err)
err = mgr.CancelTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination})
taskArgs := generateTaskArgs(t1, "")
err = mgr.CancelTask(ctx, taskArgs)
require.NoError(t, err)
}
}
@ -413,22 +414,33 @@ func TestReclaimMigrateTask(t *testing.T) {
{
// no task
mgr := newMigrateMgr(t)
err := mgr.ReclaimTask(ctx, idc, "", nil, proto.VunitLocation{}, &client.AllocVunitInfo{})
err := mgr.ReclaimTask(ctx, &api.TaskArgs{})
require.Error(t, err)
}
{
mgr := newMigrateMgr(t)
t1 := mockGenMigrateTask(proto.TaskTypeManualMigrate, idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap)
location := t1.Destination
location.Vuid += 1
mgr.workQueue.AddPreparedTask(idc, t1.TaskID, t1)
// update failed
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(gomock.Any(), gomock.Any()).Return(&client.AllocVunitInfo{VunitLocation: location}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateMigrateTask(any, any).Return(errMock)
err := mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{})
taskArgs := generateTaskArgs(t1, "")
err := mgr.ReclaimTask(ctx, taskArgs)
require.True(t, errors.Is(err, errMock))
// update success
task, err := mgr.workQueue.Query(t1.SourceIDC, t1.TaskID)
require.NoError(t, err)
t1 = task.(*proto.MigrateTask)
taskArgs = generateTaskArgs(t1, "")
location = t1.Sources[0]
location.Vuid += 2
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().AllocVolumeUnit(gomock.Any(), gomock.Any()).Return(&client.AllocVunitInfo{VunitLocation: location}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateMigrateTask(any, any).Return(nil)
err = mgr.ReclaimTask(ctx, idc, t1.TaskID, t1.Sources, t1.Destination, &client.AllocVunitInfo{})
err = mgr.ReclaimTask(ctx, taskArgs)
require.NoError(t, err)
}
}
@ -439,7 +451,7 @@ func TestCompleteMigrateTask(t *testing.T) {
{
// no task
mgr := newMigrateMgr(t)
err := mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc})
err := mgr.CompleteTask(ctx, &api.TaskArgs{})
require.Error(t, err)
}
{
@ -449,18 +461,20 @@ func TestCompleteMigrateTask(t *testing.T) {
// update failed
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateMigrateTask(any, any).Return(errMock)
err := mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination})
taskArgs := generateTaskArgs(t1, "")
err := mgr.CompleteTask(ctx, taskArgs)
require.NoError(t, err)
// no task in queue
err = mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t1.TaskID, Src: t1.Sources, Dest: t1.Destination})
err = mgr.CompleteTask(ctx, taskArgs)
require.Error(t, err)
// update success
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().UpdateMigrateTask(any, any).Return(nil)
t2 := mockGenMigrateTask(proto.TaskTypeManualMigrate, idc, 4, 100, proto.MigrateStatePrepared, MockMigrateVolInfoMap)
mgr.workQueue.AddPreparedTask(idc, t2.TaskID, t2)
err = mgr.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskID: t2.TaskID, Src: t2.Sources, Dest: t2.Destination})
args := generateTaskArgs(t2, "")
err = mgr.CompleteTask(ctx, args)
require.NoError(t, err)
}
}
@ -522,18 +536,18 @@ func TestAddMigrateTask(t *testing.T) {
require.Equal(t, 1, inited)
require.Equal(t, 0, prepared)
require.Equal(t, 0, completed)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.MigrateTask{t1}, nil)
task, err := t1.Task()
require.NoError(t, err)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasks(any, any).Return([]*proto.Task{task}, nil)
tasks, err := mgr.ListAllTask(ctx)
require.NoError(t, err)
require.Equal(t, 1, len(tasks))
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetMigrateTask(any, any, any).Return(t1, nil)
task, err := mgr.GetTask(ctx, t1.TaskID)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetMigrateTask(any, any, any).Return(task, nil)
tsk, err := mgr.GetTask(ctx, t1.TaskID)
require.NoError(t, err)
require.Equal(t, t1.TaskID, task.TaskID)
require.Equal(t, t1.TaskID, tsk.TaskID)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.MigrateTask{}, nil)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().ListAllMigrateTasksByDiskID(any, any, any).Return([]*proto.Task{}, nil)
_, err = mgr.ListAllTaskByDiskID(ctx, proto.DiskID(1))
require.NoError(t, err)
}
@ -607,7 +621,9 @@ func TestMigrateQueryTask(t *testing.T) {
_, err := mgr.QueryTask(ctx, taskID)
require.ErrorIs(t, errMock, err)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetMigrateTask(any, any, any).Return(&proto.MigrateTask{}, nil)
t1, err := mockGenMigrateTask(proto.TaskTypeManualMigrate, "z0", 4, 100, proto.MigrateStateInited, MockMigrateVolInfoMap).Task()
require.NoError(t, err)
mgr.clusterMgrCli.(*MockClusterMgrAPI).EXPECT().GetMigrateTask(any, any, any).Return(t1, nil)
_, err = mgr.QueryTask(ctx, taskID)
require.NoError(t, err)
}

34
blobstore/scheduler/scheduler_mock_test.go
View File

@ -196,10 +196,10 @@ func (m *MockMigrater) EXPECT() *MockMigraterMockRecorder {
}
// AcquireTask mocks base method.
func (m *MockMigrater) AcquireTask(arg0 context.Context, arg1 string) (proto.MigrateTask, error) {
func (m *MockMigrater) AcquireTask(arg0 context.Context, arg1 string) (*proto.Task, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "AcquireTask", arg0, arg1)
ret0, _ := ret[0].(proto.MigrateTask)
ret0, _ := ret[0].(*proto.Task)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@ -237,7 +237,7 @@ func (mr *MockMigraterMockRecorder) AddTask(arg0, arg1 interface{}) *gomock.Call
}
// CancelTask mocks base method.
func (m *MockMigrater) CancelTask(arg0 context.Context, arg1 *scheduler.OperateTaskArgs) error {
func (m *MockMigrater) CancelTask(arg0 context.Context, arg1 *scheduler.TaskArgs) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "CancelTask", arg0, arg1)
ret0, _ := ret[0].(error)
@ -287,7 +287,7 @@ func (mr *MockMigraterMockRecorder) Close() *gomock.Call {
}
// CompleteTask mocks base method.
func (m *MockMigrater) CompleteTask(arg0 context.Context, arg1 *scheduler.OperateTaskArgs) error {
func (m *MockMigrater) CompleteTask(arg0 context.Context, arg1 *scheduler.TaskArgs) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "CompleteTask", arg0, arg1)
ret0, _ := ret[0].(error)
@ -475,10 +475,10 @@ func (mr *MockMigraterMockRecorder) Progress(arg0 interface{}) *gomock.Call {
}
// QueryTask mocks base method.
func (m *MockMigrater) QueryTask(arg0 context.Context, arg1 string) (*scheduler.MigrateTaskDetail, error) {
func (m *MockMigrater) QueryTask(arg0 context.Context, arg1 string) (*scheduler.TaskRet, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "QueryTask", arg0, arg1)
ret0, _ := ret[0].(*scheduler.MigrateTaskDetail)
ret0, _ := ret[0].(*scheduler.TaskRet)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@ -490,17 +490,17 @@ func (mr *MockMigraterMockRecorder) QueryTask(arg0, arg1 interface{}) *gomock.Ca
}
// ReclaimTask mocks base method.
func (m *MockMigrater) ReclaimTask(arg0 context.Context, arg1, arg2 string, arg3 []proto.VunitLocation, arg4 proto.VunitLocation, arg5 *client.AllocVunitInfo) error {
func (m *MockMigrater) ReclaimTask(arg0 context.Context, arg1 *scheduler.TaskArgs) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ReclaimTask", arg0, arg1, arg2, arg3, arg4, arg5)
ret := m.ctrl.Call(m, "ReclaimTask", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// ReclaimTask indicates an expected call of ReclaimTask.
func (mr *MockMigraterMockRecorder) ReclaimTask(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call {
func (mr *MockMigraterMockRecorder) ReclaimTask(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReclaimTask", reflect.TypeOf((*MockMigrater)(nil).ReclaimTask), arg0, arg1, arg2, arg3, arg4, arg5)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReclaimTask", reflect.TypeOf((*MockMigrater)(nil).ReclaimTask), arg0, arg1)
}
// RenewalTask mocks base method.
@ -517,6 +517,20 @@ func (mr *MockMigraterMockRecorder) RenewalTask(arg0, arg1, arg2 interface{}) *g
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RenewalTask", reflect.TypeOf((*MockMigrater)(nil).RenewalTask), arg0, arg1, arg2)
}
// ReportTask mocks base method.
func (m *MockMigrater) ReportTask(arg0 context.Context, arg1 *scheduler.TaskArgs) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ReportTask", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// ReportTask indicates an expected call of ReportTask.
func (mr *MockMigraterMockRecorder) ReportTask(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportTask", reflect.TypeOf((*MockMigrater)(nil).ReportTask), arg0, arg1)
}
// ReportWorkerTaskStats mocks base method.
func (m *MockMigrater) ReportWorkerTaskStats(arg0 *scheduler.TaskReportArgs) {
m.ctrl.T.Helper()

67
blobstore/scheduler/service.go
View File

@ -53,7 +53,7 @@ type Service struct {
clusterMgrCli client.ClusterMgrAPI
}
func (svr *Service) mgrByType(typ proto.TaskType) (Migrator, error) {
func (svr *Service) mgrByType(typ proto.TaskType) (BaseMigrator, error) {
switch typ {
case proto.TaskTypeDiskRepair:
return svr.diskRepairMgr, nil
@ -63,6 +63,15 @@ func (svr *Service) mgrByType(typ proto.TaskType) (Migrator, error) {
return svr.diskDropMgr, nil
case proto.TaskTypeManualMigrate:
return svr.manualMigMgr, nil
case proto.TaskTypeShardDiskRepair:
// todo
return nil, nil
case proto.TaskTypeShardInspect:
return nil, errIllegalTaskType
case proto.TaskTypeShardMigrate:
return nil, errIllegalTaskType
case proto.TaskTypeShardDiskDrop:
return nil, errIllegalTaskType
default:
return nil, errIllegalTaskType
}
@ -89,7 +98,7 @@ func (svr *Service) HTTPTaskAcquire(c *rpc.Context) {
// acquire task ordered: returns disk repair task first and other random
ctx := c.Request.Context()
migrators := []Migrator{svr.diskRepairMgr, svr.manualMigMgr, svr.diskDropMgr, svr.balanceMgr}
migrators := []BaseMigrator{svr.diskRepairMgr, svr.manualMigMgr, svr.diskDropMgr, svr.balanceMgr}
shuffledMigrators := migrators[1:]
rand.Shuffle(len(shuffledMigrators), func(i, j int) {
shuffledMigrators[i], shuffledMigrators[j] = shuffledMigrators[j], shuffledMigrators[i]
@ -105,15 +114,12 @@ func (svr *Service) HTTPTaskAcquire(c *rpc.Context) {
// HTTPTaskReclaim reclaim task
func (svr *Service) HTTPTaskReclaim(c *rpc.Context) {
args := new(api.OperateTaskArgs)
args := new(api.TaskArgs)
if err := c.ParseArgs(args); err != nil {
c.RespondError(err)
return
}
if !client.ValidMigrateTask(args.TaskType, args.TaskID) {
c.RespondError(errcode.ErrIllegalArguments)
return
}
ctx := c.Request.Context()
reclaimer, err := svr.mgrByType(args.TaskType)
if err != nil {
@ -121,30 +127,16 @@ func (svr *Service) HTTPTaskReclaim(c *rpc.Context) {
return
}
if int(args.Dest.Vuid.Index()) >= len(args.Src) || args.Dest.Vuid.Index() != args.Src[args.Dest.Vuid.Index()].Vuid.Index() {
c.RespondError(errcode.ErrIllegalArguments)
return
}
newDst, err := base.AllocVunitSafe(ctx, svr.clusterMgrCli, args.Src[args.Dest.Vuid.Index()].Vuid, args.Src)
if err != nil {
c.RespondError(err)
return
}
c.RespondError(reclaimer.ReclaimTask(ctx, args.IDC, args.TaskID, args.Src, args.Dest, newDst))
c.RespondError(reclaimer.ReclaimTask(ctx, args))
}
// HTTPTaskCancel cancel task
func (svr *Service) HTTPTaskCancel(c *rpc.Context) {
args := new(api.OperateTaskArgs)
args := new(api.TaskArgs)
if err := c.ParseArgs(args); err != nil {
c.RespondError(err)
return
}
if !client.ValidMigrateTask(args.TaskType, args.TaskID) {
c.RespondError(errcode.ErrIllegalArguments)
return
}
ctx := c.Request.Context()
canceler, err := svr.mgrByType(args.TaskType)
if err != nil {
@ -156,15 +148,11 @@ func (svr *Service) HTTPTaskCancel(c *rpc.Context) {
// HTTPTaskComplete complete task
func (svr *Service) HTTPTaskComplete(c *rpc.Context) {
args := new(api.OperateTaskArgs)
args := new(api.TaskArgs)
if err := c.ParseArgs(args); err != nil {
c.RespondError(err)
return
}
if !client.ValidMigrateTask(args.TaskType, args.TaskID) {
c.RespondError(errcode.ErrIllegalArguments)
return
}
ctx := c.Request.Context()
completer, err := svr.mgrByType(args.TaskType)
if err != nil {
@ -178,9 +166,9 @@ func (svr *Service) HTTPTaskComplete(c *rpc.Context) {
func (svr *Service) HTTPInspectAcquire(c *rpc.Context) {
ctx := c.Request.Context()
task, _ := svr.inspectMgr.AcquireInspect(ctx)
if task != nil {
c.RespondJSON(task)
inspectTask, _ := svr.inspectMgr.AcquireInspect(ctx)
if inspectTask != nil {
c.RespondJSON(inspectTask)
return
}
c.RespondError(errcode.ErrNothingTodo)
@ -236,26 +224,25 @@ func (svr *Service) HTTPTaskRenewal(c *rpc.Context) {
// HTTPTaskReport reports task stats
func (svr *Service) HTTPTaskReport(c *rpc.Context) {
args := new(api.TaskReportArgs)
args := new(api.TaskArgs)
if err := c.ParseArgs(args); err != nil {
c.RespondError(err)
return
}
if !client.ValidMigrateTask(args.TaskType, args.TaskID) {
c.RespondError(errcode.ErrIllegalArguments)
return
}
reporter, err := svr.mgrByType(args.TaskType)
if err != nil {
c.RespondError(err)
return
}
reporter.ReportWorkerTaskStats(args)
if err := reporter.ReportTask(c.Request.Context(), args); err != nil {
c.RespondError(err)
return
}
c.Respond()
}
// HTTPMigrateTaskDetail returns migrate task detail.
func (svr *Service) HTTPMigrateTaskDetail(c *rpc.Context) {
// HTTPTaskDetail returns migrate task detail.
func (svr *Service) HTTPTaskDetail(c *rpc.Context) {
args := new(api.MigrateTaskDetailArgs)
if err := c.ParseArgs(args); err != nil {
c.RespondError(err)
@ -276,7 +263,7 @@ func (svr *Service) HTTPMigrateTaskDetail(c *rpc.Context) {
c.RespondError(rpc.NewError(http.StatusNotFound, "NotFound", err))
return
}
c.RespondJSON(detail)
c.RespondWith(http.StatusOK, rpc.MIMEJSON, detail.Data)
}
// HTTPDiskMigratingStats returns disk migrating stats

64
blobstore/scheduler/service_test.go
View File

@ -58,20 +58,16 @@ func newMockService(t *testing.T) *Service {
clusterTopology := NewMockClusterTopology(ctr)
// return disk repair task
diskRepairMgr.EXPECT().AcquireTask(any, any).Return(proto.MigrateTask{TaskType: proto.TaskTypeDiskRepair}, nil)
diskRepairMgr.EXPECT().AcquireTask(any, any).Return(&proto.Task{TaskType: proto.TaskTypeDiskRepair}, nil)
// reclaim repair task
diskRepairMgr.EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil)
clusterMgrCli.EXPECT().AllocVolumeUnit(any, any).Return(&client.AllocVunitInfo{}, nil)
diskRepairMgr.EXPECT().ReclaimTask(any, any).Return(nil)
// reclaim balance task
balanceMgr.EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil)
clusterMgrCli.EXPECT().AllocVolumeUnit(any, any).Return(&client.AllocVunitInfo{}, nil)
balanceMgr.EXPECT().ReclaimTask(any, any).Return(nil)
// reclaim disk drop task
diskDropMgr.EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil)
clusterMgrCli.EXPECT().AllocVolumeUnit(any, any).Return(&client.AllocVunitInfo{}, nil)
diskDropMgr.EXPECT().ReclaimTask(any, any).Return(nil)
// reclaim manual migrate task
manualMgr.EXPECT().ReclaimTask(any, any, any, any, any, any).Return(nil)
clusterMgrCli.EXPECT().AllocVolumeUnit(any, any).Return(&client.AllocVunitInfo{}, nil)
manualMgr.EXPECT().ReclaimTask(any, any).Return(nil)
// cancel repair task
diskRepairMgr.EXPECT().CancelTask(any, any).Return(nil)
@ -101,13 +97,13 @@ func newMockService(t *testing.T) *Service {
manualMgr.EXPECT().RenewalTask(any, any, any).Times(3).Return(nil)
// report repair task
diskRepairMgr.EXPECT().ReportWorkerTaskStats(any).Return()
diskRepairMgr.EXPECT().ReportTask(any, any).Return(nil)
// report balance task
balanceMgr.EXPECT().ReportWorkerTaskStats(any).Return()
balanceMgr.EXPECT().ReportTask(any, any).Return(nil)
// report disk drop task
diskDropMgr.EXPECT().ReportWorkerTaskStats(any).Return()
diskDropMgr.EXPECT().ReportTask(any, any).Return(nil)
// report manual migrate task
manualMgr.EXPECT().ReportWorkerTaskStats(any).Return()
manualMgr.EXPECT().ReportTask(any, any).Return(nil)
// add manual migrate task
manualMgr.EXPECT().AddManualTask(any, any, any).Return(nil)
@ -142,10 +138,10 @@ func newMockService(t *testing.T) *Service {
inspectorMgr.EXPECT().Enabled().Return(true)
// task detail
balanceMgr.EXPECT().QueryTask(any, any).Return(nil, nil)
diskDropMgr.EXPECT().QueryTask(any, any).Return(nil, nil)
diskRepairMgr.EXPECT().QueryTask(any, any).Return(nil, nil)
manualMgr.EXPECT().QueryTask(any, any).Return(nil, nil)
balanceMgr.EXPECT().QueryTask(any, any).Return(&api.TaskRet{TaskType: proto.TaskTypeBalance}, nil)
diskDropMgr.EXPECT().QueryTask(any, any).Return(&api.TaskRet{TaskType: proto.TaskTypeShardDiskDrop}, nil)
diskRepairMgr.EXPECT().QueryTask(any, any).Return(&api.TaskRet{TaskType: proto.TaskTypeDiskRepair}, nil)
manualMgr.EXPECT().QueryTask(any, any).Return(&api.TaskRet{TaskType: proto.TaskTypeManualMigrate}, nil)
balanceMgr.EXPECT().QueryTask(any, any).Return(nil, errMock)
diskDropMgr.EXPECT().QueryTask(any, any).Return(nil, errMock)
diskRepairMgr.EXPECT().QueryTask(any, any).Return(nil, errMock)
@ -198,18 +194,23 @@ func TestServiceAPI(t *testing.T) {
require.Equal(t, proto.TaskTypeDiskRepair, task.TaskType)
for _, taskType := range taskTypes {
require.NoError(t, cli.ReclaimTask(ctx, &api.OperateTaskArgs{
IDC: idc, TaskType: taskType, TaskID: client.GenMigrateTaskID(taskType, diskID, volumeID),
Src: []proto.VunitLocation{{Vuid: 1, Host: "127.0.0.1:xx", DiskID: 1}},
}))
require.NoError(t, cli.CancelTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskType: taskType, TaskID: client.GenMigrateTaskID(taskType, diskID, volumeID)}))
require.NoError(t, cli.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskType: taskType, TaskID: client.GenMigrateTaskID(taskType, diskID, volumeID)}))
require.NoError(t, cli.ReportTask(ctx, &api.TaskReportArgs{TaskType: taskType, TaskID: client.GenMigrateTaskID(taskType, diskID, volumeID)}))
taskArgs, err := (&api.OperateTaskArgs{
IDC: idc, TaskType: taskType,
TaskID: client.GenMigrateTaskID(taskType, diskID, volumeID),
}).TaskArgs()
require.NoError(t, err)
require.NoError(t, cli.ReclaimTask(ctx, taskArgs))
require.NoError(t, cli.CancelTask(ctx, taskArgs))
require.NoError(t, cli.CompleteTask(ctx, taskArgs))
args, err := (&api.TaskReportArgs{TaskType: taskType, TaskID: client.GenMigrateTaskID(taskType, diskID, volumeID)}).TaskArgs()
require.NoError(t, err)
require.NoError(t, cli.ReportTask(ctx, args))
}
require.Error(t, cli.ReclaimTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskType: "task"}))
require.Error(t, cli.CancelTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskType: "task"}))
require.Error(t, cli.CompleteTask(ctx, &api.OperateTaskArgs{IDC: idc, TaskType: "task"}))
taskArgs, err := (&api.OperateTaskArgs{IDC: idc, TaskType: "task"}).TaskArgs()
require.NoError(t, err)
require.Error(t, cli.ReclaimTask(ctx, taskArgs))
require.Error(t, cli.CancelTask(ctx, taskArgs))
require.Error(t, cli.CompleteTask(ctx, taskArgs))
// renewal task
_, err = cli.RenewalTask(ctx, &api.TaskRenewalArgs{
@ -288,11 +289,4 @@ func TestServiceAPI(t *testing.T) {
require.Equal(t, int(testDisk1.UsedChunkCnt), stats.TotalTasksCnt)
require.Equal(t, 1, stats.MigratedTasksCnt)
}
// reclaim failed
err = cli.ReclaimTask(ctx, &api.OperateTaskArgs{
IDC: idc, TaskType: proto.TaskTypeDiskRepair,
TaskID: client.GenMigrateTaskID(proto.TaskTypeDiskRepair, diskID, volumeID),
})
require.Error(t, err)
}

223
blobstore/scheduler/shard_migrate.go Normal file
View File

@ -0,0 +1,223 @@
// Copyright 2024 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package scheduler
import (
"context"
"errors"
"time"
api "github.com/cubefs/cubefs/blobstore/api/scheduler"
"github.com/cubefs/cubefs/blobstore/common/counter"
errcode "github.com/cubefs/cubefs/blobstore/common/errors"
"github.com/cubefs/cubefs/blobstore/common/proto"
"github.com/cubefs/cubefs/blobstore/common/taskswitch"
"github.com/cubefs/cubefs/blobstore/common/trace"
"github.com/cubefs/cubefs/blobstore/scheduler/base"
"github.com/cubefs/cubefs/blobstore/scheduler/client"
"github.com/cubefs/cubefs/blobstore/util/closer"
)
// ShardMigrateConfig migrate config
type ShardMigrateConfig struct {
ClusterID proto.ClusterID `json:"-"` // fill in config.go
base.TaskCommonConfig
}
type ShardMigrator interface {
BaseMigrator
Stats() api.ShardTaskStat
taskswitch.ISwitcher
closer.Closer
Load() error
Run()
}
type ShardMigrateMgr struct {
closer.Closer
taskType proto.TaskType
taskSwitch taskswitch.ISwitcher
prepareQueue *base.TaskQueue // store inited task
workQueue *base.ShardTaskQueue // store prepared task
finishQueue *base.TaskQueue // store completed task
clusterMgrCli client.ClusterMgrAPI
finishTaskCounter counter.Counter
taskStatsMgr *base.TaskStatsMgr
cfg *ShardMigrateConfig
}
func (mgr *ShardMigrateMgr) AcquireTask(ctx context.Context, idc string) (task *proto.Task, err error) {
span := trace.SpanFromContextSafe(ctx)
task = new(proto.Task)
task.ModuleType = proto.TypeShardNode
if !mgr.taskSwitch.Enabled() {
return task, proto.ErrTaskPaused
}
_, migTask, _ := mgr.workQueue.Acquire(idc)
if migTask != nil {
t := *migTask.(*proto.ShardMigrateTask)
data, err := t.Marshal()
if err != nil {
return task, err
}
task.Data = data
span.Infof("acquire %s taskId: %s", mgr.taskType, t.TaskID)
return task, nil
}
return task, proto.ErrTaskEmpty
}
func (mgr *ShardMigrateMgr) CancelTask(ctx context.Context, args *api.TaskArgs) error {
mgr.taskStatsMgr.CancelTask()
arg := &api.ShardTaskArgs{}
err := arg.Unmarshal(args.Data)
if err != nil {
return err
}
if !client.ValidMigrateTask(args.TaskType, arg.TaskID) {
return errcode.ErrIllegalArguments
}
err = mgr.workQueue.Cancel(arg.IDC, arg.TaskID, arg.Source, arg.Dest)
if err != nil {
span := trace.SpanFromContextSafe(ctx)
span.Errorf("cancel shard migrate failed: task_type[%s], task_id[%s], err[%+v]", mgr.taskType, arg.TaskID, err)
}
return nil
}
func (mgr *ShardMigrateMgr) CompleteTask(ctx context.Context, args *api.TaskArgs) error {
span := trace.SpanFromContextSafe(ctx)
arg := &api.ShardTaskArgs{}
err := arg.Unmarshal(args.Data)
if err != nil {
return err
}
if !client.ValidMigrateTask(args.TaskType, arg.TaskID) {
return errcode.ErrIllegalArguments
}
completeTask, err := mgr.workQueue.Complete(arg.IDC, arg.TaskID, arg.Source, arg.Dest)
if err != nil {
span.Errorf("complete migrate task failed: task_id[%s], err[%+v]", arg.TaskID, err)
return err
}
t := completeTask.(*proto.ShardMigrateTask)
t.State = proto.ShardTaskStateWorkCompleted
task, err := t.Task()
if err != nil {
return err
}
err = mgr.clusterMgrCli.UpdateMigrateTask(ctx, task)
if err != nil {
// there is no impact if we failed to update task state in db,
// because we will do it in finishTask again, so assume complete success
span.Errorf("complete migrate task into db failed: task_id[%s], err[%+v]", t.TaskID, err)
err = nil
}
mgr.finishQueue.PushTask(arg.TaskID, t)
return nil
}
func (mgr *ShardMigrateMgr) ReclaimTask(ctx context.Context, args *api.TaskArgs) error {
return nil
}
func (mgr *ShardMigrateMgr) RenewalTask(ctx context.Context, idc, taskID string) error {
return nil
}
func (mgr *ShardMigrateMgr) QueryTask(ctx context.Context, taskID string) (*api.TaskRet, error) {
return nil, nil
}
func (mgr *ShardMigrateMgr) ReportTask(ctx context.Context, args *api.TaskArgs) (err error) {
return nil
}
func (mgr *ShardMigrateMgr) Stats() api.ShardTaskStat {
return api.ShardTaskStat{}
}
// Enabled task switch
func (mgr *ShardMigrateMgr) Enabled() bool {
return mgr.taskSwitch.Enabled()
}
func (mgr *ShardMigrateMgr) WaitEnable() {
mgr.taskSwitch.WaitEnable()
}
func (mgr *ShardMigrateMgr) Close() {
mgr.Closer.Close()
}
// Done returns a channel that's closed when object was closed.
func (mgr *ShardMigrateMgr) Done() <-chan struct{} {
return mgr.Closer.Done()
}
func (mgr *ShardMigrateMgr) Load() error {
return nil
}
func (mgr *ShardMigrateMgr) Run() {
go mgr.prepareTaskLoop()
go mgr.finishTaskLoop()
}
func (mgr *ShardMigrateMgr) prepareTaskLoop() {
go func() {
for {
mgr.taskSwitch.WaitEnable()
todo, doing := mgr.workQueue.StatsTasks()
if todo+doing >= mgr.cfg.WorkQueueSize {
time.Sleep(prepareTaskPause)
continue
}
err := mgr.prepareTask()
if errors.Is(err, base.ErrNoTaskInQueue) {
time.Sleep(prepareMigrateTaskInterval)
}
}
}()
}
func (mgr *ShardMigrateMgr) finishTaskLoop() {
for {
err := mgr.finishTask()
if errors.Is(err, base.ErrNoTaskInQueue) {
time.Sleep(finishMigrateTaskInterval)
}
}
}
func (mgr *ShardMigrateMgr) prepareTask() error {
return nil
}
func (mgr *ShardMigrateMgr) finishTask() error {
return nil
}

2
blobstore/scheduler/startup.go
View File

@ -372,7 +372,7 @@ func NewHandler(service *Service) *rpc.Router {
rpc.POST(api.PathTaskReport, service.HTTPTaskReport, rpc.OptArgsBody())
rpc.POST(api.PathTaskRenewal, service.HTTPTaskRenewal, rpc.OptArgsBody())
rpc.GET(api.PathTaskDetailURI, service.HTTPMigrateTaskDetail, rpc.OptArgsURI())
rpc.GET(api.PathTaskDetailURI, service.HTTPTaskDetail, rpc.OptArgsURI())
rpc.GET(api.PathStats, service.HTTPStats, rpc.OptArgsQuery())
rpc.GET(api.PathStatsLeader, service.HTTPStats, rpc.OptArgsQuery())
rpc.GET(api.PathStatsDiskMigrating, service.HTTPDiskMigratingStats, rpc.OptArgsQuery())

8
blobstore/scheduler/startup_test.go
View File

@ -148,10 +148,10 @@ func newMockServiceWithOpts(ctr *gomock.Controller, isLeader bool) *Service {
volumeUpdater.EXPECT().UpdateFollowerVolumeCache(any, any, any).AnyTimes().Return(nil)
volumeUpdater.EXPECT().UpdateLeaderVolumeCache(any, any).AnyTimes().Return(nil)
manualMgr.EXPECT().AcquireTask(any, any).AnyTimes().Return(proto.MigrateTask{TaskType: proto.TaskTypeManualMigrate}, nil)
diskRepairMgr.EXPECT().AcquireTask(any, any).AnyTimes().Return(proto.MigrateTask{}, errMock)
diskDropMgr.EXPECT().AcquireTask(any, any).AnyTimes().Return(proto.MigrateTask{}, errMock)
balanceMgr.EXPECT().AcquireTask(any, any).AnyTimes().Return(proto.MigrateTask{}, errMock)
manualMgr.EXPECT().AcquireTask(any, any).AnyTimes().Return(&proto.Task{TaskType: proto.TaskTypeManualMigrate}, nil)
diskRepairMgr.EXPECT().AcquireTask(any, any).AnyTimes().Return(&proto.Task{}, errMock)
diskDropMgr.EXPECT().AcquireTask(any, any).AnyTimes().Return(&proto.Task{}, errMock)
balanceMgr.EXPECT().AcquireTask(any, any).AnyTimes().Return(&proto.Task{}, errMock)
clusterTopology.EXPECT().UpdateVolume(any).AnyTimes().Return(&client.VolumeInfoSimple{}, nil)
clusterMgrCli.EXPECT().GetConfig(any, any).AnyTimes().Return("", errMock)

12
blobstore/testing/mocks/api_scheduler.go
View File

@ -52,10 +52,10 @@ func (mr *MockISchedulerMockRecorder) AcquireInspectTask(arg0 interface{}) *gomo
}
// AcquireTask mocks base method.
func (m *MockIScheduler) AcquireTask(arg0 context.Context, arg1 *scheduler.AcquireArgs) (*proto.MigrateTask, error) {
func (m *MockIScheduler) AcquireTask(arg0 context.Context, arg1 *scheduler.AcquireArgs) (*proto.Task, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "AcquireTask", arg0, arg1)
ret0, _ := ret[0].(*proto.MigrateTask)
ret0, _ := ret[0].(*proto.Task)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@ -81,7 +81,7 @@ func (mr *MockISchedulerMockRecorder) AddManualMigrateTask(arg0, arg1 interface{
}
// CancelTask mocks base method.
func (m *MockIScheduler) CancelTask(arg0 context.Context, arg1 *scheduler.OperateTaskArgs) error {
func (m *MockIScheduler) CancelTask(arg0 context.Context, arg1 *scheduler.TaskArgs) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "CancelTask", arg0, arg1)
ret0, _ := ret[0].(error)
@ -109,7 +109,7 @@ func (mr *MockISchedulerMockRecorder) CompleteInspectTask(arg0, arg1 interface{}
}
// CompleteTask mocks base method.
func (m *MockIScheduler) CompleteTask(arg0 context.Context, arg1 *scheduler.OperateTaskArgs) error {
func (m *MockIScheduler) CompleteTask(arg0 context.Context, arg1 *scheduler.TaskArgs) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "CompleteTask", arg0, arg1)
ret0, _ := ret[0].(error)
@ -168,7 +168,7 @@ func (mr *MockISchedulerMockRecorder) LeaderStats(arg0 interface{}) *gomock.Call
}
// ReclaimTask mocks base method.
func (m *MockIScheduler) ReclaimTask(arg0 context.Context, arg1 *scheduler.OperateTaskArgs) error {
func (m *MockIScheduler) ReclaimTask(arg0 context.Context, arg1 *scheduler.TaskArgs) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ReclaimTask", arg0, arg1)
ret0, _ := ret[0].(error)
@ -197,7 +197,7 @@ func (mr *MockISchedulerMockRecorder) RenewalTask(arg0, arg1 interface{}) *gomoc
}
// ReportTask mocks base method.
func (m *MockIScheduler) ReportTask(arg0 context.Context, arg1 *scheduler.TaskReportArgs) error {
func (m *MockIScheduler) ReportTask(arg0 context.Context, arg1 *scheduler.TaskArgs) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ReportTask", arg0, arg1)
ret0, _ := ret[0].(error)