diff --git a/blobstore/api/shardnode/del_blob.go b/blobstore/api/shardnode/del_blob.go index c20157198..707108da6 100644 --- a/blobstore/api/shardnode/del_blob.go +++ b/blobstore/api/shardnode/del_blob.go @@ -34,7 +34,26 @@ func (c *Client) DeleteBlobRaw(ctx context.Context, host string, args DeleteBlob return c.doRequest(ctx, host, "/blob/delete/raw", &args, nil) } -func (c *Client) DeleteBlobStats(ctx context.Context, host string, args DeleteBlobStatsArgs) (ret DeleteBlobStatsRet, err error) { +func (c *Client) DeleteBlobStats(ctx context.Context, host string, args ShardnodeTaskStatsArgs) (ret ShardnodeTaskStatsRet, err error) { err = c.doRequest(ctx, host, "/blob/delete/stats", &args, &ret) return } + +func (r *RepairSliceArgs) GetShardKeys(tagNum int) []string { + if r == nil { + return nil + } + keys := make([]string, util.Max(2, tagNum)) + keys[0] = util.Any2String(r.Vid) + keys[1] = util.Any2String(r.Bid) + return keys +} + +func (c *Client) RepairSlice(ctx context.Context, host string, args RepairSliceArgs) error { + return c.doRequest(ctx, host, "/slice/repair", &args, nil) +} + +func (c *Client) RepairSliceStats(ctx context.Context, host string, args ShardnodeTaskStatsArgs) (ret ShardnodeTaskStatsRet, err error) { + err = c.doRequest(ctx, host, "/slice/repair/stats", &args, &ret) + return +} diff --git a/blobstore/api/shardnode/shardnode.pb.go b/blobstore/api/shardnode/shardnode.pb.go index c6f65a982..ed2148d3e 100644 --- a/blobstore/api/shardnode/shardnode.pb.go +++ b/blobstore/api/shardnode/shardnode.pb.go @@ -2678,24 +2678,29 @@ func (m *DeleteBlobRawRet) XXX_DiscardUnknown() { var xxx_messageInfo_DeleteBlobRawRet proto.InternalMessageInfo -type DeleteBlobStatsArgs struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` +type RepairSliceArgs struct { + Header ShardOpHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header"` + Bid github_com_cubefs_cubefs_blobstore_common_proto.BlobID `protobuf:"varint,2,opt,name=bid,proto3,casttype=github.com/cubefs/cubefs/blobstore/common/proto.BlobID" json:"bid,omitempty"` + Vid github_com_cubefs_cubefs_blobstore_common_proto.Vid `protobuf:"varint,3,opt,name=vid,proto3,casttype=github.com/cubefs/cubefs/blobstore/common/proto.Vid" json:"vid,omitempty"` + BadIdx []uint32 `protobuf:"varint,4,rep,packed,name=bad_idx,json=badIdx,proto3" json:"bad_idx,omitempty"` + Reason string `protobuf:"bytes,5,opt,name=reason,proto3" json:"reason,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (m *DeleteBlobStatsArgs) Reset() { *m = DeleteBlobStatsArgs{} } -func (m *DeleteBlobStatsArgs) String() string { return proto.CompactTextString(m) } -func (*DeleteBlobStatsArgs) ProtoMessage() {} -func (*DeleteBlobStatsArgs) Descriptor() ([]byte, []int) { +func (m *RepairSliceArgs) Reset() { *m = RepairSliceArgs{} } +func (m *RepairSliceArgs) String() string { return proto.CompactTextString(m) } +func (*RepairSliceArgs) ProtoMessage() {} +func (*RepairSliceArgs) Descriptor() ([]byte, []int) { return fileDescriptor_9d3815ca0e5f30f0, []int{47} } -func (m *DeleteBlobStatsArgs) XXX_Unmarshal(b []byte) error { +func (m *RepairSliceArgs) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *DeleteBlobStatsArgs) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *RepairSliceArgs) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_DeleteBlobStatsArgs.Marshal(b, m, deterministic) + return xxx_messageInfo_RepairSliceArgs.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) @@ -2705,19 +2710,132 @@ func (m *DeleteBlobStatsArgs) XXX_Marshal(b []byte, deterministic bool) ([]byte, return b[:n], nil } } -func (m *DeleteBlobStatsArgs) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteBlobStatsArgs.Merge(m, src) +func (m *RepairSliceArgs) XXX_Merge(src proto.Message) { + xxx_messageInfo_RepairSliceArgs.Merge(m, src) } -func (m *DeleteBlobStatsArgs) XXX_Size() int { +func (m *RepairSliceArgs) XXX_Size() int { return m.Size() } -func (m *DeleteBlobStatsArgs) XXX_DiscardUnknown() { - xxx_messageInfo_DeleteBlobStatsArgs.DiscardUnknown(m) +func (m *RepairSliceArgs) XXX_DiscardUnknown() { + xxx_messageInfo_RepairSliceArgs.DiscardUnknown(m) } -var xxx_messageInfo_DeleteBlobStatsArgs proto.InternalMessageInfo +var xxx_messageInfo_RepairSliceArgs proto.InternalMessageInfo -type DeleteBlobStatsRet struct { +func (m *RepairSliceArgs) GetHeader() ShardOpHeader { + if m != nil { + return m.Header + } + return ShardOpHeader{} +} + +func (m *RepairSliceArgs) GetBid() github_com_cubefs_cubefs_blobstore_common_proto.BlobID { + if m != nil { + return m.Bid + } + return 0 +} + +func (m *RepairSliceArgs) GetVid() github_com_cubefs_cubefs_blobstore_common_proto.Vid { + if m != nil { + return m.Vid + } + return 0 +} + +func (m *RepairSliceArgs) GetBadIdx() []uint32 { + if m != nil { + return m.BadIdx + } + return nil +} + +func (m *RepairSliceArgs) GetReason() string { + if m != nil { + return m.Reason + } + return "" +} + +type RepairSliceRet struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *RepairSliceRet) Reset() { *m = RepairSliceRet{} } +func (m *RepairSliceRet) String() string { return proto.CompactTextString(m) } +func (*RepairSliceRet) ProtoMessage() {} +func (*RepairSliceRet) Descriptor() ([]byte, []int) { + return fileDescriptor_9d3815ca0e5f30f0, []int{48} +} +func (m *RepairSliceRet) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *RepairSliceRet) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_RepairSliceRet.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *RepairSliceRet) XXX_Merge(src proto.Message) { + xxx_messageInfo_RepairSliceRet.Merge(m, src) +} +func (m *RepairSliceRet) XXX_Size() int { + return m.Size() +} +func (m *RepairSliceRet) XXX_DiscardUnknown() { + xxx_messageInfo_RepairSliceRet.DiscardUnknown(m) +} + +var xxx_messageInfo_RepairSliceRet proto.InternalMessageInfo + +type ShardnodeTaskStatsArgs struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ShardnodeTaskStatsArgs) Reset() { *m = ShardnodeTaskStatsArgs{} } +func (m *ShardnodeTaskStatsArgs) String() string { return proto.CompactTextString(m) } +func (*ShardnodeTaskStatsArgs) ProtoMessage() {} +func (*ShardnodeTaskStatsArgs) Descriptor() ([]byte, []int) { + return fileDescriptor_9d3815ca0e5f30f0, []int{49} +} +func (m *ShardnodeTaskStatsArgs) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ShardnodeTaskStatsArgs) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ShardnodeTaskStatsArgs.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ShardnodeTaskStatsArgs) XXX_Merge(src proto.Message) { + xxx_messageInfo_ShardnodeTaskStatsArgs.Merge(m, src) +} +func (m *ShardnodeTaskStatsArgs) XXX_Size() int { + return m.Size() +} +func (m *ShardnodeTaskStatsArgs) XXX_DiscardUnknown() { + xxx_messageInfo_ShardnodeTaskStatsArgs.DiscardUnknown(m) +} + +var xxx_messageInfo_ShardnodeTaskStatsArgs proto.InternalMessageInfo + +type ShardnodeTaskStatsRet struct { Enable bool `protobuf:"varint,1,opt,name=enable,proto3" json:"enable,omitempty"` SuccessPerMin string `protobuf:"bytes,2,opt,name=success_per_min,json=successPerMin,proto3" json:"success_per_min,omitempty"` FailedPerMin string `protobuf:"bytes,3,opt,name=failed_per_min,json=failedPerMin,proto3" json:"failed_per_min,omitempty"` @@ -2728,18 +2846,18 @@ type DeleteBlobStatsRet struct { XXX_sizecache int32 `json:"-"` } -func (m *DeleteBlobStatsRet) Reset() { *m = DeleteBlobStatsRet{} } -func (m *DeleteBlobStatsRet) String() string { return proto.CompactTextString(m) } -func (*DeleteBlobStatsRet) ProtoMessage() {} -func (*DeleteBlobStatsRet) Descriptor() ([]byte, []int) { - return fileDescriptor_9d3815ca0e5f30f0, []int{48} +func (m *ShardnodeTaskStatsRet) Reset() { *m = ShardnodeTaskStatsRet{} } +func (m *ShardnodeTaskStatsRet) String() string { return proto.CompactTextString(m) } +func (*ShardnodeTaskStatsRet) ProtoMessage() {} +func (*ShardnodeTaskStatsRet) Descriptor() ([]byte, []int) { + return fileDescriptor_9d3815ca0e5f30f0, []int{50} } -func (m *DeleteBlobStatsRet) XXX_Unmarshal(b []byte) error { +func (m *ShardnodeTaskStatsRet) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *DeleteBlobStatsRet) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *ShardnodeTaskStatsRet) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_DeleteBlobStatsRet.Marshal(b, m, deterministic) + return xxx_messageInfo_ShardnodeTaskStatsRet.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) @@ -2749,47 +2867,47 @@ func (m *DeleteBlobStatsRet) XXX_Marshal(b []byte, deterministic bool) ([]byte, return b[:n], nil } } -func (m *DeleteBlobStatsRet) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteBlobStatsRet.Merge(m, src) +func (m *ShardnodeTaskStatsRet) XXX_Merge(src proto.Message) { + xxx_messageInfo_ShardnodeTaskStatsRet.Merge(m, src) } -func (m *DeleteBlobStatsRet) XXX_Size() int { +func (m *ShardnodeTaskStatsRet) XXX_Size() int { return m.Size() } -func (m *DeleteBlobStatsRet) XXX_DiscardUnknown() { - xxx_messageInfo_DeleteBlobStatsRet.DiscardUnknown(m) +func (m *ShardnodeTaskStatsRet) XXX_DiscardUnknown() { + xxx_messageInfo_ShardnodeTaskStatsRet.DiscardUnknown(m) } -var xxx_messageInfo_DeleteBlobStatsRet proto.InternalMessageInfo +var xxx_messageInfo_ShardnodeTaskStatsRet proto.InternalMessageInfo -func (m *DeleteBlobStatsRet) GetEnable() bool { +func (m *ShardnodeTaskStatsRet) GetEnable() bool { if m != nil { return m.Enable } return false } -func (m *DeleteBlobStatsRet) GetSuccessPerMin() string { +func (m *ShardnodeTaskStatsRet) GetSuccessPerMin() string { if m != nil { return m.SuccessPerMin } return "" } -func (m *DeleteBlobStatsRet) GetFailedPerMin() string { +func (m *ShardnodeTaskStatsRet) GetFailedPerMin() string { if m != nil { return m.FailedPerMin } return "" } -func (m *DeleteBlobStatsRet) GetTotalErrCnt() uint64 { +func (m *ShardnodeTaskStatsRet) GetTotalErrCnt() uint64 { if m != nil { return m.TotalErrCnt } return 0 } -func (m *DeleteBlobStatsRet) GetErrStats() []string { +func (m *ShardnodeTaskStatsRet) GetErrStats() []string { if m != nil { return m.ErrStats } @@ -2844,127 +2962,135 @@ func init() { proto.RegisterType((*DBStatsRet)(nil), "cubefs.blobstore.api.shardnode.DBStatsRet") proto.RegisterType((*DeleteBlobRawArgs)(nil), "cubefs.blobstore.api.shardnode.DeleteBlobRawArgs") proto.RegisterType((*DeleteBlobRawRet)(nil), "cubefs.blobstore.api.shardnode.DeleteBlobRawRet") - proto.RegisterType((*DeleteBlobStatsArgs)(nil), "cubefs.blobstore.api.shardnode.DeleteBlobStatsArgs") - proto.RegisterType((*DeleteBlobStatsRet)(nil), "cubefs.blobstore.api.shardnode.DeleteBlobStatsRet") + proto.RegisterType((*RepairSliceArgs)(nil), "cubefs.blobstore.api.shardnode.RepairSliceArgs") + proto.RegisterType((*RepairSliceRet)(nil), "cubefs.blobstore.api.shardnode.RepairSliceRet") + proto.RegisterType((*ShardnodeTaskStatsArgs)(nil), "cubefs.blobstore.api.shardnode.ShardnodeTaskStatsArgs") + proto.RegisterType((*ShardnodeTaskStatsRet)(nil), "cubefs.blobstore.api.shardnode.ShardnodeTaskStatsRet") } func init() { proto.RegisterFile("shardnode.proto", fileDescriptor_9d3815ca0e5f30f0) } var fileDescriptor_9d3815ca0e5f30f0 = []byte{ - // 1803 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x59, 0xcd, 0x6f, 0x1c, 0x49, - 0x15, 0xa7, 0xe7, 0xcb, 0xe3, 0x37, 0x1f, 0xce, 0xf6, 0x9a, 0x30, 0x0a, 0xc2, 0x63, 0x75, 0xb2, - 0xac, 0x05, 0xcb, 0x58, 0x24, 0x08, 0x0e, 0x0b, 0x24, 0x1e, 0x0f, 0xd9, 0xcc, 0x26, 0xde, 0x5d, - 0x7a, 0x12, 0x4b, 0x20, 0xa1, 0x56, 0x4d, 0x57, 0xcd, 0xb8, 0x49, 0x7f, 0x0c, 0x55, 0x35, 0x21, - 0xd9, 0x13, 0xe2, 0xc2, 0x09, 0x21, 0xfe, 0x00, 0x84, 0xe0, 0x4f, 0xe0, 0x82, 0x04, 0xe2, 0xc4, - 0x81, 0x3d, 0x72, 0xe3, 0x36, 0x42, 0x73, 0xe1, 0x7f, 0xf0, 0x09, 0xd5, 0xab, 0xea, 0xee, 0x49, - 0xb2, 0x5e, 0x67, 0xec, 0xb1, 0x45, 0xd8, 0x4b, 0x32, 0xf5, 0xfa, 0x7d, 0xfc, 0xde, 0x7b, 0xf5, - 0x3e, 0xba, 0x0d, 0x1b, 0xe2, 0x88, 0x70, 0x1a, 0x27, 0x94, 0x75, 0x26, 0x3c, 0x91, 0x89, 0xbd, - 0xe5, 0x4f, 0x87, 0x6c, 0x24, 0x3a, 0xc3, 0x30, 0x19, 0x0a, 0x99, 0x70, 0xd6, 0x21, 0x93, 0xa0, - 0x93, 0x71, 0x5d, 0xdb, 0x1c, 0x27, 0xe3, 0x04, 0x59, 0x77, 0xd5, 0x2f, 0x2d, 0x75, 0xed, 0x1d, - 0x2d, 0xb5, 0x9b, 0x49, 0xed, 0xfa, 0x49, 0x14, 0x25, 0xf1, 0x2e, 0x0a, 0x06, 0xf1, 0x78, 0x97, - 0x93, 0x78, 0x6c, 0x6c, 0x5c, 0xfb, 0xfa, 0x4b, 0xdc, 0x64, 0x12, 0xec, 0xfa, 0xe1, 0x54, 0x48, - 0xc6, 0xa3, 0x31, 0xd7, 0x52, 0x86, 0x79, 0xe7, 0x24, 0xd5, 0x1a, 0x84, 0x22, 0x1b, 0xce, 0xb7, - 0x4f, 0xe2, 0xe4, 0x64, 0x24, 0xf1, 0x1f, 0xcd, 0xe8, 0xf8, 0x50, 0xea, 0x4b, 0x16, 0xd9, 0x57, - 0xa1, 0x10, 0xd0, 0x96, 0xb5, 0x6d, 0xed, 0xac, 0x77, 0x2b, 0xf3, 0x59, 0xbb, 0xd0, 0xef, 0xb9, - 0x85, 0x80, 0xda, 0xfb, 0x50, 0x19, 0x05, 0x2c, 0xa4, 0xa2, 0x55, 0xd8, 0x2e, 0xee, 0xd4, 0x6e, - 0xbe, 0xd5, 0xf9, 0xec, 0xa0, 0x74, 0xee, 0x2a, 0xee, 0x6e, 0xe9, 0x93, 0x59, 0xfb, 0x0b, 0xae, - 0x11, 0x75, 0x26, 0x50, 0x46, 0xb2, 0xfd, 0xc3, 0xcc, 0x4a, 0xa3, 0xbb, 0xa7, 0xad, 0x1c, 0xcf, - 0xda, 0xdf, 0x19, 0x07, 0xf2, 0x68, 0x3a, 0xec, 0xf8, 0x49, 0xb4, 0x6b, 0x70, 0x7f, 0xa6, 0xa3, - 0xda, 0x86, 0x01, 0xb8, 0x09, 0xe5, 0x27, 0x24, 0x9c, 0xb2, 0x56, 0x61, 0xdb, 0xda, 0xa9, 0xbb, - 0xfa, 0xe0, 0xfc, 0xa9, 0x08, 0x8d, 0x81, 0xc2, 0xf4, 0xe1, 0xe4, 0x1e, 0x23, 0x94, 0x71, 0x9b, - 0x40, 0x55, 0x4c, 0x88, 0xcf, 0x3c, 0x03, 0xa0, 0xd4, 0xbd, 0x3b, 0x9f, 0xb5, 0xd7, 0x06, 0x8a, - 0x76, 0x36, 0x14, 0x46, 0xd4, 0x5d, 0x43, 0xbd, 0x7d, 0x6a, 0xff, 0x04, 0xd6, 0x68, 0x20, 0x1e, - 0x2b, 0x0b, 0x05, 0x74, 0xb1, 0x37, 0x9f, 0xb5, 0x2b, 0xbd, 0x40, 0x3c, 0x46, 0x03, 0xdf, 0x5e, - 0xd6, 0x80, 0x96, 0x74, 0x2b, 0x4a, 0x69, 0x9f, 0xda, 0x0f, 0xa1, 0x24, 0xa6, 0x01, 0x6d, 0x15, - 0x51, 0xf7, 0x9d, 0xf9, 0xac, 0x5d, 0x1a, 0x4c, 0x03, 0x7a, 0x3c, 0x6b, 0x7f, 0x6b, 0x69, 0xe8, - 0xd3, 0x80, 0xba, 0xa8, 0xcd, 0x76, 0xa0, 0x8e, 0xf8, 0x0f, 0x19, 0x17, 0x41, 0x12, 0xb7, 0x4a, - 0x2a, 0x36, 0xee, 0x73, 0x34, 0x9b, 0x41, 0x83, 0x27, 0x53, 0xc9, 0xbc, 0x27, 0x86, 0xa9, 0x8c, - 0x01, 0xbc, 0x73, 0x3c, 0x6b, 0x7f, 0x77, 0x59, 0xd3, 0xae, 0x52, 0x64, 0x14, 0xbb, 0x75, 0xbe, - 0x70, 0x72, 0x7e, 0x67, 0x41, 0xb3, 0x1f, 0x0b, 0xc6, 0xa5, 0xba, 0x92, 0x7b, 0x7c, 0x2c, 0xec, - 0xfb, 0x50, 0x39, 0xc2, 0xfc, 0x61, 0xce, 0x6a, 0x37, 0xbf, 0x71, 0xda, 0xf5, 0x7b, 0x2e, 0xe9, - 0xe9, 0x35, 0xd4, 0x2a, 0xec, 0xef, 0x43, 0x29, 0x90, 0x2c, 0xc2, 0xe4, 0xd4, 0x6e, 0xde, 0x38, - 0x4d, 0x95, 0x02, 0x61, 0x34, 0xa0, 0x9c, 0xb3, 0x01, 0x8d, 0x1c, 0x9e, 0xcb, 0x24, 0x02, 0x7e, - 0x34, 0xa1, 0x44, 0xb2, 0xff, 0x59, 0xc0, 0x39, 0x3c, 0x05, 0x78, 0x0a, 0xcd, 0x1e, 0x0b, 0xd9, - 0x45, 0xe1, 0xd5, 0x4d, 0xa4, 0xf0, 0x62, 0x13, 0x51, 0x38, 0x72, 0xb3, 0x0a, 0x07, 0x87, 0xda, - 0x7b, 0x4c, 0x5e, 0x2e, 0x88, 0x07, 0x00, 0xc6, 0xa6, 0xcb, 0x64, 0x16, 0x5a, 0xeb, 0x8c, 0xa1, - 0xfd, 0x83, 0x05, 0xf5, 0x07, 0x81, 0xb8, 0x30, 0x1f, 0x2a, 0x13, 0xce, 0x46, 0xc1, 0x53, 0xed, - 0x87, 0x6b, 0x4e, 0x8a, 0x1e, 0x11, 0xfe, 0x98, 0x71, 0x6c, 0x02, 0xeb, 0xae, 0x39, 0xa9, 0x26, - 0xe8, 0x27, 0xd3, 0x58, 0x9a, 0xea, 0xd5, 0x07, 0x27, 0x81, 0x5a, 0x0a, 0x51, 0xb9, 0x7c, 0x07, - 0xca, 0x0a, 0xba, 0x68, 0x59, 0xd8, 0xc9, 0x97, 0xf1, 0x59, 0x0b, 0xda, 0x5b, 0x00, 0x31, 0x7b, - 0x2a, 0x0f, 0x34, 0x04, 0x0d, 0x6d, 0x81, 0xe2, 0xfc, 0xb9, 0x08, 0xf5, 0x3d, 0x4a, 0xd1, 0x33, - 0x0c, 0xca, 0x42, 0x47, 0xb4, 0x2e, 0xb0, 0x23, 0x16, 0x74, 0x3b, 0x5a, 0x51, 0x47, 0xdc, 0x87, - 0x32, 0x4e, 0x68, 0x8c, 0x71, 0xed, 0xe6, 0xdb, 0x2f, 0xc7, 0x49, 0x4b, 0x76, 0xd2, 0x81, 0xde, - 0x71, 0x15, 0x7b, 0x1a, 0x2a, 0x94, 0xb5, 0xef, 0x42, 0x79, 0x1a, 0x07, 0x52, 0xb4, 0x4a, 0x18, - 0xec, 0xaf, 0x7d, 0x7a, 0xb0, 0xf3, 0x39, 0xaf, 0xaf, 0xc3, 0xa3, 0x38, 0x90, 0xa9, 0x1e, 0x14, - 0xbf, 0xac, 0xd6, 0xdb, 0x80, 0x5a, 0x9a, 0x38, 0x55, 0x9f, 0xbf, 0x29, 0xc2, 0x86, 0xee, 0x1c, - 0xaf, 0x79, 0x2e, 0x7f, 0x61, 0xc1, 0x86, 0x8e, 0x2c, 0x7a, 0xf3, 0xf0, 0xd9, 0x84, 0x99, 0xf9, - 0x79, 0x38, 0x9f, 0xb5, 0x5f, 0x7c, 0x74, 0x3c, 0x6b, 0xdf, 0x5e, 0xda, 0xd8, 0xf3, 0x2a, 0xdc, - 0x17, 0x75, 0xda, 0x3d, 0x28, 0xa9, 0x54, 0x62, 0x69, 0x9e, 0xe5, 0x22, 0xa0, 0xb4, 0x73, 0x25, - 0x9d, 0x34, 0x59, 0x8e, 0xfe, 0x52, 0x80, 0x2f, 0x3d, 0xe4, 0x24, 0x16, 0x23, 0xc6, 0x91, 0xf8, - 0x00, 0x7b, 0xc7, 0xeb, 0x9b, 0xab, 0x9f, 0x42, 0x9d, 0x32, 0x21, 0xbd, 0x14, 0xb9, 0xce, 0xd3, - 0xbd, 0xf9, 0xac, 0x0d, 0x3d, 0x26, 0xe4, 0xb9, 0xd1, 0x03, 0x4d, 0xb5, 0x50, 0xa7, 0x05, 0x57, - 0x3f, 0x25, 0x76, 0x2a, 0xac, 0x7f, 0xb7, 0xa0, 0xfe, 0x1e, 0x93, 0xaf, 0xf7, 0xbd, 0x77, 0x7e, - 0x84, 0x03, 0x36, 0xbd, 0x2b, 0xf6, 0xfb, 0x50, 0xc6, 0x66, 0x65, 0x66, 0x53, 0xe7, 0xd5, 0x2f, - 0x61, 0x3f, 0x1e, 0x25, 0x69, 0x47, 0x42, 0x15, 0xce, 0xaf, 0x0b, 0xd0, 0xdc, 0xe7, 0x8c, 0x48, - 0xd6, 0x0d, 0x93, 0xe1, 0xea, 0x67, 0x9f, 0x0d, 0xa5, 0x98, 0x44, 0xcc, 0x8c, 0x17, 0xfc, 0x6d, - 0x8f, 0xa1, 0xea, 0x27, 0x94, 0x45, 0x09, 0x4d, 0xcb, 0xf7, 0xfe, 0x7c, 0xd6, 0xae, 0xee, 0x27, - 0x94, 0x1d, 0x24, 0x54, 0xd5, 0xed, 0xbb, 0xaf, 0x1e, 0xac, 0x54, 0x53, 0x27, 0x15, 0x77, 0x33, - 0xe5, 0xca, 0xb8, 0x08, 0x3e, 0x66, 0x66, 0x8e, 0xe2, 0x6f, 0xfb, 0x2b, 0x00, 0x22, 0x0c, 0x7c, - 0xe6, 0xe1, 0x13, 0xd5, 0x7f, 0x1b, 0xee, 0x3a, 0x52, 0x06, 0xc1, 0xc7, 0xcc, 0xf9, 0x00, 0x1a, - 0x79, 0x38, 0x54, 0xb0, 0xbf, 0x07, 0x25, 0x65, 0xd3, 0xc4, 0xe2, 0xfa, 0x89, 0xe3, 0x43, 0x27, - 0x4d, 0x49, 0xa5, 0x95, 0xae, 0x58, 0x9c, 0x18, 0x53, 0x77, 0x69, 0xb1, 0x75, 0xee, 0xe3, 0x5e, - 0xb4, 0x22, 0xf0, 0xe9, 0x5a, 0x74, 0x31, 0xf0, 0x57, 0xb3, 0x16, 0xc5, 0x7a, 0x2d, 0x4a, 0x3d, - 0xbe, 0x0d, 0x65, 0xc4, 0x62, 0xd6, 0xa2, 0x25, 0x5c, 0xd6, 0x72, 0xa7, 0x6e, 0x45, 0x3f, 0x4b, - 0x97, 0xee, 0xcb, 0xcb, 0x69, 0xb6, 0x70, 0x1b, 0x27, 0x9d, 0x5f, 0x59, 0xd0, 0x74, 0x99, 0x24, - 0x41, 0x7c, 0x79, 0x45, 0xbb, 0x09, 0xe5, 0x90, 0x11, 0xa1, 0x2b, 0xb6, 0xe4, 0xea, 0x83, 0x82, - 0x96, 0x03, 0x51, 0xd0, 0xfe, 0x61, 0x41, 0x7d, 0xc0, 0x48, 0x78, 0x61, 0xc0, 0xb0, 0x6c, 0x0b, - 0x0b, 0x05, 0x9d, 0x82, 0x2d, 0x2e, 0x80, 0xed, 0x42, 0x05, 0x4b, 0x3a, 0x5d, 0xd8, 0x6e, 0x9c, - 0x72, 0x0d, 0x06, 0x8a, 0x39, 0xb5, 0xa5, 0x25, 0xd5, 0x12, 0x95, 0x3a, 0xa2, 0x1c, 0xfb, 0x5b, - 0x01, 0x9a, 0x7b, 0x61, 0x98, 0xf8, 0xc8, 0xfb, 0x39, 0x68, 0x94, 0x07, 0x50, 0x1f, 0x91, 0x20, - 0x64, 0xd4, 0xc3, 0x80, 0x60, 0xab, 0x5c, 0x2e, 0x92, 0x35, 0x2d, 0x8f, 0x24, 0x67, 0x00, 0x8d, - 0x3c, 0x7c, 0xaa, 0x52, 0xf3, 0x1c, 0x59, 0x67, 0xce, 0xd1, 0xbf, 0xca, 0x00, 0x18, 0xd4, 0x81, - 0x24, 0x52, 0x64, 0xd3, 0xd7, 0x5a, 0xe9, 0x26, 0x73, 0x1d, 0x1a, 0x64, 0x32, 0x09, 0x03, 0x46, - 0xbd, 0x20, 0xa6, 0xec, 0xa9, 0xb9, 0x7d, 0x75, 0x43, 0xec, 0x2b, 0x9a, 0xdd, 0x86, 0x5a, 0x88, - 0x89, 0xf4, 0x8e, 0x12, 0x21, 0xcd, 0x65, 0x04, 0x4d, 0xba, 0x97, 0x08, 0x69, 0x4f, 0xa0, 0x69, - 0x18, 0xd2, 0xfd, 0xa3, 0x84, 0x19, 0x7d, 0x7f, 0x3e, 0x6b, 0xd7, 0xf5, 0xc2, 0x72, 0xee, 0x2d, - 0xa4, 0x1e, 0xe6, 0x7a, 0xa8, 0x3d, 0xce, 0x20, 0x61, 0x50, 0xca, 0xd9, 0x67, 0x32, 0xd0, 0xe6, - 0xce, 0x15, 0x1a, 0xe3, 0x9a, 0xfa, 0x6d, 0xb7, 0x60, 0x2d, 0x64, 0x84, 0xc7, 0x8c, 0xb7, 0x2a, - 0xdb, 0xd6, 0x4e, 0xd5, 0x4d, 0x8f, 0x2f, 0xbf, 0xef, 0xac, 0x5d, 0xc4, 0xfb, 0x4e, 0xfe, 0x8e, - 0x57, 0x5d, 0xc5, 0x3b, 0xde, 0xfa, 0xf9, 0xde, 0xf1, 0x7a, 0x50, 0xe5, 0x64, 0x24, 0xd5, 0x8d, - 0x6c, 0x01, 0xe2, 0x71, 0x4e, 0xc4, 0x83, 0x9f, 0x6e, 0x15, 0xa7, 0x51, 0x91, 0x49, 0x3a, 0xcf, - 0xa0, 0xa9, 0xc6, 0xda, 0x61, 0x12, 0x4e, 0x23, 0xdd, 0x6d, 0x16, 0x9b, 0x81, 0x75, 0x81, 0xcd, - 0xc0, 0xa1, 0xd0, 0xc8, 0x4d, 0xab, 0x4a, 0x1d, 0x40, 0xe9, 0x49, 0x40, 0x75, 0x9d, 0x36, 0xba, - 0xb7, 0x55, 0x59, 0x1d, 0x06, 0x54, 0x1c, 0xcf, 0xda, 0xb7, 0x96, 0x4d, 0xe2, 0xa1, 0xaa, 0x2a, - 0xa5, 0xcc, 0xf9, 0x8f, 0xa5, 0xcd, 0x5c, 0xda, 0x6a, 0x4e, 0xa0, 0x8a, 0xe9, 0xcf, 0x3f, 0xe8, - 0xea, 0x4f, 0xc6, 0x8a, 0x76, 0xc6, 0x4f, 0xc6, 0x5a, 0xd4, 0x5d, 0x43, 0xbd, 0x7d, 0x9a, 0x6f, - 0x28, 0xc5, 0xc5, 0x0d, 0xe5, 0xf7, 0x45, 0x78, 0x23, 0xf3, 0xb4, 0x4b, 0x04, 0x53, 0x5b, 0xf8, - 0xff, 0x81, 0xb7, 0x8b, 0x5f, 0xb0, 0x57, 0xd7, 0x6d, 0x37, 0xa1, 0xac, 0xbb, 0x2c, 0xb6, 0x47, - 0x57, 0x1f, 0x14, 0x95, 0x4d, 0x12, 0xff, 0xc8, 0x2c, 0xec, 0xfa, 0x90, 0x97, 0x6c, 0xe5, 0x5c, - 0x25, 0xeb, 0x78, 0x7a, 0xcd, 0xcd, 0x5e, 0xb0, 0x3e, 0x84, 0x0a, 0x3a, 0x99, 0x8e, 0xa6, 0x6f, - 0x9e, 0x36, 0xd8, 0x5f, 0x4a, 0x6f, 0x36, 0xa7, 0x50, 0x8d, 0xd3, 0x84, 0xfa, 0xc3, 0xfd, 0x03, - 0xa2, 0xe6, 0x9f, 0xba, 0xea, 0xce, 0x75, 0xa8, 0xa5, 0x67, 0x65, 0x6f, 0x13, 0xca, 0x42, 0x0d, - 0x30, 0xfd, 0x17, 0x1b, 0x57, 0x1f, 0x9c, 0xdf, 0x5a, 0x50, 0xeb, 0x75, 0x71, 0xb2, 0x5d, 0x46, - 0x7d, 0x5c, 0x87, 0x35, 0x3a, 0xf4, 0xf2, 0x1d, 0xa4, 0x0b, 0xa8, 0xbe, 0xfb, 0x01, 0x89, 0x98, - 0x5b, 0xa1, 0x43, 0xf5, 0xbf, 0xf3, 0xcb, 0x02, 0x80, 0xc1, 0xa4, 0x80, 0xdb, 0x50, 0x9a, 0x0a, - 0x66, 0x06, 0xae, 0x8b, 0xbf, 0xed, 0x1d, 0xb8, 0xa2, 0x0c, 0x7a, 0x3e, 0xf1, 0x8f, 0x98, 0x37, - 0x15, 0x64, 0x9c, 0xee, 0x6b, 0x4d, 0x45, 0xdf, 0x57, 0xe4, 0x47, 0x8a, 0x6a, 0xdf, 0x82, 0xab, - 0x98, 0x5d, 0x8f, 0xc4, 0xd4, 0x1b, 0x05, 0xa1, 0x64, 0xdc, 0xf0, 0xeb, 0xfa, 0x79, 0x13, 0x9f, - 0xee, 0xc5, 0xf4, 0x2e, 0x3e, 0xd3, 0x42, 0x6f, 0x41, 0x33, 0x62, 0x91, 0x24, 0xc3, 0x30, 0x55, - 0xae, 0x97, 0x96, 0x46, 0x4a, 0xd5, 0x6c, 0xef, 0x80, 0x3d, 0x0c, 0x13, 0xff, 0xb1, 0x37, 0x09, - 0xe2, 0x98, 0x51, 0xc3, 0x8a, 0x33, 0xd0, 0xbd, 0x82, 0x4f, 0x3e, 0xc2, 0x07, 0x19, 0xb7, 0x4c, - 0x24, 0x09, 0xbd, 0x88, 0x45, 0x09, 0x7f, 0x66, 0xb8, 0x2b, 0x9a, 0x1b, 0x9f, 0x1c, 0xe0, 0x03, - 0xe4, 0x76, 0xfe, 0x68, 0xc1, 0x1b, 0x0b, 0x0b, 0x39, 0xf9, 0xf9, 0xea, 0xb7, 0xc1, 0x3b, 0x50, - 0xd6, 0x5b, 0x57, 0x61, 0xe9, 0xad, 0x4b, 0x0b, 0x3a, 0x36, 0x5c, 0x79, 0x0e, 0xa3, 0xda, 0x61, - 0xbf, 0x08, 0x6f, 0xe6, 0xb4, 0xec, 0x62, 0x39, 0x7f, 0xb5, 0xc0, 0x7e, 0x81, 0xae, 0x92, 0x7b, - 0x15, 0x2a, 0x2c, 0x56, 0x11, 0x45, 0x87, 0xaa, 0xae, 0x39, 0xd9, 0x5f, 0x85, 0x0d, 0x31, 0xf5, - 0x7d, 0x26, 0x84, 0x37, 0x61, 0xdc, 0x8b, 0x82, 0xd8, 0x2c, 0xad, 0x0d, 0x43, 0xfe, 0x88, 0xf1, - 0x83, 0x20, 0xb6, 0x6f, 0x40, 0xd3, 0x2c, 0x90, 0x29, 0x9b, 0xde, 0x8a, 0xcc, 0x5a, 0x69, 0xb8, - 0x1c, 0x68, 0xe8, 0xd0, 0x33, 0xce, 0x3d, 0x3f, 0x7b, 0xbb, 0xab, 0x21, 0xf1, 0x07, 0x9c, 0xef, - 0xc7, 0xd2, 0xfe, 0x32, 0xac, 0xab, 0xa7, 0xba, 0x46, 0xca, 0xdb, 0xc5, 0x9d, 0x75, 0xb7, 0xca, - 0x38, 0x47, 0xa4, 0xdd, 0x6b, 0x9f, 0xcc, 0xb7, 0xac, 0x7f, 0xce, 0xb7, 0xac, 0x7f, 0xcf, 0xb7, - 0xac, 0x1f, 0xd7, 0x3b, 0xbb, 0xef, 0x66, 0x21, 0x1e, 0x56, 0x30, 0x3c, 0xb7, 0xfe, 0x1b, 0x00, - 0x00, 0xff, 0xff, 0x0e, 0xd2, 0xb9, 0xdf, 0x0d, 0x1e, 0x00, 0x00, + // 1899 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x59, 0xcd, 0x6f, 0x23, 0x49, + 0x15, 0xa7, 0xfd, 0x15, 0xe7, 0xf9, 0x23, 0xd9, 0x26, 0x04, 0x6b, 0x10, 0x71, 0xd4, 0x33, 0xcb, + 0x46, 0xb0, 0x38, 0x62, 0x06, 0xc1, 0x61, 0x61, 0x67, 0xe2, 0x98, 0xd9, 0xf1, 0xce, 0x64, 0x77, + 0x68, 0x67, 0x22, 0x81, 0x84, 0x5a, 0x65, 0x57, 0xd9, 0x69, 0xd2, 0xee, 0x36, 0x55, 0xed, 0x21, + 0xb3, 0x27, 0xc4, 0x85, 0x13, 0x42, 0xfc, 0x01, 0x08, 0xc1, 0x9f, 0xc0, 0x05, 0x09, 0x09, 0x0e, + 0x70, 0x60, 0x8f, 0xdc, 0xb8, 0x59, 0xc8, 0x17, 0xfe, 0x87, 0x9c, 0xd0, 0x7b, 0x55, 0xdd, 0xf6, + 0x4c, 0x36, 0x9b, 0x75, 0xbe, 0xc4, 0xc0, 0x65, 0xe2, 0x7a, 0xf5, 0xde, 0xab, 0x5f, 0xbd, 0xef, + 0xea, 0x81, 0x15, 0x75, 0xc8, 0x24, 0x0f, 0x23, 0x2e, 0x1a, 0x23, 0x19, 0xc5, 0x91, 0xbd, 0xd1, + 0x1b, 0x77, 0x45, 0x5f, 0x35, 0xba, 0x41, 0xd4, 0x55, 0x71, 0x24, 0x45, 0x83, 0x8d, 0xfc, 0x46, + 0xca, 0x75, 0x6b, 0x6d, 0x10, 0x0d, 0x22, 0x62, 0xdd, 0xc6, 0x5f, 0x5a, 0xea, 0xd6, 0xdb, 0x5a, + 0x6a, 0x3b, 0x95, 0xda, 0xee, 0x45, 0xc3, 0x61, 0x14, 0x6e, 0x93, 0xa0, 0x1f, 0x0e, 0xb6, 0x25, + 0x0b, 0x07, 0xe6, 0x8c, 0x5b, 0x5f, 0x3b, 0xc5, 0xcd, 0x46, 0xfe, 0x76, 0x2f, 0x18, 0xab, 0x58, + 0xc8, 0xe1, 0x40, 0x6a, 0x29, 0xc3, 0xbc, 0x75, 0x96, 0x6a, 0x0d, 0x02, 0xc9, 0x86, 0xf3, 0xad, + 0xb3, 0x38, 0x25, 0xeb, 0xc7, 0xf4, 0x8f, 0x66, 0x74, 0x7a, 0x90, 0x6b, 0xc7, 0x62, 0x68, 0xaf, + 0x43, 0xc6, 0xe7, 0x35, 0x6b, 0xd3, 0xda, 0x5a, 0x6e, 0x16, 0xa6, 0x93, 0x7a, 0xa6, 0xdd, 0x72, + 0x33, 0x3e, 0xb7, 0x77, 0xa1, 0xd0, 0xf7, 0x45, 0xc0, 0x55, 0x2d, 0xb3, 0x99, 0xdd, 0x2a, 0xdd, + 0x7d, 0xb3, 0xf1, 0xe9, 0x46, 0x69, 0x3c, 0x44, 0xee, 0x66, 0xee, 0xe3, 0x49, 0xfd, 0x73, 0xae, + 0x11, 0x75, 0x46, 0x90, 0x27, 0xb2, 0xfd, 0xfd, 0xf4, 0x94, 0x4a, 0x73, 0x47, 0x9f, 0x72, 0x32, + 0xa9, 0x7f, 0x7b, 0xe0, 0xc7, 0x87, 0xe3, 0x6e, 0xa3, 0x17, 0x0d, 0xb7, 0x0d, 0xee, 0x4f, 0xbd, + 0xa8, 0x3e, 0xc3, 0x00, 0x5c, 0x83, 0xfc, 0x73, 0x16, 0x8c, 0x45, 0x2d, 0xb3, 0x69, 0x6d, 0x95, + 0x5d, 0xbd, 0x70, 0xfe, 0x90, 0x85, 0x4a, 0x07, 0x31, 0x7d, 0x38, 0x7a, 0x24, 0x18, 0x17, 0xd2, + 0x66, 0x50, 0x54, 0x23, 0xd6, 0x13, 0x9e, 0x01, 0x90, 0x6b, 0x3e, 0x9c, 0x4e, 0xea, 0x4b, 0x1d, + 0xa4, 0x5d, 0x0c, 0x85, 0x11, 0x75, 0x97, 0x48, 0x6f, 0x9b, 0xdb, 0x3f, 0x82, 0x25, 0xee, 0xab, + 0x23, 0x3c, 0x21, 0x43, 0x57, 0x6c, 0x4d, 0x27, 0xf5, 0x42, 0xcb, 0x57, 0x47, 0x74, 0xc0, 0xb7, + 0x16, 0x3d, 0x40, 0x4b, 0xba, 0x05, 0x54, 0xda, 0xe6, 0xf6, 0x3e, 0xe4, 0xd4, 0xd8, 0xe7, 0xb5, + 0x2c, 0xe9, 0x7e, 0x30, 0x9d, 0xd4, 0x73, 0x9d, 0xb1, 0xcf, 0x4f, 0x26, 0xf5, 0x6f, 0x2e, 0x0c, + 0x7d, 0xec, 0x73, 0x97, 0xb4, 0xd9, 0x0e, 0x94, 0x09, 0xff, 0x81, 0x90, 0xca, 0x8f, 0xc2, 0x5a, + 0x0e, 0x6d, 0xe3, 0xbe, 0x44, 0xb3, 0x05, 0x54, 0x64, 0x34, 0x8e, 0x85, 0xf7, 0xdc, 0x30, 0xe5, + 0xc9, 0x80, 0x0f, 0x4e, 0x26, 0xf5, 0xef, 0x2c, 0x7a, 0xb4, 0x8b, 0x8a, 0x8c, 0x62, 0xb7, 0x2c, + 0xe7, 0x56, 0xce, 0x6f, 0x2c, 0xa8, 0xb6, 0x43, 0x25, 0x64, 0x8c, 0x21, 0xb9, 0x23, 0x07, 0xca, + 0x7e, 0x0c, 0x85, 0x43, 0xf2, 0x1f, 0xf9, 0xac, 0x74, 0xf7, 0xeb, 0xe7, 0x85, 0xdf, 0x4b, 0x4e, + 0x4f, 0xc2, 0x50, 0xab, 0xb0, 0xdf, 0x85, 0x9c, 0x1f, 0x8b, 0x21, 0x39, 0xa7, 0x74, 0xf7, 0xce, + 0x79, 0xaa, 0x10, 0x84, 0xd1, 0x40, 0x72, 0xce, 0x0a, 0x54, 0x66, 0xf0, 0x5c, 0x11, 0x13, 0xe0, + 0x67, 0x23, 0xce, 0x62, 0xf1, 0x5f, 0x0b, 0x78, 0x06, 0x0f, 0x01, 0x8f, 0xa1, 0xda, 0x12, 0x81, + 0xb8, 0x2e, 0xbc, 0xba, 0x88, 0x64, 0x5e, 0x2d, 0x22, 0x88, 0x63, 0x76, 0x2c, 0xe2, 0x90, 0x50, + 0x7a, 0x4f, 0xc4, 0x37, 0x0b, 0xe2, 0x09, 0x80, 0x39, 0xd3, 0x15, 0x71, 0x6a, 0x5a, 0xeb, 0x82, + 0xa6, 0xfd, 0x9d, 0x05, 0xe5, 0x27, 0xbe, 0xba, 0xb6, 0x3b, 0x14, 0x46, 0x52, 0xf4, 0xfd, 0x63, + 0x7d, 0x0f, 0xd7, 0xac, 0x90, 0x3e, 0x64, 0xf2, 0x48, 0x48, 0x2a, 0x02, 0xcb, 0xae, 0x59, 0x61, + 0x11, 0xec, 0x45, 0xe3, 0x30, 0x36, 0xd9, 0xab, 0x17, 0x4e, 0x04, 0xa5, 0x04, 0x22, 0x5e, 0xf9, + 0x01, 0xe4, 0x11, 0xba, 0xaa, 0x59, 0x54, 0xc9, 0x17, 0xb9, 0xb3, 0x16, 0xb4, 0x37, 0x00, 0x42, + 0x71, 0x1c, 0xef, 0x69, 0x08, 0x1a, 0xda, 0x1c, 0xc5, 0xf9, 0x63, 0x16, 0xca, 0x3b, 0x9c, 0xd3, + 0xcd, 0xc8, 0x28, 0x73, 0x15, 0xd1, 0xba, 0xc6, 0x8a, 0x98, 0xd1, 0xe5, 0xe8, 0x8a, 0x2a, 0xe2, + 0x2e, 0xe4, 0xa9, 0x43, 0x93, 0x8d, 0x4b, 0x77, 0xdf, 0x3a, 0x6d, 0x27, 0x2d, 0xd9, 0x48, 0x1a, + 0x7a, 0xc3, 0x45, 0xf6, 0xc4, 0x54, 0x24, 0x6b, 0x3f, 0x84, 0xfc, 0x38, 0xf4, 0x63, 0x55, 0xcb, + 0x91, 0xb1, 0xbf, 0xfa, 0xc9, 0xc6, 0x9e, 0xf5, 0x79, 0x1d, 0x0e, 0xcf, 0x42, 0x3f, 0x4e, 0xf4, + 0x90, 0xf8, 0x4d, 0x95, 0xde, 0x0a, 0x94, 0x12, 0xc7, 0x61, 0x7e, 0xfe, 0x2a, 0x0b, 0x2b, 0xba, + 0x72, 0xbc, 0xe6, 0xbe, 0xfc, 0x99, 0x05, 0x2b, 0xda, 0xb2, 0x74, 0x9b, 0xfd, 0x17, 0x23, 0x61, + 0xfa, 0xe7, 0xc1, 0x74, 0x52, 0x7f, 0x75, 0xeb, 0x64, 0x52, 0xbf, 0xbf, 0xf0, 0x61, 0x2f, 0xab, + 0x70, 0x5f, 0xd5, 0x69, 0xb7, 0x20, 0x87, 0xae, 0xa4, 0xd4, 0xbc, 0x48, 0x20, 0x90, 0xb4, 0xb3, + 0x9a, 0x74, 0x9a, 0xd4, 0x47, 0x7f, 0xca, 0xc0, 0x17, 0xf7, 0x25, 0x0b, 0x55, 0x5f, 0x48, 0x22, + 0x3e, 0xa1, 0xda, 0xf1, 0xfa, 0xfa, 0xea, 0xc7, 0x50, 0xe6, 0x42, 0xc5, 0x5e, 0x82, 0x5c, 0xfb, + 0xe9, 0xd1, 0x74, 0x52, 0x87, 0x96, 0x50, 0xf1, 0xa5, 0xd1, 0x03, 0x4f, 0xb4, 0x70, 0xa7, 0x06, + 0xeb, 0x9f, 0x60, 0x3b, 0x34, 0xeb, 0xdf, 0x2c, 0x28, 0xbf, 0x27, 0xe2, 0xd7, 0x3b, 0xee, 0x9d, + 0x1f, 0x50, 0x83, 0x4d, 0x62, 0xc5, 0x7e, 0x1f, 0xf2, 0x54, 0xac, 0x4c, 0x6f, 0x6a, 0x7c, 0xf6, + 0x20, 0x6c, 0x87, 0xfd, 0x28, 0xa9, 0x48, 0xa4, 0xc2, 0xf9, 0x65, 0x06, 0xaa, 0xbb, 0x52, 0xb0, + 0x58, 0x34, 0x83, 0xa8, 0x7b, 0xf5, 0xbd, 0xcf, 0x86, 0x5c, 0xc8, 0x86, 0xc2, 0xb4, 0x17, 0xfa, + 0x6d, 0x0f, 0xa0, 0xd8, 0x8b, 0xb8, 0x18, 0x46, 0x3c, 0x49, 0xdf, 0xc7, 0xd3, 0x49, 0xbd, 0xb8, + 0x1b, 0x71, 0xb1, 0x17, 0x71, 0xcc, 0xdb, 0x77, 0x3e, 0xbb, 0xb1, 0x12, 0x4d, 0x8d, 0x44, 0xdc, + 0x4d, 0x95, 0xe3, 0xe1, 0xca, 0xff, 0x48, 0x98, 0x3e, 0x4a, 0xbf, 0xed, 0x2f, 0x03, 0xa8, 0xc0, + 0xef, 0x09, 0x8f, 0x76, 0xb0, 0xfe, 0x56, 0xdc, 0x65, 0xa2, 0x74, 0xfc, 0x8f, 0x84, 0xf3, 0x01, + 0x54, 0x66, 0xe6, 0x40, 0x63, 0x7f, 0x17, 0x72, 0x78, 0xa6, 0xb1, 0xc5, 0xed, 0x33, 0xdb, 0x87, + 0x76, 0x1a, 0x4a, 0x25, 0x99, 0x8e, 0x2c, 0x4e, 0x48, 0xae, 0xbb, 0x31, 0xdb, 0x3a, 0x8f, 0x69, + 0x2e, 0xba, 0x22, 0xf0, 0xc9, 0x58, 0x74, 0x3d, 0xf0, 0xaf, 0x66, 0x2c, 0x0a, 0xf5, 0x58, 0x94, + 0xdc, 0xf8, 0x3e, 0xe4, 0x09, 0x8b, 0x19, 0x8b, 0x16, 0xb8, 0xb2, 0x96, 0x3b, 0x77, 0x2a, 0xfa, + 0x49, 0x32, 0x74, 0xdf, 0x9c, 0x4f, 0xd3, 0x81, 0xdb, 0x5c, 0xd2, 0xf9, 0x85, 0x05, 0x55, 0x57, + 0xc4, 0xcc, 0x0f, 0x6f, 0x2e, 0x69, 0xd7, 0x20, 0x1f, 0x08, 0xa6, 0x74, 0xc6, 0xe6, 0x5c, 0xbd, + 0x40, 0x68, 0x33, 0x20, 0x08, 0xed, 0xef, 0x16, 0x94, 0x3b, 0x82, 0x05, 0xd7, 0x06, 0x8c, 0xd2, + 0x36, 0x33, 0x97, 0xd0, 0x09, 0xd8, 0xec, 0x1c, 0xd8, 0x26, 0x14, 0x28, 0xa5, 0x93, 0x81, 0xed, + 0xce, 0x39, 0x61, 0xd0, 0x41, 0xe6, 0xe4, 0x2c, 0x2d, 0x89, 0x43, 0x54, 0x72, 0x11, 0xbc, 0xd8, + 0x9f, 0x33, 0x50, 0xdd, 0x09, 0x82, 0xa8, 0x47, 0xbc, 0xff, 0x07, 0x85, 0x72, 0x0f, 0xca, 0x7d, + 0xe6, 0x07, 0x82, 0x7b, 0x64, 0x10, 0x2a, 0x95, 0x8b, 0x59, 0xb2, 0xa4, 0xe5, 0x89, 0xe4, 0x74, + 0xa0, 0x32, 0x33, 0x1f, 0x66, 0xea, 0xcc, 0x47, 0xd6, 0x85, 0x7d, 0xf4, 0xcf, 0x3c, 0x00, 0x19, + 0xb5, 0x13, 0xb3, 0x58, 0xa5, 0xdd, 0xd7, 0xba, 0xd2, 0x49, 0xe6, 0x36, 0x54, 0xd8, 0x68, 0x14, + 0xf8, 0x82, 0x7b, 0x7e, 0xc8, 0xc5, 0xb1, 0x89, 0xbe, 0xb2, 0x21, 0xb6, 0x91, 0x66, 0xd7, 0xa1, + 0x14, 0x90, 0x23, 0xbd, 0xc3, 0x48, 0xc5, 0x26, 0x18, 0x41, 0x93, 0x1e, 0x45, 0x2a, 0xb6, 0x47, + 0x50, 0x35, 0x0c, 0xc9, 0xfc, 0x91, 0x23, 0x8f, 0xbe, 0x3f, 0x9d, 0xd4, 0xcb, 0x7a, 0x60, 0xb9, + 0xf4, 0x14, 0x52, 0x0e, 0x66, 0x7a, 0xb8, 0x3d, 0x48, 0x21, 0x91, 0x51, 0xf2, 0xe9, 0x67, 0x32, + 0xd0, 0xc7, 0x5d, 0xca, 0x34, 0xe6, 0x6a, 0xf8, 0xdb, 0xae, 0xc1, 0x52, 0x20, 0x98, 0x0c, 0x85, + 0xac, 0x15, 0x36, 0xad, 0xad, 0xa2, 0x9b, 0x2c, 0x4f, 0xbf, 0x77, 0x96, 0xae, 0xe3, 0xbd, 0x33, + 0x7b, 0xe3, 0x15, 0xaf, 0xe2, 0x8d, 0xb7, 0x7c, 0xb9, 0x37, 0x5e, 0x0b, 0x8a, 0x92, 0xf5, 0x63, + 0x8c, 0xc8, 0x1a, 0x10, 0x1e, 0xe7, 0x4c, 0x3c, 0xf4, 0xe9, 0x16, 0x39, 0x8d, 0x8a, 0x54, 0xd2, + 0x79, 0x01, 0x55, 0x6c, 0x6b, 0x07, 0x51, 0x30, 0x1e, 0xea, 0x6a, 0x33, 0x5f, 0x0c, 0xac, 0x6b, + 0x2c, 0x06, 0x0e, 0x87, 0xca, 0xec, 0x68, 0xcc, 0xd4, 0x0e, 0xe4, 0x9e, 0xfb, 0x5c, 0xe7, 0x69, + 0xa5, 0x79, 0x1f, 0xd3, 0xea, 0xc0, 0xe7, 0xea, 0x64, 0x52, 0xbf, 0xb7, 0xa8, 0x13, 0x0f, 0x30, + 0xab, 0x50, 0x99, 0xf3, 0x6f, 0x4b, 0x1f, 0x73, 0x63, 0xa3, 0x39, 0x83, 0x22, 0xb9, 0x7f, 0xf6, + 0x41, 0x57, 0x7f, 0x32, 0x46, 0xda, 0x05, 0x3f, 0x19, 0x6b, 0x51, 0x77, 0x89, 0xf4, 0xb6, 0xf9, + 0x6c, 0x42, 0xc9, 0xce, 0x4f, 0x28, 0xbf, 0xcd, 0xc2, 0x1b, 0xe9, 0x4d, 0x9b, 0x4c, 0x09, 0x9c, + 0xc2, 0xff, 0x07, 0x6e, 0x3b, 0xff, 0x05, 0xfb, 0xea, 0xaa, 0xed, 0x1a, 0xe4, 0x75, 0x95, 0xa5, + 0xf2, 0xe8, 0xea, 0x05, 0x52, 0xc5, 0x28, 0xea, 0x1d, 0x9a, 0x81, 0x5d, 0x2f, 0x66, 0x29, 0x5b, + 0xb8, 0x54, 0xca, 0x3a, 0x9e, 0x1e, 0x73, 0xd3, 0x07, 0xd6, 0x87, 0x50, 0xa0, 0x4b, 0x26, 0xad, + 0xe9, 0x1b, 0xe7, 0x35, 0xf6, 0x53, 0xee, 0x4d, 0xfb, 0x14, 0xa9, 0x71, 0xaa, 0x50, 0xde, 0xdf, + 0xdd, 0x63, 0xd8, 0xff, 0x30, 0xd4, 0x9d, 0xdb, 0x50, 0x4a, 0xd6, 0x78, 0xde, 0x1a, 0xe4, 0x15, + 0x36, 0x30, 0xfd, 0x3f, 0x36, 0xae, 0x5e, 0x38, 0xbf, 0xb6, 0xa0, 0xd4, 0x6a, 0x52, 0x67, 0xbb, + 0x89, 0xfc, 0xb8, 0x0d, 0x4b, 0xbc, 0xeb, 0xcd, 0x66, 0x90, 0x26, 0x90, 0xfa, 0xe6, 0x07, 0x6c, + 0x28, 0xdc, 0x02, 0xef, 0xe2, 0x5f, 0xe7, 0xe7, 0x19, 0x00, 0x83, 0x09, 0x81, 0xdb, 0x90, 0x1b, + 0x2b, 0x61, 0x1a, 0xae, 0x4b, 0xbf, 0xed, 0x2d, 0x58, 0xc5, 0x03, 0xbd, 0x1e, 0xeb, 0x1d, 0x0a, + 0x6f, 0xac, 0xd8, 0x20, 0x99, 0xd7, 0xaa, 0x48, 0xdf, 0x45, 0xf2, 0x33, 0xa4, 0xda, 0xf7, 0x60, + 0x9d, 0xbc, 0xeb, 0xb1, 0x90, 0x7b, 0x7d, 0x3f, 0x88, 0x85, 0x34, 0xfc, 0x3a, 0x7f, 0x3e, 0x4f, + 0xbb, 0x3b, 0x21, 0x7f, 0x48, 0x7b, 0x5a, 0xe8, 0x4d, 0xa8, 0x0e, 0xc5, 0x30, 0x66, 0xdd, 0x20, + 0x51, 0xae, 0x87, 0x96, 0x4a, 0x42, 0xd5, 0x6c, 0x6f, 0x83, 0xdd, 0x0d, 0xa2, 0xde, 0x91, 0x37, + 0xf2, 0xc3, 0x50, 0x70, 0xc3, 0x4a, 0x3d, 0xd0, 0x5d, 0xa5, 0x9d, 0xa7, 0xb4, 0x91, 0x72, 0xc7, + 0x51, 0xcc, 0x02, 0x6f, 0x28, 0x86, 0x91, 0x7c, 0x61, 0xb8, 0x0b, 0x9a, 0x9b, 0x76, 0xf6, 0x68, + 0x83, 0xb8, 0x9d, 0xdf, 0x5b, 0xf0, 0xc6, 0xdc, 0x40, 0xce, 0x7e, 0x7a, 0xf5, 0xd3, 0xe0, 0x03, + 0xc8, 0xeb, 0xa9, 0x2b, 0xb3, 0xf0, 0xd4, 0xa5, 0x05, 0x1d, 0x1b, 0x56, 0x5f, 0xc2, 0x88, 0x33, + 0xec, 0x5f, 0x33, 0xb0, 0xe2, 0x8a, 0x11, 0xf3, 0xe5, 0x35, 0x0d, 0xb1, 0x1d, 0xc8, 0x76, 0xd3, + 0xaf, 0x1f, 0x3b, 0xd3, 0x49, 0x3d, 0xdb, 0xa4, 0x82, 0xb0, 0x70, 0x6c, 0x22, 0xe8, 0x76, 0xcb, + 0x45, 0x6d, 0xf6, 0x53, 0xc8, 0x3e, 0x4f, 0x3f, 0x20, 0xbd, 0x8b, 0x4a, 0x0f, 0x48, 0xe9, 0x85, + 0x9a, 0x0f, 0xaa, 0xc2, 0x50, 0xef, 0x32, 0x2c, 0x8d, 0xc7, 0xf4, 0x3e, 0xa8, 0xe8, 0x50, 0x6f, + 0x32, 0xde, 0xe6, 0xc7, 0x6e, 0xa1, 0x4b, 0x7f, 0xf1, 0x19, 0x2a, 0x05, 0x53, 0xe6, 0x23, 0xed, + 0xb2, 0x6b, 0x56, 0xce, 0x2a, 0xbe, 0xbd, 0x52, 0x1b, 0xa2, 0x59, 0x6b, 0xb0, 0xde, 0x49, 0xcc, + 0xb3, 0xcf, 0xd4, 0x51, 0x9a, 0xb2, 0xce, 0x5f, 0x2c, 0xf8, 0xc2, 0xe9, 0x2d, 0xcc, 0x9c, 0x75, + 0x28, 0x88, 0x10, 0xc3, 0x95, 0xcc, 0x5e, 0x74, 0xcd, 0xca, 0xfe, 0x0a, 0xac, 0xa8, 0x71, 0xaf, + 0x27, 0x94, 0xf2, 0x46, 0x42, 0x7a, 0x43, 0x3f, 0x34, 0x2f, 0x82, 0x8a, 0x21, 0x3f, 0x15, 0x72, + 0xcf, 0x0f, 0xed, 0x3b, 0x50, 0x35, 0xd3, 0x79, 0xc2, 0xa6, 0x47, 0x4e, 0x33, 0xb3, 0x1b, 0x2e, + 0x07, 0x2a, 0x3a, 0xae, 0x85, 0x94, 0x5e, 0x2f, 0x7d, 0x3a, 0x97, 0x88, 0xf8, 0x3d, 0x29, 0x77, + 0xc3, 0xd8, 0xfe, 0x12, 0x2c, 0xe3, 0xae, 0x2e, 0x40, 0xf9, 0xcd, 0xec, 0xd6, 0xb2, 0x5b, 0x14, + 0x52, 0x12, 0xd2, 0xe6, 0xad, 0x8f, 0xa7, 0x1b, 0xd6, 0x3f, 0xa6, 0x1b, 0xd6, 0xbf, 0xa6, 0x1b, + 0xd6, 0x0f, 0xcb, 0x8d, 0xed, 0x77, 0xd2, 0x40, 0xe8, 0x16, 0xc8, 0xaa, 0xf7, 0xfe, 0x13, 0x00, + 0x00, 0xff, 0xff, 0xa8, 0x97, 0x5e, 0xfc, 0x6a, 0x1f, 0x00, 0x00, } func (m *Item) Marshal() (dAtA []byte, err error) { @@ -5008,7 +5134,7 @@ func (m *DeleteBlobRawRet) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *DeleteBlobStatsArgs) Marshal() (dAtA []byte, err error) { +func (m *RepairSliceArgs) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -5018,12 +5144,84 @@ func (m *DeleteBlobStatsArgs) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *DeleteBlobStatsArgs) MarshalTo(dAtA []byte) (int, error) { +func (m *RepairSliceArgs) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *DeleteBlobStatsArgs) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *RepairSliceArgs) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Reason) > 0 { + i -= len(m.Reason) + copy(dAtA[i:], m.Reason) + i = encodeVarintShardnode(dAtA, i, uint64(len(m.Reason))) + i-- + dAtA[i] = 0x2a + } + if len(m.BadIdx) > 0 { + dAtA29 := make([]byte, len(m.BadIdx)*10) + var j28 int + for _, num := range m.BadIdx { + for num >= 1<<7 { + dAtA29[j28] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j28++ + } + dAtA29[j28] = uint8(num) + j28++ + } + i -= j28 + copy(dAtA[i:], dAtA29[:j28]) + i = encodeVarintShardnode(dAtA, i, uint64(j28)) + i-- + dAtA[i] = 0x22 + } + if m.Vid != 0 { + i = encodeVarintShardnode(dAtA, i, uint64(m.Vid)) + i-- + dAtA[i] = 0x18 + } + if m.Bid != 0 { + i = encodeVarintShardnode(dAtA, i, uint64(m.Bid)) + i-- + dAtA[i] = 0x10 + } + { + size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShardnode(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + return len(dAtA) - i, nil +} + +func (m *RepairSliceRet) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RepairSliceRet) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *RepairSliceRet) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int @@ -5035,7 +5233,7 @@ func (m *DeleteBlobStatsArgs) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *DeleteBlobStatsRet) Marshal() (dAtA []byte, err error) { +func (m *ShardnodeTaskStatsArgs) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -5045,12 +5243,39 @@ func (m *DeleteBlobStatsRet) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *DeleteBlobStatsRet) MarshalTo(dAtA []byte) (int, error) { +func (m *ShardnodeTaskStatsArgs) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *DeleteBlobStatsRet) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *ShardnodeTaskStatsArgs) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + return len(dAtA) - i, nil +} + +func (m *ShardnodeTaskStatsRet) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ShardnodeTaskStatsRet) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ShardnodeTaskStatsRet) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int @@ -6006,7 +6231,38 @@ func (m *DeleteBlobRawRet) Size() (n int) { return n } -func (m *DeleteBlobStatsArgs) Size() (n int) { +func (m *RepairSliceArgs) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = m.Header.Size() + n += 1 + l + sovShardnode(uint64(l)) + if m.Bid != 0 { + n += 1 + sovShardnode(uint64(m.Bid)) + } + if m.Vid != 0 { + n += 1 + sovShardnode(uint64(m.Vid)) + } + if len(m.BadIdx) > 0 { + l = 0 + for _, e := range m.BadIdx { + l += sovShardnode(uint64(e)) + } + n += 1 + sovShardnode(uint64(l)) + l + } + l = len(m.Reason) + if l > 0 { + n += 1 + l + sovShardnode(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *RepairSliceRet) Size() (n int) { if m == nil { return 0 } @@ -6018,7 +6274,19 @@ func (m *DeleteBlobStatsArgs) Size() (n int) { return n } -func (m *DeleteBlobStatsRet) Size() (n int) { +func (m *ShardnodeTaskStatsArgs) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ShardnodeTaskStatsRet) Size() (n int) { if m == nil { return 0 } @@ -11183,7 +11451,7 @@ func (m *DeleteBlobRawRet) Unmarshal(dAtA []byte) error { } return nil } -func (m *DeleteBlobStatsArgs) Unmarshal(dAtA []byte) error { +func (m *RepairSliceArgs) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -11206,10 +11474,240 @@ func (m *DeleteBlobStatsArgs) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: DeleteBlobStatsArgs: wiretype end group for non-group") + return fmt.Errorf("proto: RepairSliceArgs: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: DeleteBlobStatsArgs: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: RepairSliceArgs: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShardnode + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShardnode + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Bid", wireType) + } + m.Bid = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Bid |= github_com_cubefs_cubefs_blobstore_common_proto.BlobID(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Vid", wireType) + } + m.Vid = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Vid |= github_com_cubefs_cubefs_blobstore_common_proto.Vid(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType == 0 { + var v uint32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.BadIdx = append(m.BadIdx, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthShardnode + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthShardnode + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + var count int + for _, integer := range dAtA[iNdEx:postIndex] { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.BadIdx) == 0 { + m.BadIdx = make([]uint32, 0, elementCount) + } + for iNdEx < postIndex { + var v uint32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.BadIdx = append(m.BadIdx, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field BadIdx", wireType) + } + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Reason", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthShardnode + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthShardnode + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Reason = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipShardnode(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShardnode + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RepairSliceRet) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RepairSliceRet: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RepairSliceRet: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { default: @@ -11234,7 +11732,7 @@ func (m *DeleteBlobStatsArgs) Unmarshal(dAtA []byte) error { } return nil } -func (m *DeleteBlobStatsRet) Unmarshal(dAtA []byte) error { +func (m *ShardnodeTaskStatsArgs) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -11257,10 +11755,61 @@ func (m *DeleteBlobStatsRet) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: DeleteBlobStatsRet: wiretype end group for non-group") + return fmt.Errorf("proto: ShardnodeTaskStatsArgs: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: DeleteBlobStatsRet: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: ShardnodeTaskStatsArgs: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipShardnode(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShardnode + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ShardnodeTaskStatsRet) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShardnode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ShardnodeTaskStatsRet: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ShardnodeTaskStatsRet: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: diff --git a/blobstore/api/shardnode/shardnode.proto b/blobstore/api/shardnode/shardnode.proto index 3f8c8cf5a..981fe6f2b 100644 --- a/blobstore/api/shardnode/shardnode.proto +++ b/blobstore/api/shardnode/shardnode.proto @@ -262,9 +262,19 @@ message DeleteBlobRawArgs { message DeleteBlobRawRet {} -message DeleteBlobStatsArgs {} +message RepairSliceArgs { + ShardOpHeader header = 1 [(gogoproto.nullable) = false]; + uint64 bid = 2 [(gogoproto.customname) = "Bid", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.BlobID"]; + uint32 vid = 3 [(gogoproto.customname) = "Vid", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.Vid"]; + repeated uint32 bad_idx = 4 [(gogoproto.customname) = "BadIdx"]; + string reason = 5; +} -message DeleteBlobStatsRet { +message RepairSliceRet {} + +message ShardnodeTaskStatsArgs {} + +message ShardnodeTaskStatsRet { bool enable = 1; string success_per_min = 2; string failed_per_min = 3; diff --git a/blobstore/cli/rpc2.go b/blobstore/cli/rpc2.go index 86d1df291..a75d13b68 100644 --- a/blobstore/cli/rpc2.go +++ b/blobstore/cli/rpc2.go @@ -30,8 +30,8 @@ var types = map[string]func() rpc2.Codec{ "shardnode.GetBlobArgs": func() rpc2.Codec { return new(shardnode.GetBlobArgs) }, "shardnode.GetBlobRet": func() rpc2.Codec { return new(shardnode.GetBlobRet) }, - "shardnode.DeleteBlobStatsArgs": func() rpc2.Codec { return new(shardnode.DeleteBlobStatsArgs) }, - "shardnode.DeleteBlobStatsRet": func() rpc2.Codec { return new(shardnode.DeleteBlobStatsRet) }, + "shardnode.ShardnodeTaskStatsArgs": func() rpc2.Codec { return new(shardnode.ShardnodeTaskStatsArgs) }, + "shardnode.ShardnodeTaskStatsRet": func() rpc2.Codec { return new(shardnode.ShardnodeTaskStatsRet) }, "nil": func() rpc2.Codec { return nil }, "rpc2.NoParameter": func() rpc2.Codec { return rpc2.NoParameter }, diff --git a/blobstore/shardnode/base/statistics_metrics.go b/blobstore/shardnode/base/statistics_metrics.go index 99d5150b1..097648708 100644 --- a/blobstore/shardnode/base/statistics_metrics.go +++ b/blobstore/shardnode/base/statistics_metrics.go @@ -19,6 +19,10 @@ import ( "sort" "strings" "sync" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/cubefs/cubefs/blobstore/common/proto" ) // ErrorStats error stats @@ -98,3 +102,75 @@ func errStrFormat(err error) string { strSlice := strings.Split(err.Error(), ":") return strings.TrimSpace(strSlice[len(strSlice)-1]) } + +const ( + Namespace = "shardnode" + ShardRepair = "shard_repair" // same as scheduler + ChunkMissMigrateAbnormal = "chunk_miss_migrate" +) + +type AbnormalReporter struct { + lock sync.RWMutex + abnormalReporter *prometheus.GaugeVec + reportedVuids map[proto.Vuid]struct{} +} + +// NewAbnormalReporter returns abnormal reporter +func NewAbnormalReporter(clusterID proto.ClusterID, taskType string, abnormalKind string) *AbnormalReporter { + abnormalReporter := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "task", + Name: "abnormal_task", + Help: "abnormal task", + ConstLabels: map[string]string{ + "cluster_id": fmt.Sprintf("%d", clusterID), + "task_type": taskType, + "kind": abnormalKind, + }, + }, []string{"diskID", "vuid"}) + if err := prometheus.Register(abnormalReporter); err != nil { + if are, ok := err.(prometheus.AlreadyRegisteredError); ok { + abnormalReporter = are.ExistingCollector.(*prometheus.GaugeVec) + return &AbnormalReporter{ + abnormalReporter: abnormalReporter, + reportedVuids: make(map[proto.Vuid]struct{}), + } + } + panic(err) + } + return &AbnormalReporter{ + abnormalReporter: abnormalReporter, + reportedVuids: make(map[proto.Vuid]struct{}), + } +} + +// ReportAbnormal report abnormal task +func (abr *AbnormalReporter) ReportAbnormal(diskID proto.DiskID, vuid proto.Vuid) { + abr.abnormalReporter.WithLabelValues( + fmt.Sprintf("%d", diskID), + fmt.Sprintf("%d", vuid), + ).Set(1) +} + +// CancelAbnormal cancel abnormal report +func (abr *AbnormalReporter) CancelAbnormal(diskID proto.DiskID, vuid proto.Vuid) { + abr.abnormalReporter.WithLabelValues( + fmt.Sprintf("%d", diskID), + fmt.Sprintf("%d", vuid), + ).Set(0) +} + +// IsVuidReported check if vuid abnormal is reported +func (abr *AbnormalReporter) IsVuidReported(vuid proto.Vuid) bool { + abr.lock.RLock() + defer abr.lock.RUnlock() + _, ok := abr.reportedVuids[vuid] + return ok +} + +// SetVuidReported set vuid reported +func (abr *AbnormalReporter) SetVuidReported(vuid proto.Vuid) { + abr.lock.Lock() + abr.reportedVuids[vuid] = struct{}{} + abr.lock.Unlock() +} diff --git a/blobstore/shardnode/base/statistics_metrics_test.go b/blobstore/shardnode/base/statistics_metrics_test.go index 41d41cb35..54a99e90c 100644 --- a/blobstore/shardnode/base/statistics_metrics_test.go +++ b/blobstore/shardnode/base/statistics_metrics_test.go @@ -20,6 +20,8 @@ import ( "testing" "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/common/proto" ) func TestErrorStats(t *testing.T) { @@ -61,3 +63,15 @@ func TestErrStrFormat(t *testing.T) { require.Equal(t, "fake error", errStrFormat(err2)) require.Equal(t, "", errStrFormat(err3)) } + +func TestAbnormalReport(t *testing.T) { + rp := NewAbnormalReporter(proto.ClusterID(1), "", "") + diskID := proto.DiskID(1) + vuid := proto.Vuid(100) + rp.ReportAbnormal(diskID, vuid) + rp.CancelAbnormal(diskID, vuid) + + rp.SetVuidReported(vuid) + require.True(t, rp.IsVuidReported(vuid)) + require.False(t, rp.IsVuidReported(vuid+1)) +} diff --git a/blobstore/shardnode/base/transport.go b/blobstore/shardnode/base/transport.go index 06932457a..b5734c69e 100644 --- a/blobstore/shardnode/base/transport.go +++ b/blobstore/shardnode/base/transport.go @@ -34,6 +34,8 @@ type ( GetConfig(ctx context.Context, key string) (string, error) ShardReport(ctx context.Context, reports []clustermgr.ShardUnitInfo) ([]clustermgr.ShardTask, error) GetRouteUpdate(ctx context.Context, routeVersion proto.RouteVersion) (proto.RouteVersion, []clustermgr.CatalogChangeItem, error) + GetService(ctx context.Context, name string) ([]string, error) + GetBlobnodeDiskInfo(ctx context.Context, diskID proto.DiskID) (*clustermgr.BlobNodeDiskInfo, error) NodeTransport SpaceTransport AllocVolTransport @@ -78,6 +80,7 @@ type ( BlobTransport interface { DeleteSliceUnit(ctx context.Context, info proto.VunitLocation, bid proto.BlobID) (err error) MarkDeleteSliceUnit(ctx context.Context, info proto.VunitLocation, bid proto.BlobID) (err error) + RepairSlice(ctx context.Context, host string, volInfo *snproto.VolumeInfoSimple, repairMsg *snproto.SliceRepairMsg) (err error) } VolumeTransport interface { @@ -291,6 +294,18 @@ func (t *transport) GetConfig(ctx context.Context, key string) (string, error) { return t.cmClient.GetConfig(ctx, key) } +func (t *transport) GetService(ctx context.Context, name string) ([]string, error) { + nodes, err := t.cmClient.GetService(ctx, clustermgr.GetServiceArgs{Name: name}) + if err != nil { + return nil, err + } + hosts := make([]string, 0, len(nodes.Nodes)) + for _, node := range nodes.Nodes { + hosts = append(hosts, node.Host) + } + return hosts, nil +} + func (t *transport) AllocBid(ctx context.Context, count uint64) (proto.BlobID, error) { ret, err := t.cmClient.AllocBid(ctx, &clustermgr.BidScopeArgs{Count: count}) if err != nil { @@ -360,6 +375,17 @@ func (t *transport) MarkDeleteSliceUnit(ctx context.Context, info proto.VunitLoc }) } +func (t *transport) RepairSlice(ctx context.Context, host string, volInfo *snproto.VolumeInfoSimple, repairMsg *snproto.SliceRepairMsg) (err error) { + task := proto.ShardRepairTask{ + Bid: repairMsg.Bid, + CodeMode: volInfo.CodeMode, + Sources: volInfo.VunitLocations, + BadIdxs: sliceUint32ToInt(repairMsg.BadIdx), + Reason: repairMsg.Reason, + } + return t.bnClient.RepairShard(ctx, host, &task) +} + func (t *transport) ListVolume(ctx context.Context, marker proto.Vid, count int) (volInfo []*snproto.VolumeInfoSimple, retVid proto.Vid, err error) { vols, err := t.cmClient.ListVolume(ctx, &clustermgr.ListVolumeArgs{Marker: marker, Count: count}) if err != nil { @@ -383,3 +409,15 @@ func (t *transport) GetVolumeInfo(ctx context.Context, vid proto.Vid) (ret *snpr ret.Set(info) return ret, nil } + +func (t *transport) GetBlobnodeDiskInfo(ctx context.Context, diskID proto.DiskID) (*clustermgr.BlobNodeDiskInfo, error) { + return t.cmClient.DiskInfo(ctx, diskID) +} + +func sliceUint32ToInt(s []uint32) []uint8 { + var ret []uint8 + for _, e := range s { + ret = append(ret, uint8(e)) + } + return ret +} diff --git a/blobstore/shardnode/blob.go b/blobstore/shardnode/blob.go index 1e009018a..1ac257044 100644 --- a/blobstore/shardnode/blob.go +++ b/blobstore/shardnode/blob.go @@ -114,6 +114,14 @@ func (s *service) deleteBlobRaw(ctx context.Context, req *shardnode.DeleteBlobRa return s.blobDelMgr.Delete(ctx, req) } -func (s *service) deleteBlobStats() *shardnode.DeleteBlobStatsRet { +func (s *service) deleteBlobStats() *shardnode.ShardnodeTaskStatsRet { return s.blobDelMgr.Stats() } + +func (s *service) repairSlice(ctx context.Context, req *shardnode.RepairSliceArgs) error { + return s.sliceRepairMgr.Repair(ctx, req) +} + +func (s *service) repairSliceStats() *shardnode.ShardnodeTaskStatsRet { + return s.sliceRepairMgr.Stats() +} diff --git a/blobstore/shardnode/httpservice.go b/blobstore/shardnode/httpservice.go index b7a9f6c9e..e616dd583 100644 --- a/blobstore/shardnode/httpservice.go +++ b/blobstore/shardnode/httpservice.go @@ -15,9 +15,6 @@ package shardnode import ( - "encoding/json" - "net/http" - "github.com/cubefs/cubefs/blobstore/api/shardnode" "github.com/cubefs/cubefs/blobstore/common/rpc" "github.com/cubefs/cubefs/blobstore/common/trace" @@ -43,25 +40,17 @@ func (s *HttpService) HttpShardStats(c *rpc.Context) { c.RespondError(err) return } - - data, err := json.Marshal(ret) - if err != nil { - c.RespondError(err) - return - } - - c.RespondWith(http.StatusOK, rpc.MIMEJSON, data) + c.RespondJSON(ret) } func (s *HttpService) HttpDeleteBlobStats(c *rpc.Context) { ret := s.deleteBlobStats() - data, err := json.Marshal(ret) - if err != nil { - c.RespondError(err) - return - } + c.RespondJSON(ret) +} - c.RespondWith(http.StatusOK, rpc.MIMEJSON, data) +func (s *HttpService) HttpRepairSliceStats(c *rpc.Context) { + ret := s.repairSliceStats() + c.RespondJSON(ret) } func newHttpHandler(service *HttpService) *rpc.Router { @@ -69,6 +58,7 @@ func newHttpHandler(service *HttpService) *rpc.Router { rpc.GET("/shard/stats", service.HttpShardStats, rpc.OptArgsQuery()) rpc.GET("/blob/delete/stats", service.HttpDeleteBlobStats) + rpc.GET("/slice/repair/stats", service.HttpRepairSliceStats) return rpc.DefaultRouter } diff --git a/blobstore/shardnode/httpservice_test.go b/blobstore/shardnode/httpservice_test.go index 509eeb556..5996a5353 100644 --- a/blobstore/shardnode/httpservice_test.go +++ b/blobstore/shardnode/httpservice_test.go @@ -84,12 +84,20 @@ func TestHttpService_HTTPGet(t *testing.T) { require.Equal(t, suid, ret1.Suid) // Test delete blob stats - ret2 := &shardnode.DeleteBlobStatsRet{} + ret2 := &shardnode.ShardnodeTaskStatsRet{} resp2, err := client.Get(ctx, "http://127.0.0.1:11000/blob/delete/stats") require.Nil(t, err) json.NewDecoder(resp2.Body).Decode(ret2) resp2.Body.Close() require.NotNil(t, ret2) + + // Test repair shard stats + ret3 := &shardnode.ShardnodeTaskStatsRet{} + resp3, err := client.Get(ctx, "http://127.0.0.1:11000/slice/repair/stats") + require.Nil(t, err) + json.NewDecoder(resp3.Body).Decode(ret3) + resp3.Body.Close() + require.NotNil(t, ret3) } // Test setUpHttp function diff --git a/blobstore/shardnode/message/blob.go b/blobstore/shardnode/message/blob.go deleted file mode 100644 index 54003a84f..000000000 --- a/blobstore/shardnode/message/blob.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2025 The CubeFS Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. See the License for the specific language governing -// permissions and limitations under the License. - -package message - -import ( - "context" - "fmt" - - snapi "github.com/cubefs/cubefs/blobstore/api/shardnode" - "github.com/cubefs/cubefs/blobstore/common/proto" -) - -func (m *BlobDeleteMgr) SlicesToDeleteMsgItems(ctx context.Context, slices []proto.Slice, shardKeys []string) ([]snapi.Item, error) { - return m.slicesToDeleteMsgItems(ctx, slices, shardKeys) -} - -func (m *BlobDeleteMgr) Delete(ctx context.Context, req *snapi.DeleteBlobRawArgs) error { - return m.insertDeleteMsg(ctx, req) -} - -func (m *BlobDeleteMgr) Stats() *snapi.DeleteBlobStatsRet { - deleteSuccessCounter, deleteFailedCounter := m.getTaskStats() - delErrStats, delTotalErrCnt := m.getErrorStats() - return &snapi.DeleteBlobStatsRet{ - Enable: m.enabled(), - SuccessPerMin: fmt.Sprint(deleteSuccessCounter), - FailedPerMin: fmt.Sprint(deleteFailedCounter), - TotalErrCnt: delTotalErrCnt, - ErrStats: delErrStats, - } -} diff --git a/blobstore/shardnode/message/delete.go b/blobstore/shardnode/message/delete.go index bf108a460..302f65166 100644 --- a/blobstore/shardnode/message/delete.go +++ b/blobstore/shardnode/message/delete.go @@ -16,6 +16,7 @@ package message import ( "context" + "fmt" "time" snapi "github.com/cubefs/cubefs/blobstore/api/shardnode" @@ -58,6 +59,26 @@ func (m *BlobDeleteMgr) Start() { m.messageMgr.run() } +func (m *BlobDeleteMgr) SlicesToDeleteMsgItems(ctx context.Context, slices []proto.Slice, shardKeys []string) ([]snapi.Item, error) { + return m.slicesToDeleteMsgItems(ctx, slices, shardKeys) +} + +func (m *BlobDeleteMgr) Delete(ctx context.Context, req *snapi.DeleteBlobRawArgs) error { + return m.insertDeleteMsg(ctx, req) +} + +func (m *BlobDeleteMgr) Stats() *snapi.ShardnodeTaskStatsRet { + deleteSuccessCounter, deleteFailedCounter := m.getTaskStats() + delErrStats, delTotalErrCnt := m.getErrorStats() + return &snapi.ShardnodeTaskStatsRet{ + Enable: m.enabled(), + SuccessPerMin: fmt.Sprint(deleteSuccessCounter), + FailedPerMin: fmt.Sprint(deleteFailedCounter), + TotalErrCnt: delTotalErrCnt, + ErrStats: delErrStats, + } +} + func (m *BlobDeleteMgr) ItemToMessageExt(item interface{}) (snproto.MessageExt, error) { msgItem, ok := item.(messageItem) if !ok { @@ -259,10 +280,10 @@ func (m *BlobDeleteMgr) deleteSliceUnit(ctx context.Context, info proto.VunitLoc if markerDel { stage = DeleteStageMarkDelete - err = m.cfg.Transport.MarkDeleteSliceUnit(ctx, info, sliceId) + err = m.cfg.BlobTransport.MarkDeleteSliceUnit(ctx, info, sliceId) } else { stage = DeleteStageDelete - err = m.cfg.Transport.DeleteSliceUnit(ctx, info, sliceId) + err = m.cfg.BlobTransport.DeleteSliceUnit(ctx, info, sliceId) } if err != nil { diff --git a/blobstore/shardnode/message/delete_test.go b/blobstore/shardnode/message/delete_test.go index 0483bcf2d..b60cf6c90 100644 --- a/blobstore/shardnode/message/delete_test.go +++ b/blobstore/shardnode/message/delete_test.go @@ -51,7 +51,7 @@ func TestNewTestBlobDeleteMgr(t *testing.T) { cfg := &BlobDelMgrConfig{ TaskSwitchMgr: taskSwitchMgr, ShardGetter: sg, - Transport: mocks.NewMockBlobTransport(ctr(t)), + BlobTransport: mocks.NewMockBlobTransport(ctr(t)), VolCache: mocks.NewMockIVolumeCache(ctr(t)), MessageCfg: MessageCfg{ FailedMsgChannelSize: 5, @@ -337,8 +337,8 @@ func TestBlobDeleteMgr_DeleteSlice_UpdateVolume(t *testing.T) { tp.EXPECT().MarkDeleteSliceUnit(any, any, any).Return(nil).Times(3) mgr := newMinimalBlobDeleteMgr(&MessageMgrConfig{ - Transport: tp, - VolCache: vc, + BlobTransport: tp, + VolCache: vc, }) msgExt := &delMsgExt{msg: snproto.DeleteMsg{ @@ -368,8 +368,8 @@ func TestBlobDeleteMgr_DeleteSlice_UpdateVolume2(t *testing.T) { }).Times(6) mgr := newMinimalBlobDeleteMgr(&MessageMgrConfig{ - Transport: tp, - VolCache: vc, + BlobTransport: tp, + VolCache: vc, }) msgExt := &delMsgExt{msg: snproto.DeleteMsg{ @@ -389,7 +389,7 @@ func TestBlobDeleteMgr_DeleteShard_BackToInitStage(t *testing.T) { volInfo := newMockSimpleVolumeInfo(vid) mgr := newMinimalBlobDeleteMgr(&MessageMgrConfig{ - Transport: tp, + BlobTransport: tp, }) idx := 0 @@ -415,7 +415,7 @@ func TestBlobDeleteMgr_DeleteShard_AssumeSuccess(t *testing.T) { volInfo := newMockSimpleVolumeInfo(vid) mgr := newMinimalBlobDeleteMgr(&MessageMgrConfig{ - Transport: tp, + BlobTransport: tp, }) idx := 0 diff --git a/blobstore/shardnode/message/processor.go b/blobstore/shardnode/message/processor.go index abad185f7..ece361bfa 100644 --- a/blobstore/shardnode/message/processor.go +++ b/blobstore/shardnode/message/processor.go @@ -84,7 +84,7 @@ type MessageMgrConfig struct { executor MessageExecutor TaskSwitchMgr *taskswitch.SwitchMgr ShardGetter ShardGetter - Transport base.BlobTransport + BlobTransport base.BlobTransport VolCache base.IVolumeCache MessageCfg } diff --git a/blobstore/shardnode/message/processor_test.go b/blobstore/shardnode/message/processor_test.go index d8a6baf17..38b2163d6 100644 --- a/blobstore/shardnode/message/processor_test.go +++ b/blobstore/shardnode/message/processor_test.go @@ -58,7 +58,7 @@ func TestBlobMessageMgr_New(t *testing.T) { cfg := &BlobDelMgrConfig{ TaskSwitchMgr: taskSwitchMgr, ShardGetter: sg, - Transport: mocks.NewMockBlobTransport(ctr(t)), + BlobTransport: mocks.NewMockBlobTransport(ctr(t)), VolCache: mocks.NewMockIVolumeCache(ctr(t)), MessageCfg: MessageCfg{ FailedMsgChannelSize: 5, @@ -531,10 +531,10 @@ func newTestPriorityConfig() TierConfig { func newTestBlobMessageMgr(t *testing.T, sg ShardGetter, tp base.BlobTransport, vc base.IVolumeCache, executor MessageExecutor, messageType snproto.MessageType) *messageMgr { cfg := &MessageMgrConfig{ - executor: executor, - ShardGetter: sg, - Transport: tp, - VolCache: vc, + executor: executor, + ShardGetter: sg, + BlobTransport: tp, + VolCache: vc, MessageCfg: MessageCfg{ RetryTimes: 3, FailedMsgChannelSize: 5, diff --git a/blobstore/shardnode/message/repair.go b/blobstore/shardnode/message/repair.go index 372a7a198..8663ebad6 100644 --- a/blobstore/shardnode/message/repair.go +++ b/blobstore/shardnode/message/repair.go @@ -16,32 +16,65 @@ package message import ( "context" + "fmt" + "time" + "github.com/cubefs/cubefs/blobstore/api/scheduler" snapi "github.com/cubefs/cubefs/blobstore/api/shardnode" + apierr "github.com/cubefs/cubefs/blobstore/common/errors" "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/rpc2" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/shardnode/base" snproto "github.com/cubefs/cubefs/blobstore/shardnode/proto" + "github.com/cubefs/cubefs/blobstore/shardnode/storage" "github.com/cubefs/cubefs/blobstore/util/errors" + "github.com/cubefs/cubefs/blobstore/util/retry" + "github.com/cubefs/cubefs/blobstore/util/selector" ) +// ErrBlobnodeServiceUnavailable worker service unavailable +var ErrBlobnodeServiceUnavailable = errors.New("blobnode service unavailable") + +// OrphanSlice orphan slice identification. +type OrphanSlice struct { + ClusterID proto.ClusterID `json:"cluster_id"` + Vid proto.Vid `json:"vid"` + Bid proto.BlobID `json:"bid"` +} + +type SliceRepairMgrConfig struct { + *MessageMgrConfig + Transport base.Transport + BlobNodeSelector selector.Selector + SCClient scheduler.ITaskInfoNotifier +} + // SliceRepairMgr handles repair messages type SliceRepairMgr struct { *messageMgr + transport base.Transport + scClient scheduler.ITaskInfoNotifier + blobNodeSelector selector.Selector + chunkMissMigrateReporter *base.AbnormalReporter } -func NewShardRepairMgr(cfg *MessageMgrConfig) (*SliceRepairMgr, error) { +func NewSliceRepairMgr(cfg *SliceRepairMgrConfig) (*SliceRepairMgr, error) { if cfg.messageType == 0 { cfg.messageType = snproto.MessageTypeRepair } repairMgr := &SliceRepairMgr{} cfg.executor = repairMgr - msgMgr, err := newMessageMgr(cfg) + + msgMgr, err := newMessageMgr(cfg.MessageMgrConfig) if err != nil { return nil, err } repairMgr.messageMgr = msgMgr repairMgr.Start() + return repairMgr, nil } @@ -49,6 +82,22 @@ func (m *SliceRepairMgr) Start() { m.messageMgr.run() } +func (r *SliceRepairMgr) Repair(ctx context.Context, req *snapi.RepairSliceArgs) error { + return r.insertRepairMsg(ctx, req) +} + +func (r *SliceRepairMgr) Stats() *snapi.ShardnodeTaskStatsRet { + repairSuccessCounter, repairFailedCounter := r.getTaskStats() + repairErrStats, repairTotalErrCnt := r.getErrorStats() + return &snapi.ShardnodeTaskStatsRet{ + Enable: r.enabled(), + SuccessPerMin: fmt.Sprint(repairSuccessCounter), + FailedPerMin: fmt.Sprint(repairFailedCounter), + TotalErrCnt: repairTotalErrCnt, + ErrStats: repairErrStats, + } +} + func (m *SliceRepairMgr) ItemToMessageExt(item interface{}) (snproto.MessageExt, error) { msgItem, ok := item.(messageItem) if !ok { @@ -66,10 +115,203 @@ func (m *SliceRepairMgr) ItemToMessageExt(item interface{}) (snproto.MessageExt, } // ExecuteWithCheckVolConsistency implements MessageExecutor interface for SliceRepairMgr -// TODO: implement repair logic func (m *SliceRepairMgr) ExecuteWithCheckVolConsistency(ctx context.Context, vid proto.Vid, ret interface{}) error { - // TODO: implement repair logic - return errors.New("repair logic not implemented yet") + return m.cfg.VolCache.DoubleCheckedRun(ctx, vid, func(info *snproto.VolumeInfoSimple) (newVol *snproto.VolumeInfoSimple, _err error) { + executeRet, ok := ret.(*executeRet) + if !ok { + return nil, errors.New("invalid execute ret") + } + + me, ok := executeRet.msgExt.(*repairMsgExt) + if !ok { + return nil, errors.New("not a repair message") + } + return m.tryRepair(executeRet.ctx, info, &me.msg) + }) +} + +func (m *SliceRepairMgr) tryRepair(ctx context.Context, volInfo *snproto.VolumeInfoSimple, repairMsg *snproto.SliceRepairMsg) (*snproto.VolumeInfoSimple, error) { + span := trace.SpanFromContextSafe(ctx) + + newVol, err := m.repairSlice(ctx, volInfo, repairMsg) + if err == nil { + return newVol, nil + } + + if err == ErrBlobnodeServiceUnavailable { + return volInfo, err + } + + if isErrDiskNotFound(err) { + m.processDiskNotFoundErr(ctx, volInfo, repairMsg) + } + + newVol, err1 := m.cfg.VolCache.UpdateVolume(volInfo.Vid) + if err1 != nil || newVol.EqualWith(volInfo) { + // if update volInfo failed or volInfo not updated, don't need retry + span.Warnf("new volInfo is same or clusterTopology.UpdateVolume failed: vid[%d], vol cache update err[%+v], repair err[%+v]", + volInfo.Vid, err1, err) + return volInfo, err + } + + return m.repairSlice(ctx, newVol, repairMsg) +} + +func (m *SliceRepairMgr) repairSlice(ctx context.Context, volInfo *snproto.VolumeInfoSimple, repairMsg *snproto.SliceRepairMsg) (*snproto.VolumeInfoSimple, error) { + span := trace.SpanFromContextSafe(ctx) + + span.Infof("repair slice: msg[%+v], vol info[%+v]", repairMsg, volInfo) + + hosts := m.blobNodeSelector.GetRandomN(1) + if len(hosts) == 0 { + return volInfo, ErrBlobnodeServiceUnavailable + } + workerHost := hosts[0] + + err := m.cfg.BlobTransport.RepairSlice(ctx, workerHost, volInfo, repairMsg) + if err == nil { + return volInfo, nil + } + + if isOrphanSlice(err) { + m.saveOrphanSlice(ctx, repairMsg) + } + + return volInfo, err +} + +func (m *SliceRepairMgr) saveOrphanSlice(ctx context.Context, repairMsg *snproto.SliceRepairMsg) { + span := trace.SpanFromContextSafe(ctx) + + slice := OrphanSlice{ + ClusterID: m.cfg.ClusterID, + Vid: repairMsg.Vid, + Bid: repairMsg.Bid, + } + span.Infof("save orphan slice: [%+v]", slice) + + insistOn(ctx, "save orphan slice", func() error { + return m.executeLogger.Encode(slice) + }) +} + +func (m *SliceRepairMgr) processDiskNotFoundErr(ctx context.Context, volInfo *snproto.VolumeInfoSimple, repairMsg *snproto.SliceRepairMsg) { + span := trace.SpanFromContextSafe(ctx) + for _, idx := range repairMsg.BadIdx { + vunitInfo := volInfo.VunitLocations[idx] + + if m.chunkMissMigrateReporter.IsVuidReported(vunitInfo.Vuid) { + span.Warnf("chunk is miss migrate and already reported, vunitInfo: %+v", vunitInfo) + continue + } + + // check disk status + disk, err := m.transport.GetBlobnodeDiskInfo(ctx, vunitInfo.DiskID) + if err != nil { + span.Errorf("get diskinfo failed, vunitInfo: %+v, err: %s", volInfo, err.Error()) + continue + } + // maybe disk is broken but not repaired, and restarted, retry repair next time + if disk.Status <= proto.DiskStatusRepairing { + continue + } + // disk is repaired or dropped, means volInfo is too old, get new volInfo from cm + vol, err := m.transport.GetVolumeInfo(ctx, vunitInfo.Vuid.Vid()) + if err != nil { + span.Errorf("get volumeinfo failed, vunitInfo: %+v, err: %s", volInfo, err.Error()) + continue + } + if !vol.EqualWith(volInfo) { + continue + } + + ret, err := m.scClient.CheckTaskExist(ctx, &scheduler.CheckTaskExistArgs{ + TaskType: proto.TaskTypeManualMigrate, + DiskID: vunitInfo.DiskID, + Vuid: vunitInfo.Vuid, + }) + if err != nil { + span.Errorf("check task exist failed, vunitInfo: %+v, err: %s", volInfo, err.Error()) + continue + } + if ret.Exist { + m.chunkMissMigrateReporter.SetVuidReported(vunitInfo.Vuid) + continue + } + m.chunkMissMigrateReporter.ReportAbnormal(vunitInfo.DiskID, vunitInfo.Vuid) + m.chunkMissMigrateReporter.SetVuidReported(vunitInfo.Vuid) + } +} + +func (m *SliceRepairMgr) insertRepairMsg(ctx context.Context, req *snapi.RepairSliceArgs) error { + shard, err := m.getShard(req.Header.DiskID, req.Header.Suid) + if err != nil { + return err + } + + span := trace.SpanFromContextSafe(ctx) + ts := m.tsGen.GenerateTs() + shardKeys := req.GetShardKeys(shard.ShardingSubRangeCount()) + tier := snproto.TierSingleIdx + if len(req.BadIdx) > 1 { + tier = snproto.TierMultiIdx + } + + mk := newMsgKey() + defer mk.release() + + mk.msgType = snproto.MessageTypeRepair + mk.tier = tier + mk.ts = ts + mk.vid = req.Vid + mk.bid = req.Bid + mk.shardKeys = shardKeys + key := mk.encode() + + msg := snproto.SliceRepairMsg{ + Bid: req.Bid, + Vid: req.Vid, + BadIdx: req.BadIdx, + Reason: req.Reason, + ReqId: span.TraceID(), + } + + raw, err := msg.Marshal() + if err != nil { + return err + } + itm := snapi.Item{ + ID: string(key), + Fields: []snapi.Field{ + { + ID: snproto.SliceRepairMsgFieldID, + Value: raw, + }, + }, + } + + oph := storage.OpHeader{ + RouteVersion: req.Header.RouteVersion, + ShardKeys: shardKeys, + } + return shard.InsertItem(ctx, oph, []byte(itm.ID), itm) +} + +func isOrphanSlice(err error) bool { + return rpc2.DetectStatusCode(err) == apierr.CodeOrphanShard +} + +func isErrDiskNotFound(err error) bool { + return rpc2.DetectStatusCode(err) == apierr.CodeDiskNotFound +} + +func insistOn(ctx context.Context, errMsg string, on func() error) { + span := trace.SpanFromContextSafe(ctx) + attempt := 0 + retry.Insist(time.Second, on, func(err error) { + attempt++ + span.Errorf("insist attempt-%d: %s %s", attempt, errMsg, err.Error()) + }) } func itemToRepairMsg(itm snapi.Item) (msg snproto.SliceRepairMsg, err error) { diff --git a/blobstore/shardnode/message/repair_test.go b/blobstore/shardnode/message/repair_test.go new file mode 100644 index 000000000..be6800198 --- /dev/null +++ b/blobstore/shardnode/message/repair_test.go @@ -0,0 +1,511 @@ +// Copyright 2025 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package message + +import ( + "context" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/api/scheduler" + snapi "github.com/cubefs/cubefs/blobstore/api/shardnode" + apierr "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/recordlog" + "github.com/cubefs/cubefs/blobstore/common/rpc2" + "github.com/cubefs/cubefs/blobstore/common/taskswitch" + "github.com/cubefs/cubefs/blobstore/shardnode/base" + snproto "github.com/cubefs/cubefs/blobstore/shardnode/proto" + "github.com/cubefs/cubefs/blobstore/shardnode/storage" + "github.com/cubefs/cubefs/blobstore/testing/mocks" + mock "github.com/cubefs/cubefs/blobstore/testing/mockshardnode" + "github.com/cubefs/cubefs/blobstore/util/errors" + "github.com/cubefs/cubefs/blobstore/util/selector" +) + +func TestNewSliceRepairMgr(t *testing.T) { + cmClient := mocks.NewMockClientAPI(ctr(t)) + cmClient.EXPECT().GetConfig(any, any).Return("", nil).AnyTimes() + taskSwitchMgr := taskswitch.NewSwitchMgr(cmClient) + + sh := mock.NewMockSpaceShardHandler(ctr(t)) + sg := mock.NewMockMessageMgrShardGetter(ctr(t)) + sg.EXPECT().GetAllShards().Return([]storage.ShardHandler{sh}).AnyTimes() + + testDir, err := os.MkdirTemp(os.TempDir(), "repair_log") + require.NoError(t, err) + defer os.RemoveAll(testDir) + + cfg := &SliceRepairMgrConfig{ + MessageMgrConfig: &MessageMgrConfig{ + TaskSwitchMgr: taskSwitchMgr, + ShardGetter: sg, + BlobTransport: mocks.NewMockBlobTransport(ctr(t)), + VolCache: mocks.NewMockIVolumeCache(ctr(t)), + MessageCfg: MessageCfg{ + FailedMsgChannelSize: 5, + ProduceTaskPoolSize: 4, + RateLimit: 100.0, + RateLimitBurst: 10, + MaxExecuteSliceNum: 1000, + TierConfig: newTestPriorityConfig(), + MessageLog: recordlog.Config{Dir: testDir}, + }, + }, + } + mgr, err := NewSliceRepairMgr(cfg) + require.Nil(t, err) + + // test stats + ret := mgr.Stats() + require.NotNil(t, ret) + mgr.Close() +} + +func TestSliceRepairMgr_ExecuteWithCheckVolConsistency(t *testing.T) { + // mock blob transport + blobTp := mocks.NewMockBlobTransport(ctr(t)) + blobTp.EXPECT().RepairSlice(any, any, any, any).DoAndReturn(func(ctx context.Context, host string, volInfo *snproto.VolumeInfoSimple, repairMsg *snproto.SliceRepairMsg) error { + return nil + }).AnyTimes() + + // mock blobnode selector + blobNodeSelector := mocks.NewMockSelector(ctr(t)) + blobNodeSelector.EXPECT().GetRandomN(1).Return([]string{"blobnode-1"}).AnyTimes() + + // mock volume info + vid := proto.Vid(1) + volInfo := newMockSimpleVolumeInfo(vid) + + // mock volume cache transport + volTp := mocks.NewMockVolumeTransport(ctr(t)) + volTp.EXPECT().GetVolumeInfo(any, any).Return(volInfo, nil).AnyTimes() + + vc := base.NewVolumeCache(volTp, 1) + mgr := newTestSliceRepairMgr(t, nil, blobTp, vc, nil, nil, blobNodeSelector) + + bid := proto.BlobID(1) + msg := &repairMsgExt{ + msg: snproto.SliceRepairMsg{ + Bid: bid, + Vid: vid, + BadIdx: []uint32{0}, + Reason: "test", + }, + } + + repairRet := &executeRet{ + msgExt: msg, + ctx: context.Background(), + } + + // repair + err := mgr.ExecuteWithCheckVolConsistency(context.Background(), vid, repairRet) + require.Nil(t, err) +} + +func TestSliceRepairMgr_ExecuteWithCheckVolConsistency_BlobnodeUnavailable(t *testing.T) { + // mock blobnode selector - return empty list + blobNodeSelector := mocks.NewMockSelector(ctr(t)) + blobNodeSelector.EXPECT().GetRandomN(1).Return([]string{}).AnyTimes() + + // mock volume info + vid := proto.Vid(1) + volInfo := newMockSimpleVolumeInfo(vid) + + // mock volume cache transport + volTp := mocks.NewMockVolumeTransport(ctr(t)) + volTp.EXPECT().GetVolumeInfo(any, any).Return(volInfo, nil).AnyTimes() + + vc := base.NewVolumeCache(volTp, 1) + mgr := newTestSliceRepairMgr(t, nil, nil, vc, nil, nil, blobNodeSelector) + + bid := proto.BlobID(1) + msg := &repairMsgExt{ + msg: snproto.SliceRepairMsg{ + Bid: bid, + Vid: vid, + BadIdx: []uint32{0}, + Reason: "test", + }, + } + + repairRet := &executeRet{ + msgExt: msg, + ctx: context.Background(), + } + + // repair should fail with ErrBlobnodeServiceUnavailable + err := mgr.ExecuteWithCheckVolConsistency(context.Background(), vid, repairRet) + require.NotNil(t, err) + require.Equal(t, ErrBlobnodeServiceUnavailable, err) +} + +func TestSliceRepairMgr_ExecuteWithCheckVolConsistency_OrphanShard(t *testing.T) { + // mock blob transport - return orphan slice error + blobTp := mocks.NewMockBlobTransport(ctr(t)) + blobTp.EXPECT().RepairSlice(any, any, any, any).DoAndReturn(func(ctx context.Context, host string, volInfo *snproto.VolumeInfoSimple, repairMsg *snproto.SliceRepairMsg) error { + return rpc2.NewError(apierr.CodeOrphanShard, "orphan slice", "") + }).AnyTimes() + + // mock blobnode selector + blobNodeSelector := mocks.NewMockSelector(ctr(t)) + blobNodeSelector.EXPECT().GetRandomN(1).Return([]string{"blobnode-1"}).AnyTimes() + + // mock volume info + vid := proto.Vid(1) + volInfo := newMockSimpleVolumeInfo(vid) + + // mock volume cache transport + volTp := mocks.NewMockVolumeTransport(ctr(t)) + volTp.EXPECT().GetVolumeInfo(any, any).Return(volInfo, nil).AnyTimes() + + // mock execute logger + executeLogger := mocks.NewMockRecordLogEncoder(ctr(t)) + executeLogger.EXPECT().Encode(any).Return(nil).AnyTimes() + + vc := base.NewVolumeCache(volTp, 1) + mgr := newTestSliceRepairMgr(t, nil, blobTp, vc, nil, nil, blobNodeSelector) + mgr.executeLogger = executeLogger + + bid := proto.BlobID(1) + msg := &repairMsgExt{ + msg: snproto.SliceRepairMsg{ + Bid: bid, + Vid: vid, + BadIdx: []uint32{0}, + Reason: "test", + }, + } + + repairRet := &executeRet{ + msgExt: msg, + ctx: context.Background(), + } + + mgr.cfg.ClusterID = 1 + err := mgr.ExecuteWithCheckVolConsistency(context.Background(), vid, repairRet) + require.NotNil(t, err) + require.Equal(t, apierr.CodeOrphanShard, rpc2.DetectStatusCode(err)) +} + +func TestSliceRepairMgr_ExecuteWithCheckVolConsistency_DiskNotFound(t *testing.T) { + // mock blob transport - return disk not found error + blobTp := mocks.NewMockBlobTransport(ctr(t)) + blobTp.EXPECT().RepairSlice(any, any, any, any).DoAndReturn(func(ctx context.Context, host string, volInfo *snproto.VolumeInfoSimple, repairMsg *snproto.SliceRepairMsg) error { + return rpc2.NewError(apierr.CodeDiskNotFound, "disk not found", "") + }).AnyTimes() + + // mock blobnode selector + blobNodeSelector := mocks.NewMockSelector(ctr(t)) + blobNodeSelector.EXPECT().GetRandomN(1).Return([]string{"blobnode-1"}).AnyTimes() + + // mock volume info + vid := proto.Vid(1) + volInfo := newMockSimpleVolumeInfo(vid) + + // mock volume cache transport + volTp := mocks.NewMockVolumeTransport(ctr(t)) + volTp.EXPECT().GetVolumeInfo(any, any).Return(volInfo, nil).AnyTimes() + + // mock transport for GetBlobnodeDiskInfo and GetVolumeInfo + transport := mocks.NewMockTransport(ctr(t)) + transport.EXPECT().GetBlobnodeDiskInfo(any, any).Return(&clustermgr.BlobNodeDiskInfo{ + DiskInfo: clustermgr.DiskInfo{ + Status: proto.DiskStatusNormal, + }, + DiskHeartBeatInfo: clustermgr.DiskHeartBeatInfo{ + DiskID: volInfo.VunitLocations[0].DiskID, + }, + }, nil).AnyTimes() + transport.EXPECT().GetVolumeInfo(any, any).Return(volInfo, nil).AnyTimes() + + // mock scheduler client + scClient := mocks.NewMockIScheduler(ctr(t)) + scClient.EXPECT().CheckTaskExist(any, any).Return(&scheduler.CheckTaskExistResp{Exist: false}, nil).AnyTimes() + + vc := base.NewVolumeCache(volTp, 1) + mgr := newTestSliceRepairMgr(t, nil, blobTp, vc, transport, scClient, blobNodeSelector) + + bid := proto.BlobID(1) + msg := &repairMsgExt{ + msg: snproto.SliceRepairMsg{ + Bid: bid, + Vid: vid, + BadIdx: []uint32{0}, + Reason: "test", + }, + } + + repairRet := &executeRet{ + msgExt: msg, + ctx: context.Background(), + } + + // repair should fail with disk not found error + err := mgr.ExecuteWithCheckVolConsistency(context.Background(), vid, repairRet) + require.NotNil(t, err) + require.Equal(t, apierr.CodeDiskNotFound, rpc2.DetectStatusCode(err)) +} + +func TestSliceRepairMgr_ExecuteWithCheckVolConsistency_UpdateVolume(t *testing.T) { + // mock volume info + vid := proto.Vid(1) + volInfo := newMockSimpleVolumeInfo(vid) + newVolInfo := newMockSimpleVolumeInfo(vid) + newVolInfo.VunitLocations[0].DiskID = volInfo.VunitLocations[0].DiskID + 100 + + // mock volume cache transport + firstTime := true + volTp := mocks.NewMockVolumeTransport(ctr(t)) + volTp.EXPECT().GetVolumeInfo(any, any).DoAndReturn(func(ctx context.Context, vid proto.Vid) (*snproto.VolumeInfoSimple, error) { + if firstTime { + firstTime = false + return volInfo, nil + } + return newVolInfo, nil + }).Times(2) + + vc := base.NewVolumeCache(volTp, 1) + + // mock blob transport - first time fail, second time success + firstTimeRepair := true + blobTp := mocks.NewMockBlobTransport(ctr(t)) + blobTp.EXPECT().RepairSlice(any, any, any, any).DoAndReturn(func(ctx context.Context, host string, volInfo *snproto.VolumeInfoSimple, repairMsg *snproto.SliceRepairMsg) error { + if firstTimeRepair { + firstTimeRepair = false + return rpc2.NewError(int32(apierr.CodeDiskBroken), "disk broken", "") + } + return nil + }).Times(2) + + // mock blobnode selector + blobNodeSelector := mocks.NewMockSelector(ctr(t)) + blobNodeSelector.EXPECT().GetRandomN(1).Return([]string{"blobnode-1"}).AnyTimes() + + mgr := newMinimalSliceRepairMgr(&SliceRepairMgrConfig{ + MessageMgrConfig: &MessageMgrConfig{ + BlobTransport: blobTp, + VolCache: vc, + }, + BlobNodeSelector: blobNodeSelector, + }) + + bid := proto.BlobID(1) + msg := &repairMsgExt{ + msg: snproto.SliceRepairMsg{ + Bid: bid, + Vid: vid, + BadIdx: []uint32{0}, + Reason: "test", + }, + } + + repairRet := &executeRet{ + msgExt: msg, + ctx: context.Background(), + } + + // repair should succeed after volume update + err := mgr.ExecuteWithCheckVolConsistency(context.Background(), vid, repairRet) + require.Nil(t, err) +} + +func TestSliceRepairMgr_ExecuteWithCheckVolConsistency_UpdateVolume2(t *testing.T) { + // mock volume info + vid := proto.Vid(1) + volInfo := newMockSimpleVolumeInfo(vid) + newVolInfo := newMockSimpleVolumeInfo(vid) + newVolInfo.VunitLocations[0].DiskID = volInfo.VunitLocations[0].DiskID + 100 + + // mock volume cache transport + firstTime := true + volTp := mocks.NewMockVolumeTransport(ctr(t)) + volTp.EXPECT().GetVolumeInfo(any, any).DoAndReturn(func(ctx context.Context, vid proto.Vid) (*snproto.VolumeInfoSimple, error) { + if firstTime { + firstTime = false + return volInfo, nil + } + return newVolInfo, nil + }).Times(2) + + vc := base.NewVolumeCache(volTp, 1) + + // mock blob transport - always fail + blobTp := mocks.NewMockBlobTransport(ctr(t)) + blobTp.EXPECT().RepairSlice(any, any, any, any).DoAndReturn(func(ctx context.Context, host string, volInfo *snproto.VolumeInfoSimple, repairMsg *snproto.SliceRepairMsg) error { + return rpc2.NewError(int32(apierr.CodeDiskBroken), "disk broken", "") + }).AnyTimes() + + // mock blobnode selector + blobNodeSelector := mocks.NewMockSelector(ctr(t)) + blobNodeSelector.EXPECT().GetRandomN(1).Return([]string{"blobnode-1"}).AnyTimes() + + mgr := newMinimalSliceRepairMgr(&SliceRepairMgrConfig{ + MessageMgrConfig: &MessageMgrConfig{ + BlobTransport: blobTp, + VolCache: vc, + }, + BlobNodeSelector: blobNodeSelector, + }) + + bid := proto.BlobID(1) + msg := &repairMsgExt{ + msg: snproto.SliceRepairMsg{ + Bid: bid, + Vid: vid, + BadIdx: []uint32{0}, + Reason: "test", + }, + } + + repairRet := &executeRet{ + msgExt: msg, + ctx: context.Background(), + } + + // repair should fail even after volume update (volume not changed) + err := mgr.ExecuteWithCheckVolConsistency(context.Background(), vid, repairRet) + require.NotNil(t, err) +} + +func TestSliceRepairMgr_RepairShard(t *testing.T) { + // mock blob transport + blobTp := mocks.NewMockBlobTransport(ctr(t)) + blobTp.EXPECT().RepairSlice(any, any, any, any).DoAndReturn(func(ctx context.Context, host string, volInfo *snproto.VolumeInfoSimple, repairMsg *snproto.SliceRepairMsg) error { + return nil + }).AnyTimes() + + // mock blobnode selector + blobNodeSelector := mocks.NewMockSelector(ctr(t)) + blobNodeSelector.EXPECT().GetRandomN(1).Return([]string{"blobnode-1"}).AnyTimes() + + // mock volume info + vid := proto.Vid(1) + volInfo := newMockSimpleVolumeInfo(vid) + + mgr := newMinimalSliceRepairMgr(&SliceRepairMgrConfig{ + MessageMgrConfig: &MessageMgrConfig{ + BlobTransport: blobTp, + }, + BlobNodeSelector: blobNodeSelector, + }) + + repairMsg := &snproto.SliceRepairMsg{ + Bid: proto.BlobID(1), + Vid: vid, + BadIdx: []uint32{0}, + Reason: "test", + } + + // repair should succeed + newVol, err := mgr.repairSlice(context.Background(), volInfo, repairMsg) + require.Nil(t, err) + require.Equal(t, volInfo, newVol) +} + +func TestSliceRepairMgr_RepairShard_Error(t *testing.T) { + // mock blob transport - return error + blobTp := mocks.NewMockBlobTransport(ctr(t)) + blobTp.EXPECT().RepairSlice(any, any, any, any).DoAndReturn(func(ctx context.Context, host string, volInfo *snproto.VolumeInfoSimple, repairMsg *snproto.SliceRepairMsg) error { + return errors.New("repair failed") + }).AnyTimes() + + // mock blobnode selector + blobNodeSelector := mocks.NewMockSelector(ctr(t)) + blobNodeSelector.EXPECT().GetRandomN(1).Return([]string{"blobnode-1"}).AnyTimes() + + // mock volume info + vid := proto.Vid(1) + volInfo := newMockSimpleVolumeInfo(vid) + + mgr := newMinimalSliceRepairMgr(&SliceRepairMgrConfig{ + MessageMgrConfig: &MessageMgrConfig{ + BlobTransport: blobTp, + }, + BlobNodeSelector: blobNodeSelector, + }) + + repairMsg := &snproto.SliceRepairMsg{ + Bid: proto.BlobID(1), + Vid: vid, + BadIdx: []uint32{0}, + Reason: "test", + } + + // repair should fail + newVol, err := mgr.repairSlice(context.Background(), volInfo, repairMsg) + require.NotNil(t, err) + require.Equal(t, volInfo, newVol) +} + +func TestSliceRepairMgr_InsertRepairMsg(t *testing.T) { + // mock shard + shard := mock.NewMockSpaceShardHandler(ctr(t)) + shard.EXPECT().InsertItem(any, any, any, any).Return(nil).AnyTimes() + shard.EXPECT().ShardingSubRangeCount().Return(2).AnyTimes() + + // mock shard getter + sg := mock.NewMockMessageMgrShardGetter(ctr(t)) + sg.EXPECT().GetShard(any, any).Return(shard, nil).AnyTimes() + + mgr := newTestSliceRepairMgr(t, sg, nil, nil, nil, nil, nil) + + args := &snapi.RepairSliceArgs{ + Header: snapi.ShardOpHeader{ + DiskID: 1, + Suid: 1, + }, + Bid: proto.BlobID(1), + Vid: proto.Vid(1), + BadIdx: []uint32{0, 1}, + Reason: "test", + } + err := mgr.Repair(context.Background(), args) + require.Nil(t, err) +} + +// newMinimalSliceRepairMgr creates a minimal SliceRepairMgr for testing +func newMinimalSliceRepairMgr(cfg *SliceRepairMgrConfig) *SliceRepairMgr { + baseMgr := &messageMgr{ + cfg: cfg.MessageMgrConfig, + } + mgr := &SliceRepairMgr{ + messageMgr: baseMgr, + transport: cfg.Transport, + blobNodeSelector: cfg.BlobNodeSelector, + scClient: cfg.SCClient, + } + baseMgr.cfg.executor = mgr + return mgr +} + +func newTestSliceRepairMgr(t *testing.T, sg ShardGetter, tp base.BlobTransport, vc base.IVolumeCache, transport base.Transport, scClient scheduler.IScheduler, selector selector.Selector) *SliceRepairMgr { + baseMgr := newTestBlobMessageMgr(t, sg, tp, vc, nil, snproto.MessageTypeRepair) + + mgr := &SliceRepairMgr{ + messageMgr: baseMgr, + transport: transport, + scClient: scClient, + blobNodeSelector: selector, + chunkMissMigrateReporter: base.NewAbnormalReporter(0, base.ShardRepair, base.ChunkMissMigrateAbnormal), + } + // Set executor to ShardRepairMgr itself + baseMgr.cfg.executor = mgr + return mgr +} diff --git a/blobstore/shardnode/message/shard_reader_test.go b/blobstore/shardnode/message/shard_reader_test.go index 0d98236e3..9d1d61843 100644 --- a/blobstore/shardnode/message/shard_reader_test.go +++ b/blobstore/shardnode/message/shard_reader_test.go @@ -289,24 +289,23 @@ func TestShardListReader_IsProtected(t *testing.T) { require.True(t, reader.isProtected(time.Hour)) } -func TestDelMsgExt_IsProtected(t *testing.T) { +func TestMessageExt_IsProtected(t *testing.T) { // test msg unprotected oldTime := time.Now().Add(-2 * time.Hour).Unix() - unprotectedMsg := &delMsgExt{ + msg := &delMsgExt{ msg: snproto.DeleteMsg{ Time: oldTime, }, } - require.False(t, unprotectedMsg.IsProtected(time.Hour)) + require.False(t, msg.IsProtected(time.Hour)) - // test msg protected - now := time.Now().Unix() - protectedMsg := &delMsgExt{ - msg: snproto.DeleteMsg{ - Time: now, + msg2 := &repairMsgExt{ + msg: snproto.SliceRepairMsg{ + Time: time.Now().Unix(), }, } - require.True(t, protectedMsg.IsProtected(time.Hour)) + + require.True(t, msg2.IsProtected(time.Hour)) } func TestItemToDelMsg(t *testing.T) { @@ -343,8 +342,56 @@ func TestItemToDelMsg(t *testing.T) { require.Error(t, err) } +func TestItemToRepairMsg(t *testing.T) { + // test valid item + msg := snproto.SliceRepairMsg{ + Bid: proto.BlobID(100), + Vid: proto.Vid(1), + BadIdx: []uint32{0, 1}, + Reason: "test", + ReqId: "test_req_id", + } + item := snapi.Item{ + ID: "test_key", + Fields: []snapi.Field{ + { + ID: snproto.SliceRepairMsgFieldID, + Value: marshalRepairMsg(msg), + }, + }, + } + + result, err := itemToRepairMsg(item) + require.NoError(t, err) + require.Equal(t, msg.Bid, result.Bid) + require.Equal(t, msg.Vid, result.Vid) + require.Equal(t, msg.BadIdx, result.BadIdx) + require.Equal(t, msg.Reason, result.Reason) + require.Equal(t, msg.ReqId, result.ReqId) + + // test invalid item + invalidItem := snapi.Item{ + ID: "invalid_key", + Fields: []snapi.Field{ + { + ID: snproto.SliceRepairMsgFieldID, + Value: []byte("invalid_data"), + }, + }, + } + + _, err = itemToRepairMsg(invalidItem) + require.Error(t, err) +} + // marshal DeleteMsg to bytes func marshalDeleteMsg(msg snproto.DeleteMsg) []byte { data, _ := msg.Marshal() return data } + +// marshal RepairMsg to bytes +func marshalRepairMsg(msg snproto.SliceRepairMsg) []byte { + data, _ := msg.Marshal() + return data +} diff --git a/blobstore/shardnode/rpcservice.go b/blobstore/shardnode/rpcservice.go index 662130cc1..2f94ab664 100644 --- a/blobstore/shardnode/rpcservice.go +++ b/blobstore/shardnode/rpcservice.go @@ -418,6 +418,22 @@ func (s *RpcService) DBStats(w rpc2.ResponseWriter, req *rpc2.Request) error { return w.WriteOK(&ret) } +func (s *RpcService) RepairSlice(w rpc2.ResponseWriter, req *rpc2.Request) error { + ctx := req.Context() + span := req.Span() + args := &shardnode.RepairSliceArgs{} + if err := req.ParseParameter(args); err != nil { + return err + } + span.Infof("receive RepairSlice request, args:%+v", args) + return s.repairSlice(ctx, args) +} + +func (s *RpcService) RepairSliceStats(w rpc2.ResponseWriter, req *rpc2.Request) error { + ret := s.repairSliceStats() + return w.WriteOK(ret) +} + func initConfig(args []string) (*cmd.Config, error) { config.Init("f", "", "shardnode.conf") if err := config.Load(&conf); err != nil { @@ -460,6 +476,9 @@ func newHandler(s *RpcService) *rpc2.Router { handler.Register("/blob/delete/raw", s.DeleteBlobRaw) handler.Register("/blob/delete/stats", s.DeleteBlobStats) + handler.Register("/slice/repair", s.RepairSlice) + handler.Register("/slice/repair/stats", s.RepairSliceStats) + return handler } diff --git a/blobstore/shardnode/rpcservice_test.go b/blobstore/shardnode/rpcservice_test.go index ea80361ee..5ff8d8fc0 100644 --- a/blobstore/shardnode/rpcservice_test.go +++ b/blobstore/shardnode/rpcservice_test.go @@ -145,6 +145,10 @@ func newMockService(t *testing.T, cfg mockServiceCfg) (*service, func(), error) require.NoError(t, err) defer os.RemoveAll(delLogDir) + repairLogDir, err := os.MkdirTemp(os.TempDir(), "repair_log") + require.NoError(t, err) + defer os.RemoveAll(repairLogDir) + dm, _ := message.NewBlobDeleteMgr(&message.BlobDelMgrConfig{ TaskSwitchMgr: taskSwitchMgr, ShardGetter: sg2, @@ -156,6 +160,15 @@ func newMockService(t *testing.T, cfg mockServiceCfg) (*service, func(), error) }) s.blobDelMgr = dm + shardRepairMgr, _ := message.NewSliceRepairMgr(&message.SliceRepairMgrConfig{ + MessageMgrConfig: &message.MessageMgrConfig{ + TaskSwitchMgr: taskSwitchMgr, + ShardGetter: sg2, + MessageCfg: message.MessageCfg{MessageLog: recordlog.Config{Dir: repairLogDir}}, + }, + }) + s.sliceRepairMgr = shardRepairMgr + // set disk s.disks = make(map[proto.DiskID]*storage.Disk) for id, d := range cfg.disks { @@ -308,7 +321,25 @@ func TestRpcService_Blob(t *testing.T) { require.Nil(t, err) // delete blob stats - stats, err := cli.DeleteBlobStats(context.Background(), tcpAddrBlob, shardnode.DeleteBlobStatsArgs{}) + stats, err := cli.DeleteBlobStats(context.Background(), tcpAddrBlob, shardnode.ShardnodeTaskStatsArgs{}) + require.Nil(t, err) + require.NotNil(t, stats) + + // repair shard + err = cli.RepairSlice(context.Background(), tcpAddrBlob, shardnode.RepairSliceArgs{ + Header: shardnode.ShardOpHeader{ + DiskID: 1, + Suid: 1, + }, + Bid: proto.BlobID(1), + Vid: proto.Vid(1), + BadIdx: []uint32{0, 1}, + Reason: "test", + }) + require.Nil(t, err) + + // repair shard stats + stats, err = cli.RepairSliceStats(context.Background(), tcpAddrBlob, shardnode.ShardnodeTaskStatsArgs{}) require.Nil(t, err) require.NotNil(t, stats) } diff --git a/blobstore/shardnode/startup.go b/blobstore/shardnode/startup.go index 82294fc7f..805eecc3c 100644 --- a/blobstore/shardnode/startup.go +++ b/blobstore/shardnode/startup.go @@ -402,6 +402,13 @@ func initServiceConfig(cfg *Config) { defaulter.LessOrEqual(&cfg.DeleteBlobCfg.MaxExecuteSliceNum, uint64(64)) defaulter.LessOrEqual(&cfg.DeleteBlobCfg.TierConfig.SafeMessageTimeout.Duration, 12*time.Hour) defaulter.LessOrEqual(&cfg.DeleteBlobCfg.TierConfig.PunishTimeout.Duration, 1*time.Minute) + defaulter.LessOrEqual(&cfg.SliceRepairCfg.RateLimit, float64(1024)) + defaulter.LessOrEqual(&cfg.SliceRepairCfg.RateLimitBurst, 64) + defaulter.LessOrEqual(&cfg.SliceRepairCfg.FailedMsgChannelSize, 10<<10) + defaulter.LessOrEqual(&cfg.SliceRepairCfg.ProduceTaskPoolSize, 16) + defaulter.LessOrEqual(&cfg.SliceRepairCfg.MaxExecuteSliceNum, uint64(64)) + defaulter.LessOrEqual(&cfg.SliceRepairCfg.TierConfig.SafeMessageTimeout.Duration, 12*time.Hour) + defaulter.LessOrEqual(&cfg.SliceRepairCfg.TierConfig.PunishTimeout.Duration, 1*time.Minute) } func isDiskInfoMatch(a, b clustermgr.ShardNodeDiskInfo) bool { diff --git a/blobstore/shardnode/svr.go b/blobstore/shardnode/svr.go index bc4ce401e..2f3635203 100644 --- a/blobstore/shardnode/svr.go +++ b/blobstore/shardnode/svr.go @@ -24,6 +24,7 @@ import ( bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode" cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/api/scheduler" shardnodeapi "github.com/cubefs/cubefs/blobstore/api/shardnode" "github.com/cubefs/cubefs/blobstore/cmd" apierr "github.com/cubefs/cubefs/blobstore/common/errors" @@ -40,6 +41,7 @@ import ( "github.com/cubefs/cubefs/blobstore/shardnode/storage" "github.com/cubefs/cubefs/blobstore/shardnode/storage/store" "github.com/cubefs/cubefs/blobstore/util/closer" + "github.com/cubefs/cubefs/blobstore/util/selector" "github.com/cubefs/cubefs/blobstore/util/taskpool" ) @@ -79,7 +81,9 @@ type Config struct { WaitReOpenDiskIntervalS int64 `json:"wait_re_open_disk_interval_s"` ShardCheckAndClearIntervalH int64 `json:"shard_check_and_clear_interval_h"` - DeleteBlobCfg message.MessageCfg `json:"blob_delete_cfg"` + DeleteBlobCfg message.MessageCfg `json:"blob_delete_cfg"` + ScClientConfig scheduler.Config `json:"sc_client_config"` + SliceRepairCfg message.MessageCfg `json:"slice_repair_cfg"` } // newService returns the singleton service instance @@ -155,19 +159,39 @@ func createService(cfg *Config) *service { }) svr.catalog = c - cfg.DeleteBlobCfg.ClusterID = cfg.NodeConfig.ClusterID taskSwitchMgr := taskswitch.NewSwitchMgr(cmClient) - dm, err := message.NewBlobDeleteMgr(&message.BlobDelMgrConfig{ + cfg.DeleteBlobCfg.ClusterID = cfg.NodeConfig.ClusterID + blboDeleteMgr, err := message.NewBlobDeleteMgr(&message.BlobDelMgrConfig{ TaskSwitchMgr: taskSwitchMgr, ShardGetter: svr, - Transport: transport, + BlobTransport: transport, VolCache: base.NewVolumeCache(transport, 10*time.Second), MessageCfg: cfg.DeleteBlobCfg, }) if err != nil { span.Fatalf("new blob delete mgr failed, err: %s", err.Error()) } - svr.blobDelMgr = dm + svr.blobDelMgr = blboDeleteMgr + + cfg.SliceRepairCfg.ClusterID = cfg.NodeConfig.ClusterID + sliceRepairMgr, err := message.NewSliceRepairMgr(&message.SliceRepairMgrConfig{ + MessageMgrConfig: &message.MessageMgrConfig{ + TaskSwitchMgr: taskSwitchMgr, + ShardGetter: svr, + BlobTransport: transport, + VolCache: base.NewVolumeCache(transport, 10*time.Second), + MessageCfg: cfg.SliceRepairCfg, + }, + Transport: transport, + BlobNodeSelector: selector.MakeSelector(60*1000, func() (hosts []string, err error) { + return transport.GetService(context.Background(), proto.ServiceNameWorker) + }), + SCClient: scheduler.New(&cfg.ScClientConfig, cmClient, cfg.NodeConfig.ClusterID), + }) + if err != nil { + span.Fatalf("new slice repair mgr failed, err: %s", err.Error()) + } + svr.sliceRepairMgr = sliceRepairMgr go svr.loop(ctx) span.Infof("service started success") @@ -182,7 +206,8 @@ type service struct { taskPool taskpool.TaskPool groupRun singleflight.Group - blobDelMgr *message.BlobDeleteMgr + blobDelMgr *message.BlobDeleteMgr + sliceRepairMgr *message.SliceRepairMgr cfg Config lock sync.RWMutex @@ -219,5 +244,6 @@ func (s *service) getAllDisks() []*storage.Disk { func (s *service) close() { s.closer.Close() s.blobDelMgr.Close() + s.sliceRepairMgr.Close() s.cfg.RaftConfig.Transport.Close() } diff --git a/blobstore/shardnode/svr_test.go b/blobstore/shardnode/svr_test.go index e44cfa270..4586377cb 100644 --- a/blobstore/shardnode/svr_test.go +++ b/blobstore/shardnode/svr_test.go @@ -82,6 +82,11 @@ func TestSvr_Loop(t *testing.T) { defer os.RemoveAll(delLogDir) cfg.DeleteBlobCfg.MessageLog.Dir = delLogDir + repairLogDir, err := os.MkdirTemp(os.TempDir(), "repair_log") + require.NoError(t, err) + defer os.RemoveAll(repairLogDir) + cfg.SliceRepairCfg.MessageLog.Dir = repairLogDir + s := newService(cfg) time.Sleep(3 * time.Second) s.closer.Close() @@ -99,6 +104,11 @@ func TestSvr_HandleEIO(t *testing.T) { defer os.RemoveAll(delLogDir) cfg.DeleteBlobCfg.MessageLog.Dir = delLogDir + repairLogDir, err := os.MkdirTemp(os.TempDir(), "repair_log") + require.NoError(t, err) + defer os.RemoveAll(repairLogDir) + cfg.SliceRepairCfg.MessageLog.Dir = repairLogDir + s := newService(cfg) disk := &storage.Disk{} disk.SetDiskInfo(cmapi.ShardNodeDiskInfo{ @@ -128,6 +138,11 @@ func TestSingletonPattern(t *testing.T) { defer os.RemoveAll(delLogDir) cfg.DeleteBlobCfg.MessageLog.Dir = delLogDir + repairLogDir, err := os.MkdirTemp(os.TempDir(), "repair_log") + require.NoError(t, err) + defer os.RemoveAll(repairLogDir) + cfg.SliceRepairCfg.MessageLog.Dir = repairLogDir + // First call to newService service1 := newService(cfg) require.NotNil(t, service1) diff --git a/blobstore/testing/mocks/shardnode_transport.go b/blobstore/testing/mocks/shardnode_transport.go index eb5d2499b..a5961b891 100644 --- a/blobstore/testing/mocks/shardnode_transport.go +++ b/blobstore/testing/mocks/shardnode_transport.go @@ -113,6 +113,21 @@ func (mr *MockTransportMockRecorder) GetAllSpaces(ctx interface{}) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllSpaces", reflect.TypeOf((*MockTransport)(nil).GetAllSpaces), ctx) } +// GetBlobnodeDiskInfo mocks base method. +func (m *MockTransport) GetBlobnodeDiskInfo(ctx context.Context, diskID proto.DiskID) (*clustermgr.BlobNodeDiskInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetBlobnodeDiskInfo", ctx, diskID) + ret0, _ := ret[0].(*clustermgr.BlobNodeDiskInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetBlobnodeDiskInfo indicates an expected call of GetBlobnodeDiskInfo. +func (mr *MockTransportMockRecorder) GetBlobnodeDiskInfo(ctx, diskID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBlobnodeDiskInfo", reflect.TypeOf((*MockTransport)(nil).GetBlobnodeDiskInfo), ctx, diskID) +} + // GetConfig mocks base method. func (m *MockTransport) GetConfig(ctx context.Context, key string) (string, error) { m.ctrl.T.Helper() @@ -188,6 +203,21 @@ func (mr *MockTransportMockRecorder) GetRouteUpdate(ctx, routeVersion interface{ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRouteUpdate", reflect.TypeOf((*MockTransport)(nil).GetRouteUpdate), ctx, routeVersion) } +// GetService mocks base method. +func (m *MockTransport) GetService(ctx context.Context, name string) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetService", ctx, name) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetService indicates an expected call of GetService. +func (mr *MockTransportMockRecorder) GetService(ctx, name interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetService", reflect.TypeOf((*MockTransport)(nil).GetService), ctx, name) +} + // GetSpace mocks base method. func (m *MockTransport) GetSpace(ctx context.Context, sid proto.SpaceID) (*clustermgr.Space, error) { m.ctrl.T.Helper() @@ -334,6 +364,20 @@ func (mr *MockTransportMockRecorder) RegisterDisk(ctx, disk interface{}) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterDisk", reflect.TypeOf((*MockTransport)(nil).RegisterDisk), ctx, disk) } +// RepairSlice mocks base method. +func (m *MockTransport) RepairSlice(ctx context.Context, host string, volInfo *proto0.VolumeInfoSimple, repairMsg *proto0.SliceRepairMsg) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RepairSlice", ctx, host, volInfo, repairMsg) + ret0, _ := ret[0].(error) + return ret0 +} + +// RepairSlice indicates an expected call of RepairSlice. +func (mr *MockTransportMockRecorder) RepairSlice(ctx, host, volInfo, repairMsg interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RepairSlice", reflect.TypeOf((*MockTransport)(nil).RepairSlice), ctx, host, volInfo, repairMsg) +} + // ResolveNodeAddr mocks base method. func (m *MockTransport) ResolveNodeAddr(ctx context.Context, diskID proto.DiskID) (string, error) { m.ctrl.T.Helper() @@ -873,6 +917,20 @@ func (mr *MockBlobTransportMockRecorder) MarkDeleteSliceUnit(ctx, info, bid inte return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkDeleteSliceUnit", reflect.TypeOf((*MockBlobTransport)(nil).MarkDeleteSliceUnit), ctx, info, bid) } +// RepairSlice mocks base method. +func (m *MockBlobTransport) RepairSlice(ctx context.Context, host string, volInfo *proto0.VolumeInfoSimple, repairMsg *proto0.SliceRepairMsg) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RepairSlice", ctx, host, volInfo, repairMsg) + ret0, _ := ret[0].(error) + return ret0 +} + +// RepairSlice indicates an expected call of RepairSlice. +func (mr *MockBlobTransportMockRecorder) RepairSlice(ctx, host, volInfo, repairMsg interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RepairSlice", reflect.TypeOf((*MockBlobTransport)(nil).RepairSlice), ctx, host, volInfo, repairMsg) +} + // MockVolumeTransport is a mock of VolumeTransport interface. type MockVolumeTransport struct { ctrl *gomock.Controller