mirror of
https://github.com/cubefs/cubefs.git
synced 2026-08-02 02:00:56 +00:00
feat(flashnode): support read block.#1000151055
Signed-off-by: baihailong <baihailong@oppo.com>
This commit is contained in:
baihailong
chihe
parent
c094de085f
commit
c7aac8a558
@ -330,6 +330,7 @@ func newCmdFlashGroupList(client *master.MasterClient) *cobra.Command {
|
||||
|
||||
stdoutln("[Flash Groups]")
|
||||
slots := make([]*slotInfo, 0)
|
||||
reservedSlots := make([]*slotInfo, 0)
|
||||
tbl := table{formatFlashGroupViewTile}
|
||||
for _, group := range fgView.FlashGroups {
|
||||
sort.Slice(group.Slots, func(i, j int) bool {
|
||||
@ -341,7 +342,16 @@ func newCmdFlashGroupList(client *master.MasterClient) *cobra.Command {
|
||||
slot: slot,
|
||||
})
|
||||
}
|
||||
tbl = tbl.append(arow(group.ID, group.Weight, len(group.Slots), group.Status, group.SlotStatus, len(group.PendingSlots), group.Step, group.FlashNodeCount))
|
||||
sort.Slice(group.ReservedSlots, func(i, j int) bool {
|
||||
return group.ReservedSlots[i] < group.ReservedSlots[j]
|
||||
})
|
||||
for _, slot := range group.ReservedSlots {
|
||||
reservedSlots = append(reservedSlots, &slotInfo{
|
||||
fgID: group.ID,
|
||||
slot: slot,
|
||||
})
|
||||
}
|
||||
tbl = tbl.append(arow(group.ID, group.Weight, len(group.Slots), len(group.ReservedSlots), group.Status, group.SlotStatus, len(group.PendingSlots), group.Step, group.FlashNodeCount, group.IsReducingSlots))
|
||||
}
|
||||
stdoutln(alignTable(tbl...))
|
||||
|
||||
@ -357,6 +367,15 @@ func newCmdFlashGroupList(client *master.MasterClient) *cobra.Command {
|
||||
}
|
||||
stdoutlnf("num:%d slot:%d fg:%d percent:%0.5f%%", i+1, info.slot, info.fgID, info.percent)
|
||||
}
|
||||
stdoutln("ReservedSlots:")
|
||||
for i, info := range reservedSlots {
|
||||
if i < len(reservedSlots)-1 {
|
||||
info.percent = float64(reservedSlots[i+1].slot-info.slot) * 100 / math.MaxUint32
|
||||
} else {
|
||||
info.percent = float64(math.MaxUint32-info.slot) * 100 / math.MaxUint32
|
||||
}
|
||||
stdoutlnf("num:%d slot:%d fg:%d percent:%0.5f%%", i+1, info.slot, info.fgID, info.percent)
|
||||
}
|
||||
return
|
||||
},
|
||||
}
|
||||
@ -462,6 +481,7 @@ func newCmdFlashGroupGraph(client *master.MasterClient) *cobra.Command {
|
||||
groupn := make(map[uint64]int)
|
||||
groupStatusMap := make(map[uint64]string)
|
||||
slots := make([]slotInfo, 0)
|
||||
reservedSlots := make([]slotInfo, 0)
|
||||
for _, fg := range fgView.FlashGroups {
|
||||
groups[fg.ID] = fg
|
||||
groupn[fg.ID] = 0
|
||||
@ -476,11 +496,22 @@ func newCmdFlashGroupGraph(client *master.MasterClient) *cobra.Command {
|
||||
slot: slot,
|
||||
})
|
||||
}
|
||||
for _, slot := range fg.ReservedSlots {
|
||||
if _, in := set[slot]; in {
|
||||
continue
|
||||
}
|
||||
groupn[fg.ID]++
|
||||
reservedSlots = append(reservedSlots, slotInfo{
|
||||
fgID: fg.ID,
|
||||
slot: slot,
|
||||
})
|
||||
}
|
||||
}
|
||||
sort.Slice(slots, func(i, j int) bool {
|
||||
return slots[i].slot < slots[j].slot
|
||||
})
|
||||
stdoutln("[Flash Groups]")
|
||||
stdoutln("[Slots]")
|
||||
tbl := table{arow("Slot", "ID", "Status", "Count", "Ref", "Proportion")}
|
||||
for idx, slot := range slots {
|
||||
g := groups[slot.fgID]
|
||||
@ -494,6 +525,24 @@ func newCmdFlashGroupGraph(client *master.MasterClient) *cobra.Command {
|
||||
}
|
||||
stdoutln(alignTable(tbl...))
|
||||
|
||||
sort.Slice(reservedSlots, func(i, j int) bool {
|
||||
return reservedSlots[i].slot < reservedSlots[j].slot
|
||||
})
|
||||
stdoutln("[ReservedSlots]")
|
||||
tbl1 := table{arow("Slot", "ID", "Status", "Count", "Ref", "Proportion")}
|
||||
for idx, slot := range reservedSlots {
|
||||
g := groups[slot.fgID]
|
||||
var p string
|
||||
if idx == len(reservedSlots)-1 {
|
||||
p = proportion(slot.slot, math.MaxUint32)
|
||||
} else {
|
||||
p = proportion(slot.slot, reservedSlots[idx+1].slot)
|
||||
}
|
||||
tbl1 = tbl1.append(arow(slot.slot, g.ID, g.Status.String(), g.FlashNodeCount, groupn[g.ID], p))
|
||||
}
|
||||
|
||||
stdoutln(alignTable(tbl1...))
|
||||
|
||||
fnView, err := client.NodeAPI().ListFlashNodes(-1)
|
||||
if err != nil {
|
||||
return
|
||||
|
||||
@ -1481,7 +1481,7 @@ var (
|
||||
"STORAGE CLASS", "INODE COUNT", "USED SIZE", "QUOTA")
|
||||
formatFlashNodeSimpleViewTableTitle = arow("Zone", "ID", "Address", "Active", "Enable", "FlashGroupID", "ReportTime")
|
||||
formatFlashNodeViewTableTitle = append(formatFlashNodeSimpleViewTableTitle[:], "DataPath", "HitRate", "Evicts", "Limit", "MaxAlloc", "HasAlloc", "Num", "Status")
|
||||
formatFlashGroupViewTile = arow("ID", "Weight", "Slots", "Status", "SlotStatus", "PendingSlots", "Step", "FlashNodeCount")
|
||||
formatFlashGroupViewTile = arow("ID", "Weight", "Slots", "ReservedSlots", "Status", "SlotStatus", "PendingSlots", "Step", "FlashNodeCount", "ReducingSlots")
|
||||
QosHeader = fmt.Sprintf(qosPattern, "NAME", "TOTAL-MB", "USED-MB")
|
||||
)
|
||||
|
||||
@ -1508,6 +1508,8 @@ func formatFlashGroupView(fg *proto.FlashGroupAdminView) string {
|
||||
fmt.Sprintf(" ID:%v\n", fg.ID) +
|
||||
fmt.Sprintf(" Weight:%v\n", fg.Weight) +
|
||||
fmt.Sprintf(" Slots:%v\n", fg.Slots) +
|
||||
fmt.Sprintf(" ReservedSlots:%v\n", fg.ReservedSlots) +
|
||||
fmt.Sprintf(" IsReducingSlots:%v\n", fg.IsReducingSlots) +
|
||||
fmt.Sprintf(" Status:%v\n", fg.Status) +
|
||||
fmt.Sprintf(" SlotStatus:%v\n", fg.SlotStatus) +
|
||||
fmt.Sprintf(" PedningSlots:%v\n", fg.PendingSlots) +
|
||||
|
||||
@ -13,6 +13,7 @@ import (
|
||||
const (
|
||||
PageSize = 4 * 1024
|
||||
CACHE_BLOCK_SIZE = 1 << 20
|
||||
CACHE_OBJECT_BLOCK_SIZE = 4 * 1024 * 1024
|
||||
ReadCacheTimeout = 1 // second
|
||||
DefaultCacheTTLSec = 5 * 24 * 3600
|
||||
FlashGroupDefaultWeight = 1
|
||||
@ -259,20 +260,60 @@ func (pbh *PutBlockHead) String() string {
|
||||
return fmt.Sprintf("PutBlockHead[UniKey(%v) BlockLen(%v) TTL(%v)]", pbh.UniKey, pbh.BlockLen, pbh.TTL)
|
||||
}
|
||||
|
||||
func (cr *CacheReadRequestBase) DecodeBinaryFrom(b []byte) {
|
||||
keyLen := binary.BigEndian.Uint16(b[:2])
|
||||
cr.Key = string(b[2 : 2+keyLen])
|
||||
off := 2 + uint32(keyLen)
|
||||
cr.TTL = int64(binary.BigEndian.Uint64(b[off : off+8])) // TTL (uint64, 8 bytes)
|
||||
off += 8
|
||||
cr.Slot = binary.BigEndian.Uint64(b[off : off+8]) // Slot (uint64, 8 bytes)
|
||||
off += 8
|
||||
cr.Offset = binary.BigEndian.Uint64(b[off : off+8]) // Offset (uint64, 8 bytes)
|
||||
off += 8
|
||||
cr.Size_ = binary.BigEndian.Uint64(b[off : off+8]) // Size (uint64, 8 bytes)
|
||||
}
|
||||
|
||||
func (cr *CacheReadRequestBase) EncodeBinaryTo(b []byte) {
|
||||
binary.BigEndian.PutUint16(b[:2], uint16(len(cr.Key))) // Length of Key (uint16, 2bytes)
|
||||
copy(b[2:2+len(cr.Key)], cr.Key) // Key
|
||||
off := 2 + len(cr.Key)
|
||||
binary.BigEndian.PutUint64(b[off:off+8], uint64(cr.TTL)) // TTL (uint64, 8 bytes)
|
||||
off += 8
|
||||
binary.BigEndian.PutUint64(b[off:off+8], cr.Slot) // Slot (uint64, 8 bytes)
|
||||
off += 8
|
||||
binary.BigEndian.PutUint64(b[off:off+8], cr.Offset) // Offset (uint64, 8 bytes)
|
||||
off += 8
|
||||
binary.BigEndian.PutUint64(b[off:off+8], cr.Size_) // Size (uint64, 8 bytes)
|
||||
}
|
||||
|
||||
func (cr *CacheReadRequestBase) EncodeBinaryLen() int {
|
||||
return 2 + len(cr.Key) + 8 + 8 + 8 + 8
|
||||
}
|
||||
|
||||
func (cr *CacheReadRequestBase) String() string {
|
||||
if cr == nil {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("cacheReadRequestBase[Key(%v) TTL(%v) Slot(%v) Offset(%v) Size(%v)]",
|
||||
cr.Key, cr.TTL, cr.Slot, cr.Offset, cr.Size_)
|
||||
}
|
||||
|
||||
type FlashGroupsAdminView struct {
|
||||
FlashGroups []FlashGroupAdminView
|
||||
}
|
||||
|
||||
type FlashGroupAdminView struct {
|
||||
ID uint64
|
||||
Slots []uint32
|
||||
Weight uint32
|
||||
Status FlashGroupStatus
|
||||
SlotStatus SlotStatus
|
||||
PendingSlots []uint32
|
||||
Step uint32
|
||||
FlashNodeCount int
|
||||
ZoneFlashNodes map[string][]*FlashNodeViewInfo
|
||||
ID uint64
|
||||
Slots []uint32
|
||||
ReservedSlots []uint32
|
||||
IsReducingSlots bool
|
||||
Weight uint32
|
||||
Status FlashGroupStatus
|
||||
SlotStatus SlotStatus
|
||||
PendingSlots []uint32
|
||||
Step uint32
|
||||
FlashNodeCount int
|
||||
ZoneFlashNodes map[string][]*FlashNodeViewInfo
|
||||
}
|
||||
|
||||
type FlashNodeViewInfo struct {
|
||||
|
||||
396
proto/distributed_cache.pb.go
Normal file → Executable file
396
proto/distributed_cache.pb.go
Normal file → Executable file
@ -5,20 +5,17 @@ package proto
|
||||
|
||||
import (
|
||||
fmt "fmt"
|
||||
_ "github.com/gogo/protobuf/gogoproto"
|
||||
proto "github.com/golang/protobuf/proto"
|
||||
io "io"
|
||||
math "math"
|
||||
math_bits "math/bits"
|
||||
|
||||
_ "github.com/gogo/protobuf/gogoproto"
|
||||
proto "github.com/golang/protobuf/proto"
|
||||
)
|
||||
|
||||
// Reference imports to suppress errors if they are not otherwise used.
|
||||
var (
|
||||
_ = proto.Marshal
|
||||
_ = fmt.Errorf
|
||||
_ = math.Inf
|
||||
)
|
||||
var _ = proto.Marshal
|
||||
var _ = fmt.Errorf
|
||||
var _ = math.Inf
|
||||
|
||||
// This is a compile-time assertion to ensure that this generated file
|
||||
// is compatible with the proto package it is being compiled against.
|
||||
@ -384,47 +381,129 @@ func (m *PutBlockHead) GetTTL() uint64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
type CacheReadRequestBase struct {
|
||||
Key string `protobuf:"bytes,1,opt,name=Key,proto3" json:"Key,omitempty"`
|
||||
TTL int64 `protobuf:"varint,2,opt,name=TTL,proto3" json:"TTL,omitempty"`
|
||||
Slot uint64 `protobuf:"varint,3,opt,name=Slot,proto3" json:"Slot,omitempty"`
|
||||
Offset uint64 `protobuf:"varint,4,opt,name=Offset,proto3" json:"Offset,omitempty"`
|
||||
Size_ uint64 `protobuf:"varint,5,opt,name=Size,proto3" json:"Size,omitempty"`
|
||||
XXX_NoUnkeyedLiteral struct{} `json:"-"`
|
||||
XXX_unrecognized []byte `json:"-"`
|
||||
XXX_sizecache int32 `json:"-"`
|
||||
}
|
||||
|
||||
func (m *CacheReadRequestBase) Reset() { *m = CacheReadRequestBase{} }
|
||||
func (*CacheReadRequestBase) ProtoMessage() {}
|
||||
func (*CacheReadRequestBase) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_9a29821e5022ef34, []int{5}
|
||||
}
|
||||
func (m *CacheReadRequestBase) XXX_Unmarshal(b []byte) error {
|
||||
return m.Unmarshal(b)
|
||||
}
|
||||
func (m *CacheReadRequestBase) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
|
||||
if deterministic {
|
||||
return xxx_messageInfo_CacheReadRequestBase.Marshal(b, m, deterministic)
|
||||
} else {
|
||||
b = b[:cap(b)]
|
||||
n, err := m.MarshalToSizedBuffer(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b[:n], nil
|
||||
}
|
||||
}
|
||||
func (m *CacheReadRequestBase) XXX_Merge(src proto.Message) {
|
||||
xxx_messageInfo_CacheReadRequestBase.Merge(m, src)
|
||||
}
|
||||
func (m *CacheReadRequestBase) XXX_Size() int {
|
||||
return m.Size()
|
||||
}
|
||||
func (m *CacheReadRequestBase) XXX_DiscardUnknown() {
|
||||
xxx_messageInfo_CacheReadRequestBase.DiscardUnknown(m)
|
||||
}
|
||||
|
||||
var xxx_messageInfo_CacheReadRequestBase proto.InternalMessageInfo
|
||||
|
||||
func (m *CacheReadRequestBase) GetKey() string {
|
||||
if m != nil {
|
||||
return m.Key
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (m *CacheReadRequestBase) GetTTL() int64 {
|
||||
if m != nil {
|
||||
return m.TTL
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (m *CacheReadRequestBase) GetSlot() uint64 {
|
||||
if m != nil {
|
||||
return m.Slot
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (m *CacheReadRequestBase) GetOffset() uint64 {
|
||||
if m != nil {
|
||||
return m.Offset
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (m *CacheReadRequestBase) GetSize_() uint64 {
|
||||
if m != nil {
|
||||
return m.Size_
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func init() {
|
||||
proto.RegisterType((*DataSource)(nil), "proto.DataSource")
|
||||
proto.RegisterType((*CacheRequest)(nil), "proto.CacheRequest")
|
||||
proto.RegisterType((*CacheReadRequest)(nil), "proto.CacheReadRequest")
|
||||
proto.RegisterType((*CachePrepareRequest)(nil), "proto.CachePrepareRequest")
|
||||
proto.RegisterType((*PutBlockHead)(nil), "proto.PutBlockHead")
|
||||
proto.RegisterType((*CacheReadRequestBase)(nil), "proto.CacheReadRequestBase")
|
||||
}
|
||||
|
||||
func init() { proto.RegisterFile("distributed_cache.proto", fileDescriptor_9a29821e5022ef34) }
|
||||
|
||||
var fileDescriptor_9a29821e5022ef34 = []byte{
|
||||
// 463 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x52, 0xc1, 0x6e, 0xd3, 0x40,
|
||||
0x10, 0xad, 0xeb, 0xc4, 0xa1, 0x93, 0x20, 0xc2, 0x16, 0xc1, 0xaa, 0x07, 0x63, 0xe5, 0x64, 0x09,
|
||||
0x91, 0x4a, 0xe5, 0x80, 0xc4, 0xb1, 0x84, 0xaa, 0x11, 0x15, 0x44, 0xdb, 0xd2, 0x03, 0x17, 0xe4,
|
||||
0xd8, 0x93, 0x64, 0x85, 0xeb, 0x2d, 0xde, 0xb5, 0x54, 0x90, 0xf8, 0x8f, 0x1e, 0xf9, 0x12, 0xce,
|
||||
0x1c, 0xb9, 0x72, 0x43, 0xe1, 0x47, 0x90, 0x67, 0xd7, 0xad, 0x5b, 0x8e, 0x9c, 0x3c, 0xef, 0xcd,
|
||||
0x7a, 0xe6, 0xbd, 0xb7, 0x0b, 0x8f, 0x32, 0xa9, 0x4d, 0x29, 0xe7, 0x95, 0xc1, 0xec, 0x43, 0x9a,
|
||||
0xa4, 0x2b, 0x1c, 0x9f, 0x97, 0xca, 0x28, 0xd6, 0xa5, 0xcf, 0xce, 0xd3, 0xa5, 0x34, 0xab, 0x6a,
|
||||
0x3e, 0x4e, 0xd5, 0xd9, 0xee, 0x52, 0x2d, 0xd5, 0x2e, 0xd1, 0xf3, 0x6a, 0x41, 0x88, 0x00, 0x55,
|
||||
0xf6, 0xaf, 0xd1, 0x77, 0x0f, 0x60, 0x92, 0x98, 0xe4, 0x58, 0x55, 0x65, 0x8a, 0x2c, 0x04, 0x38,
|
||||
0x90, 0x39, 0xbe, 0x5d, 0x2c, 0x34, 0x1a, 0xee, 0x45, 0x5e, 0xdc, 0x11, 0x2d, 0x86, 0x45, 0xd0,
|
||||
0x9f, 0x25, 0xa5, 0x91, 0x46, 0xaa, 0x62, 0x3a, 0xe1, 0x9b, 0x74, 0xa0, 0x4d, 0xb1, 0x1d, 0xb8,
|
||||
0xf3, 0xea, 0xc2, 0x60, 0x61, 0xa6, 0x13, 0xee, 0x53, 0xfb, 0x0a, 0xb3, 0x11, 0x0c, 0x6c, 0xed,
|
||||
0xe6, 0x77, 0xa8, 0x7f, 0x83, 0x63, 0x0c, 0x3a, 0xc7, 0xf2, 0x0b, 0xf2, 0x2e, 0xf5, 0xa8, 0x66,
|
||||
0x0f, 0xa0, 0x7b, 0xa8, 0xb4, 0xd1, 0x3c, 0x88, 0xfc, 0x78, 0x4b, 0x58, 0xf0, 0xa2, 0x73, 0xf9,
|
||||
0xed, 0xf1, 0xc6, 0xe8, 0x97, 0x07, 0x83, 0x97, 0x75, 0x0c, 0x02, 0x3f, 0x55, 0xa8, 0x0d, 0x7b,
|
||||
0x08, 0xc1, 0xa9, 0xca, 0xab, 0x33, 0x24, 0xf9, 0x5b, 0xc2, 0xa1, 0x7a, 0xc8, 0xb4, 0x50, 0x19,
|
||||
0x3a, 0xd1, 0x16, 0xb0, 0x18, 0xee, 0x1d, 0xc8, 0x0b, 0xcc, 0x5a, 0xae, 0xad, 0xea, 0xdb, 0x34,
|
||||
0xe3, 0xd0, 0x3b, 0xc5, 0x52, 0x4b, 0x55, 0x90, 0xee, 0xbb, 0xa2, 0x81, 0xec, 0x09, 0xf4, 0x6c,
|
||||
0x7c, 0x9a, 0x77, 0x23, 0x3f, 0xee, 0xef, 0xdd, 0xb7, 0xe1, 0x8e, 0xaf, 0x83, 0x15, 0xcd, 0x09,
|
||||
0x36, 0x04, 0xff, 0xe4, 0xe4, 0x88, 0x07, 0x91, 0x17, 0xfb, 0xa2, 0x2e, 0xc9, 0x71, 0xae, 0x0c,
|
||||
0xef, 0x39, 0xc7, 0xb9, 0x32, 0xce, 0xdb, 0x57, 0x18, 0x3a, 0x6b, 0x49, 0xd6, 0xd8, 0x7b, 0x7e,
|
||||
0xd3, 0x2e, 0x99, 0xec, 0xef, 0x6d, 0xbb, 0x8d, 0xed, 0x96, 0xf8, 0x27, 0x17, 0x67, 0xd0, 0x06,
|
||||
0x10, 0xdc, 0x0a, 0xdc, 0xbf, 0x0e, 0xdc, 0xad, 0x37, 0xb0, 0x4d, 0x13, 0x66, 0x25, 0x9e, 0x27,
|
||||
0x25, 0xfe, 0xb7, 0x82, 0xfa, 0x71, 0xe5, 0x89, 0x5e, 0xbd, 0x51, 0x19, 0x6a, 0xbe, 0x49, 0x77,
|
||||
0xd9, 0x62, 0xdc, 0xd6, 0xf7, 0x30, 0x98, 0x55, 0x66, 0x3f, 0x57, 0xe9, 0xc7, 0x43, 0x4c, 0xb2,
|
||||
0x5a, 0xf7, 0xbb, 0x42, 0xbe, 0xc6, 0xcf, 0xcd, 0x7d, 0x5a, 0x54, 0x3f, 0x34, 0x3a, 0x74, 0x84,
|
||||
0x05, 0x39, 0xf2, 0xc5, 0x15, 0x6e, 0x42, 0xb6, 0x96, 0xea, 0xd2, 0xce, 0xde, 0x1f, 0xfe, 0x58,
|
||||
0x87, 0xde, 0xcf, 0x75, 0xe8, 0xfd, 0x5e, 0x87, 0xde, 0xe5, 0x9f, 0x70, 0x63, 0x1e, 0x90, 0xea,
|
||||
0x67, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x42, 0xb3, 0xc8, 0x05, 0x57, 0x03, 0x00, 0x00,
|
||||
// 497 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x92, 0xcf, 0x6e, 0xd3, 0x40,
|
||||
0x10, 0xc6, 0xeb, 0x38, 0x7f, 0xe8, 0x24, 0x88, 0xb0, 0xad, 0x60, 0xd5, 0x83, 0xb1, 0x72, 0xb2,
|
||||
0x84, 0x48, 0xa5, 0x72, 0x40, 0xe2, 0x18, 0x42, 0xd5, 0x88, 0x0a, 0x22, 0xb7, 0xf4, 0xc0, 0x05,
|
||||
0x39, 0xf6, 0x24, 0x59, 0xe1, 0x7a, 0x8b, 0x77, 0x2d, 0x15, 0x24, 0x24, 0x1e, 0xa3, 0x47, 0x9e,
|
||||
0x84, 0x33, 0x47, 0xae, 0xdc, 0x50, 0x78, 0x11, 0xe4, 0xd9, 0x75, 0xe3, 0xa6, 0xdc, 0x38, 0x79,
|
||||
0xbe, 0xd9, 0xf5, 0xcc, 0x7c, 0xbf, 0x59, 0x78, 0x98, 0x08, 0xa5, 0x73, 0x31, 0x2b, 0x34, 0x26,
|
||||
0xef, 0xe3, 0x28, 0x5e, 0xe2, 0xf0, 0x22, 0x97, 0x5a, 0xb2, 0x16, 0x7d, 0xf6, 0x9e, 0x2c, 0x84,
|
||||
0x5e, 0x16, 0xb3, 0x61, 0x2c, 0xcf, 0xf7, 0x17, 0x72, 0x21, 0xf7, 0x29, 0x3d, 0x2b, 0xe6, 0xa4,
|
||||
0x48, 0x50, 0x64, 0xfe, 0x1a, 0x7c, 0x77, 0x00, 0xc6, 0x91, 0x8e, 0x4e, 0x64, 0x91, 0xc7, 0xc8,
|
||||
0x3c, 0x80, 0x43, 0x91, 0xe2, 0x9b, 0xf9, 0x5c, 0xa1, 0xe6, 0x8e, 0xef, 0x04, 0xcd, 0xb0, 0x96,
|
||||
0x61, 0x3e, 0x74, 0xa7, 0x51, 0xae, 0x85, 0x16, 0x32, 0x9b, 0x8c, 0x79, 0x83, 0x2e, 0xd4, 0x53,
|
||||
0x6c, 0x0f, 0xee, 0xbc, 0xbc, 0xd4, 0x98, 0xe9, 0xc9, 0x98, 0xbb, 0x74, 0x7c, 0xad, 0xd9, 0x00,
|
||||
0x7a, 0x26, 0xb6, 0xf5, 0x9b, 0x74, 0x7e, 0x23, 0xc7, 0x18, 0x34, 0x4f, 0xc4, 0x67, 0xe4, 0x2d,
|
||||
0x3a, 0xa3, 0x98, 0xed, 0x42, 0xeb, 0x48, 0x2a, 0xad, 0x78, 0xdb, 0x77, 0x83, 0xed, 0xd0, 0x88,
|
||||
0xe7, 0xcd, 0xab, 0x6f, 0x8f, 0xb6, 0x06, 0xbf, 0x1c, 0xe8, 0xbd, 0x28, 0x31, 0x84, 0xf8, 0xb1,
|
||||
0x40, 0xa5, 0xd9, 0x03, 0x68, 0x9f, 0xc9, 0xb4, 0x38, 0x47, 0x1a, 0x7f, 0x3b, 0xb4, 0xaa, 0x2c,
|
||||
0x32, 0xc9, 0x64, 0x82, 0x76, 0x68, 0x23, 0x58, 0x00, 0xf7, 0x0e, 0xc5, 0x25, 0x26, 0x35, 0xd7,
|
||||
0x66, 0xea, 0xcd, 0x34, 0xe3, 0xd0, 0x39, 0xc3, 0x5c, 0x09, 0x99, 0xd1, 0xdc, 0x77, 0xc3, 0x4a,
|
||||
0xb2, 0xc7, 0xd0, 0x31, 0xf8, 0x14, 0x6f, 0xf9, 0x6e, 0xd0, 0x3d, 0xb8, 0x6f, 0xe0, 0x0e, 0xd7,
|
||||
0x60, 0xc3, 0xea, 0x06, 0xeb, 0x83, 0x7b, 0x7a, 0x7a, 0xcc, 0xdb, 0xbe, 0x13, 0xb8, 0x61, 0x19,
|
||||
0x92, 0xe3, 0x54, 0x6a, 0xde, 0xb1, 0x8e, 0x53, 0xa9, 0xad, 0xb7, 0x2f, 0xd0, 0xb7, 0xd6, 0xa2,
|
||||
0xa4, 0xb2, 0xf7, 0xec, 0xa6, 0x5d, 0x32, 0xd9, 0x3d, 0xd8, 0xb1, 0x1d, 0xeb, 0x47, 0xe1, 0x2d,
|
||||
0x2e, 0xd6, 0xa0, 0x01, 0xd0, 0xde, 0x00, 0xee, 0xae, 0x81, 0xdb, 0xf6, 0x1a, 0x76, 0xa8, 0xc2,
|
||||
0x34, 0xc7, 0x8b, 0x28, 0xc7, 0xff, 0x9e, 0xa0, 0x7c, 0x5c, 0x69, 0xa4, 0x96, 0xaf, 0x65, 0x82,
|
||||
0x8a, 0x37, 0x68, 0x97, 0xb5, 0x8c, 0xed, 0xfa, 0x0e, 0x7a, 0xd3, 0x42, 0x8f, 0x52, 0x19, 0x7f,
|
||||
0x38, 0xc2, 0x28, 0x29, 0xe7, 0x7e, 0x9b, 0x89, 0x57, 0xf8, 0xa9, 0xda, 0xa7, 0x51, 0xe5, 0x43,
|
||||
0xa3, 0x4b, 0xc7, 0x98, 0x91, 0x23, 0x37, 0xbc, 0xd6, 0x15, 0x64, 0x63, 0xa9, 0x0c, 0x6d, 0xed,
|
||||
0xaf, 0x0e, 0xec, 0x6e, 0x12, 0x1d, 0x45, 0x0a, 0xcb, 0x1f, 0xd6, 0x1d, 0xca, 0xb0, 0x2a, 0xd1,
|
||||
0xb8, 0xbd, 0x27, 0x77, 0xbd, 0xa7, 0x1a, 0xd4, 0xe6, 0x3f, 0xa1, 0xb6, 0x36, 0xa1, 0x8e, 0xfa,
|
||||
0x3f, 0x56, 0x9e, 0xf3, 0x73, 0xe5, 0x39, 0xbf, 0x57, 0x9e, 0x73, 0xf5, 0xc7, 0xdb, 0x9a, 0xb5,
|
||||
0x09, 0xdc, 0xd3, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xb4, 0xc4, 0xac, 0xf4, 0xda, 0x03, 0x00,
|
||||
0x00,
|
||||
}
|
||||
|
||||
func (m *DataSource) Marshal() (dAtA []byte, err error) {
|
||||
@ -702,6 +781,60 @@ func (m *PutBlockHead) MarshalToSizedBuffer(dAtA []byte) (int, error) {
|
||||
return len(dAtA) - i, nil
|
||||
}
|
||||
|
||||
func (m *CacheReadRequestBase) Marshal() (dAtA []byte, err error) {
|
||||
size := m.Size()
|
||||
dAtA = make([]byte, size)
|
||||
n, err := m.MarshalToSizedBuffer(dAtA[:size])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return dAtA[:n], nil
|
||||
}
|
||||
|
||||
func (m *CacheReadRequestBase) MarshalTo(dAtA []byte) (int, error) {
|
||||
size := m.Size()
|
||||
return m.MarshalToSizedBuffer(dAtA[:size])
|
||||
}
|
||||
|
||||
func (m *CacheReadRequestBase) MarshalToSizedBuffer(dAtA []byte) (int, error) {
|
||||
i := len(dAtA)
|
||||
_ = i
|
||||
var l int
|
||||
_ = l
|
||||
if m.XXX_unrecognized != nil {
|
||||
i -= len(m.XXX_unrecognized)
|
||||
copy(dAtA[i:], m.XXX_unrecognized)
|
||||
}
|
||||
if m.Size_ != 0 {
|
||||
i = encodeVarintDistributedCache(dAtA, i, uint64(m.Size_))
|
||||
i--
|
||||
dAtA[i] = 0x28
|
||||
}
|
||||
if m.Offset != 0 {
|
||||
i = encodeVarintDistributedCache(dAtA, i, uint64(m.Offset))
|
||||
i--
|
||||
dAtA[i] = 0x20
|
||||
}
|
||||
if m.Slot != 0 {
|
||||
i = encodeVarintDistributedCache(dAtA, i, uint64(m.Slot))
|
||||
i--
|
||||
dAtA[i] = 0x18
|
||||
}
|
||||
if m.TTL != 0 {
|
||||
i = encodeVarintDistributedCache(dAtA, i, uint64(m.TTL))
|
||||
i--
|
||||
dAtA[i] = 0x10
|
||||
}
|
||||
if len(m.Key) > 0 {
|
||||
i -= len(m.Key)
|
||||
copy(dAtA[i:], m.Key)
|
||||
i = encodeVarintDistributedCache(dAtA, i, uint64(len(m.Key)))
|
||||
i--
|
||||
dAtA[i] = 0xa
|
||||
}
|
||||
return len(dAtA) - i, nil
|
||||
}
|
||||
|
||||
func encodeVarintDistributedCache(dAtA []byte, offset int, v uint64) int {
|
||||
offset -= sovDistributedCache(v)
|
||||
base := offset
|
||||
@ -849,6 +982,34 @@ func (m *PutBlockHead) Size() (n int) {
|
||||
return n
|
||||
}
|
||||
|
||||
func (m *CacheReadRequestBase) Size() (n int) {
|
||||
if m == nil {
|
||||
return 0
|
||||
}
|
||||
var l int
|
||||
_ = l
|
||||
l = len(m.Key)
|
||||
if l > 0 {
|
||||
n += 1 + l + sovDistributedCache(uint64(l))
|
||||
}
|
||||
if m.TTL != 0 {
|
||||
n += 1 + sovDistributedCache(uint64(m.TTL))
|
||||
}
|
||||
if m.Slot != 0 {
|
||||
n += 1 + sovDistributedCache(uint64(m.Slot))
|
||||
}
|
||||
if m.Offset != 0 {
|
||||
n += 1 + sovDistributedCache(uint64(m.Offset))
|
||||
}
|
||||
if m.Size_ != 0 {
|
||||
n += 1 + sovDistributedCache(uint64(m.Size_))
|
||||
}
|
||||
if m.XXX_unrecognized != nil {
|
||||
n += len(m.XXX_unrecognized)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func sovDistributedCache(x uint64) (n int) {
|
||||
return (math_bits.Len64(x|1) + 6) / 7
|
||||
}
|
||||
@ -1610,6 +1771,165 @@ func (m *PutBlockHead) Unmarshal(dAtA []byte) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (m *CacheReadRequestBase) Unmarshal(dAtA []byte) error {
|
||||
l := len(dAtA)
|
||||
iNdEx := 0
|
||||
for iNdEx < l {
|
||||
preIndex := iNdEx
|
||||
var wire uint64
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
if shift >= 64 {
|
||||
return ErrIntOverflowDistributedCache
|
||||
}
|
||||
if iNdEx >= l {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
b := dAtA[iNdEx]
|
||||
iNdEx++
|
||||
wire |= uint64(b&0x7F) << shift
|
||||
if b < 0x80 {
|
||||
break
|
||||
}
|
||||
}
|
||||
fieldNum := int32(wire >> 3)
|
||||
wireType := int(wire & 0x7)
|
||||
if wireType == 4 {
|
||||
return fmt.Errorf("proto: CacheReadRequestBase: wiretype end group for non-group")
|
||||
}
|
||||
if fieldNum <= 0 {
|
||||
return fmt.Errorf("proto: CacheReadRequestBase: illegal tag %d (wire type %d)", fieldNum, wire)
|
||||
}
|
||||
switch fieldNum {
|
||||
case 1:
|
||||
if wireType != 2 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType)
|
||||
}
|
||||
var stringLen uint64
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
if shift >= 64 {
|
||||
return ErrIntOverflowDistributedCache
|
||||
}
|
||||
if iNdEx >= l {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
b := dAtA[iNdEx]
|
||||
iNdEx++
|
||||
stringLen |= uint64(b&0x7F) << shift
|
||||
if b < 0x80 {
|
||||
break
|
||||
}
|
||||
}
|
||||
intStringLen := int(stringLen)
|
||||
if intStringLen < 0 {
|
||||
return ErrInvalidLengthDistributedCache
|
||||
}
|
||||
postIndex := iNdEx + intStringLen
|
||||
if postIndex < 0 {
|
||||
return ErrInvalidLengthDistributedCache
|
||||
}
|
||||
if postIndex > l {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
m.Key = string(dAtA[iNdEx:postIndex])
|
||||
iNdEx = postIndex
|
||||
case 2:
|
||||
if wireType != 0 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field TTL", wireType)
|
||||
}
|
||||
m.TTL = 0
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
if shift >= 64 {
|
||||
return ErrIntOverflowDistributedCache
|
||||
}
|
||||
if iNdEx >= l {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
b := dAtA[iNdEx]
|
||||
iNdEx++
|
||||
m.TTL |= int64(b&0x7F) << shift
|
||||
if b < 0x80 {
|
||||
break
|
||||
}
|
||||
}
|
||||
case 3:
|
||||
if wireType != 0 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field Slot", wireType)
|
||||
}
|
||||
m.Slot = 0
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
if shift >= 64 {
|
||||
return ErrIntOverflowDistributedCache
|
||||
}
|
||||
if iNdEx >= l {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
b := dAtA[iNdEx]
|
||||
iNdEx++
|
||||
m.Slot |= uint64(b&0x7F) << shift
|
||||
if b < 0x80 {
|
||||
break
|
||||
}
|
||||
}
|
||||
case 4:
|
||||
if wireType != 0 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field Offset", wireType)
|
||||
}
|
||||
m.Offset = 0
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
if shift >= 64 {
|
||||
return ErrIntOverflowDistributedCache
|
||||
}
|
||||
if iNdEx >= l {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
b := dAtA[iNdEx]
|
||||
iNdEx++
|
||||
m.Offset |= uint64(b&0x7F) << shift
|
||||
if b < 0x80 {
|
||||
break
|
||||
}
|
||||
}
|
||||
case 5:
|
||||
if wireType != 0 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType)
|
||||
}
|
||||
m.Size_ = 0
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
if shift >= 64 {
|
||||
return ErrIntOverflowDistributedCache
|
||||
}
|
||||
if iNdEx >= l {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
b := dAtA[iNdEx]
|
||||
iNdEx++
|
||||
m.Size_ |= uint64(b&0x7F) << shift
|
||||
if b < 0x80 {
|
||||
break
|
||||
}
|
||||
}
|
||||
default:
|
||||
iNdEx = preIndex
|
||||
skippy, err := skipDistributedCache(dAtA[iNdEx:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if (skippy < 0) || (iNdEx+skippy) < 0 {
|
||||
return ErrInvalidLengthDistributedCache
|
||||
}
|
||||
if (iNdEx + skippy) > l {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
|
||||
iNdEx += skippy
|
||||
}
|
||||
}
|
||||
|
||||
if iNdEx > l {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func skipDistributedCache(dAtA []byte) (n int, err error) {
|
||||
l := len(dAtA)
|
||||
iNdEx := 0
|
||||
|
||||
@ -48,4 +48,13 @@ message PutBlockHead {
|
||||
string UniKey = 1;
|
||||
int64 BlockLen = 2;
|
||||
uint64 TTL = 3;
|
||||
}
|
||||
}
|
||||
message CacheReadRequestBase {
|
||||
option (gogoproto.goproto_stringer) = false;
|
||||
|
||||
string Key = 1;
|
||||
int64 TTL = 2;
|
||||
uint64 Slot = 3;
|
||||
uint64 Offset = 4;
|
||||
uint64 Size = 5;
|
||||
}
|
||||
|
||||
@ -300,6 +300,7 @@ const (
|
||||
OpFlashNodeCacheRead uint8 = 0xDC
|
||||
OpFlashNodeCachePutBlock uint8 = 0xD8
|
||||
OpFlashNodeCacheDelete uint8 = 0xD9
|
||||
OpFlashNodeCacheReadObject uint8 = 0xDD
|
||||
OpFlashNodeSetReadIOLimits uint8 = 0xED
|
||||
OpFlashNodeSetWriteIOLimits uint8 = 0xEE
|
||||
OpFlashNodeScan uint8 = 0xD4
|
||||
@ -1096,6 +1097,40 @@ func (p *Packet) WriteToConn(c net.Conn) (err error) {
|
||||
return
|
||||
}
|
||||
|
||||
func (p *Packet) WriteToConnForOCS(c net.Conn) (err error) {
|
||||
headSize := p.CalcPacketHeaderSize()
|
||||
header, err := Buffers.Get(headSize)
|
||||
if err != nil {
|
||||
header = make([]byte, headSize)
|
||||
}
|
||||
// log.LogErrorf("action[WriteToConn] buffer get nil,opcode %v head len [%v]", p.Opcode, len(header))
|
||||
defer Buffers.Put(header)
|
||||
c.SetWriteDeadline(time.Now().Add(WriteDeadlineTime * time.Second))
|
||||
p.MarshalHeader(header)
|
||||
if _, err = c.Write(header); err == nil {
|
||||
// write dir version info.
|
||||
if p.IsVersionList() {
|
||||
d, err1 := p.MarshalVersionSlice()
|
||||
if err1 != nil {
|
||||
log.LogErrorf("MarshalVersionSlice: marshal version ifo failed, err %s", err1.Error())
|
||||
return err1
|
||||
}
|
||||
|
||||
_, err = c.Write(d)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err = c.Write(p.Arg[:int(p.ArgLen)]); err == nil {
|
||||
if p.Data != nil && p.Size != 0 {
|
||||
_, err = c.Write(p.Data[:])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// ReadFull is a wrapper function of io.ReadFull.
|
||||
func ReadFull(c net.Conn, buf *[]byte, readSize int) (err error) {
|
||||
*buf = make([]byte, readSize)
|
||||
|
||||
@ -7,6 +7,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/cubefs/cubefs/proto"
|
||||
@ -336,6 +337,16 @@ func (c *Cluster) scheduleToUpdateFlashGroupRespCache() {
|
||||
for {
|
||||
if c.partition != nil && c.partition.IsRaftLeader() {
|
||||
c.flashNodeTopo.updateClientResponse()
|
||||
// sync fg if slots changed
|
||||
c.flashNodeTopo.flashGroupMap.Range(func(_, value interface{}) bool {
|
||||
fg := value.(*FlashGroup)
|
||||
slotChanged := atomic.LoadInt32(&fg.SlotChanged) != 0
|
||||
if slotChanged {
|
||||
c.syncUpdateFlashGroup(fg)
|
||||
atomic.StoreInt32(&fg.SlotChanged, 0)
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
select {
|
||||
case <-c.stopc:
|
||||
|
||||
@ -1,7 +1,10 @@
|
||||
package flashgroupmanager
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/cubefs/cubefs/proto"
|
||||
)
|
||||
@ -12,13 +15,17 @@ const (
|
||||
)
|
||||
|
||||
type flashGroupValue struct {
|
||||
ID uint64
|
||||
Slots []uint32 // FlashGroup's position in hasher ring, set by cli. value is range of crc32.
|
||||
SlotStatus proto.SlotStatus
|
||||
PendingSlots []uint32
|
||||
Step uint32
|
||||
Weight uint32
|
||||
Status proto.FlashGroupStatus
|
||||
ID uint64
|
||||
Slots []uint32 // FlashGroup's position in hasher ring, set by cli. value is range of crc32.
|
||||
ReservedSlots []uint32
|
||||
SlotStatus proto.SlotStatus
|
||||
PendingSlots []uint32
|
||||
Step uint32
|
||||
Weight uint32
|
||||
Status proto.FlashGroupStatus
|
||||
LostAllFlashNode int32
|
||||
ReducingSlots int32
|
||||
SlotChanged int32
|
||||
}
|
||||
|
||||
type FlashGroup struct {
|
||||
@ -30,13 +37,15 @@ type FlashGroup struct {
|
||||
func (fg *FlashGroup) GetAdminView() (view proto.FlashGroupAdminView) {
|
||||
fg.lock.RLock()
|
||||
view = proto.FlashGroupAdminView{
|
||||
ID: fg.ID,
|
||||
Slots: fg.Slots,
|
||||
Weight: fg.Weight,
|
||||
Status: fg.Status,
|
||||
SlotStatus: fg.SlotStatus,
|
||||
PendingSlots: fg.PendingSlots,
|
||||
Step: fg.Step,
|
||||
ID: fg.ID,
|
||||
Slots: fg.Slots,
|
||||
ReservedSlots: fg.ReservedSlots,
|
||||
IsReducingSlots: fg.ReducingSlots != 0,
|
||||
Weight: fg.Weight,
|
||||
Status: fg.Status,
|
||||
SlotStatus: fg.SlotStatus,
|
||||
PendingSlots: fg.PendingSlots,
|
||||
Step: fg.Step,
|
||||
}
|
||||
view.ZoneFlashNodes = make(map[string][]*proto.FlashNodeViewInfo)
|
||||
view.FlashNodeCount = len(fg.flashNodes)
|
||||
@ -75,6 +84,49 @@ func argConvertFlashGroupStatus(active bool) proto.FlashGroupStatus {
|
||||
return proto.FlashGroupStatus_Inactive
|
||||
}
|
||||
|
||||
func (fg *FlashGroup) IsLostAllFlashNode() bool {
|
||||
return atomic.LoadInt32(&fg.LostAllFlashNode) != 0
|
||||
}
|
||||
|
||||
func (fg *FlashGroup) ReduceSlot() {
|
||||
reducingSlots := atomic.LoadInt32(&fg.ReducingSlots) != 0
|
||||
if reducingSlots {
|
||||
return
|
||||
}
|
||||
atomic.StoreInt32(&fg.ReducingSlots, 1)
|
||||
|
||||
ticker := time.NewTicker(10 * time.Minute)
|
||||
defer ticker.Stop()
|
||||
go func() {
|
||||
for {
|
||||
<-ticker.C
|
||||
if fg.IsLostAllFlashNode() {
|
||||
fg.executeReduceSlot()
|
||||
atomic.StoreInt32(&fg.SlotChanged, 1)
|
||||
} else {
|
||||
atomic.StoreInt32(&fg.ReducingSlots, 0)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (fg *FlashGroup) executeReduceSlot() {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
numToSelect := len(fg.Slots) / 4
|
||||
if numToSelect == 0 {
|
||||
return
|
||||
}
|
||||
selectedIndexes := rand.Perm(len(fg.Slots))[:numToSelect]
|
||||
for i := len(selectedIndexes) - 1; i >= 0; i-- {
|
||||
index := selectedIndexes[i]
|
||||
// add selected slot to ReservedSlots
|
||||
fg.ReservedSlots = append(fg.ReservedSlots, fg.Slots[index])
|
||||
// remove selected slot from Slots
|
||||
fg.Slots = append(fg.Slots[:index], fg.Slots[index+1:]...)
|
||||
}
|
||||
}
|
||||
|
||||
func (fg *FlashGroup) GetStatus() (st proto.FlashGroupStatus) {
|
||||
fg.lock.RLock()
|
||||
st = fg.Status
|
||||
@ -82,11 +134,11 @@ func (fg *FlashGroup) GetStatus() (st proto.FlashGroupStatus) {
|
||||
return
|
||||
}
|
||||
|
||||
func (fg *FlashGroup) getFlashNodeHostsEnabled() (hosts []string) {
|
||||
func (fg *FlashGroup) getFlashNodeHostsEnableAndActive() (hosts []string) {
|
||||
hosts = make([]string, 0, len(fg.flashNodes))
|
||||
fg.lock.RLock()
|
||||
for host, flashNode := range fg.flashNodes {
|
||||
if !flashNode.isEnable() {
|
||||
if !flashNode.isActiveAndEnable() {
|
||||
continue
|
||||
}
|
||||
hosts = append(hosts, host)
|
||||
|
||||
@ -69,13 +69,6 @@ func (flashNode *FlashNode) GetFlashNodeViewInfo() (info *proto.FlashNodeViewInf
|
||||
return
|
||||
}
|
||||
|
||||
func (flashNode *FlashNode) isEnable() (ok bool) {
|
||||
flashNode.RLock()
|
||||
ok = flashNode.IsEnable
|
||||
flashNode.RUnlock()
|
||||
return
|
||||
}
|
||||
|
||||
func (flashNode *FlashNode) isActiveAndEnable() (ok bool) {
|
||||
flashNode.RLock()
|
||||
ok = flashNode.IsActive && flashNode.IsEnable
|
||||
|
||||
@ -173,12 +173,36 @@ func (t *FlashNodeTopology) updateClientResponse() []byte {
|
||||
func (t *FlashNodeTopology) getFlashGroupView() (fgv *proto.FlashGroupView) {
|
||||
fgv = new(proto.FlashGroupView)
|
||||
fgv.Enable = true
|
||||
fgCount := 0
|
||||
t.flashGroupMap.Range(func(_, _ interface{}) bool {
|
||||
fgCount++
|
||||
return true
|
||||
})
|
||||
disableFlashGroupNum := 0
|
||||
maxDisableFlashGroupCount := fgCount * 2 / 3
|
||||
|
||||
t.flashGroupMap.Range(func(_, value interface{}) bool {
|
||||
fg := value.(*FlashGroup)
|
||||
if fg.GetStatus().IsActive() {
|
||||
hosts := fg.getFlashNodeHostsEnabled()
|
||||
hosts := fg.getFlashNodeHostsEnableAndActive()
|
||||
if len(hosts) == 0 {
|
||||
return true
|
||||
atomic.StoreInt32(&fg.LostAllFlashNode, 1)
|
||||
fg.ReduceSlot()
|
||||
} else if len(fg.ReservedSlots) > 0 {
|
||||
fg.Slots = append(fg.Slots, fg.ReservedSlots...)
|
||||
fg.ReservedSlots = nil
|
||||
atomic.StoreInt32(&fg.LostAllFlashNode, 0)
|
||||
atomic.StoreInt32(&fg.SlotChanged, 1)
|
||||
}
|
||||
|
||||
if len(fg.Slots) == 0 {
|
||||
disableFlashGroupNum++
|
||||
if disableFlashGroupNum >= maxDisableFlashGroupCount && len(fg.ReservedSlots) > 0 {
|
||||
fg.Slots = append(fg.Slots, fg.ReservedSlots...)
|
||||
atomic.StoreInt32(&fg.SlotChanged, 1)
|
||||
} else {
|
||||
return true
|
||||
}
|
||||
}
|
||||
fgv.FlashGroups = append(fgv.FlashGroups, &proto.FlashGroupInfo{
|
||||
ID: fg.ID,
|
||||
|
||||
@ -183,7 +183,7 @@ func (cb *CacheBlock) WriteAt(data []byte, offset, size int64) (err error) {
|
||||
}
|
||||
|
||||
// Read reads data from an extent.
|
||||
func (cb *CacheBlock) Read(ctx context.Context, data []byte, offset, size int64, waitForBlock bool) (crc uint32, err error) {
|
||||
func (cb *CacheBlock) Read(ctx context.Context, data []byte, offset, size int64, waitForBlock bool, readCrc bool) (crc uint32, err error) {
|
||||
var file *os.File
|
||||
if err = cb.ready(ctx, waitForBlock); err != nil {
|
||||
return
|
||||
@ -206,6 +206,11 @@ func (cb *CacheBlock) Read(ctx context.Context, data []byte, offset, size int64,
|
||||
if realSize >= size {
|
||||
realSize = size
|
||||
}
|
||||
|
||||
if cb.sourceType == SourceTypeBlock {
|
||||
realSize = util.PageSize
|
||||
}
|
||||
|
||||
log.LogDebugf("action[Read] read cache block:%v, offset:%d, allocSize:%d, usedSize:%d", cb.blockKey, offset, cb.allocSize, cb.usedSize)
|
||||
|
||||
if file, err = cb.GetOrOpenFileHandler(); err != nil {
|
||||
@ -216,7 +221,19 @@ func (cb *CacheBlock) Read(ctx context.Context, data []byte, offset, size int64,
|
||||
log.LogErrorf("action[Read] read cacheBlock:%v failed, filename:%v realSize:%d", cb.blockKey, file.Name(), realSize)
|
||||
return
|
||||
}
|
||||
crc = crc32.ChecksumIEEE(data)
|
||||
if readCrc {
|
||||
sliceIndex := offset / proto.PageSize
|
||||
crcOffset := cb.allocSize + HeaderSize + sliceIndex*4
|
||||
crcBuf := make([]byte, 4)
|
||||
if _, err = file.ReadAt(crcBuf, crcOffset); err != nil {
|
||||
log.LogErrorf("action[Read] read crc:%v failed, filename:%v realSize:%d", cb.blockKey, file.Name(), realSize)
|
||||
return
|
||||
}
|
||||
crc = binary.BigEndian.Uint32(crcBuf)
|
||||
} else {
|
||||
crc = crc32.ChecksumIEEE(data)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@ -877,3 +894,18 @@ func CalcAllocSizeV2(reqLen int) int {
|
||||
}
|
||||
return reqLen
|
||||
}
|
||||
|
||||
func (cb *CacheBlock) VerifyObjectReq(offset, size uint64) error {
|
||||
end := offset + size - 1
|
||||
if offset/proto.CACHE_OBJECT_BLOCK_SIZE != end/proto.CACHE_OBJECT_BLOCK_SIZE {
|
||||
log.LogErrorf("invalid range offset(%v) size(%v)", offset, size)
|
||||
return fmt.Errorf("invalid range offset(%v) size(%v)", offset, size)
|
||||
}
|
||||
|
||||
if uint64(cb.usedSize) <= end {
|
||||
log.LogWarnf("block is not read, usedSize(%v) offset(%v) size(%v)", cb.usedSize, offset, size)
|
||||
return fmt.Errorf("block is not ready")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -253,7 +253,7 @@ func TestBlockReadCache(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
cacheBlock.notifyReady()
|
||||
bytesRead := make([]byte, 1024)
|
||||
_, err = cacheBlock.Read(context.Background(), bytesRead, offset, 1024, true)
|
||||
_, err = cacheBlock.Read(context.Background(), bytesRead, offset, 1024, true, false)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, bytesRead, bytes)
|
||||
}
|
||||
@ -306,7 +306,7 @@ func testParallelOperation(t *testing.T) {
|
||||
case <-ticker.C:
|
||||
bytesRead := make([]byte, 1024)
|
||||
offset := rand.Intn(int(cacheBlock.allocSize))
|
||||
cacheBlock.Read(context.Background(), bytesRead, int64(offset), 1024, true)
|
||||
cacheBlock.Read(context.Background(), bytesRead, int64(offset), 1024, true, false)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
@ -534,6 +534,10 @@ func (c *CacheEngine) DeleteCacheBlock(key string) {
|
||||
|
||||
func (c *CacheEngine) GetCacheBlockForRead(volume string, inode, offset uint64, version uint32, size uint64) (block *CacheBlock, err error) {
|
||||
key := GenCacheBlockKey(volume, inode, offset, version)
|
||||
return c.GetCacheBlockForReadByKey(key)
|
||||
}
|
||||
|
||||
func (c *CacheEngine) GetCacheBlockForReadByKey(key string) (block *CacheBlock, err error) {
|
||||
v, ok := c.keyToDiskMap.Load(key)
|
||||
if ok {
|
||||
cacheItem := v.(*lruCacheItem)
|
||||
|
||||
@ -63,6 +63,8 @@ func (f *FlashNode) handlePacket(conn net.Conn, p *proto.Packet) (err error) {
|
||||
err = f.opCachePutBlock(conn, p)
|
||||
case proto.OpFlashNodeCacheDelete:
|
||||
err = f.opCacheDelete(conn, p)
|
||||
case proto.OpFlashNodeCacheReadObject:
|
||||
err = f.opCacheObjectGet(conn, p)
|
||||
case proto.OpFlashNodeSetReadIOLimits:
|
||||
err = f.opSetReadIOLimits(conn, p)
|
||||
case proto.OpFlashNodeSetWriteIOLimits:
|
||||
@ -386,6 +388,67 @@ func (f *FlashNode) opCachePutBlock(conn net.Conn, p *proto.Packet) (err error)
|
||||
return err
|
||||
}
|
||||
|
||||
func (f *FlashNode) opCacheObjectGet(conn net.Conn, p *proto.Packet) (err error) {
|
||||
var key string
|
||||
bgTime := stat.BeginStat()
|
||||
defer func() {
|
||||
stat.EndStat("FlashNode:opCacheObjectGet", err, bgTime, 1)
|
||||
}()
|
||||
|
||||
defer func() {
|
||||
if err != nil {
|
||||
if !proto.IsFlashNodeLimitError(err) {
|
||||
log.LogWarnf("action[opCacheObjectGet] key:[%s], logMsg:%s", key,
|
||||
p.LogMessage(p.GetOpMsg(), conn.RemoteAddr().String(), p.StartT, err))
|
||||
}
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
if e := p.WriteToConn(conn); e != nil {
|
||||
log.LogErrorf("action[opCacheObjectGet] write to conn %v", e)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
ctx, ctxCancel := context.WithTimeout(context.Background(), time.Duration(f.handleReadTimeout)*time.Millisecond)
|
||||
defer ctxCancel()
|
||||
|
||||
req := new(proto.CacheReadRequestBase)
|
||||
if err = p.UnmarshalDataPb(req); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
log.LogDebugf("opCacheObjectGet req(%v)", req)
|
||||
|
||||
uniKey := req.Key
|
||||
pDir := cachengine.MapKeyToDirectory(uniKey)
|
||||
blockKey := cachengine.GenCacheBlockKeyV2(pDir, uniKey)
|
||||
|
||||
f.updateSlotStat(req.Slot)
|
||||
block, err := f.cacheEngine.GetCacheBlockForReadByKey(blockKey)
|
||||
if err != nil {
|
||||
// if not find block, check whether should cache
|
||||
err = f.shouldCache(key)
|
||||
return
|
||||
}
|
||||
|
||||
err = block.VerifyObjectReq(req.Offset, req.Size_)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
bgTime2 := stat.BeginStat()
|
||||
// reply to client as quick as possible if hit cache
|
||||
err2 := f.limitRead.RunNoWait(int(req.Size_), false, func() {
|
||||
err = f.doObjectReadRequest(ctx, conn, req, p, block)
|
||||
})
|
||||
if err2 != nil {
|
||||
err = err2
|
||||
stat.EndStat("HitCacheRead", err, bgTime2, 1)
|
||||
return
|
||||
}
|
||||
stat.EndStat("HitCacheRead", err, bgTime2, 1)
|
||||
return
|
||||
}
|
||||
|
||||
func (f *FlashNode) opFlashNodeScan(conn net.Conn, p *proto.Packet) (err error) {
|
||||
data := p.Data
|
||||
responseAckOKToMaster(conn, p)
|
||||
@ -582,7 +645,7 @@ func (f *FlashNode) doStreamReadRequest(ctx context.Context, conn net.Conn, req
|
||||
reply.ExtentOffset = offset
|
||||
p.Size = currReadSize
|
||||
p.ExtentOffset = offset
|
||||
reply.CRC, err = block.Read(ctx, reply.Data[:], offset, int64(currReadSize), f.waitForCacheBlock)
|
||||
reply.CRC, err = block.Read(ctx, reply.Data[:], offset, int64(currReadSize), f.waitForCacheBlock, false)
|
||||
if err != nil {
|
||||
bufRelease()
|
||||
return
|
||||
@ -614,6 +677,99 @@ func (f *FlashNode) doStreamReadRequest(ctx context.Context, conn net.Conn, req
|
||||
return
|
||||
}
|
||||
|
||||
func (f *FlashNode) doObjectReadRequest(ctx context.Context, conn net.Conn, req *proto.CacheReadRequestBase, p *proto.Packet,
|
||||
block *cachengine.CacheBlock,
|
||||
) (err error) {
|
||||
const action = "action[doObjectReadRequest]"
|
||||
offset := int64(req.Offset)
|
||||
defer func() {
|
||||
if err != nil {
|
||||
// too many caching logs
|
||||
if strings.Compare(err.Error(), "require data is caching") != 0 {
|
||||
log.LogWarnf("%s cache block(%v) err:%v", action, block.String(), err)
|
||||
}
|
||||
} else {
|
||||
f.metrics.updateReadCountMetric(block.GetRootPath())
|
||||
f.metrics.updateReadBytesMetric(req.Size_, block.GetRootPath())
|
||||
}
|
||||
}()
|
||||
|
||||
// first reply to client
|
||||
firstReply := proto.NewPacket()
|
||||
firstReply.ReqID = p.ReqID
|
||||
firstReply.Size = uint32(req.Size_)
|
||||
firstReply.ResultCode = proto.OpOk
|
||||
firstReply.Opcode = p.Opcode
|
||||
firstReply.StartT = p.StartT
|
||||
if err = firstReply.WriteToConn(conn); err != nil {
|
||||
log.LogErrorf("testRead key:[%s] %s", req.Key,
|
||||
firstReply.LogMessage(firstReply.GetOpMsg(), conn.RemoteAddr().String(), firstReply.StartT, err))
|
||||
return
|
||||
}
|
||||
|
||||
// reply data to client
|
||||
end := offset + int64(req.Size_)
|
||||
for {
|
||||
err = nil
|
||||
reply := proto.NewPacket()
|
||||
reply.ReqID = p.ReqID
|
||||
reply.StartT = p.StartT
|
||||
|
||||
var bufOnce sync.Once
|
||||
buf, bufErr := proto.Buffers.Get(util.PageSize)
|
||||
bufRelease := func() {
|
||||
bufOnce.Do(func() {
|
||||
if bufErr == nil {
|
||||
proto.Buffers.Put(reply.Data[:util.PageSize])
|
||||
}
|
||||
})
|
||||
}
|
||||
if bufErr != nil {
|
||||
buf = make([]byte, util.PageSize)
|
||||
}
|
||||
reply.Data = buf
|
||||
|
||||
alignedOffset := offset / util.PageSize * util.PageSize
|
||||
reply.KernelOffset = uint64(offset)
|
||||
reply.ExtentOffset = offset - alignedOffset
|
||||
p.Size = util.PageSize
|
||||
p.ExtentOffset = offset
|
||||
|
||||
reply.CRC, err = block.Read(ctx, reply.Data[:], alignedOffset, util.PageSize, f.waitForCacheBlock, true)
|
||||
if err != nil {
|
||||
bufRelease()
|
||||
return
|
||||
}
|
||||
p.CRC = reply.CRC
|
||||
realNeedSize := uint32(util.Min(int(util.PageSize-reply.ExtentOffset), int(end-offset)))
|
||||
|
||||
reply.Size = realNeedSize
|
||||
reply.ResultCode = proto.OpOk
|
||||
reply.Opcode = p.Opcode
|
||||
p.ResultCode = proto.OpOk
|
||||
|
||||
bgTime := stat.BeginStat()
|
||||
if err = reply.WriteToConnForOCS(conn); err != nil {
|
||||
bufRelease()
|
||||
log.LogErrorf("%s key:[%s] %s", action, req.Key,
|
||||
reply.LogMessage(reply.GetOpMsg(), conn.RemoteAddr().String(), reply.StartT, err))
|
||||
return
|
||||
}
|
||||
stat.EndStat("HitCacheRead:ReplyToClient", err, bgTime, 1)
|
||||
offset = alignedOffset + util.PageSize
|
||||
bufRelease()
|
||||
if log.EnableInfo() {
|
||||
log.LogInfof("%s ReqID[%d] key:[%s] reply[%s] block[%s]", action, p.ReqID, req.Key,
|
||||
reply.LogMessage(reply.GetOpMsg(), conn.RemoteAddr().String(), reply.StartT, err), block.String())
|
||||
}
|
||||
if uint64(offset) >= req.Offset+req.Size_ {
|
||||
break
|
||||
}
|
||||
}
|
||||
p.PacketOkReply()
|
||||
return
|
||||
}
|
||||
|
||||
func (f *FlashNode) opCachePrepare(conn net.Conn, p *proto.Packet) (err error) {
|
||||
action := "action[opCachePrepare]"
|
||||
var volume string
|
||||
|
||||
@ -55,6 +55,7 @@ func testTCP(t *testing.T) {
|
||||
t.Run("ManualScan", testTCPManualScan)
|
||||
t.Run("CachePutBlock", testTCPCachePutBlock)
|
||||
t.Run("CacheDelete", testTCPCacheDelete)
|
||||
t.Run("CacheReadObject", testTCPObjectCacheRead)
|
||||
}
|
||||
|
||||
func testTCPHeartbeat(t *testing.T) {
|
||||
@ -297,3 +298,41 @@ func testShouldCache(t *testing.T) {
|
||||
require.Equal(t, countA, int64(2))
|
||||
require.Equal(t, countB, int64(8))
|
||||
}
|
||||
|
||||
func testTCPObjectCacheRead(t *testing.T) {
|
||||
conn := newTCPConn(t)
|
||||
defer conn.Close()
|
||||
p := proto.NewPacketReqID()
|
||||
r := proto.NewPacket()
|
||||
flashServer.readLimiter.SetBurst(0)
|
||||
flashServer.readLimiter.SetLimit(0)
|
||||
time.Sleep(time.Second)
|
||||
p.Opcode = proto.OpFlashNodeCacheReadObject
|
||||
require.NoError(t, p.WriteToConn(conn))
|
||||
require.NoError(t, r.ReadFromConn(conn, 3))
|
||||
require.Equal(t, proto.OpErr, r.ResultCode) // Read limited
|
||||
|
||||
flashServer.readLimiter.SetBurst(200)
|
||||
flashServer.readLimiter.SetLimit(20)
|
||||
time.Sleep(time.Second)
|
||||
require.NoError(t, p.WriteToConn(conn))
|
||||
require.NoError(t, r.ReadFromConn(conn, 3))
|
||||
require.Equal(t, proto.OpErr, r.ResultCode) // CacheReadRequestBase is nil
|
||||
|
||||
p.Size = 1
|
||||
p.Data = []byte{'{'}
|
||||
require.NoError(t, p.WriteToConn(conn))
|
||||
require.NoError(t, r.ReadFromConn(conn, 3))
|
||||
require.Equal(t, proto.OpErr, r.ResultCode) // CacheReadRequestBase invalid
|
||||
|
||||
req := new(proto.CacheReadRequestBase)
|
||||
req.Key = "1234567"
|
||||
req.Slot = 12345678
|
||||
req.Offset = 0
|
||||
req.Size_ = 1024
|
||||
req.TTL = _ttl
|
||||
p.MarshalDataPb(req)
|
||||
require.NoError(t, p.WriteToConn(conn))
|
||||
require.NoError(t, r.ReadFromConn(conn, 3))
|
||||
require.Equal(t, proto.OpErr, r.ResultCode)
|
||||
}
|
||||
|
||||
@ -1,9 +1,10 @@
|
||||
package remotecache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
@ -11,6 +12,7 @@ import (
|
||||
"runtime/debug"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/cubefs/cubefs/proto"
|
||||
@ -19,7 +21,6 @@ import (
|
||||
"github.com/cubefs/cubefs/util/auditlog"
|
||||
"github.com/cubefs/cubefs/util/btree"
|
||||
"github.com/cubefs/cubefs/util/bytespool"
|
||||
"github.com/cubefs/cubefs/util/errors"
|
||||
"github.com/cubefs/cubefs/util/exporter"
|
||||
"github.com/cubefs/cubefs/util/iputil"
|
||||
"github.com/cubefs/cubefs/util/log"
|
||||
@ -118,15 +119,12 @@ func NewRemoteCacheClient(masters []string, blockSize uint64) (rc *RemoteCacheCl
|
||||
rc.wg.Add(1)
|
||||
go rc.refresh()
|
||||
|
||||
// rc.Started = true
|
||||
|
||||
log.LogDebugf("NewRemoteCacheClient sucess")
|
||||
|
||||
return rc, err
|
||||
}
|
||||
|
||||
func (rc *RemoteCacheClient) Stop() {
|
||||
// rc.Started = false
|
||||
close(rc.stopC)
|
||||
rc.conns.Close()
|
||||
rc.wg.Wait()
|
||||
@ -411,73 +409,16 @@ func (rc *RemoteCacheClient) getMinFlashGroup() (*FlashGroup, uint32) {
|
||||
return nil, 0
|
||||
}
|
||||
|
||||
func (rc *RemoteCacheClient) getFlashGroup(key string, fixedFileOffset uint64) (uint32, *FlashGroup, uint32) {
|
||||
slot := proto.ComputeCacheBlockSlot(key, 0, fixedFileOffset)
|
||||
func (rc *RemoteCacheClient) getFlashGroupByKey(key string) (uint32, *FlashGroup, uint32) {
|
||||
slot := proto.ComputeCacheBlockSlot(key, 0, 0)
|
||||
fg, ownerSlot := rc.GetFlashGroupBySlot(slot)
|
||||
return slot, fg, ownerSlot
|
||||
}
|
||||
|
||||
func (rc *RemoteCacheClient) readFromRemoteCache(ctx context.Context, key string, offset, size uint64, cReadRequests []*CacheReadRequest) (total int, err error) {
|
||||
metric := exporter.NewTPCnt("RemoteCacheClient:readFromRemoteCache")
|
||||
metricBytes := exporter.NewCounter("RemoteCacheClient:readFromRemoteCacheBytes")
|
||||
defer func() {
|
||||
metric.SetWithLabels(err, map[string]string{"key": key})
|
||||
metricBytes.AddWithLabels(int64(total), map[string]string{"key": key})
|
||||
}()
|
||||
|
||||
var read int
|
||||
for _, req := range cReadRequests {
|
||||
if len(req.CacheRequest.Sources) == 0 {
|
||||
total += int(req.Size_)
|
||||
continue
|
||||
}
|
||||
slot, fg, ownerSlot := rc.getFlashGroup(req.CacheRequest.Volume, req.CacheRequest.FixedFileOffset)
|
||||
if fg == nil {
|
||||
err = fmt.Errorf("readFromRemoteCache failed: cannot find any flashGroups")
|
||||
return
|
||||
}
|
||||
req.CacheRequest.Slot = uint64(slot)<<32 | uint64(ownerSlot)
|
||||
// if read, err = s.client.RemoteCache.Read(ctx, fg, s.inode, req); err != nil {
|
||||
if read, err = rc.Read(ctx, fg, 0, req); err != nil {
|
||||
if !proto.IsFlashNodeLimitError(err) {
|
||||
log.LogWarnf("readFromRemoteCache: flashGroup read failed. offset(%v) size(%v) fg(%v) req(%v) err(%v)", offset, size, fg, req, err)
|
||||
}
|
||||
return
|
||||
} else {
|
||||
log.LogDebugf("readFromRemoteCache: inode(%d) cacheReadRequest version %v, source %v",
|
||||
req.CacheRequest.Inode, req.CacheRequest.Version, req.CacheRequest.Sources)
|
||||
total += read
|
||||
}
|
||||
}
|
||||
log.LogDebugf("readFromRemoteCache: cacheReadRequests(%v) offset(%v) size(%v) total(%v)", cReadRequests, offset, size, total)
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (rc *RemoteCacheClient) Get(ctx context.Context, reqId, key string, from, to int64) (r io.ReadCloser, length int, shouldCache bool, err error) {
|
||||
size := to - from
|
||||
data := make([]byte, size)
|
||||
|
||||
var cacheReadRequests []*CacheReadRequest
|
||||
cacheReadRequests, err = rc.prepareCacheRequests(key, uint64(from), uint64(size), data)
|
||||
|
||||
if err == nil {
|
||||
var read int
|
||||
if read, err = rc.readFromRemoteCache(ctx, key, uint64(from), uint64(size), cacheReadRequests); err == nil {
|
||||
remoteCacheHitMetric := exporter.NewCounter("readRemoteCacheHit")
|
||||
remoteCacheHitMetric.AddWithLabels(1, map[string]string{"key": key})
|
||||
return io.NopCloser(bytes.NewReader(data[:read])), read, false, err
|
||||
} else {
|
||||
log.LogWarnf("RemoteCacheClient:Get failed, err(%v)", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
|
||||
func (rc *RemoteCacheClient) Put(ctx context.Context, reqId, key string, r io.Reader, length int64) (err error) {
|
||||
if length <= 0 {
|
||||
log.LogWarnf("put data length is less and equal 0")
|
||||
return nil
|
||||
if length <= 0 || length > proto.CACHE_OBJECT_BLOCK_SIZE {
|
||||
log.LogWarnf("put data length %v is leq 0 or gt 4M", length)
|
||||
return fmt.Errorf("put data length %v is leq 0 or gt 4M", length)
|
||||
}
|
||||
slot := proto.ComputeCacheBlockSlot(key, 0, 0)
|
||||
fg, _ := rc.GetFlashGroupBySlot(slot)
|
||||
@ -585,15 +526,15 @@ type CacheReadRequest struct {
|
||||
Data []byte
|
||||
}
|
||||
|
||||
func NewRemoteCachePacket(reqId int64, opcode uint8) *proto.Packet {
|
||||
func NewRemoteCachePacket(reqId string, opcode uint8) *proto.Packet {
|
||||
p := new(proto.Packet)
|
||||
p.Magic = proto.ProtoMagic
|
||||
if reqId == 0 {
|
||||
p.ReqID = proto.GenerateRequestID()
|
||||
} else {
|
||||
p.ReqID = reqId
|
||||
}
|
||||
p.ReqID = proto.GenerateRequestID()
|
||||
p.Opcode = opcode
|
||||
if len(reqId) != 0 {
|
||||
p.Arg = []byte(reqId)
|
||||
p.ArgLen = uint32(len(reqId))
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
@ -603,29 +544,6 @@ func NewFlashCacheReply() *proto.Packet {
|
||||
return p
|
||||
}
|
||||
|
||||
func (rc *RemoteCacheClient) prepareCacheRequests(key string, offset, size uint64, data []byte) ([]*CacheReadRequest, error) {
|
||||
bgTime := stat.BeginStat()
|
||||
defer func() {
|
||||
stat.EndStat("prepareCacheRequests", nil, bgTime, 1)
|
||||
}()
|
||||
|
||||
var (
|
||||
cReadRequests []*CacheReadRequest
|
||||
cRequests = make([]*proto.CacheRequest, 0)
|
||||
)
|
||||
for fixedOff := offset / rc.blockSize * rc.blockSize; fixedOff < offset+size; fixedOff += rc.blockSize {
|
||||
cReq := &proto.CacheRequest{
|
||||
Volume: key,
|
||||
FixedFileOffset: fixedOff,
|
||||
TTL: rc.TTL,
|
||||
}
|
||||
cRequests = append(cRequests, cReq)
|
||||
}
|
||||
cReadRequests = rc.GetCacheReadRequests(offset, size, data, cRequests)
|
||||
log.LogDebugf("prepareCacheRequests: extent[offset=%v,size=%v] cReadRequests %v ", offset, size, cReadRequests)
|
||||
return cReadRequests, nil
|
||||
}
|
||||
|
||||
func (rc *RemoteCacheClient) GetCacheReadRequests(offset uint64, size uint64, data []byte, cRequests []*proto.CacheRequest) (cReadRequests []*CacheReadRequest) {
|
||||
cReadRequests = make([]*CacheReadRequest, 0, len(cRequests))
|
||||
startFixedOff := offset / rc.blockSize * rc.blockSize
|
||||
@ -680,7 +598,7 @@ func (rc *RemoteCacheClient) Read(ctx context.Context, fg *FlashGroup, reqId int
|
||||
log.LogWarnf("FlashGroup Read failed: fg(%v) err(%v)", fg, err)
|
||||
return
|
||||
}
|
||||
reqPacket = NewRemoteCachePacket(reqId, proto.OpFlashNodeCacheRead)
|
||||
reqPacket = NewRemoteCachePacket("", proto.OpFlashNodeCacheRead)
|
||||
if err = reqPacket.MarshalDataPb(&req.CacheReadRequest); err != nil {
|
||||
log.LogWarnf("FlashGroup Read: failed to MarshalData (%+v). err(%v)", req, err)
|
||||
return
|
||||
@ -737,7 +655,7 @@ func (rc *RemoteCacheClient) Prepare(ctx context.Context, fg *FlashGroup, req *p
|
||||
log.LogWarnf("FlashGroup prepare failed: err(%v)", err)
|
||||
return
|
||||
}
|
||||
reqPacket := NewRemoteCachePacket(0, proto.OpFlashNodeCachePrepare)
|
||||
reqPacket := NewRemoteCachePacket("", proto.OpFlashNodeCachePrepare)
|
||||
if err = reqPacket.MarshalDataPb(req); err != nil {
|
||||
log.LogWarnf("FlashGroup Prepare: failed to MarshalData (%v), err(%v)", req, err)
|
||||
return
|
||||
@ -802,3 +720,266 @@ func (rc *RemoteCacheClient) getReadReply(conn *net.TCPConn, reqPacket *proto.Pa
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (rc *RemoteCacheClient) Get(ctx context.Context, key string, reqId string, from, to int64) (r io.ReadCloser, length int64, shouldCache bool, err error) {
|
||||
log.LogDebugf("RemoteCacheClient Get: key(%v) reqId(%v) from(%v) to(%v)", key, reqId, from, to)
|
||||
if from < 0 || to-from <= 0 {
|
||||
return nil, 0, false, fmt.Errorf("invalid range: from(%v) to(%v)", from, to)
|
||||
}
|
||||
remoteCacheMetric := exporter.NewCounter("readRemoteCache")
|
||||
remoteCacheMetric.AddWithLabels(1, map[string]string{"key": key})
|
||||
if r, length, err = rc.readObjectFromRemoteCache(ctx, key, reqId, uint64(from), uint64(to-from)); err == nil {
|
||||
remoteCacheHitMetric := exporter.NewCounter("readRemoteCacheHit")
|
||||
remoteCacheHitMetric.AddWithLabels(1, map[string]string{"key": key})
|
||||
} else if errors.Is(err, proto.ErrorNotExistShouldCache) {
|
||||
shouldCache = true
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (rc *RemoteCacheClient) readObjectFromRemoteCache(ctx context.Context, key string, reqId string, offset, size uint64) (reader *RemoteCacheReader, length int64, err error) {
|
||||
metric := exporter.NewTPCnt("RemoteCacheClient:readObjectFromRemoteCache")
|
||||
metricBytes := exporter.NewCounter("RemoteCacheClient:readFromRemoteCacheBytes")
|
||||
defer func() {
|
||||
metric.SetWithLabels(err, map[string]string{"key": key})
|
||||
metricBytes.AddWithLabels(length, map[string]string{"key": key})
|
||||
}()
|
||||
|
||||
slot, fg, ownerSlot := rc.getFlashGroupByKey(key)
|
||||
if fg == nil {
|
||||
err = fmt.Errorf("readObjectFromRemoteCache failed: cannot find any flashGroups")
|
||||
return
|
||||
}
|
||||
|
||||
var req proto.CacheReadRequestBase
|
||||
req.Key = key
|
||||
req.TTL = rc.TTL
|
||||
req.Slot = uint64(slot)<<32 | uint64(ownerSlot)
|
||||
req.Offset = offset
|
||||
req.Size_ = size
|
||||
|
||||
if reader, length, err = rc.ReadObject(ctx, fg, reqId, &req); err != nil {
|
||||
if !proto.IsFlashNodeLimitError(err) {
|
||||
log.LogWarnf("readObjectFromRemoteCache: flashGroup read failed. offset(%v) size(%v) fg(%v) req(%v) reqId(%v) err(%v)", offset, size, fg, req, reqId, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
log.LogDebugf("readObjectFromRemoteCache: cacheReadRequestBase(%v) key(%v) reqId(%v)", req, key, reqId)
|
||||
return
|
||||
}
|
||||
|
||||
func (reader *RemoteCacheReader) getReadObjectReply(p *proto.Packet) (length int64, err error) {
|
||||
header, err := proto.Buffers.Get(util.PacketHeaderSize)
|
||||
if err != nil {
|
||||
header = make([]byte, util.PacketHeaderSize)
|
||||
}
|
||||
defer proto.Buffers.Put(header)
|
||||
var n int
|
||||
if n, err = io.ReadFull(reader.conn, header); err != nil {
|
||||
return
|
||||
}
|
||||
if n != util.PacketHeaderSize {
|
||||
return 0, syscall.EBADMSG
|
||||
}
|
||||
|
||||
if err = p.UnmarshalHeader(header); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err = p.TryReadExtraFieldsFromConn(reader.conn); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if p.ArgLen > 0 {
|
||||
p.Arg = make([]byte, int(p.ArgLen))
|
||||
if _, err = io.ReadFull(reader.conn, p.Arg[:int(p.ArgLen)]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
if p.ResultCode != proto.OpOk {
|
||||
size := p.Size
|
||||
p.Data = make([]byte, size)
|
||||
if _, err = io.ReadFull(reader.conn, p.Data[:size]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
err = fmt.Errorf("%v", string(p.Data))
|
||||
if !proto.IsFlashNodeLimitError(err) {
|
||||
log.LogWarnf("getReadObjectReply: ResultCode NOK, err(%v) ResultCode(%v)", err, p.ResultCode)
|
||||
}
|
||||
return
|
||||
}
|
||||
return int64(p.Size), nil
|
||||
}
|
||||
|
||||
type RemoteCacheReader struct {
|
||||
conn *net.TCPConn
|
||||
reader *bufio.Reader
|
||||
rc *RemoteCacheClient
|
||||
}
|
||||
|
||||
func (rc *RemoteCacheClient) NewRemoteCacheReader(conn *net.TCPConn) *RemoteCacheReader {
|
||||
r := &RemoteCacheReader{
|
||||
conn: conn,
|
||||
reader: bufio.NewReaderSize(conn, 4096),
|
||||
rc: rc,
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func (reader *RemoteCacheReader) Read(p []byte) (n int, err error) {
|
||||
reply := new(proto.Packet)
|
||||
var expectLen int64
|
||||
expectLen, err = reader.getReadObjectReply(reply)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
buff := bytespool.Alloc(proto.PageSize)
|
||||
defer bytespool.Free(buff)
|
||||
|
||||
_, err = io.ReadFull(reader.conn, buff)
|
||||
if err != nil {
|
||||
log.LogErrorf("RemoteCacheReader:Read err(%v)", err)
|
||||
return
|
||||
}
|
||||
|
||||
// check crc
|
||||
actualCrc := crc32.ChecksumIEEE(buff)
|
||||
if actualCrc != reply.CRC {
|
||||
err = fmt.Errorf("inconsistent CRC, offset(%v) extentOffset(%v) expect(%v) actualCrc(%v)", reply.KernelOffset, reply.ExtentOffset, reply.CRC, actualCrc)
|
||||
log.LogErrorf("RemoteCacheReader:Read check crc failed offset(%v) extentOffset(%v) expect(%v) actualCrc(%v)",
|
||||
reply.KernelOffset, reply.ExtentOffset, reply.CRC, actualCrc)
|
||||
return
|
||||
}
|
||||
|
||||
log.LogDebugf("RemoteCacheReader:Read offset(%v) extentOffset(%v) expectLen(%v) p.len(%v)", reply.KernelOffset, reply.ExtentOffset, expectLen, len(p))
|
||||
|
||||
extentOffset := reply.ExtentOffset
|
||||
copy(p[:expectLen], buff[extentOffset:extentOffset+expectLen])
|
||||
|
||||
return int(expectLen), nil
|
||||
}
|
||||
|
||||
func (reader *RemoteCacheReader) Close() error {
|
||||
log.LogDebugf("RemoteCacheReader Close, addr(%v)", reader.conn.RemoteAddr())
|
||||
reader.rc.conns.PutConnect(reader.conn, false)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (rc *RemoteCacheClient) ReadObjectFirstReply(conn *net.TCPConn, p *proto.Packet) (length int64, err error) {
|
||||
header, err := proto.Buffers.Get(util.PacketHeaderSize)
|
||||
if err != nil {
|
||||
header = make([]byte, util.PacketHeaderSize)
|
||||
}
|
||||
defer proto.Buffers.Put(header)
|
||||
var n int
|
||||
if n, err = io.ReadFull(conn, header); err != nil {
|
||||
return
|
||||
}
|
||||
if n != util.PacketHeaderSize {
|
||||
return 0, syscall.EBADMSG
|
||||
}
|
||||
|
||||
if err = p.UnmarshalHeader(header); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err = p.TryReadExtraFieldsFromConn(conn); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if p.ArgLen > 0 {
|
||||
p.Arg = make([]byte, int(p.ArgLen))
|
||||
if _, err = io.ReadFull(conn, p.Arg[:int(p.ArgLen)]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
if p.ResultCode != proto.OpOk {
|
||||
size := p.Size
|
||||
p.Data = make([]byte, size)
|
||||
if _, err = io.ReadFull(conn, p.Data[:size]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
err = fmt.Errorf("%v", string(p.Data))
|
||||
if !proto.IsFlashNodeLimitError(err) {
|
||||
log.LogWarnf("ReadObjectFirstReply: ResultCode NOK, err(%v) ResultCode(%v)", err, p.ResultCode)
|
||||
}
|
||||
return
|
||||
}
|
||||
return int64(p.Size), nil
|
||||
}
|
||||
|
||||
func (rc *RemoteCacheClient) ReadObject(ctx context.Context, fg *FlashGroup, reqId string, req *proto.CacheReadRequestBase) (reader *RemoteCacheReader, length int64, err error) {
|
||||
var (
|
||||
conn *net.TCPConn
|
||||
moved bool
|
||||
addr string
|
||||
reqPacket *proto.Packet
|
||||
)
|
||||
bgTime := stat.BeginStat()
|
||||
defer func() {
|
||||
if err != nil && strings.Contains(err.Error(), "timeout") {
|
||||
err = fmt.Errorf("read timeout")
|
||||
}
|
||||
parts := strings.Split(addr, ":")
|
||||
if len(parts) > 0 && addr != "" {
|
||||
stat.EndStat(fmt.Sprintf("flashNode:%v", parts[0]), err, bgTime, 1)
|
||||
}
|
||||
stat.EndStat("flashNode", err, bgTime, 1)
|
||||
}()
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return nil, 0, ctx.Err()
|
||||
}
|
||||
addr = fg.getFlashHost()
|
||||
if addr == "" {
|
||||
err = fmt.Errorf("no available host")
|
||||
log.LogWarnf("FlashGroup Read failed: fg(%v) err(%v)", fg, err)
|
||||
return
|
||||
}
|
||||
reqPacket = NewRemoteCachePacket(reqId, proto.OpFlashNodeCacheReadObject)
|
||||
if err = reqPacket.MarshalDataPb(req); err != nil {
|
||||
log.LogWarnf("FlashGroup Read: failed to MarshalData (%+v). err(%v)", req, err)
|
||||
return
|
||||
}
|
||||
if conn, err = rc.conns.GetConnect(addr); err != nil {
|
||||
log.LogWarnf("FlashGroup Read: get connection failed, addr(%v) reqPacket(%v) err(%v) remoteCacheMultiRead(%v)", addr, req, err, rc.RemoteCacheMultiRead)
|
||||
moved = fg.moveToUnknownRank(addr, err, rc.FlashNodeTimeoutCount)
|
||||
if rc.RemoteCacheMultiRead {
|
||||
log.LogInfof("Retrying due to GetConnect of addr(%v) failure err(%v)", addr, err)
|
||||
continue
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if err = reqPacket.WriteToConn(conn); err != nil {
|
||||
log.LogWarnf("FlashGroup Read: failed to write to addr(%v) err(%v) remoteCacheMultiRead(%v)", addr, err, rc.RemoteCacheMultiRead)
|
||||
rc.conns.PutConnect(conn, err != nil)
|
||||
moved = fg.moveToUnknownRank(addr, err, rc.FlashNodeTimeoutCount)
|
||||
if rc.RemoteCacheMultiRead {
|
||||
log.LogInfof("Retrying due to write to addr(%v) failure err(%v)", addr, err)
|
||||
continue
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
reply := new(proto.Packet)
|
||||
length, err = rc.ReadObjectFirstReply(conn, reply)
|
||||
if err != nil {
|
||||
log.LogWarnf("ReadObject getReadObjectReply from(%v) failed, reply(%v) error(%v)", conn.RemoteAddr(), reply, err)
|
||||
rc.conns.PutConnect(conn, true)
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
reader = rc.NewRemoteCacheReader(conn)
|
||||
break
|
||||
}
|
||||
|
||||
log.LogDebugf("FlashGroup Read: flashGroup(%v) addr(%v) CacheReadRequest(%v) reqPacket(%v) err(%v) moved(%v) remoteCacheMultiRead(%v)",
|
||||
fg, addr, req, reqPacket, err, moved, rc.RemoteCacheMultiRead)
|
||||
return
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user
