gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 02:00:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(master): hybrid cloud Support querying the inode count and used size of volumes and metapartitions by storage class.

Browse Source 
Signed-off-by: true1064 <true1063@163.com>
This commit is contained in:
true1064 2024-02-20 19:29:56 +08:00 committed by AmazingChi
parent 21d4c73847
commit d3f1d3d734
10 changed files with 259 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
master/api_service.go
View File

@ -5390,6 +5390,7 @@ func volStat(vol *Vol, countByMeta bool) (stat *proto.VolStatInfo) {
stat.TrashInterval = vol.TrashInterval
stat.DefaultStorageClass = vol.volStorageClass
stat.CacheDpStorageClass = vol.cacheDpStorageClass
stat.StatByStorageClass = vol.StatByStorageClass
log.LogDebugf("[volStat] vol[%v] total[%v],usedSize[%v] TrashInterval[%v] DefaultStorageClass[%v]",
vol.Name, stat.TotalSize, stat.UsedSize, stat.TrashInterval, stat.DefaultStorageClass)
if proto.IsHot(vol.VolType) {
@ -5479,25 +5480,26 @@ func (m *Server) getMetaPartition(w http.ResponseWriter, r *http.Request) {
log.LogErrorf("action[getMetaPartition]failed to get volume %v, err %v", mp.volName, err)
}
mpInfo := &proto.MetaPartitionInfo{
PartitionID: mp.PartitionID,
Start: mp.Start,
End: mp.End,
VolName: mp.volName,
MaxInodeID: mp.MaxInodeID,
InodeCount: mp.InodeCount,
DentryCount: mp.DentryCount,
Replicas: replicas,
ReplicaNum: mp.ReplicaNum,
Status: mp.Status,
IsRecover: mp.IsRecover,
Hosts: mp.Hosts,
Peers: mp.Peers,
Zones: zones,
NodeSets: nodeSets,
MissNodes: mp.MissNodes,
OfflinePeerID: mp.OfflinePeerID,
LoadResponse: mp.LoadResponse,
Forbidden: forbidden,
PartitionID: mp.PartitionID,
Start: mp.Start,
End: mp.End,
VolName: mp.volName,
MaxInodeID: mp.MaxInodeID,
InodeCount: mp.InodeCount,
DentryCount: mp.DentryCount,
Replicas: replicas,
ReplicaNum: mp.ReplicaNum,
Status: mp.Status,
IsRecover: mp.IsRecover,
Hosts: mp.Hosts,
Peers: mp.Peers,
Zones: zones,
NodeSets: nodeSets,
MissNodes: mp.MissNodes,
OfflinePeerID: mp.OfflinePeerID,
LoadResponse: mp.LoadResponse,
Forbidden: forbidden,
StatByStorageClass: mp.StatByStorageClass,
}
return mpInfo
}

146
master/meta_partition.go
View File

@ -28,60 +28,62 @@ import (
// MetaReplica defines the replica of a meta partition
type MetaReplica struct {
Addr string
start uint64 // lower bound of the inode id
end uint64 // upper bound of the inode id
dataSize uint64
nodeID uint64
MaxInodeID uint64
InodeCount uint64
DentryCount uint64
TxCnt uint64
TxRbInoCnt uint64
TxRbDenCnt uint64
FreeListLen uint64
ReportTime int64
Status int8 // unavailable, readOnly, readWrite
IsLeader bool
metaNode *MetaNode
Addr string
start uint64 // lower bound of the inode id
end uint64 // upper bound of the inode id
dataSize uint64
nodeID uint64
MaxInodeID uint64
InodeCount uint64
DentryCount uint64
TxCnt uint64
TxRbInoCnt uint64
TxRbDenCnt uint64
FreeListLen uint64
ReportTime int64
Status int8 // unavailable, readOnly, readWrite
IsLeader bool
StatByStorageClass []*proto.StatOftorageClass
metaNode *MetaNode
}
// MetaPartition defines the structure of a meta partition
type MetaPartition struct {
PartitionID uint64
Start uint64
End uint64
MaxInodeID uint64
InodeCount uint64
DentryCount uint64
FreeListLen uint64
TxCnt uint64
TxRbInoCnt uint64
TxRbDenCnt uint64
Replicas []*MetaReplica
LeaderReportTime int64
ReplicaNum uint8
Status int8
IsRecover bool
volID uint64
volName string
Hosts []string
Peers []proto.Peer
OfflinePeerID uint64
MissNodes map[string]int64
LoadResponse []*proto.MetaPartitionLoadResponse
offlineMutex sync.RWMutex
uidInfo []*proto.UidReportSpaceInfo
EqualCheckPass bool
VerSeq uint64
heartBeatDone bool
PartitionID uint64
Start uint64
End uint64
MaxInodeID uint64
InodeCount uint64
DentryCount uint64
FreeListLen uint64
TxCnt uint64
TxRbInoCnt uint64
TxRbDenCnt uint64
Replicas []*MetaReplica
LeaderReportTime int64
ReplicaNum uint8
Status int8
IsRecover bool
volID uint64
volName string
Hosts []string
Peers []proto.Peer
OfflinePeerID uint64
MissNodes map[string]int64
LoadResponse []*proto.MetaPartitionLoadResponse
offlineMutex sync.RWMutex
uidInfo []*proto.UidReportSpaceInfo
EqualCheckPass bool
VerSeq uint64
heartBeatDone bool
StatByStorageClass []*proto.StatOftorageClass
sync.RWMutex
}
func newMetaReplica(start, end uint64, metaNode *MetaNode) (mr *MetaReplica) {
mr = &MetaReplica{start: start, end: end, nodeID: metaNode.ID, Addr: metaNode.Addr}
mr.metaNode = metaNode
mr.StatByStorageClass = make([]*proto.StatOftorageClass, 0)
mr.ReportTime = time.Now().Unix()
return
}
@ -98,6 +100,7 @@ func newMetaPartition(partitionID, start, end uint64, replicaNum uint8, volName
mp.VerSeq = verSeq
mp.LoadResponse = make([]*proto.MetaPartitionLoadResponse, 0)
mp.EqualCheckPass = true
mp.StatByStorageClass = make([]*proto.StatOftorageClass, 0)
return
}
@ -426,6 +429,7 @@ func (mp *MetaPartition) updateMetaPartition(mgr *proto.MetaPartitionReport, met
mp.SetTxCnt()
mp.removeMissingReplica(metaNode.Addr)
mp.setUidInfo(mgr)
mp.setStatByStorageClass()
mp.setHeartBeatDone()
}
@ -749,6 +753,12 @@ func (mr *MetaReplica) updateMetric(mgr *proto.MetaPartitionReport) {
mr.TxRbDenCnt = mgr.TxRbDenCnt
mr.FreeListLen = mgr.FreeListLen
mr.dataSize = mgr.Size
if mgr.StatByStorageClass != nil {
mr.StatByStorageClass = mgr.StatByStorageClass
} else if len(mr.StatByStorageClass) != 0 {
// handle compatibility, report from old version metanode has no filed StatByStorageClass
mr.StatByStorageClass = make([]*proto.StatOftorageClass, 0)
}
mr.setLastReportTime()
if mr.metaNode.RdOnly && mr.Status == proto.ReadWrite {
@ -889,6 +899,54 @@ func (mp *MetaPartition) SetTxCnt() {
mp.TxCnt, mp.TxRbInoCnt, mp.TxRbDenCnt = txCnt, rbInoCnt, rbDenCnt
}
func (mp *MetaPartition) setStatByStorageClass() {
var mpStat *proto.StatOftorageClass
var ok bool
statByStorageClassMap := make(map[uint32]*proto.StatOftorageClass)
for _, r := range mp.Replicas {
if r.StatByStorageClass == nil {
continue
}
for _, rStat := range r.StatByStorageClass {
if mpStat, ok = statByStorageClassMap[rStat.StorageClass]; !ok {
mpStat = proto.NewStatOfStorageClass(rStat.StorageClass)
statByStorageClassMap[rStat.StorageClass] = mpStat
}
if rStat.InodeCount > mpStat.InodeCount {
mpStat.InodeCount = rStat.InodeCount
}
if rStat.UsedSizeBytes > mpStat.UsedSizeBytes {
mpStat.UsedSizeBytes = rStat.UsedSizeBytes
}
}
}
toSlice := make([]*proto.StatOftorageClass, 0)
for _, mpStat := range statByStorageClassMap {
toSlice = append(toSlice, mpStat)
}
mp.StatByStorageClass = toSlice
}
func (mp *MetaPartition) getAllNodeSets() (nodeSets []uint64) {
mp.RLock()
defer mp.RUnlock()
nodeSets = make([]uint64, 0)
for _, mr := range mp.Replicas {
if mr.metaNode == nil {
continue
}
if !containsID(nodeSets, mr.metaNode.NodeSetID) {
nodeSets = append(nodeSets, mr.metaNode.NodeSetID)
}
}
return
}
func (mp *MetaPartition) getLiveZones(offlineAddr string) (zones []string) {
mp.RLock()
defer mp.RUnlock()

29
master/vol.go
View File

@ -163,6 +163,7 @@ type Vol struct {
allowedStorageClass []uint32 // specifies which storageClasses the vol use, a cluster may have multiple StorageClasses
volStorageClass uint32 // specifies which storageClass is written, unless dirStorageClass is set in file path
cacheDpStorageClass uint32 // for SDK those access cache/preload dp of cold volume
StatByStorageClass []*proto.StatOftorageClass
}
func newVol(vv volValue) (vol *Vol) {
@ -238,6 +239,7 @@ func newVol(vv volValue) (vol *Vol) {
copy(vol.allowedStorageClass, vv.AllowedStorageClass)
vol.volStorageClass = vv.VolStorageClass
vol.cacheDpStorageClass = vv.CacheDpStorageClass
vol.StatByStorageClass = make([]*proto.StatOftorageClass, 0)
return
}
@ -806,10 +808,16 @@ func (vol *Vol) checkMetaPartitions(c *Cluster) {
metaPartitionInodeIdStep := gConfig.MetaPartitionInodeIdStep
maxPartitionID := vol.maxPartitionID()
mps := vol.cloneMetaPartitionMap()
var (
doSplit bool
err error
doSplit bool
err error
volStat *proto.StatOftorageClass
ok bool
statByStorageClassMap map[uint32]*proto.StatOftorageClass
)
statByStorageClassMap = make(map[uint32]*proto.StatOftorageClass)
for _, mp := range mps {
doSplit = mp.checkStatus(c.Name, true, int(vol.mpReplicaNum), maxPartitionID, metaPartitionInodeIdStep, vol.Forbidden)
if doSplit && !c.cfg.DisableAutoCreate {
@ -826,7 +834,24 @@ func (vol *Vol) checkMetaPartitions(c *Cluster) {
mp.checkEnd(c, maxPartitionID)
mp.reportMissingReplicas(c.Name, c.leaderInfo.addr, defaultMetaPartitionTimeOutSec, defaultIntervalToAlarmMissingMetaPartition)
tasks = append(tasks, mp.replicaCreationTasks(c.Name, vol.Name)...)
for _, mpStat := range mp.StatByStorageClass {
if volStat, ok = statByStorageClassMap[mpStat.StorageClass]; !ok {
volStat = proto.NewStatOfStorageClass(mpStat.StorageClass)
statByStorageClassMap[mpStat.StorageClass] = volStat
}
volStat.InodeCount += mpStat.InodeCount
volStat.UsedSizeBytes += mpStat.UsedSizeBytes
}
}
StatOfStorageClassSlice := make([]*proto.StatOftorageClass, 0)
for _, volStat = range statByStorageClassMap {
StatOfStorageClassSlice = append(StatOfStorageClassSlice, volStat)
}
vol.StatByStorageClass = StatOfStorageClassSlice
c.addMetaNodeTasks(tasks)
vol.checkSplitMetaPartition(c, metaPartitionInodeIdStep)
}

26
metanode/manager_op.go
View File

@ -115,19 +115,19 @@ func (m *metadataManager) opMasterHeartbeat(conn net.Conn, p *Packet,
mConf := partition.GetBaseConfig()
mpr := &proto.MetaPartitionReport{
PartitionID: mConf.PartitionId,
Start: mConf.Start,
End: mConf.End,
Status: proto.ReadWrite,
MaxInodeID: mConf.Cursor,
VolName: mConf.VolName,
Size: partition.DataSize(),
InodeCnt: uint64(partition.GetInodeTreeLen()),
DentryCnt: uint64(partition.GetDentryTreeLen()),
FreeListLen: uint64(partition.GetFreeListLen()),
UidInfo: partition.GetUidInfo(),
QuotaReportInfos: partition.getQuotaReportInfos(),
StorageTypes: partition.GetStorageTypes(),
PartitionID: mConf.PartitionId,
Start: mConf.Start,
End: mConf.End,
Status: proto.ReadWrite,
MaxInodeID: mConf.Cursor,
VolName: mConf.VolName,
Size: partition.DataSize(),
InodeCnt: uint64(partition.GetInodeTreeLen()),
DentryCnt: uint64(partition.GetDentryTreeLen()),
FreeListLen: uint64(partition.GetFreeListLen()),
UidInfo: partition.GetUidInfo(),
QuotaReportInfos: partition.getQuotaReportInfos(),
StatByStorageClass: partition.GetStatByStorageClass(),
}
mpr.TxCnt, mpr.TxRbInoCnt, mpr.TxRbDenCnt = partition.TxGetCnt()

30
metanode/multi_ver_test.go
View File

@ -66,21 +66,21 @@ var tlog *testing.T
func newPartition(conf *MetaPartitionConfig, manager *metadataManager) (mp *metaPartition) {
mp = &metaPartition{
config: conf,
dentryTree: NewBtree(),
inodeTree: NewBtree(),
extendTree: NewBtree(),
multipartTree: NewBtree(),
stopC: make(chan bool),
storeChan: make(chan *storeMsg, 100),
freeList: newFreeList(),
extDelCh: make(chan []proto.ExtentKey, defaultDelExtentsCnt),
extReset: make(chan struct{}),
vol: NewVol(),
manager: manager,
verSeq: conf.VerSeq,
storageTypes: make([]uint32, 0),
fmList: newForbiddenMigrationList(proto.ForbiddenMigrationRenewalPeriod),
config: conf,
dentryTree: NewBtree(),
inodeTree: NewBtree(),
extendTree: NewBtree(),
multipartTree: NewBtree(),
stopC: make(chan bool),
storeChan: make(chan *storeMsg, 100),
freeList: newFreeList(),
extDelCh: make(chan []proto.ExtentKey, defaultDelExtentsCnt),
extReset: make(chan struct{}),
vol: NewVol(),
manager: manager,
verSeq: conf.VerSeq,
statByStorageClass: make([]*proto.StatOftorageClass, 0),
fmList: newForbiddenMigrationList(proto.ForbiddenMigrationRenewalPeriod),
}
mp.config.Cursor = 0
mp.config.End = 100000

51
metanode/partition.go
View File

@ -285,7 +285,7 @@ type MetaPartition interface {
IsEnableAuditLog() bool
SetEnableAuditLog(status bool)
UpdateVolumeView(dataView *proto.DataPartitionsView, volumeView *proto.SimpleVolView)
GetStorageTypes() []uint32
GetStatByStorageClass() []*proto.StatOftorageClass
}
type UidManager struct {
@ -508,7 +508,7 @@ type metaPartition struct {
manager *metadataManager
isLoadingMetaPartition bool
ebsClient *blobstore.BlobStoreClient
volType int
volType int // kept in hybrid cloud for compatibility
isFollowerRead bool
uidManager *UidManager
xattrLock sync.Mutex
@ -524,6 +524,7 @@ type metaPartition struct {
enablePersistAccessTime bool
accessTimeValidInterval uint64
storageTypes []uint32
statByStorageClass []*proto.StatOftorageClass
fmList *forbiddenMigrationList
}
@ -593,38 +594,42 @@ func (mp *metaPartition) updateSize() {
select {
case <-timer.C:
size := uint64(0)
storageTypes := make([]uint32, 0)
statByStorageClassMap := make(map[uint32]*proto.StatOftorageClass)
var statOfStorageClass *proto.StatOftorageClass
var ok bool
mp.inodeTree.GetTree().Ascend(func(item BtreeItem) bool {
inode := item.(*Inode)
size += inode.Size
storageTypes = append(storageTypes, inode.StorageClass)
if statOfStorageClass, ok = statByStorageClassMap[inode.StorageClass]; !ok {
statOfStorageClass = proto.NewStatOfStorageClass(inode.StorageClass)
statByStorageClassMap[inode.StorageClass] = statOfStorageClass
}
statOfStorageClass.InodeCount++
statOfStorageClass.UsedSizeBytes += inode.Size
return true
})
mp.size = size
mp.storageTypes = uniqueUint32Slice(storageTypes)
log.LogDebugf("[updateSize] update mp[%v] size(%d) success,inodeCount(%d),dentryCount(%d)", mp.config.PartitionId, size, mp.inodeTree.Len(), mp.dentryTree.Len())
toSlice := make([]*proto.StatOftorageClass, 0)
for _, stat := range statByStorageClassMap {
toSlice = append(toSlice, stat)
}
mp.statByStorageClass = toSlice
log.LogDebugf("[updateSize] update mp(%d) size(%d) success, inodeCount(%d), dentryCount(%d)",
mp.config.PartitionId, size, mp.inodeTree.Len(), mp.dentryTree.Len())
case <-mp.stopC:
log.LogDebugf("[updateSize] stop update mp[%v] size,inodeCount(%d),dentryCount(%d)", mp.config.PartitionId, mp.inodeTree.Len(), mp.dentryTree.Len())
log.LogDebugf("[updateSize] stop update mp[%v] size, inodeCount(%d), dentryCount(%d)",
mp.config.PartitionId, mp.inodeTree.Len(), mp.dentryTree.Len())
return
}
}
}()
}
func uniqueUint32Slice(input []uint32) []uint32 {
uniqueMap := make(map[uint32]bool)
uniqueSlice := []uint32{}
for _, num := range input {
if !uniqueMap[num] {
uniqueMap[num] = true
uniqueSlice = append(uniqueSlice, num)
}
}
return uniqueSlice
}
func (mp *metaPartition) ForceSetMetaPartitionToLoadding() {
mp.isLoadingMetaPartition = true
}
@ -1767,6 +1772,6 @@ func (mp *metaPartition) GetAccessTimeValidInterval() time.Duration {
return time.Duration(interval)
}
func (mp *metaPartition) GetStorageTypes() []uint32 {
return mp.storageTypes
func (mp *metaPartition) GetStatByStorageClass() []*proto.StatOftorageClass {
return mp.statByStorageClass
}

2
metanode/partition_fsm.go
View File

@ -982,7 +982,7 @@ func (mp *metaPartition) HandleLeaderChange(leader uint64) {
exporter.Warning(fmt.Sprintf("[HandleLeaderChange] pid %v init root inode id: %s.", mp.config.PartitionId, err.Error()))
}
ino := NewInode(id, proto.Mode(os.ModePerm|os.ModeDir))
//TODO:
//TODO:hchi,tangjingyu: decide storageClass of rootInode
//ino.StorageClass = uint32(legacyReplicaStorageClass)
ino.StorageClass = proto.StorageClass_Replica_HDD
go mp.initInode(ino)

34
proto/admin_proto.go
View File

@ -838,23 +838,23 @@ type OpLog struct {
// MetaPartitionReport defines the meta partition report.
type MetaPartitionReport struct {
PartitionID uint64
Start uint64
End uint64
Status int
Size uint64
MaxInodeID uint64
IsLeader bool
VolName string
InodeCnt uint64
DentryCnt uint64
TxCnt uint64
TxRbInoCnt uint64
TxRbDenCnt uint64
FreeListLen uint64
UidInfo []*UidReportSpaceInfo
QuotaReportInfos []*QuotaReportInfo
StorageTypes []uint32
PartitionID uint64
Start uint64
End uint64
Status int
Size uint64
MaxInodeID uint64
IsLeader bool
VolName string
InodeCnt uint64
DentryCnt uint64
TxCnt uint64
TxRbInoCnt uint64
TxRbDenCnt uint64
FreeListLen uint64
UidInfo []*UidReportSpaceInfo
QuotaReportInfos []*QuotaReportInfo
StatByStorageClass []*StatOftorageClass
}
// MetaNodeHeartbeatResponse defines the response to the meta node heartbeat request.

14
proto/meta_proto.go
View File

@ -141,3 +141,17 @@ func (quotaInfo *QuotaInfo) IsOverQuotaBytes() (isOver bool) {
}
return
}
type StatOftorageClass struct {
StorageClass uint32
InodeCount uint64
UsedSizeBytes uint64
}
func NewStatOfStorageClass(storageClass uint32) *StatOftorageClass {
return &StatOftorageClass{
StorageClass: storageClass,
InodeCount: 0,
UsedSizeBytes: 0,
}
}

40
proto/model.go
View File

@ -83,25 +83,26 @@ type DataNodeInfo struct {
// MetaPartition defines the structure of a meta partition
type MetaPartitionInfo struct {
PartitionID uint64
Start uint64
End uint64
MaxInodeID uint64
InodeCount uint64
DentryCount uint64
VolName string
Replicas []*MetaReplicaInfo
ReplicaNum uint8
Status int8
IsRecover bool
Hosts []string
Peers []Peer
Zones []string
NodeSets []uint64
OfflinePeerID uint64
MissNodes map[string]int64
LoadResponse []*MetaPartitionLoadResponse
Forbidden bool
PartitionID uint64
Start uint64
End uint64
MaxInodeID uint64
InodeCount uint64
DentryCount uint64
VolName string
Replicas []*MetaReplicaInfo
ReplicaNum uint8
Status int8
IsRecover bool
Hosts []string
Peers []Peer
Zones []string
NodeSets []uint64
OfflinePeerID uint64
MissNodes map[string]int64
LoadResponse []*MetaPartitionLoadResponse
Forbidden bool
StatByStorageClass []*StatOftorageClass
}
// MetaReplica defines the replica of a meta partition
@ -282,6 +283,7 @@ type VolStatInfo struct {
TrashInterval int64 `json:"TrashIntervalV2"`
DefaultStorageClass uint32
CacheDpStorageClass uint32
StatByStorageClass []*StatOftorageClass
}
// DataPartition represents the structure of storing the file contents.