mirror of
https://github.com/cubefs/cubefs.git
synced 2026-08-02 02:00:56 +00:00
fix(sdk): classifiy statistic file size and count for refresher tool.#22490031
Signed-off-by: baihailong <baihailong@oppo.com>
This commit is contained in:
baihailong
AmazingChi
parent
e9fb2da0b1
commit
f56ef231a1
@ -869,12 +869,23 @@ func (d *Dir) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fus
|
||||
d.super.sc.Put(ino, &summaryInfo)
|
||||
}
|
||||
|
||||
files := summaryInfo.Files
|
||||
filesHdd := summaryInfo.FilesHdd
|
||||
fileSsd := summaryInfo.FilesSsd
|
||||
filesBlobStore := summaryInfo.FilesBlobStore
|
||||
|
||||
fbytesHdd := summaryInfo.FbytesHdd
|
||||
fbytesSsd := summaryInfo.FbytesSsd
|
||||
fbytesBlobStore := summaryInfo.FbytesBlobStore
|
||||
|
||||
subdirs := summaryInfo.Subdirs
|
||||
fbytes := summaryInfo.Fbytes
|
||||
summaryStr := "Files:" + strconv.FormatInt(int64(files), 10) + "," +
|
||||
"Dirs:" + strconv.FormatInt(int64(subdirs), 10) + "," +
|
||||
"Bytes:" + strconv.FormatInt(int64(fbytes), 10)
|
||||
|
||||
summaryStr := "FilesHdd:" + strconv.FormatInt(int64(filesHdd), 10) + "," +
|
||||
"FilesSsd:" + strconv.FormatInt(int64(fileSsd), 10) + "," +
|
||||
"FilesBlobStore:" + strconv.FormatInt(int64(filesBlobStore), 10) + "," +
|
||||
"BytesHdd:" + strconv.FormatInt(int64(fbytesHdd), 10) + "," +
|
||||
"BytesSsd:" + strconv.FormatInt(int64(fbytesSsd), 10) + "," +
|
||||
"BytesBlobStore:" + strconv.FormatInt(int64(fbytesBlobStore), 10) + "," +
|
||||
"Dirs:" + strconv.FormatInt(int64(subdirs), 10)
|
||||
value = []byte(summaryStr)
|
||||
|
||||
} else {
|
||||
|
||||
@ -136,9 +136,13 @@ type (
|
||||
}
|
||||
|
||||
SummaryInfo struct {
|
||||
Files int64
|
||||
Subdirs int64
|
||||
Fbytes int64
|
||||
Subdirs int64
|
||||
FilesHdd int64
|
||||
FilesSsd int64
|
||||
FilesBlobStore int64
|
||||
FbytesHdd int64
|
||||
FbytesSsd int64
|
||||
FbytesBlobStore int64
|
||||
}
|
||||
|
||||
File struct {
|
||||
@ -926,9 +930,13 @@ func (c *Client) GetSummary(path string, useCache string, goroutineNum int32) (s
|
||||
if strings.ToLower(useCache) == "true" {
|
||||
cacheSummaryInfo := c.sc.Get(info.Inode)
|
||||
if cacheSummaryInfo != nil {
|
||||
summary.Files = cacheSummaryInfo.Files
|
||||
summary.FilesHdd = cacheSummaryInfo.FilesHdd
|
||||
summary.FilesSsd = cacheSummaryInfo.FilesSsd
|
||||
summary.FilesBlobStore = cacheSummaryInfo.FilesBlobStore
|
||||
summary.FbytesHdd = cacheSummaryInfo.FbytesHdd
|
||||
summary.FbytesSsd = cacheSummaryInfo.FbytesSsd
|
||||
summary.FbytesBlobStore = cacheSummaryInfo.FbytesBlobStore
|
||||
summary.Subdirs = cacheSummaryInfo.Subdirs
|
||||
summary.Fbytes = cacheSummaryInfo.Fbytes
|
||||
return summary, nil
|
||||
}
|
||||
}
|
||||
@ -944,9 +952,14 @@ func (c *Client) GetSummary(path string, useCache string, goroutineNum int32) (s
|
||||
if strings.ToLower(useCache) != "false" {
|
||||
c.sc.Put(info.Inode, &summaryInfo)
|
||||
}
|
||||
summary.Files = summaryInfo.Files
|
||||
|
||||
summary.FilesHdd = summaryInfo.FilesHdd
|
||||
summary.FilesSsd = summaryInfo.FilesSsd
|
||||
summary.FilesBlobStore = summaryInfo.FilesBlobStore
|
||||
summary.FbytesHdd = summaryInfo.FbytesHdd
|
||||
summary.FbytesSsd = summaryInfo.FbytesSsd
|
||||
summary.FbytesBlobStore = summaryInfo.FbytesBlobStore
|
||||
summary.Subdirs = summaryInfo.Subdirs
|
||||
summary.Fbytes = summaryInfo.Fbytes
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
|
||||
@ -48,9 +48,13 @@ struct cfs_stat_info {
|
||||
};
|
||||
|
||||
struct cfs_summary_info {
|
||||
int64_t files;
|
||||
int64_t filesHdd;
|
||||
int64_t filesSsd;
|
||||
int64_t filesBlobStore;
|
||||
int64_t fbytesHdd;
|
||||
int64_t fbytesSsd;
|
||||
int64_t fbytesBlobStore;
|
||||
int64_t subdirs;
|
||||
int64_t fbytes;
|
||||
};
|
||||
|
||||
struct cfs_dirent {
|
||||
|
||||
@ -42,9 +42,13 @@ struct cfs_stat_info {
|
||||
};
|
||||
|
||||
struct cfs_summary_info {
|
||||
int64_t files;
|
||||
int64_t filesHdd;
|
||||
int64_t filesSsd;
|
||||
int64_t filesBlobStore;
|
||||
int64_t fbytesHdd;
|
||||
int64_t fbytesSsd;
|
||||
int64_t fbytesBlobStore;
|
||||
int64_t subdirs;
|
||||
int64_t fbytes;
|
||||
};
|
||||
|
||||
struct cfs_dirent {
|
||||
@ -1370,9 +1374,13 @@ func cfs_getsummary(id C.int64_t, path *C.char, summary *C.struct_cfs_summary_in
|
||||
if strings.ToLower(C.GoString(useCache)) == "true" {
|
||||
cacheSummaryInfo := c.sc.Get(info.Inode)
|
||||
if cacheSummaryInfo != nil {
|
||||
summary.files = C.int64_t(cacheSummaryInfo.Files)
|
||||
summary.filesHdd = C.int64_t(cacheSummaryInfo.FilesHdd)
|
||||
summary.filesSsd = C.int64_t(cacheSummaryInfo.FilesSsd)
|
||||
summary.filesBlobStore = C.int64_t(cacheSummaryInfo.FilesBlobStore)
|
||||
summary.fbytesHdd = C.int64_t(cacheSummaryInfo.FbytesHdd)
|
||||
summary.fbytesSsd = C.int64_t(cacheSummaryInfo.FbytesSsd)
|
||||
summary.fbytesBlobStore = C.int64_t(cacheSummaryInfo.FbytesBlobStore)
|
||||
summary.subdirs = C.int64_t(cacheSummaryInfo.Subdirs)
|
||||
summary.fbytes = C.int64_t(cacheSummaryInfo.Fbytes)
|
||||
return statusOK
|
||||
}
|
||||
}
|
||||
@ -1388,9 +1396,14 @@ func cfs_getsummary(id C.int64_t, path *C.char, summary *C.struct_cfs_summary_in
|
||||
if strings.ToLower(C.GoString(useCache)) != "false" {
|
||||
c.sc.Put(info.Inode, &summaryInfo)
|
||||
}
|
||||
summary.files = C.int64_t(summaryInfo.Files)
|
||||
|
||||
summary.filesHdd = C.int64_t(summaryInfo.FilesHdd)
|
||||
summary.filesSsd = C.int64_t(summaryInfo.FilesSsd)
|
||||
summary.filesBlobStore = C.int64_t(summaryInfo.FilesBlobStore)
|
||||
summary.fbytesHdd = C.int64_t(summaryInfo.FbytesHdd)
|
||||
summary.fbytesSsd = C.int64_t(summaryInfo.FbytesSsd)
|
||||
summary.fbytesBlobStore = C.int64_t(summaryInfo.FbytesBlobStore)
|
||||
summary.subdirs = C.int64_t(summaryInfo.Subdirs)
|
||||
summary.fbytes = C.int64_t(summaryInfo.Fbytes)
|
||||
return statusOK
|
||||
}
|
||||
|
||||
|
||||
@ -21,36 +21,17 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/cubefs/cubefs/proto"
|
||||
"github.com/cubefs/cubefs/util/errors"
|
||||
"github.com/cubefs/cubefs/util/log"
|
||||
)
|
||||
|
||||
func (mp *metaPartition) UpdateXAttr(req *proto.UpdateXAttrRequest, p *Packet) (err error) {
|
||||
newValueList := strings.Split(req.Value, ",")
|
||||
if len(newValueList) < 3 {
|
||||
err = errors.New("Wrong number of parameters")
|
||||
log.LogErrorf("action[UpdateXAttr],Wrong number of parameters")
|
||||
p.PacketErrorWithBody(proto.OpArgMismatchErr, []byte(err.Error()))
|
||||
return
|
||||
}
|
||||
filesInc, err := strconv.ParseInt(newValueList[0], 10, 64)
|
||||
if err != nil {
|
||||
log.LogErrorf("action[UpdateXAttr],The parameter must be an integer: err(%v)", err)
|
||||
p.PacketErrorWithBody(proto.OpArgMismatchErr, []byte(err.Error()))
|
||||
return
|
||||
}
|
||||
dirsInc, err := strconv.ParseInt(newValueList[1], 10, 64)
|
||||
if err != nil {
|
||||
log.LogErrorf("action[UpdateXAttr],The parameter must be an integer: err(%v)", err)
|
||||
p.PacketErrorWithBody(proto.OpArgMismatchErr, []byte(err.Error()))
|
||||
return
|
||||
}
|
||||
bytesInc, err := strconv.ParseInt(newValueList[2], 10, 64)
|
||||
if err != nil {
|
||||
log.LogErrorf("action[UpdateXAttr],The parameter must be an integer: err(%v)", err)
|
||||
p.PacketErrorWithBody(proto.OpArgMismatchErr, []byte(err.Error()))
|
||||
return
|
||||
}
|
||||
filesHddInc, _ := strconv.ParseInt(newValueList[0], 10, 64)
|
||||
filesSsdInc, _ := strconv.ParseInt(newValueList[1], 10, 64)
|
||||
filesBlobStoreInc, _ := strconv.ParseInt(newValueList[2], 10, 64)
|
||||
bytesHddInc, _ := strconv.ParseInt(newValueList[3], 10, 64)
|
||||
bytesSsdInc, _ := strconv.ParseInt(newValueList[4], 10, 64)
|
||||
bytesBlobStoreInc, _ := strconv.ParseInt(newValueList[5], 10, 64)
|
||||
dirsInc, _ := strconv.ParseInt(newValueList[6], 10, 64)
|
||||
|
||||
mp.xattrLock.Lock()
|
||||
defer mp.xattrLock.Unlock()
|
||||
@ -59,15 +40,30 @@ func (mp *metaPartition) UpdateXAttr(req *proto.UpdateXAttrRequest, p *Packet) (
|
||||
extend := treeItem.(*Extend)
|
||||
if value, exist := extend.Get([]byte(req.Key)); exist {
|
||||
oldValueList := strings.Split(string(value), ",")
|
||||
oldFiles, _ := strconv.ParseInt(oldValueList[0], 10, 64)
|
||||
oldDirs, _ := strconv.ParseInt(oldValueList[1], 10, 64)
|
||||
oldBytes, _ := strconv.ParseInt(oldValueList[2], 10, 64)
|
||||
newFiles := oldFiles + filesInc
|
||||
oldFilesHdd, _ := strconv.ParseInt(oldValueList[0], 10, 64)
|
||||
oldFilesSsd, _ := strconv.ParseInt(oldValueList[1], 10, 64)
|
||||
oldFilesBlobStore, _ := strconv.ParseInt(oldValueList[2], 10, 64)
|
||||
oldBytesHdd, _ := strconv.ParseInt(oldValueList[3], 10, 64)
|
||||
oldBytesSsd, _ := strconv.ParseInt(oldValueList[4], 10, 64)
|
||||
oldBytesBlobStore, _ := strconv.ParseInt(oldValueList[5], 10, 64)
|
||||
oldDirs, _ := strconv.ParseInt(oldValueList[6], 10, 64)
|
||||
|
||||
newFilesHdd := oldFilesHdd + filesHddInc
|
||||
newFilesSsd := oldFilesSsd + filesSsdInc
|
||||
newFilesBlobStore := oldFilesBlobStore + filesBlobStoreInc
|
||||
newBytesHdd := oldBytesHdd + bytesHddInc
|
||||
newBytesSsd := oldBytesSsd + bytesSsdInc
|
||||
newBytesBlobStore := oldBytesBlobStore + bytesBlobStoreInc
|
||||
newDirs := oldDirs + dirsInc
|
||||
newBytes := oldBytes + bytesInc
|
||||
newValue := strconv.FormatInt(int64(newFiles), 10) + "," +
|
||||
strconv.FormatInt(int64(newDirs), 10) + "," +
|
||||
strconv.FormatInt(int64(newBytes), 10)
|
||||
|
||||
newValue := strconv.FormatInt(int64(newFilesHdd), 10) + "," +
|
||||
strconv.FormatInt(int64(newFilesSsd), 10) + "," +
|
||||
strconv.FormatInt(int64(newFilesBlobStore), 10) + "," +
|
||||
strconv.FormatInt(int64(newBytesHdd), 10) + "," +
|
||||
strconv.FormatInt(int64(newBytesSsd), 10) + "," +
|
||||
strconv.FormatInt(int64(newBytesBlobStore), 10) + "," +
|
||||
strconv.FormatInt(int64(newDirs), 10)
|
||||
|
||||
extend := NewExtend(req.Inode)
|
||||
extend.Put([]byte(req.Key), []byte(newValue), mp.verSeq)
|
||||
if _, err = mp.putExtend(opFSMUpdateXAttr, extend); err != nil {
|
||||
|
||||
@ -609,7 +609,17 @@ func (client *ExtentClient) Truncate(mw *meta.MetaWrapper, parentIno uint64, ino
|
||||
log.LogError(errors.Stack(err))
|
||||
}
|
||||
if mw.EnableSummary {
|
||||
go mw.UpdateSummary_ll(parentIno, 0, 0, int64(size)-int64(oldSize))
|
||||
var bytesHddInc, bytesSsdInc, bytesBlobStoreInc int64
|
||||
if info.StorageClass == proto.StorageClass_Replica_HDD {
|
||||
bytesHddInc = int64(size) - int64(oldSize)
|
||||
}
|
||||
if info.StorageClass == proto.StorageClass_Replica_SSD {
|
||||
bytesSsdInc = int64(size) - int64(oldSize)
|
||||
}
|
||||
if info.StorageClass == proto.StorageClass_BlobStore {
|
||||
bytesBlobStoreInc = int64(size) - int64(oldSize)
|
||||
}
|
||||
go mw.UpdateSummary_ll(parentIno, 0, 0, 0, bytesHddInc, bytesSsdInc, bytesBlobStoreInc, 0)
|
||||
}
|
||||
|
||||
return err
|
||||
|
||||
286
sdk/meta/api.go
286
sdk/meta/api.go
@ -197,15 +197,23 @@ create_dentry:
|
||||
}
|
||||
|
||||
if mw.EnableSummary {
|
||||
var filesInc, dirsInc int64
|
||||
var filesHddInc, filesSsdInc, filesBlobStoreInc, dirsInc int64
|
||||
if proto.IsDir(mode) {
|
||||
dirsInc = 1
|
||||
} else {
|
||||
filesInc = 1
|
||||
if info.StorageClass == proto.StorageClass_Replica_HDD {
|
||||
filesHddInc = 1
|
||||
}
|
||||
if info.StorageClass == proto.StorageClass_Replica_SSD {
|
||||
filesSsdInc = 1
|
||||
}
|
||||
if info.StorageClass == proto.StorageClass_BlobStore {
|
||||
filesBlobStoreInc = 1
|
||||
}
|
||||
}
|
||||
// go mw.UpdateSummary_ll(parentID, filesInc, dirsInc, 0)
|
||||
job := func() {
|
||||
mw.UpdateSummary_ll(parentID, filesInc, dirsInc, 0)
|
||||
mw.UpdateSummary_ll(parentID, filesHddInc, filesSsdInc, filesBlobStoreInc, 0, 0, 0, dirsInc)
|
||||
}
|
||||
tx.SetOnCommit(job)
|
||||
}
|
||||
@ -325,13 +333,21 @@ create_dentry:
|
||||
return nil, statusToErrno(status)
|
||||
}
|
||||
if mw.EnableSummary {
|
||||
var filesInc, dirsInc int64
|
||||
var filesHddInc, filesSsdInc, filesBlobStoreInc, dirsInc int64
|
||||
if proto.IsDir(mode) {
|
||||
dirsInc = 1
|
||||
} else {
|
||||
filesInc = 1
|
||||
if info.StorageClass == proto.StorageClass_Replica_HDD {
|
||||
filesHddInc = 1
|
||||
}
|
||||
if info.StorageClass == proto.StorageClass_Replica_SSD {
|
||||
filesSsdInc = 1
|
||||
}
|
||||
if info.StorageClass == proto.StorageClass_BlobStore {
|
||||
filesBlobStoreInc = 1
|
||||
}
|
||||
}
|
||||
go mw.UpdateSummary_ll(parentID, filesInc, dirsInc, 0)
|
||||
go mw.UpdateSummary_ll(parentID, filesHddInc, filesSsdInc, filesBlobStoreInc, 0, 0, 0, dirsInc)
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
@ -774,11 +790,19 @@ func (mw *MetaWrapper) txDelete_ll(parentID uint64, name string, isDir bool, ful
|
||||
// go func() {
|
||||
if proto.IsDir(mode) {
|
||||
job = func() {
|
||||
mw.UpdateSummary_ll(parentID, 0, -1, 0)
|
||||
mw.UpdateSummary_ll(parentID, 0, 0, 0, 0, 0, 0, -1)
|
||||
}
|
||||
} else {
|
||||
job = func() {
|
||||
mw.UpdateSummary_ll(parentID, -1, 0, -int64(info.Size))
|
||||
if info.StorageClass == proto.StorageClass_Replica_HDD {
|
||||
mw.UpdateSummary_ll(parentID, -1, 0, 0, -int64(info.Size), 0, 0, 0)
|
||||
}
|
||||
if info.StorageClass == proto.StorageClass_Replica_SSD {
|
||||
mw.UpdateSummary_ll(parentID, 0, -1, 0, 0, -int64(info.Size), 0, 0)
|
||||
}
|
||||
if info.StorageClass == proto.StorageClass_BlobStore {
|
||||
mw.UpdateSummary_ll(parentID, 0, 0, -1, 0, 0, -int64(info.Size), 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
tx.SetOnCommit(job)
|
||||
@ -929,9 +953,17 @@ func (mw *MetaWrapper) Delete_ll_EX(parentID uint64, name string, isDir bool, ve
|
||||
if verSeq == 0 && mw.EnableSummary {
|
||||
go func() {
|
||||
if proto.IsDir(mode) {
|
||||
mw.UpdateSummary_ll(parentID, 0, -1, 0)
|
||||
mw.UpdateSummary_ll(parentID, 0, 0, 0, 0, 0, 0, -1)
|
||||
} else {
|
||||
mw.UpdateSummary_ll(parentID, -1, 0, -int64(info.Size))
|
||||
if info.StorageClass == proto.StorageClass_Replica_HDD {
|
||||
mw.UpdateSummary_ll(parentID, -1, 0, 0, -int64(info.Size), 0, 0, 0)
|
||||
}
|
||||
if info.StorageClass == proto.StorageClass_Replica_SSD {
|
||||
mw.UpdateSummary_ll(parentID, 0, -1, 0, 0, -int64(info.Size), 0, 0)
|
||||
}
|
||||
if info.StorageClass == proto.StorageClass_BlobStore {
|
||||
mw.UpdateSummary_ll(parentID, 0, 0, -1, 0, 0, -int64(info.Size), 0)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
@ -1048,9 +1080,17 @@ func (mw *MetaWrapper) deletewithcond_ll(parentID, cond uint64, name string, isD
|
||||
if mw.EnableSummary {
|
||||
go func() {
|
||||
if proto.IsDir(mode) {
|
||||
mw.UpdateSummary_ll(parentID, 0, -1, 0)
|
||||
mw.UpdateSummary_ll(parentID, 0, 0, 0, 0, 0, 0, -1)
|
||||
} else {
|
||||
mw.UpdateSummary_ll(parentID, -1, 0, -int64(info.Size))
|
||||
if info.StorageClass == proto.StorageClass_Replica_HDD {
|
||||
mw.UpdateSummary_ll(parentID, -1, 0, 0, -int64(info.Size), 0, 0, 0)
|
||||
}
|
||||
if info.StorageClass == proto.StorageClass_Replica_SSD {
|
||||
mw.UpdateSummary_ll(parentID, 0, -1, 0, 0, -int64(info.Size), 0, 0)
|
||||
}
|
||||
if info.StorageClass == proto.StorageClass_BlobStore {
|
||||
mw.UpdateSummary_ll(parentID, 0, 0, -1, 0, 0, -int64(info.Size), 0)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
@ -1198,8 +1238,18 @@ func (mw *MetaWrapper) txRename_ll(srcParentID uint64, srcName string, dstParent
|
||||
dstInodeInfo, _ = mw.InodeGet_ll(dstInode)
|
||||
sizeInc := srcInodeInfo.Size - dstInodeInfo.Size
|
||||
job = func() {
|
||||
mw.UpdateSummary_ll(srcParentID, -1, 0, -int64(srcInodeInfo.Size))
|
||||
mw.UpdateSummary_ll(dstParentID, 0, 0, int64(sizeInc))
|
||||
if dstInodeInfo.StorageClass == proto.StorageClass_Replica_HDD {
|
||||
mw.UpdateSummary_ll(srcParentID, -1, 0, 0, -int64(srcInodeInfo.Size), 0, 0, 0)
|
||||
mw.UpdateSummary_ll(dstParentID, 0, 0, 0, int64(sizeInc), 0, 0, 0)
|
||||
}
|
||||
if dstInodeInfo.StorageClass == proto.StorageClass_Replica_SSD {
|
||||
mw.UpdateSummary_ll(srcParentID, -1, 0, 0, 0, -int64(srcInodeInfo.Size), 0, 0)
|
||||
mw.UpdateSummary_ll(dstParentID, 0, 0, 0, 0, int64(sizeInc), 0, 0)
|
||||
}
|
||||
if dstInodeInfo.StorageClass == proto.StorageClass_BlobStore {
|
||||
mw.UpdateSummary_ll(srcParentID, -1, 0, 0, 0, 0, -int64(srcInodeInfo.Size), 0)
|
||||
mw.UpdateSummary_ll(dstParentID, 0, 0, 0, 0, 0, int64(sizeInc), 0)
|
||||
}
|
||||
}
|
||||
tx.SetOnCommit(job)
|
||||
return
|
||||
@ -1208,14 +1258,24 @@ func (mw *MetaWrapper) txRename_ll(srcParentID uint64, srcName string, dstParent
|
||||
if proto.IsRegular(srcMode) {
|
||||
log.LogDebugf("txRename_ll: update summary when file dentry is replaced")
|
||||
job = func() {
|
||||
mw.UpdateSummary_ll(srcParentID, -1, 0, -sizeInc)
|
||||
mw.UpdateSummary_ll(dstParentID, 1, 0, sizeInc)
|
||||
if srcInodeInfo.StorageClass == proto.StorageClass_Replica_HDD {
|
||||
mw.UpdateSummary_ll(srcParentID, -1, 0, 0, -sizeInc, 0, 0, 0)
|
||||
mw.UpdateSummary_ll(dstParentID, 1, 0, 0, sizeInc, 0, 0, 0)
|
||||
}
|
||||
if srcInodeInfo.StorageClass == proto.StorageClass_Replica_SSD {
|
||||
mw.UpdateSummary_ll(srcParentID, 0, -1, 0, 0, -sizeInc, 0, 0)
|
||||
mw.UpdateSummary_ll(dstParentID, 0, 1, 0, 0, sizeInc, 0, 0)
|
||||
}
|
||||
if srcInodeInfo.StorageClass == proto.StorageClass_BlobStore {
|
||||
mw.UpdateSummary_ll(srcParentID, 0, 0, -1, 0, 0, -sizeInc, 0)
|
||||
mw.UpdateSummary_ll(dstParentID, 0, 0, 1, 0, 0, sizeInc, 0)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.LogDebugf("txRename_ll: update summary when dir dentry is replaced")
|
||||
job = func() {
|
||||
mw.UpdateSummary_ll(srcParentID, 0, -1, 0)
|
||||
mw.UpdateSummary_ll(dstParentID, 0, 1, 0)
|
||||
mw.UpdateSummary_ll(srcParentID, 0, 0, 0, 0, 0, 0, -1)
|
||||
mw.UpdateSummary_ll(dstParentID, 0, 0, 0, 0, 0, 0, 1)
|
||||
}
|
||||
}
|
||||
tx.SetOnCommit(job)
|
||||
@ -1375,8 +1435,18 @@ func (mw *MetaWrapper) rename_ll(srcParentID uint64, srcName string, dstParentID
|
||||
if mw.EnableSummary {
|
||||
sizeInc := srcInodeInfo.Size - dstInodeInfo.Size
|
||||
go func() {
|
||||
mw.UpdateSummary_ll(srcParentID, -1, 0, -int64(srcInodeInfo.Size))
|
||||
mw.UpdateSummary_ll(dstParentID, 0, 0, int64(sizeInc))
|
||||
if dstInodeInfo.StorageClass == proto.StorageClass_Replica_HDD {
|
||||
mw.UpdateSummary_ll(srcParentID, -1, 0, 0, -int64(srcInodeInfo.Size), 0, 0, 0)
|
||||
mw.UpdateSummary_ll(dstParentID, 0, 0, 0, int64(sizeInc), 0, 0, 0)
|
||||
}
|
||||
if dstInodeInfo.StorageClass == proto.StorageClass_Replica_SSD {
|
||||
mw.UpdateSummary_ll(srcParentID, -1, 0, 0, 0, -int64(srcInodeInfo.Size), 0, 0)
|
||||
mw.UpdateSummary_ll(dstParentID, 0, 0, 0, 0, int64(sizeInc), 0, 0)
|
||||
}
|
||||
if dstInodeInfo.StorageClass == proto.StorageClass_BlobStore {
|
||||
mw.UpdateSummary_ll(srcParentID, -1, 0, 0, 0, 0, -int64(srcInodeInfo.Size), 0)
|
||||
mw.UpdateSummary_ll(dstParentID, 0, 0, 0, 0, 0, int64(sizeInc), 0)
|
||||
}
|
||||
}()
|
||||
}
|
||||
} else {
|
||||
@ -1385,14 +1455,24 @@ func (mw *MetaWrapper) rename_ll(srcParentID uint64, srcName string, dstParentID
|
||||
if proto.IsRegular(mode) {
|
||||
// file
|
||||
go func() {
|
||||
mw.UpdateSummary_ll(srcParentID, -1, 0, -sizeInc)
|
||||
mw.UpdateSummary_ll(dstParentID, 1, 0, sizeInc)
|
||||
if srcInodeInfo.StorageClass == proto.StorageClass_Replica_HDD {
|
||||
mw.UpdateSummary_ll(srcParentID, -1, 0, 0, -sizeInc, 0, 0, 0)
|
||||
mw.UpdateSummary_ll(dstParentID, 1, 0, 0, sizeInc, 0, 0, 0)
|
||||
}
|
||||
if srcInodeInfo.StorageClass == proto.StorageClass_Replica_SSD {
|
||||
mw.UpdateSummary_ll(srcParentID, 0, -1, 0, 0, -sizeInc, 0, 0)
|
||||
mw.UpdateSummary_ll(dstParentID, 0, 1, 0, 0, sizeInc, 0, 0)
|
||||
}
|
||||
if srcInodeInfo.StorageClass == proto.StorageClass_BlobStore {
|
||||
mw.UpdateSummary_ll(srcParentID, 0, 0, -1, 0, 0, -sizeInc, 0)
|
||||
mw.UpdateSummary_ll(dstParentID, 0, 0, 1, 0, 0, sizeInc, 0)
|
||||
}
|
||||
}()
|
||||
} else {
|
||||
// dir
|
||||
go func() {
|
||||
mw.UpdateSummary_ll(srcParentID, 0, -1, 0)
|
||||
mw.UpdateSummary_ll(dstParentID, 0, 1, 0)
|
||||
mw.UpdateSummary_ll(srcParentID, 0, 0, 0, 0, 0, 0, -1)
|
||||
mw.UpdateSummary_ll(dstParentID, 0, 0, 0, 0, 0, 0, 1)
|
||||
}()
|
||||
}
|
||||
}
|
||||
@ -1536,7 +1616,16 @@ func (mw *MetaWrapper) SplitExtentKey(parentInode, inode uint64, ek proto.Extent
|
||||
newInfo, _ := mw.InodeGet_ll(inode)
|
||||
if oldInfo != nil && newInfo != nil {
|
||||
if int64(oldInfo.Size) < int64(newInfo.Size) {
|
||||
mw.UpdateSummary_ll(parentInode, 0, 0, int64(newInfo.Size)-int64(oldInfo.Size))
|
||||
incSize := int64(newInfo.Size) - int64(oldInfo.Size)
|
||||
if newInfo.StorageClass == proto.StorageClass_Replica_HDD {
|
||||
mw.UpdateSummary_ll(parentInode, 0, 0, 0, incSize, 0, 0, 0)
|
||||
}
|
||||
if newInfo.StorageClass == proto.StorageClass_Replica_SSD {
|
||||
mw.UpdateSummary_ll(parentInode, 0, 0, 0, 0, incSize, 0, 0)
|
||||
}
|
||||
if newInfo.StorageClass == proto.StorageClass_BlobStore {
|
||||
mw.UpdateSummary_ll(parentInode, 0, 0, 0, 0, 0, incSize, 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
@ -1569,7 +1658,16 @@ func (mw *MetaWrapper) AppendExtentKey(parentInode, inode uint64, ek proto.Exten
|
||||
newInfo, _ := mw.InodeGet_ll(inode)
|
||||
if oldInfo != nil && newInfo != nil {
|
||||
if int64(oldInfo.Size) < int64(newInfo.Size) {
|
||||
mw.UpdateSummary_ll(parentInode, 0, 0, int64(newInfo.Size)-int64(oldInfo.Size))
|
||||
incSize := int64(newInfo.Size) - int64(oldInfo.Size)
|
||||
if newInfo.StorageClass == proto.StorageClass_Replica_HDD {
|
||||
mw.UpdateSummary_ll(parentInode, 0, 0, 0, incSize, 0, 0, 0)
|
||||
}
|
||||
if newInfo.StorageClass == proto.StorageClass_Replica_SSD {
|
||||
mw.UpdateSummary_ll(parentInode, 0, 0, 0, 0, incSize, 0, 0)
|
||||
}
|
||||
if newInfo.StorageClass == proto.StorageClass_BlobStore {
|
||||
mw.UpdateSummary_ll(parentInode, 0, 0, 0, 0, 0, incSize, 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
@ -2279,8 +2377,9 @@ func (mw *MetaWrapper) XAttrsList_ll(inode uint64) ([]string, error) {
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func (mw *MetaWrapper) UpdateSummary_ll(parentIno uint64, filesInc int64, dirsInc int64, bytesInc int64) {
|
||||
if filesInc == 0 && dirsInc == 0 && bytesInc == 0 {
|
||||
func (mw *MetaWrapper) UpdateSummary_ll(parentIno uint64, filesHddInc int64, filesSsdInc int64, filesBlobStoreInc int64,
|
||||
bytesHddInc int64, bytesSsdInc int64, bytesBlobStoreInc int64, dirsInc int64) {
|
||||
if filesHddInc == 0 && filesSsdInc == 0 && filesBlobStoreInc == 0 && bytesHddInc == 0 && bytesSsdInc == 0 && bytesBlobStoreInc == 0 && dirsInc == 0 {
|
||||
return
|
||||
}
|
||||
mp := mw.getPartitionByInode(parentIno)
|
||||
@ -2289,7 +2388,8 @@ func (mw *MetaWrapper) UpdateSummary_ll(parentIno uint64, filesInc int64, dirsIn
|
||||
return
|
||||
}
|
||||
for cnt := 0; cnt < UpdateSummaryRetry; cnt++ {
|
||||
err := mw.updateXAttrs(mp, parentIno, filesInc, dirsInc, bytesInc)
|
||||
err := mw.updateXAttrs(mp, parentIno, filesHddInc, filesSsdInc, filesBlobStoreInc,
|
||||
bytesHddInc, bytesSsdInc, bytesBlobStoreInc, dirsInc)
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
@ -2310,9 +2410,13 @@ func (mw *MetaWrapper) ReadDirOnly_ll(parentID uint64) ([]proto.Dentry, error) {
|
||||
}
|
||||
|
||||
type SummaryInfo struct {
|
||||
Files int64
|
||||
Subdirs int64
|
||||
Fbytes int64
|
||||
Subdirs int64
|
||||
FilesHdd int64
|
||||
FilesSsd int64
|
||||
FilesBlobStore int64
|
||||
FbytesHdd int64
|
||||
FbytesSsd int64
|
||||
FbytesBlobStore int64
|
||||
}
|
||||
|
||||
func (mw *MetaWrapper) GetSummary_ll(parentIno uint64, goroutineNum int32) (SummaryInfo, error) {
|
||||
@ -2339,7 +2443,7 @@ func (mw *MetaWrapper) GetSummary_ll(parentIno uint64, goroutineNum int32) (Summ
|
||||
|
||||
go mw.getDirSummary(&summaryInfo, inodeCh, errCh)
|
||||
for err := range errCh {
|
||||
return SummaryInfo{0, 0, 0}, err
|
||||
return SummaryInfo{0, 0, 0, 0, 0, 0, 0}, err
|
||||
}
|
||||
return summaryInfo, nil
|
||||
} else {
|
||||
@ -2353,14 +2457,18 @@ func (mw *MetaWrapper) GetSummary_ll(parentIno uint64, goroutineNum int32) (Summ
|
||||
}()
|
||||
go func(summaryInfo *SummaryInfo) {
|
||||
for summary := range summaryCh {
|
||||
summaryInfo.Files = summaryInfo.Files + summary.Files
|
||||
summaryInfo.FilesHdd = summaryInfo.FilesHdd + summary.FilesHdd
|
||||
summaryInfo.FilesSsd = summaryInfo.FilesSsd + summary.FilesSsd
|
||||
summaryInfo.FilesBlobStore = summaryInfo.FilesBlobStore + summary.FilesBlobStore
|
||||
summaryInfo.FbytesHdd = summaryInfo.FbytesHdd + summary.FbytesHdd
|
||||
summaryInfo.FilesSsd = summaryInfo.FilesSsd + summary.FilesSsd
|
||||
summaryInfo.FbytesBlobStore = summaryInfo.FbytesBlobStore + summary.FbytesBlobStore
|
||||
summaryInfo.Subdirs = summaryInfo.Subdirs + summary.Subdirs
|
||||
summaryInfo.Fbytes = summaryInfo.Fbytes + summary.Fbytes
|
||||
}
|
||||
close(errCh)
|
||||
}(&summaryInfo)
|
||||
for err := range errCh {
|
||||
return SummaryInfo{0, 0, 0}, err
|
||||
return SummaryInfo{0, 0, 0, 0, 0, 0, 0}, err
|
||||
}
|
||||
return summaryInfo, nil
|
||||
}
|
||||
@ -2409,12 +2517,21 @@ func (mw *MetaWrapper) getDirSummary(summaryInfo *SummaryInfo, inodeCh <-chan ui
|
||||
for _, xattrInfo := range xattrInfos {
|
||||
if xattrInfo.XAttrs[SummaryKey] != "" {
|
||||
summaryList := strings.Split(xattrInfo.XAttrs[SummaryKey], ",")
|
||||
files, _ := strconv.ParseInt(summaryList[0], 10, 64)
|
||||
subdirs, _ := strconv.ParseInt(summaryList[1], 10, 64)
|
||||
fbytes, _ := strconv.ParseInt(summaryList[2], 10, 64)
|
||||
summaryInfo.Files += files
|
||||
filesHdd, _ := strconv.ParseInt(summaryList[0], 10, 64)
|
||||
filesSsd, _ := strconv.ParseInt(summaryList[1], 10, 64)
|
||||
filesBlobStore, _ := strconv.ParseInt(summaryList[2], 10, 64)
|
||||
fbytesHdd, _ := strconv.ParseInt(summaryList[3], 10, 64)
|
||||
fbytesSsd, _ := strconv.ParseInt(summaryList[4], 10, 64)
|
||||
fbytesBlobStore, _ := strconv.ParseInt(summaryList[5], 10, 64)
|
||||
subdirs, _ := strconv.ParseInt(summaryList[6], 10, 64)
|
||||
|
||||
summaryInfo.FilesHdd += filesHdd
|
||||
summaryInfo.FilesSsd += filesSsd
|
||||
summaryInfo.FilesBlobStore += filesBlobStore
|
||||
summaryInfo.FbytesHdd += fbytesHdd
|
||||
summaryInfo.FbytesSsd += fbytesSsd
|
||||
summaryInfo.FbytesBlobStore += fbytesBlobStore
|
||||
summaryInfo.Subdirs += subdirs
|
||||
summaryInfo.Fbytes += fbytes
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -2426,12 +2543,21 @@ func (mw *MetaWrapper) getDirSummary(summaryInfo *SummaryInfo, inodeCh <-chan ui
|
||||
for _, xattrInfo := range xattrInfos {
|
||||
if xattrInfo.XAttrs[SummaryKey] != "" {
|
||||
summaryList := strings.Split(xattrInfo.XAttrs[SummaryKey], ",")
|
||||
files, _ := strconv.ParseInt(summaryList[0], 10, 64)
|
||||
subdirs, _ := strconv.ParseInt(summaryList[1], 10, 64)
|
||||
fbytes, _ := strconv.ParseInt(summaryList[2], 10, 64)
|
||||
summaryInfo.Files += files
|
||||
filesHdd, _ := strconv.ParseInt(summaryList[0], 10, 64)
|
||||
filesSsd, _ := strconv.ParseInt(summaryList[1], 10, 64)
|
||||
filesBlobStore, _ := strconv.ParseInt(summaryList[2], 10, 64)
|
||||
fbytesHdd, _ := strconv.ParseInt(summaryList[3], 10, 64)
|
||||
fbytesSsd, _ := strconv.ParseInt(summaryList[4], 10, 64)
|
||||
fbytesBlobStore, _ := strconv.ParseInt(summaryList[5], 10, 64)
|
||||
subdirs, _ := strconv.ParseInt(summaryList[6], 10, 64)
|
||||
|
||||
summaryInfo.FilesHdd += filesHdd
|
||||
summaryInfo.FilesSsd += filesSsd
|
||||
summaryInfo.FilesBlobStore += filesBlobStore
|
||||
summaryInfo.FbytesHdd += fbytesHdd
|
||||
summaryInfo.FbytesSsd += fbytesSsd
|
||||
summaryInfo.FbytesBlobStore += fbytesBlobStore
|
||||
summaryInfo.Subdirs += subdirs
|
||||
summaryInfo.Fbytes += fbytes
|
||||
}
|
||||
}
|
||||
close(errch)
|
||||
@ -2445,11 +2571,7 @@ func (mw *MetaWrapper) getSummaryOrigin(parentIno uint64, summaryCh chan<- Summa
|
||||
}
|
||||
}()
|
||||
var subdirsList []uint64
|
||||
retSummaryInfo := SummaryInfo{
|
||||
Files: 0,
|
||||
Subdirs: 0,
|
||||
Fbytes: 0,
|
||||
}
|
||||
retSummaryInfo := SummaryInfo{0, 0, 0, 0, 0, 0, 0}
|
||||
children, err := mw.ReadDir_ll(parentIno)
|
||||
if err != nil {
|
||||
errCh <- err
|
||||
@ -2465,8 +2587,18 @@ func (mw *MetaWrapper) getSummaryOrigin(parentIno uint64, summaryCh chan<- Summa
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
retSummaryInfo.Files += 1
|
||||
retSummaryInfo.Fbytes += int64(fileInfo.Size)
|
||||
if fileInfo.StorageClass == proto.StorageClass_Replica_HDD {
|
||||
retSummaryInfo.FilesHdd += 1
|
||||
retSummaryInfo.FbytesHdd += int64(fileInfo.Size)
|
||||
}
|
||||
if fileInfo.StorageClass == proto.StorageClass_Replica_SSD {
|
||||
retSummaryInfo.FilesSsd += 1
|
||||
retSummaryInfo.FbytesSsd += int64(fileInfo.Size)
|
||||
}
|
||||
if fileInfo.StorageClass == proto.StorageClass_BlobStore {
|
||||
retSummaryInfo.FilesBlobStore += 1
|
||||
retSummaryInfo.FbytesBlobStore += int64(fileInfo.Size)
|
||||
}
|
||||
}
|
||||
}
|
||||
summaryCh <- retSummaryInfo
|
||||
@ -2516,22 +2648,30 @@ func (mw *MetaWrapper) refreshSummary(parentIno uint64, errCh chan<- error, wg *
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
oldSummaryInfo := SummaryInfo{0, 0, 0}
|
||||
oldSummaryInfo := SummaryInfo{0, 0, 0, 0, 0, 0, 0}
|
||||
if summaryXAttrInfo.XAttrs[SummaryKey] != "" {
|
||||
summaryList := strings.Split(summaryXAttrInfo.XAttrs[SummaryKey], ",")
|
||||
files, _ := strconv.ParseInt(summaryList[0], 10, 64)
|
||||
subdirs, _ := strconv.ParseInt(summaryList[1], 10, 64)
|
||||
fbytes, _ := strconv.ParseInt(summaryList[2], 10, 64)
|
||||
filesHdd, _ := strconv.ParseInt(summaryList[0], 10, 64)
|
||||
filesSsd, _ := strconv.ParseInt(summaryList[1], 10, 64)
|
||||
filesBlobStore, _ := strconv.ParseInt(summaryList[2], 10, 64)
|
||||
fbytesHdd, _ := strconv.ParseInt(summaryList[3], 10, 64)
|
||||
fbytesSsd, _ := strconv.ParseInt(summaryList[4], 10, 64)
|
||||
fbytesBlobStore, _ := strconv.ParseInt(summaryList[5], 10, 64)
|
||||
subdirs, _ := strconv.ParseInt(summaryList[6], 10, 64)
|
||||
oldSummaryInfo = SummaryInfo{
|
||||
Files: files,
|
||||
Subdirs: subdirs,
|
||||
Fbytes: fbytes,
|
||||
FilesHdd: filesHdd,
|
||||
FilesSsd: filesSsd,
|
||||
FilesBlobStore: filesBlobStore,
|
||||
FbytesHdd: fbytesHdd,
|
||||
FbytesSsd: fbytesSsd,
|
||||
FbytesBlobStore: fbytesBlobStore,
|
||||
Subdirs: subdirs,
|
||||
}
|
||||
} else {
|
||||
oldSummaryInfo = SummaryInfo{0, 0, 0}
|
||||
oldSummaryInfo = SummaryInfo{0, 0, 0, 0, 0, 0, 0}
|
||||
}
|
||||
|
||||
newSummaryInfo := SummaryInfo{0, 0, 0}
|
||||
newSummaryInfo := SummaryInfo{0, 0, 0, 0, 0, 0, 0}
|
||||
|
||||
var subdirsList []uint64
|
||||
children, err := mw.ReadDir_ll(parentIno)
|
||||
@ -2549,15 +2689,29 @@ func (mw *MetaWrapper) refreshSummary(parentIno uint64, errCh chan<- error, wg *
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
newSummaryInfo.Files += 1
|
||||
newSummaryInfo.Fbytes += int64(fileInfo.Size)
|
||||
if fileInfo.StorageClass == proto.StorageClass_Replica_HDD {
|
||||
newSummaryInfo.FilesHdd += 1
|
||||
newSummaryInfo.FbytesHdd += int64(fileInfo.Size)
|
||||
}
|
||||
if fileInfo.StorageClass == proto.StorageClass_Replica_SSD {
|
||||
newSummaryInfo.FilesSsd += 1
|
||||
newSummaryInfo.FbytesSsd += int64(fileInfo.Size)
|
||||
}
|
||||
if fileInfo.StorageClass == proto.StorageClass_BlobStore {
|
||||
newSummaryInfo.FilesBlobStore += 1
|
||||
newSummaryInfo.FbytesBlobStore += int64(fileInfo.Size)
|
||||
}
|
||||
}
|
||||
}
|
||||
go mw.UpdateSummary_ll(
|
||||
parentIno,
|
||||
newSummaryInfo.Files-oldSummaryInfo.Files,
|
||||
newSummaryInfo.FilesHdd-oldSummaryInfo.FilesHdd,
|
||||
newSummaryInfo.FilesSsd-oldSummaryInfo.FilesSsd,
|
||||
newSummaryInfo.FilesBlobStore-oldSummaryInfo.FilesBlobStore,
|
||||
newSummaryInfo.FbytesHdd-oldSummaryInfo.FbytesHdd,
|
||||
newSummaryInfo.FbytesSsd-oldSummaryInfo.FbytesSsd,
|
||||
newSummaryInfo.FbytesBlobStore-oldSummaryInfo.FbytesBlobStore,
|
||||
newSummaryInfo.Subdirs-oldSummaryInfo.Subdirs,
|
||||
newSummaryInfo.Fbytes-oldSummaryInfo.Fbytes,
|
||||
)
|
||||
|
||||
for _, subdirIno := range subdirsList {
|
||||
|
||||
@ -2468,7 +2468,8 @@ func (mw *MetaWrapper) readdironly(mp *MetaPartition, parentID uint64) (status i
|
||||
return statusOK, resp.Children, nil
|
||||
}
|
||||
|
||||
func (mw *MetaWrapper) updateXAttrs(mp *MetaPartition, inode uint64, filesInc int64, dirsInc int64, bytesInc int64) error {
|
||||
func (mw *MetaWrapper) updateXAttrs(mp *MetaPartition, inode uint64, filesHddInc int64, filesSsdInc int64, filesBlobStoreInc int64,
|
||||
bytesHddInc int64, bytesSsdInc int64, bytesBlobStoreInc int64, dirsInc int64) error {
|
||||
var err error
|
||||
|
||||
bgTime := stat.BeginStat()
|
||||
@ -2476,7 +2477,14 @@ func (mw *MetaWrapper) updateXAttrs(mp *MetaPartition, inode uint64, filesInc in
|
||||
stat.EndStat("updateXAttrs", err, bgTime, 1)
|
||||
}()
|
||||
|
||||
value := strconv.FormatInt(int64(filesInc), 10) + "," + strconv.FormatInt(int64(dirsInc), 10) + "," + strconv.FormatInt(int64(bytesInc), 10)
|
||||
value := strconv.FormatInt(int64(filesHddInc), 10) + "," +
|
||||
strconv.FormatInt(int64(filesSsdInc), 10) + "," +
|
||||
strconv.FormatInt(int64(filesBlobStoreInc), 10) + "," +
|
||||
strconv.FormatInt(int64(bytesHddInc), 10) + "," +
|
||||
strconv.FormatInt(int64(bytesSsdInc), 10) + "," +
|
||||
strconv.FormatInt(int64(bytesBlobStoreInc), 10) + "," +
|
||||
strconv.FormatInt(int64(dirsInc), 10)
|
||||
|
||||
req := &proto.UpdateXAttrRequest{
|
||||
VolName: mw.volname,
|
||||
PartitionId: mp.PartitionID,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user
