gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 02:00:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(lcnode/master): lcnode support bucket lifecycle and vol snapshot delete

Browse Source 
Signed-off-by: zhaochenyang <zhaochenyang@oppo.com>
This commit is contained in:
zhaochenyang 2023-06-29 12:32:28 +08:00 committed by leonrayang
parent d117ee2daa
commit a6d125f5ce
45 changed files with 2874 additions and 3538 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
lcnode/const.go
View File

@ -1,9 +1,68 @@
// Copyright 2023 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package lcnode
var (
defaultUChanInitCapacity int = 10000
defaultS3ScanRoutineNumPerTask int = 100
defaultSnapShotRoutineNumPerTask int = 100
maxRoutineNumPerTask int = 5000
defaultMaxChanUnm int = 10000000 //worst case: may take up around 400MB of heap space
import (
"regexp"
"github.com/cubefs/cubefs/proto"
"golang.org/x/time/rate"
)
const (
configListen = proto.ListenPort
configMasterAddr = proto.MasterAddr
configBatchExpirationGetNumStr = "batchExpirationGetNum"
configScanCheckIntervalStr = "scanCheckInterval"
configLcScanRoutineNumPerTaskStr = "lcScanRoutineNumPerTask"
configLcScanLimitPerSecondStr = "lcScanLimitPerSecond"
configSnapshotRoutineNumPerTaskStr = "snapshotRoutineNumPerTask"
)
// Default of configuration value
const (
defaultListen = "80"
ModuleName = "lcNode"
defaultBatchExpirationGetNum = 100
maxBatchExpirationGetNum = 10000
defaultScanCheckInterval = 60
defaultLcScanRoutineNumPerTask = 100
defaultLcScanLimitPerSecond = rate.Inf
defaultLcScanLimitBurst = 1000
maxLcScanRoutineNumPerTask = 5000
maxDirChanNum = 1000000
defaultReadDirLimit = 1000
defaultMasterIntervalToCheckHeartbeat = 6
noHeartBeatTimes = 3 // number of times that no heartbeat reported
defaultLcNodeTimeOutSec = noHeartBeatTimes * defaultMasterIntervalToCheckHeartbeat
defaultIntervalToCheckRegister = 2 * defaultLcNodeTimeOutSec
defaultUnboundedChanInitCapacity = 10000
)
var (
// Regular expression used to verify the configuration of the service listening port.
// A valid service listening port configuration is a string containing only numbers.
regexpListen = regexp.MustCompile("^(\\d)+$")
batchExpirationGetNum int
scanCheckInterval int64
lcScanRoutineNumPerTask int
lcScanLimitPerSecond rate.Limit
snapshotRoutineNumPerTask int
)

186
lcnode/lc_op.go Normal file
View File

@ -0,0 +1,186 @@
// Copyright 2023 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package lcnode
import (
"bytes"
"encoding/json"
"net"
"sync/atomic"
"time"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/util/errors"
"github.com/cubefs/cubefs/util/log"
)
func (l *LcNode) opMasterHeartbeat(conn net.Conn, p *proto.Packet, remoteAddr string) (err error) {
go func() {
p.PacketOkReply()
if err := p.WriteToConn(conn); err != nil {
log.LogErrorf("ack master response: %s", err.Error())
}
}()
data := p.Data
var (
req = &proto.HeartBeatRequest{}
resp = &proto.LcNodeHeartbeatResponse{
LcScanningTasks: make(map[string]*proto.LcNodeRuleTaskResponse, 0),
}
adminTask = &proto.AdminTask{
Request: req,
}
)
go func() {
start := time.Now()
decode := json.NewDecoder(bytes.NewBuffer(data))
decode.UseNumber()
if err = decode.Decode(adminTask); err != nil {
resp.Status = proto.TaskFailed
resp.Result = err.Error()
goto end
}
l.scannerMutex.RLock()
for _, scanner := range l.lcScanners {
result := &proto.LcNodeRuleTaskResponse{
ID: scanner.ID,
LcNodeRuleTaskStatistics: proto.LcNodeRuleTaskStatistics{
Volume: scanner.Volume,
RuleId: scanner.rule.ID,
TotalInodeScannedNum: atomic.LoadInt64(&scanner.currentStat.TotalInodeScannedNum),
FileScannedNum: atomic.LoadInt64(&scanner.currentStat.FileScannedNum),
DirScannedNum: atomic.LoadInt64(&scanner.currentStat.DirScannedNum),
ExpiredNum: atomic.LoadInt64(&scanner.currentStat.ExpiredNum),
ErrorSkippedNum: atomic.LoadInt64(&scanner.currentStat.ErrorSkippedNum),
},
}
resp.LcScanningTasks[scanner.ID] = result
}
for _, scanner := range l.snapshotScanners {
info := &proto.SnapshotVerDelTaskResponse{
ID: scanner.ID,
SnapshotStatistics: proto.SnapshotStatistics{
VerInfo: proto.VerInfo{
VolName: scanner.getTaskVolName(),
VerSeq: scanner.getTaskVerSeq(),
},
TotalInodeNum: atomic.LoadInt64(&scanner.currentStat.TotalInodeNum),
FileNum: atomic.LoadInt64(&scanner.currentStat.FileNum),
DirNum: atomic.LoadInt64(&scanner.currentStat.DirNum),
ErrorSkippedNum: atomic.LoadInt64(&scanner.currentStat.ErrorSkippedNum),
},
}
resp.SnapshotScanningTasks[scanner.ID] = info
}
l.scannerMutex.RUnlock()
resp.Status = proto.TaskSucceeds
end:
adminTask.Request = nil
adminTask.Response = resp
l.respondToMaster(adminTask)
data, _ := json.Marshal(resp)
log.LogInfof("%s pkt %s, resp success req:%v; respAdminTask: %v, resp: %v, cost %s",
remoteAddr, p.String(), req, adminTask, string(data), time.Since(start).String())
}()
l.lastHeartbeat = time.Now()
return
}
func (l *LcNode) opLcScan(conn net.Conn, p *proto.Packet) (err error) {
go func() {
p.PacketOkReply()
if err := p.WriteToConn(conn); err != nil {
log.LogErrorf("ack master response: %s", err.Error())
}
}()
data := p.Data
var (
req = &proto.LcNodeRuleTaskRequest{}
resp = &proto.LcNodeRuleTaskResponse{}
adminTask = &proto.AdminTask{
Request: req,
}
)
decoder := json.NewDecoder(bytes.NewBuffer(data))
decoder.UseNumber()
if err = decoder.Decode(adminTask); err != nil {
resp.Status = proto.TaskFailed
resp.Result = err.Error()
adminTask.Request = nil
adminTask.Response = resp
l.respondToMaster(adminTask)
return
}
if err = l.startLcScan(adminTask); err != nil {
l.respondToMaster(adminTask)
return
}
return
}
func (l *LcNode) respondToMaster(task *proto.AdminTask) (err error) {
// handle panic
defer func() {
if r := recover(); r != nil {
switch data := r.(type) {
case error:
err = data
default:
err = errors.New(data.(string))
}
}
}()
if err = l.mc.NodeAPI().ResponseLcNodeTask(task); err != nil {
err = errors.Trace(err, "try respondToMaster failed")
}
return
}
func (l *LcNode) opSnapshotVerDel(conn net.Conn, p *proto.Packet) (err error) {
go func() {
p.PacketOkReply()
if err := p.WriteToConn(conn); err != nil {
log.LogErrorf("ack master response: %s", err.Error())
}
}()
data := p.Data
var (
req = &proto.SnapshotVerDelTaskRequest{}
resp = &proto.SnapshotVerDelTaskResponse{}
adminTask = &proto.AdminTask{
Request: req,
}
)
decoder := json.NewDecoder(bytes.NewBuffer(data))
decoder.UseNumber()
if err = decoder.Decode(adminTask); err != nil {
resp.Status = proto.TaskFailed
resp.Result = err.Error()
adminTask.Response = resp
_ = l.respondToMaster(adminTask)
} else {
err = l.startSnapshotScan(adminTask)
}
return
}

552
lcnode/lc_scanner.go Normal file
View File

@ -0,0 +1,552 @@
// Copyright 2023 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package lcnode
import (
"context"
"os"
"strings"
"sync/atomic"
"syscall"
"time"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/sdk/meta"
"github.com/cubefs/cubefs/util/log"
"github.com/cubefs/cubefs/util/routinepool"
"github.com/cubefs/cubefs/util/unboundedchan"
"golang.org/x/time/rate"
)
const (
pathSep = "/"
)
type LcScanner struct {
ID string
Volume string
mw *meta.MetaWrapper
lcnode *LcNode
adminTask *proto.AdminTask
rule *proto.Rule
dirChan *unboundedchan.UnboundedChan
fileChan *unboundedchan.UnboundedChan
fileRPoll *routinepool.RoutinePool
dirRPoll *routinepool.RoutinePool
batchDentries *proto.BatchDentries
currentStat *proto.LcNodeRuleTaskStatistics
limiter *rate.Limiter
now time.Time
stopC chan bool
}
func NewS3Scanner(adminTask *proto.AdminTask, l *LcNode) (*LcScanner, error) {
request := adminTask.Request.(*proto.LcNodeRuleTaskRequest)
scanTask := request.Task
var err error
var metaConfig = &meta.MetaConfig{
Volume: scanTask.VolName,
Masters: l.masters,
Authenticate: false,
ValidateOwner: false,
}
var metaWrapper *meta.MetaWrapper
if metaWrapper, err = meta.NewMetaWrapper(metaConfig); err != nil {
return nil, err
}
scanner := &LcScanner{
ID: scanTask.Id,
Volume: scanTask.VolName,
lcnode: l,
mw: metaWrapper,
adminTask: adminTask,
rule: scanTask.Rule,
dirChan: unboundedchan.NewUnboundedChan(defaultUnboundedChanInitCapacity),
fileChan: unboundedchan.NewUnboundedChan(defaultUnboundedChanInitCapacity),
dirRPoll: routinepool.NewRoutinePool(lcScanRoutineNumPerTask),
fileRPoll: routinepool.NewRoutinePool(lcScanRoutineNumPerTask),
batchDentries: proto.NewBatchDentries(),
currentStat: &proto.LcNodeRuleTaskStatistics{},
limiter: rate.NewLimiter(lcScanLimitPerSecond, defaultLcScanLimitBurst),
now: time.Now(),
stopC: make(chan bool, 0),
}
return scanner, nil
}
func (l *LcNode) startLcScan(adminTask *proto.AdminTask) (err error) {
request := adminTask.Request.(*proto.LcNodeRuleTaskRequest)
log.LogInfof("startLcScan: scan task(%v) received!", request.Task)
resp := &proto.LcNodeRuleTaskResponse{}
adminTask.Response = resp
l.scannerMutex.Lock()
if _, ok := l.lcScanners[request.Task.Id]; ok {
log.LogInfof("startLcScan: scan task(%v) is already running!", request.Task)
l.scannerMutex.Unlock()
return
}
var scanner *LcScanner
scanner, err = NewS3Scanner(adminTask, l)
if err != nil {
log.LogErrorf("startLcScan: NewS3Scanner err(%v)", err)
resp.Status = proto.TaskFailed
resp.Result = err.Error()
l.scannerMutex.Unlock()
return
}
l.lcScanners[scanner.ID] = scanner
l.scannerMutex.Unlock()
if err = scanner.Start(); err != nil {
return
}
return
}
func (s *LcScanner) Start() (err error) {
response := s.adminTask.Response.(*proto.LcNodeRuleTaskResponse)
parentId, prefixDirs, err := s.FindPrefixInode()
if err != nil {
log.LogErrorf("startScan err(%v): volume(%v), rule id(%v), scanning done!",
err, s.Volume, s.rule.ID)
t := time.Now()
response.ID = s.ID
response.EndTime = &t
response.Status = proto.TaskSucceeds
response.Done = true
s.lcnode.scannerMutex.Lock()
delete(s.lcnode.lcScanners, s.ID)
s.lcnode.scannerMutex.Unlock()
return
}
go s.scan()
var currentPath string
if len(prefixDirs) > 0 {
currentPath = strings.Join(prefixDirs, pathSep)
}
firstDentry := &proto.ScanDentry{
Inode: parentId,
Path: strings.TrimPrefix(currentPath, pathSep),
Type: uint32(os.ModeDir),
}
t := time.Now()
response.StartTime = &t
log.LogInfof("startScan: first dentry(%v) in dirChan!", firstDentry)
s.dirChan.In <- firstDentry
go s.checkScanning()
return
}
func (s *LcScanner) FindPrefixInode() (inode uint64, prefixDirs []string, err error) {
prefixDirs = make([]string, 0)
var prefix string
if s.rule.Filter != nil {
prefix = s.rule.Filter.Prefix
}
var dirs []string
if prefix != "" {
dirs = strings.Split(prefix, "/")
log.LogInfof("FindPrefixInode: volume(%v), prefix(%v), dirs(%v), len(%v)", s.Volume, prefix, dirs, len(dirs))
}
if len(dirs) <= 1 {
return proto.RootIno, prefixDirs, nil
}
var parentId = proto.RootIno
for index, dir := range dirs {
// Because lookup can only retrieve dentry whose name exactly matches,
// so do not lookup the last part.
if index+1 == len(dirs) {
break
}
curIno, curMode, err := s.mw.Lookup_ll(parentId, dir)
// If the part except the last part does not match exactly the same dentry, there is
// no path matching the path prefix. An ENOENT error is returned to the caller.
if err == syscall.ENOENT {
log.LogErrorf("FindPrefixInode: find directories fail ENOENT: parentId(%v) dir(%v)", parentId, dir)
return 0, nil, syscall.ENOENT
}
if err != nil && err != syscall.ENOENT {
log.LogErrorf("FindPrefixInode: find directories fail: prefix(%v) err(%v)", prefix, err)
return 0, nil, err
}
// Because the file cannot have the next level members,
// if there is a directory in the middle of the prefix,
// it means that there is no file matching the prefix.
if !os.FileMode(curMode).IsDir() {
return 0, nil, syscall.ENOENT
}
prefixDirs = append(prefixDirs, dir)
parentId = curIno
}
inode = parentId
return
}
func (s *LcScanner) scan() {
log.LogInfof("Enter scan %+v", s)
defer func() {
log.LogInfof("Exit scan %+v", s)
}()
var prefix string
if s.rule.Filter != nil {
prefix = s.rule.Filter.Prefix
}
for {
select {
case <-s.stopC:
return
case val, ok := <-s.fileChan.Out:
if !ok {
log.LogErrorf("fileChan closed")
return
}
dentry := val.(*proto.ScanDentry)
if !strings.HasPrefix(dentry.Path, prefix) {
continue
}
job := func() {
s.handleFile(dentry)
}
_, err := s.fileRPoll.Submit(job)
if err != nil {
log.LogErrorf("fileRPoll closed")
}
default:
select {
case <-s.stopC:
return
case val, ok := <-s.fileChan.Out:
if !ok {
log.LogErrorf("fileChan closed")
return
}
dentry := val.(*proto.ScanDentry)
if !strings.HasPrefix(dentry.Path, prefix) {
continue
}
job := func() {
s.handleFile(dentry)
}
_, err := s.fileRPoll.Submit(job)
if err != nil {
log.LogErrorf("fileRPoll closed")
}
case val, ok := <-s.dirChan.Out:
if !ok {
log.LogErrorf("dirChan closed")
return
}
dentry := val.(*proto.ScanDentry)
job := func() {}
if s.dirChan.Len() > maxDirChanNum {
job = func() {
s.handleDirLimitDepthFirst(dentry)
}
} else {
job = func() {
s.handleDirLimitBreadthFirst(dentry)
}
}
_, err := s.dirRPoll.Submit(job)
if err != nil {
log.LogErrorf("handleDir failed, err(%v)", err)
}
}
}
}
}
func (s *LcScanner) handleFile(dentry *proto.ScanDentry) {
s.limiter.Wait(context.Background())
log.LogDebugf("handleFile dentry: %+v, fileChan.Len: %v", dentry, s.fileChan.Len())
atomic.AddInt64(&s.currentStat.FileScannedNum, 1)
atomic.AddInt64(&s.currentStat.TotalInodeScannedNum, 1)
s.batchDentries.Append(dentry)
if s.batchDentries.Len() >= batchExpirationGetNum {
s.batchHandleFile()
}
}
func (s *LcScanner) batchHandleFile() {
dentries, inodes := s.batchDentries.BatchGetAndClear()
var expiredDentries []*proto.ScanDentry
inodesInfo := s.mw.BatchInodeGet(inodes)
for _, info := range inodesInfo {
if s.inodeExpired(info, s.rule.Expire) {
d := dentries[info.Inode]
if d != nil {
expiredDentries = append(expiredDentries, d)
}
}
}
var getPath = func() (path []string) {
for _, d := range expiredDentries {
path = append(path, d.Path)
}
return
}
log.LogDebugf("batchHandleFile num: %v, expired num: %v, expired path: %v", len(inodesInfo), len(expiredDentries), getPath())
if len(expiredDentries) > 0 {
s.mw.BatchDelete_ll(expiredDentries)
atomic.AddInt64(&s.currentStat.ExpiredNum, int64(len(expiredDentries)))
}
}
func (s *LcScanner) inodeExpired(inode *proto.InodeInfo, cond *proto.ExpirationConfig) bool {
if inode == nil || cond == nil {
return false
}
now := s.now.Unix()
if cond.Days > 0 {
if now-inode.ModifyTime.Unix() < int64(cond.Days*24*60*60) {
return false
}
}
if cond.Date != nil {
if now < cond.Date.Unix() {
return false
}
}
return true
}
//scan dir tree in depth when size of dirChan.In grow too much.
//consider 40 Bytes is the ave size of dentry, 100 million ScanDentries may take up to around 4GB of Memory
func (s *LcScanner) handleDirLimitDepthFirst(dentry *proto.ScanDentry) {
log.LogDebugf("handleDirLimitDepthFirst dentry: %+v, dirChan.Len: %v", dentry, s.dirChan.Len())
marker := ""
done := false
for !done {
children, err := s.mw.ReadDirLimit_ll(dentry.Inode, marker, uint64(defaultReadDirLimit))
if err != nil && err != syscall.ENOENT {
atomic.AddInt64(&s.currentStat.ErrorSkippedNum, 1)
log.LogErrorf("handleDirLimitDepthFirst ReadDirLimit_ll err %v, dentry %v, marker %v", err, dentry, marker)
return
}
if marker == "" {
atomic.AddInt64(&s.currentStat.DirScannedNum, 1)
atomic.AddInt64(&s.currentStat.TotalInodeScannedNum, 1)
}
if err == syscall.ENOENT {
done = true
break
}
if marker != "" {
if len(children) >= 1 && marker == children[0].Name {
if len(children) <= 1 {
done = true
break
} else {
children = children[1:]
}
}
}
files := make([]*proto.ScanDentry, 0)
dirs := make([]*proto.ScanDentry, 0)
for _, child := range children {
childDentry := &proto.ScanDentry{
ParentId: dentry.Inode,
Name: child.Name,
Inode: child.Inode,
Path: strings.TrimPrefix(dentry.Path+pathSep+child.Name, pathSep),
Type: child.Type,
}
if os.FileMode(childDentry.Type).IsDir() {
dirs = append(dirs, childDentry)
} else {
files = append(files, childDentry)
}
}
for _, file := range files {
s.fileChan.In <- file
}
for _, dir := range dirs {
s.handleDirLimitDepthFirst(dir)
}
childrenNr := len(children)
if (marker == "" && childrenNr < defaultReadDirLimit) || (marker != "" && childrenNr+1 < defaultReadDirLimit) {
done = true
} else {
marker = children[childrenNr-1].Name
}
}
}
func (s *LcScanner) handleDirLimitBreadthFirst(dentry *proto.ScanDentry) {
log.LogDebugf("handleDirLimitBreadthFirst dentry: %+v, dirChan.Len: %v", dentry, s.dirChan.Len())
marker := ""
done := false
for !done {
children, err := s.mw.ReadDirLimit_ll(dentry.Inode, marker, uint64(defaultReadDirLimit))
if err != nil && err != syscall.ENOENT {
atomic.AddInt64(&s.currentStat.ErrorSkippedNum, 1)
log.LogErrorf("handleDirLimitBreadthFirst ReadDirLimit_ll err %v, dentry %v, marker %v", err, dentry, marker)
return
}
if marker == "" {
atomic.AddInt64(&s.currentStat.DirScannedNum, 1)
atomic.AddInt64(&s.currentStat.TotalInodeScannedNum, 1)
}
if err == syscall.ENOENT {
done = true
break
}
if marker != "" {
if len(children) >= 1 && marker == children[0].Name {
if len(children) <= 1 {
done = true
break
} else {
children = children[1:]
}
}
}
for _, child := range children {
childDentry := &proto.ScanDentry{
ParentId: dentry.Inode,
Name: child.Name,
Inode: child.Inode,
Path: strings.TrimPrefix(dentry.Path+pathSep+child.Name, pathSep),
Type: child.Type,
}
if !os.FileMode(childDentry.Type).IsDir() {
s.fileChan.In <- childDentry
} else {
s.dirChan.In <- childDentry
}
}
childrenNr := len(children)
if (marker == "" && childrenNr < defaultReadDirLimit) || (marker != "" && childrenNr+1 < defaultReadDirLimit) {
done = true
} else {
marker = children[childrenNr-1].Name
}
}
}
func (s *LcScanner) checkScanning() {
dur := time.Second * time.Duration(scanCheckInterval)
taskCheckTimer := time.NewTimer(dur)
for {
select {
case <-s.stopC:
log.LogInfof("stop checking scan")
return
case <-taskCheckTimer.C:
if s.DoneScanning() {
if s.batchDentries.Len() > 0 {
log.LogInfof("checkScanning last batchDentries")
s.batchHandleFile()
} else {
log.LogInfof("checkScanning completed for task(%v)", s.adminTask)
taskCheckTimer.Stop()
t := time.Now()
response := s.adminTask.Response.(*proto.LcNodeRuleTaskResponse)
response.EndTime = &t
response.Status = proto.TaskSucceeds
response.Done = true
response.ID = s.ID
response.Volume = s.Volume
response.RuleId = s.rule.ID
response.ExpiredNum = s.currentStat.ExpiredNum
response.FileScannedNum = s.currentStat.FileScannedNum
response.DirScannedNum = s.currentStat.DirScannedNum
response.TotalInodeScannedNum = s.currentStat.TotalInodeScannedNum
response.ErrorSkippedNum = s.currentStat.ErrorSkippedNum
s.lcnode.scannerMutex.Lock()
delete(s.lcnode.lcScanners, s.ID)
s.Stop()
s.lcnode.scannerMutex.Unlock()
s.lcnode.respondToMaster(s.adminTask)
return
}
}
taskCheckTimer.Reset(dur)
}
}
}
func (s *LcScanner) DoneScanning() bool {
log.LogInfof("dirChan.Len(%v) fileChan.Len(%v) fileRPoll.RunningNum(%v) dirRPoll.RunningNum(%v)",
s.dirChan.Len(), s.fileChan.Len(), s.fileRPoll.RunningNum(), s.dirRPoll.RunningNum())
return s.dirChan.Len() == 0 && s.fileChan.Len() == 0 && s.fileRPoll.RunningNum() == 0 && s.dirRPoll.RunningNum() == 0
}
func (s *LcScanner) Stop() {
close(s.stopC)
s.fileRPoll.WaitAndClose()
s.dirRPoll.WaitAndClose()
close(s.dirChan.In)
close(s.fileChan.In)
log.LogInfof("scanner(%v) stopped", s.ID)
}

278
lcnode/lcnode.go
View File

@ -1,278 +0,0 @@
package lcnode
import (
"errors"
"fmt"
"github.com/cubefs/cubefs/cmd/common"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/sdk/master"
"github.com/cubefs/cubefs/util"
"github.com/cubefs/cubefs/util/config"
"github.com/cubefs/cubefs/util/exporter"
"github.com/cubefs/cubefs/util/log"
"net/http"
"regexp"
"runtime"
"strconv"
"strings"
"sync"
"time"
)
const (
// String type configuration type, used to configure the listening port number of the service.
// Example:
// {
// "listen": "80"
// }
configListen = proto.ListenPort
// String array configuration item, used to configure the hostname or IP address of the cluster master node.
// The ObjectNode needs to communicate with the Master during the startup and running process to update the
// cluster, user and volume information.
// Example:
// {
// "masterAddr":[
// "master1.chubao.io",
// "master2.chubao.io",
// "master3.chubao.io"
// ]
// }
configMasterAddr = proto.MasterAddr
batchExpirationGetNumStr = "batchExpirationGetNum"
scanCheckIntervalStr = "scanCheckInterval"
s3ScanRoutineNumPerTaskStr = "s3ScanRoutineNumPerTask"
snapShotRoutineNumPerTaskStr = "snapShotRoutineNumPerTask"
)
// Default of configuration value
const (
defaultListen = "80"
ModuleName = "lcNode"
defaultBatchExpirationGetNum = 100
maxBatchExpirationGetNum = 10000
defaultScanCheckInterval = 60
defaultMasterIntervalToCheckHeartbeat = 6
noHeartBeatTimes = 3 // number of times that no heartbeat reported
defaultLcNodeTimeOutSec = noHeartBeatTimes * defaultMasterIntervalToCheckHeartbeat
defaultIntervalToCheckRegister = 2 * defaultLcNodeTimeOutSec
)
var (
// Regular expression used to verify the configuration of the service listening port.
// A valid service listening port configuration is a string containing only numbers.
regexpListen = regexp.MustCompile("^(\\d)+$")
batchExpirationGetNum int
scanCheckInterval int64
s3ScanRoutineNumPerTask int
snapShotRoutingNumPerTask int
)
type LcNode struct {
listen string
localServerAddr string
clusterID string
nodeID uint64
httpServer *http.Server
//tcpListener net.Listener
masters []string
mc *master.MasterClient
scannerMutex sync.RWMutex
s3Scanners map[string]*S3Scanner
snapshotScanners map[string]*SnapshotScanner
stopC chan bool
lastHeartbeat time.Time
control common.Control
}
func NewServer() *LcNode {
return &LcNode{}
}
func (l *LcNode) Start(cfg *config.Config) (err error) {
runtime.GOMAXPROCS(runtime.NumCPU())
return l.control.Start(l, cfg, handleStart)
}
// Shutdown shuts down the current lcnode.
func (l *LcNode) Shutdown() {
l.control.Shutdown(l, handleShutdown)
}
// Sync keeps lcnode in sync.
func (l *LcNode) Sync() {
l.control.Sync()
}
func (l *LcNode) loadConfig(cfg *config.Config) (err error) {
// parse listen
listen := cfg.GetString(configListen)
if len(listen) == 0 {
listen = defaultListen
}
if match := regexpListen.MatchString(listen); !match {
err = errors.New("invalid listen configuration")
return
}
l.listen = listen
log.LogInfof("loadConfig: setup config: %v(%v)", configListen, listen)
// parse master config
masters := cfg.GetStringSlice(configMasterAddr)
if len(masters) == 0 {
return config.NewIllegalConfigError(configMasterAddr)
}
log.LogInfof("loadConfig: setup config: %v(%v)", configMasterAddr, strings.Join(masters, ","))
l.masters = masters
l.mc = master.NewMasterClient(masters, false)
l.s3Scanners = make(map[string]*S3Scanner, 0)
l.snapshotScanners = make(map[string]*SnapshotScanner, 0)
begns := cfg.GetString(batchExpirationGetNumStr)
var batchNum int64
if begns != "" {
if batchNum, err = strconv.ParseInt(begns, 10, 64); err != nil {
return fmt.Errorf("%v,err:%v", proto.ErrInvalidCfg, err.Error())
}
}
batchExpirationGetNum = int(batchNum)
if batchExpirationGetNum <= 0 || batchExpirationGetNum > maxBatchExpirationGetNum {
batchExpirationGetNum = defaultBatchExpirationGetNum
}
log.LogInfof("loadConfig: setup config: %v(%v)", batchExpirationGetNumStr, batchExpirationGetNum)
scis := cfg.GetString(scanCheckIntervalStr)
if scis != "" {
if scanCheckInterval, err = strconv.ParseInt(scis, 10, 64); err != nil {
return fmt.Errorf("%v,err:%v", proto.ErrInvalidCfg, err.Error())
}
}
if scanCheckInterval <= 0 {
scanCheckInterval = defaultScanCheckInterval
}
log.LogInfof("loadConfig: setup config: %v(%v)", scanCheckIntervalStr, scanCheckInterval)
var routineNum int64
s3RoutineNum := cfg.GetString(s3ScanRoutineNumPerTaskStr)
if s3RoutineNum != "" {
if routineNum, err = strconv.ParseInt(s3RoutineNum, 10, 64); err != nil {
return fmt.Errorf("%v,err:%v", proto.ErrInvalidCfg, err.Error())
}
}
s3ScanRoutineNumPerTask = int(routineNum)
if s3ScanRoutineNumPerTask <= 0 || s3ScanRoutineNumPerTask > maxRoutineNumPerTask {
s3ScanRoutineNumPerTask = defaultS3ScanRoutineNumPerTask
}
log.LogInfof("loadConfig: setup config: %v(%v)", s3ScanRoutineNumPerTaskStr, s3ScanRoutineNumPerTask)
routineNum = 0
snapRoutineNum := cfg.GetString(snapShotRoutineNumPerTaskStr)
if snapRoutineNum != "" {
if routineNum, err = strconv.ParseInt(snapRoutineNum, 10, 64); err != nil {
return fmt.Errorf("%v,err:%v", proto.ErrInvalidCfg, err.Error())
}
}
snapShotRoutingNumPerTask = int(routineNum)
if snapShotRoutingNumPerTask <= 0 || snapShotRoutingNumPerTask > maxRoutineNumPerTask {
snapShotRoutingNumPerTask = defaultSnapShotRoutineNumPerTask
}
log.LogInfof("loadConfig: setup config: %v(%v)", snapShotRoutineNumPerTaskStr, snapShotRoutingNumPerTask)
return
}
// register the lcnode on the master to report the information such as IsIPV4 address.
// The startup of a lcnode will be blocked until the registration succeeds.
func (l *LcNode) register(cfg *config.Config) {
var (
err error
)
timer := time.NewTimer(0)
// get the IsIPV4 address, cluster ID and node ID from the master
for {
select {
case <-timer.C:
var ci *proto.ClusterInfo
if ci, err = l.mc.AdminAPI().GetClusterInfo(); err != nil {
log.LogErrorf("action[registerToMaster] cannot get ip from master(%v) err(%v).",
l.mc.Leader(), err)
timer.Reset(2 * time.Second)
continue
}
masterAddr := l.mc.Leader()
l.clusterID = ci.Cluster
localIP := string(ci.Ip)
l.localServerAddr = fmt.Sprintf("%s:%v", localIP, l.listen)
if !util.IsIPV4(localIP) {
log.LogErrorf("action[registerToMaster] got an invalid local ip(%v) from master(%v).",
localIP, masterAddr)
timer.Reset(2 * time.Second)
continue
}
// register this lcnode on the master
var nodeID uint64
if nodeID, err = l.mc.NodeAPI().AddLcNode(l.localServerAddr); err != nil {
log.LogErrorf("action[registerToMaster] cannot register this node to master[%v] err(%v).",
masterAddr, err)
timer.Reset(2 * time.Second)
continue
}
//exporter.RegistConsul(l.clusterID, ModuleName, cfg)
l.nodeID = nodeID
log.LogInfof("register: register LcNode: nodeID(%v)", l.nodeID)
return
case <-l.stopC:
timer.Stop()
return
}
}
}
func (l *LcNode) checkRegister(cfg *config.Config) {
for {
if time.Since(l.lastHeartbeat) > time.Second*time.Duration(defaultLcNodeTimeOutSec) {
l.StopScanners()
log.LogWarnf("Lcnode might be deregistered from master, retry registering...")
l.register(cfg)
l.lastHeartbeat = time.Now()
}
time.Sleep(time.Second * defaultIntervalToCheckRegister)
}
}
func handleStart(s common.Server, cfg *config.Config) (err error) {
l, ok := s.(*LcNode)
if !ok {
return errors.New("Invalid node Type!")
}
l.stopC = make(chan bool, 0)
// parse config
if err = l.loadConfig(cfg); err != nil {
return
}
l.register(cfg)
l.lastHeartbeat = time.Now()
go l.checkRegister(cfg)
if err = l.startServer(); err != nil {
return
}
exporter.Init(ModuleName, cfg)
exporter.RegistConsul(l.clusterID, ModuleName, cfg)
log.LogInfo("lcnode start successfully")
return
}
func handleShutdown(s common.Server) {
l, ok := s.(*LcNode)
if !ok {
return
}
//close(l.stopC)
l.stopServer()
}

550
lcnode/s3_scanner.go
View File

@ -1,550 +0,0 @@
package lcnode
import (
"errors"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/sdk/meta"
"github.com/cubefs/cubefs/util/log"
"github.com/cubefs/cubefs/util/routinepool"
"github.com/cubefs/cubefs/util/unboundedchan"
"os"
"strings"
"sync/atomic"
"syscall"
"time"
)
type S3Scanner struct {
ID string
RoutineID int64
Volume string
mw *meta.MetaWrapper
lcnode *LcNode
adminTask *proto.AdminTask
filter *proto.ScanFilter
abortFilter *proto.AbortIncompleteMultiPartFilter
dirChan *unboundedchan.UnboundedChan
fileChan *unboundedchan.UnboundedChan
fileRPoll *routinepool.RoutinePool
dirRPoll *routinepool.RoutinePool
batchDentries *fileDentries
currentStat proto.S3TaskStatistics
stopC chan bool
}
//func NewS3Scanner(scanTask *proto.RuleTask, RoutineID int64, l *LcNode) (*S3Scanner, error) {
func NewS3Scanner(adminTask *proto.AdminTask, l *LcNode) (*S3Scanner, error) {
request := adminTask.Request.(*proto.RuleTaskRequest)
scanTask := request.Task
var err error
filter, abortFilter := scanTask.Rule.GetScanFilter()
if filter == nil && abortFilter == nil {
return nil, errors.New("invalid rule")
}
var metaConfig = &meta.MetaConfig{
Volume: scanTask.VolName,
Masters: l.masters,
Authenticate: false,
ValidateOwner: false,
}
var metaWrapper *meta.MetaWrapper
if metaWrapper, err = meta.NewMetaWrapper(metaConfig); err != nil {
return nil, err
}
scanner := &S3Scanner{
ID: scanTask.Id,
RoutineID: request.RoutineID,
Volume: scanTask.VolName,
mw: metaWrapper,
lcnode: l,
adminTask: adminTask,
filter: filter,
abortFilter: abortFilter,
dirChan: unboundedchan.NewUnboundedChan(defaultUChanInitCapacity),
fileChan: unboundedchan.NewUnboundedChan(defaultUChanInitCapacity),
fileRPoll: routinepool.NewRoutinePool(int(s3ScanRoutineNumPerTask)),
dirRPoll: routinepool.NewRoutinePool(int(s3ScanRoutineNumPerTask)),
stopC: make(chan bool, 0),
}
scanner.batchDentries = newFileDentries()
return scanner, nil
}
func (s *S3Scanner) FindPrefixInode() (inode uint64, mode uint32, err error) {
// if prefix and marker are both not empty, use marker
var dirs []string
if s.filter.Prefix != "" {
dirs = strings.Split(s.filter.Prefix, "/")
log.LogDebugf("FindPrefixInode: Prefix(%v), dirs(%v) dirs len(%v)", s.filter.Prefix, dirs, len(dirs))
}
if len(dirs) <= 0 {
return proto.RootIno, 0, nil
}
var parentId = proto.RootIno
for _, dir := range dirs {
if dir == "" {
continue
}
curIno, curMode, err := s.mw.Lookup_ll(parentId, dir)
// If the part except the last part does not match exactly the same dentry, there is
// no path matching the path prefix. An ENOENT error is returned to the caller.
if err == syscall.ENOENT {
log.LogErrorf("FindPrefixInode: find directories fail: parentId(%v) dir(%v)", parentId, dir)
return 0, 0, syscall.ENOENT
}
if err != nil && err != syscall.ENOENT {
log.LogErrorf("FindPrefixInode: find directories fail: prefix(%v) err(%v)", s.filter.Prefix, err)
return 0, 0, err
}
//if !os.FileMode(curMode).IsDir() {
// return 0, syscall.ENOENT
//}
parentId = curIno
inode = curIno
mode = curMode
}
return
}
func (s *S3Scanner) Stop() {
close(s.stopC)
s.fileRPoll.WaitAndClose()
s.dirRPoll.WaitAndClose()
close(s.dirChan.In)
close(s.fileChan.In)
log.LogDebugf("scanner(%v) stopped", s.ID)
}
func (s *S3Scanner) Start() (err error) {
request := s.adminTask.Request.(*proto.RuleTaskRequest)
response := s.adminTask.Response.(*proto.RuleTaskResponse)
perfixInode, mode, err := s.FindPrefixInode()
if err != nil {
log.LogWarnf("startScan: node path(%v) found in volume(%v), err(%v), scanning done!",
s.filter.Prefix, request.Task.VolName, err)
t := time.Now()
response.EndTime = &t
response.Status = proto.TaskSucceeds
s.lcnode.scannerMutex.Lock()
delete(s.lcnode.s3Scanners, s.ID)
s.lcnode.scannerMutex.Unlock()
//l.respondToMaster(s.adminTask)
return
}
go s.scan()
prefixDentry := &proto.ScanDentry{
Inode: perfixInode,
Type: mode,
}
t := time.Now()
response.StartTime = &t
if os.FileMode(mode).IsDir() {
log.LogDebugf("startScan: first dir entry(%v) in!", prefixDentry)
s.dirChan.In <- prefixDentry
} else {
log.LogDebugf("startScan: first file entry(%v) in!", prefixDentry)
s.fileChan.In <- prefixDentry
}
if s.abortFilter != nil {
scanMultipart := func() {
s.incompleteMultiPartScan(response)
}
_, _ = s.fileRPoll.Submit(scanMultipart)
}
go s.checkScanning()
return
}
func (s *S3Scanner) DoneScanning() bool {
log.LogDebugf("dirChan.Len(%v) fileChan.Len(%v) fileRPoll.RunningNum(%v) dirRPoll.RunningNum(%v)",
s.dirChan.Len(), s.fileChan.Len(), s.fileRPoll.RunningNum(), s.dirRPoll.RunningNum())
return s.dirChan.Len() == 0 && s.fileChan.Len() == 0 && s.fileRPoll.RunningNum() == 0 && s.dirRPoll.RunningNum() == 0
}
func (s *S3Scanner) batchAbortIncompleteMultiPart(expriredInfos []*proto.ExpiredMultipartInfo, resp *proto.RuleTaskResponse) (err error) {
for _, info := range expriredInfos {
// release part data
for _, inode := range info.Inodes {
log.LogWarnf("BatchAbortIncompleteMultiPart: unlink part inode: path(%v) multipartID(%v) inode(%v)",
info.Path, info.MultipartId, inode)
if _, err = s.mw.InodeUnlink_ll(inode); err != nil {
log.LogErrorf("BatchAbortIncompleteMultiPart: meta inode unlink fail: path(%v) multipartID(%v)inode(%v) err(%v)",
info.Path, info.MultipartId, inode, err)
}
log.LogWarnf("BatchAbortIncompleteMultiPart: evict part inode: path(%v) multipartID(%v) inode(%v)",
info.Path, info.MultipartId, inode)
if err = s.mw.Evict(inode); err != nil {
log.LogErrorf("BatchAbortIncompleteMultiPart: meta inode evict fail: path(%v) multipartID(%v)inode(%v) err(%v)",
info.Path, info.MultipartId, inode, err)
}
log.LogDebugf("BatchAbortIncompleteMultiPart: multipart part data released: path(%v) multipartID(%v)inode(%v)",
info.Path, info.MultipartId, inode)
}
if err = s.mw.RemoveMultipart_ll(info.Path, info.MultipartId); err != nil {
log.LogErrorf("BatchAbortIncompleteMultiPart: meta abort multipart fail: path(%v) multipartID(%v) err(%v)",
info.Path, info.MultipartId, err)
return err
}
atomic.AddInt64(&resp.AbortedIncompleteMultipartNum, 1)
}
return nil
}
func (s *S3Scanner) incompleteMultiPartScan(resp *proto.RuleTaskResponse) {
expriredInfos, err := s.mw.BatchGetExpiredMultipart(s.abortFilter.Prefix, s.abortFilter.DaysAfterInitiation)
if err == syscall.ENOENT {
log.LogDebugf("incompleteMultiPartScan: no expired multipart")
return
}
_ = s.batchAbortIncompleteMultiPart(expriredInfos, resp)
}
func (s *S3Scanner) handlFile(dentry *proto.ScanDentry) {
log.LogDebugf("scan file dentry(%v)", dentry)
if dentry.DelInode {
_, err := s.mw.Delete_ll(dentry.ParentId, dentry.Name, false)
if err != nil {
log.LogErrorf("delete dentry(%v) failed, err(%v)", dentry, err)
} else {
log.LogDebugf("dentry(%v) deleted!", dentry)
//atomic.AddInt64(&resp.ExpiredNum, 1)
atomic.AddInt64(&s.currentStat.ExpiredNum, 1)
}
} else {
s.batchDentries.Append(dentry)
if s.batchDentries.Len() >= batchExpirationGetNum {
expireInfos := s.mw.BatchInodeExpirationGet(s.batchDentries.GetDentries(), s.filter.ExpireCond)
for _, info := range expireInfos {
if info.Expired {
info.Dentry.DelInode = true
s.fileChan.In <- info.Dentry
}
}
var scannedNum int64 = int64(s.batchDentries.Len())
atomic.AddInt64(&s.currentStat.FileScannedNum, scannedNum)
atomic.AddInt64(&s.currentStat.TotalInodeScannedNum, scannedNum)
s.batchDentries.Clear()
}
}
}
func (s *S3Scanner) handleDir(dentry *proto.ScanDentry, resp *proto.RuleTaskResponse) {
log.LogDebugf("scan dir dentry(%v)", dentry)
children, err := s.mw.ReadDir_ll(dentry.Inode)
if err != nil && err != syscall.ENOENT {
atomic.AddInt64(&resp.ErrorSkippedNum, 1)
log.LogWarnf("ReadDir_ll failed")
return
}
if err == syscall.ENOENT {
return
}
atomic.AddInt64(&resp.DirScannedNum, 1)
atomic.AddInt64(&resp.TotalInodeScannedNum, 1)
for _, child := range children {
childDentry := &proto.ScanDentry{
DelInode: false,
ParentId: dentry.Inode,
Name: child.Name,
Inode: child.Inode,
Type: child.Type,
}
if os.FileMode(childDentry.Type).IsDir() {
s.dirChan.In <- childDentry
} else {
s.fileChan.In <- childDentry
}
}
}
func (s *S3Scanner) handleDirLimit(dentry *proto.ScanDentry) {
log.LogDebugf("scan dir dentry(%v)", dentry)
marker := ""
done := false
for !done {
children, err := s.mw.ReadDirLimit_ll(dentry.Inode, marker, uint64(s3ScanRoutineNumPerTask))
if err != nil && err != syscall.ENOENT {
atomic.AddInt64(&s.currentStat.ErrorSkippedNum, 1)
log.LogWarnf("ReadDir_ll failed")
return
}
if marker == "" {
atomic.AddInt64(&s.currentStat.DirScannedNum, 1)
atomic.AddInt64(&s.currentStat.TotalInodeScannedNum, 1)
}
if err == syscall.ENOENT {
done = true
break
}
if marker != "" {
if len(children) >= 1 && marker == children[0].Name {
if len(children) <= 1 {
done = true
break
} else {
children = children[1:]
}
}
}
for _, child := range children {
childDentry := &proto.ScanDentry{
DelInode: false,
ParentId: dentry.Inode,
Name: child.Name,
Inode: child.Inode,
Type: child.Type,
}
if os.FileMode(childDentry.Type).IsDir() {
s.dirChan.In <- childDentry
} else {
s.fileChan.In <- childDentry
}
}
childrenNr := len(children)
if (marker == "" && childrenNr < s3ScanRoutineNumPerTask) || (marker != "" && childrenNr+1 < s3ScanRoutineNumPerTask) {
done = true
} else {
marker = children[childrenNr-1].Name
}
}
}
//handleDirLimitDepthFirst used to scan dir tree in depth when size of dirChan.In grow too much.
//consider 40 Bytes is the ave size of proto.ScanDentry, 100 million ScanDentries may take up to around 4GB of Memory
func (s *S3Scanner) handleDirLimitDepthFirst(dentry *proto.ScanDentry) {
log.LogDebugf("scan dir dentry(%v)", dentry)
marker := ""
done := false
for !done {
children, err := s.mw.ReadDirLimit_ll(dentry.Inode, marker, uint64(s3ScanRoutineNumPerTask))
if err != nil && err != syscall.ENOENT {
atomic.AddInt64(&s.currentStat.ErrorSkippedNum, 1)
log.LogWarnf("ReadDir_ll failed")
return
}
if marker == "" {
atomic.AddInt64(&s.currentStat.DirScannedNum, 1)
atomic.AddInt64(&s.currentStat.TotalInodeScannedNum, 1)
}
if err == syscall.ENOENT {
done = true
break
}
if marker != "" {
if len(children) >= 1 && marker == children[0].Name {
if len(children) <= 1 {
done = true
break
} else {
children = children[1:]
}
}
}
files := make([]*proto.ScanDentry, 0)
dirs := make([]*proto.ScanDentry, 0)
for _, child := range children {
childDentry := &proto.ScanDentry{
DelInode: false,
ParentId: dentry.Inode,
Name: child.Name,
Inode: child.Inode,
Type: child.Type,
}
if os.FileMode(childDentry.Type).IsDir() {
dirs = append(dirs, childDentry)
} else {
files = append(files, childDentry)
}
}
for _, file := range files {
s.fileChan.In <- file
}
for _, dir := range dirs {
s.handleDirLimitDepthFirst(dir)
}
childrenNr := len(children)
if (marker == "" && childrenNr < s3ScanRoutineNumPerTask) || (marker != "" && childrenNr+1 < s3ScanRoutineNumPerTask) {
done = true
} else {
marker = children[childrenNr-1].Name
}
}
}
func (s *S3Scanner) getDirJob(dentry *proto.ScanDentry) (job func()) {
//no need to lock
if s.dirChan.Len()+s3ScanRoutineNumPerTask > defaultMaxChanUnm {
log.LogDebugf("getDirJob: Depth first job")
job = func() {
s.handleDirLimitDepthFirst(dentry)
}
} else {
log.LogDebugf("getDirJob: Breadth first job")
job = func() {
s.handleDirLimit(dentry)
}
}
return
}
func (s *S3Scanner) scan() {
log.LogDebugf("Enter scan")
defer func() {
log.LogDebugf("Exit scan")
}()
for {
select {
case <-s.stopC:
return
case val, ok := <-s.fileChan.Out:
if !ok {
log.LogWarnf("fileChan closed")
} else {
dentry := val.(*proto.ScanDentry)
job := func() {
s.handlFile(dentry)
}
_, err := s.fileRPoll.Submit(job)
if err != nil {
log.LogErrorf("handlFile failed, err(%v)", err)
}
}
default:
select {
case <-s.stopC:
return
case val, ok := <-s.fileChan.Out:
if !ok {
log.LogWarnf("fileChan closed")
} else {
dentry := val.(*proto.ScanDentry)
job := func() {
s.handlFile(dentry)
}
_, err := s.fileRPoll.Submit(job)
if err != nil {
log.LogErrorf("handlFile failed, err(%v)", err)
}
}
case val, ok := <-s.dirChan.Out:
if !ok {
log.LogWarnf("dirChan closed")
} else {
dentry := val.(*proto.ScanDentry)
job := s.getDirJob(dentry)
_, err := s.dirRPoll.Submit(job)
if err != nil {
log.LogErrorf("handleDir failed, err(%v)", err)
}
}
}
}
}
}
//func (s *S3Scanner) checkScanning(adminTask *proto.AdminTask, resp *proto.RuleTaskResponse) {
func (s *S3Scanner) checkScanning() {
dur := time.Second * time.Duration(scanCheckInterval)
taskCheckTimer := time.NewTimer(dur)
for {
select {
case <-s.stopC:
log.LogDebugf("stop checking scan")
return
case <-taskCheckTimer.C:
if s.DoneScanning() {
if s.batchDentries.Len() > 0 {
log.LogDebugf("last batch dentries scan")
expireInfos := s.mw.BatchInodeExpirationGet(s.batchDentries.GetDentries(), s.filter.ExpireCond)
for _, info := range expireInfos {
if info.Expired {
info.Dentry.DelInode = true
s.fileChan.In <- info.Dentry
}
}
var scannedNum int64 = int64(s.batchDentries.Len())
atomic.AddInt64(&s.currentStat.FileScannedNum, scannedNum)
atomic.AddInt64(&s.currentStat.TotalInodeScannedNum, scannedNum)
s.batchDentries.Clear()
} else {
log.LogInfof("scan completed for task(%v)", s.adminTask)
taskCheckTimer.Stop()
t := time.Now()
response := s.adminTask.Response.(*proto.RuleTaskResponse)
response.EndTime = &t
response.Status = proto.TaskSucceeds
response.Done = true
response.ID = s.ID
response.RoutineID = s.RoutineID
response.Volume = s.Volume
response.Prefix = s.filter.Prefix
response.ExpiredNum = s.currentStat.ExpiredNum
response.FileScannedNum = s.currentStat.FileScannedNum
response.DirScannedNum = s.currentStat.DirScannedNum
response.TotalInodeScannedNum = s.currentStat.TotalInodeScannedNum
response.AbortedIncompleteMultipartNum = s.currentStat.AbortedIncompleteMultipartNum
response.ErrorSkippedNum = s.currentStat.ErrorSkippedNum
s.lcnode.scannerMutex.Lock()
delete(s.lcnode.s3Scanners, s.ID)
s.Stop()
s.lcnode.scannerMutex.Unlock()
s.lcnode.respondToMaster(s.adminTask)
return
}
}
taskCheckTimer.Reset(dur)
}
}
}

626
lcnode/server.go
View File

@ -1,29 +1,259 @@
// Copyright 2023 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package lcnode
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net"
"runtime"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/cubefs/cubefs/cmd/common"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/sdk/master"
"github.com/cubefs/cubefs/util"
"github.com/cubefs/cubefs/util/config"
"github.com/cubefs/cubefs/util/errors"
"github.com/cubefs/cubefs/util/exporter"
"github.com/cubefs/cubefs/util/log"
"golang.org/x/time/rate"
)
//type ScanDentry struct {
// DelInode bool //if Type is file, and DelInode is true, then Inode and Dentry(ParentId, Name) is to be deleted
// ParentId uint64 // FileID value of the parent inode.
// Name string // Name of the current dentry.
// Inode uint64 // FileID value of the current inode.
// Type uint32
//}
type LcNode struct {
listen string
localServerAddr string
clusterID string
nodeID uint64
masters []string
mc *master.MasterClient
scannerMutex sync.RWMutex
stopC chan bool
lastHeartbeat time.Time
control common.Control
lcScanners map[string]*LcScanner
snapshotScanners map[string]*SnapshotScanner
}
func NewServer() *LcNode {
return &LcNode{
lcScanners: make(map[string]*LcScanner),
snapshotScanners: make(map[string]*SnapshotScanner),
}
}
func (l *LcNode) Start(cfg *config.Config) (err error) {
runtime.GOMAXPROCS(runtime.NumCPU())
return l.control.Start(l, cfg, doStart)
}
func (l *LcNode) Shutdown() {
l.control.Shutdown(l, doShutdown)
}
func (l *LcNode) Sync() {
l.control.Sync()
}
func doStart(s common.Server, cfg *config.Config) (err error) {
l, ok := s.(*LcNode)
if !ok {
return errors.New("Invalid node Type!")
}
l.stopC = make(chan bool, 0)
if err = l.parseConfig(cfg); err != nil {
return
}
l.register()
l.lastHeartbeat = time.Now()
go l.checkRegister()
if err = l.startServer(); err != nil {
return
}
exporter.Init(ModuleName, cfg)
exporter.RegistConsul(l.clusterID, ModuleName, cfg)
log.LogInfo("lcnode start successfully")
return
}
func doShutdown(s common.Server) {
l, ok := s.(*LcNode)
if !ok {
return
}
l.stopServer()
}
func (l *LcNode) parseConfig(cfg *config.Config) (err error) {
// parse listen
listen := cfg.GetString(configListen)
if len(listen) == 0 {
listen = defaultListen
}
if match := regexpListen.MatchString(listen); !match {
err = errors.New("invalid listen configuration")
return
}
l.listen = listen
log.LogInfof("loadConfig: setup config: %v(%v)", configListen, listen)
// parse master config
masters := cfg.GetStringSlice(configMasterAddr)
if len(masters) == 0 {
return config.NewIllegalConfigError(configMasterAddr)
}
log.LogInfof("loadConfig: setup config: %v(%v)", configMasterAddr, strings.Join(masters, ","))
l.masters = masters
l.mc = master.NewMasterClient(masters, false)
// parse batchExpirationGetNum
begns := cfg.GetString(configBatchExpirationGetNumStr)
var batchNum int64
if begns != "" {
if batchNum, err = strconv.ParseInt(begns, 10, 64); err != nil {
return fmt.Errorf("%v,err:%v", proto.ErrInvalidCfg, err.Error())
}
}
batchExpirationGetNum = int(batchNum)
if batchExpirationGetNum <= 0 || batchExpirationGetNum > maxBatchExpirationGetNum {
batchExpirationGetNum = defaultBatchExpirationGetNum
}
log.LogInfof("loadConfig: setup config: %v(%v)", configBatchExpirationGetNumStr, batchExpirationGetNum)
// parse scanCheckInterval
scis := cfg.GetString(configScanCheckIntervalStr)
if scis != "" {
if scanCheckInterval, err = strconv.ParseInt(scis, 10, 64); err != nil {
return fmt.Errorf("%v,err:%v", proto.ErrInvalidCfg, err.Error())
}
}
if scanCheckInterval <= 0 {
scanCheckInterval = defaultScanCheckInterval
}
log.LogInfof("loadConfig: setup config: %v(%v)", configScanCheckIntervalStr, scanCheckInterval)
// parse lcScanRoutineNumPerTask
var routineNum int64
lcScanRoutineNum := cfg.GetString(configLcScanRoutineNumPerTaskStr)
if lcScanRoutineNum != "" {
if routineNum, err = strconv.ParseInt(lcScanRoutineNum, 10, 64); err != nil {
return fmt.Errorf("%v,err:%v", proto.ErrInvalidCfg, err.Error())
}
}
lcScanRoutineNumPerTask = int(routineNum)
if lcScanRoutineNumPerTask <= 0 || lcScanRoutineNumPerTask > maxLcScanRoutineNumPerTask {
lcScanRoutineNumPerTask = defaultLcScanRoutineNumPerTask
}
log.LogInfof("loadConfig: setup config: %v(%v)", configLcScanRoutineNumPerTaskStr, lcScanRoutineNumPerTask)
// parse snapshotRoutineNumPerTask
routineNum = 0
snapRoutineNum := cfg.GetString(configSnapshotRoutineNumPerTaskStr)
if snapRoutineNum != "" {
if routineNum, err = strconv.ParseInt(snapRoutineNum, 10, 64); err != nil {
return fmt.Errorf("%v,err:%v", proto.ErrInvalidCfg, err.Error())
}
}
snapshotRoutineNumPerTask = int(routineNum)
if snapshotRoutineNumPerTask <= 0 || snapshotRoutineNumPerTask > maxLcScanRoutineNumPerTask {
snapshotRoutineNumPerTask = defaultLcScanRoutineNumPerTask
}
log.LogInfof("loadConfig: setup config: %v(%v)", configSnapshotRoutineNumPerTaskStr, snapshotRoutineNumPerTask)
// parse lcScanLimitPerSecond
var limitNum int64
lcScanLimit := cfg.GetString(configLcScanLimitPerSecondStr)
if lcScanLimit != "" {
if limitNum, err = strconv.ParseInt(lcScanLimit, 10, 64); err != nil {
return fmt.Errorf("%v,err:%v", proto.ErrInvalidCfg, err.Error())
}
}
if limitNum <= 0 {
lcScanLimitPerSecond = defaultLcScanLimitPerSecond
} else {
lcScanLimitPerSecond = rate.Limit(limitNum)
}
log.LogInfof("loadConfig: setup config: %v(%v)", configLcScanLimitPerSecondStr, lcScanLimitPerSecond)
return
}
func (l *LcNode) register() {
var err error
timer := time.NewTimer(0)
// get the IsIPV4 address, cluster ID and node ID from the master
for {
select {
case <-timer.C:
var ci *proto.ClusterInfo
if ci, err = l.mc.AdminAPI().GetClusterInfo(); err != nil {
log.LogErrorf("action[registerToMaster] cannot get ip from master(%v) err(%v).",
l.mc.Leader(), err)
timer.Reset(2 * time.Second)
continue
}
masterAddr := l.mc.Leader()
l.clusterID = ci.Cluster
localIP := ci.Ip
l.localServerAddr = fmt.Sprintf("%s:%v", localIP, l.listen)
if !util.IsIPV4(localIP) {
log.LogErrorf("action[registerToMaster] got an invalid local ip(%v) from master(%v).",
localIP, masterAddr)
timer.Reset(2 * time.Second)
continue
}
// register this lcnode on the master
var nodeID uint64
if nodeID, err = l.mc.NodeAPI().AddLcNode(l.localServerAddr); err != nil {
log.LogErrorf("action[registerToMaster] cannot register this node to master[%v] err(%v).",
masterAddr, err)
timer.Reset(2 * time.Second)
continue
}
l.nodeID = nodeID
log.LogInfof("register: register LcNode: nodeID(%v)", l.nodeID)
return
case <-l.stopC:
timer.Stop()
return
}
}
}
func (l *LcNode) checkRegister() {
for {
if time.Since(l.lastHeartbeat) > time.Second*time.Duration(defaultLcNodeTimeOutSec) {
l.stopScanners()
log.LogWarnf("Lcnode might be deregistered from master, retry registering...")
l.register()
l.lastHeartbeat = time.Now()
}
time.Sleep(time.Second * defaultIntervalToCheckRegister)
}
}
// startServer binds and listens to the specified port.
func (l *LcNode) startServer() (err error) {
log.LogInfo("Start: startServer")
addr := fmt.Sprintf(":%v", l.listen)
@ -53,33 +283,6 @@ func (l *LcNode) startServer() (err error) {
return
}
func (l *LcNode) stopServer() {
if l.stopC != nil {
defer func() {
if r := recover(); r != nil {
log.LogErrorf("action[StopTcpServer],err:%v", r)
}
}()
l.StopScanners()
close(l.stopC)
}
}
func (l *LcNode) StopScanners() {
l.scannerMutex.Lock()
defer l.scannerMutex.Unlock()
for _, s := range l.s3Scanners {
s.Stop()
delete(l.s3Scanners, s.ID)
}
for _, s := range l.snapshotScanners {
s.Stop()
delete(l.snapshotScanners, s.ID)
}
}
// Read data from the specified tcp connection until the connection is closed by the remote or the tcp service is down.
func (l *LcNode) serveConn(conn net.Conn, stopC chan bool) {
defer conn.Close()
c := conn.(*net.TCPConn)
@ -93,14 +296,14 @@ func (l *LcNode) serveConn(conn net.Conn, stopC chan bool) {
default:
}
p := &proto.Packet{}
if err := p.ReadFromConnWithVer(conn, proto.NoReadDeadlineTime); err != nil {
if err := p.ReadFromConn(conn, proto.NoReadDeadlineTime); err != nil {
if err != io.EOF {
log.LogError("serve LcNode: ", err.Error())
log.LogErrorf("serveConn ReadFromConn err: %v", err)
}
return
}
if err := l.handlePacket(conn, p, remoteAddr); err != nil {
log.LogErrorf("serve handlePacket fail: %v", err)
log.LogErrorf("serveConn handlePacket err: %v", err)
}
}
}
@ -111,9 +314,9 @@ func (l *LcNode) handlePacket(conn net.Conn, p *proto.Packet, remoteAddr string)
case proto.OpLcNodeHeartbeat:
err = l.opMasterHeartbeat(conn, p, remoteAddr)
case proto.OpLcNodeScan:
err = l.opLcScan(conn, p, remoteAddr)
err = l.opLcScan(conn, p)
case proto.OpLcNodeSnapshotVerDel:
err = l.opSnapshotVerDel(conn, p, remoteAddr)
err = l.opSnapshotVerDel(conn, p)
default:
err = fmt.Errorf("%s unknown Opcode: %d, reqId: %d", remoteAddr,
p.Opcode, p.GetReqID())
@ -125,331 +328,28 @@ func (l *LcNode) handlePacket(conn net.Conn, p *proto.Packet, remoteAddr string)
return
}
// Reply operation results to the master.
func (l *LcNode) respondToMaster(task *proto.AdminTask) (err error) {
// handle panic
defer func() {
if r := recover(); r != nil {
switch data := r.(type) {
case error:
err = data
default:
err = errors.New(data.(string))
func (l *LcNode) stopServer() {
if l.stopC != nil {
defer func() {
if r := recover(); r != nil {
log.LogErrorf("action[StopTcpServer],err:%v", r)
}
}
}()
if err = l.mc.NodeAPI().ResponseLcNodeTask(task); err != nil {
err = errors.Trace(err, "try respondToMaster failed")
}
return
}
}()
type fileDentries struct {
sync.RWMutex
dentries []*proto.ScanDentry
}
func newFileDentries() *fileDentries {
return &fileDentries{
dentries: make([]*proto.ScanDentry, 0),
l.stopScanners()
close(l.stopC)
}
}
func (f *fileDentries) GetDentries() []*proto.ScanDentry {
f.RLock()
defer f.RUnlock()
return f.dentries
}
func (f *fileDentries) Append(dentry *proto.ScanDentry) {
f.Lock()
defer f.Unlock()
f.dentries = append(f.dentries, dentry)
}
func (f *fileDentries) Len() int {
f.RLock()
defer f.RUnlock()
return len(f.dentries)
}
func (f *fileDentries) Clear() {
f.Lock()
defer f.Unlock()
f.dentries = f.dentries[:0]
}
func (l *LcNode) scanning() bool {
func (l *LcNode) stopScanners() {
l.scannerMutex.Lock()
defer l.scannerMutex.Unlock()
return len(l.s3Scanners) > 0
}
func (l *LcNode) startSnapshotScan(adminTask *proto.AdminTask) (err error) {
request := adminTask.Request.(*proto.SnapshotVerDelTaskRequest)
log.LogInfof("startSnapshotScan: scan task(%v) received!", request.Task)
response := &proto.SnapshotVerDelTaskResponse{}
adminTask.Response = response
defer func() {
if err != nil {
l.respondToMaster(adminTask)
}
}()
l.scannerMutex.Lock()
if _, ok := l.snapshotScanners[request.Task.Key()]; ok {
log.LogInfof("startSnapshotScan: scan task(%v) is already running!", request.Task)
l.scannerMutex.Unlock()
return
}
var scanner *SnapshotScanner
scanner, err = NewSnapshotScanner(adminTask, l)
if err != nil {
log.LogErrorf("startSnapshotScan: NewSnapshotScanner err(%v)", err)
response.Status = proto.TaskFailed
response.Result = err.Error()
l.scannerMutex.Unlock()
return
}
l.snapshotScanners[request.Task.Key()] = scanner
l.scannerMutex.Unlock()
if err = scanner.Start(); err != nil {
return
}
return
}
func (l *LcNode) startS3Scan(adminTask *proto.AdminTask) (err error) {
request := adminTask.Request.(*proto.RuleTaskRequest)
log.LogInfof("startS3Scan: scan task(%v) of routine(%v) received!", request.Task, request.RoutineID)
resp := &proto.RuleTaskResponse{}
adminTask.Response = resp
defer func() {
if err != nil {
//resp.Status = proto.TaskFailed
//resp.Result = err.Error()
//adminTask.Request = nil
l.respondToMaster(adminTask)
}
}()
l.scannerMutex.Lock()
if _, ok := l.s3Scanners[request.Task.Id]; ok {
log.LogInfof("startS3Scan: scan task(%v) of routine(%v) is already running!", request.Task, request.RoutineID)
l.scannerMutex.Unlock()
return
}
var scanner *S3Scanner
//scanner, err = NewS3Scanner(request.Task, request.RoutineID, l)
scanner, err = NewS3Scanner(adminTask, l)
if err != nil {
log.LogErrorf("startS3Scan: NewS3Scanner err(%v)", err)
resp.Status = proto.TaskFailed
resp.Result = err.Error()
l.scannerMutex.Unlock()
return
}
l.s3Scanners[scanner.ID] = scanner
l.scannerMutex.Unlock()
if err = scanner.Start(); err != nil {
return
}
/*var perfixInode uint64
var mode uint32
perfixInode, mode, err = scanner.FindPrefixInode()
if err != nil {
log.LogWarnf("startScan: node path(%v) found in volume(%v), err(%v), scanning done!",
scanner.filter.Prefix, request.Task.VolName, err)
t := time.Now()
resp.EndTime = &t
resp.Status = proto.TaskSucceeds
l.scannerMutex.Lock()
delete(l.s3Scanners, scanner.ID)
l.scannerMutex.Unlock()
return
}
go scanner.scan()
prefixDentry := &proto.ScanDentry{
Inode: perfixInode,
Type: mode,
}
t := time.Now()
resp.StartTime = &t
if os.FileMode(mode).IsDir() {
log.LogDebugf("startScan: first dir entry(%v) in!", prefixDentry)
scanner.dirChan.In <- prefixDentry
} else {
log.LogDebugf("startScan: first file entry(%v) in!", prefixDentry)
scanner.fileChan.In <- prefixDentry
}
if scanner.abortFilter != nil {
scanMultipart := func() {
scanner.incompleteMultiPartScan(resp)
}
_, _ = scanner.fileRPoll.Submit(scanMultipart)
}
go scanner.checkScanning(adminTask, resp)*/
return err
}
func (l *LcNode) opSnapshotVerDel(conn net.Conn, p *proto.Packet, remoteAddr string) (err error) {
go func() {
p.PacketOkReply()
if err := p.WriteToConn(conn); err != nil {
log.LogErrorf("ack master response: %s", err.Error())
}
}()
data := p.Data
var (
req = &proto.SnapshotVerDelTaskRequest{}
resp = &proto.SnapshotVerDelTaskResponse{}
adminTask = &proto.AdminTask{
Request: req,
}
)
decoder := json.NewDecoder(bytes.NewBuffer(data))
decoder.UseNumber()
if err = decoder.Decode(adminTask); err != nil {
resp.Status = proto.TaskFailed
resp.Result = err.Error()
//adminTask.Request = nil
adminTask.Response = resp
_ = l.respondToMaster(adminTask)
} else {
err = l.startSnapshotScan(adminTask)
}
return
}
func (l *LcNode) opLcScan(conn net.Conn, p *proto.Packet, remoteAddr string) (err error) {
go func() {
p.PacketOkReply()
if err := p.WriteToConn(conn); err != nil {
log.LogErrorf("ack master response: %s", err.Error())
}
}()
data := p.Data
var (
req = &proto.RuleTaskRequest{}
resp = &proto.RuleTaskResponse{}
adminTask = &proto.AdminTask{
Request: req,
}
)
decoder := json.NewDecoder(bytes.NewBuffer(data))
decoder.UseNumber()
if err = decoder.Decode(adminTask); err != nil {
resp.Status = proto.TaskFailed
resp.Result = err.Error()
//adminTask.Request = nil
adminTask.Response = resp
_ = l.respondToMaster(adminTask)
} else {
err = l.startS3Scan(adminTask)
}
return
}
func (l *LcNode) opMasterHeartbeat(conn net.Conn, p *proto.Packet, remoteAddr string) (err error) {
go func() {
p.PacketOkReply()
if err := p.WriteToConn(conn); err != nil {
log.LogErrorf("ack master response: %s", err.Error())
}
}()
data := p.Data
var (
req = &proto.HeartBeatRequest{}
resp = &proto.LcNodeHeartbeatResponse{
S3ScanningTasks: make(map[string]*proto.S3ScanInfo, 0),
SnapshotScanningTasks: make(map[string]*proto.SnapshotScanInfo, 0),
}
adminTask = &proto.AdminTask{
Request: req,
}
)
go func() {
start := time.Now()
decode := json.NewDecoder(bytes.NewBuffer(data))
decode.UseNumber()
if err = decode.Decode(adminTask); err != nil {
resp.Status = proto.TaskFailed
resp.Result = err.Error()
goto end
}
//collect status?
l.scannerMutex.RLock()
for _, scanner := range l.s3Scanners {
info := &proto.S3ScanInfo{
S3ScanTaskInfo: proto.S3ScanTaskInfo{
Id: scanner.ID,
RoutineId: scanner.RoutineID,
},
S3TaskStatistics: proto.S3TaskStatistics{
Volume: scanner.Volume,
Prefix: scanner.filter.Prefix,
TotalInodeScannedNum: atomic.LoadInt64(&scanner.currentStat.TotalInodeScannedNum),
FileScannedNum: atomic.LoadInt64(&scanner.currentStat.FileScannedNum),
DirScannedNum: atomic.LoadInt64(&scanner.currentStat.DirScannedNum),
ExpiredNum: atomic.LoadInt64(&scanner.currentStat.ExpiredNum),
ErrorSkippedNum: atomic.LoadInt64(&scanner.currentStat.ErrorSkippedNum),
AbortedIncompleteMultipartNum: atomic.LoadInt64(&scanner.currentStat.AbortedIncompleteMultipartNum),
},
}
resp.S3ScanningTasks[scanner.ID] = info
}
for _, scanner := range l.snapshotScanners {
info := &proto.SnapshotScanInfo{
DelVerTaskInfo: proto.DelVerTaskInfo{
Id: scanner.ID,
},
SnapshotStatistics: proto.SnapshotStatistics{
VerInfo: proto.VerInfo{
VolName: scanner.getTaskVolName(),
VerSeq: scanner.getTaskVerSeq(),
},
TotalInodeNum: atomic.LoadInt64(&scanner.currentStat.TotalInodeNum),
FileNum: atomic.LoadInt64(&scanner.currentStat.FileNum),
DirNum: atomic.LoadInt64(&scanner.currentStat.DirNum),
ErrorSkippedNum: atomic.LoadInt64(&scanner.currentStat.ErrorSkippedNum),
},
}
resp.SnapshotScanningTasks[scanner.ID] = info
}
l.scannerMutex.RUnlock()
resp.Status = proto.TaskSucceeds
end:
adminTask.Request = nil
adminTask.Response = resp
l.respondToMaster(adminTask)
data, _ := json.Marshal(resp)
log.LogInfof("%s pkt %s, resp success req:%v; respAdminTask: %v, resp: %v, cost %s",
remoteAddr, p.String(), req, adminTask, string(data), time.Since(start).String())
}()
l.lastHeartbeat = time.Now()
return
for _, s := range l.lcScanners {
s.Stop()
delete(l.lcScanners, s.ID)
}
for _, s := range l.snapshotScanners {
s.Stop()
delete(l.snapshotScanners, s.ID)
}
}

100
lcnode/snapshot_scanner.go
View File

@ -1,19 +1,33 @@
// Copyright 2023 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package lcnode
import (
"os"
"sync/atomic"
"syscall"
"time"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/sdk/meta"
"github.com/cubefs/cubefs/util/log"
"github.com/cubefs/cubefs/util/routinepool"
"github.com/cubefs/cubefs/util/unboundedchan"
"os"
"sync/atomic"
"syscall"
"time"
)
const (
//snap scan type
SnapScanTypeAll int = 0
SnapScanTypeOnlyFile int = 1
SnapScanTypeOnlyDepth int = 2
@ -56,8 +70,8 @@ func NewSnapshotScanner(adminTask *proto.AdminTask, l *LcNode) (*SnapshotScanner
mw: metaWrapper,
lcnode: l,
adminTask: adminTask,
inodeChan: unboundedchan.NewUnboundedChan(defaultUChanInitCapacity),
rPoll: routinepool.NewRoutinePool(snapShotRoutingNumPerTask),
inodeChan: unboundedchan.NewUnboundedChan(defaultUnboundedChanInitCapacity),
rPoll: routinepool.NewRoutinePool(snapshotRoutineNumPerTask),
currentStat: proto.SnapshotStatistics{},
scanType: SnapScanTypeAll,
stopC: make(chan bool, 0),
@ -65,6 +79,42 @@ func NewSnapshotScanner(adminTask *proto.AdminTask, l *LcNode) (*SnapshotScanner
return scanner, nil
}
func (l *LcNode) startSnapshotScan(adminTask *proto.AdminTask) (err error) {
request := adminTask.Request.(*proto.SnapshotVerDelTaskRequest)
log.LogInfof("startSnapshotScan: scan task(%v) received!", request.Task)
response := &proto.SnapshotVerDelTaskResponse{}
adminTask.Response = response
defer func() {
if err != nil {
l.respondToMaster(adminTask)
}
}()
l.scannerMutex.Lock()
if _, ok := l.snapshotScanners[request.Task.Key()]; ok {
log.LogInfof("startSnapshotScan: scan task(%v) is already running!", request.Task)
l.scannerMutex.Unlock()
return
}
var scanner *SnapshotScanner
scanner, err = NewSnapshotScanner(adminTask, l)
if err != nil {
log.LogErrorf("startSnapshotScan: NewSnapshotScanner err(%v)", err)
response.Status = proto.TaskFailed
response.Result = err.Error()
l.scannerMutex.Unlock()
return
}
l.snapshotScanners[request.Task.Key()] = scanner
l.scannerMutex.Unlock()
if err = scanner.Start(); err != nil {
return
}
return
}
func (s *SnapshotScanner) getTaskVolName() string {
request := s.adminTask.Request.(*proto.SnapshotVerDelTaskRequest)
return request.Task.VolName
@ -134,7 +184,7 @@ func (s *SnapshotScanner) getDirJob(dentry *proto.ScanDentry) (job func()) {
return
}
if s.inodeChan.Len()+snapShotRoutingNumPerTask > defaultMaxChanUnm {
if s.inodeChan.Len() > maxDirChanNum {
log.LogDebugf("getDirJob: Depth first job")
job = func() {
s.handlVerDelDepthFirst(dentry)
@ -179,7 +229,6 @@ func (s *SnapshotScanner) handlVerDelDepthFirst(dentry *proto.ScanDentry) {
err error
)
//scanDentries := make([]*proto.ScanDentry, 0)
if os.FileMode(dentry.Type).IsDir() {
marker := ""
done := false
@ -187,19 +236,19 @@ func (s *SnapshotScanner) handlVerDelDepthFirst(dentry *proto.ScanDentry) {
onlyDir := s.scanType == SnapScanTypeOnlyDir || s.scanType == SnapScanTypeOnlyDirAndDepth
for !done {
children, err = s.mw.ReadDirLimitByVer(dentry.Inode, marker, uint64(snapShotRoutingNumPerTask), s.getTaskVerSeq(), onlyDir)
children, err = s.mw.ReadDirLimitByVer(dentry.Inode, marker, uint64(snapshotRoutineNumPerTask), s.getTaskVerSeq(), onlyDir)
if err != nil && err != syscall.ENOENT {
log.LogErrorf("action[handlVerDelDepthFirst] ReadDirLimitByVer failed, parent[%v] maker[%v] limit[%v] verSeq[%v] err[%v]",
dentry.Inode, marker, uint64(snapShotRoutingNumPerTask), s.getTaskVerSeq(), err)
dentry.Inode, marker, uint64(snapshotRoutineNumPerTask), s.getTaskVerSeq(), err)
return
}
log.LogDebugf("action[handlVerDelDepthFirst] ReadDirLimitByVer parent[%v] maker[%v] limit[%v] verSeq[%v] children[%v]",
dentry.Inode, marker, uint64(snapShotRoutingNumPerTask), s.getTaskVerSeq(), len(children))
dentry.Inode, marker, uint64(snapshotRoutineNumPerTask), s.getTaskVerSeq(), len(children))
if err == syscall.ENOENT {
done = true
log.LogErrorf("action[handlVerDelDepthFirst] ReadDirLimitByVer failed, parent[%v] maker[%v] limit[%v] verSeq[%v] err[%v]",
dentry.Inode, marker, uint64(snapShotRoutingNumPerTask), s.getTaskVerSeq(), err)
dentry.Inode, marker, uint64(snapshotRoutineNumPerTask), s.getTaskVerSeq(), err)
break
}
@ -208,7 +257,7 @@ func (s *SnapshotScanner) handlVerDelDepthFirst(dentry *proto.ScanDentry) {
if len(children) <= 1 {
done = true
log.LogDebugf("action[handlVerDelDepthFirst] ReadDirLimit_ll done, parent[%v] maker[%v] limit[%v] verSeq[%v] children[%v]",
dentry.Inode, marker, uint64(snapShotRoutingNumPerTask), s.getTaskVerSeq(), children)
dentry.Inode, marker, uint64(snapshotRoutineNumPerTask), s.getTaskVerSeq(), children)
break
} else {
skippedChild := children[0]
@ -224,22 +273,18 @@ func (s *SnapshotScanner) handlVerDelDepthFirst(dentry *proto.ScanDentry) {
for _, child := range children {
childDentry := &proto.ScanDentry{
DelInode: false,
ParentId: dentry.Inode,
Name: child.Name,
Inode: child.Inode,
Type: child.Type,
}
if os.FileMode(childDentry.Type).IsDir() {
//s.inodeChan.In <- childDentry
dirs = append(dirs, childDentry)
} else {
//scanDentries = append(scanDentries, childDentry)
files = append(files, childDentry)
}
}
//for _, file := range scanDentries {
for _, file := range files {
if ino, err = s.mw.Delete_Ver_ll(file.ParentId, file.Name, false, s.getTaskVerSeq()); err != nil {
log.LogErrorf("action[handlVerDelDepthFirst] Delete_Ver_ll failed, file(parent[%v] child name[%v]) verSeq[%v] err[%v]",
@ -253,13 +298,13 @@ func (s *SnapshotScanner) handlVerDelDepthFirst(dentry *proto.ScanDentry) {
}
}
//scanDentries = scanDentries[:0]
for _, dir := range dirs {
s.handlVerDelDepthFirst(dir)
}
childrenNr := len(children)
if (marker == "" && childrenNr < snapShotRoutingNumPerTask) || (marker != "" && childrenNr+1 < snapShotRoutingNumPerTask) {
if (marker == "" && childrenNr < snapshotRoutineNumPerTask) || (marker != "" && childrenNr+1 < snapshotRoutineNumPerTask) {
log.LogDebugf("action[handlVerDelDepthFirst] ReadDirLimit_ll done, parent[%v]",
dentry.Inode)
done = true
@ -288,7 +333,6 @@ func (s *SnapshotScanner) handlVerDelDepthFirst(dentry *proto.ScanDentry) {
}
//func (s *SnapshotScanner) handlVerDel(dentry *proto.ScanDentry, scanType int) {
func (s *SnapshotScanner) handlVerDel(dentry *proto.ScanDentry) {
var (
children []proto.Dentry
@ -298,7 +342,6 @@ func (s *SnapshotScanner) handlVerDel(dentry *proto.ScanDentry) {
scanDentries := make([]*proto.ScanDentry, 0)
if os.FileMode(dentry.Type).IsDir() {
//totalChildrenNum := 0
totalChildDirNum := 0
totalChildFileNum := 0
marker := ""
@ -307,19 +350,19 @@ func (s *SnapshotScanner) handlVerDel(dentry *proto.ScanDentry) {
onlyDir := s.scanType == SnapScanTypeOnlyDir || s.scanType == SnapScanTypeOnlyDirAndDepth
for !done {
children, err = s.mw.ReadDirLimitByVer(dentry.Inode, marker, uint64(snapShotRoutingNumPerTask), s.getTaskVerSeq(), onlyDir)
children, err = s.mw.ReadDirLimitByVer(dentry.Inode, marker, uint64(snapshotRoutineNumPerTask), s.getTaskVerSeq(), onlyDir)
if err != nil && err != syscall.ENOENT {
log.LogErrorf("action[handlVerDel] ReadDirLimitByVer failed, parent[%v] maker[%v] limit[%v] verSeq[%v] err[%v]",
dentry.Inode, marker, uint64(snapShotRoutingNumPerTask), s.getTaskVerSeq(), err)
dentry.Inode, marker, uint64(snapshotRoutineNumPerTask), s.getTaskVerSeq(), err)
return
}
log.LogDebugf("action[handlVerDel] ReadDirLimitByVer parent[%v] maker[%v] limit[%v] verSeq[%v] children[%v]",
dentry.Inode, marker, uint64(snapShotRoutingNumPerTask), s.getTaskVerSeq(), len(children))
dentry.Inode, marker, uint64(snapshotRoutineNumPerTask), s.getTaskVerSeq(), len(children))
if err == syscall.ENOENT {
done = true
log.LogErrorf("action[handlVerDel] ReadDirLimitByVer failed, parent[%v] maker[%v] limit[%v] verSeq[%v] err[%v]",
dentry.Inode, marker, uint64(snapShotRoutingNumPerTask), s.getTaskVerSeq(), err)
dentry.Inode, marker, uint64(snapshotRoutineNumPerTask), s.getTaskVerSeq(), err)
break
}
@ -328,7 +371,7 @@ func (s *SnapshotScanner) handlVerDel(dentry *proto.ScanDentry) {
if len(children) <= 1 {
done = true
log.LogDebugf("action[handlVerDel] ReadDirLimit_ll done, parent[%v] maker[%v] limit[%v] verSeq[%v] children[%v]",
dentry.Inode, marker, uint64(snapShotRoutingNumPerTask), s.getTaskVerSeq(), children)
dentry.Inode, marker, uint64(snapshotRoutineNumPerTask), s.getTaskVerSeq(), children)
break
} else {
skippedChild := children[0]
@ -341,7 +384,6 @@ func (s *SnapshotScanner) handlVerDel(dentry *proto.ScanDentry) {
for _, child := range children {
childDentry := &proto.ScanDentry{
DelInode: false,
ParentId: dentry.Inode,
Name: child.Name,
Inode: child.Inode,
@ -373,7 +415,7 @@ func (s *SnapshotScanner) handlVerDel(dentry *proto.ScanDentry) {
}
scanDentries = scanDentries[:0]
childrenNr := len(children)
if (marker == "" && childrenNr < snapShotRoutingNumPerTask) || (marker != "" && childrenNr+1 < snapShotRoutingNumPerTask) {
if (marker == "" && childrenNr < snapshotRoutineNumPerTask) || (marker != "" && childrenNr+1 < snapshotRoutineNumPerTask) {
log.LogDebugf("action[handlVerDel] ReadDirLimit_ll done, parent[%v] total childrenNr[%v] marker[%v]",
dentry.Inode, totalChildFileNum+totalChildDirNum, marker)
done = true

251
master/api_service.go
View File

@ -4824,48 +4824,6 @@ func (m *Server) queryDataNodeDecoFailedDps(w http.ResponseWriter, r *http.Reque
sendOkReply(w, r, newSuccessHTTPReply(dps))
}
func (m *Server) setConLcNodeNum(w http.ResponseWriter, r *http.Request) {
var (
count uint64
err error
)
if count, err = parseRequestToGetConcurrentLcNode(r); err != nil {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
return
}
if err = m.cluster.setMaxConcurrentLcNodes(count); err != nil {
sendErrReply(w, r, newErrHTTPReply(err))
return
}
sendOkReply(w, r, newSuccessHTTPReply(fmt.Sprintf("set MaxConcurrentLcNodes to %v successfully", count)))
}
func (m *Server) getAllLcNodeInfo(w http.ResponseWriter, r *http.Request) {
var (
rsp *proto.LcNodeInfoResponse
err error
)
if rsp, err = m.cluster.getAllLcNodeInfo(); err != nil {
sendErrReply(w, r, newErrHTTPReply(err))
return
}
sendOkReply(w, r, newSuccessHTTPReply(rsp))
}
// handle tasks such as heartbeat，expiration scanning, etc.
func (m *Server) handleLcNodeTaskResponse(w http.ResponseWriter, r *http.Request) {
tr, err := parseRequestToGetTaskResponse(r)
if err != nil {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
return
}
sendOkReply(w, r, newSuccessHTTPReply(fmt.Sprintf("%v", http.StatusOK)))
m.cluster.handleLcNodeTaskResponse(tr.OperatorAddr, tr)
}
func parseReqToDecoDataNodeProgress(r *http.Request) (nodeAddr string, err error) {
if err = r.ParseForm(); err != nil {
return
@ -5190,97 +5148,6 @@ func (m *Server) DeleteQuota(w http.ResponseWriter, r *http.Request) {
return
}
func (m *Server) addLcNode(w http.ResponseWriter, r *http.Request) {
var (
nodeAddr string
id uint64
err error
)
if nodeAddr, err = parseAndExtractNodeAddr(r); err != nil {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
return
}
if !checkIp(nodeAddr) {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: fmt.Errorf("addr not legal").Error()})
return
}
if id, err = m.cluster.addLcNode(nodeAddr); err != nil {
sendErrReply(w, r, newErrHTTPReply(err))
return
}
sendOkReply(w, r, newSuccessHTTPReply(id))
}
func (m *Server) SetBucketLifecycle(w http.ResponseWriter, r *http.Request) {
var (
bytes []byte
err error
)
if bytes, err = ioutil.ReadAll(r.Body); err != nil {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
return
}
var req = proto.SetBucketLifecycleRequest{}
if err = json.Unmarshal(bytes, &req); err != nil {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
return
}
if _, err = m.cluster.getVol(req.VolName); err != nil {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeVolNotExists, Msg: err.Error()})
return
}
_ = m.cluster.SetBucketLifecycle(&req)
sendOkReply(w, r, newSuccessHTTPReply(fmt.Sprintf("PutBucketLifecycleConfiguration successful ")))
}
func (m *Server) GetBucketLifecycle(w http.ResponseWriter, r *http.Request) {
var (
err error
name string
lcConf *proto.LcConfiguration
)
if name, err = parseAndExtractName(r); err != nil {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
return
}
if _, err = m.cluster.getVol(name); err != nil {
sendErrReply(w, r, newErrHTTPReply(proto.ErrVolNotExists))
return
}
lcConf = m.cluster.GetBucketLifecycle(name)
if lcConf == nil {
sendErrReply(w, r, newErrHTTPReply(proto.ErrNoSuchLifecycleConfiguration))
}
sendOkReply(w, r, newSuccessHTTPReply(lcConf))
}
func (m *Server) DelBucketLifecycle(w http.ResponseWriter, r *http.Request) {
var (
err error
name string
)
if name, err = parseAndExtractName(r); err != nil {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
return
}
if _, err = m.cluster.getVol(name); err != nil {
sendErrReply(w, r, newErrHTTPReply(proto.ErrVolNotExists))
return
}
m.cluster.DelBucketLifecycle(name)
msg := fmt.Sprintf("delete vol[%v] lifecycle successfully", name)
log.LogWarn(msg)
sendOkReply(w, r, newSuccessHTTPReply(msg))
}
func (m *Server) ListQuota(w http.ResponseWriter, r *http.Request) {
var (
err error
@ -5520,3 +5387,121 @@ func (m *Server) queryBadDisks(w http.ResponseWriter, r *http.Request) {
sendOkReply(w, r, newSuccessHTTPReply(infos))
}
func (m *Server) addLcNode(w http.ResponseWriter, r *http.Request) {
var (
nodeAddr string
id uint64
err error
)
if nodeAddr, err = parseAndExtractNodeAddr(r); err != nil {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
return
}
if !checkIp(nodeAddr) {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: fmt.Errorf("addr not legal").Error()})
return
}
if id, err = m.cluster.addLcNode(nodeAddr); err != nil {
sendErrReply(w, r, newErrHTTPReply(err))
return
}
sendOkReply(w, r, newSuccessHTTPReply(id))
}
// handle tasks such as heartbeat，expiration scanning, etc.
func (m *Server) handleLcNodeTaskResponse(w http.ResponseWriter, r *http.Request) {
tr, err := parseRequestToGetTaskResponse(r)
if err != nil {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
return
}
sendOkReply(w, r, newSuccessHTTPReply(fmt.Sprintf("%v", http.StatusOK)))
m.cluster.handleLcNodeTaskResponse(tr.OperatorAddr, tr)
}
func (m *Server) SetBucketLifecycle(w http.ResponseWriter, r *http.Request) {
var (
bytes []byte
err error
)
if bytes, err = ioutil.ReadAll(r.Body); err != nil {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
return
}
var req = proto.LcConfiguration{}
if err = json.Unmarshal(bytes, &req); err != nil {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
return
}
if _, err = m.cluster.getVol(req.VolName); err != nil {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeVolNotExists, Msg: err.Error()})
return
}
_ = m.cluster.SetBucketLifecycle(&req)
sendOkReply(w, r, newSuccessHTTPReply(fmt.Sprintf("PutBucketLifecycleConfiguration successful ")))
}
func (m *Server) GetBucketLifecycle(w http.ResponseWriter, r *http.Request) {
var (
err error
name string
lcConf *proto.LcConfiguration
)
if name, err = parseAndExtractName(r); err != nil {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
return
}
if _, err = m.cluster.getVol(name); err != nil {
sendErrReply(w, r, newErrHTTPReply(proto.ErrVolNotExists))
return
}
lcConf = m.cluster.GetBucketLifecycle(name)
if lcConf == nil {
sendErrReply(w, r, newErrHTTPReply(proto.ErrNoSuchLifecycleConfiguration))
}
sendOkReply(w, r, newSuccessHTTPReply(lcConf))
}
func (m *Server) DelBucketLifecycle(w http.ResponseWriter, r *http.Request) {
var (
err error
name string
)
if name, err = parseAndExtractName(r); err != nil {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
return
}
if _, err = m.cluster.getVol(name); err != nil {
sendErrReply(w, r, newErrHTTPReply(proto.ErrVolNotExists))
return
}
m.cluster.DelBucketLifecycle(name)
msg := fmt.Sprintf("delete vol[%v] lifecycle successfully", name)
log.LogWarn(msg)
sendOkReply(w, r, newSuccessHTTPReply(msg))
}
func (m *Server) lcnodeInfo(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
return
}
switch r.FormValue("op") {
case "info":
var (
rsp *LcNodeInfoResponse
err error
)
if rsp, err = m.cluster.getAllLcNodeInfo(); err != nil {
sendErrReply(w, r, newErrHTTPReply(err))
return
}
sendOkReply(w, r, newSuccessHTTPReply(rsp))
case "start":
if m.cluster.partition != nil && m.cluster.partition.IsRaftLeader() {
m.cluster.startLcScan()
sendOkReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeSuccess})
} else {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: "not leader"})
}
default:
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: "invalid op"})
}
}

349
master/cluster.go
View File

@ -93,7 +93,9 @@ type Cluster struct {
dentryCountNotEqualMP *sync.Map
ac *authSDK.AuthClient
authenticate bool
lcNodes sync.Map
lcMgr *lifecycleManager
snapshotMgr *snapshotDelManager
}
type followerReadManager struct {
@ -331,6 +333,8 @@ func newCluster(name string, leaderInfo *LeaderInfo, fsm *MetadataFsm, partition
c.dentryCountNotEqualMP = new(sync.Map)
c.lcMgr = newLifecycleManager()
c.lcMgr.cluster = c
c.snapshotMgr = newSnapshotManager()
c.snapshotMgr.cluster = c
return
}
@ -353,8 +357,8 @@ func (c *Cluster) scheduleTask() {
c.scheduleToCheckDecommissionDataNode()
c.scheduleToCheckDecommissionDisk()
c.scheduleToCheckDataReplicas()
c.scheduleToScanS3Expiration()
c.scheduleToDelSnapshotVer()
c.scheduleToLcScan()
c.scheduleToSnapshotScanDelVer()
}
func (c *Cluster) masterAddr() (addr string) {
@ -442,38 +446,6 @@ func (c *Cluster) scheduleToCheckDataPartitions() {
}()
}
func (c *Cluster) scheduleToScanS3Expiration() {
go func() {
for {
if c.partition != nil && c.partition.IsRaftLeader() {
c.scanS3Expiration()
}
time.Sleep(time.Second * time.Duration(c.cfg.IntervalToScanS3Expiration))
}
}()
}
func (c *Cluster) scheduleToDelSnapshotVer() {
//make sure resume all the processing ver deleting tasks before checking
waitTime := time.Second * defaultIntervalToCheck
waited := false
go func() {
for {
if c.partition != nil && c.partition.IsRaftLeader() {
if !waited {
log.LogInfof("wait for %v seconds once after becoming leader to make sure all the ver deleting tasks are resumed",
waitTime)
time.Sleep(waitTime)
waited = true
}
c.getDeletingSnapshotVer()
c.checkSnapshotStrategy()
}
time.Sleep(time.Second * defaultIntervalToCheckHeartbeat)
}
}()
}
func (c *Cluster) scheduleToCheckVolStatus() {
go func() {
//check vols after switching leader two minutes
@ -570,52 +542,6 @@ func (c *Cluster) checkDataPartitions() {
}
}
func (c *Cluster) scanS3Expiration() {
defer func() {
if r := recover(); r != nil {
log.LogWarnf("scanS3Expiration occurred panic,err[%v]", r)
WarnBySpecialKey(fmt.Sprintf("%v_%v_scheduling_job_panic", c.Name, ModuleName),
"scanS3Expiration occurred panic")
}
}()
c.lcMgr.rescheduleScanRoutine()
}
func (c *Cluster) getDeletingSnapshotVer() {
if c.partition != nil && c.partition.IsRaftLeader() {
vols := c.allVols()
for volName, vol := range vols {
volVerInfoList := vol.VersionMgr.getVersionList()
for _, volVerInfo := range volVerInfoList.VerList {
if volVerInfo.Status == proto.VersionDeleting {
verInfo := &LcVerInfo{
VerInfo: proto.VerInfo{
VolName: volName,
VerSeq: volVerInfo.Ver,
},
dTime: volVerInfo.DelTime,
}
c.lcMgr.snapshotMgr.volVerInfos.AddVerInfo(verInfo)
}
}
}
} else {
log.LogWarnf("getDeletingSnapshotVer: master is not leader")
}
}
func (c *Cluster) checkSnapshotStrategy() {
vols := c.allVols()
for _, vol := range vols {
if !proto.IsHot(vol.VolType) {
continue
}
vol.VersionMgr.checkSnapshotStrategy()
}
}
func (c *Cluster) doLoadDataPartitions() {
defer func() {
if r := recover(); r != nil {
@ -757,9 +683,7 @@ func (c *Cluster) checkMetaNodeHeartbeat() {
func (c *Cluster) checkLcNodeHeartbeat() {
tasks := make([]*proto.AdminTask, 0)
diedNodes := make([]string, 0)
var currentRunningLcnodesNum uint64 = 0
c.lcMgr.lcNodes.Range(func(addr, lcNode interface{}) bool {
c.lcNodes.Range(func(addr, lcNode interface{}) bool {
node := lcNode.(*LcNode)
node.checkLiveness()
if !node.IsActive {
@ -767,12 +691,6 @@ func (c *Cluster) checkLcNodeHeartbeat() {
diedNodes = append(diedNodes, node.Addr)
return true
}
currentRunningLcnodesNum++
if currentRunningLcnodesNum > c.cfg.MaxConcurrentLcNodes {
log.LogInfof("checkLcNodeHeartbeat: max concurrent lcnodes reached, lcnode(%v) is prepare to step down", node.Addr)
diedNodes = append(diedNodes, node.Addr)
return true
}
task := node.createHeartbeatTask(c.masterAddr())
tasks = append(tasks, task)
return true
@ -1781,7 +1699,7 @@ func (c *Cluster) metaNode(addr string) (metaNode *MetaNode, err error) {
}
func (c *Cluster) lcNode(addr string) (lcNode *LcNode, err error) {
value, ok := c.lcMgr.lcNodes.Load(addr)
value, ok := c.lcNodes.Load(addr)
if !ok {
err = errors.Trace(lcNodeNotFound(addr), "%v not found", addr)
return
@ -3283,7 +3201,7 @@ func (c *Cluster) allMasterNodes() (masterNodes []proto.NodeView) {
}
func (c *Cluster) lcNodeCount() (len int) {
c.lcMgr.lcNodes.Range(func(key, value interface{}) bool {
c.lcNodes.Range(func(key, value interface{}) bool {
len++
return true
})
@ -3617,25 +3535,6 @@ func (c *Cluster) setMaxConcurrentLcNodes(count uint64) (err error) {
return
}
func (c *Cluster) getAllLcNodeInfo() (rsp *proto.LcNodeInfoResponse, err error) {
rsp = &proto.LcNodeInfoResponse{
MaxConcurrentLcNodeNum: int(c.cfg.MaxConcurrentLcNodes),
RunningLcNodeNum: c.lcNodeCount(),
Infos: make([]*proto.LcNodeStatInfo, 0),
}
c.lcMgr.lcNodes.Range(func(addr, value interface{}) bool {
ln := &proto.LcNodeStatInfo{
Addr: addr.(string),
Busy: c.lcMgr.lnStates.IsNodeBusy(addr.(string)),
}
rsp.Infos = append(rsp.Infos, ln)
return true
})
return
}
func (c *Cluster) clearVols() {
c.volMutex.Lock()
defer c.volMutex.Unlock()
@ -3860,15 +3759,6 @@ func (c *Cluster) checkDecommissionDisk() {
})
}
func (c *Cluster) clearLcNodes() {
c.lcMgr.lcNodes.Range(func(key, value interface{}) bool {
lcNode := value.(*LcNode)
c.lcMgr.lcNodes.Delete(key)
lcNode.clean()
return true
})
}
func (c *Cluster) TryDecommissionDisk(disk *DecommissionDisk) {
var (
node *DataNode
@ -4089,83 +3979,34 @@ func (c *Cluster) parseAndCheckClientIDKey(r *http.Request, Type proto.MsgType)
return
}
func (c *Cluster) delLcNode(nodeAddr string) (err error) {
tasks := c.lcMgr.lnStates.RemoveNode(nodeAddr)
if len(tasks) != 0 {
s3Tasks := make([]*proto.S3ScanTaskInfo, 0)
snapshotTaskIds := make([]string, 0)
for _, task := range tasks {
switch t := task.(type) {
case *proto.S3ScanTaskInfo:
s3Tasks = append(s3Tasks, t)
case *proto.DelVerTaskInfo:
snapshotTaskIds = append(snapshotTaskIds, t.Id)
default:
}
}
if c.lcMgr.lcScan != nil {
c.lcMgr.lcScan.redoTasks(s3Tasks)
}
c.lcMgr.snapshotMgr.volVerInfos.ReturnProcessingVerInfo(snapshotTaskIds)
}
val, loaded := c.lcMgr.lcNodes.LoadAndDelete(nodeAddr)
if loaded {
ln := val.(*LcNode)
c.lcMgr.lnMutex.Lock()
defer c.lcMgr.lnMutex.Unlock()
if err = c.syncDeleteLcNode(ln); err != nil {
err = fmt.Errorf("action[delLcNode],clusterID[%v] lcNodeAddr:%v err:%v ", c.Name, nodeAddr, err.Error())
return
}
}
log.LogInfof("action[delLcNode],clusterID[%v] lcNodeAddr:%v", c.Name, nodeAddr)
return
}
func (c *Cluster) addLcNode(nodeAddr string) (id uint64, err error) {
var ln *LcNode
if node, ok := c.lcMgr.lcNodes.Load(nodeAddr); ok {
ln = node.(*LcNode)
if value, ok := c.lcNodes.Load(nodeAddr); ok {
ln = value.(*LcNode)
log.LogInfof("action[addLcNode] already add nodeAddr: %v, id: %v", nodeAddr, ln.ID)
return ln.ID, nil
}
ln = newLcNode(nodeAddr, c.Name)
// allocate dataNode id
// allocate LcNode id
if id, err = c.idAlloc.allocateCommonID(); err != nil {
goto errHandler
}
ln.ID = id
log.LogInfof("action[addLcNode] lcnode id[%v]", id)
c.lcMgr.lnMutex.Lock()
if c.lcNodeCount() >= int(c.cfg.MaxConcurrentLcNodes) {
c.lcMgr.lnMutex.Unlock()
err = errors.New("max concurrent LcNodes reached!")
goto errHandler
}
log.LogInfof("action[addLcNode] allocateCommonID: %v", id)
if err = c.syncAddLcNode(ln); err != nil {
c.lcMgr.lnMutex.Unlock()
goto errHandler
}
c.lcMgr.lcNodes.Store(nodeAddr, ln)
c.lcMgr.lnMutex.Unlock()
log.LogInfof("action[addLcNode],clusterID[%v] lcNodeAddr:%v",
c.Name, nodeAddr)
c.lcMgr.lnStates.Lock()
delete(c.lcMgr.lnStates.workingNodes, nodeAddr)
c.lcMgr.lnStates.idleNodes[nodeAddr] = nodeAddr
c.lcMgr.lnStates.Unlock()
log.LogInfof("action[addLcNode], lcnode(%v) is set idle", nodeAddr)
c.lcNodes.Store(nodeAddr, ln)
c.lcMgr.lcNodeStatus.Lock()
delete(c.lcMgr.lcNodeStatus.workingNodes, nodeAddr)
c.lcMgr.lcNodeStatus.idleNodes[nodeAddr] = nodeAddr
c.lcMgr.lcNodeStatus.Unlock()
c.snapshotMgr.lcNodeStatus.Lock()
delete(c.snapshotMgr.lcNodeStatus.workingNodes, nodeAddr)
c.snapshotMgr.lcNodeStatus.idleNodes[nodeAddr] = nodeAddr
c.snapshotMgr.lcNodeStatus.Unlock()
log.LogInfof("action[addLcNode], clusterID[%v], lcNodeAddr: %v, id: %v, add idleNodes", c.Name, nodeAddr, ln.ID)
return
errHandler:
err = fmt.Errorf("action[addLcNode],clusterID[%v] lcNodeAddr:%v err:%v ", c.Name, nodeAddr, err.Error())
log.LogError(errors.Stack(err))
@ -4173,12 +4014,142 @@ errHandler:
return
}
func (c *Cluster) SetBucketLifecycle(req *proto.SetBucketLifecycleRequest) error {
func (c *Cluster) getAllLcNodeInfo() (rsp *LcNodeInfoResponse, err error) {
rsp = &LcNodeInfoResponse{
Infos: make([]*LcNodeStatInfo, 0),
}
c.lcNodes.Range(func(addr, value interface{}) bool {
t := c.lcMgr.lcNodeStatus.getWorkingTask(addr.(string))
var TaskId string
if t != nil {
TaskId = t.(*proto.RuleTask).Id
}
ln := &LcNodeStatInfo{
Addr: addr.(string),
TaskId: TaskId,
}
rsp.Infos = append(rsp.Infos, ln)
return true
})
rsp.LcConfigurations = c.lcMgr.lcConfigurations
rsp.LcRuleTaskStatus = c.lcMgr.lcRuleTaskStatus
return
}
func (c *Cluster) clearLcNodes() {
c.lcNodes.Range(func(key, value interface{}) bool {
lcNode := value.(*LcNode)
c.lcNodes.Delete(key)
lcNode.clean()
return true
})
}
func (c *Cluster) delLcNode(nodeAddr string) (err error) {
t := c.lcMgr.lcNodeStatus.removeNode(nodeAddr)
c.lcMgr.lcRuleTaskStatus.redoTask(t.(*proto.RuleTask))
t = c.snapshotMgr.lcNodeStatus.removeNode(nodeAddr)
c.snapshotMgr.volVerInfos.RedoProcessingVerInfo(t.(string))
lcNode, err := c.lcNode(nodeAddr)
if err != nil {
log.LogErrorf("action[delLcNode], clusterID:%v, lcNodeAddr:%v, load err:%v ", c.Name, nodeAddr, err)
return
}
if err = c.syncDeleteLcNode(lcNode); err != nil {
log.LogErrorf("action[delLcNode], clusterID:%v, lcNodeAddr:%v syncDeleteLcNode err:%v ", c.Name, nodeAddr, err)
return
}
val, loaded := c.lcNodes.LoadAndDelete(nodeAddr)
log.LogInfof("action[delLcNode], clusterID:%v, lcNodeAddr:%v, LoadAndDelete result val:%v, loaded:%v", c.Name, nodeAddr, val, loaded)
return
}
func (c *Cluster) scheduleToLcScan() {
go func() {
for {
now := time.Now()
next := now.Add(time.Hour * 24)
next = time.Date(next.Year(), next.Month(), next.Day(), 1, 0, 0, 0, next.Location())
t := time.NewTimer(next.Sub(now))
<-t.C
if c.partition != nil && c.partition.IsRaftLeader() {
c.startLcScan()
}
}
}()
}
func (c *Cluster) startLcScan() {
defer func() {
if r := recover(); r != nil {
log.LogWarnf("startLcScan occurred panic,err[%v]", r)
WarnBySpecialKey(fmt.Sprintf("%v_%v_scheduling_job_panic", c.Name, ModuleName),
"startLcScan occurred panic")
}
}()
c.lcMgr.startLcScan()
}
func (c *Cluster) scheduleToSnapshotScanDelVer() {
c.snapshotMgr.Start()
//make sure resume all the processing ver deleting tasks before checking
waitTime := time.Second * defaultIntervalToCheck
waited := false
go func() {
for {
if c.partition != nil && c.partition.IsRaftLeader() {
if !waited {
log.LogInfof("wait for %v seconds once after becoming leader to make sure all the ver deleting tasks are resumed",
waitTime)
time.Sleep(waitTime)
waited = true
}
c.getDeletingSnapshotVer()
c.checkSnapshotStrategy()
}
time.Sleep(time.Second * defaultIntervalToCheck)
}
}()
}
func (c *Cluster) getDeletingSnapshotVer() {
if c.partition != nil && c.partition.IsRaftLeader() {
vols := c.allVols()
for volName, vol := range vols {
volVerInfoList := vol.VersionMgr.getVersionList()
for _, volVerInfo := range volVerInfoList.VerList {
if volVerInfo.Status == proto.VersionDeleting {
verInfo := &LcVerInfo{
VerInfo: proto.VerInfo{
VolName: volName,
VerSeq: volVerInfo.Ver,
},
dTime: volVerInfo.DelTime,
}
c.snapshotMgr.volVerInfos.AddVerInfo(verInfo)
}
}
}
} else {
log.LogWarnf("getDeletingSnapshotVer: master is not leader")
}
}
func (c *Cluster) checkSnapshotStrategy() {
vols := c.allVols()
for _, vol := range vols {
if !proto.IsHot(vol.VolType) {
continue
}
vol.VersionMgr.checkSnapshotStrategy()
}
}
func (c *Cluster) SetBucketLifecycle(req *proto.LcConfiguration) error {
lcConf := &proto.LcConfiguration{
VolName: req.VolName,
Rules: req.Rules,
}
if c.lcMgr.GetS3BucketLifecycle(req.VolName) != nil {
if err := c.syncUpdateLcConf(lcConf); err != nil {
err = fmt.Errorf("action[SetS3BucketLifecycle],clusterID[%v] vol:%v err:%v ", c.Name, lcConf.VolName, err.Error())
@ -4192,30 +4163,24 @@ func (c *Cluster) SetBucketLifecycle(req *proto.SetBucketLifecycleRequest) error
Warn(c.Name, err.Error())
}
}
_ = c.lcMgr.SetS3BucketLifecycle(lcConf)
log.LogInfof("action[SetS3BucketLifecycle],clusterID[%v] vol:%v", c.Name, lcConf.VolName)
return nil
}
func (c *Cluster) GetBucketLifecycle(VolName string) (lcConf *proto.LcConfiguration) {
lcConf = c.lcMgr.GetS3BucketLifecycle(VolName)
log.LogInfof("action[GetS3BucketLifecycle],clusterID[%v] vol:%v", c.Name, lcConf.VolName)
log.LogInfof("action[GetS3BucketLifecycle],clusterID[%v] vol:%v", c.Name, VolName)
return
}
func (c *Cluster) DelBucketLifecycle(VolName string) {
lcConf := &proto.LcConfiguration{
VolName: VolName,
}
if err := c.syncDeleteLcConf(lcConf); err != nil {
err = fmt.Errorf("action[DelS3BucketLifecycle],clusterID[%v] vol:%v err:%v ", c.Name, VolName, err.Error())
log.LogError(errors.Stack(err))
Warn(c.Name, err.Error())
}
c.lcMgr.DelS3BucketLifecycle(VolName)
log.LogInfof("action[DelS3BucketLifecycle],clusterID[%v] vol:%v", c.Name, VolName)
return

256
master/cluster_task.go
View File

@ -58,7 +58,6 @@ func (c *Cluster) addMetaNodeTasks(tasks []*proto.AdminTask) {
}
func (c *Cluster) addLcNodeTasks(tasks []*proto.AdminTask) {
for _, t := range tasks {
if t == nil {
continue
@ -71,261 +70,6 @@ func (c *Cluster) addLcNodeTasks(tasks []*proto.AdminTask) {
}
}
func (c *Cluster) handleLcNodeTaskResponse(nodeAddr string, task *proto.AdminTask) {
if task == nil {
log.LogInfof("action[handleLcNodeTaskResponse] receive addr[%v] task response,but task is nil", nodeAddr)
return
}
log.LogDebugf("action[handleLcNodeTaskResponse] receive addr[%v] task:%v", nodeAddr, task.ToString())
var (
err error
lcNode *LcNode
)
if lcNode, err = c.lcNode(nodeAddr); err != nil {
goto errHandler
}
lcNode.TaskManager.DelTask(task)
if err = unmarshalTaskResponse(task); err != nil {
goto errHandler
}
switch task.OpCode {
case proto.OpLcNodeScan:
bytes, err := json.Marshal(task.Request)
if err != nil {
log.LogErrorf("action[handleLcNodeTaskResponse] Marshal task request failed!")
return
}
var request interface{}
request = &proto.RuleTaskRequest{}
if err = json.Unmarshal(bytes, request); err != nil {
log.LogErrorf("action[handleLcNodeTaskResponse] Unmarshal task request failed!")
return
}
task.Request = request
req := task.Request.(*proto.RuleTaskRequest)
response := task.Response.(*proto.RuleTaskResponse)
err = c.handleLcNodeS3ScanResp(task.OperatorAddr, req, response)
case proto.OpLcNodeSnapshotVerDel:
response := task.Response.(*proto.SnapshotVerDelTaskResponse)
err = c.handleLcNodeSnapshotScanResp(task.OperatorAddr, response)
case proto.OpLcNodeHeartbeat:
response := task.Response.(*proto.LcNodeHeartbeatResponse)
err = c.handleLcNodeHeartbeatResp(task.OperatorAddr, response)
default:
err = fmt.Errorf(fmt.Sprintf("unknown operate code %v", task.OpCode))
goto errHandler
}
if err != nil {
goto errHandler
}
return
errHandler:
log.LogErrorf("process task[%v] failed,err:%v", task.ToString(), err)
return
}
func (c *Cluster) handleLcNodeSnapshotScanResp(nodeAddr string, resp *proto.SnapshotVerDelTaskResponse) (err error) {
//1.mark done for VersionMgr
var vol *Vol
vol, err = c.getVol(resp.VolName)
if err != nil {
log.LogErrorf("action[handleLcNodeSnapshotScanResp] snapshot task(%v) scanning completed by %v, results(%v), volume(%v) is not found",
resp.ID, nodeAddr, resp, resp.VolName)
return
} else {
_ = vol.VersionMgr.DelVer(resp.VerSeq)
}
//2. mark done for snapshotMgr
taskInfo := &proto.DelVerTaskInfo{Id: resp.ID}
c.lcMgr.lnStates.ReleaseTask(taskInfo)
verInfo := c.lcMgr.snapshotMgr.volVerInfos.RemoveProcessingVerInfo(taskInfo.Id)
if verInfo != nil {
log.LogInfof("action[handleLcNodeSnapshotScanResp] snapshot task(%v) volume(%v) verseq(%v) scanning completed by %v, results(%v).",
taskInfo.Id, verInfo.VolName, verInfo.VerSeq, nodeAddr, resp)
} else {
log.LogErrorf("action[handleLcNodeSnapshotScanResp] snapshot task(%v) scanning completed by %v, results(%v).",
taskInfo.Id, nodeAddr, resp)
}
return
}
func (c *Cluster) handleLcNodeS3ScanResp(nodeAddr string, req *proto.RuleTaskRequest, resp *proto.RuleTaskResponse) (err error) {
log.LogDebugf("action[handleLcNodeS3ScanResp] Rule(%v) in task(%v) of routine(%v) Enter", req.Task.Rule.ID, req.Task.Id, req.RoutineID)
defer func() {
log.LogDebugf("action[handleLcNodeS3ScanResp] Rule(%v) in task(%v) of routine(%v) Exit", req.Task.Rule.ID, req.Task.Id, req.RoutineID)
}()
taskInfo := &proto.S3ScanTaskInfo{
Id: req.Task.Id,
RoutineId: req.RoutineID,
}
c.lcMgr.lnStates.ReleaseTask(taskInfo)
lcScan := c.lcMgr.getScanRoutine()
if lcScan != nil {
log.LogDebugf("action[handleLcNodeS3ScanResp] Rule(%v) in task(%v) of routine(%v), current routine(%v)",
req.Task.Rule.ID, req.Task.Id, req.RoutineID, lcScan.Id)
lcScan.Lock()
if lcScan.Id == req.RoutineID {
for i, ruleTask := range lcScan.RuleStatus.Scanning {
if req.Task.Id == ruleTask.Id {
lcScan.RuleStatus.Scanning = append(lcScan.RuleStatus.Scanning[:i], lcScan.RuleStatus.Scanning[i+1:]...)
if resp.Status == proto.TaskFailed {
log.LogWarnf("action[handleLcNodeS3ScanResp] Rule(%v) in task(%v) of routine(%v) scanning failed, current routine(%v), results(%v).",
req.Task.Rule.ID, req.Task.Id, req.RoutineID, lcScan.Id, resp)
lcScan.RuleStatus.ToBeScanned = append(lcScan.RuleStatus.ToBeScanned, ruleTask)
lcScan.Unlock()
return nil
} else {
lcScan.RuleStatus.Scanned = append(lcScan.RuleStatus.Scanned, ruleTask)
//lcScan.RuleStatus.Results = append(lcScan.RuleStatus.Results, resp)
lcScan.RuleStatus.Results[resp.ID] = resp
break
}
}
}
} else {
lcScan.Unlock()
log.LogWarnf("action[handleLcNodeS3ScanResp] Rule(%v) in task(%v) of routine(%v),", req.Task.Rule.ID, req.Task.Id, req.RoutineID)
if resp.Status == proto.TaskSucceeds {
log.LogInfof("action[handleLcNodeS3ScanResp] Rule(%v) in task(%v) of routine(%v) scanning completed, current routine(%v),"+
" task startTime(%v), task endTime(%v), totalScanned(%v), fileScanned(%v), dirScanned(%v), ExpiredNum(%v), "+
"ErrorSkippedNum(%v), AbortedIncompleteMultipartNum(%v)",
req.Task.Rule.ID, req.Task.Id, req.RoutineID, lcScan.Id, resp.StartTime.String(), resp.EndTime.String(),
resp.TotalInodeScannedNum, resp.FileScannedNum, resp.DirScannedNum, resp.ExpiredNum, resp.ErrorSkippedNum, resp.AbortedIncompleteMultipartNum)
}
return
}
lcScan.Unlock()
log.LogInfof("action[handleLcNodeS3ScanResp] Rule(%v) in task(%v) of routine(%v) scanning completed, current routine(%v), results(%v).",
req.Task.Rule.ID, req.Task.Id, req.RoutineID, lcScan.Id, resp)
if !lcScan.Scanning() {
lcScan.Stop()
}
}
return
}
func (c *Cluster) handleLcNodeHeartbeatResp(nodeAddr string, resp *proto.LcNodeHeartbeatResponse) (err error) {
var (
lcNode *LcNode
logMsg string
infos []lcnodeTaskInfo
)
log.LogDebugf("action[handleLcNodeHeartbeatResp] clusterID[%v] receive lcNode[%v] heartbeat", c.Name, nodeAddr)
if resp.Status != proto.TaskSucceeds {
Warn(c.Name, fmt.Sprintf("action[handleLcNodeHeartbeatResp] clusterID[%v] lcNode[%v] heartbeat task failed",
c.Name, nodeAddr))
return
}
if lcNode, err = c.lcNode(nodeAddr); err != nil {
goto errHandler
}
lcNode.Lock()
lcNode.IsActive = true
lcNode.ReportTime = time.Now()
lcNode.Unlock()
//c.lcMgr.lnStates.UpdateNodeTask(nodeAddr, resp.S3ScanningTasks)
if !resp.IsLcNodeIdle() {
log.LogDebugf("action[handleLcNodeHeartbeatResp], lcNode[%v] is scanning", nodeAddr)
lcScan := c.lcMgr.getScanRoutine()
if lcScan != nil {
lcScan.Lock()
for _, task := range resp.S3ScanningTasks {
if lcScan.Id == task.RoutineId {
tmpResp := &proto.RuleTaskResponse{
ID: task.Id,
RoutineID: task.RoutineId,
Done: false,
S3TaskStatistics: task.S3TaskStatistics,
}
lcScan.RuleStatus.Results[task.Id] = tmpResp
log.LogDebugf("action[handleLcNodeHeartbeatResp], lcScan.Id(%v) task.RoutineId(%v) rsp(%v)",
lcScan.Id, task.RoutineId, tmpResp)
}
}
lcScan.Unlock()
}
for _, task := range resp.SnapshotScanningTasks {
tmpResp := &proto.SnapshotVerDelTaskResponse{
ID: task.Id,
SnapshotStatistics: task.SnapshotStatistics,
Done: false,
}
c.lcMgr.snapshotMgr.volVerInfos.Lock()
c.lcMgr.snapshotMgr.volVerInfos.TaskResults[task.Id] = tmpResp
if pInfo, ok := c.lcMgr.snapshotMgr.volVerInfos.ProcessingVerInfos[task.Id]; ok {
pInfo.updateTime = time.Now()
log.LogDebugf("action[handleLcNodeHeartbeatResp], snapshot scan taskid(%v) update time",
task.Id)
} else {
//in case of master restart or leader changing, resume processing status
if vol, err := c.getVol(task.VolName); err == nil {
volVerInfoList := vol.VersionMgr.getVersionList()
for _, volVerInfo := range volVerInfoList.VerList {
if volVerInfo.Ver == task.VerSeq && volVerInfo.Status == proto.VersionDeleting {
pInfo := &ProcessingVerInfo{
LcVerInfo: LcVerInfo{
VerInfo: proto.VerInfo{
VolName: task.VolName,
VerSeq: task.VerSeq,
},
dTime: volVerInfo.DelTime,
},
updateTime: time.Now(),
}
c.lcMgr.snapshotMgr.volVerInfos.ProcessingVerInfos[task.Id] = pInfo
log.LogInfof("action[handleLcNodeHeartbeatResp], task (%v) processing status is resumed",
task.Id)
}
}
} else {
log.LogErrorf("action[handleLcNodeHeartbeatResp], snapshot scan vol(%v) not found", task.VolName)
}
}
c.lcMgr.snapshotMgr.volVerInfos.Unlock()
info := &proto.DelVerTaskInfo{Id: task.Key()}
infos = append(infos, info)
log.LogDebugf("action[handleLcNodeHeartbeatResp], snapshot scan taskid(%v) rsp(%v)",
task.Id, tmpResp)
}
c.lcMgr.lnStates.UpdateNodeRunningTask(nodeAddr, infos, true)
} else {
log.LogDebugf("action[handleLcNodeHeartbeatResp], lcNode[%v] is idle", nodeAddr)
c.lcMgr.lnStates.UpdateNodeRunningTask(nodeAddr, infos, false)
c.lcMgr.NotifyIdleLcNode()
}
logMsg = fmt.Sprintf("action[handleLcNodeHeartbeatResp], lcNode:%v, ReportTime:%v success", lcNode.Addr, lcNode.ReportTime.Unix())
log.LogInfof(logMsg)
return
errHandler:
logMsg = fmt.Sprintf("nodeAddr %v heartbeat error :%v", nodeAddr, err.Error())
log.LogError(logMsg)
return
}
func (c *Cluster) waitForResponseToLoadDataPartition(partitions []*DataPartition) {
var wg sync.WaitGroup

7
master/http_server.go
View File

@ -409,11 +409,8 @@ func (m *Server) registerAPIRoutes(router *mux.Router) {
Path(proto.AddLcNode).
HandlerFunc(m.addLcNode)
router.NewRoute().Methods(http.MethodGet, http.MethodPost).
Path(proto.AdminSetConLcNodeNum).
HandlerFunc(m.setConLcNodeNum)
router.NewRoute().Methods(http.MethodGet, http.MethodPost).
Path(proto.AdminGetAllLcNodeInfo).
HandlerFunc(m.getAllLcNodeInfo)
Path(proto.AdminLcNode).
HandlerFunc(m.lcnodeInfo)
// node task response APIs
router.NewRoute().Methods(http.MethodGet, http.MethodPost).

848
master/lifecycle.go
View File

@ -1,848 +0,0 @@
// Copyright 2018 The Chubao Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package master
import (
"github.com/cubefs/cubefs/util/errors"
"math"
"sync"
"time"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/util/log"
)
//type scanTaskInfo struct {
// Id string
// routineId int64
//}
type lcnodeTaskInfo interface{}
type nodesState struct {
sync.RWMutex
//workingNodes map[string]map[string]*proto.S3ScanTaskInfo
workingNodes map[string]map[string]lcnodeTaskInfo
idleNodes map[string]string
}
func (ns *nodesState) getOneIdleNode() (nodeAddr string, err error) {
if len(ns.idleNodes) > 0 {
for n := range ns.idleNodes {
nodeAddr = n
break
}
} else {
return "", errors.New("no idle nodes available")
}
return nodeAddr, nil
}
//func (ns *nodesState) addTaskToNode(info *proto.S3ScanTaskInfo, nodeAddr string) {
func (ns *nodesState) addTaskToNode(info lcnodeTaskInfo, nodeAddr string) {
switch t := info.(type) {
case *proto.S3ScanTaskInfo:
if taskMap, ok := ns.workingNodes[nodeAddr]; !ok {
taskMap = make(map[string]lcnodeTaskInfo, 0)
taskMap[t.Id] = info
ns.workingNodes[nodeAddr] = taskMap
} else {
if _, ok = taskMap[t.Id]; !ok {
taskMap[t.Id] = info
} else {
log.LogWarnf("s3 scan task(%v) in routine(%v) is already running in nodeAddr(%v)", t.Id, t.RoutineId, nodeAddr)
}
}
case *proto.DelVerTaskInfo:
if taskMap, ok := ns.workingNodes[nodeAddr]; !ok {
taskMap = make(map[string]lcnodeTaskInfo, 0)
taskMap[t.Id] = info
ns.workingNodes[nodeAddr] = taskMap
} else {
if _, ok = taskMap[t.Id]; !ok {
taskMap[t.Id] = info
} else {
log.LogWarnf("delete vol version task(%v) is already running in nodeAddr(%v)", t.Id, nodeAddr)
}
}
default:
log.LogErrorf("Unknown task type: %v", t)
}
}
func (ns *nodesState) IsNodeBusy(nodeAddr string) bool {
ns.RLock()
defer ns.RUnlock()
var working, idle bool
_, working = ns.workingNodes[nodeAddr]
_, idle = ns.idleNodes[nodeAddr]
return working && !idle
}
//func (ns *nodesState) AddTaskToNode(info *proto.S3ScanTaskInfo, maxConcurrentLcNodes uint64) (nodeAddr string, err error) {
func (ns *nodesState) AddTaskToNode(info lcnodeTaskInfo, maxConcurrentLcNodes uint64) (nodeAddr string, err error) {
ns.Lock()
defer ns.Unlock()
nodeAddr, err = ns.getOneIdleNode()
if err != nil {
return nodeAddr, err
}
if _, ok := info.(*proto.S3ScanTaskInfo); ok && maxConcurrentLcNodes > 0 {
if len(ns.workingNodes) >= int(maxConcurrentLcNodes) {
return "", errors.New("max concurrent lcnodes reached")
}
}
delete(ns.idleNodes, nodeAddr)
ns.addTaskToNode(info, nodeAddr)
return
}
//func (ns *nodesState) UpdateNodeTask(nodeAddr string, infos []lcnodeTaskInfo) {
// if len(infos) == 0 {
// ns.Lock()
// delete(ns.workingNodes, nodeAddr)
// ns.idleNodes[nodeAddr] = nodeAddr
// ns.Unlock()
// return
// }
// ns.Lock()
// defer ns.Unlock()
//
// if _, ok := ns.workingNodes[nodeAddr]; !ok {
// for _, info := range infos {
// ns.addTaskToNode(info, nodeAddr)
// }
// delete(ns.idleNodes, nodeAddr)
// } else {
// delete(ns.workingNodes, nodeAddr)
// for _, info := range infos {
// ns.addTaskToNode(info, nodeAddr)
// }
//
// }
//
//}
func (ns *nodesState) UpdateNodeRunningTask(nodeAddr string, infos []lcnodeTaskInfo, incrementUpdate bool) {
ns.Lock()
defer ns.Unlock()
if !incrementUpdate {
if len(infos) == 0 {
delete(ns.workingNodes, nodeAddr)
ns.idleNodes[nodeAddr] = nodeAddr
return
} else {
delete(ns.workingNodes, nodeAddr)
for _, info := range infos {
ns.addTaskToNode(info, nodeAddr)
}
delete(ns.idleNodes, nodeAddr)
}
} else {
for _, info := range infos {
ns.addTaskToNode(info, nodeAddr)
}
if len(infos) != 0 {
delete(ns.idleNodes, nodeAddr)
}
}
}
//func (ns *nodesState) ReleaseTask(taskId string, RoutineID int64) {
// ns.Lock()
// defer ns.Unlock()
// for nodeAddr, taskMap := range ns.workingNodes {
// if info, ok := taskMap[taskId]; ok {
// if RoutineID == info.RoutineId {
// delete(taskMap, taskId)
// log.LogInfof("task(%v) in routine(%v) is released", taskId, RoutineID)
// if len(taskMap) == 0 {
// delete(ns.workingNodes, nodeAddr)
// ns.idleNodes[nodeAddr] = nodeAddr
// log.LogInfof("lcnode(%v) is set idle", nodeAddr)
// }
// }
//
// }
// }
//}
func (ns *nodesState) ReleaseTask(info lcnodeTaskInfo) {
ns.Lock()
defer ns.Unlock()
for nodeAddr, taskMap := range ns.workingNodes {
switch t := info.(type) {
case *proto.S3ScanTaskInfo:
if i, ok := taskMap[t.Id]; ok {
if it, ok := i.(*proto.S3ScanTaskInfo); ok {
if t.RoutineId == it.RoutineId {
delete(taskMap, t.Id)
log.LogInfof("s3 task(%v) in routine(%v) is released", t.Id, t.RoutineId)
if len(taskMap) == 0 {
delete(ns.workingNodes, nodeAddr)
ns.idleNodes[nodeAddr] = nodeAddr
log.LogInfof("lcnode(%v) is set idle", nodeAddr)
}
}
}
}
case *proto.DelVerTaskInfo:
if i, ok := taskMap[t.Id]; ok {
if _, ok := i.(*proto.DelVerTaskInfo); ok {
delete(taskMap, t.Id)
log.LogInfof("snap task(%v) is released", t.Id)
if len(taskMap) == 0 {
delete(ns.workingNodes, nodeAddr)
ns.idleNodes[nodeAddr] = nodeAddr
log.LogInfof("lcnode(%v) is set idle", nodeAddr)
}
}
}
default:
log.LogErrorf("Unknown task type: %v", t)
}
}
}
func (ns *nodesState) RemoveNode(nodeAddr string) (taskInfos []lcnodeTaskInfo) {
ns.Lock()
defer ns.Unlock()
if taskMap, ok := ns.workingNodes[nodeAddr]; ok {
for _, info := range taskMap {
taskInfos = append(taskInfos, info)
}
}
log.LogInfof("RemoveNode, %v tasks [%v] are still in processing on %v", len(taskInfos), taskInfos, nodeAddr)
delete(ns.idleNodes, nodeAddr)
delete(ns.workingNodes, nodeAddr)
return
}
type LcVerInfo struct {
proto.VerInfo
dTime time.Time
}
type ProcessingVerInfo struct {
LcVerInfo
updateTime time.Time
}
type VolVerInfos struct {
sync.RWMutex
VerInfos map[string]*LcVerInfo
//ProcessingVerInfos map[string]*LcVerInfo
ProcessingVerInfos map[string]*ProcessingVerInfo
TaskResults map[string]*proto.SnapshotVerDelTaskResponse
}
func (vvi *VolVerInfos) AddVerInfo(vi *LcVerInfo) {
vvi.Lock()
defer vvi.Unlock()
if pInfo, ok := vvi.ProcessingVerInfos[vi.Key()]; ok {
if time.Since(pInfo.updateTime) < time.Second*time.Duration(defaultIntervalToCheckDelVerTaskExpiration*defaultIntervalToCheck) {
log.LogDebugf("VerInfo: %v is already in processing", vi)
return
} else {
log.LogWarnf("VerInfo: %v is expired", vi)
}
}
vvi.VerInfos[vi.Key()] = vi
log.LogDebugf("Adding VerInfo: %v", vi)
}
func (vvi *VolVerInfos) RemoveProcessingVerInfo(verInfoKey string) (pInfo *ProcessingVerInfo) {
vvi.Lock()
defer vvi.Unlock()
var ok bool
if pInfo, ok = vvi.ProcessingVerInfos[verInfoKey]; !ok {
return
} else {
delete(vvi.ProcessingVerInfos, verInfoKey)
}
return
}
func (vvi *VolVerInfos) RemoveVerInfo(verInfoKey string) {
vvi.Lock()
defer vvi.Unlock()
delete(vvi.VerInfos, verInfoKey)
delete(vvi.ProcessingVerInfos, verInfoKey)
}
func (vvi *VolVerInfos) FetchOldestVerInfo() (info *LcVerInfo) {
var min int64 = math.MaxInt64
minKey := ""
vvi.Lock()
defer vvi.Unlock()
if len(vvi.VerInfos) == 0 {
return nil
}
for _, i := range vvi.VerInfos {
if i.dTime.Unix() < min {
min = i.dTime.Unix()
minKey = i.Key()
}
}
info = vvi.VerInfos[minKey]
delete(vvi.VerInfos, minKey)
pInfo := &ProcessingVerInfo{
LcVerInfo: *info,
updateTime: time.Now(),
}
vvi.ProcessingVerInfos[minKey] = pInfo
return
}
func (vvi *VolVerInfos) ReturnProcessingVerInfo(keys []string) {
vvi.Lock()
defer vvi.Unlock()
log.LogInfof("ReturnProcessingVerInfo, verinfo: %v", keys)
for _, k := range keys {
if pInfo, ok := vvi.ProcessingVerInfos[k]; ok {
vvi.VerInfos[k] = &pInfo.LcVerInfo
delete(vvi.ProcessingVerInfos, k)
log.LogInfof("ReturnProcessingVerInfo, remove %v from processing", vvi.VerInfos[k])
}
}
}
/*func (vvi *VolVerInfos) ReturnProcessingVerInfo(info *LcVerInfo) {
vvi.Lock()
defer vvi.Unlock()
if _, ok := vvi.VerInfos[info.Key()]; !ok {
vvi.VerInfos[info.Key()] = info
}
delete(vvi.ProcessingVerInfos, info.Key())
}*/
type snapshotDelManager struct {
volVerInfos VolVerInfos
idleNodeCh chan struct{}
exitCh chan struct{}
lcMgr *lifecycleManager
}
//func (m *snapshotDelManager) AddVerInfo(vi *LcVerInfo) {
// m.lcMgr.snapshotMgr.volVerInfos.Lock()
// defer m.lcMgr.snapshotMgr.volVerInfos.Unlock()
// if _, ok := m.lcMgr.snapshotMgr.volVerInfos.ProcessingVerInfos[vi.Key()]; ok {
// if m.isVerInfoInProcessing(vi) {
// log.LogDebugf("VerInfo: %v is already in processing", vi)
// return
// } else {
// log.LogWarnf("VerInfo: %v is not in processing", vi)
// }
//
// }
// m.lcMgr.snapshotMgr.volVerInfos.VerInfos[vi.Key()] = vi
// log.LogDebugf("Adding VerInfo: %v", vi)
//}
//func (m *snapshotDelManager) isVerInfoInProcessing(vi *LcVerInfo) bool {
// m.lcMgr.lnStates.RLock()
// defer m.lcMgr.lnStates.RUnlock()
// for nodeAddr, taskMap := range m.lcMgr.lnStates.workingNodes {
// if _, ok := taskMap[vi.Key()]; ok {
// log.LogDebugf("task %v is already in processing on %v", vi, nodeAddr)
// return true
// }
// }
// return false
//}
func (m *snapshotDelManager) NotifyIdleLcNode() {
m.volVerInfos.RLock()
defer m.volVerInfos.RUnlock()
if len(m.volVerInfos.VerInfos) > 0 {
select {
case m.idleNodeCh <- struct{}{}:
log.LogDebugf("snapshotDelManager NotifyIdleLcNode, scan routine notified!")
default:
log.LogDebugf("snapshotDelManager NotifyIdleLcNode, skipping notify!")
}
}
}
func (m *snapshotDelManager) Start() {
go m.process()
}
func (m *snapshotDelManager) Stop() {
defer func() {
if r := recover(); r != nil {
log.LogErrorf("snapshot process Stop,err:%v", r)
}
}()
close(m.exitCh)
}
func (m *snapshotDelManager) process() {
defer func() {
log.LogDebugf("snapshot process exit")
}()
for {
select {
case <-m.exitCh:
return
case <-m.idleNodeCh:
aTasks := make([]*proto.AdminTask, 0)
vi := m.volVerInfos.FetchOldestVerInfo()
if vi == nil {
log.LogDebugf("no snapshot ver del task available")
break
}
info := &proto.DelVerTaskInfo{Id: vi.Key()}
nodeAddr, err := m.lcMgr.lnStates.AddTaskToNode(info, 0)
if err != nil {
log.LogWarnf("AddTaskToNode failed: err(%v)", err)
rtTasks := make([]string, 0)
rtTasks = append(rtTasks, vi.Key())
m.volVerInfos.ReturnProcessingVerInfo(rtTasks)
break
}
val, ok := m.lcMgr.lcNodes.Load(nodeAddr)
if !ok {
log.LogErrorf("nodeAddr(%v) is not available for scanning!", nodeAddr)
m.lcMgr.lnStates.RemoveNode(nodeAddr)
break
}
task := &proto.SnapshotVerDelTask{
VerInfo: proto.VerInfo{
VolName: vi.VolName,
VerSeq: vi.VerSeq,
},
}
node := val.(*LcNode)
aTask := node.createSnapshotVerDelTask(task, m.lcMgr.cluster.masterAddr())
aTasks = append(aTasks, aTask)
m.lcMgr.cluster.addLcNodeTasks(aTasks)
log.LogDebugf("add snapshot version del task(%v) to lcnode(%v)", *task, nodeAddr)
}
}
}
type lifecycleManager struct {
cluster *Cluster
//lcnodes
lnMutex sync.RWMutex
lcNodes sync.Map
//lcnode working state
lnStates *nodesState
//s3 lifecycle configuration
s3ConfMutex sync.RWMutex
s3LcConfs map[string]*proto.LcConfiguration
//s3 scan routine
scanMutex sync.RWMutex
lcScan *scanRoutine
//snapshot
snapshotMgr *snapshotDelManager
}
//type RuleTask struct {
// Id string
// Rule *proto.Rule
//}
type lcRuleStatus struct {
Scanned []*proto.RuleTask
Scanning []*proto.RuleTask
ToBeScanned []*proto.RuleTask
//Results []*proto.RuleTaskResponse
Results map[string]*proto.RuleTaskResponse
}
type scanRoutine struct {
sync.RWMutex
s3LcMgr *lifecycleManager
Id int64
stopped bool
StartTime *time.Time
EndTime *time.Time
RuleStatus lcRuleStatus
exitCh chan struct{}
idleNodeCh chan struct{}
}
func newLifecycleManager() *lifecycleManager {
log.LogInfof("action[newLifecycleManager] construct")
lcMgr := &lifecycleManager{
s3LcConfs: make(map[string]*proto.LcConfiguration),
lnStates: &nodesState{
workingNodes: make(map[string]map[string]lcnodeTaskInfo),
idleNodes: make(map[string]string),
},
}
lcMgr.snapshotMgr = &snapshotDelManager{
volVerInfos: VolVerInfos{
VerInfos: make(map[string]*LcVerInfo, 0),
ProcessingVerInfos: make(map[string]*ProcessingVerInfo, 0),
TaskResults: make(map[string]*proto.SnapshotVerDelTaskResponse, 0),
},
exitCh: make(chan struct{}),
idleNodeCh: make(chan struct{}),
lcMgr: lcMgr,
}
lcMgr.snapshotMgr.Start()
return lcMgr
}
func newScanRoutine(s3LcMgr *lifecycleManager) *scanRoutine {
lcScan := &scanRoutine{
stopped: true,
s3LcMgr: s3LcMgr,
RuleStatus: lcRuleStatus{
Scanned: make([]*proto.RuleTask, 0),
Scanning: make([]*proto.RuleTask, 0),
ToBeScanned: make([]*proto.RuleTask, 0),
Results: make(map[string]*proto.RuleTaskResponse, 0),
},
exitCh: make(chan struct{}),
idleNodeCh: make(chan struct{}),
}
return lcScan
}
func (s *scanRoutine) Start(tasks []*proto.RuleTask) {
t := time.Now()
s.StartTime = &t
s.Id = t.Unix()
s.RuleStatus.ToBeScanned = append(s.RuleStatus.ToBeScanned, tasks...)
s.stopped = false
ruleNum := len(s.RuleStatus.ToBeScanned)
log.LogInfof("Start scan, rule num(%v)", ruleNum)
go s.process()
}
func (s *scanRoutine) redoTasks(taskInfos []*proto.S3ScanTaskInfo) {
s.Lock()
defer s.Unlock()
for _, info := range taskInfos {
if info.RoutineId == s.Id {
var task *proto.RuleTask
for i := 0; i < len(s.RuleStatus.Scanning); i++ {
if s.RuleStatus.Scanning[i].Id == info.Id {
task = s.RuleStatus.Scanning[i]
s.RuleStatus.Scanning = append(s.RuleStatus.Scanning[:i], s.RuleStatus.Scanning[i+1:]...)
break
}
}
s.RuleStatus.ToBeScanned = append(s.RuleStatus.ToBeScanned, task)
} else {
log.LogWarnf("redoTasks: skip expired task(%v) in routine(%v), current scan routine(%v)",
info.Id, info.RoutineId, s.Id)
continue
}
}
}
func (s *scanRoutine) process() {
defer func() {
log.LogDebugf("scan routine(%v) process exit", s.Id)
}()
for {
select {
case <-s.exitCh:
return
case <-s.idleNodeCh:
log.LogDebugf("scan routine notified")
s.Lock()
for len(s.RuleStatus.ToBeScanned) > 0 {
aTasks := make([]*proto.AdminTask, 0)
var task = s.RuleStatus.ToBeScanned[0]
info := &proto.S3ScanTaskInfo{
Id: task.Id,
RoutineId: s.Id,
}
maxConcurrentLcNodes := s.s3LcMgr.cluster.cfg.MaxConcurrentLcNodes
nodeAddr, err := s.s3LcMgr.lnStates.AddTaskToNode(info, maxConcurrentLcNodes)
if err != nil {
log.LogWarnf("scan routine: err(%v)", err)
break
}
val, ok := s.s3LcMgr.lcNodes.Load(nodeAddr)
if !ok {
log.LogErrorf("nodeAddr(%v) is not available for scanning!", nodeAddr)
s.s3LcMgr.lnStates.RemoveNode(nodeAddr)
break
}
s.RuleStatus.ToBeScanned = s.RuleStatus.ToBeScanned[1:]
s.RuleStatus.Scanning = append(s.RuleStatus.Scanning, task)
node := val.(*LcNode)
aTask := node.createLcScanTask(s.s3LcMgr.lcScan.Id, task, s.s3LcMgr.cluster.masterAddr())
aTasks = append(aTasks, aTask)
s.s3LcMgr.cluster.addLcNodeTasks(aTasks)
log.LogDebugf("scan routine add lifecycle scan task(%v) to lcnode(%v) in routine(%v)", *task, nodeAddr, s.Id)
if len(s.RuleStatus.ToBeScanned) == 0 {
s.Unlock()
log.LogDebugf("scan routine all rule task is scheduled!")
return
}
}
s.Unlock()
}
}
}
func (s *scanRoutine) Scanning() bool {
s.RLock()
defer s.RUnlock()
return s.scanning() && !s.stopped
}
func (s *scanRoutine) scanning() bool {
return len(s.RuleStatus.ToBeScanned) != 0 || len(s.RuleStatus.Scanning) != 0
}
func (s *scanRoutine) printStatistics(tobeScanned, scanning []string) {
var totalInodeScannedNum int64
var FileScannedNum int64
var DirScannedNum int64
var ExpiredNum int64
var ErrorSkippedNum int64
var AbortedIncompleteMultipartNum int64
for _, r := range s.RuleStatus.Results {
totalInodeScannedNum += r.TotalInodeScannedNum
FileScannedNum += r.FileScannedNum
DirScannedNum += r.DirScannedNum
ExpiredNum += r.ExpiredNum
ErrorSkippedNum += r.ErrorSkippedNum
AbortedIncompleteMultipartNum += r.AbortedIncompleteMultipartNum
}
if len(tobeScanned) == 0 && len(scanning) == 0 {
log.LogInfof("Expiration scan for routine(%v) is completed, startTime(%v), endTime(%v), totalScanned(%v), ExpiredNum(%v), AbortedIncompleteMultipartNum(%v)",
s.Id, s.StartTime.String(), s.EndTime.String(), totalInodeScannedNum, ExpiredNum, AbortedIncompleteMultipartNum)
} else {
log.LogWarnf("Expiration scan for routine(%v) is not completed, startTime(%v), endTime(%v), (%v) tasks are not started, (%v) tasks are scanning, totalScanned(%v), ExpiredNum(%v), AbortedIncompleteMultipartNum(%v))",
s.Id, s.StartTime.String(), s.EndTime.String(), len(tobeScanned), len(scanning), totalInodeScannedNum, ExpiredNum, AbortedIncompleteMultipartNum)
}
}
func (s *scanRoutine) Stop() {
defer func() {
if r := recover(); r != nil {
log.LogErrorf("scanRoutine Stop,err:%v", r)
}
}()
s.Lock()
defer s.Unlock()
tobeScanned := make([]string, 0)
scanning := make([]string, 0)
if !s.stopped {
t := time.Now()
s.EndTime = &t
if s.scanning() {
for _, t := range s.RuleStatus.ToBeScanned {
tobeScanned = append(tobeScanned, t.Id)
}
for _, t := range s.RuleStatus.Scanning {
scanning = append(scanning, t.Id)
}
s.RuleStatus.ToBeScanned = nil
s.RuleStatus.Scanning = nil
}
close(s.exitCh)
close(s.idleNodeCh)
s.stopped = true
s.printStatistics(tobeScanned, scanning)
}
log.LogInfof("Scan routine(%v) stopped", s.Id)
}
func (lcMgr *lifecycleManager) genRuleTasks() []*proto.RuleTask {
lcMgr.s3ConfMutex.RLock()
defer lcMgr.s3ConfMutex.RUnlock()
tasks := make([]*proto.RuleTask, 0)
for _, v := range lcMgr.s3LcConfs {
ts := v.GenRuleTasks()
if len(ts) > 0 {
tasks = append(tasks, ts...)
}
}
return tasks
}
func (lcMgr *lifecycleManager) setNewScanRoutine(scan *scanRoutine) {
lcMgr.scanMutex.RLock()
oldScan := lcMgr.lcScan
lcMgr.scanMutex.RUnlock()
if oldScan != nil {
oldScan.Stop()
log.LogInfof("setNewScanRoutine: old routine(%v) is stopped!", oldScan.Id)
}
lcMgr.scanMutex.Lock()
lcMgr.lcScan = scan
lcMgr.scanMutex.Unlock()
log.LogInfof("setNewScanRoutine: new routine(%v) is started!", scan.Id)
}
func (lcMgr *lifecycleManager) getScanRoutine() *scanRoutine {
lcMgr.scanMutex.RLock()
defer lcMgr.scanMutex.RUnlock()
return lcMgr.lcScan
}
func (lcMgr *lifecycleManager) NotifyIdleLcNode() {
//snapshot version delete first
lcMgr.snapshotMgr.NotifyIdleLcNode()
lcScan := lcMgr.getScanRoutine()
if lcScan != nil {
select {
case lcScan.idleNodeCh <- struct{}{}:
log.LogDebugf("action[handleLcNodeHeartbeatResp], scan routine notified!")
default:
log.LogDebugf("action[handleLcNodeHeartbeatResp], skipping notify looper")
}
}
}
func (lcMgr *lifecycleManager) genEnabledRuleTasks() []*proto.RuleTask {
lcMgr.s3ConfMutex.RLock()
defer lcMgr.s3ConfMutex.RUnlock()
tasks := make([]*proto.RuleTask, 0)
for _, v := range lcMgr.s3LcConfs {
ts := v.GenEnabledRuleTasks()
if len(ts) > 0 {
tasks = append(tasks, ts...)
}
}
return tasks
}
func (lcMgr *lifecycleManager) rescheduleScanRoutine() {
tasks := lcMgr.genEnabledRuleTasks()
if len(tasks) <= 0 {
log.LogDebugf("rescheduleScanRoutine: no enabled lifecycle rule task to schedule!")
return
} else {
log.LogDebugf("rescheduleScanRoutine: %v lifecycle rule tasks to schedule!", len(tasks))
}
lcScan := lcMgr.getScanRoutine()
if lcScan != nil && lcScan.Scanning() {
lcScan.RLock()
log.LogWarnf("rescheduleScanRoutine: scanning is not completed, lcScan(%v)", lcScan)
toBeScannedMap := make(map[string]*proto.RuleTask, 0)
ScannedMap := make(map[string]*proto.RuleTask, 0)
ScanningMap := make(map[string]*proto.RuleTask, 0)
for _, ruleTask := range lcScan.RuleStatus.ToBeScanned {
toBeScannedMap[ruleTask.Id] = ruleTask
}
for _, ruleTask := range lcScan.RuleStatus.Scanned {
ScannedMap[ruleTask.Id] = ruleTask
}
for _, ruleTask := range lcScan.RuleStatus.Scanning {
ScanningMap[ruleTask.Id] = ruleTask
}
toBeScannedSlice := make([]*proto.RuleTask, 0)
ScannedSlice := make([]*proto.RuleTask, 0)
ScanningSlice := make([]*proto.RuleTask, 0)
NewTasksSlice := make([]*proto.RuleTask, 0)
for _, r := range tasks {
if _, ok := toBeScannedMap[r.Id]; ok {
toBeScannedSlice = append(toBeScannedSlice, r)
} else if _, ok := ScannedMap[r.Id]; ok {
ScannedSlice = append(ScannedSlice, r)
} else if _, ok := ScanningMap[r.Id]; ok {
ScanningSlice = append(ScanningSlice, r)
} else {
NewTasksSlice = append(NewTasksSlice, r)
}
}
tasks = tasks[:0]
tasks = append(tasks, toBeScannedSlice...)
tasks = append(tasks, NewTasksSlice...)
tasks = append(tasks, ScannedSlice...)
tasks = append(tasks, ScanningSlice...)
lcScan.RUnlock()
log.LogWarnf("rescheduleScanRoutine: tasks are rearranged")
}
newLcScan := newScanRoutine(lcMgr)
newLcScan.Start(tasks)
lcMgr.setNewScanRoutine(newLcScan)
}
func (lcMgr *lifecycleManager) SetS3BucketLifecycle(lcConf *proto.LcConfiguration) error {
lcMgr.s3ConfMutex.Lock()
defer lcMgr.s3ConfMutex.Unlock()
lcMgr.s3LcConfs[lcConf.VolName] = lcConf
return nil
}
func (lcMgr *lifecycleManager) GetS3BucketLifecycle(VolName string) (lcConf *proto.LcConfiguration) {
lcMgr.s3ConfMutex.RLock()
defer lcMgr.s3ConfMutex.RUnlock()
var ok bool
lcConf, ok = lcMgr.s3LcConfs[VolName]
if !ok {
return nil
}
return lcConf
}
func (lcMgr *lifecycleManager) DelS3BucketLifecycle(VolName string) {
lcMgr.s3ConfMutex.Lock()
defer lcMgr.s3ConfMutex.Unlock()
delete(lcMgr.s3LcConfs, VolName)
}

341
master/lifecycle_manager.go Normal file
View File

@ -0,0 +1,341 @@
// Copyright 2023 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package master
import (
"sync"
"time"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/util/log"
)
type lifecycleManager struct {
sync.RWMutex
cluster *Cluster
lcConfigurations map[string]*proto.LcConfiguration
lcNodeStatus *lcNodeStatus
lcRuleTaskStatus *lcRuleTaskStatus
idleLcNodeCh chan struct{}
exitCh chan struct{}
}
type lcNodeStatus struct {
sync.RWMutex
idleNodes map[string]string //ip:ip
workingNodes map[string]interface{} //ip:task
}
type lcRuleTaskStatus struct {
sync.RWMutex
Scanned []*proto.RuleTask
Scanning []*proto.RuleTask
ToBeScanned []*proto.RuleTask
Results map[string]*proto.LcNodeRuleTaskResponse
StartTime *time.Time
EndTime *time.Time
}
func newLifecycleManager() *lifecycleManager {
log.LogInfof("action[newLifecycleManager] construct")
lcMgr := &lifecycleManager{
lcConfigurations: make(map[string]*proto.LcConfiguration),
lcNodeStatus: &lcNodeStatus{
workingNodes: make(map[string]interface{}),
idleNodes: make(map[string]string),
},
lcRuleTaskStatus: &lcRuleTaskStatus{
Results: make(map[string]*proto.LcNodeRuleTaskResponse),
},
idleLcNodeCh: make(chan struct{}),
exitCh: make(chan struct{}),
}
return lcMgr
}
func (lcMgr *lifecycleManager) startLcScan() {
tasks := lcMgr.genEnabledRuleTasks()
if len(tasks) <= 0 {
log.LogDebugf("startLcScan: no enabled lifecycle rule task to schedule!")
return
} else {
log.LogDebugf("startLcScan: %v lifecycle rule tasks to schedule!", len(tasks))
}
// stop if already scanning
if lcMgr.scanning() {
log.LogWarnf("rescheduleScanRoutine: scanning is not completed, lcRuleTaskStatus(%v)", lcMgr.lcRuleTaskStatus)
return
}
// start scan init
lcMgr.lcRuleTaskStatus = &lcRuleTaskStatus{
Results: make(map[string]*proto.LcNodeRuleTaskResponse),
}
lcMgr.lcNodeStatus.workingNodes = make(map[string]interface{})
lcMgr.lcRuleTaskStatus.ToBeScanned = append(lcMgr.lcRuleTaskStatus.ToBeScanned, tasks...)
go lcMgr.scheduleLcScanProcess()
}
// generate tasks for every bucket
func (lcMgr *lifecycleManager) genEnabledRuleTasks() []*proto.RuleTask {
lcMgr.RLock()
defer lcMgr.RUnlock()
tasks := make([]*proto.RuleTask, 0)
for _, v := range lcMgr.lcConfigurations {
ts := v.GenEnabledRuleTasks()
if len(ts) > 0 {
tasks = append(tasks, ts...)
}
}
return tasks
}
func (lcMgr *lifecycleManager) scanning() bool {
log.LogInfof("scanning lcNodeStatus: %v", lcMgr.lcNodeStatus)
if len(lcMgr.lcRuleTaskStatus.ToBeScanned) > 0 {
return true
}
//handling exceptions
if len(lcMgr.lcRuleTaskStatus.Scanning) > 0 {
scanning := lcMgr.lcRuleTaskStatus.Scanning
var nodes []string
lcMgr.cluster.lcNodes.Range(func(addr, value interface{}) bool {
nodes = append(nodes, addr.(string))
return true
})
for _, task := range scanning {
workingNodes := lcMgr.lcNodeStatus.workingNodes
var node string
for nodeAddr, t := range workingNodes {
if task.Id == t.(*proto.RuleTask).Id {
node = nodeAddr
}
}
if exist(node, nodes) {
log.LogInfof("scanning task: %v, exist node: %v, all nodes: %v", task, node, nodes)
return true
}
log.LogInfof("scanning task: %v, but not exist node: %v, all nodes: %v", task, node, nodes)
}
}
return false
}
func exist(node string, nodes []string) bool {
for _, n := range nodes {
if node == n {
return true
}
}
return false
}
func (lcMgr *lifecycleManager) scheduleLcScanProcess() {
log.LogInfof("start scheduleLcScanProcess, rule num(%v)", len(lcMgr.lcRuleTaskStatus.ToBeScanned))
now := time.Now()
lcMgr.lcRuleTaskStatus.StartTime = &now
for lcMgr.scanning() {
log.LogDebugf("wait idleLcNodeCh... ToBeScanned num(%v), Scanning num(%v)", len(lcMgr.lcRuleTaskStatus.ToBeScanned), len(lcMgr.lcRuleTaskStatus.Scanning))
select {
case <-lcMgr.exitCh:
log.LogDebug("exitCh notified")
return
case <-lcMgr.idleLcNodeCh:
log.LogDebug("idleLcNodeCh notified")
// get ToBeScanned task
task := lcMgr.lcRuleTaskStatus.getOneToBeScannedTask()
if task == nil {
log.LogWarn("no ToBeScanned task")
continue
}
// get idle lcnode
nodeAddr := lcMgr.lcNodeStatus.getOneIdleNode(task)
if nodeAddr == "" {
log.LogWarn("no idle lcnode")
continue
}
val, ok := lcMgr.cluster.lcNodes.Load(nodeAddr)
if !ok {
log.LogErrorf("nodeAddr(%v) is not available for scanning!", nodeAddr)
t := lcMgr.lcNodeStatus.removeNode(nodeAddr)
lcMgr.lcRuleTaskStatus.redoTask(t.(*proto.RuleTask))
continue
}
node := val.(*LcNode)
adminTask := node.createLcScanTask(lcMgr.cluster.masterAddr(), task)
lcMgr.cluster.addLcNodeTasks([]*proto.AdminTask{adminTask})
log.LogDebugf("add lifecycle scan task(%v) to lcnode(%v)", *task, nodeAddr)
}
}
now = time.Now()
lcMgr.lcRuleTaskStatus.EndTime = &now
log.LogInfof("finish scheduleLcScanProcess, lcRuleTaskStatus results(%v)", lcMgr.lcRuleTaskStatus.Results)
}
func (lcMgr *lifecycleManager) notifyIdleLcNode() {
select {
case lcMgr.idleLcNodeCh <- struct{}{}:
log.LogDebug("action[handleLcNodeHeartbeatResp], lifecycleManager scan routine notified!")
default:
log.LogDebug("action[handleLcNodeHeartbeatResp], lifecycleManager skipping notify!")
}
}
func (lcMgr *lifecycleManager) SetS3BucketLifecycle(lcConf *proto.LcConfiguration) error {
lcMgr.Lock()
defer lcMgr.Unlock()
lcMgr.lcConfigurations[lcConf.VolName] = lcConf
return nil
}
func (lcMgr *lifecycleManager) GetS3BucketLifecycle(VolName string) (lcConf *proto.LcConfiguration) {
lcMgr.RLock()
defer lcMgr.RUnlock()
var ok bool
lcConf, ok = lcMgr.lcConfigurations[VolName]
if !ok {
return nil
}
return lcConf
}
func (lcMgr *lifecycleManager) DelS3BucketLifecycle(VolName string) {
lcMgr.Lock()
defer lcMgr.Unlock()
delete(lcMgr.lcConfigurations, VolName)
}
func (rs *lcRuleTaskStatus) getOneToBeScannedTask() (task *proto.RuleTask) {
rs.Lock()
defer rs.Unlock()
if len(rs.ToBeScanned) > 0 {
task = rs.ToBeScanned[0]
rs.ToBeScanned = rs.ToBeScanned[1:]
rs.Scanning = append(rs.Scanning, task)
}
return
}
func (rs *lcRuleTaskStatus) redoTask(task *proto.RuleTask) {
rs.Lock()
defer rs.Unlock()
if task != nil {
for i := 0; i < len(rs.Scanning); i++ {
if rs.Scanning[i].Id == task.Id {
rs.Scanning = append(rs.Scanning[:i], rs.Scanning[i+1:]...)
break
}
}
rs.ToBeScanned = append(rs.ToBeScanned, task)
}
return
}
func (rs *lcRuleTaskStatus) finishTask(task *proto.RuleTask, resp *proto.LcNodeRuleTaskResponse) {
rs.Lock()
defer rs.Unlock()
if task != nil {
for i := 0; i < len(rs.Scanning); i++ {
if rs.Scanning[i].Id == task.Id {
rs.Scanning = append(rs.Scanning[:i], rs.Scanning[i+1:]...)
break
}
}
rs.Scanned = append(rs.Scanned, task)
rs.Results[resp.ID] = resp
}
return
}
func (rs *lcRuleTaskStatus) deleteScanningTask(task *proto.RuleTask) {
rs.Lock()
defer rs.Unlock()
if task != nil {
for i := 0; i < len(rs.Scanning); i++ {
if rs.Scanning[i].Id == task.Id {
rs.Scanning = append(rs.Scanning[:i], rs.Scanning[i+1:]...)
break
}
}
}
return
}
func (ns *lcNodeStatus) getOneIdleNode(task interface{}) (nodeAddr string) {
ns.Lock()
defer ns.Unlock()
if len(ns.idleNodes) > 0 {
for n := range ns.idleNodes {
nodeAddr = n
delete(ns.idleNodes, nodeAddr)
ns.workingNodes[nodeAddr] = task
return
}
}
return
}
func (ns *lcNodeStatus) removeNode(nodeAddr string) (task interface{}) {
ns.Lock()
defer ns.Unlock()
task = ns.workingNodes[nodeAddr]
delete(ns.idleNodes, nodeAddr)
delete(ns.workingNodes, nodeAddr)
return
}
func (ns *lcNodeStatus) releaseNode(nodeAddr string) (task interface{}) {
ns.Lock()
defer ns.Unlock()
task = ns.workingNodes[nodeAddr]
delete(ns.workingNodes, nodeAddr)
ns.idleNodes[nodeAddr] = nodeAddr
return task
}
func (ns *lcNodeStatus) getWorkingTask(nodeAddr string) (task interface{}) {
ns.RLock()
defer ns.RUnlock()
return ns.workingNodes[nodeAddr]
}
//----------------------------------------------
type LcNodeStatInfo struct {
Addr string
TaskId string
}
type LcNodeInfoResponse struct {
Infos []*LcNodeStatInfo
LcConfigurations map[string]*proto.LcConfiguration
LcRuleTaskStatus *lcRuleTaskStatus
}
//-----------------------------------------------

18
master/lc_node.go → master/lifecycle_node.go
View File

@ -1,4 +1,4 @@
// Copyright 2018 The Chubao Authors.
// Copyright 2023 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -15,7 +15,6 @@
package master
import (
"fmt"
"sync"
"time"
@ -66,22 +65,21 @@ func (lcNode *LcNode) createHeartbeatTask(masterAddr string) (task *proto.AdminT
return
}
func (lcNode *LcNode) createLcScanTask(routineId int64, rTask *proto.RuleTask, masterAddr string) (task *proto.AdminTask) {
request := &proto.RuleTaskRequest{
func (lcNode *LcNode) createLcScanTask(masterAddr string, ruleTask *proto.RuleTask) (task *proto.AdminTask) {
request := &proto.LcNodeRuleTaskRequest{
MasterAddr: masterAddr,
Task: rTask,
RoutineID: routineId,
LcNodeAddr: lcNode.Addr,
Task: ruleTask,
}
reqID := fmt.Sprintf("%v_%v", routineId, rTask.Id)
task = proto.NewAdminTaskEx(proto.OpLcNodeScan, lcNode.Addr, request, reqID)
task = proto.NewAdminTaskEx(proto.OpLcNodeScan, lcNode.Addr, request, ruleTask.Id)
return
}
func (lcNode *LcNode) createSnapshotVerDelTask(sTask *proto.SnapshotVerDelTask, masterAddr string) (task *proto.AdminTask) {
func (lcNode *LcNode) createSnapshotVerDelTask(masterAddr string, sTask *proto.SnapshotVerDelTask) (task *proto.AdminTask) {
request := &proto.SnapshotVerDelTaskRequest{
MasterAddr: masterAddr,
LcNodeAddr: lcNode.Addr,
Task: sTask,
}
task = proto.NewAdminTaskEx(proto.OpLcNodeSnapshotVerDel, lcNode.Addr, request, request.Task.Key())

215
master/lifecycle_task.go Normal file
View File

@ -0,0 +1,215 @@
// Copyright 2023 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package master
import (
"encoding/json"
"fmt"
"time"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/util/log"
)
func (c *Cluster) handleLcNodeTaskResponse(nodeAddr string, task *proto.AdminTask) {
if task == nil {
log.LogInfof("lc action[handleLcNodeTaskResponse] receive addr[%v] task response,but task is nil", nodeAddr)
return
}
log.LogInfof("lc action[handleLcNodeTaskResponse] receive addr[%v] task:%v", nodeAddr, task.ToString())
var (
err error
lcNode *LcNode
)
if lcNode, err = c.lcNode(nodeAddr); err != nil {
goto errHandler
}
lcNode.TaskManager.DelTask(task)
if err = unmarshalTaskResponse(task); err != nil {
goto errHandler
}
switch task.OpCode {
case proto.OpLcNodeHeartbeat:
response := task.Response.(*proto.LcNodeHeartbeatResponse)
err = c.handleLcNodeHeartbeatResp(task.OperatorAddr, response)
case proto.OpLcNodeScan:
bytes, err := json.Marshal(task.Request)
if err != nil {
log.LogErrorf("lc action[handleLcNodeTaskResponse] Marshal task request failed!")
return
}
var request interface{}
request = &proto.LcNodeRuleTaskRequest{}
if err = json.Unmarshal(bytes, request); err != nil {
log.LogErrorf("lc action[handleLcNodeTaskResponse] Unmarshal task request failed!")
return
}
task.Request = request
req := task.Request.(*proto.LcNodeRuleTaskRequest)
response := task.Response.(*proto.LcNodeRuleTaskResponse)
err = c.handleLcNodeLcScanResp(req, response)
case proto.OpLcNodeSnapshotVerDel:
req := task.Request.(*proto.SnapshotVerDelTaskRequest)
response := task.Response.(*proto.SnapshotVerDelTaskResponse)
err = c.handleLcNodeSnapshotScanResp(req, response)
default:
err = fmt.Errorf(fmt.Sprintf("lc unknown operate code %v", task.OpCode))
goto errHandler
}
if err != nil {
goto errHandler
}
return
errHandler:
log.LogErrorf("lc handleLcNodeTaskResponse failed, task: %v, err: %v", task.ToString(), err)
return
}
func (c *Cluster) handleLcNodeLcScanResp(req *proto.LcNodeRuleTaskRequest, resp *proto.LcNodeRuleTaskResponse) (err error) {
log.LogDebugf("action[handleLcNodeLcScanResp] Rule(%v) in task(%v) Enter", req.Task.Rule.ID, req.Task.Id)
defer func() {
log.LogDebugf("action[handleLcNodeLcScanResp] Rule(%v) in task(%v) Exit", req.Task.Rule.ID, req.Task.Id)
}()
c.lcMgr.lcNodeStatus.releaseNode(req.LcNodeAddr)
if resp.Status == proto.TaskFailed {
c.lcMgr.lcRuleTaskStatus.redoTask(req.Task)
} else {
c.lcMgr.lcRuleTaskStatus.finishTask(req.Task, resp)
}
log.LogInfof("action[handleLcNodeLcScanResp] task(%v) scanning completed, current rule(%v), vol(%v), results(%v).",
req.Task.Id, req.Task.Rule.ID, req.Task.VolName, resp)
return
}
func (c *Cluster) handleLcNodeHeartbeatResp(nodeAddr string, resp *proto.LcNodeHeartbeatResponse) (err error) {
var (
lcNode *LcNode
logMsg string
)
log.LogDebugf("action[handleLcNodeHeartbeatResp] clusterID[%v] receive lcNode[%v] heartbeat", c.Name, nodeAddr)
if resp.Status != proto.TaskSucceeds {
Warn(c.Name, fmt.Sprintf("action[handleLcNodeHeartbeatResp] clusterID[%v] lcNode[%v] heartbeat task failed, err[%v]",
c.Name, nodeAddr, resp.Result))
return
}
if lcNode, err = c.lcNode(nodeAddr); err != nil {
goto errHandler
}
lcNode.Lock()
lcNode.IsActive = true
lcNode.ReportTime = time.Now()
lcNode.Unlock()
//handle LcScanningTasks
if len(resp.LcScanningTasks) != 0 {
for _, taskRsp := range resp.LcScanningTasks {
c.lcMgr.lcRuleTaskStatus.Lock()
c.lcMgr.lcRuleTaskStatus.Results[taskRsp.ID] = taskRsp
c.lcMgr.lcRuleTaskStatus.Unlock()
log.LogDebugf("action[handleLcNodeHeartbeatResp], lcNode(%v) scanning LcScanningTasks rsp(%v)", nodeAddr, taskRsp)
}
} else {
log.LogDebugf("action[handleLcNodeHeartbeatResp], lcNode[%v] is idle for LcScanningTasks", nodeAddr)
task := c.lcMgr.lcNodeStatus.releaseNode(nodeAddr)
c.lcMgr.lcRuleTaskStatus.deleteScanningTask(task.(*proto.RuleTask))
c.lcMgr.notifyIdleLcNode()
}
//handle SnapshotScanningTasks
if len(resp.SnapshotScanningTasks) != 0 {
for _, taskRsp := range resp.SnapshotScanningTasks {
c.snapshotMgr.volVerInfos.Lock()
c.snapshotMgr.volVerInfos.TaskResults[taskRsp.ID] = taskRsp
if pInfo, ok := c.snapshotMgr.volVerInfos.ProcessingVerInfos[taskRsp.ID]; ok {
pInfo.updateTime = time.Now()
log.LogDebugf("action[handleLcNodeHeartbeatResp], snapshot scan taskid(%v) update time",
taskRsp.ID)
} else {
//in case of master restart or leader changing, resume processing status
if vol, err := c.getVol(taskRsp.VolName); err == nil {
volVerInfoList := vol.VersionMgr.getVersionList()
for _, volVerInfo := range volVerInfoList.VerList {
if volVerInfo.Ver == taskRsp.VerSeq && volVerInfo.Status == proto.VersionDeleting {
pInfo := &ProcessingVerInfo{
LcVerInfo: LcVerInfo{
VerInfo: proto.VerInfo{
VolName: taskRsp.VolName,
VerSeq: taskRsp.VerSeq,
},
dTime: volVerInfo.DelTime,
},
updateTime: time.Now(),
}
c.snapshotMgr.volVerInfos.ProcessingVerInfos[taskRsp.ID] = pInfo
log.LogInfof("action[handleLcNodeHeartbeatResp], task (%v) processing status is resumed",
taskRsp.ID)
}
}
} else {
log.LogErrorf("action[handleLcNodeHeartbeatResp], snapshot scan vol(%v) not found", taskRsp.VolName)
}
}
c.snapshotMgr.volVerInfos.Unlock()
log.LogDebugf("action[handleLcNodeHeartbeatResp], SnapshotScanningTasks rsp(%v)", taskRsp)
}
} else {
log.LogDebugf("action[handleLcNodeHeartbeatResp], lcNode[%v] is idle for SnapshotScanningTasks", nodeAddr)
task := c.snapshotMgr.lcNodeStatus.releaseNode(nodeAddr)
c.snapshotMgr.volVerInfos.RemoveProcessingVerInfo(task.(string))
c.snapshotMgr.notifyIdleLcNode()
}
logMsg = fmt.Sprintf("action[handleLcNodeHeartbeatResp], lcNode:%v, ReportTime:%v success", lcNode.Addr, lcNode.ReportTime.Unix())
log.LogInfof(logMsg)
return
errHandler:
logMsg = fmt.Sprintf("nodeAddr %v heartbeat error :%v", nodeAddr, err.Error())
log.LogError(logMsg)
return
}
func (c *Cluster) handleLcNodeSnapshotScanResp(req *proto.SnapshotVerDelTaskRequest, resp *proto.SnapshotVerDelTaskResponse) (err error) {
c.snapshotMgr.lcNodeStatus.releaseNode(req.LcNodeAddr)
if resp.Status == proto.TaskFailed {
c.snapshotMgr.volVerInfos.RedoProcessingVerInfo(resp.ID)
} else {
//1.mark done for VersionMgr
var vol *Vol
vol, err = c.getVol(resp.VolName)
if err != nil {
log.LogErrorf("action[handleLcNodeSnapshotScanResp] snapshot task(%v) scanning completed by %v, results(%v), volume(%v) is not found",
resp.ID, req.LcNodeAddr, resp, resp.VolName)
return
} else {
_ = vol.VersionMgr.DelVer(resp.VerSeq)
}
//2. mark done for snapshotMgr
c.snapshotMgr.volVerInfos.RemoveProcessingVerInfo(resp.ID)
}
return
}

2
master/metadata_fsm.go
View File

@ -130,7 +130,7 @@ func (mf *MetadataFsm) Apply(command []byte, index uint64) (resp interface{}, er
switch cmd.Op {
case opSyncDeleteDataNode, opSyncDeleteMetaNode, opSyncDeleteVol, opSyncDeleteDataPartition, opSyncDeleteMetaPartition,
opSyncDeleteUserInfo, opSyncDeleteAKUser, opSyncDeleteVolUser, opSyncDeleteQuota, opSyncDeleteLcConf:
opSyncDeleteUserInfo, opSyncDeleteAKUser, opSyncDeleteVolUser, opSyncDeleteQuota, opSyncDeleteLcNode, opSyncDeleteLcConf:
if err = mf.delKeyAndPutIndex(cmd.K, cmdMap); err != nil {
panic(err)
}

43
master/metadata_fsm_op.go
View File

@ -442,18 +442,6 @@ func newNodeSetGrpValue(nset *nodeSetGroup) (nsv *domainNodeSetGrpValue) {
return
}
type lcNodeValue struct {
ID uint64
Addr string
}
func newLcNodeValue(lcNode *LcNode) *lcNodeValue {
return &lcNodeValue{
ID: lcNode.ID,
Addr: lcNode.Addr,
}
}
// RaftCmd defines the Raft commands.
type RaftCmd struct {
Op uint32 `json:"op"`
@ -1540,7 +1528,6 @@ func (c *Cluster) startDecommissionListTraverse() (err error) {
return
}
// key=#ln#id#Addr,value = json.Marshal(lnv)
func (c *Cluster) syncAddLcNode(ln *LcNode) (err error) {
return c.syncPutLcNodeInfo(opSyncAddLcNode, ln)
}
@ -1556,7 +1543,7 @@ func (c *Cluster) syncUpdateLcNode(ln *LcNode) (err error) {
func (c *Cluster) syncPutLcNodeInfo(opType uint32, ln *LcNode) (err error) {
metadata := new(RaftCmd)
metadata.Op = opType
metadata.K = lcNodePrefix + strconv.FormatUint(ln.ID, 10) + keySeparator + ln.Addr
metadata.K = lcNodePrefix + ln.Addr
lnv := newLcNodeValue(ln)
metadata.V, err = json.Marshal(lnv)
if err != nil {
@ -1565,35 +1552,40 @@ func (c *Cluster) syncPutLcNodeInfo(opType uint32, ln *LcNode) (err error) {
return c.submit(metadata)
}
type lcNodeValue struct {
ID uint64
Addr string
}
func newLcNodeValue(lcNode *LcNode) *lcNodeValue {
return &lcNodeValue{
ID: lcNode.ID,
Addr: lcNode.Addr,
}
}
func (c *Cluster) loadLcNodes() (err error) {
result, err := c.fsm.store.SeekForPrefix([]byte(lcNodePrefix))
if err != nil {
err = fmt.Errorf("action[loadLcNodes],err:%v", err.Error())
return err
}
log.LogInfof("action[loadLcNodes], result count %v", len(result))
for _, value := range result {
lnv := &lcNodeValue{}
if err = json.Unmarshal(value, lnv); err != nil {
err = fmt.Errorf("action[loadLcNodes],value:%v,unmarshal err:%v", string(value), err)
return
}
log.LogInfof("action[loadLcNodes], load lcNode[%v], lcNodeID[%v]", lnv.Addr, lnv.ID)
lcNode := newLcNode(lnv.Addr, c.Name)
lcNode.ID = lnv.ID
oldln, ok := c.lcMgr.lcNodes.Load(lcNode.Addr)
if ok {
if oldln.(*LcNode).ID <= lcNode.ID {
continue
}
}
c.lcMgr.lcNodes.Store(lcNode.Addr, lcNode)
log.LogInfof("action[loadLcNodes],lcNode[%v],lcNodeID[%v]", lcNode.Addr, lcNode.ID)
c.lcNodes.Store(lcNode.Addr, lcNode)
log.LogInfof("action[loadLcNodes], store lcNode[%v], lcNodeID[%v]", lcNode.Addr, lcNode.ID)
}
return
}
// key=#lcConf#volName,value = json.Marshal(lcConf)
func (c *Cluster) syncAddLcConf(lcConf *bsProto.LcConfiguration) (err error) {
return c.syncPutLcConfInfo(opSyncAddLcConf, lcConf)
}
@ -1630,7 +1622,6 @@ func (c *Cluster) loadLcConfs() (err error) {
err = fmt.Errorf("action[loadLcConfs],value:%v,unmarshal err:%v", string(value), err)
return
}
_ = c.lcMgr.SetS3BucketLifecycle(lcConf)
log.LogInfof("action[loadLcConfs],vol[%v]", lcConf.VolName)
}

102
master/monitor_metrics.go
View File

@ -76,13 +76,17 @@ const (
MetricNodesetMpReplicaCount = "nodeset_mp_replica_count"
MetricNodesetDpReplicaCount = "nodeset_dp_replica_count"
MetricLcNodesCount = "lcNodes_count"
MetricLcNodesConcurrentCount = "lcNodes_concurrent"
Metrics3LcTotalScanned = "s3Lc_Total_Scanned"
Metrics3LcTotalFileScanned = "s3Lc_Total_File_Scanned"
Metrics3LcTotalDirScanned = "s3Lc_Total_DirS_canned"
Metrics3LcTotalExpired = "s3Lc_Total_Expired"
Metrics3LcAbortedMultipartUpload = "s3Lc_Aborted_Multipart_Upload"
MetricLcNodesCount = "lc_nodes_count"
MetricLcTotalScanned = "lc_total_scanned"
MetricLcTotalFileScanned = "lc_total_file_scanned"
MetricLcTotalDirScanned = "lc_total_dirs_scanned"
MetricLcTotalExpired = "lc_total_expired"
)
var WarnMetrics *warningMetrics
@ -137,14 +141,12 @@ type monitorMetrics struct {
inconsistentMps map[string]string
nodesetIds map[uint64]string
lcNodesCount *exporter.Gauge
lcNodesConcurrentCount *exporter.Gauge
s3LcTotalScanned *exporter.GaugeVec
s3LcTotalFileScanned *exporter.GaugeVec
s3LcTotalDirScanned *exporter.GaugeVec
s3LcTotalExpired *exporter.GaugeVec
s3LcAbortedMultipartUpload *exporter.GaugeVec
s3VolNames map[string]struct{}
lcNodesCount *exporter.Gauge
lcVolNames map[string]struct{}
lcTotalScanned *exporter.GaugeVec
lcTotalFileScanned *exporter.GaugeVec
lcTotalDirScanned *exporter.GaugeVec
lcTotalExpired *exporter.GaugeVec
}
func newMonitorMetrics(c *Cluster) *monitorMetrics {
@ -154,7 +156,7 @@ func newMonitorMetrics(c *Cluster) *monitorMetrics {
nodesetInactiveDataNodesCount: make(map[uint64]int64),
nodesetInactiveMetaNodesCount: make(map[uint64]int64),
inconsistentMps: make(map[string]string),
s3VolNames: make(map[string]struct{}),
lcVolNames: make(map[string]struct{}),
}
}
@ -457,13 +459,12 @@ func (mm *monitorMetrics) start() {
mm.nodesetDataUsageRatio = exporter.NewGaugeVec(MetricNodesetDataUsageRadio, "", []string{"nodeset"})
mm.nodesetMpReplicaCount = exporter.NewGaugeVec(MetricNodesetMpReplicaCount, "", []string{"nodeset"})
mm.nodesetDpReplicaCount = exporter.NewGaugeVec(MetricNodesetDpReplicaCount, "", []string{"nodeset"})
mm.lcNodesConcurrentCount = exporter.NewGauge(MetricLcNodesConcurrentCount)
mm.s3LcTotalScanned = exporter.NewGaugeVec(Metrics3LcTotalScanned, "", []string{"volName", "type"})
mm.s3LcTotalFileScanned = exporter.NewGaugeVec(Metrics3LcTotalFileScanned, "", []string{"volName", "type"})
mm.s3LcTotalDirScanned = exporter.NewGaugeVec(Metrics3LcTotalDirScanned, "", []string{"volName", "type"})
mm.s3LcTotalExpired = exporter.NewGaugeVec(Metrics3LcTotalExpired, "", []string{"volName", "type"})
mm.s3LcAbortedMultipartUpload = exporter.NewGaugeVec(Metrics3LcAbortedMultipartUpload, "", []string{"volName", "type"})
mm.lcNodesCount = exporter.NewGauge(MetricLcNodesCount)
mm.lcTotalScanned = exporter.NewGaugeVec(MetricLcTotalScanned, "", []string{"volName", "type"})
mm.lcTotalFileScanned = exporter.NewGaugeVec(MetricLcTotalFileScanned, "", []string{"volName", "type"})
mm.lcTotalDirScanned = exporter.NewGaugeVec(MetricLcTotalDirScanned, "", []string{"volName", "type"})
mm.lcTotalExpired = exporter.NewGaugeVec(MetricLcTotalExpired, "", []string{"volName", "type"})
go mm.statMetrics()
}
@ -526,14 +527,12 @@ func (mm *monitorMetrics) doStat() {
mm.setNotWritableDataNodesCount()
mm.setNotWritableMetaNodesCount()
mm.setMpInconsistentErrorMetric()
mm.setInactiveDataNodesCountMetric()
mm.setInactiveMetaNodesCountMetric()
mm.setMpAndDpMetrics()
mm.setNodesetMetrics()
mm.setInactiveDataNodesCountMetric()
mm.setInactiveMetaNodesCountMetric()
mm.setIdleLcNodesCount()
mm.setS3LcMetrics()
mm.setLcMetrics()
}
func (mm *monitorMetrics) setMpAndDpMetrics() {
@ -754,10 +753,6 @@ func (mm *monitorMetrics) setDiskErrorMetric() {
}
}
func (mm *monitorMetrics) setIdleLcNodesCount() {
mm.lcNodesConcurrentCount.Set(float64(mm.cluster.cfg.MaxConcurrentLcNodes))
}
func (mm *monitorMetrics) setInactiveMetaNodesCountMetric() {
var inactiveMetaNodesCount int64
@ -875,46 +870,38 @@ func (mm *monitorMetrics) clearInconsistentMps() {
}
func (mm *monitorMetrics) deleteS3LcVolMetric(volName string) {
mm.s3LcTotalScanned.DeleteLabelValues(volName, "total")
mm.s3LcTotalFileScanned.DeleteLabelValues(volName, "file")
mm.s3LcTotalDirScanned.DeleteLabelValues(volName, "dir")
mm.s3LcTotalExpired.DeleteLabelValues(volName, "expired")
mm.s3LcAbortedMultipartUpload.DeleteLabelValues(volName, "aborted")
mm.lcTotalScanned.DeleteLabelValues(volName, "total")
mm.lcTotalFileScanned.DeleteLabelValues(volName, "file")
mm.lcTotalDirScanned.DeleteLabelValues(volName, "dir")
mm.lcTotalExpired.DeleteLabelValues(volName, "expired")
}
func (mm *monitorMetrics) setS3LcMetrics() {
lcScan := mm.cluster.lcMgr.getScanRoutine()
if lcScan != nil {
volumeScanStatistics := make(map[string]proto.S3TaskStatistics, 0)
lcScan.RLock()
for _, rst := range lcScan.RuleStatus.Results {
key := rst.Volume + "[" + rst.Prefix + "]"
if _, ok := volumeScanStatistics[key]; ok && rst.Done {
volumeScanStatistics[key] = proto.S3TaskStatistics{}
} else {
volumeScanStatistics[key] = rst.S3TaskStatistics
}
func (mm *monitorMetrics) setLcMetrics() {
lcTaskStatus := mm.cluster.lcMgr.lcRuleTaskStatus
volumeScanStatistics := make(map[string]proto.LcNodeRuleTaskStatistics, 0)
lcTaskStatus.RLock()
for _, r := range lcTaskStatus.Results {
key := r.Volume + "[" + r.RuleId + "]"
if _, ok := volumeScanStatistics[key]; ok && r.Done {
volumeScanStatistics[key] = proto.LcNodeRuleTaskStatistics{}
} else {
volumeScanStatistics[key] = r.LcNodeRuleTaskStatistics
}
lcScan.RUnlock()
for vol, stat := range volumeScanStatistics {
key := stat.Volume + "[" + stat.Prefix + "]"
mm.s3VolNames[key] = struct{}{}
mm.s3LcTotalScanned.SetWithLabelValues(float64(stat.TotalInodeScannedNum), vol, "total")
mm.s3LcTotalFileScanned.SetWithLabelValues(float64(stat.FileScannedNum), vol, "file")
mm.s3LcTotalDirScanned.SetWithLabelValues(float64(stat.DirScannedNum), vol, "dir")
mm.s3LcTotalExpired.SetWithLabelValues(float64(stat.ExpiredNum), vol, "expired")
mm.s3LcAbortedMultipartUpload.SetWithLabelValues(float64(stat.AbortedIncompleteMultipartNum), vol, "aborted")
}
}
lcTaskStatus.RUnlock()
for key, stat := range volumeScanStatistics {
mm.lcVolNames[key] = struct{}{}
mm.lcTotalScanned.SetWithLabelValues(float64(stat.TotalInodeScannedNum), key, "total")
mm.lcTotalFileScanned.SetWithLabelValues(float64(stat.FileScannedNum), key, "file")
mm.lcTotalDirScanned.SetWithLabelValues(float64(stat.DirScannedNum), key, "dir")
mm.lcTotalExpired.SetWithLabelValues(float64(stat.ExpiredNum), key, "expired")
}
}
func (mm *monitorMetrics) clearS3LcMetrics() {
for vol := range mm.s3VolNames {
func (mm *monitorMetrics) clearLcMetrics() {
for vol := range mm.lcVolNames {
mm.deleteS3LcVolMetric(vol)
delete(mm.s3VolNames, vol)
delete(mm.lcVolNames, vol)
}
}
@ -1026,7 +1013,7 @@ func (mm *monitorMetrics) resetAllLeaderMetrics() {
mm.clearInactiveDataNodesCountMetric()
mm.clearInconsistentMps()
mm.clearNodesetMetrics()
mm.clearS3LcMetrics()
mm.clearLcMetrics()
mm.dataNodesCount.Set(0)
mm.metaNodesCount.Set(0)
@ -1048,5 +1035,4 @@ func (mm *monitorMetrics) resetAllLeaderMetrics() {
mm.ReplicaMissingDPCount.Set(0)
mm.MpMissingLeaderCount.Set(0)
mm.DpMissingLeaderCount.Set(0)
mm.lcNodesConcurrentCount.Set(0)
}

2
master/multi_ver_snapshot.go
View File

@ -99,7 +99,7 @@ func (verMgr *VolVersionManager) loadMultiVersion(c *Cluster, val []byte) (err e
}
func (verMgr *VolVersionManager) CommitVer() (ver *proto.VolVersionInfo) {
log.LogDebugf("action[CommitVer] vol %v %v", verMgr.vol.Name, verMgr)
log.LogDebugf("action[CommitVer] op %v vol %v %v", verMgr.prepareCommit.op, verMgr.vol.Name, verMgr)
if verMgr.prepareCommit.op == proto.CreateVersionPrepare {
ver = verMgr.prepareCommit.prepareInfo
commitVer := &proto.VolVersionInfo{

2
master/operate_util.go
View File

@ -106,7 +106,7 @@ func unmarshalTaskResponse(task *proto.AdminTask) (err error) {
case proto.OpLcNodeHeartbeat:
response = &proto.LcNodeHeartbeatResponse{}
case proto.OpLcNodeScan:
response = &proto.RuleTaskResponse{}
response = &proto.LcNodeRuleTaskResponse{}
case proto.OpLcNodeSnapshotVerDel:
response = &proto.SnapshotVerDelTaskResponse{}

199
master/snapshot_manager.go Normal file
View File

@ -0,0 +1,199 @@
// Copyright 2023 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package master
import (
"math"
"sync"
"time"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/util/log"
)
type LcVerInfo struct {
proto.VerInfo
dTime time.Time
}
type ProcessingVerInfo struct {
LcVerInfo
updateTime time.Time
}
type VolVerInfos struct {
sync.RWMutex
VerInfos map[string]*LcVerInfo
ProcessingVerInfos map[string]*ProcessingVerInfo
TaskResults map[string]*proto.SnapshotVerDelTaskResponse
}
func (vvi *VolVerInfos) AddVerInfo(vi *LcVerInfo) {
vvi.Lock()
defer vvi.Unlock()
if pInfo, ok := vvi.ProcessingVerInfos[vi.Key()]; ok {
if time.Since(pInfo.updateTime) < time.Second*time.Duration(3*defaultIntervalToCheck) {
log.LogDebugf("VerInfo: %v is already in processing", vi)
return
} else {
log.LogWarnf("VerInfo: %v is expired", vi)
}
}
vvi.VerInfos[vi.Key()] = vi
log.LogDebugf("Adding VerInfo: %v", vi)
}
func (vvi *VolVerInfos) RemoveProcessingVerInfo(verInfoKey string) (pInfo *ProcessingVerInfo) {
vvi.Lock()
defer vvi.Unlock()
var ok bool
if pInfo, ok = vvi.ProcessingVerInfos[verInfoKey]; !ok {
return
} else {
delete(vvi.ProcessingVerInfos, verInfoKey)
}
return
}
func (vvi *VolVerInfos) RemoveVerInfo(verInfoKey string) {
vvi.Lock()
defer vvi.Unlock()
delete(vvi.VerInfos, verInfoKey)
delete(vvi.ProcessingVerInfos, verInfoKey)
}
func (vvi *VolVerInfos) FetchOldestVerInfo() (info *LcVerInfo) {
var min int64 = math.MaxInt64
minKey := ""
vvi.Lock()
defer vvi.Unlock()
if len(vvi.VerInfos) == 0 {
return nil
}
for _, i := range vvi.VerInfos {
if i.dTime.Unix() < min {
min = i.dTime.Unix()
minKey = i.Key()
}
}
info = vvi.VerInfos[minKey]
delete(vvi.VerInfos, minKey)
pInfo := &ProcessingVerInfo{
LcVerInfo: *info,
updateTime: time.Now(),
}
vvi.ProcessingVerInfos[minKey] = pInfo
return
}
func (vvi *VolVerInfos) RedoProcessingVerInfo(verInfoKey string) {
vvi.Lock()
defer vvi.Unlock()
log.LogInfof("RedoProcessingVerInfo, verinfo: %v", verInfoKey)
if pInfo, ok := vvi.ProcessingVerInfos[verInfoKey]; ok {
vvi.VerInfos[verInfoKey] = &pInfo.LcVerInfo
delete(vvi.ProcessingVerInfos, verInfoKey)
}
}
type snapshotDelManager struct {
cluster *Cluster
volVerInfos *VolVerInfos
lcNodeStatus *lcNodeStatus
idleNodeCh chan struct{}
exitCh chan struct{}
}
func newSnapshotManager() *snapshotDelManager {
log.LogInfof("action[newSnapshotManager] construct")
snapshotMgr := &snapshotDelManager{
volVerInfos: &VolVerInfos{
VerInfos: make(map[string]*LcVerInfo, 0),
ProcessingVerInfos: make(map[string]*ProcessingVerInfo, 0),
TaskResults: make(map[string]*proto.SnapshotVerDelTaskResponse, 0),
},
lcNodeStatus: &lcNodeStatus{
workingNodes: make(map[string]interface{}),
idleNodes: make(map[string]string),
},
idleNodeCh: make(chan struct{}),
exitCh: make(chan struct{}),
}
return snapshotMgr
}
func (m *snapshotDelManager) Start() {
go m.process()
}
func (m *snapshotDelManager) process() {
for {
select {
case <-m.exitCh:
log.LogInfo("snapshotDelManager process exit")
return
case <-m.idleNodeCh:
vi := m.volVerInfos.FetchOldestVerInfo()
if vi == nil {
log.LogDebugf("no snapshot ver del task available")
continue
}
nodeAddr := m.lcNodeStatus.getOneIdleNode(vi.Key())
if nodeAddr == "" {
log.LogWarn("no idle lcnode")
continue
}
val, ok := m.cluster.lcNodes.Load(nodeAddr)
if !ok {
log.LogErrorf("nodeAddr(%v) is not available for scanning!", nodeAddr)
t := m.lcNodeStatus.removeNode(nodeAddr)
m.volVerInfos.RedoProcessingVerInfo(t.(string))
continue
}
task := &proto.SnapshotVerDelTask{
VerInfo: proto.VerInfo{
VolName: vi.VolName,
VerSeq: vi.VerSeq,
},
}
node := val.(*LcNode)
adminTask := node.createSnapshotVerDelTask(m.cluster.masterAddr(), task)
m.cluster.addLcNodeTasks([]*proto.AdminTask{adminTask})
log.LogDebugf("add snapshot version del task(%v) to lcnode(%v)", *task, nodeAddr)
}
}
}
func (m *snapshotDelManager) notifyIdleLcNode() {
m.volVerInfos.RLock()
defer m.volVerInfos.RUnlock()
if len(m.volVerInfos.VerInfos) > 0 {
select {
case m.idleNodeCh <- struct{}{}:
log.LogDebug("action[handleLcNodeHeartbeatResp], snapshotDelManager scan routine notified!")
default:
log.LogDebug("action[handleLcNodeHeartbeatResp], snapshotDelManager skipping notify!")
}
}
}

2
metanode/const.go
View File

@ -92,8 +92,6 @@ type (
// Client -> MetaNode
GetUniqIDResp = proto.GetUniqIDResponse
// Client -> MetaNode
InodeGetExpirationReqBatch = proto.BatchInodeGetExpirationRequest
)
// op code should be fixed, order change will cause raft fsm log apply fail

2
metanode/manager.go
View File

@ -283,8 +283,6 @@ func (m *metadataManager) HandleMetadataOperation(conn net.Conn, p *Packet, remo
// multi version
case proto.OpVersionOperation:
err = m.opMultiVersionOp(conn, p, remoteAddr)
case proto.OpMetaBatchInodeExpirationGet:
err = m.opMetaBatchInodeExpirationGet(conn, p, remoteAddr)
case proto.OpGetExpiredMultipart:
err = m.opGetExpiredMultipart(conn, p, remoteAddr)
default:

26
metanode/manager_op.go
View File

@ -1574,32 +1574,6 @@ func (m *metadataManager) opRemoveMetaPartitionRaftMember(conn net.Conn,
return
}
func (m *metadataManager) opMetaBatchInodeExpirationGet(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
req := &proto.BatchInodeGetExpirationRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
m.respondToClient(conn, p)
err = errors.NewErrorf("[%v] req: %v, resp: %v", p.GetOpMsgWithReqAndResult(), req, err.Error())
return
}
mp, err := m.getPartition(req.PartitionID)
if err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
m.respondToClient(conn, p)
err = errors.NewErrorf("[%v] req: %v, resp: %v", p.GetOpMsgWithReqAndResult(), req, err.Error())
return
}
if !m.serveProxy(conn, mp, p) {
return
}
err = mp.InodeExpirationGetBatch(req, p)
m.respondToClient(conn, p)
log.LogDebugf("%s [opMetaBatchInodeGet] req: %d - %v, resp: %v, "+
"body: %s", remoteAddr, p.GetReqID(), req, p.GetResultMsg(), p.Data)
return
}
func (m *metadataManager) opMetaBatchInodeGet(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
req := &proto.BatchInodeGetRequest{}

1
metanode/partition.go
View File

@ -142,7 +142,6 @@ type OpInode interface {
TxUnlinkInode(req *proto.TxUnlinkInodeRequest, p *Packet) (err error)
TxCreateInodeLink(req *proto.TxLinkInodeRequest, p *Packet) (err error)
QuotaCreateInode(req *proto.QuotaCreateInodeRequest, p *Packet) (err error)
InodeExpirationGetBatch(req *InodeGetExpirationReqBatch, p *Packet) (err error)
}
type OpExtend interface {

2
metanode/partition_op_extent.go
View File

@ -129,7 +129,7 @@ func (mp *metaPartition) ExtentAppendWithCheck(req *proto.AppendExtentKeyWithChe
// extent key verSeq not set value since marshal will not include verseq
// use inode verSeq instead
inoParm.verSeq = mp.verSeq
inoParm.setVer(mp.verSeq)
inoParm.Extents.Append(ext)
log.LogDebugf("ExtentAppendWithCheck: ino(%v) mp(%v) verSeq (%v)", req.Inode, req.PartitionID, mp.verSeq)

109
metanode/partition_op_inode.go
View File

@ -19,6 +19,7 @@ import (
"encoding/json"
"fmt"
"net/url"
"strings"
"time"
"github.com/cubefs/cubefs/proto"
@ -323,7 +324,7 @@ func (mp *metaPartition) UnlinkInode(req *UnlinkInoReq, p *Packet) (err error) {
} else {
ino := NewInode(req.Inode, 0)
ino.setVer(req.VerSeq)
log.LogDebugf("action[UnlinkInode] verseq %v ino %v", ino.verSeq, ino)
log.LogDebugf("action[UnlinkInode] verseq %v ino %v", req.VerSeq, ino)
item := mp.inodeTree.Get(ino)
if item == nil {
err = fmt.Errorf("inode %v reqeust cann't found", ino)
@ -342,11 +343,6 @@ func (mp *metaPartition) UnlinkInode(req *UnlinkInoReq, p *Packet) (err error) {
r, err = mp.submit(opFSMUnlinkInode, val)
}
log.LogDebugf("action[UnlinkInode] ino %v submit", ino)
if req.DenVerSeq == item.(*Inode).getVer() {
ino.Flag |= InodeDelTop
}
if err != nil {
p.PacketErrorWithBody(proto.OpAgain, []byte(err.Error()))
return
@ -523,107 +519,6 @@ func (mp *metaPartition) InodeGet(req *InodeGetReq, p *Packet) (err error) {
return
}
func (mp *metaPartition) inodeExpired(inode *Inode, cond *proto.InodeExpireCondition) bool {
if inode == nil || cond == nil {
return false
}
now := Now.GetCurrentTime().Unix()
if cond.ExpirationInfo.Days > 0 {
if now-inode.ModifyTime < int64(cond.ExpirationInfo.Days*24*60*60) {
return false
}
}
if cond.ExpirationInfo.Date != nil {
if now < cond.ExpirationInfo.Date.Unix() {
return false
}
}
if cond.ObjectSizeGreaterThan > 0 {
if inode.Size <= uint64(cond.ObjectSizeGreaterThan) {
return false
}
}
if cond.ObjectSizeLessThan > 0 {
if inode.Size >= uint64(cond.ObjectSizeLessThan) {
return false
}
}
if len(cond.Tags) > 0 {
treeItem := mp.extendTree.Get(NewExtend(inode.Inode))
if treeItem != nil {
extend := treeItem.(*Extend)
if val, exist := extend.Get([]byte("oss:tagging")); exist {
tagParser := func(src string) map[string]string {
values, err := url.ParseQuery(src)
if err != nil {
return nil
}
tagMap := make(map[string]string, 0)
for key, value := range values {
tagMap[key] = value[0]
}
if len(tagMap) == 0 {
return nil
}
return tagMap
}
tags := tagParser(string(val))
for _, tag := range cond.Tags {
if tagVal, exist := tags[tag.Key]; exist {
if tagVal != tag.Value {
return false
}
} else {
return false
}
}
} else {
return false
}
}
}
return true
}
func (mp *metaPartition) InodeExpirationGetBatch(req *InodeGetExpirationReqBatch, p *Packet) (err error) {
resp := &proto.BatchInodeGetExpirationResponse{
ExpirationResults: make([]*proto.ExpireInfo, 0),
}
ino := NewInode(0, 0)
for _, dentry := range req.Dentries {
ino.Inode = dentry.Inode
retMsg := mp.getInode(ino, false)
expireInfo := &proto.ExpireInfo{
Dentry: dentry,
Expired: false,
}
if retMsg.Status == proto.OpOk {
if mp.inodeExpired(retMsg.Msg, req.Cond) {
expireInfo.Expired = true
}
}
resp.ExpirationResults = append(resp.ExpirationResults, expireInfo)
}
data, err := json.Marshal(resp)
if err != nil {
p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
return
}
p.PacketOkWithBody(data)
return
}
// InodeGetBatch executes the inodeBatchGet command from the client.
func (mp *metaPartition) InodeGetBatch(req *InodeGetReqBatch, p *Packet) (err error) {

137
objectnode/lifecycle.go
View File

@ -1,4 +1,4 @@
// Copyright 2019 The ChubaoFS Authors.
// Copyright 2023 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -16,19 +16,31 @@ package objectnode
import (
"encoding/xml"
"github.com/cubefs/cubefs/util/log"
"net/http"
"time"
)
type ExpirationStatus string
const (
ExpirationStatusEnabled ExpirationStatus = "Enabled"
ExpirationStatusDisabled ExpirationStatus = "Disabled"
ExpirationStatusEnabled = "Enabled"
ExpirationStatusDisabled = "Disabled"
)
const (
RuleMaxCounts = 1000
MaxIdLength = 255
)
var (
LifeCycleErrTooManyRules = &ErrorCode{ErrorCode: "InvalidArgument", ErrorMessage: "Rules Number should not exceed allowed limit of 1000.", StatusCode: http.StatusBadRequest}
LifeCycleErrMissingRules = &ErrorCode{ErrorCode: "InvalidArgument", ErrorMessage: "No Lifecycle Rules found in request.", StatusCode: http.StatusBadRequest}
LifeCycleErrMissingActions = &ErrorCode{ErrorCode: "InvalidArgument", ErrorMessage: "At least one action needs to be specified in a rule.", StatusCode: http.StatusBadRequest}
LifeCycleErrMissingRuleID = &ErrorCode{ErrorCode: "InvalidArgument", ErrorMessage: "No Lifecycle Rule ID in request.", StatusCode: http.StatusBadRequest}
LifeCycleErrTooLongRuleID = &ErrorCode{ErrorCode: "InvalidArgument", ErrorMessage: "ID length should not exceed allowed limit of 255.", StatusCode: http.StatusBadRequest}
LifeCycleErrSameRuleID = &ErrorCode{ErrorCode: "InvalidArgument", ErrorMessage: "Rule ID must be unique. Found same ID for more than one rule.", StatusCode: http.StatusBadRequest}
LifeCycleErrDateType = &ErrorCode{ErrorCode: "InvalidArgument", ErrorMessage: "'Date' must be at midnight GMT.", StatusCode: http.StatusBadRequest}
LifeCycleErrDaysType = &ErrorCode{ErrorCode: "InvalidArgument", ErrorMessage: "'Days' for Expiration action must be a positive integer.", StatusCode: http.StatusBadRequest}
LifeCycleErrMalformedXML = &ErrorCode{ErrorCode: "MalformedXML", ErrorMessage: "The XML you provided was not well-formed or did not validate against our published schema.", StatusCode: http.StatusBadRequest}
NoSuchLifecycleConfiguration = &ErrorCode{ErrorCode: "NoSuchLifecycleConfiguration", ErrorMessage: "The lifecycle configuration does not exist.", StatusCode: http.StatusNotFound}
)
type LifeCycle struct {
@ -38,47 +50,22 @@ type LifeCycle struct {
}
type Rule struct {
XMLName xml.Name `xml:"Rule"`
AbortMultipartUpload *AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty"`
Expire *Expiration `xml:"Expiration"`
Filter *FilterConfig `xml:"Filter"`
ID string `xml:"ID"`
Prefix string `xml:"Prefix,omitempty"`
Status ExpirationStatus `xml:"Status"`
}
type AbortIncompleteMultipartUpload struct {
XMLName xml.Name `xml:"AbortIncompleteMultipartUpload"`
DaysAfterInitiation int `xml:"DaysAfterInitiation"`
XMLName xml.Name `xml:"Rule"`
Expire *Expiration `xml:"Expiration"`
Filter *Filter `xml:"Filter"`
ID string `xml:"ID"`
Status string `xml:"Status"`
}
type Expiration struct {
XMLName xml.Name `xml:"Expiration"`
Date *time.Time `xml:"Date,omitempty"`
Days int `xml:"Days,omitempty"`
Days *int `xml:"Days,omitempty"`
}
type FilterConfig struct {
XMLName xml.Name `xml:"Filter"`
And *AndOpr `xml:"And,omitempty"`
ObjectSizeGreaterThan int64 `xml:"ObjectSizeGreaterThan,omitempty"`
ObjectSizeLessThan int64 `xml:"ObjectSizeLessThan,omitempty"`
Prefix string `xml:"Prefix,omitempty"`
Tag *TagConfig `xml:"Tag,omitempty"`
}
type AndOpr struct {
XMLName xml.Name `xml:"And"`
ObjectSizeGreaterThan int64 `xml:"ObjectSizeGreaterThan,omitempty"`
ObjectSizeLessThan int64 `xml:"ObjectSizeLessThan,omitempty"`
Prefix string `xml:"Prefix,omitempty"`
Tag []*TagConfig `xml:"Tag,omitempty"`
}
type TagConfig struct {
XMLName xml.Name `xml:"Tag"`
Key string `xml:"Key"`
Value string `xml:"Value"`
type Filter struct {
XMLName xml.Name `xml:"Filter"`
Prefix string `xml:"Prefix,omitempty"`
}
func NewLifeCycle() *LifeCycle {
@ -88,26 +75,72 @@ func NewLifeCycle() *LifeCycle {
}
func (l *LifeCycle) Validate() (bool, *ErrorCode) {
var errorCode *ErrorCode
if len(l.Rules) > RuleMaxCounts {
return false, LifeCycleRulesGreaterThen1K
return false, LifeCycleErrTooManyRules
}
if len(l.Rules) <= 0 {
return false, LifeCycleRulesLessThenOne
return false, LifeCycleErrMissingRules
}
ruleIds := make(map[string]int)
isRuleIdExist := make(map[string]bool)
for _, rule := range l.Rules {
if _, ok := ruleIds[rule.ID]; ok {
return false, LifeCycleRulesInvalid
_, ok := isRuleIdExist[rule.ID]
if !ok {
isRuleIdExist[rule.ID] = true
} else {
ruleIds[rule.ID] = 1
return false, LifeCycleErrSameRuleID
}
log.LogDebugf("Validate: rule : (%v)", rule)
if rule.Expire == nil {
return false, LifeCycleAtLeastOneAction
if err := rule.valid(); err != nil {
return false, err
}
}
return true, errorCode
return true, nil
}
func (r *Rule) valid() *ErrorCode {
if len(r.ID) == 0 {
return LifeCycleErrMissingRuleID
}
if len(r.ID) > MaxIdLength {
return LifeCycleErrTooLongRuleID
}
if r.Status != ExpirationStatusEnabled && r.Status != ExpirationStatusDisabled {
return LifeCycleErrMalformedXML
}
if r.Expire == nil {
return LifeCycleErrMissingActions
}
if err := r.Expire.validExpiration(); err != nil {
return err
}
return nil
}
func (e *Expiration) validExpiration() *ErrorCode {
// Date and Days cannot be set at the same time
if e.Date != nil && e.Days != nil {
return LifeCycleErrMalformedXML
}
// Date and Days cannot both be nil
if e.Date == nil && e.Days == nil {
return LifeCycleErrMalformedXML
}
// Date must be midnight UTC
if e.Date != nil {
date := e.Date.In(time.UTC)
if !(date.Hour() == 0 && date.Minute() == 0 && date.Second() == 0 && date.Nanosecond() == 0) {
return LifeCycleErrDateType
}
} else if e.Days != nil {
// Days must be greater than 0
if *e.Days <= 0 {
return LifeCycleErrDaysType
}
}
return nil
}

159
objectnode/lifecycle_handler.go
View File

@ -1,4 +1,4 @@
// Copyright 2019 The ChubaoFS Authors.
// Copyright 2023 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -16,22 +16,21 @@ package objectnode
import (
"encoding/xml"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/util/log"
"io"
"io/ioutil"
"net/http"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/util/log"
)
//Get bucket lifecycle configuration
// API reference: https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/API/API_GetBucketLifecycleConfiguration.html
func (o *ObjectNode) getBucketLifecycleConfigurationHandler(w http.ResponseWriter, r *http.Request) {
var err error
var errorCode *ErrorCode
defer func() {
if errorCode != nil {
_ = errorCode.ServeResponse(w, r)
}
o.errorResponse(w, r, err, errorCode)
}()
var param = ParseRequestParam(r)
@ -46,7 +45,7 @@ func (o *ObjectNode) getBucketLifecycleConfigurationHandler(w http.ResponseWrite
var lcConf *proto.LcConfiguration
if lcConf, err = o.mc.AdminAPI().GetBucketLifecycle(param.Bucket()); err != nil {
log.LogErrorf("getBucketLifecycleConfigurationHandler failed: bucket[%v] err(%v)", param.Bucket(), err)
log.LogErrorf("getBucketLifecycle failed: bucket[%v] err(%v)", param.Bucket(), err)
errorCode = NoSuchLifecycleConfiguration
return
}
@ -56,56 +55,29 @@ func (o *ObjectNode) getBucketLifecycleConfigurationHandler(w http.ResponseWrite
for _, lc := range lcConf.Rules {
rule := &Rule{
ID: lc.ID,
Prefix: lc.Prefix,
Status: ExpirationStatus(lc.Status),
}
if lc.AbortMultipartUpload != nil {
rule.AbortMultipartUpload = &AbortIncompleteMultipartUpload{
DaysAfterInitiation: lc.AbortMultipartUpload.DaysAfterInitiation,
}
Status: lc.Status,
}
if lc.Expire != nil {
rule.Expire = &Expiration{
Date: lc.Expire.Date,
Days: lc.Expire.Days,
rule.Expire = &Expiration{}
if lc.Expire.Date != nil {
rule.Expire.Date = lc.Expire.Date
}
if lc.Expire.Days != 0 {
rule.Expire.Days = &lc.Expire.Days
}
}
if lc.Filter != nil {
rule.Filter = &FilterConfig{
ObjectSizeGreaterThan: lc.Filter.ObjectSizeGreaterThan,
ObjectSizeLessThan: lc.Filter.ObjectSizeLessThan,
Prefix: lc.Filter.Prefix,
}
if lc.Filter.And != nil {
rule.Filter.And = &AndOpr{
ObjectSizeGreaterThan: lc.Filter.And.ObjectSizeGreaterThan,
ObjectSizeLessThan: lc.Filter.And.ObjectSizeLessThan,
Prefix: lc.Filter.And.Prefix,
Tag: make([]*TagConfig, 0),
}
for _, at := range lc.Filter.And.Tags {
tag := &TagConfig{
Key: at.Key,
Value: at.Value,
}
rule.Filter.And.Tag = append(rule.Filter.And.Tag, tag)
}
}
if lc.Filter.Tag != nil {
rule.Filter.Tag = &TagConfig{
Key: lc.Filter.Tag.Key,
Value: lc.Filter.Tag.Value,
}
rule.Filter = &Filter{
Prefix: lc.Filter.Prefix,
}
}
lifeCycle.Rules = append(lifeCycle.Rules, rule)
}
var data []byte
data, err = xml.Marshal(lifeCycle)
if err != nil {
log.LogErrorf("getBucketLifecycleConfigurationHandler failed: bucket[%v] err(%v)", param.Bucket(), err)
log.LogErrorf("getBucketLifecycle failed: bucket[%v] err(%v)", param.Bucket(), err)
errorCode = NoSuchLifecycleConfiguration
return
}
@ -115,16 +87,13 @@ func (o *ObjectNode) getBucketLifecycleConfigurationHandler(w http.ResponseWrite
}
//Put bucket lifecycle configuration
// API reference: https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html
func (o *ObjectNode) putBucketLifecycleConfigurationHandler(w http.ResponseWriter, r *http.Request) {
var err error
var errorCode *ErrorCode
defer func() {
if errorCode != nil {
_ = errorCode.ServeResponse(w, r)
}
o.errorResponse(w, r, err, errorCode)
}()
var param = ParseRequestParam(r)
@ -138,109 +107,73 @@ func (o *ObjectNode) putBucketLifecycleConfigurationHandler(w http.ResponseWrite
}
var requestBody []byte
if requestBody, err = ioutil.ReadAll(r.Body); err != nil {
log.LogErrorf("putBucketLifecycleConfigurationHandler: read request body data fail: requestID(%v) err(%v)", GetRequestID(r), err)
_ = InvalidArgument.ServeResponse(w, r)
if requestBody, err = ioutil.ReadAll(r.Body); err != nil && err != io.EOF {
log.LogErrorf("putBucketLifecycle failed: read request body data err: requestID(%v) err(%v)", GetRequestID(r), err)
errorCode = &ErrorCode{
ErrorCode: http.StatusText(http.StatusBadRequest),
ErrorMessage: err.Error(),
StatusCode: http.StatusBadRequest,
}
return
}
var lifeCycle = NewLifeCycle()
if err = UnmarshalXMLEntity(requestBody, lifeCycle); err != nil {
log.LogWarnf("putBucketLifecycleConfigurationHandler: decode request body fail: requestID(%v) err(%v)", GetRequestID(r), err)
_ = InvalidArgument.ServeResponse(w, r)
log.LogWarnf("putBucketLifecycle failed: decode request body err: requestID(%v) err(%v)", GetRequestID(r), err)
errorCode = LifeCycleErrMalformedXML
return
}
validateRes, errorCode := lifeCycle.Validate()
if !validateRes {
log.LogErrorf("putBucketLifecycleConfigurationHandler: lifeCycle validate fail: requestID(%v) lifeCycle(%v) err(%v)", GetRequestID(r), lifeCycle, err)
_ = errorCode.ServeResponse(w, r)
ok, errorCode := lifeCycle.Validate()
if !ok {
log.LogErrorf("putBucketLifecycle failed: validate err: requestID(%v) lifeCycle(%v) err(%v)", GetRequestID(r), lifeCycle, errorCode)
return
}
req := proto.SetBucketLifecycleRequest{
req := proto.LcConfiguration{
VolName: param.Bucket(),
Rules: make([]*proto.Rule, 0),
}
for _, lr := range lifeCycle.Rules {
rule := &proto.Rule{
ID: lr.ID,
Prefix: lr.Prefix,
Status: string(lr.Status),
Status: lr.Status,
}
if lr.AbortMultipartUpload != nil {
rule.AbortMultipartUpload = &proto.AbortIncompleteMultipartUpload{
DaysAfterInitiation: lr.AbortMultipartUpload.DaysAfterInitiation,
}
}
if lr.Expire != nil {
rule.Expire = &proto.Expiration{
Date: lr.Expire.Date,
Days: lr.Expire.Days,
rule.Expire = &proto.ExpirationConfig{}
if lr.Expire.Date != nil {
rule.Expire.Date = lr.Expire.Date
}
if lr.Expire.Days != nil {
rule.Expire.Days = *lr.Expire.Days
}
}
if lr.Filter != nil {
rule.Filter = &proto.FilterConfig{
ObjectSizeGreaterThan: lr.Filter.ObjectSizeGreaterThan,
ObjectSizeLessThan: lr.Filter.ObjectSizeLessThan,
Prefix: lr.Filter.Prefix,
Prefix: lr.Filter.Prefix,
}
if lr.Filter.And != nil {
rule.Filter.And = &proto.AndOpr{
ObjectSizeGreaterThan: lr.Filter.And.ObjectSizeGreaterThan,
ObjectSizeLessThan: lr.Filter.And.ObjectSizeLessThan,
Prefix: lr.Filter.And.Prefix,
Tags: make([]*proto.TagConfig, 0),
}
for _, at := range lr.Filter.And.Tag {
tag := &proto.TagConfig{
Key: at.Key,
Value: at.Value,
}
rule.Filter.And.Tags = append(rule.Filter.And.Tags, tag)
}
}
if lr.Filter.Tag != nil {
rule.Filter.Tag = &proto.TagConfig{
Key: lr.Filter.Tag.Key,
Value: lr.Filter.Tag.Value,
}
}
}
if valid, err := rule.Validate(); !valid {
log.LogErrorf("putBucketLifecycleConfigurationHandler: rule validate fail: requestID(%v) rule(%v) err(%v)", GetRequestID(r), rule, err)
errorCode = LifeCycleRulesInvalid
_ = errorCode.ServeResponse(w, r)
return
}
req.Rules = append(req.Rules, rule)
}
if err = o.mc.AdminAPI().SetBucketLifecycle(&req); err != nil {
log.LogErrorf("putBucketLifecycleConfigurationHandler failed: bucket[%v] err(%v)", param.Bucket(), err)
_ = InternalErrorCode(err).ServeResponse(w, r)
log.LogErrorf("putBucketLifecycle failed: SetBucketLifecycle err: bucket[%v] err(%v)", param.Bucket(), err)
errorCode = InternalErrorCode(err)
return
}
log.LogInfof("putBucketLifecycle success: requestID(%v) volume(%v) lifeCycle(%v)",
GetRequestID(r), param.Bucket(), lifeCycle)
}
//Delete Bucket Lifecycle
// API reference: https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/API/API_DeleteBucketLifecycle.html
func (o *ObjectNode) deleteBucketLifecycleConfigurationHandler(w http.ResponseWriter, r *http.Request) {
var err error
var errorCode *ErrorCode
defer func() {
if errorCode != nil {
_ = errorCode.ServeResponse(w, r)
}
o.errorResponse(w, r, err, errorCode)
}()
var param = ParseRequestParam(r)
@ -254,8 +187,8 @@ func (o *ObjectNode) deleteBucketLifecycleConfigurationHandler(w http.ResponseWr
}
if err = o.mc.AdminAPI().DelBucketLifecycle(param.Bucket()); err != nil {
log.LogErrorf("deleteBucketLifecycleConfigurationHandler failed: bucket[%v] err(%v)", param.Bucket(), err)
_ = InternalErrorCode(err).ServeResponse(w, r)
log.LogErrorf("deleteBucketLifecycle failed: bucket[%v] err(%v)", param.Bucket(), err)
errorCode = InternalErrorCode(err)
return
}
w.WriteHeader(http.StatusNoContent)

126
objectnode/lifecycle_test.go Normal file
View File

@ -0,0 +1,126 @@
// Copyright 2023 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package objectnode
import (
"encoding/xml"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestLifecycleConfiguration(t *testing.T) {
LifecycleXml := `
<LifecycleConfiguration>
<Rule>
<Filter>
<Prefix>logs/</Prefix>
</Filter>
<ID>id1</ID>
<Status>Enabled</Status>
<Expiration>
<Days>365</Days>
</Expiration>
</Rule>
<Rule>
<Filter>
<Prefix>logs/</Prefix>
</Filter>
<ID>id1</ID>
<Status>Enabled</Status>
<Expiration>
<Days>365</Days>
</Expiration>
</Rule>
</LifecycleConfiguration>
`
var l1 = NewLifeCycle()
err := xml.Unmarshal([]byte(LifecycleXml), l1)
require.NoError(t, err)
//same id
_, err = l1.Validate()
require.Equal(t, err, LifeCycleErrSameRuleID)
//id = ""
l1.Rules[0].ID = ""
_, err = l1.Validate()
require.Equal(t, err, LifeCycleErrMissingRuleID)
//len(id) > 255
var id string
for i := 0; i < 256; i++ {
id += "a"
}
l1.Rules[0].ID = id
_, err = l1.Validate()
require.Equal(t, err, LifeCycleErrTooLongRuleID)
l1.Rules[0].ID = "id"
//invalid status
l1.Rules[0].Status = ""
_, err = l1.Validate()
require.Equal(t, err, LifeCycleErrMalformedXML)
l1.Rules[0].Status = "Enabled"
//days < 0
day := -1
l1.Rules[0].Expire.Days = &day
_, err = l1.Validate()
require.Equal(t, err, LifeCycleErrDaysType)
day = 0
l1.Rules[0].Expire.Days = &day
//date
l1.Rules[0].Expire.Days = nil
now := time.Now().In(time.UTC)
ti := time.Date(now.Year(), now.Month(), now.Day(), 1, 0, 0, 0, time.UTC)
l1.Rules[0].Expire.Date = &ti
_, err = l1.Validate()
require.Equal(t, err, LifeCycleErrDateType)
//days and date all nil
l1.Rules[0].Expire.Days = nil
l1.Rules[0].Expire.Date = nil
_, err = l1.Validate()
require.Equal(t, err, LifeCycleErrMalformedXML)
//days and date
day = 1
l1.Rules[0].Expire.Days = &day
ti = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
l1.Rules[0].Expire.Date = &ti
_, err = l1.Validate()
require.Equal(t, err, LifeCycleErrMalformedXML)
l1.Rules[0].Expire.Date = nil
day = 1
l1.Rules[0].Expire.Days = &day
l1.Rules[1].Expire = nil
_, err = l1.Validate()
require.Equal(t, err, LifeCycleErrMissingActions)
//no err
l1.Rules = l1.Rules[:1]
ok, _ := l1.Validate()
require.Equal(t, true, ok)
l1.Rules = l1.Rules[:0]
_, err = l1.Validate()
require.Equal(t, err, LifeCycleErrMissingRules)
}

1
objectnode/result_error.go
View File

@ -136,7 +136,6 @@ var (
LifeCycleRulesLessThenOne = &ErrorCode{ErrorCode: "InvalidRequest", ErrorMessage: "At least one lifecycle rule must be specified.", StatusCode: http.StatusBadRequest}
LifeCycleAtLeastOneAction = &ErrorCode{ErrorCode: "InvalidRequest", ErrorMessage: "At least one action must be specified in a lifecycle rule.", StatusCode: http.StatusBadRequest}
LifeCycleRulesInvalid = &ErrorCode{ErrorCode: "InvalidRequest", ErrorMessage: "Lifecycle rule is invalid.", StatusCode: http.StatusBadRequest}
NoSuchLifecycleConfiguration = &ErrorCode{ErrorCode: "NoSuchLifecycleConfiguration", ErrorMessage: "The lifecycle configuration does not exist.", StatusCode: http.StatusNotFound}
)
func HttpStatusErrorCode(code int) *ErrorCode {

41
proto/admin_proto.go
View File

@ -83,6 +83,9 @@ const (
AdminSetConLcNodeNum = "/admin/setConLcNodeNum"
AdminGetAllLcNodeInfo = "/admin/getAllLcNodeInfo"
AdminLcNode = "/admin/lcnode"
//graphql master api
AdminClusterAPI = "/api/cluster"
AdminUserAPI = "/api/user"
@ -624,46 +627,12 @@ type MetaNodeHeartbeatResponse struct {
CpuUtil float64 `json:"cpuUtil"`
}
type S3ScanTaskInfo struct {
Id string
RoutineId int64
}
type S3ScanInfo struct {
S3ScanTaskInfo
S3TaskStatistics
}
type DelVerTaskInfo struct {
Id string
}
type SnapshotScanInfo struct {
DelVerTaskInfo
SnapshotStatistics
}
// LcNodeHeartbeatResponse defines the response to the lc node heartbeat.
type LcNodeHeartbeatResponse struct {
S3ScanningTasks map[string]*S3ScanInfo
SnapshotScanningTasks map[string]*SnapshotScanInfo
Status uint8
Result string
}
func (r *LcNodeHeartbeatResponse) IsLcNodeIdle() bool {
return len(r.S3ScanningTasks) == 0 && len(r.SnapshotScanningTasks) == 0
}
type LcNodeStatInfo struct {
Addr string
Busy bool
}
type LcNodeInfoResponse struct {
MaxConcurrentLcNodeNum int
RunningLcNodeNum int
Infos []*LcNodeStatInfo
LcScanningTasks map[string]*LcNodeRuleTaskResponse
SnapshotScanningTasks map[string]*SnapshotVerDelTaskResponse
}
// DeleteFileRequest defines the request to delete a file.

3
proto/fs_proto.go
View File

@ -503,7 +503,8 @@ type DeleteDentryResponse struct {
// BatchDeleteDentryResponse defines the response to the request of deleting a dentry.
type BatchDeleteDentryResponse struct {
Items []*struct {
ParentID uint64 `json:"pino"`
Items []*struct {
Inode uint64 `json:"ino"`
Status uint8 `json:"status"`
} `json:"items"`

369
proto/lifecycle.go
View File

@ -1,4 +1,4 @@
// Copyright 2018 The Chubao Authors.
// Copyright 2023 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -15,41 +15,39 @@
package proto
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/cubefs/cubefs/util/errors"
"github.com/cubefs/cubefs/util/log"
"sync"
"time"
"github.com/cubefs/cubefs/util/log"
)
//master
type LcConfiguration struct {
VolName string
Rules []*Rule
}
func (lcConf *LcConfiguration) GenRuleTasks() []*RuleTask {
tasks := make([]*RuleTask, 0)
for _, r := range lcConf.Rules {
var err error
var encoded []byte
if encoded, err = json.Marshal(r); err != nil {
continue
}
var idSlice = []byte(lcConf.VolName)
idSlice = append(idSlice, encoded...)
task := &RuleTask{
Id: fmt.Sprintf("%x", md5.Sum(idSlice)),
VolName: lcConf.VolName,
Rule: r,
}
tasks = append(tasks, task)
}
return tasks
type Rule struct {
Expire *ExpirationConfig
Filter *FilterConfig
ID string
Status string
}
type ExpirationConfig struct {
Date *time.Time
Days int
}
type FilterConfig struct {
Prefix string
}
const (
RuleEnabled string = "Enabled"
RuleDisabled string = "Disabled"
)
func (lcConf *LcConfiguration) GenEnabledRuleTasks() []*RuleTask {
tasks := make([]*RuleTask, 0)
for _, r := range lcConf.Rules {
@ -57,16 +55,8 @@ func (lcConf *LcConfiguration) GenEnabledRuleTasks() []*RuleTask {
log.LogDebugf("GenEnabledRuleTasks: skip disabled rule(%v) in volume(%v)", r.ID, lcConf.VolName)
continue
}
var err error
var encoded []byte
if encoded, err = json.Marshal(r); err != nil {
continue
}
var idSlice = []byte(lcConf.VolName)
idSlice = append(idSlice, encoded...)
task := &RuleTask{
Id: fmt.Sprintf("%x", md5.Sum(idSlice)),
Id: fmt.Sprintf("%s:%s", lcConf.VolName, r.ID),
VolName: lcConf.VolName,
Rule: r,
}
@ -76,285 +66,84 @@ func (lcConf *LcConfiguration) GenEnabledRuleTasks() []*RuleTask {
return tasks
}
//lcnode
// ----------------------------------------------
// lcnode <-> master
// LcNodeRuleTask
type ScanDentry struct {
ParentId uint64 `json:"pid"` // FileID value of the parent inode.
Inode uint64 `json:"inode"` // FileID value of the current inode.
Name string `json:"name"` // Name of the current dentry.
Type uint32 `json:"type"`
DelInode bool `json:"delino"` //if Type is file, and DelInode is true, then Inode and Dentry(ParentId, Name) is to be deleted
type LcNodeRuleTaskRequest struct {
MasterAddr string
LcNodeAddr string
Task *RuleTask
}
//meta
type InodeExpireCondition struct {
ExpirationInfo *Expiration
ObjectSizeGreaterThan int64 `json:"sizegt"`
ObjectSizeLessThan int64 `json:"sizelt"`
Tags []*TagConfig
}
// BatchInodeGetExpirationRequest defines the request to get the inode in batch.
type BatchInodeGetExpirationRequest struct {
VolName string `json:"vol"`
PartitionID uint64 `json:"pid"`
Dentries []*ScanDentry `json:"dentries"`
Cond *InodeExpireCondition `json:"cond"`
}
type ExpireInfo struct {
Dentry *ScanDentry `json:"dentry"`
Expired bool `json:"expired"`
}
// BatchInodeGetExpirationResponse defines the response to the request of getting the inode in batch.
type BatchInodeGetExpirationResponse struct {
ExpirationResults []*ExpireInfo `json:"rsts"`
}
//s3 rules
var (
RuleEnabled string = "Enabled"
RuleDisabled string = "Disabled"
)
type RuleTask struct {
Id string
VolName string
Rule *Rule
}
type RuleTaskRequest struct {
MasterAddr string
RoutineID int64
Task *RuleTask
}
type S3TaskStatistics struct {
Volume string
Prefix string
TotalInodeScannedNum int64
FileScannedNum int64
DirScannedNum int64
ExpiredNum int64
ErrorSkippedNum int64
AbortedIncompleteMultipartNum int64
}
type RuleTaskResponse struct {
type LcNodeRuleTaskResponse struct {
ID string
RoutineID int64
S3TaskStatistics
StartTime *time.Time
EndTime *time.Time
Done bool
Status uint8
Result string
LcNodeRuleTaskStatistics
}
type AbortIncompleteMultipartUpload struct {
DaysAfterInitiation int
type LcNodeRuleTaskStatistics struct {
Volume string
RuleId string
TotalInodeScannedNum int64
FileScannedNum int64
DirScannedNum int64
ExpiredNum int64
ErrorSkippedNum int64
}
type Expiration struct {
Date *time.Time
Days int
// ----------------------------------
// lcnode <-> meta
type ScanDentry struct {
ParentId uint64 `json:"pid"` // FileID value of the parent inode.
Inode uint64 `json:"inode"` // FileID value of the current inode.
Name string `json:"name"` // Name of the current dentry.
Path string `json:"path"` // Path of the current dentry.
Type uint32 `json:"type"` // Type of the current dentry.
}
type AndOpr struct {
ObjectSizeGreaterThan int64
ObjectSizeLessThan int64
Prefix string
Tags []*TagConfig
type BatchDentries struct {
sync.RWMutex
dentries map[uint64]*ScanDentry
}
type TagConfig struct {
Key string
Value string
}
type FilterConfig struct {
And *AndOpr
ObjectSizeGreaterThan int64
ObjectSizeLessThan int64
Prefix string
Tag *TagConfig
}
type Rule struct {
AbortMultipartUpload *AbortIncompleteMultipartUpload
Expire *Expiration
Filter *FilterConfig
ID string
Prefix string
Status string
}
type SetBucketLifecycleRequest struct {
VolName string
Rules []*Rule
}
type ScanFilter struct {
ExpireCond *InodeExpireCondition
Prefix string
}
type AbortIncompleteMultiPartFilter struct {
DaysAfterInitiation int
Prefix string
}
func (r *Rule) GetAbortIncompleteMultiPartFilter() *AbortIncompleteMultiPartFilter {
if valid, _ := r.Validate(); !valid {
return nil
func NewBatchDentries() *BatchDentries {
return &BatchDentries{
dentries: make(map[uint64]*ScanDentry, 0),
}
if r.AbortMultipartUpload != nil {
if r.AbortMultipartUpload.DaysAfterInitiation > 0 {
return &AbortIncompleteMultiPartFilter{
DaysAfterInitiation: r.AbortMultipartUpload.DaysAfterInitiation,
}
}
}
func (f *BatchDentries) Append(dentry *ScanDentry) {
f.Lock()
defer f.Unlock()
f.dentries[dentry.Inode] = dentry
}
func (f *BatchDentries) Len() int {
f.RLock()
defer f.RUnlock()
return len(f.dentries)
}
func (f *BatchDentries) BatchGetAndClear() (map[uint64]*ScanDentry, []uint64) {
f.Lock()
defer f.Unlock()
var dentries = f.dentries
var inodes []uint64
for i := range f.dentries {
inodes = append(inodes, i)
}
return nil
}
func (r *Rule) GetScanFilter() (filter *ScanFilter, abortFilter *AbortIncompleteMultiPartFilter) {
if valid, _ := r.Validate(); !valid {
return nil, nil
}
filter = &ScanFilter{
ExpireCond: &InodeExpireCondition{
ExpirationInfo: r.Expire,
Tags: make([]*TagConfig, 0),
},
}
if r.Filter == nil {
filter.Prefix = r.Prefix
} else {
if r.Prefix != "" {
filter.Prefix = r.Prefix
} else if r.Filter.Prefix != "" {
filter.Prefix = r.Filter.Prefix
} else {
filter.Prefix = r.Filter.And.Prefix
}
if r.Filter.Tag != nil {
filter.ExpireCond.Tags = append(filter.ExpireCond.Tags, r.Filter.Tag)
} else {
if r.Filter.And != nil {
filter.ExpireCond.Tags = r.Filter.And.Tags
}
}
if r.Filter.ObjectSizeGreaterThan > 0 {
filter.ExpireCond.ObjectSizeGreaterThan = r.Filter.ObjectSizeGreaterThan
} else {
if r.Filter.And != nil {
filter.ExpireCond.ObjectSizeGreaterThan = r.Filter.And.ObjectSizeGreaterThan
}
}
if r.Filter.ObjectSizeLessThan > 0 {
filter.ExpireCond.ObjectSizeLessThan = r.Filter.ObjectSizeLessThan
} else {
if r.Filter.And != nil {
filter.ExpireCond.ObjectSizeLessThan = r.Filter.And.ObjectSizeLessThan
}
}
}
if r.AbortMultipartUpload != nil {
if r.AbortMultipartUpload.DaysAfterInitiation > 0 {
abortFilter = &AbortIncompleteMultiPartFilter{
DaysAfterInitiation: r.AbortMultipartUpload.DaysAfterInitiation,
Prefix: filter.Prefix,
}
}
}
return filter, abortFilter
}
func (r *Rule) Validate() (valid bool, err error) {
//https://docs.aws.amazon.com/AmazonS3/latest/userguide/lifecycle-configuration-examples.html
if r.Expire == nil {
return false, errors.New("Expiration can not be empty!")
} else {
if r.Expire.Date != nil && r.Expire.Days > 0 || r.Expire.Days < 0 {
return false, errors.New("Expiration days or date error!")
}
}
if r.Filter != nil {
if r.Prefix != "" && r.Filter.Prefix != "" {
return false, errors.New("Prefix error!")
}
if r.Filter.ObjectSizeLessThan > 0 && r.Filter.ObjectSizeGreaterThan > 0 {
if r.Filter.ObjectSizeLessThan < r.Filter.ObjectSizeGreaterThan {
return false, errors.New("ObjectSize filter error!")
}
}
if r.Filter.And != nil {
//"When using more than one filter, you must wrap the filters in an <And> element. "
if r.Prefix != "" || r.Filter.Prefix != "" || r.Filter.Tag != nil || r.Filter.ObjectSizeGreaterThan > 0 || r.Filter.ObjectSizeLessThan > 0 {
return false, errors.New("And operator conflicts!")
}
if r.Filter.And.ObjectSizeLessThan > 0 && r.Filter.And.ObjectSizeGreaterThan > 0 {
if r.Filter.And.ObjectSizeLessThan < r.Filter.And.ObjectSizeGreaterThan {
return false, errors.New("ObjectSize filter error!")
}
}
}
}
return true, nil
}
type VerInfo struct {
VolName string
VerSeq uint64
}
func (vi *VerInfo) Key() string {
return fmt.Sprintf("%s_%d", vi.VolName, vi.VerSeq)
}
//snapshot version delete
type SnapshotVerDelTask struct {
VerInfo
}
type SnapshotVerDelTaskRequest struct {
MasterAddr string
Task *SnapshotVerDelTask
}
type SnapshotStatistics struct {
VerInfo
TotalInodeNum int64
FileNum int64
DirNum int64
ErrorSkippedNum int64
}
type SnapshotVerDelTaskResponse struct {
ID string
SnapshotStatistics
StartTime *time.Time
EndTime *time.Time
Done bool
Status uint8
Result string
f.dentries = make(map[uint64]*ScanDentry, 0)
return dentries, inodes
}

5
proto/packet.go
View File

@ -102,9 +102,6 @@ const (
OpMetaExtentAddWithCheck uint8 = 0x3A // Append extent key with discard extents check
OpMetaReadDirLimit uint8 = 0x3D
//Operations: LcNode -> MetaNode
OpMetaBatchInodeExpirationGet uint8 = 0x3E
// Operations: Master -> MetaNode
OpCreateMetaPartition uint8 = 0x40
OpMetaNodeHeartbeat uint8 = 0x41
@ -412,6 +409,8 @@ func (p *Packet) GetOpMsg() (m string) {
m = "OpQuotaCreateDentry"
case OpMetaDeleteDentry:
m = "OpMetaDeleteDentry"
case OpMetaBatchDeleteDentry:
m = "OpMetaBatchDeleteDentry"
case OpMetaOpen:
m = "OpMetaOpen"
case OpMetaReleaseOpen:

44
proto/snapshot.go Normal file
View File

@ -0,0 +1,44 @@
package proto
import (
"fmt"
"time"
)
type VerInfo struct {
VolName string
VerSeq uint64
}
func (vi *VerInfo) Key() string {
return fmt.Sprintf("%s_%d", vi.VolName, vi.VerSeq)
}
//snapshot version delete
type SnapshotVerDelTask struct {
VerInfo
}
type SnapshotVerDelTaskRequest struct {
MasterAddr string
LcNodeAddr string
Task *SnapshotVerDelTask
}
type SnapshotStatistics struct {
VerInfo
TotalInodeNum int64
FileNum int64
DirNum int64
ErrorSkippedNum int64
}
type SnapshotVerDelTaskResponse struct {
ID string
SnapshotStatistics
StartTime *time.Time
EndTime *time.Time
Done bool
Status uint8
Result string
}

6
sdk/master/api_admin.go
View File

@ -853,9 +853,8 @@ func (api *AdminAPI) GetVerList(volName string) (verList *proto.VolVersionInfoLi
return
}
func (api *AdminAPI) SetBucketLifecycle(req *proto.SetBucketLifecycleRequest) (err error) {
func (api *AdminAPI) SetBucketLifecycle(req *proto.LcConfiguration) (err error) {
var request = newAPIRequest(http.MethodPost, proto.SetBucketLifecycle)
var encoded []byte
if encoded, err = json.Marshal(req); err != nil {
return
@ -866,7 +865,6 @@ func (api *AdminAPI) SetBucketLifecycle(req *proto.SetBucketLifecycleRequest) (e
}
return
}
func (api *AdminAPI) GetBucketLifecycle(volume string) (lcConf *proto.LcConfiguration, err error) {
var request = newAPIRequest(http.MethodGet, proto.GetBucketLifecycle)
request.addParam("name", volume)
@ -880,13 +878,11 @@ func (api *AdminAPI) GetBucketLifecycle(volume string) (lcConf *proto.LcConfigur
}
return
}
func (api *AdminAPI) DelBucketLifecycle(volume string) (err error) {
var request = newAPIRequest(http.MethodGet, proto.DeleteBucketLifecycle)
request.addParam("name", volume)
if _, err = api.mc.serveRequest(request); err != nil {
return
}
return
}

92
sdk/meta/api.go
View File

@ -363,45 +363,6 @@ func (mw *MetaWrapper) BatchGetExpiredMultipart(prefix string, days int) (expire
return
}
func (mw *MetaWrapper) BatchInodeExpirationGet(dentries []*proto.ScanDentry, cond *proto.InodeExpireCondition) []*proto.ExpireInfo {
var wg sync.WaitGroup
batchInfos := make([]*proto.ExpireInfo, 0)
resp := make(chan []*proto.ExpireInfo, BatchIgetRespBuf)
candidates := make(map[uint64][]*proto.ScanDentry)
// Target partition does not have to be very accurate.
for _, d := range dentries {
mp := mw.getPartitionByInode(d.Inode)
if mp == nil {
continue
}
if _, ok := candidates[mp.PartitionID]; !ok {
candidates[mp.PartitionID] = make([]*proto.ScanDentry, 0, 256)
}
candidates[mp.PartitionID] = append(candidates[mp.PartitionID], d)
}
for id, ds := range candidates {
mp := mw.getPartitionByID(id)
if mp == nil {
continue
}
wg.Add(1)
go mw.batchExpriratrionGet(&wg, mp, ds, cond, resp)
}
go func() {
wg.Wait()
close(resp)
}()
for infos := range resp {
batchInfos = append(batchInfos, infos...)
}
return batchInfos
}
func (mw *MetaWrapper) InodeGet_ll(inode uint64) (*proto.InodeInfo, error) {
mp := mw.getPartitionByInode(inode)
if mp == nil {
@ -692,6 +653,59 @@ func (mw *MetaWrapper) txDelete_ll(parentID uint64, name string, isDir bool) (in
return info, preErr
}
func (mw *MetaWrapper) BatchDelete_ll(dentries []*proto.ScanDentry) {
mw.batchDelete_ll(dentries)
}
func (mw *MetaWrapper) batchDelete_ll(dentries []*proto.ScanDentry) {
pidDentries := make(map[uint64][]proto.Dentry)
for _, d := range dentries {
if _, ok := pidDentries[d.ParentId]; !ok {
pidDentries[d.ParentId] = make([]proto.Dentry, 0, 256)
}
pidDentries[d.ParentId] = append(pidDentries[d.ParentId], proto.Dentry{
Name: d.Name,
Inode: d.Inode,
Type: d.Type,
})
}
respCh := make(chan *proto.BatchDeleteDentryResponse, BatchIgetRespBuf)
var wg sync.WaitGroup
for pid, ds := range pidDentries {
parentMP := mw.getPartitionByInode(pid)
if parentMP == nil {
continue
}
wg.Add(1)
go mw.batchddelete(&wg, parentMP, pid, ds, respCh)
}
go func() {
wg.Wait()
close(respCh)
}()
for resp := range respCh {
parentID := resp.ParentID
for _, r := range resp.Items {
if r.Status == proto.OpOk {
mp := mw.getPartitionByInode(r.Inode)
if mp == nil {
log.LogErrorf("batchDelete_ll: No inode partition, ino(%v)", r.Inode)
continue
}
status, info, err := mw.iunlink(mp, r.Inode, mw.Client.GetLatestVer(), 0)
if err != nil || status != statusOK {
continue
}
if mw.EnableSummary {
go func() {
mw.UpdateSummary_ll(parentID, -1, 0, -int64(info.Size))
}()
}
}
}
}
}
/*
* Note that the return value of InodeInfo might be nil without error,
* and the caller should make sure InodeInfo is valid before using it.

111
sdk/meta/operation.go
View File

@ -879,6 +879,56 @@ func (mw *MetaWrapper) canDeleteInode(mp *MetaPartition, info *proto.InodeInfo,
return true, nil
}
func (mw *MetaWrapper) batchddelete(wg *sync.WaitGroup, mp *MetaPartition, parentID uint64, dens []proto.Dentry, respCh chan *proto.BatchDeleteDentryResponse) {
defer wg.Done()
var err error
bgTime := stat.BeginStat()
defer func() {
stat.EndStat("batchddelete", err, bgTime, 1)
}()
req := &proto.BatchDeleteDentryRequest{
VolName: mw.volname,
PartitionID: mp.PartitionID,
ParentID: parentID,
Dens: dens,
}
packet := proto.NewPacketReqID()
packet.Opcode = proto.OpMetaBatchDeleteDentry
packet.PartitionID = mp.PartitionID
err = packet.MarshalData(req)
if err != nil {
log.LogErrorf("batchddelete: req(%v) err(%v)", *req, err)
return
}
metric := exporter.NewTPCnt(packet.GetOpMsg())
defer func() {
metric.SetWithLabels(err, map[string]string{exporter.Vol: mw.volname})
}()
packet, err = mw.sendToMetaPartition(mp, packet)
if err != nil {
log.LogErrorf("batchddelete: packet(%v) mp(%v) req(%v) err(%v)", packet, mp, *req, err)
return
}
status := parseStatus(packet.ResultCode)
if status != statusOK {
err = errors.New(packet.GetResultMsg())
log.LogErrorf("batchddelete: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg())
return
}
resp := new(proto.BatchDeleteDentryResponse)
err = packet.UnmarshalData(resp)
if err != nil {
log.LogErrorf("batchddelete: packet(%v) mp(%v) err(%v) PacketData(%v)", packet, mp, err, string(packet.Data))
return
}
resp.ParentID = parentID
select {
case respCh <- resp:
default:
}
log.LogDebugf("batchddelete success: packet(%v) mp(%v) req(%v) resp(%v)", packet, mp, *req, *resp)
return
}
func (mw *MetaWrapper) lookup(mp *MetaPartition, parentID uint64, name string, verSeq uint64) (status int, inode uint64, mode uint32, err error) {
bgTime := stat.BeginStat()
@ -993,67 +1043,6 @@ func (mw *MetaWrapper) iget(mp *MetaPartition, inode uint64, verSeq uint64) (sta
return statusOK, resp.Info, nil
}
func (mw *MetaWrapper) batchExpriratrionGet(wg *sync.WaitGroup, mp *MetaPartition, dentries []*proto.ScanDentry, cond *proto.InodeExpireCondition, respCh chan []*proto.ExpireInfo) {
defer wg.Done()
var (
err error
)
bgTime := stat.BeginStat()
defer func() {
stat.EndStat("batchExpriratrionGet", err, bgTime, 1)
}()
req := &proto.BatchInodeGetExpirationRequest{
VolName: mw.volname,
PartitionID: mp.PartitionID,
Dentries: dentries,
Cond: cond,
}
packet := proto.NewPacketReqID()
packet.Opcode = proto.OpMetaBatchInodeExpirationGet
packet.PartitionID = mp.PartitionID
err = packet.MarshalData(req)
if err != nil {
return
}
metric := exporter.NewTPCnt(packet.GetOpMsg())
defer func() {
metric.SetWithLabels(err, map[string]string{exporter.Vol: mw.volname})
}()
packet, err = mw.sendToMetaPartition(mp, packet)
if err != nil {
log.LogErrorf("batchIget: packet(%v) mp(%v) req(%v) err(%v)", packet, mp, *req, err)
return
}
status := parseStatus(packet.ResultCode)
if status != statusOK {
err = errors.New(packet.GetResultMsg())
log.LogErrorf("batchIget: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg())
return
}
resp := new(proto.BatchInodeGetExpirationResponse)
err = packet.UnmarshalData(resp)
if err != nil {
log.LogErrorf("batchIget: packet(%v) mp(%v) err(%v) PacketData(%v)", packet, mp, err, string(packet.Data))
return
}
if len(resp.ExpirationResults) == 0 {
return
}
select {
case respCh <- resp.ExpirationResults:
default:
}
}
func (mw *MetaWrapper) batchIget(wg *sync.WaitGroup, mp *MetaPartition, inodes []uint64, respCh chan []*proto.InodeInfo) {
defer wg.Done()
var (

17
util/routinepool/pool.go
View File

@ -1,8 +1,23 @@
// Copyright 2023 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package routinepool
import (
"github.com/cubefs/cubefs/util/errors"
"sync/atomic"
"github.com/cubefs/cubefs/util/errors"
)
const (

14
util/routinepool/pool_test.go
View File

@ -1,3 +1,17 @@
// Copyright 2023 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package routinepool
import (

14
util/unboundedchan/ringbuffer.go
View File

@ -1,3 +1,17 @@
// Copyright 2023 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package unboundedchan
import "errors"

14
util/unboundedchan/ringbuffer_test.go
View File

@ -1,3 +1,17 @@
// Copyright 2023 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package unboundedchan
import (

14
util/unboundedchan/unbounded_chan.go
View File

@ -1,3 +1,17 @@
// Copyright 2023 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package unboundedchan
import "sync/atomic"

105
util/unboundedchan/unbounded_chan_test.go
View File

@ -1,42 +1,67 @@
// Copyright 2023 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package unboundedchan
//func TestWriteReadUnboundedChan(t *testing.T) {
// //concurrent write and read unbounded chan
// UChan := NewUnboundedChan(10)
// var wg sync.WaitGroup
// for i := 0; i < 100; i++ {
// wg.Add(1)
// go func(i int) {
// defer wg.Done()
// t.Logf("In <- %v", i)
// UChan.In <- i
// }(i)
// }
//
// go func() {
// var total int
// values := make(map[int]int)
// for v := range UChan.Out {
// val, ok := v.(int)
// t.Logf("val(%v), ok(%v)", val, ok)
// v1, ok := values[val]
// if ok {
// t.Fatalf("value(%v) is not expected to be in UnboundedChan multiple times", v1)
// } else {
// values[val] = val
// }
// if v1 >= 100 || v1 < 0 {
// t.Fatalf("value(%v) is expected in range 0-99", v1)
// }
// total++
// }
// if total != 100 {
// t.Fatalf("expected total num(%v), got(%v)", 100, total)
// }
// t.Logf("total: %v", total)
// }()
//
// wg.Wait()
// close(UChan.In)
//
//}
import (
"sync"
"testing"
)
func TestWriteReadUnboundedChan(t *testing.T) {
//concurrent write and read unbounded chan
UChan := NewUnboundedChan(10)
var wg sync.WaitGroup
for i := 0; i < 100; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
t.Logf("In <- %v", i)
UChan.In <- i
}(i)
}
var wg2 sync.WaitGroup
wg2.Add(1)
go func() {
defer wg2.Done()
var total int
values := make(map[int]int)
for v := range UChan.Out {
val, ok := v.(int)
t.Logf("val(%v), ok(%v)", val, ok)
v1, ok := values[val]
if ok {
t.Fatalf("value(%v) is not expected to be in UnboundedChan multiple times", v1)
} else {
values[val] = val
}
if v1 >= 100 || v1 < 0 {
t.Fatalf("value(%v) is expected in range 0-99", v1)
}
total++
}
if total != 100 {
t.Fatalf("expected total num(%v), got(%v)", 100, total)
}
t.Logf("total: %v", total)
}()
wg.Wait()
close(UChan.In)
t.Logf("UChan closed")
wg2.Wait()
t.Logf("finish")
}