fix(lcnode): hybrid cloud support start one task and stop one task

#22423763
	#22502668
Signed-off-by: zhaochenyang <zhaochenyang@oppo.com>
This commit is contained in:
zhaochenyang 2024-08-16 15:03:00 +08:00 committed by AmazingChi
parent f385a3b49d
commit 56b4c27f7a
8 changed files with 116 additions and 67 deletions

View File

@ -189,6 +189,8 @@ func (l *LcNode) startLcScan(adminTask *proto.AdminTask) (err error) {
if err != nil {
log.LogErrorf("startLcScan: NewS3Scanner err(%v)", err)
resp.ID = request.Task.Id
resp.Volume = request.Task.VolName
resp.Rule = request.Task.Rule
resp.LcNode = l.localServerAddr
resp.Status = proto.TaskFailed
resp.Done = true

View File

@ -394,6 +394,7 @@ func (l *LcNode) httpServiceStopScanner(w http.ResponseWriter, r *http.Request)
http.Error(w, "invalid task id", http.StatusBadRequest)
return
}
log.LogInfof("receive httpServiceStopScanner id: %v", id)
l.scannerMutex.RLock()
scanner, ok := l.lcScanners[id]
@ -420,6 +421,12 @@ func (l *LcNode) httpServiceGetFile(w http.ResponseWriter, r *http.Request) {
return
}
vol := r.FormValue("vol")
var isMigrationExtent bool
mek := r.FormValue("mek")
if mek == "true" {
isMigrationExtent = true
}
var ino uint64
if ino, err = strconv.ParseUint(r.FormValue("ino"), 10, 64); err != nil {
http.Error(w, fmt.Sprintf("ParseUint ino err: %v", err.Error()), http.StatusBadRequest)
@ -488,6 +495,7 @@ func (l *LcNode) httpServiceGetFile(w http.ResponseWriter, r *http.Request) {
defer extentClient.CloseStream(ino)
t := &TransitionMgr{
ec: extentClient,
ecForW: extentClient,
}
e := &proto.ScanDentry{
@ -495,7 +503,7 @@ func (l *LcNode) httpServiceGetFile(w http.ResponseWriter, r *http.Request) {
Inode: ino,
StorageClass: uint32(sc),
}
if err = t.readFromExtentClient(e, w, true, 0, 0); err != nil {
if err = t.readFromExtentClient(e, w, isMigrationExtent, 0, 0); err != nil {
http.Error(w, fmt.Sprintf("readFromExtentClient err: %v", err.Error()), http.StatusBadRequest)
return
}

View File

@ -7086,7 +7086,8 @@ func (m *Server) adminLcNode(w http.ResponseWriter, r *http.Request) {
case "start":
if m.cluster.partition != nil && m.cluster.partition.IsRaftLeader() {
vol := r.FormValue("vol")
success, msg := m.cluster.lcMgr.startLcScan(vol)
rid := r.FormValue("rid")
success, msg := m.cluster.lcMgr.startLcScan(vol, rid)
if !success {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: msg})
} else {
@ -7098,7 +7099,8 @@ func (m *Server) adminLcNode(w http.ResponseWriter, r *http.Request) {
case "stop":
if m.cluster.partition != nil && m.cluster.partition.IsRaftLeader() {
vol := r.FormValue("vol")
success, msg := m.cluster.lcMgr.stopLcScan(vol)
rid := r.FormValue("rid")
success, msg := m.cluster.lcMgr.stopLcScan(vol, rid)
if !success {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: msg})
} else {

View File

@ -5416,7 +5416,7 @@ func (c *Cluster) getAllLcNodeInfo(vol, done string) (rsp *LcNodeInfoResponse, e
}
for k, v := range tmpLcRuleTaskStatus.Results {
if strings.HasPrefix(k, vol) {
if vol == "" || vol == v.Volume {
if done == "true" && v.Done {
rsp.LcRuleTaskStatus.Results[k] = v
continue
@ -5432,8 +5432,10 @@ func (c *Cluster) getAllLcNodeInfo(vol, done string) (rsp *LcNodeInfoResponse, e
}
for k, v := range tmpLcRuleTaskStatus.ToBeScanned {
if strings.HasPrefix(k, vol) && (done == "false" || done == "") {
rsp.LcRuleTaskStatus.ToBeScanned[k] = v
if vol == "" || vol == v.VolName {
if done == "" || done == "false" {
rsp.LcRuleTaskStatus.ToBeScanned[k] = v
}
}
}
@ -5549,7 +5551,7 @@ func (c *Cluster) scheduleToLcScan() {
func (c *Cluster) startLcScan() {
for {
success, msg := c.lcMgr.startLcScan("")
success, msg := c.lcMgr.startLcScan("", "")
if !success {
log.LogErrorf("%v, retry after 1min", msg)
time.Sleep(time.Minute)

View File

@ -90,7 +90,15 @@ func exist(task *proto.RuleTask, doing []*proto.LcNodeRuleTaskResponse, todo []*
return false
}
func (lcMgr *lifecycleManager) startLcScan(vol string) (success bool, msg string) {
func (lcMgr *lifecycleManager) startLcScan(vol, rid string) (success bool, msg string) {
log.LogInfof("startLcScan vol: %v, rid: %v", vol, rid)
if vol == "" && rid != "" {
success = false
msg = "startLcScan failed: rid must be used with vol"
log.LogInfo(msg)
return
}
start := time.Now()
lcMgr.lcRuleTaskStatus.StartTime = &start
lcMgr.lcRuleTaskStatus.EndTime = nil
@ -127,13 +135,17 @@ func (lcMgr *lifecycleManager) startLcScan(vol string) (success bool, msg string
lcMgr.lcRuleTaskStatus.Unlock()
}
var tid string
if vol != "" {
if rid != "" {
tid = fmt.Sprintf("%s:%s", vol, rid)
}
lcMgr.lcRuleTaskStatus.Lock()
for id, result := range lcMgr.lcRuleTaskStatus.Results {
if !result.Done {
doing = append(doing, result)
} else {
if result.Volume == vol {
if result.Volume == vol && (tid == "" || tid == id) {
if err := lcMgr.cluster.syncDeleteLcResult(result); err != nil {
success = false
msg = fmt.Sprintf("startLcScan failed: syncDeleteLcResult: %v err: %v, need retry", id, err)
@ -156,6 +168,9 @@ func (lcMgr *lifecycleManager) startLcScan(vol string) (success bool, msg string
// decide which task should be started
var taskTodo []*proto.RuleTask
for _, task := range tasks {
if tid != "" && task.Id != tid {
continue
}
if !exist(task, doing, todo) {
taskTodo = append(taskTodo, task)
}
@ -218,13 +233,18 @@ func (lcMgr *lifecycleManager) genRuleTask(vol, taskId string) *proto.RuleTask {
return nil
}
func (lcMgr *lifecycleManager) stopLcScan(vol string) (success bool, msg string) {
func (lcMgr *lifecycleManager) stopLcScan(vol, rid string) (success bool, msg string) {
log.LogInfof("stopLcScan vol: %v, rid: %v", vol, rid)
if vol == "" {
success = false
msg = "stopLcScan failed: invalid vol name"
log.LogInfo(msg)
return
}
var tid string
if rid != "" {
tid = fmt.Sprintf("%s:%s", vol, rid)
}
var doing []*proto.LcNodeRuleTaskResponse
var stopTodo []string
@ -233,14 +253,14 @@ func (lcMgr *lifecycleManager) stopLcScan(vol string) (success bool, msg string)
lcMgr.lcRuleTaskStatus.Lock()
for id, result := range lcMgr.lcRuleTaskStatus.Results {
if !result.Done && vol == result.Volume {
if !result.Done && vol == result.Volume && (tid == "" || tid == id) {
doing = append(doing, result)
hasTasks = true
delete(lcMgr.lcRuleTaskStatus.Results, id)
}
}
for id, task := range lcMgr.lcRuleTaskStatus.ToBeScanned {
if vol == task.VolName {
if vol == task.VolName && (tid == "" || tid == id) {
if err := lcMgr.cluster.syncDeleteLcTask(task); err != nil {
success = false
msg = fmt.Sprintf("stopLcScan failed: syncDeleteLcTask: %v err: %v, need retry", id, err)

View File

@ -79,12 +79,12 @@ const (
MetricNodesetDpReplicaCount = "nodeset_dp_replica_count"
MetricLcNodesCount = "lc_nodes_count"
MetricLcVolStatus = "lc_vol_status"
MetricLcVolScanned = "lc_vol_scanned"
MetricLcVolFileScanned = "lc_vol_file_scanned"
MetricLcVolDirScanned = "lc_vol_dirs_scanned"
MetricLcVolExpired = "lc_vol_expired"
MetricLcTotalExpiredNum = "lc_total_expired_num"
MetricLcTotalMigrateBytes = "lc_total_migrate_Bytes"
MetricLcVolError = "lc_vol_error"
MetricLcTotalExpiredNum = "lc_vol_total_expired_num"
MetricLcTotalMigrateBytes = "lc_vol_total_migrate_Bytes"
)
const (
@ -146,14 +146,14 @@ type monitorMetrics struct {
replicaCntMap map[uint64]struct{}
nodesetIds map[uint64]string
lcNodesCount *exporter.Gauge
lcVolNames map[string]struct{}
lcVolScanned *exporter.GaugeVec
lcVolFileScanned *exporter.GaugeVec
lcVolDirScanned *exporter.GaugeVec
lcVolExpired *exporter.GaugeVec
lcTotalExpiredNum *exporter.Counter
lcTotalMigrateBytes *exporter.Counter
lcNodesCount *exporter.Gauge
lcId map[string]struct{}
lcVolStatus *exporter.GaugeVec
lcVolScanned *exporter.GaugeVec
lcVolExpired *exporter.GaugeVec
lcVolError *exporter.GaugeVec
lcVolTotalMigrateNum *exporter.Counter
lcVolTotalMigrateBytes *exporter.Counter
}
func newMonitorMetrics(c *Cluster) *monitorMetrics {
@ -165,7 +165,7 @@ func newMonitorMetrics(c *Cluster) *monitorMetrics {
nodesetInactiveMetaNodesCount: make(map[uint64]int64),
inconsistentMps: make(map[string]string),
replicaCntMap: make(map[uint64]struct{}),
lcVolNames: make(map[string]struct{}),
lcId: make(map[string]struct{}),
}
}
@ -512,12 +512,12 @@ func (mm *monitorMetrics) start() {
mm.nodesetDpReplicaCount = exporter.NewGaugeVec(MetricNodesetDpReplicaCount, "", []string{"nodeset"})
mm.lcNodesCount = exporter.NewGauge(MetricLcNodesCount)
mm.lcVolScanned = exporter.NewGaugeVec(MetricLcVolScanned, "", []string{"volName", "type"})
mm.lcVolFileScanned = exporter.NewGaugeVec(MetricLcVolFileScanned, "", []string{"volName", "type"})
mm.lcVolDirScanned = exporter.NewGaugeVec(MetricLcVolDirScanned, "", []string{"volName", "type"})
mm.lcVolExpired = exporter.NewGaugeVec(MetricLcVolExpired, "", []string{"volName", "type"})
mm.lcTotalExpiredNum = exporter.NewCounter(MetricLcTotalExpiredNum)
mm.lcTotalMigrateBytes = exporter.NewCounter(MetricLcTotalMigrateBytes)
mm.lcVolStatus = exporter.NewGaugeVec(MetricLcVolStatus, "", []string{"id"})
mm.lcVolScanned = exporter.NewGaugeVec(MetricLcVolScanned, "", []string{"id", "type"})
mm.lcVolExpired = exporter.NewGaugeVec(MetricLcVolExpired, "", []string{"id", "type"})
mm.lcVolError = exporter.NewGaugeVec(MetricLcVolError, "", []string{"id", "type"})
mm.lcVolTotalMigrateNum = exporter.NewCounter(MetricLcTotalExpiredNum)
mm.lcVolTotalMigrateBytes = exporter.NewCounter(MetricLcTotalMigrateBytes)
go mm.statMetrics()
}
@ -969,57 +969,55 @@ func (mm *monitorMetrics) clearInconsistentMps() {
}
}
func (mm *monitorMetrics) deleteS3LcVolMetric(volName string) {
mm.lcVolScanned.DeleteLabelValues(volName, "total")
mm.lcVolFileScanned.DeleteLabelValues(volName, "file")
mm.lcVolDirScanned.DeleteLabelValues(volName, "dir")
mm.lcVolExpired.DeleteLabelValues(volName, "expired")
mm.lcVolExpired.DeleteLabelValues(volName, "m_to_hdd")
mm.lcVolExpired.DeleteLabelValues(volName, "m_to_ebs")
func (mm *monitorMetrics) deleteS3LcVolMetric(id string) {
mm.lcVolStatus.DeleteLabelValues(id)
mm.lcVolScanned.DeleteLabelValues(id, "file")
mm.lcVolScanned.DeleteLabelValues(id, "dir")
mm.lcVolExpired.DeleteLabelValues(id, "delete")
mm.lcVolExpired.DeleteLabelValues(id, "hdd")
mm.lcVolExpired.DeleteLabelValues(id, "blobstore")
mm.lcVolExpired.DeleteLabelValues(id, "skip")
mm.lcVolError.DeleteLabelValues(id, "delete")
mm.lcVolError.DeleteLabelValues(id, "hdd")
mm.lcVolError.DeleteLabelValues(id, "blobstore")
mm.lcVolError.DeleteLabelValues(id, "readdir")
}
func (mm *monitorMetrics) setLcMetrics() {
lcTaskStatus := mm.cluster.lcMgr.lcRuleTaskStatus
volumeScanStatistics := make(map[string]proto.LcNodeRuleTaskStatistics)
lcTaskStatus.RLock()
for k, r := range lcTaskStatus.Results {
if _, ok := volumeScanStatistics[k]; ok && r.Done {
volumeScanStatistics[k] = proto.LcNodeRuleTaskStatistics{}
for id, r := range lcTaskStatus.Results {
if r.Done {
mm.lcVolStatus.SetWithLabelValues(0, id)
volumeScanStatistics[id] = proto.LcNodeRuleTaskStatistics{}
} else {
volumeScanStatistics[k] = r.LcNodeRuleTaskStatistics
mm.lcVolStatus.SetWithLabelValues(1, id)
volumeScanStatistics[id] = r.LcNodeRuleTaskStatistics
}
}
lcTaskStatus.RUnlock()
var expiredNum int64
var migrateToHddNum int64
var migrateToEbsNum int64
var migrateToHddBytes int64
var migrateToEbsBytes int64
for key, stat := range volumeScanStatistics {
mm.lcVolNames[key] = struct{}{}
mm.lcVolScanned.SetWithLabelValues(float64(stat.TotalFileScannedNum+stat.TotalDirScannedNum), key, "total")
mm.lcVolFileScanned.SetWithLabelValues(float64(stat.TotalFileScannedNum), key, "file")
mm.lcVolDirScanned.SetWithLabelValues(float64(stat.TotalDirScannedNum), key, "dir")
mm.lcVolExpired.SetWithLabelValues(float64(stat.ExpiredDeleteNum), key, "expired")
mm.lcVolExpired.SetWithLabelValues(float64(stat.ExpiredMToHddNum), key, "m_to_hdd")
mm.lcVolExpired.SetWithLabelValues(float64(stat.ExpiredMToBlobstoreNum), key, "m_to_ebs")
expiredNum += stat.ExpiredDeleteNum
migrateToHddNum += stat.ExpiredMToHddNum
migrateToEbsNum += stat.ExpiredMToBlobstoreNum
migrateToHddBytes += stat.ExpiredMToHddBytes
migrateToEbsBytes += stat.ExpiredMToBlobstoreBytes
for id, stat := range volumeScanStatistics {
mm.lcId[id] = struct{}{}
mm.lcVolScanned.SetWithLabelValues(float64(stat.TotalFileScannedNum), id, "file")
mm.lcVolScanned.SetWithLabelValues(float64(stat.TotalDirScannedNum), id, "dir")
mm.lcVolExpired.SetWithLabelValues(float64(stat.ExpiredDeleteNum), id, "delete")
mm.lcVolExpired.SetWithLabelValues(float64(stat.ExpiredMToHddNum), id, "hdd")
mm.lcVolExpired.SetWithLabelValues(float64(stat.ExpiredMToBlobstoreNum), id, "blobstore")
mm.lcVolExpired.SetWithLabelValues(float64(stat.ExpiredSkipNum), id, "skip")
mm.lcVolError.SetWithLabelValues(float64(stat.ErrorDeleteNum), id, "delete")
mm.lcVolError.SetWithLabelValues(float64(stat.ErrorMToHddNum), id, "hdd")
mm.lcVolError.SetWithLabelValues(float64(stat.ErrorMToBlobstoreNum), id, "blobstore")
mm.lcVolError.SetWithLabelValues(float64(stat.ErrorReadDirNum), id, "readdir")
mm.lcVolTotalMigrateNum.SetWithLabels(float64(stat.ExpiredMToHddNum+stat.ExpiredMToBlobstoreNum), map[string]string{"id": id})
mm.lcVolTotalMigrateBytes.SetWithLabels(float64(stat.ExpiredMToHddBytes+stat.ExpiredMToBlobstoreBytes), map[string]string{"id": id})
}
mm.lcTotalExpiredNum.SetWithLabels(float64(expiredNum), map[string]string{"type": "expired"})
mm.lcTotalExpiredNum.SetWithLabels(float64(migrateToHddNum), map[string]string{"type": "m_to_hdd"})
mm.lcTotalExpiredNum.SetWithLabels(float64(migrateToEbsNum), map[string]string{"type": "m_to_ebs"})
mm.lcTotalMigrateBytes.SetWithLabels(float64(migrateToHddBytes), map[string]string{"type": "m_to_hdd"})
mm.lcTotalMigrateBytes.SetWithLabels(float64(migrateToEbsBytes), map[string]string{"type": "m_to_ebs"})
}
func (mm *monitorMetrics) clearLcMetrics() {
for vol := range mm.lcVolNames {
for vol := range mm.lcId {
mm.deleteS3LcVolMetric(vol)
delete(mm.lcVolNames, vol)
delete(mm.lcId, vol)
}
}

View File

@ -240,6 +240,15 @@ func TestLifecycleConfigurationTransition2(t *testing.T) {
l1.Rules[0].Expiration.Days = &day3
err = proto.ValidRules(l1.Rules)
require.NoError(t, err)
// invalid id
l1.Rules[0].ID = "a a"
err = proto.ValidRules(l1.Rules)
require.Equal(t, err, proto.LifeCycleErrInvalidRuleID)
l1.Rules[0].ID = "abc1234AAA.-v"
err = proto.ValidRules(l1.Rules)
require.NoError(t, err)
}
func TestLifecycleConfigurationTransition3(t *testing.T) {

View File

@ -17,6 +17,7 @@ package proto
import (
"errors"
"fmt"
"regexp"
"strings"
"sync"
"time"
@ -79,6 +80,7 @@ var (
LifeCycleErrMissingActions = errors.New("At least one action needs to be specified in a rule")
LifeCycleErrMissingRuleID = errors.New("No Lifecycle Rule ID in request")
LifeCycleErrTooLongRuleID = errors.New("ID length should not exceed allowed limit of 255")
LifeCycleErrInvalidRuleID = errors.New("Invalid Rule ID")
LifeCycleErrSameRuleID = errors.New("Rule ID must be unique. Found same ID for more than one rule")
LifeCycleErrDateType = errors.New("'Date' must be at midnight GMT")
LifeCycleErrDaysType = errors.New("'Days' for Expiration action must be a positive integer")
@ -162,6 +164,8 @@ func (r *Rule) GetPrefix() string {
return prefix
}
var regexRuleId = regexp.MustCompile(`^[A-Za-z0-9.-]+$`)
func validRule(r *Rule) error {
if len(r.ID) == 0 {
return LifeCycleErrMissingRuleID
@ -169,6 +173,10 @@ func validRule(r *Rule) error {
if len(r.ID) > MaxIdLength {
return LifeCycleErrTooLongRuleID
}
if !regexRuleId.MatchString(r.ID) {
return LifeCycleErrInvalidRuleID
}
if r.Status != RuleEnabled && r.Status != RuleDisabled {
return LifeCycleErrMalformedXML
}