refactor(lcnode): hybrid cloud lifecycle transition

1. close extent client
2. use StorageClass_Replica_HDD and StorageClass_BlobStore
3. scanner init AllowedStorageClass
4. enhance lifecycle parameter check
5. add extent client for write

Signed-off-by: zhaochenyang <zhaochenyang@oppo.com>

lcnode/lc_scanner.go

@@ -108,6 +108,17 @@ func NewS3Scanner(adminTask *proto.AdminTask, l *LcNode) (*LcScanner, error) {
 		return nil, err
 	}
 
+	var volumeInfo *proto.SimpleVolView
+	volumeInfo, err = l.mc.AdminAPI().GetVolumeSimpleInfo(scanner.Volume)
+	if err != nil {
+		log.LogErrorf("NewVolume: get volume info from master failed: volume(%v) err(%v)", scanner.Volume, err)
+		return nil, err
+	}
+	if volumeInfo.Status == 1 {
+		log.LogWarnf("NewVolume: volume has been marked for deletion: volume(%v) status(%v - 0:normal/1:markDelete)",
+			scanner.Volume, volumeInfo.Status)
+		return nil, proto.ErrVolNotExists
+	}
 	var extentConfig = &stream.ExtentConfig{
 		Volume:                      scanner.Volume,
 		Masters:                     l.masters,
@@ -117,16 +128,23 @@ func NewS3Scanner(adminTask *proto.AdminTask, l *LcNode) (*LcScanner, error) {
 		OnGetExtents:                metaWrapper.GetExtents,
 		OnTruncate:                  metaWrapper.Truncate,
 		OnRenewalForbiddenMigration: metaWrapper.RenewalForbiddenMigration,
+		AllowedStorageClass:         volumeInfo.AllowedStorageClass,
 	}
 	var extentClient *stream.ExtentClient
 	if extentClient, err = stream.NewExtentClient(extentConfig); err != nil {
 		log.LogErrorf("NewExtentClient err: %v", err)
 		return nil, err
 	}
+	var extentClientForW *stream.ExtentClient
+	if extentClientForW, err = stream.NewExtentClient(extentConfig); err != nil {
+		log.LogErrorf("NewExtentClient err: %v", err)
+		return nil, err
+	}
 
 	scanner.transitionMgr = &TransitionMgr{
 		volume:    scanner.Volume,
 		ec:        extentClient,
+		ecForW:    extentClientForW,
 		ebsClient: ebsClient,
 	}
 
@@ -356,11 +374,11 @@ func (s *LcScanner) handleFile(dentry *proto.ScanDentry) {
 		_, err := s.mw.DeleteWithCond_ll(dentry.ParentId, dentry.Inode, dentry.Name, os.FileMode(dentry.Type).IsDir(), dentry.Path)
 		if err != nil {
 			atomic.AddInt64(&s.currentStat.ErrorSkippedNum, 1)
-			log.LogWarnf("batchHandleFile DeleteWithCond_ll err: %v, dentry: %+v, skip it", err, dentry)
+			log.LogWarnf("delete DeleteWithCond_ll err: %v, dentry: %+v, skip it", err, dentry)
 			return
 		}
 		if err = s.mw.Evict(dentry.Inode, dentry.Path); err != nil {
-			log.LogWarnf("batchHandleFile Evict err: %v, dentry: %+v", err, dentry)
+			log.LogWarnf("delete Evict err: %v, dentry: %+v", err, dentry)
 		}
 		atomic.AddInt64(&s.currentStat.ExpiredNum, 1)
 
@@ -447,7 +465,7 @@ func (s *LcScanner) inodeExpired(inode *proto.InodeInfo, condE *proto.Expiration
 	if condT != nil {
 		for _, cond := range condT {
 			if cond.StorageClass == proto.OpTypeStorageClassEBS {
-				if expired(s.now.Unix(), inode.CreateTime.Unix(), cond.Days, cond.Date) && inode.StorageClass < proto.StorageTypeEBS {
+				if expired(s.now.Unix(), inode.CreateTime.Unix(), cond.Days, cond.Date) && inode.StorageClass < proto.StorageClass_BlobStore {
 					op = proto.OpTypeStorageClassEBS
 					return
 				}
@@ -455,7 +473,7 @@ func (s *LcScanner) inodeExpired(inode *proto.InodeInfo, condE *proto.Expiration
 		}
 		for _, cond := range condT {
 			if cond.StorageClass == proto.OpTypeStorageClassHDD {
-				if expired(s.now.Unix(), inode.CreateTime.Unix(), cond.Days, cond.Date) && inode.StorageClass < proto.StorageTypeHDD {
+				if expired(s.now.Unix(), inode.CreateTime.Unix(), cond.Days, cond.Date) && inode.StorageClass < proto.StorageClass_Replica_HDD {
 					op = proto.OpTypeStorageClassHDD
 					return
 				}
@@ -667,5 +685,7 @@ func (s *LcScanner) Stop() {
 	close(s.dirChan.In)
 	close(s.fileChan.In)
 	s.mw.Close()
+	s.transitionMgr.ec.Close()
+	s.transitionMgr.ecForW.Close()
 	log.LogInfof("scanner(%v) stopped", s.ID)
 }

lcnode/lc_scanner_test.go

@@ -38,6 +38,7 @@ func TestLcScanner(t *testing.T) {
 		transitionMgr: &TransitionMgr{
 			volume:    "test_vol",
 			ec:        NewMockExtentClient(),
+			ecForW:    NewMockExtentClient(),
 			ebsClient: NewMockEbsClient(),
 		},
 		adminTask: &proto.AdminTask{

lcnode/lc_transition.go

@@ -32,6 +32,7 @@ type ExtentApi interface {
 	Read(inode uint64, data []byte, offset int, size int, storageClass uint32, isMigration bool) (read int, err error)
 	Write(inode uint64, offset int, data []byte, flags int, checkFunc func() error, storageClass uint32, isMigration bool) (write int, err error)
 	Flush(inode uint64) error
+	Close() error
 }
 
 type EbsApi interface {
@@ -41,18 +42,28 @@ type EbsApi interface {
 
 type TransitionMgr struct {
 	volume    string
-	ec        ExtentApi
+	ec        ExtentApi // extent client for read
+	ecForW    ExtentApi // extent client for write
 	ebsClient EbsApi
 }
 
 func (t *TransitionMgr) migrate(e *proto.ScanDentry) (err error) {
 	if err = t.ec.OpenStream(e.Inode, false, false); err != nil {
-		log.LogErrorf("migrate: OpenStream fail, inode(%v) err: %v", e.Inode, err)
+		log.LogErrorf("migrate: ec OpenStream fail, inode(%v) err: %v", e.Inode, err)
 		return
 	}
 	defer func() {
 		if closeErr := t.ec.CloseStream(e.Inode); err != nil {
-			log.LogErrorf("migrate: CloseStream fail, inode(%v) err: %v", e.Inode, closeErr)
+			log.LogErrorf("migrate: ec CloseStream fail, inode(%v) err: %v", e.Inode, closeErr)
+		}
+	}()
+	if err = t.ecForW.OpenStream(e.Inode, false, false); err != nil {
+		log.LogErrorf("migrate: ecForW OpenStream fail, inode(%v) err: %v", e.Inode, err)
+		return
+	}
+	defer func() {
+		if closeErr := t.ecForW.CloseStream(e.Inode); err != nil {
+			log.LogErrorf("migrate: ecForW CloseStream fail, inode(%v) err: %v", e.Inode, closeErr)
 		}
 	}()
 
@@ -84,9 +95,9 @@ func (t *TransitionMgr) migrate(e *proto.ScanDentry) (err error) {
 			return
 		}
 		if readN > 0 {
-			writeN, err = t.ec.Write(e.Inode, writeOffset, buf[:readN], 0, nil, proto.OpTypeToStorageType(e.Op), true)
+			writeN, err = t.ecForW.Write(e.Inode, writeOffset, buf[:readN], 0, nil, proto.OpTypeToStorageType(e.Op), true)
 			if err != nil {
-				log.LogErrorf("migrate: ebs write err: %v, inode(%v), target offset(%v)", err, e.Inode, writeOffset)
+				log.LogErrorf("migrate: ecForW write err: %v, inode(%v), target offset(%v)", err, e.Inode, writeOffset)
 				return
 			}
 			readOffset += readN
@@ -101,8 +112,8 @@ func (t *TransitionMgr) migrate(e *proto.ScanDentry) (err error) {
 		}
 	}
 
-	if err = t.ec.Flush(e.Inode); err != nil {
-		log.LogErrorf("migrate: ec flush err: %v, inode(%v)", err, e.Inode)
+	if err = t.ecForW.Flush(e.Inode); err != nil {
+		log.LogErrorf("migrate: ecForW flush err: %v, inode(%v)", err, e.Inode)
 		return
 	}
 
@@ -162,7 +173,12 @@ func (t *TransitionMgr) readFromExtentClient(e *proto.ScanDentry, writer io.Writ
 		}
 		buf = buf[:readSize]
 
-		readN, err = t.ec.Read(e.Inode, buf, readOffset, readSize, e.StorageClass, isMigrationExtent)
+		if isMigrationExtent {
+			readN, err = t.ecForW.Read(e.Inode, buf, readOffset, readSize, e.StorageClass, isMigrationExtent)
+		} else {
+			readN, err = t.ec.Read(e.Inode, buf, readOffset, readSize, e.StorageClass, isMigrationExtent)
+		}
+
 		if err != nil && err != io.EOF {
 			return
 		}

lcnode/lc_transition_test.go

@@ -61,6 +61,10 @@ func (m *MockExtentClient) Flush(inode uint64) error {
 	return nil
 }
 
+func (m *MockExtentClient) Close() error {
+	return nil
+}
+
 type MockEbsClient struct {
 	data []byte
 }

master/api_service.go

@@ -6886,14 +6886,39 @@ func (m *Server) SetBucketLifecycle(w http.ResponseWriter, r *http.Request) {
 		sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
 		return
 	}
-	if _, err = m.cluster.getVol(req.VolName); err != nil {
+	var vol *Vol
+	if vol, err = m.cluster.getVol(req.VolName); err != nil {
 		sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeVolNotExists, Msg: err.Error()})
 		return
 	}
+
+	if err = proto.ValidRules(req.Rules); err != nil {
+		sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
+		return
+	}
+	//lifecycle transition storage class must in vol allowedStorageClass
+	for _, rule := range req.Rules {
+		for _, t := range rule.Transitions {
+			if !allowedStorageClass(t.StorageClass, vol.allowedStorageClass) {
+				sendErrReply(w, r, newErrHTTPReply(proto.ErrNoSupportStorageClass))
+				return
+			}
+		}
+	}
+
 	_ = m.cluster.SetBucketLifecycle(&req)
 	sendOkReply(w, r, newSuccessHTTPReply("PutBucketLifecycleConfiguration successful"))
 }
 
+func allowedStorageClass(sc string, allowed []uint32) bool {
+	for _, a := range allowed {
+		if proto.OpTypeToStorageType(sc) == a {
+			return true
+		}
+	}
+	return false
+}
+
 func (m *Server) GetBucketLifecycle(w http.ResponseWriter, r *http.Request) {
 	var (
 		err    error
@@ -6910,7 +6935,7 @@ func (m *Server) GetBucketLifecycle(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	if _, err = m.cluster.getVol(name); err != nil {
-		sendErrReply(w, r, newErrHTTPReply(proto.ErrVolNotExists))
+		sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeVolNotExists, Msg: err.Error()})
 		return
 	}
 	lcConf = m.cluster.GetBucketLifecycle(name)

objectnode/lifecycle.go

@@ -15,28 +15,17 @@
 package objectnode
 
 import (
+	"encoding/xml"
+	"io"
+	"io/ioutil"
 	"net/http"
-	"time"
 
 	"github.com/cubefs/cubefs/proto"
-)
-
-const (
-	RuleMaxCounts = 1000
-	MaxIdLength   = 255
+	"github.com/cubefs/cubefs/util/log"
 )
 
 var (
-	LifeCycleErrTooManyRules     = &ErrorCode{ErrorCode: "InvalidArgument", ErrorMessage: "Rules Number should not exceed allowed limit of 1000.", StatusCode: http.StatusBadRequest}
-	LifeCycleErrMissingRules     = &ErrorCode{ErrorCode: "InvalidArgument", ErrorMessage: "No Lifecycle Rules found in request.", StatusCode: http.StatusBadRequest}
-	LifeCycleErrMissingActions   = &ErrorCode{ErrorCode: "InvalidArgument", ErrorMessage: "At least one action needs to be specified in a rule.", StatusCode: http.StatusBadRequest}
-	LifeCycleErrMissingRuleID    = &ErrorCode{ErrorCode: "InvalidArgument", ErrorMessage: "No Lifecycle Rule ID in request.", StatusCode: http.StatusBadRequest}
-	LifeCycleErrTooLongRuleID    = &ErrorCode{ErrorCode: "InvalidArgument", ErrorMessage: "ID length should not exceed allowed limit of 255.", StatusCode: http.StatusBadRequest}
-	LifeCycleErrSameRuleID       = &ErrorCode{ErrorCode: "InvalidArgument", ErrorMessage: "Rule ID must be unique. Found same ID for more than one rule.", StatusCode: http.StatusBadRequest}
-	LifeCycleErrDateType         = &ErrorCode{ErrorCode: "InvalidArgument", ErrorMessage: "'Date' must be at midnight GMT.", StatusCode: http.StatusBadRequest}
-	LifeCycleErrDaysType         = &ErrorCode{ErrorCode: "InvalidArgument", ErrorMessage: "'Days' for Expiration action must be a positive integer.", StatusCode: http.StatusBadRequest}
-	LifeCycleErrStorageClass     = &ErrorCode{ErrorCode: "InvalidArgument", ErrorMessage: "'StorageClass' must be different for 'Transition' actions in same 'Rule'", StatusCode: http.StatusBadRequest}
-	LifeCycleErrMalformedXML     = &ErrorCode{ErrorCode: "MalformedXML", ErrorMessage: "The XML you provided was not well-formed or did not validate against our published schema.", StatusCode: http.StatusBadRequest}
+	LifeCycleErrMalformedXML     = &ErrorCode{ErrorCode: "InvalidArgument", ErrorMessage: "The XML you provided was not well-formed or did not validate against our published schema.", StatusCode: http.StatusBadRequest}
 	NoSuchLifecycleConfiguration = &ErrorCode{ErrorCode: "NoSuchLifecycleConfiguration", ErrorMessage: "The lifecycle configuration does not exist.", StatusCode: http.StatusNotFound}
 )
 
@@ -48,179 +37,132 @@ func NewLifecycleConfiguration() *LifecycleConfiguration {
 	return &LifecycleConfiguration{}
 }
 
-func (l *LifecycleConfiguration) Validate() (bool, *ErrorCode) {
-	if len(l.Rules) > RuleMaxCounts {
-		return false, LifeCycleErrTooManyRules
+// API reference: https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/API/API_GetBucketLifecycleConfiguration.html
+func (o *ObjectNode) getBucketLifecycleConfigurationHandler(w http.ResponseWriter, r *http.Request) {
+	var err error
+	var errorCode *ErrorCode
+
+	defer func() {
+		o.errorResponse(w, r, err, errorCode)
+	}()
+
+	param := ParseRequestParam(r)
+	if param.Bucket() == "" {
+		errorCode = InvalidBucketName
+		return
 	}
-	if len(l.Rules) <= 0 {
-		return false, LifeCycleErrMissingRules
+	if _, err = o.vm.Volume(param.Bucket()); err != nil {
+		errorCode = NoSuchBucket
+		return
 	}
 
-	isRuleIdExist := make(map[string]bool)
-	for _, rule := range l.Rules {
-		_, ok := isRuleIdExist[rule.ID]
-		if !ok {
-			isRuleIdExist[rule.ID] = true
-		} else {
-			return false, LifeCycleErrSameRuleID
-		}
-		if err := validRule(rule); err != nil {
-			return false, err
+	var lcConf *proto.LcConfiguration
+	if lcConf, err = o.mc.AdminAPI().GetBucketLifecycle(param.Bucket()); err != nil {
+		log.LogErrorf("getBucketLifecycle failed: requestID(%v) bucket[%v] err(%v)", GetRequestID(r), param.Bucket(), err)
+		if err.Error() == proto.ErrNoSuchLifecycleConfiguration.Error() {
+			errorCode = NoSuchLifecycleConfiguration
 		}
+		return
 	}
 
-	return true, nil
+	var lifeCycle = NewLifecycleConfiguration()
+	lifeCycle.Rules = lcConf.Rules
+	var data []byte
+	data, err = xml.Marshal(lifeCycle)
+	if err != nil {
+		log.LogErrorf("getBucketLifecycle failed: requestID(%v) bucket[%v] err(%v)", GetRequestID(r), param.Bucket(), err)
+		return
+	}
+
+	writeSuccessResponseXML(w, data)
+	return
+
 }
 
-func validRule(r *proto.Rule) *ErrorCode {
-	if len(r.ID) == 0 {
-		return LifeCycleErrMissingRuleID
+// API reference: https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html
+func (o *ObjectNode) putBucketLifecycleConfigurationHandler(w http.ResponseWriter, r *http.Request) {
+	var err error
+	var errorCode *ErrorCode
+
+	defer func() {
+		o.errorResponse(w, r, err, errorCode)
+	}()
+
+	param := ParseRequestParam(r)
+	if param.Bucket() == "" {
+		errorCode = InvalidBucketName
+		return
 	}
-	if len(r.ID) > MaxIdLength {
-		return LifeCycleErrTooLongRuleID
-	}
-	if r.Status != proto.RuleEnabled && r.Status != proto.RuleDisabled {
-		return LifeCycleErrMalformedXML
+	if _, err = o.vm.Volume(param.Bucket()); err != nil {
+		errorCode = NoSuchBucket
+		return
 	}
 
-	if r.Expiration == nil && r.Transitions == nil {
-		return LifeCycleErrMissingActions
+	_, errorCode = VerifyContentLength(r, BodyLimit)
+	if errorCode != nil {
+		return
+	}
+	var requestBody []byte
+	if requestBody, err = ioutil.ReadAll(r.Body); err != nil && err != io.EOF {
+		log.LogErrorf("putBucketLifecycle failed: read request body data err: requestID(%v) err(%v)", GetRequestID(r), err)
+		return
 	}
 
-	if r.Expiration != nil {
-		if err := validExpiration(r.Expiration); err != nil {
-			return err
+	var lifeCycle = NewLifecycleConfiguration()
+	if err = UnmarshalXMLEntity(requestBody, lifeCycle); err != nil {
+		log.LogWarnf("putBucketLifecycle failed: decode request body err: requestID(%v) err(%v)", GetRequestID(r), err)
+		errorCode = LifeCycleErrMalformedXML
+		return
+	}
+
+	if err = proto.ValidRules(lifeCycle.Rules); err != nil {
+		errorCode = &ErrorCode{
+			ErrorCode:    "InvalidArgument",
+			ErrorMessage: err.Error(),
+			StatusCode:   http.StatusBadRequest,
 		}
+		log.LogErrorf("putBucketLifecycle failed: validate err: requestID(%v) lifeCycle(%v) err(%v)", GetRequestID(r), lifeCycle, errorCode)
+		return
 	}
 
-	if r.Transitions != nil {
-		daysMap := make(map[string]int)
-		dateMap := make(map[string]*time.Time)
-		singleMap := make(map[string]int)
-		for _, transition := range r.Transitions {
-			singleMap[transition.StorageClass]++
-			if singleMap[transition.StorageClass] > 1 {
-				return LifeCycleErrStorageClass
-			}
-			if err := validTransition(transition, dateMap, daysMap); err != nil {
-				return err
-			}
-		}
-
-		if err := validTransitions(dateMap, daysMap, r.Expiration); err != nil {
-			return err
+	req := proto.LcConfiguration{
+		VolName: param.Bucket(),
+		Rules:   lifeCycle.Rules,
+	}
+	if err = o.mc.AdminAPI().SetBucketLifecycle(&req); err != nil {
+		log.LogErrorf("putBucketLifecycle failed: SetBucketLifecycle err: requestID(%v) bucket[%v] err(%v)", GetRequestID(r), param.Bucket(), err)
+		if err.Error() == proto.ErrNoSupportStorageClass.Error() {
+			errorCode = LifeCycleErrMalformedXML
 		}
+		return
 	}
 
-	return nil
+	log.LogInfof("putBucketLifecycle success: requestID(%v) volume(%v) lifeCycle(%v)",
+		GetRequestID(r), param.Bucket(), lifeCycle)
 }
 
-func validExpiration(e *proto.Expiration) *ErrorCode {
-	// Date and Days cannot be set at the same time
-	if e.Date != nil && e.Days != nil {
-		return LifeCycleErrMalformedXML
+// API reference: https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/API/API_DeleteBucketLifecycle.html
+func (o *ObjectNode) deleteBucketLifecycleConfigurationHandler(w http.ResponseWriter, r *http.Request) {
+	var err error
+	var errorCode *ErrorCode
+
+	defer func() {
+		o.errorResponse(w, r, err, errorCode)
+	}()
+
+	param := ParseRequestParam(r)
+	if param.Bucket() == "" {
+		errorCode = InvalidBucketName
+		return
 	}
-	// Date and Days cannot both be nil
-	if e.Date == nil && e.Days == nil {
-		return LifeCycleErrMalformedXML
-	}
-	// Date must be midnight UTC
-	if e.Date != nil {
-		date := e.Date.In(time.UTC)
-		if !(date.Hour() == 0 && date.Minute() == 0 && date.Second() == 0 && date.Nanosecond() == 0) {
-			return LifeCycleErrDateType
-		}
-	} else if e.Days != nil {
-		// Days must be greater than 0
-		if *e.Days <= 0 {
-			return LifeCycleErrDaysType
-		}
+	if _, err = o.vm.Volume(param.Bucket()); err != nil {
+		errorCode = NoSuchBucket
+		return
 	}
 
-	return nil
-}
-
-var transitionStorageClass = map[string]bool{
-	proto.OpTypeStorageClassHDD: true,
-	proto.OpTypeStorageClassEBS: true,
-}
-
-func validTransition(t *proto.Transition, dateMap map[string]*time.Time, daysMap map[string]int) *ErrorCode {
-	// Date and Days cannot be set at the same time
-	if t.Date != nil && t.Days != nil {
-		return LifeCycleErrMalformedXML
-	}
-	// Date and Days cannot both be nil
-	if t.Date == nil && t.Days == nil {
-		return LifeCycleErrMalformedXML
-	}
-	// StorageClass must be the specified
-	if !transitionStorageClass[t.StorageClass] {
-		return LifeCycleErrMalformedXML
-	}
-	// Date must be midnight UTC
-	if t.Date != nil {
-		date := t.Date.In(time.UTC)
-		if !(date.Hour() == 0 && date.Minute() == 0 && date.Second() == 0 && date.Nanosecond() == 0) {
-			return LifeCycleErrDateType
-		}
-		dateMap[t.StorageClass] = t.Date
-	} else if t.Days != nil {
-		// Days must be greater than 0
-		if *t.Days <= 0 {
-			return LifeCycleErrDaysType
-		}
-		daysMap[t.StorageClass] = *t.Days
-	}
-
-	return nil
-}
-
-func validTransitions(dateMap map[string]*time.Time, daysMap map[string]int, expiration *proto.Expiration) *ErrorCode {
-	// transitions and expiration must be all in date form or all in days form
-	if len(dateMap) > 0 && len(daysMap) > 0 {
-		return LifeCycleErrMalformedXML
-	}
-
-	if len(dateMap) > 0 {
-		var s []*time.Time
-		if c, ok := dateMap[proto.OpTypeStorageClassHDD]; ok {
-			s = append(s, c)
-		}
-		if c, ok := dateMap[proto.OpTypeStorageClassEBS]; ok {
-			s = append(s, c)
-		}
-		for i := 0; i < len(s)-1; i++ {
-			if !s[i+1].After(*s[i]) {
-				return LifeCycleErrMalformedXML
-			}
-		}
-		if expiration != nil {
-			if expiration.Days != nil || !expiration.Date.After(*s[len(s)-1]) {
-				return LifeCycleErrMalformedXML
-			}
-		}
-	}
-
-	if len(daysMap) > 0 {
-		var s []int
-		if c, ok := daysMap[proto.OpTypeStorageClassHDD]; ok {
-			s = append(s, c)
-		}
-		if c, ok := daysMap[proto.OpTypeStorageClassEBS]; ok {
-			s = append(s, c)
-		}
-		for i := 0; i < len(s)-1; i++ {
-			if s[i+1] <= s[i] {
-				return LifeCycleErrMalformedXML
-			}
-		}
-		if expiration != nil {
-			if expiration.Date != nil || *expiration.Days <= s[len(s)-1] {
-				return LifeCycleErrMalformedXML
-			}
-		}
-	}
-
-	return nil
+	if err = o.mc.AdminAPI().DelBucketLifecycle(param.Bucket()); err != nil {
+		log.LogErrorf("deleteBucketLifecycle failed: bucket[%v] err(%v)", param.Bucket(), err)
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
 }

objectnode/lifecycle_test.go

@@ -53,14 +53,13 @@ func TestLifecycleConfiguration(t *testing.T) {
 	err := xml.Unmarshal([]byte(LifecycleXml), l1)
 	require.NoError(t, err)
 
-	// same id
-	_, err = l1.Validate()
-	require.Equal(t, err, LifeCycleErrSameRuleID)
+	err = proto.ValidRules(l1.Rules)
+	require.Equal(t, err, proto.LifeCycleErrSameRuleID)
 
 	// id = ""
 	l1.Rules[0].ID = ""
-	_, err = l1.Validate()
-	require.Equal(t, err, LifeCycleErrMissingRuleID)
+	err = proto.ValidRules(l1.Rules)
+	require.Equal(t, err, proto.LifeCycleErrMissingRuleID)
 
 	// len(id) > 255
 	var id string
@@ -68,21 +67,21 @@ func TestLifecycleConfiguration(t *testing.T) {
 		id += "a"
 	}
 	l1.Rules[0].ID = id
-	_, err = l1.Validate()
-	require.Equal(t, err, LifeCycleErrTooLongRuleID)
+	err = proto.ValidRules(l1.Rules)
+	require.Equal(t, err, proto.LifeCycleErrTooLongRuleID)
 	l1.Rules[0].ID = "id"
 
 	// invalid status
 	l1.Rules[0].Status = ""
-	_, err = l1.Validate()
-	require.Equal(t, err, LifeCycleErrMalformedXML)
+	err = proto.ValidRules(l1.Rules)
+	require.Equal(t, err, proto.LifeCycleErrMalformedXML)
 	l1.Rules[0].Status = "Enabled"
 
 	// days < 0
 	day := -1
 	l1.Rules[0].Expiration.Days = &day
-	_, err = l1.Validate()
-	require.Equal(t, err, LifeCycleErrDaysType)
+	err = proto.ValidRules(l1.Rules)
+	require.Equal(t, err, proto.LifeCycleErrDaysType)
 	day = 0
 	l1.Rules[0].Expiration.Days = &day
 
@@ -91,39 +90,39 @@ func TestLifecycleConfiguration(t *testing.T) {
 	now := time.Now().In(time.UTC)
 	ti := time.Date(now.Year(), now.Month(), now.Day(), 1, 0, 0, 0, time.UTC)
 	l1.Rules[0].Expiration.Date = &ti
-	_, err = l1.Validate()
-	require.Equal(t, err, LifeCycleErrDateType)
+	err = proto.ValidRules(l1.Rules)
+	require.Equal(t, err, proto.LifeCycleErrDateType)
 
 	//days and date all nil
 	l1.Rules[0].Expiration.Days = nil
 	l1.Rules[0].Expiration.Date = nil
-	_, err = l1.Validate()
-	require.Equal(t, err, LifeCycleErrMalformedXML)
+	err = proto.ValidRules(l1.Rules)
+	require.Equal(t, err, proto.LifeCycleErrMalformedXML)
 
 	// days and date
 	day = 1
 	l1.Rules[0].Expiration.Days = &day
 	ti = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
 	l1.Rules[0].Expiration.Date = &ti
-	_, err = l1.Validate()
-	require.Equal(t, err, LifeCycleErrMalformedXML)
+	err = proto.ValidRules(l1.Rules)
+	require.Equal(t, err, proto.LifeCycleErrMalformedXML)
 
 	l1.Rules[0].Expiration.Date = nil
 	day = 1
 	l1.Rules[0].Expiration.Days = &day
 
 	l1.Rules[1].Expiration = nil
-	_, err = l1.Validate()
-	require.Equal(t, err, LifeCycleErrMissingActions)
+	err = proto.ValidRules(l1.Rules)
+	require.Equal(t, err, proto.LifeCycleErrMissingActions)
 
 	// no err
 	l1.Rules = l1.Rules[:1]
-	ok, _ := l1.Validate()
-	require.Equal(t, true, ok)
+	err = proto.ValidRules(l1.Rules)
+	require.NoError(t, err)
 
 	l1.Rules = l1.Rules[:0]
-	_, err = l1.Validate()
-	require.Equal(t, err, LifeCycleErrMissingRules)
+	err = proto.ValidRules(l1.Rules)
+	require.Equal(t, err, proto.LifeCycleErrMissingRules)
 }
 
 func TestLifecycleConfigurationTransition1(t *testing.T) {
@@ -146,31 +145,31 @@ func TestLifecycleConfigurationTransition1(t *testing.T) {
 
 	// test validTransition
 	l1.Rules[0].Transitions[0].Days = nil
-	_, err = l1.Validate()
-	require.Equal(t, LifeCycleErrMalformedXML, err)
+	err = proto.ValidRules(l1.Rules)
+	require.Equal(t, proto.LifeCycleErrMalformedXML, err)
 
 	day := 1
 	now := time.Now().In(time.UTC)
 	ti := time.Date(now.Year(), now.Month(), now.Day(), 1, 0, 0, 0, time.UTC)
 	l1.Rules[0].Transitions[0].Days = &day
 	l1.Rules[0].Transitions[0].Date = &ti
-	_, err = l1.Validate()
-	require.Equal(t, LifeCycleErrMalformedXML, err)
+	err = proto.ValidRules(l1.Rules)
+	require.Equal(t, proto.LifeCycleErrMalformedXML, err)
 
 	l1.Rules[0].Transitions[0].Days = nil
 	l1.Rules[0].Transitions[0].StorageClass = "SSS"
-	_, err = l1.Validate()
-	require.Equal(t, LifeCycleErrMalformedXML, err)
+	err = proto.ValidRules(l1.Rules)
+	require.Equal(t, proto.LifeCycleErrMalformedXML, err)
 
 	l1.Rules[0].Transitions[0].StorageClass = "HDD"
-	_, err = l1.Validate()
-	require.Equal(t, LifeCycleErrDateType, err)
+	err = proto.ValidRules(l1.Rules)
+	require.Equal(t, proto.LifeCycleErrDateType, err)
 
 	l1.Rules[0].Transitions[0].Date = nil
 	day = 0
 	l1.Rules[0].Transitions[0].Days = &day
-	_, err = l1.Validate()
-	require.Equal(t, LifeCycleErrDaysType, err)
+	err = proto.ValidRules(l1.Rules)
+	require.Equal(t, proto.LifeCycleErrDaysType, err)
 }
 
 func TestLifecycleConfigurationTransition2(t *testing.T) {
@@ -195,50 +194,50 @@ func TestLifecycleConfigurationTransition2(t *testing.T) {
 	err := xml.Unmarshal([]byte(LifecycleXml), l1)
 	require.NoError(t, err)
 
-	_, err = l1.Validate()
-	require.Equal(t, LifeCycleErrStorageClass, err)
+	err = proto.ValidRules(l1.Rules)
+	require.Equal(t, proto.LifeCycleErrStorageClass, err)
 
 	//test validTransitions
-	l1.Rules[0].Transitions[1].StorageClass = "EBS"
+	l1.Rules[0].Transitions[1].StorageClass = "BLOBSTORE"
 	now := time.Now().In(time.UTC)
 	ti := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
 	l1.Rules[0].Transitions[1].Days = nil
 	l1.Rules[0].Transitions[1].Date = &ti
-	_, err = l1.Validate()
-	require.Equal(t, LifeCycleErrMalformedXML, err)
+	err = proto.ValidRules(l1.Rules)
+	require.Equal(t, proto.LifeCycleErrMalformedXML, err)
 
 	ti = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
 	l1.Rules[0].Transitions[0].Days = nil
 	l1.Rules[0].Transitions[0].Date = &ti
-	_, err = l1.Validate()
-	require.Equal(t, LifeCycleErrMalformedXML, err)
+	err = proto.ValidRules(l1.Rules)
+	require.Equal(t, proto.LifeCycleErrMalformedXML, err)
 
 	t2 := time.Date(now.Year(), now.Month(), now.Day()+1, 0, 0, 0, 0, time.UTC)
 	l1.Rules[0].Transitions[1].Date = &t2
 	l1.Rules[0].Expiration = &proto.Expiration{
 		Date: &t2,
 	}
-	_, err = l1.Validate()
-	require.Equal(t, LifeCycleErrMalformedXML, err)
+	err = proto.ValidRules(l1.Rules)
+	require.Equal(t, proto.LifeCycleErrMalformedXML, err)
 
 	day1, day2 := 1, 2
 	l1.Rules[0].Transitions[0].Days = &day2
 	l1.Rules[0].Transitions[0].Date = nil
 	l1.Rules[0].Transitions[1].Days = &day1
 	l1.Rules[0].Transitions[1].Date = nil
-	_, err = l1.Validate()
-	require.Equal(t, LifeCycleErrMalformedXML, err)
+	err = proto.ValidRules(l1.Rules)
+	require.Equal(t, proto.LifeCycleErrMalformedXML, err)
 
 	l1.Rules[0].Transitions[0].Days = &day1
 	l1.Rules[0].Transitions[1].Days = &day2
 	l1.Rules[0].Expiration.Date = nil
 	l1.Rules[0].Expiration.Days = &day2
 
-	_, err = l1.Validate()
-	require.Equal(t, LifeCycleErrMalformedXML, err)
+	err = proto.ValidRules(l1.Rules)
+	require.Equal(t, proto.LifeCycleErrMalformedXML, err)
 
 	day3 := 3
 	l1.Rules[0].Expiration.Days = &day3
-	ok, _ := l1.Validate()
-	require.Equal(t, true, ok)
+	err = proto.ValidRules(l1.Rules)
+	require.NoError(t, err)
 }

proto/errors.go

@@ -93,7 +93,6 @@ var (
 	ErrVolNotDelete                            = errors.New("vol was not previously deleted or already deleted")
 	ErrVolHasDeleted                           = errors.New("vol has been deleted")
 	ErrCodeVersionOp                           = errors.New("version op failed")
-	ErrNoSuchLifecycleConfiguration            = errors.New("The lifecycle configuration does not exist")
 	ErrNoNodeSetToUpdateDecommissionDiskFactor = errors.New("no node set available for updating decommission disk factor")
 	ErrNoNodeSetToQueryDecommissionDiskLimit   = errors.New("no node set available for query decommission disk limit")
 	ErrNodeSetNotExists                        = errors.New("node set not exists")
@@ -109,6 +108,8 @@ var (
 	ErrVolNameRegExpNotMatch                   = errors.New("name can only be number and letters")
 	ErrSnapshotNotEnabled                      = errors.New("cluster not enable snapshot")
 	ErrMemberChange                            = errors.New("raft prev member change is not finished.")
+	ErrNoSuchLifecycleConfiguration            = errors.New("The lifecycle configuration does not exist")
+	ErrNoSupportStorageClass                   = errors.New("Lifecycle storage class not allowed")
 )
 
 // http response error code and error message definitions
@@ -178,6 +179,8 @@ const (
 	ErrCodeZoneNumError
 	ErrCodeVersionOpError
 	ErrCodeNodeSetNotExists
+	ErrCodeNoSuchLifecycleConfiguration
+	ErrCodeNoSupportStorageClass
 )
 
 // Err2CodeMap error map to code
@@ -243,6 +246,8 @@ var Err2CodeMap = map[error]int32{
 	ErrZoneNum:                         ErrCodeZoneNumError,
 	ErrCodeVersionOp:                   ErrCodeVersionOpError,
 	ErrNodeSetNotExists:                ErrCodeNodeSetNotExists,
+	ErrNoSuchLifecycleConfiguration:    ErrCodeNoSuchLifecycleConfiguration,
+	ErrNoSupportStorageClass:           ErrCodeNoSupportStorageClass,
 }
 
 func ParseErrorCode(code int32) error {
@@ -317,6 +322,8 @@ var code2ErrMap = map[int32]error{
 	ErrCodeNodeSetNotExists:                ErrNodeSetNotExists,
 	ErrCodeVolNotDelete:                    ErrVolNotDelete,
 	ErrCodeVolHasDeleted:                   ErrVolHasDeleted,
+	ErrCodeNoSuchLifecycleConfiguration:    ErrNoSuchLifecycleConfiguration,
+	ErrCodeNoSupportStorageClass:           ErrNoSupportStorageClass,
 }
 
 type GeneralResp struct {

proto/lifecycle.go

@@ -15,6 +15,7 @@
 package proto
 
 import (
+	"errors"
 	"fmt"
 	"sync"
 	"time"
@@ -23,24 +24,22 @@ import (
 )
 
 const (
-	RuleEnabled  string = "Enabled"
-	RuleDisabled string = "Disabled"
+	RuleEnabled   string = "Enabled"
+	RuleDisabled  string = "Disabled"
+	RuleMaxCounts        = 1000
+	MaxIdLength          = 255
 
 	OpTypeDelete          = "DELETE"
 	OpTypeStorageClassHDD = "HDD"
-	OpTypeStorageClassEBS = "EBS"
-
-	StorageTypeSSD uint32 = 1
-	StorageTypeHDD uint32 = 2
-	StorageTypeEBS uint32 = 3
+	OpTypeStorageClassEBS = "BLOBSTORE"
 )
 
 func OpTypeToStorageType(op string) uint32 {
 	switch op {
 	case OpTypeStorageClassHDD:
-		return StorageTypeHDD
+		return StorageClass_Replica_HDD
 	case OpTypeStorageClassEBS:
-		return StorageTypeEBS
+		return StorageClass_BlobStore
 	}
 	return 0
 }
@@ -73,6 +72,191 @@ type Transition struct {
 	StorageClass string     `json:"StorageClass,omitempty" xml:"StorageClass,omitempty" bson:"StorageClass,omitempty"`
 }
 
+var (
+	LifeCycleErrTooManyRules   = errors.New("Rules number should not exceed allowed limit of 1000")
+	LifeCycleErrMissingRules   = errors.New("No Lifecycle Rules found in request")
+	LifeCycleErrMissingActions = errors.New("At least one action needs to be specified in a rule")
+	LifeCycleErrMissingRuleID  = errors.New("No Lifecycle Rule ID in request")
+	LifeCycleErrTooLongRuleID  = errors.New("ID length should not exceed allowed limit of 255")
+	LifeCycleErrSameRuleID     = errors.New("Rule ID must be unique. Found same ID for more than one rule")
+	LifeCycleErrDateType       = errors.New("'Date' must be at midnight GMT")
+	LifeCycleErrDaysType       = errors.New("'Days' for Expiration action must be a positive integer")
+	LifeCycleErrStorageClass   = errors.New("'StorageClass' must be different for 'Transition' actions in same 'Rule'")
+	LifeCycleErrMalformedXML   = errors.New("The XML you provided was not well-formed or did not validate against our published schema")
+)
+
+func ValidRules(Rules []*Rule) error {
+	if len(Rules) > RuleMaxCounts {
+		return LifeCycleErrTooManyRules
+	}
+	if len(Rules) <= 0 {
+		return LifeCycleErrMissingRules
+	}
+
+	isRuleIdExist := make(map[string]bool)
+	for _, rule := range Rules {
+		_, ok := isRuleIdExist[rule.ID]
+		if !ok {
+			isRuleIdExist[rule.ID] = true
+		} else {
+			return LifeCycleErrSameRuleID
+		}
+		if err := validRule(rule); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func validRule(r *Rule) error {
+	if len(r.ID) == 0 {
+		return LifeCycleErrMissingRuleID
+	}
+	if len(r.ID) > MaxIdLength {
+		return LifeCycleErrTooLongRuleID
+	}
+	if r.Status != RuleEnabled && r.Status != RuleDisabled {
+		return LifeCycleErrMalformedXML
+	}
+
+	if r.Expiration == nil && r.Transitions == nil {
+		return LifeCycleErrMissingActions
+	}
+
+	if r.Expiration != nil {
+		if err := validExpiration(r.Expiration); err != nil {
+			return err
+		}
+	}
+
+	if r.Transitions != nil {
+		daysMap := make(map[string]int)
+		dateMap := make(map[string]*time.Time)
+		singleMap := make(map[string]int)
+		for _, transition := range r.Transitions {
+			singleMap[transition.StorageClass]++
+			if singleMap[transition.StorageClass] > 1 {
+				return LifeCycleErrStorageClass
+			}
+			if err := validTransition(transition, dateMap, daysMap); err != nil {
+				return err
+			}
+		}
+
+		if err := validTransitions(dateMap, daysMap, r.Expiration); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func validExpiration(e *Expiration) error {
+	// Date and Days cannot be set at the same time
+	if e.Date != nil && e.Days != nil {
+		return LifeCycleErrMalformedXML
+	}
+	// Date and Days cannot both be nil
+	if e.Date == nil && e.Days == nil {
+		return LifeCycleErrMalformedXML
+	}
+	// Date must be midnight UTC
+	if e.Date != nil {
+		date := e.Date.In(time.UTC)
+		if !(date.Hour() == 0 && date.Minute() == 0 && date.Second() == 0 && date.Nanosecond() == 0) {
+			return LifeCycleErrDateType
+		}
+	} else if e.Days != nil {
+		// Days must be greater than 0
+		if *e.Days <= 0 {
+			return LifeCycleErrDaysType
+		}
+	}
+
+	return nil
+}
+
+func validTransition(t *Transition, dateMap map[string]*time.Time, daysMap map[string]int) error {
+	// Date and Days cannot be set at the same time
+	if t.Date != nil && t.Days != nil {
+		return LifeCycleErrMalformedXML
+	}
+	// Date and Days cannot both be nil
+	if t.Date == nil && t.Days == nil {
+		return LifeCycleErrMalformedXML
+	}
+	// StorageClass must be the specified
+	if t.StorageClass != OpTypeStorageClassHDD && t.StorageClass != OpTypeStorageClassEBS {
+		return LifeCycleErrMalformedXML
+	}
+	// Date must be midnight UTC
+	if t.Date != nil {
+		date := t.Date.In(time.UTC)
+		if !(date.Hour() == 0 && date.Minute() == 0 && date.Second() == 0 && date.Nanosecond() == 0) {
+			return LifeCycleErrDateType
+		}
+		dateMap[t.StorageClass] = t.Date
+	} else if t.Days != nil {
+		// Days must be greater than 0
+		if *t.Days <= 0 {
+			return LifeCycleErrDaysType
+		}
+		daysMap[t.StorageClass] = *t.Days
+	}
+
+	return nil
+}
+
+func validTransitions(dateMap map[string]*time.Time, daysMap map[string]int, expiration *Expiration) error {
+	// transitions and expiration must be all in date form or all in days form
+	if len(dateMap) > 0 && len(daysMap) > 0 {
+		return LifeCycleErrMalformedXML
+	}
+
+	if len(dateMap) > 0 {
+		var s []*time.Time
+		if c, ok := dateMap[OpTypeStorageClassHDD]; ok {
+			s = append(s, c)
+		}
+		if c, ok := dateMap[OpTypeStorageClassEBS]; ok {
+			s = append(s, c)
+		}
+		for i := 0; i < len(s)-1; i++ {
+			if !s[i+1].After(*s[i]) {
+				return LifeCycleErrMalformedXML
+			}
+		}
+		if expiration != nil {
+			if expiration.Days != nil || !expiration.Date.After(*s[len(s)-1]) {
+				return LifeCycleErrMalformedXML
+			}
+		}
+	}
+
+	if len(daysMap) > 0 {
+		var s []int
+		if c, ok := daysMap[OpTypeStorageClassHDD]; ok {
+			s = append(s, c)
+		}
+		if c, ok := daysMap[OpTypeStorageClassEBS]; ok {
+			s = append(s, c)
+		}
+		for i := 0; i < len(s)-1; i++ {
+			if s[i+1] <= s[i] {
+				return LifeCycleErrMalformedXML
+			}
+		}
+		if expiration != nil {
+			if expiration.Date != nil || *expiration.Days <= s[len(s)-1] {
+				return LifeCycleErrMalformedXML
+			}
+		}
+	}
+
+	return nil
+}
+
 func (lcConf *LcConfiguration) GenEnabledRuleTasks() []*RuleTask {
 	tasks := make([]*RuleTask, 0)
 	for _, r := range lcConf.Rules {