feature: object storage interface

Add ObjectNode provides S3-compatibile APIs.
Fusion Storage interface expose two interface (POSIX and S3-compatible) for file operation.

Signed-off-by: Mofei Zhang <mofei2816@gmail.com>

.gitignore

@@ -1,4 +1,3 @@
-.gitignore
 build/bin
 build/rocksdb
 build/snappy

.gitlab-ci.yml

@@ -0,0 +1,50 @@
+before_script:
+  - go version
+  - echo $CI_PROJECT_DIR
+  - export GOPATH=/export/workspace/go
+  - echo $GOPATH
+  - echo ${CI_COMMIT_REF_NAME}
+  - rm -fr $GOPATH/src/github.com/chubaofs/chubaofs
+  - ln -s $CI_PROJECT_DIR $GOPATH/src/github.com/chubaofs/
+  - echo $CI_PROJECT_DIR
+  - echo $GOPATH/src/github.com/chubaofs/
+  - cd $GOPATH/src/github.com/chubaofs/chubaofs
+
+stages:
+  - build
+  - deploy
+  - restart
+  - ltptest
+
+build_main:
+  stage: build
+  tags:
+    - cfs
+  script:
+    - sh /export/App/cfsci/scripts/build.sh
+    
+deploy:
+  stage: deploy
+  tags:
+    - cfs
+  script:
+    - sh /export/App/cfsci/scripts/deploy.sh
+    
+restart:
+  stage: restart
+  tags:
+    - cfs
+  script:
+    - sh /export/App/cfsci/scripts/stop-ltp.sh
+    - sh /export/App/cfsci/scripts/stop-client.sh
+    - sh /export/App/cfsci/scripts/stop-server.sh
+    - sh /export/App/cfsci/scripts/start-server.sh
+    - sh /export/App/cfsci/scripts/start-client.sh
+
+ltptest:
+  stage: ltptest
+  tags:
+    - cfs
+  script:
+    - sh /export/App/cfsci/scripts/start-ltp.sh
+

MAINTAINERS.md

@@ -5,13 +5,12 @@ The ChubaoFS maintainers are:
 * Shuoran Liu <sjzlsr@126.com> @sjzlsr: Client and SDK
 * Hongyin Zhu <zzhniy.163.niy@163.com> @zhuhyc: Master
 * Jianxing Zhao <znlstar@163.com> @znlstar: MetaNode
-* Mofei Zhang <mofei2816@gmail.com> @mervinkid: MetaNode and  S3 Gateway
+* Mofei Zhang <mofei2816@gmail.com> @mervinkid: MetaNode and ObjectNode
 * Tianpeng Li <skypigltp@gmail.com> @Skypigltp: DataNode and Raft
-* Yubo Li <pangbolee@gmail.com> @yuboLee: S3 Gateway and Console
+* Yubo Li <pangbolee@gmail.com> @yuboLee: ObjectNode and Console
 * Wei Ding <wding109@gmail.com> @wding109: Research and Open Source Strategy
-* Zhengyi Zhu  <zhengyi.zhu.hust@gmail.com> @zhuzhengyi: Monitoring and S3 Gateway
+* Zhengyi Zhu  <zhengyi.zhu.hust@gmail.com> @zhuzhengyi: Monitoring and ObjectNode 
 * Liying Zhang <zly7755@163.com> @Vivian7755: Product Management & Advocate
 * Junyuan Zeng <jzeng04@gmail.com> @jzeng4: Authorization Node
 * Xihao Xu<xxscott@163.com> @xxscott: CSI Driver
-* Wenjia Wu<buaa1214wwj@126.com> @wenjia322: Authorization Node
-* Chengyu Liu<chengyu_l@126.comv> @@chengyu-l: Helm and CSI Driver
+* Wenjia Wu<buaa1214wwj@126.com> @wenjia322: Authorization Node

authnode/api_service.go

@@ -178,7 +178,7 @@ func genAuthRaftNodeOpResp(req *proto.APIAccessReq, ts int64, key []byte, msg st
 	}
 
 	if message, err = cryptoutil.EncodeMessage(jresp, key); err != nil {
-		err = fmt.Errorf("encdoe message for response failed %s", err.Error())
+		err = fmt.Errorf("encode message for response failed %s", err.Error())
 		return
 	}
 
@@ -302,17 +302,27 @@ func (m *Server) handleGetKey(keyInfo *keystore.KeyInfo) (res *keystore.KeyInfo,
 }
 
 func (m *Server) handleAddCaps(keyInfo *keystore.KeyInfo) (res *keystore.KeyInfo, err error) {
-	if res, err = m.cluster.AddCaps(keyInfo.ID, keyInfo); err != nil {
-		return
+	if keyInfo.ID == "" {
+		var akInfo *keystore.AccessKeyInfo
+		if akInfo, err = m.cluster.GetAKInfo(keyInfo.AccessKey); err != nil {
+			return
+		}
+		return m.cluster.AddCaps(akInfo.ID, keyInfo)
+	} else {
+		return m.cluster.AddCaps(keyInfo.ID, keyInfo)
 	}
-	return
 }
 
 func (m *Server) handleDeleteCaps(keyInfo *keystore.KeyInfo) (res *keystore.KeyInfo, err error) {
-	if res, err = m.cluster.DeleteCaps(keyInfo.ID, keyInfo); err != nil {
-		return
+	if keyInfo.ID == "" {
+		var akInfo *keystore.AccessKeyInfo
+		if akInfo, err = m.cluster.GetAKInfo(keyInfo.AccessKey); err != nil {
+			return
+		}
+		return m.cluster.DeleteCaps(akInfo.ID, keyInfo)
+	} else {
+		return m.cluster.DeleteCaps(keyInfo.ID, keyInfo)
 	}
-	return
 }
 
 func (m *Server) extractClientReqInfo(r *http.Request) (plaintext []byte, err error) {
@@ -335,6 +345,28 @@ func (m *Server) extractClientReqInfo(r *http.Request) (plaintext []byte, err er
 	return
 }
 
+func (m *Server) osCapsOp(writer http.ResponseWriter, request *http.Request) {
+	//TODO
+	/*
+		case proto.MsgAuthOSAddCapsReq:
+			fallthrough
+		case proto.MsgAuthOSDeleteCapsReq:
+			if err = keyInfo.IsValidAK(); err != nil {
+				sendErrReply(w, r, &proto.HTTPAuthReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
+				return
+			}
+			if err = keyInfo.IsValidCaps(); err != nil {
+				sendErrReply(w, r, &proto.HTTPAuthReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
+				return
+			}
+		case proto.MsgAuthOSGetCapsReq:
+			if err = keyInfo.IsValidAK(); err != nil {
+				sendErrReply(w, r, &proto.HTTPAuthReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
+				return
+			}
+	*/
+}
+
 func (m *Server) genTicket(key []byte, serviceID string, IP string, caps []byte) (ticket cryptoutil.Ticket) {
 	currentTime := time.Now().Unix()
 	ticket.Version = cryptoutil.TicketVersion
@@ -354,14 +386,14 @@ func (m *Server) getSecretKey(id string) (key []byte, err error) {
 	if keyInfo, err = m.getSecretKeyInfo(id); err != nil {
 		return
 	}
-	return keyInfo.Key, err
+	return keyInfo.AuthKey, err
 }
 
 func (m *Server) getSecretKeyInfo(id string) (keyInfo *keystore.KeyInfo, err error) {
 	if id == proto.AuthServiceID {
 		keyInfo = &keystore.KeyInfo{
-			Key:  m.cluster.AuthSecretKey,
-			Caps: []byte(`{"API": ["*:*:*"]}`),
+			AuthKey: m.cluster.AuthSecretKey,
+			Caps:    []byte(`{"API": ["*:*:*"]}`),
 		}
 	} else {
 		if keyInfo, err = m.cluster.GetKey(id); err != nil {
@@ -417,7 +449,7 @@ func (m *Server) genGetTicketAuthResp(req *proto.AuthGetTicketReq, ts int64, r *
 	if keyInfo, err = m.getSecretKeyInfo(resp.ClientID); err != nil {
 		return
 	}
-	clientKey = keyInfo.Key
+	clientKey = keyInfo.AuthKey
 	if message, err = cryptoutil.EncodeMessage(jresp, clientKey); err != nil {
 		return
 	}
@@ -459,7 +491,7 @@ func genAuthAPIAccessResp(req *proto.APIAccessReq, keyInfo *keystore.KeyInfo, ts
 	}
 
 	if message, err = cryptoutil.EncodeMessage(jresp, key); err != nil {
-		err = fmt.Errorf("encdoe message for response failed %s", err.Error())
+		err = fmt.Errorf("encode message for response failed %s", err.Error())
 		return
 	}
 

authnode/authnode_manager.go

@@ -40,6 +40,9 @@ func (m *Server) handleLeaderChange(leader uint64) {
 		if err := m.cluster.loadKeystore(); err != nil {
 			panic(err)
 		}
+		if err := m.cluster.loadAKstore(); err != nil {
+			panic(err)
+		}
 		m.metaReady = true
 	}
 }

authnode/cluster.go

@@ -17,6 +17,7 @@ package authnode
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/chubaofs/chubaofs/util"
 	"time"
 
 	"github.com/chubaofs/chubaofs/proto"
@@ -57,6 +58,7 @@ func newCluster(name string, leaderInfo *LeaderInfo, fsm *KeystoreFsm, partition
 	c.fsm = fsm
 	c.partition = partition
 	c.fsm.keystore = make(map[string]*keystore.KeyInfo, 0)
+	c.fsm.accessKeystore = make(map[string]*keystore.AccessKeyInfo, 0)
 	return
 }
 
@@ -88,17 +90,28 @@ func (c *Cluster) checkLeaderAddr() {
 func (c *Cluster) CreateNewKey(id string, keyInfo *keystore.KeyInfo) (res *keystore.KeyInfo, err error) {
 	c.fsm.opKeyMutex.Lock()
 	defer c.fsm.opKeyMutex.Unlock()
+	accessKeyInfo := &keystore.AccessKeyInfo{
+		AccessKey: keyInfo.AccessKey,
+		ID:        keyInfo.ID,
+	}
 	if _, err = c.fsm.GetKey(id); err == nil {
 		err = proto.ErrDuplicateKey
 		goto errHandler
 	}
 	keyInfo.Ts = time.Now().Unix()
-	keyInfo.Key = cryptoutil.GenSecretKey([]byte(c.AuthRootKey), keyInfo.Ts, id)
+	keyInfo.AuthKey = cryptoutil.GenSecretKey([]byte(c.AuthRootKey), keyInfo.Ts, id)
+	//TODO check duplicate
+	keyInfo.AccessKey = util.RandomString(16, util.Numeric|util.LowerLetter|util.UpperLetter)
+	keyInfo.SecretKey = util.RandomString(32, util.Numeric|util.LowerLetter|util.UpperLetter)
 	if err = c.syncAddKey(keyInfo); err != nil {
 		goto errHandler
 	}
+	if err = c.syncAddAccessKey(accessKeyInfo); err != nil {
+		goto errHandler
+	}
 	res = keyInfo
 	c.fsm.PutKey(keyInfo)
+	c.fsm.PutAKInfo(accessKeyInfo)
 	return
 errHandler:
 	err = fmt.Errorf("action[CreateNewKey], clusterID[%v] ID:%v, err:%v ", c.Name, keyInfo, err.Error())
@@ -138,6 +151,19 @@ errHandler:
 	return
 }
 
+// GetKey get a key from the AKstore
+func (c *Cluster) GetAKInfo(accessKey string) (akInfo *keystore.AccessKeyInfo, err error) {
+	if akInfo, err = c.fsm.GetAKInfo(accessKey); err != nil {
+		err = proto.ErrAccessKeyNotExists
+		goto errHandler
+	}
+	return
+errHandler:
+	err = fmt.Errorf("action[GetAKInfo], clusterID[%v] ID:%v, err:%v ", c.Name, accessKey, err.Error())
+	log.LogError(errors.Stack(err))
+	return
+}
+
 // AddCaps add caps to the key
 func (c *Cluster) AddCaps(id string, keyInfo *keystore.KeyInfo) (res *keystore.KeyInfo, err error) {
 	var (

authnode/const.go

@@ -33,4 +33,7 @@ const (
 	idSeparator  = "$" // To seperate ID of server that submits raft changes
 	keyAcronym   = "key"
 	ksPrefix     = keySeparator + keyAcronym + keySeparator
+
+	akAcronym = "ak"
+	akPrefix  = keySeparator + akAcronym + keySeparator
 )

authnode/http_server.go

@@ -92,6 +92,12 @@ func (m *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		fallthrough
 	case proto.AdminRemoveRaftNode:
 		m.raftNodeOp(w, r)
+	case proto.OSAddCaps:
+		fallthrough
+	case proto.OSDeleteCaps:
+		fallthrough
+	case proto.OSGetCaps:
+		m.osCapsOp(w, r)
 	default:
 		sendErrReply(w, r, &proto.HTTPAuthReply{Code: proto.ErrCodeParamError, Msg: "Invalid requst URL"})
 	}
@@ -107,7 +113,9 @@ func (m *Server) handleFunctions() {
 	http.Handle(proto.AdminGetCaps, m.handlerWithInterceptor())
 	http.Handle(proto.AdminAddRaftNode, m.handlerWithInterceptor())
 	http.Handle(proto.AdminRemoveRaftNode, m.handlerWithInterceptor())
-
+	http.Handle(proto.OSAddCaps, m.handlerWithInterceptor())
+	http.Handle(proto.OSDeleteCaps, m.handlerWithInterceptor())
+	http.Handle(proto.OSGetCaps, m.handlerWithInterceptor())
 	return
 }
 

authnode/keystore_cache_op.go

@@ -29,6 +29,31 @@ func (mf *KeystoreFsm) GetKey(id string) (u *keystore.KeyInfo, err error) {
 func (mf *KeystoreFsm) DeleteKey(id string) {
 	mf.ksMutex.Lock()
 	defer mf.ksMutex.Unlock()
-	delete((mf.keystore), id)
+	delete(mf.keystore, id)
+	return
+}
+
+func (mf *KeystoreFsm) PutAKInfo(akInfo *keystore.AccessKeyInfo) {
+	mf.aksMutex.Lock()
+	defer mf.aksMutex.Unlock()
+	if _, ok := (mf.accessKeystore)[akInfo.AccessKey]; !ok {
+		(mf.accessKeystore)[akInfo.AccessKey] = akInfo
+	}
+}
+
+func (mf *KeystoreFsm) GetAKInfo(accessKey string) (akInfo *keystore.AccessKeyInfo, err error) {
+	mf.aksMutex.RLock()
+	defer mf.aksMutex.RUnlock()
+	akInfo, ok := (mf.accessKeystore)[accessKey]
+	if !ok {
+		err = proto.ErrAccessKeyNotExists
+	}
+	return
+}
+
+func (mf *KeystoreFsm) DeleteAKInfo(accessKey string) {
+	mf.aksMutex.Lock()
+	defer mf.aksMutex.Unlock()
+	//TODO
 	return
 }

authnode/keystore_fsm.go

@@ -51,10 +51,12 @@ type KeystoreFsm struct {
 	peerChangeHandler   raftPeerChangeHandler
 	snapshotHandler     raftApplySnapshotHandler
 
-	keystore   map[string]*keystore.KeyInfo
-	ksMutex    sync.RWMutex // keystore mutex
-	opKeyMutex sync.RWMutex // operations on key mutex
-	id         uint64       // current id of server
+	keystore       map[string]*keystore.KeyInfo
+	accessKeystore map[string]*keystore.AccessKeyInfo
+	ksMutex        sync.RWMutex // keystore mutex
+	aksMutex       sync.RWMutex //accesskeystore mutex
+	opKeyMutex     sync.RWMutex // operations on key mutex
+	id             uint64       // current id of server
 }
 
 func newKeystoreFsm(store *raftstore.RocksDBStore, retainsLog uint64, rs *raft.RaftServer) (fsm *KeystoreFsm) {
@@ -144,7 +146,7 @@ func (mf *KeystoreFsm) Apply(command []byte, index uint64) (resp interface{}, er
 		// of cache may happen in newly demoted leader node. Therefore, we use the following
 		// statement: "id" indicates which server has changed keystore cache (typical leader).
 		if mf.id != leader {
-			mf.DeleteKey(string(keyInfo.Key))
+			mf.DeleteKey(keyInfo.ID)
 			log.LogInfof("action[Apply], Successfully delete key in node[%d]", mf.id)
 		} else {
 			log.LogInfof("action[Apply], Already delete key in node[%d]", mf.id)

authnode/keystore_fsm_op.go

@@ -52,6 +52,8 @@ func (m *RaftCmd) setOpType() {
 	switch keyArr[1] {
 	case keyAcronym:
 		m.Op = opSyncAddKey
+	case akAcronym:
+		m.Op = opSyncAddKey
 	default:
 		log.LogWarnf("action[setOpType] unknown opCode[%v]", keyArr[1])
 	}
@@ -73,6 +75,10 @@ func (c *Cluster) syncAddKey(keyInfo *keystore.KeyInfo) (err error) {
 	return c.syncPutKeyInfo(opSyncAddKey, keyInfo)
 }
 
+func (c *Cluster) syncAddAccessKey(akInfo *keystore.AccessKeyInfo) (err error) {
+	return c.syncPutAccessKeyInfo(opSyncAddKey, akInfo)
+}
+
 func (c *Cluster) syncAddCaps(keyInfo *keystore.KeyInfo) (err error) {
 	return c.syncPutKeyInfo(opSyncAddCaps, keyInfo)
 }
@@ -96,6 +102,17 @@ func (c *Cluster) syncPutKeyInfo(opType uint32, keyInfo *keystore.KeyInfo) (err
 	return c.submit(keydata)
 }
 
+func (c *Cluster) syncPutAccessKeyInfo(opType uint32, accessKeyInfo *keystore.AccessKeyInfo) (err error) {
+	keydata := new(RaftCmd)
+	keydata.Op = opType
+	keydata.K = akPrefix + accessKeyInfo.AccessKey + idSeparator + strconv.FormatUint(c.fsm.id, 10)
+	vv := *accessKeyInfo
+	if keydata.V, err = json.Marshal(vv); err != nil {
+		return errors.New(err.Error())
+	}
+	return c.submit(keydata)
+}
+
 func (c *Cluster) loadKeystore() (err error) {
 	ks := make(map[string]*keystore.KeyInfo, 0)
 	log.LogInfof("action[loadKeystore]")
@@ -128,6 +145,38 @@ func (c *Cluster) clearKeystore() {
 	c.fsm.keystore = nil
 }
 
+func (c *Cluster) loadAKstore() (err error) {
+	aks := make(map[string]*keystore.AccessKeyInfo, 0)
+	log.LogInfof("action[loadAccessKeystore]")
+	result, err := c.fsm.store.SeekForPrefix([]byte(akPrefix))
+	if err != nil {
+		err = fmt.Errorf("action[loadAccessKeystore], err: %v", err.Error())
+		return err
+	}
+	for _, value := range result {
+		ak := &keystore.AccessKeyInfo{}
+		if err = json.Unmarshal(value, ak); err != nil {
+			err = fmt.Errorf("action[loadAccessKeystore], value: %v, unmarshal err: %v", string(value), err)
+			return err
+		}
+		if _, ok := aks[ak.AccessKey]; !ok {
+			aks[ak.AccessKey] = ak
+		}
+		log.LogInfof("action[loadAccessKeystore], access key[%v]", ak)
+	}
+	c.fsm.aksMutex.Lock()
+	defer c.fsm.aksMutex.Unlock()
+	c.fsm.accessKeystore = aks
+
+	return
+}
+
+func (c *Cluster) clearAKstore() {
+	c.fsm.aksMutex.Lock()
+	defer c.fsm.aksMutex.Unlock()
+	c.fsm.accessKeystore = nil
+}
+
 func (c *Cluster) addRaftNode(nodeID uint64, addr string) (err error) {
 	peer := proto.Peer{ID: nodeID}
 	_, err = c.partition.ChangeMember(proto.ConfAddNode, peer, []byte(addr))

authnode/server.go

@@ -189,7 +189,10 @@ func (m *Server) Start(cfg *config.Config) (err error) {
 		log.LogError(errors.Stack(err))
 		return
 	}
-	m.rocksDBStore = raftstore.NewRocksDBStore(m.storeDir, LRUCacheSize, WriteBufferSize)
+	if m.rocksDBStore, err = raftstore.NewRocksDBStore(m.storeDir, LRUCacheSize, WriteBufferSize); err != nil {
+		log.LogErrorf("Start: init RocksDB fail: err(%v)", err)
+		return
+	}
 	if err = m.createRaftServer(); err != nil {
 		log.LogError(errors.Stack(err))
 		return

authtool/authtool.go

@@ -181,7 +181,7 @@ func getTicket() {
 	if err1 != nil {
 		panic(err1)
 	}
-	key, err2 := cryptoutil.Base64Decode(cfg.GetString("key"))
+	key, err2 := cryptoutil.Base64Decode(cfg.GetString("auth_key"))
 	if err2 != nil {
 		panic(err2)
 	}
@@ -464,11 +464,11 @@ func main() {
 				panic(err)
 			}
 			keyInfo := keystore.KeyInfo{
-				ID:   "AuthService",
-				Key:  random,
-				Ts:   time.Now().Unix(),
-				Role: "AuthService",
-				Caps: []byte(`{"*"}`),
+				ID:      "AuthService",
+				AuthKey: random,
+				Ts:      time.Now().Unix(),
+				Role:    "AuthService",
+				Caps:    []byte(`{"*"}`),
 			}
 			keyInfo.DumpJSONFile(*output[i])
 		}

build/build.sh

@@ -12,38 +12,25 @@ BuildTime=$(date +%Y-%m-%d\ %H:%M)
 LDFlags="-X main.CommitID=${CommitID} -X main.BranchName=${BranchName} -X 'main.BuildTime=${BuildTime}'"
 MODFLAGS=""
 
-RM="rm -rf"
-[[ -x "/usr/bin/rm" ]] && RM="/usr/bin/rm -rf"
-[[ -x "/bin/rm" ]] && RM="/bin/rm -rf"
-
 NPROC=$(nproc 2>/dev/null)
 NPROC=${NPROC:-"1"}
 
 GCC_LIBRARY_PATH="/lib /lib64 /usr/lib /usr/lib64 /usr/local/lib /usr/local/lib64"
+cgo_cflags=""
+cgo_ldflags="-lstdc++ -lm"
 
 [[ $(uname -s) != "Linux" ]] && { echo "ChubaoFS only support Linux os"; exit 1; }
 
-TMPDIR=${HOME}/tmp/$$
-mkdir -p ${TMPDIR}
-
-set_go_path() {
-    export GOPATH=${TMPDIR}
-    mkdir -p $GOPATH/src/github.com/chubaofs
-    SrcPath=$GOPATH/src/github.com/chubaofs/chubaofs
-    if [[  ! -e "$SrcPath" ]] ; then
-        ln -s $RootPath $SrcPath 2>/dev/null
-    fi
-}
-
 build_snappy() {
-    SnappySrcPath=${RocksdbBuildPath}/third-party/snappy-1.1.7
-    SnappyBuildPath=${SnappySrcPath}/build
+    found=$(find ${GCC_LIBRARY_PATH}  -name libsnappy.a -o -name libsnappy.so 2>/dev/null | wc -l)
+    if [[ ${found} -gt 0 ]] ; then
+        cgo_ldflags="${cgo_ldflags} -lsnappy"
+        return
+    fi
+    SnappySrcPath=${VendorPath}/snappy-1.1.7
+    SnappyBuildPath=${BuildOutPath}/snappy
     found=$(find ${SnappyBuildPath} -name libsnappy.a 2>/dev/null | wc -l)
     if [[ ${found} -eq 0 ]] ; then
-        if [[ ! -d ${RocksdbBuildPath} ]] ; then
-            mkdir -p ${RocksdbBuildPath}
-            cp -rf ${RocksdbSrcPath}/* ${RocksdbBuildPath}
-        fi
         mkdir -p ${SnappyBuildPath}
         echo "build snappy..."
         pushd ${SnappyBuildPath} >/dev/null
@@ -55,11 +42,13 @@ build_snappy() {
 }
 
 build_rocksdb() {
+    found=$(find ${GCC_LIBRARY_PATH} -name librocksdb.a -o -name librocksdb.so 2>/dev/null | wc -l)
+    if [[ ${found} -gt 0 ]] ; then
+        cgo_ldflags="${cgo_ldflags} -lrocksdb"
+        return
+    fi
     RocksdbSrcPath=${VendorPath}/rocksdb-5.9.2
     RocksdbBuildPath=${BuildOutPath}/rocksdb
-
-    build_snappy
-
     found=$(find ${RocksdbBuildPath} -name librocksdb.a 2>/dev/null | wc -l)
     if [[ ${found} -eq 0 ]] ; then
         if [[ ! -d ${RocksdbBuildPath} ]] ; then
@@ -76,20 +65,32 @@ build_rocksdb() {
     cgo_ldflags="${cgo_ldflags} -L${RocksdbBuildPath} -lrocksdb"
 }
 
-set_server_deps() {
-    cgo_cflags=""
-    cgo_ldflags=""
-
+pre_build() {
+    build_snappy
     build_rocksdb
 
+    rocksdb_libs=( z bz2 lz4 zstd )
+    for p in ${rocksdb_libs[*]} ; do
+        found=$(find /usr -name lib${p}.so 2>/dev/null | wc -l)
+        if [[ ${found} -gt 0 ]] ; then
+            cgo_ldflags="${cgo_ldflags} -l${p}"
+        fi
+    done
+
     export CGO_CFLAGS=${cgo_cflags}
     export CGO_LDFLAGS="${cgo_ldflags}"
     export GO111MODULE=off
+    export GOPATH=/tmp/cfs/go
+
+    mkdir -p $GOPATH/src/github.com/chubaofs
+    SrcPath=$GOPATH/src/github.com/chubaofs/chubaofs
+    if [[  ! -e "$SrcPath" ]] ; then
+        ln -s $RootPath $SrcPath 2>/dev/null
+    fi
 }
 
 run_test() {
-    set_go_path
-    set_server_deps
+    pre_build
     pushd $SrcPath >/dev/null
     echo "run test "
     go test -ldflags "${LDFlags}" ./...
@@ -97,17 +98,15 @@ run_test() {
 }
 
 build_server() {
-    set_go_path
-    set_server_deps
+    pre_build
     pushd $SrcPath >/dev/null
     echo -n "build cfs-server "
     go build $MODFLAGS -ldflags "${LDFlags}" -o ${BuildBinPath}/cfs-server ${SrcPath}/cmd/*.go && echo "success" || echo "failed"
     popd >/dev/null
-    unset CGO_LDFLAGS CGO_CFLAGS
 }
 
 build_client() {
-    set_go_path
+    pre_build
     pushd $SrcPath >/dev/null
     echo -n "build cfs-client "
     go build $MODFLAGS -ldflags "${LDFlags}" -o ${BuildBinPath}/cfs-client ${SrcPath}/client/*.go  && echo "success" || echo "failed"
@@ -115,7 +114,7 @@ build_client() {
 }
 
 build_client2() {
-    set_go_path
+    pre_build
     pushd $SrcPath >/dev/null
     echo -n "build cfs-client2 "
     go build $MODFLAGS -ldflags "${LDFlags}" -o ${BuildBinPath}/cfs-client2 ${SrcPath}/clientv2/*.go  && echo "success" || echo "failed"
@@ -123,7 +122,7 @@ build_client2() {
 }
 
 build_authtool() {
-    set_go_path
+    pre_build
     pushd $SrcPath >/dev/null
     echo -n "build cfs-authtool "
     go build $MODFLAGS -ldflags "${LDFlags}" -o ${BuildBinPath}/cfs-authtool ${SrcPath}/authtool/*.go  && echo "success" || echo "failed"
@@ -131,12 +130,12 @@ build_authtool() {
 }
 
 clean() {
-    ${RM} ${BuildBinPath}
+    rm -rf ${BuildBinPath}
 }
 
 dist_clean() {
-    ${RM} ${BuildBinPath}
-    ${RM} ${BuildOutPath}
+    rm -rf ${BuildBinPath}
+    rm -rf ${BuildOutPath}
 }
 
 cmd=${1:-"all"}
@@ -170,5 +169,3 @@ case "$cmd" in
     *)
         ;;
 esac
-
-${RM} ${TMPDIR}

client/fs/super.go

@@ -81,7 +81,7 @@ var (
 // NewSuper returns a new Super.
 func NewSuper(opt *MountOption) (s *Super, err error) {
 	s = new(Super)
-	s.mw, err = meta.NewMetaWrapper(opt.Volname, opt.Owner, opt.Master, opt.Authenticate, opt.TicketMess)
+	s.mw, err = meta.NewMetaWrapper(opt.Volname, opt.Owner, opt.Master, opt.Authenticate, true, &opt.TicketMess)
 	if err != nil {
 		return nil, errors.Trace(err, "NewMetaWrapper failed!")
 	}

clientv2/fs/super.go

@@ -75,7 +75,7 @@ var (
 
 func NewSuper(opt *MountOption) (s *Super, err error) {
 	s = new(Super)
-	s.mw, err = meta.NewMetaWrapper(opt.Volname, opt.Owner, opt.Master, opt.Authenticate, opt.TicketMess)
+	s.mw, err = meta.NewMetaWrapper(opt.Volname, opt.Owner, opt.Master, opt.Authenticate, true, &opt.TicketMess)
 	if err != nil {
 		return nil, errors.Trace(err, "NewMetaWrapper failed!")
 	}

cmd/cfg/meta.json

@@ -7,7 +7,7 @@
     "logDir": "/export/Logs/metanode",
     "raftDir": "/export/Data/metanode/raft",
     "raftHeartbeatPort": "9093",
-    "raftReplicatePort": "9094",
+    "raftReplicaPort": "9094",
     "totalMem":  "17179869184",
     "warnLogDir":"/export/home/tomcat/UMP-Monitor/logs/",
     "consulAddr": "http://consul.prometheus-cfs.local",

cmd/cfg/objectnode.json

@@ -0,0 +1,15 @@
+{
+  "role": "objectnode",
+  "domains": [
+    "object.cfs.local"
+  ],
+  "listen": 80,
+  "masters": [
+    "master1.cfs.local:80",
+    "master2.cfs.local:80",
+    "master3.cfs.local:80"
+  ],
+  "logLevel": "info",
+  "logDir": "/export/Logs/objectnode",
+  "region": "cn_bj"
+}

cmd/cmd.go

@@ -28,6 +28,8 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/chubaofs/chubaofs/objectnode"
+
 	"github.com/jacobsa/daemonize"
 
 	"github.com/chubaofs/chubaofs/authnode"
@@ -58,6 +60,7 @@ const (
 	RoleMeta   = "metanode"
 	RoleData   = "datanode"
 	RoleAuth   = "authnode"
+	RoleObject = "objectnode"
 )
 
 const (
@@ -65,6 +68,7 @@ const (
 	ModuleMeta   = "metaNode"
 	ModuleData   = "dataNode"
 	ModuleAuth   = "authNode"
+	ModuleObject = "objectNode"
 )
 
 const (
@@ -173,6 +177,9 @@ func main() {
 	case RoleAuth:
 		server = authnode.NewServer()
 		module = ModuleAuth
+	case RoleObject:
+		server = objectnode.NewServer()
+		module = ModuleObject
 	default:
 		daemonize.SignalOutcome(fmt.Errorf("Fatal: role mismatch: %v", role))
 		os.Exit(1)

datanode/data_partition_repair.go

@@ -126,7 +126,7 @@ func (dp *DataPartition) repair(extentType uint8) {
 
 	log.LogInfof("action[repair] partition(%v) GoodTinyExtents(%v) BadTinyExtents(%v)"+
 		" finish cost[%vms] masterAddr(%v).", dp.partitionID, dp.extentStore.AvailableTinyExtentCnt(),
-		dp.extentStore.BrokenTinyExtentCnt(), (end-start)/int64(time.Millisecond), MasterHelper.Nodes())
+		dp.extentStore.BrokenTinyExtentCnt(), (end-start)/int64(time.Millisecond), MasterClient.Nodes())
 }
 
 func (dp *DataPartition) buildDataPartitionRepairTask(repairTasks []*DataPartitionRepairTask, extentType uint8, tinyExtents []uint64) (err error) {

datanode/partition.go

@@ -587,28 +587,16 @@ func (dp *DataPartition) compareReplicas(v1, v2 []string) (equals bool) {
 // Fetch the replica information from the master.
 func (dp *DataPartition) fetchReplicasFromMaster() (isLeader bool, replicas []string, err error) {
 
-	var (
-		bufs []byte
-	)
-	params := make(map[string]string)
-	params["id"] = strconv.Itoa(int(dp.partitionID))
-	params["name"] = dp.volumeID
-	if bufs, err = MasterHelper.Request("GET", proto.AdminGetDataPartition, params, nil); err != nil {
+	var partition *master.DataPartition
+	if partition, err = MasterClient.AdminAPI().GetDataPartition(dp.volumeID, dp.partitionID); err != nil {
 		isLeader = false
 		return
 	}
-	response := &master.DataPartition{}
-	replicas = make([]string, 0)
-	if err = json.Unmarshal(bufs, &response); err != nil {
-		isLeader = false
-		replicas = nil
-		return
-	}
-	for _, host := range response.Hosts {
+	for _, host := range partition.Hosts {
 		replicas = append(replicas, host)
 	}
-	if response.Hosts != nil && len(response.Hosts) >= 1 {
-		leaderAddr := strings.Split(response.Hosts[0], ":")
+	if partition.Hosts != nil && len(partition.Hosts) >= 1 {
+		leaderAddr := strings.Split(partition.Hosts[0], ":")
 		if len(leaderAddr) == 2 && strings.TrimSpace(leaderAddr[0]) == LocalIP {
 			isLeader = true
 		}

datanode/server.go

@@ -15,8 +15,8 @@
 package datanode
 
 import (
-	"encoding/json"
 	"fmt"
+	"github.com/chubaofs/chubaofs/master"
 	"net"
 	"net/http"
 	"regexp"
@@ -34,6 +34,7 @@ import (
 	"github.com/chubaofs/chubaofs/proto"
 	"github.com/chubaofs/chubaofs/raftstore"
 	"github.com/chubaofs/chubaofs/repl"
+	masterSDK "github.com/chubaofs/chubaofs/sdk/master"
 	"github.com/chubaofs/chubaofs/util"
 	"github.com/chubaofs/chubaofs/util/config"
 	"github.com/chubaofs/chubaofs/util/exporter"
@@ -47,7 +48,7 @@ var (
 
 	LocalIP      string
 	gConnPool    = util.NewConnectPool()
-	MasterHelper = util.NewMasterHelper()
+	MasterClient = masterSDK.NewMasterClient(nil, false)
 )
 
 const (
@@ -87,6 +88,7 @@ type DataNode struct {
 	raftHeartbeat   string
 	raftReplica     string
 	raftStore       raftstore.RaftStore
+
 	tcpListener     net.Listener
 	stopC           chan bool
 	state           uint32
@@ -194,13 +196,13 @@ func (s *DataNode) parseConfig(cfg *config.Config) (err error) {
 		return fmt.Errorf("Err:masterAddr unavalid")
 	}
 	for _, ip := range cfg.GetArray(ConfigKeyMasterAddr) {
-		MasterHelper.AddNode(ip.(string))
+		MasterClient.AddNode(ip.(string))
 	}
 	s.cellName = cfg.GetString(ConfigKeyCell)
 	if s.cellName == "" {
 		s.cellName = DefaultCellName
 	}
-	log.LogDebugf("action[parseConfig] load masterAddrs(%v).", MasterHelper.Nodes())
+	log.LogDebugf("action[parseConfig] load masterAddrs(%v).", MasterClient.Nodes())
 	log.LogDebugf("action[parseConfig] load port(%v).", s.port)
 	log.LogDebugf("action[parseConfig] load cellName(%v).", s.cellName)
 	return
@@ -258,7 +260,6 @@ func (s *DataNode) startSpaceManager(cfg *config.Config) (err error) {
 func (s *DataNode) register(cfg *config.Config) {
 	var (
 		err  error
-		data []byte
 	)
 
 	timer := time.NewTimer(0)
@@ -267,20 +268,18 @@ func (s *DataNode) register(cfg *config.Config) {
 	for {
 		select {
 		case <-timer.C:
-			data, err = MasterHelper.Request(http.MethodGet, proto.AdminGetIP, nil, nil)
-			masterAddr := MasterHelper.Leader()
-			if err != nil {
+			var ci *proto.ClusterInfo
+			if ci, err = MasterClient.AdminAPI().GetClusterInfo(); err != nil {
 				log.LogErrorf("action[registerToMaster] cannot get ip from master(%v) err(%v).",
-					masterAddr, err)
+					MasterClient.Leader(), err)
 				timer.Reset(2 * time.Second)
 				continue
 			}
-			cInfo := new(proto.ClusterInfo)
-			json.Unmarshal(data, cInfo)
+			masterAddr := MasterClient.Leader()
+			s.clusterID = ci.Cluster
 			if LocalIP == "" {
-				LocalIP = string(cInfo.Ip)
+				LocalIP = string(ci.Ip)
 			}
-			s.clusterID = cInfo.Cluster
 			s.localServerAddr = fmt.Sprintf("%s:%v", LocalIP, s.port)
 			if !util.IsIPV4(LocalIP) {
 				log.LogErrorf("action[registerToMaster] got an invalid local ip(%v) from master(%v).",
@@ -290,21 +289,16 @@ func (s *DataNode) register(cfg *config.Config) {
 			}
 
 			// register this data node on the master
-			params := make(map[string]string)
-			params["addr"] = fmt.Sprintf("%s:%v", LocalIP, s.port)
-			data, err = MasterHelper.Request(http.MethodPost, proto.AddDataNode, params, nil)
-			if err != nil {
+			var nodeID uint64
+			if nodeID, err = MasterClient.NodeAPI().AddDataNode(fmt.Sprintf("%s:%v", LocalIP, s.port)); err != nil {
 				log.LogErrorf("action[registerToMaster] cannot register this node to master[%v] err(%v).",
 					masterAddr, err)
 				timer.Reset(2 * time.Second)
 				continue
 			}
-
 			exporter.RegistConsul(s.clusterID, ModuleName, cfg)
-
-			nodeID := strings.TrimSpace(string(data))
-			s.nodeID, err = strconv.ParseUint(nodeID, 10, 64)
-			log.LogDebugf("[tempDebug] nodeID(%v)", s.nodeID)
+			s.nodeID = nodeID
+			log.LogDebugf("register: register DataNode: nodeID(%v)", s.nodeID)
 			return
 		case <-s.stopC:
 			timer.Stop()
@@ -319,23 +313,21 @@ type DataNodeInfo struct {
 }
 
 func (s *DataNode) checkLocalPartitionMatchWithMaster() (err error) {
-	params := make(map[string]string)
-	params["addr"] = s.localServerAddr
-	var data interface{}
+	var convert = func(node *master.DataNode) *DataNodeInfo {
+		result := &DataNodeInfo{}
+		result.Addr = node.Addr
+		result.PersistenceDataPartitions = node.PersistenceDataPartitions
+		return result
+	}
+	var dataNode *master.DataNode
 	for i := 0; i < 3; i++ {
-		data, err = MasterHelper.Request(http.MethodGet, proto.GetDataNode, params, nil)
-		if err != nil {
+		if dataNode, err = MasterClient.NodeAPI().GetDataNode(s.localServerAddr); err != nil {
 			log.LogErrorf("checkLocalPartitionMatchWithMaster error %v", err)
 			continue
 		}
 		break
 	}
-	dinfo := new(DataNodeInfo)
-	if err = json.Unmarshal(data.([]byte), dinfo); err != nil {
-		err = fmt.Errorf("checkLocalPartitionMatchWithMaster jsonUnmarsh failed %v", err)
-		log.LogErrorf(err.Error())
-		return
-	}
+	dinfo := convert(dataNode)
 	if len(dinfo.PersistenceDataPartitions) == 0 {
 		return
 	}

datanode/wrap_operator.go

@@ -187,10 +187,7 @@ func (s *DataNode) handlePacketToCreateDataPartition(p *repl.Packet) {
 
 // Handle OpHeartbeat packet.
 func (s *DataNode) handleHeartbeatPacket(p *repl.Packet) {
-	var (
-		data []byte
-		err  error
-	)
+	var err  error
 	task := &proto.AdminTask{}
 	err = json.Unmarshal(p.Data, task)
 	defer func() {
@@ -210,8 +207,8 @@ func (s *DataNode) handleHeartbeatPacket(p *repl.Packet) {
 		s.buildHeartBeatResponse(response)
 
 		if task.OpCode == proto.OpDataNodeHeartbeat {
-			bytes, _ := json.Marshal(task.Request)
-			json.Unmarshal(bytes, request)
+			marshaled, _ := json.Marshal(task.Request)
+			_ = json.Unmarshal(marshaled, request)
 			response.Status = proto.TaskSucceeds
 		} else {
 			response.Status = proto.TaskFailed
@@ -219,11 +216,7 @@ func (s *DataNode) handleHeartbeatPacket(p *repl.Packet) {
 			response.Result = err.Error()
 		}
 		task.Response = response
-		if data, err = json.Marshal(task); err != nil {
-			return
-		}
-		_, err = MasterHelper.Request("POST", proto.GetDataNodeTaskResponse, nil, data)
-		if err != nil {
+		if err = MasterClient.NodeAPI().ResponseDataNodeTask(task); err != nil {
 			err = errors.Trace(err, "heartbeat to master(%v) failed.", request.MasterAddr)
 			log.LogErrorf(err.Error())
 			return
@@ -312,15 +305,7 @@ func (s *DataNode) asyncLoadDataPartition(task *proto.AdminTask) {
 		response.Result = err.Error()
 	}
 	task.Response = response
-	data, err := json.Marshal(task)
-	if err != nil {
-		response.PartitionId = uint64(request.PartitionId)
-		response.Status = proto.TaskFailed
-		response.Result = err.Error()
-		err = fmt.Errorf("from master Task(%v) failed,error(%v)", task.ToString(), response.Result)
-	}
-	_, err = MasterHelper.Request("POST", proto.GetDataNodeTaskResponse, nil, data)
-	if err != nil {
+	if err = MasterClient.NodeAPI().ResponseDataNodeTask(task); err != nil {
 		err = errors.Trace(err, "load DataPartition failed,PartitionID(%v)", request.PartitionId)
 		log.LogError(errors.Stack(err))
 	}

docker/Dockerfile-cfs

@@ -1,11 +0,0 @@
-FROM centos:7 AS base
-RUN curl -o /etc/yum.repos.d/epel-7.repo http://mirrors.aliyun.com/repo/epel-7.repo && \
-        yum install -y bind-utils xfsprogs jq fuse
-RUN mkdir -p /cfs/bin /cfs/conf /cfs/logs /cfs/data
-
-FROM base AS server 
-COPY build/bin/cfs-server /cfs/bin/
-
-FROM base AS client 
-COPY build/bin/cfs-client /cfs/bin/
-

docker/authnode/authnode1.json

@@ -4,7 +4,7 @@
  "port": "8080",
  "prof":"10088",
  "id":"1",
- "peers": "1:192.168.0.14:8080,2:192.168.0.15:8081",
+ "peers": "1:192.168.0.14:8080,2:192.168.0.15:8081,3:192.168.0.16:8082",
  "retainLogs":"2",
  "logDir": "/export/Logs/authnode",
  "logLevel":"info",

docker/authnode/authnode2.json

@@ -4,7 +4,7 @@
  "port": "8081",
  "prof":"10088",
  "id":"2",
- "peers": "1:192.168.0.14:8080,2:192.168.0.15:8081",
+ "peers": "1:192.168.0.14:8080,2:192.168.0.15:8081,3:192.168.0.16:8082",
  "retainLogs":"2",
  "logDir": "/export/Logs/authnode",
  "logLevel":"info",

docker/authnode/run_docker4auth.sh

@@ -3,8 +3,8 @@
 #write authkey to authnode.json
 cd ..
 cd ..
-cp ./build/bin/cfs-server /home/wuwenjia/gocode/src/github.com/chubaofs/chubaofs/docker/authnode/
-cp ./build/bin/cfs-authtool /home/wuwenjia/gocode/src/github.com/chubaofs/chubaofs/docker/authnode/
+cp ./build/bin/cfs-server /home/wuwenjia/gocode/src/github.com/chubaofs/chubaofs/docker/authnode/.
+cp ./build/bin/cfs-authtool /home/wuwenjia/gocode/src/github.com/chubaofs/chubaofs/docker/authnode/.
 cd docker/authnode
 ./cfs-authtool authkey
 authnodeKey=$(sed -n '3p' authservice.json | sed 's/key/authServiceKey/g')

docker/build_docker_cfs.sh

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-
-Version="1.0.0"
-if [[ -n "$1" ]] ;then
-	# docker image tag of ChubaoFS
-    Version=$1
-fi
-
-RootPath=$(cd $(dirname $0)/..; pwd)
-
-source ${RootPath}/build/build.sh
-CfsServer="chubaofs/cfs-server:$Version"
-CfsClient="chubaofs/cfs-client:$Version"
-docker build -t ${CfsServer} -f ${RootPath}/docker/Dockerfile-cfs --target server ${RootPath}
-docker build -t ${CfsClient} -f ${RootPath}/docker/Dockerfile-cfs --target client ${RootPath}

docker/conf/metanode.json

@@ -8,9 +8,15 @@
     "exporterPort": 9500,
     "logLevel": "info",
     "logDir": "/cfs/log",
+    "warnLogDir":"/cfs/log",
     "totalMem":"536870912",
     "metadataDir": "/cfs/data/meta",
     "raftDir": "/cfs/data/raft",
+    "masterAddrs": [
+        "192.168.0.11:17010",
+        "192.168.0.12:17010",
+        "192.168.0.13:17010"
+    ],
     "masterAddr": [
         "192.168.0.11:17010",
         "192.168.0.12:17010",

docker/conf/objectnode.json

@@ -0,0 +1,12 @@
+{
+  "role": "objectnode",
+  "hosts": "127.0.0.1",
+  "logDir": "/cfs/log/",
+  "logLevel": "debug",
+  "listen": 80,
+  "masters": [
+      "192.168.0.11:17010",
+      "192.168.0.12:17010",
+      "192.168.0.13:17010"
+  ]
+}

docker/docker-compose-auth.yml

@@ -27,6 +27,7 @@ services:
       - datanode2
       - datanode3
       - datanode4
+      - s3node1
     networks:
       authnode_extnetwork:
 
@@ -103,6 +104,7 @@ services:
     privileged: true
     networks:
       authnode_extnetwork:
+        ipv4_address: 192.168.0.21
 
   metanode2:
     image: chubaofs/cfs-base:1.0
@@ -121,6 +123,7 @@ services:
     privileged: true
     networks:
       authnode_extnetwork:
+        ipv4_address: 192.168.0.22
 
   metanode3:
     image: chubaofs/cfs-base:1.0
@@ -139,6 +142,7 @@ services:
     privileged: true
     networks:
       authnode_extnetwork:
+        ipv4_address: 192.168.0.23
 
   datanode1:
     image: chubaofs/cfs-base:1.0
@@ -158,6 +162,7 @@ services:
     privileged: true
     networks:
       authnode_extnetwork:
+        ipv4_address: 192.168.0.31
 
   datanode2:
     image: chubaofs/cfs-base:1.0
@@ -177,6 +182,7 @@ services:
     privileged: true
     networks:
       authnode_extnetwork:
+        ipv4_address: 192.168.0.32
 
   datanode3:
     image: chubaofs/cfs-base:1.0
@@ -196,6 +202,7 @@ services:
     privileged: true
     networks:
       authnode_extnetwork:
+        ipv4_address: 192.168.0.33
 
   datanode4:
     image: chubaofs/cfs-base:1.0
@@ -215,6 +222,22 @@ services:
     privileged: true
     networks:
       authnode_extnetwork:
+        ipv4_address: 192.168.0.34
+
+  s3node1:
+    image: chubaofs/cfs-base:1.0
+    ports:
+      - "80"
+    volumes:
+      - ./bin:/cfs/bin
+      - ./conf/s3node.json:/cfs/conf/s3node.json
+      - ./script/start_s3node.sh:/cfs/script/start.sh
+    command: /bin/sh /cfs/script/start.sh
+    restart: on-failure
+    privileged: true
+    networks:
+      authnode_extnetwork:
+        ipv4_address: 192.168.0.31
 
   client:
     image: chubaofs/cfs-base:1.0

docker/docker-compose.yml

@@ -30,6 +30,7 @@ services:
             - datanode2
             - datanode3
             - datanode4
+            - objectnode1
         networks:
             extnetwork:
 
@@ -226,6 +227,21 @@ services:
             extnetwork:
                 ipv4_address: 192.168.0.34
 
+    objectnode1:
+        image: chubaofs/cfs-base:1.0
+        ports:
+            - "80"
+        volumes:
+            - ./bin:/cfs/bin
+            - ./conf/objectnode.json:/cfs/conf/objectnode.json
+            - ./script/start_objectnode.sh:/cfs/script/start.sh
+        command: /bin/sh /cfs/script/start.sh
+        restart: on-failure
+        privileged: true
+        networks:
+            extnetwork:
+                ipv4_address: 192.168.0.41
+
     client:
         image: chubaofs/cfs-base:1.0
         ports:

docker/run_docker.sh

@@ -11,10 +11,11 @@ help() {
 
 Usage: ./run_docker.sh [ -h | --help ] [ -d | --disk </disk/path> ] [ -l | --ltptest ]
     -h, --help              show help info
-    -d, --disk </disk/path>     set datanode local disk path
-    -b, --build             build chubaofs server and cliente
-    -s, --server            start chubaofs servers docker image
-    -c, --client            start chubaofs client docker image
+    -d, --disk </disk/path>     set ChubaoFS DataNode local disk path
+    -b, --build             build ChubaoFS server and client
+    -s, --server            start ChubaoFS servers docker image
+    -o, --object-node       start ChubaoFS
+    -c, --client            start ChubaoFS client docker image
     -m, --monitor           start monitor web ui
     -l, --ltptest           run ltp test
     -r, --run               run servers, client and monitor
@@ -44,6 +45,10 @@ start_client() {
     docker-compose -f ${RootPath}/docker/docker-compose.yml run client bash -c "/cfs/script/start_client.sh ; /bin/bash"
 }
 
+start_objectnode() {
+    docker-compose -f ${RootPath}/docker/docker-compose.yml up -d objectnode
+}
+
 start_monitor() {
     docker-compose -f ${RootPath}/docker/docker-compose.yml up -d monitor
 }
@@ -62,6 +67,7 @@ run() {
     build
     start_monitor
     start_servers
+    start_objectnode
     start_client
 }
 
@@ -137,6 +143,7 @@ case "-$cmd" in
     -build) build ;;
     -run_servers) start_servers ;;
     -run_client) start_client ;;
+    -run_s3node) start_s3node ;;
     -run_monitor) start_monitor ;;
     -run_ltptest) run_ltptest ;;
     -clean) clean ;;

docker/run_docker_auth.sh

@@ -4,6 +4,8 @@ RootPath=$(cd $(dirname $0)/..; pwd)
 GOPATH=/go
 export DiskPath="$RootPath/docker/disk"
 
+MIN_DNDISK_AVAIL_SIZE_GB=10
+
 help() {
     cat <<EOF
 
@@ -13,6 +15,7 @@ Usage: ./run_docker_auth.sh [ -h | --help ] [ -d | --disk </disk/path> ] [ -l |
     -b, --build             build chubaofs server and cliente
     -s, --server            start chubaofs servers docker image
     -c, --client            start chubaofs client docker image
+    -s3, --s3node           start chubaofs s3node
     -m, --monitor           start monitor web ui
     -l, --ltptest           run ltp test
     -r, --run               run servers, client and monitor
@@ -33,6 +36,7 @@ build() {
 
 # start server
 start_servers() {
+    isDiskAvailable $DiskPath
     mkdir -p ${DiskPath}/{1..4}
     docker-compose -f ${RootPath}/docker/docker-compose-auth.yml up -d servers
 }
@@ -41,6 +45,10 @@ start_client() {
     docker-compose -f ${RootPath}/docker/docker-compose-auth.yml run client bash -c "/cfs/script/start_client.sh ; /bin/bash"
 }
 
+start_s3node() {
+    docker-compose -f ${RootPath}/docker/docker-compose-auth.yml up -d s3node1
+}
+
 start_monitor() {
     docker-compose -f ${RootPath}/docker/docker-compose-auth.yml up -d monitor
 }
@@ -82,6 +90,9 @@ for opt in ${ARGS[*]} ; do
         -s|--server)
             cmd=run_servers
             ;;
+        -s3|--s3node)
+            cmd=run_s3node
+            ;;
         -c|--client)
             cmd=run_client
             ;;
@@ -96,12 +107,27 @@ for opt in ${ARGS[*]} ; do
     esac
 done
 
+function isDiskAvailable() {
+    Disk=${1:-"need diskpath"}
+    [[ -d $Disk ]] || mkdir -p $Disk
+    if [[ ! -d $Disk ]] ; then
+        echo "error: $DiskPath must be exist and at least 10GB free size"
+        exit 1
+    fi
+    avail_sectors=$(df  $Disk | tail -1 | awk '{print $4}')
+    avail_GB=$(( $avail_sectors / 1024 / 1024 / 2  ))
+    if (( $avail_GB < $MIN_DNDISK_AVAIL_SIZE_GB )) ; then
+        echo "$Disk: avaible size $avail_GB GB < Min Disk avaible size $MIN_DNDISK_AVAIL_SIZE_GB GB" ;
+        exit 1
+    fi
+}
+
 for opt in ${ARGS[*]} ; do
     case "-$1" in
         --d|---disk)
             shift
             export DiskPath=${1:?"need disk dir path"}
-            [[ -d $DiskPath ]] || { echo "error: $DiskPath must be exist and at least 30GB free size"; exit 1; }
+            isDiskAvailable $DiskPath
             shift
             ;;
         -)
@@ -119,9 +145,9 @@ case "-$cmd" in
     -build) build ;;
     -run_servers) start_servers ;;
     -run_client) start_client ;;
+    -run_s3node) start_s3node ;;
     -run_monitor) start_monitor ;;
     -run_ltptest) run_ltptest ;;
     -clean) clean ;;
     *) help ;;
 esac
-

docker/script/start_client.sh

@@ -94,8 +94,8 @@ print_error_info() {
 
 start_client() {
     echo -n "start client "
+    /cfs/bin/cfs-client -c  /cfs/conf/client.json
     for((i=0; i<$TryTimes; i++)) ; do
-        nohup /cfs/bin/cfs-client -c /cfs/conf/client.json >/cfs/log/cfs.out 2>&1 &
         sleep 2
         sta=$(stat $MntPoint 2>/dev/null | tr ":：" " "  | awk '/Inode/{print $4}')
         if [[ "x$sta" == "x1" ]] ; then

docker/script/start_client_auth.sh

@@ -94,8 +94,8 @@ print_error_info() {
 
 start_client() {
     echo -n "start client "
+    /cfs/bin/cfs-client -c  /cfs/conf/client.json
     for((i=0; i<$TryTimes; i++)) ; do
-        nohup /cfs/bin/cfs-client -c /cfs/conf/client.json >/cfs/log/cfs.out 2>&1 &
         sleep 2
         sta=$(stat $MntPoint 2>/dev/null | tr ":：" " "  | awk '/Inode/{print $4}')
         if [[ "x$sta" == "x1" ]] ; then
@@ -112,5 +112,4 @@ getLeaderAddr
 check_status "MetaNode"
 check_status "DataNode"
 create_vol
-start_client
-
+start_client

docker/script/start_objectnode.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+rm -rf /cfs/disk/* /cfs/log/*
+mkdir -p /cfs/bin /cfs/log
+
+echo "start objectnode"
+/cfs/bin/cfs-server -f -c /cfs/conf/objectnode.json ; sleep 99999999d
+

docs/source/index.rst

@@ -24,6 +24,7 @@ Welcome to ChubaoFS(Chubao File System)
    user-guide/master
    user-guide/metanode
    user-guide/datanode
+   user-guide/objectnode
    user-guide/client
    user-guide/monitor
    user-guide/fuse

docs/source/user-guide/objectnode.rst

@@ -0,0 +1,57 @@
+Object Storage Service
+==============================
+
+How To Provide Object Storage Service with ObjectNode
+-----------------------
+
+Start a ObjectNode process by execute the server binary of ChubaoFS you built with ``-c`` argument and specify configuration file.
+
+.. code-block:: bash
+
+   nohup cfs-server -c s3gateway.json &
+
+
+Configurations
+-----------------------
+
+.. csv-table:: Properties
+   :header: "Key", "Type", "Description", "Mandatory"
+
+   "role", "string", "Role of process and must be set to *objectnode*", "Yes"
+   "listen", "string", "Listen and accept port of the server. Default: 80", "Yes"
+   "region", "string", "Region of this gateway. Used by S3-like interface signature validation. Default: cfs_default", "No"
+   "domains", "string slice", "
+   | Format: *DOMAIN*.
+   | DOMAIN: Domain of S3-like interface which makes wildcard domain support", "No"
+   "logDir", "string", "Log directory", "Yes"
+   "logLevel", "string", "Level operation for logging. Default is *error*", "No"
+   "masters", "string slice", "
+   | Format: *HOST:PORT*.
+   | HOST: Hostname, domain or IP address of master (resource manager).
+   | PORT: port number which listened by this master", "Yes"
+   "exporterPort", "string", "Port for monitor system", "No"
+   "prof", "string", "Pprof port", "Yes"
+
+
+**Example:**
+
+.. code-block:: json
+
+   {
+        "role": "objectnode",
+        "listen": 80,
+        "region": "test",
+        "domains": [
+            "object.cfs.local"
+        ],
+        "logDir": "/opt/cfs/objectnode/logs",
+        "logLevel": "debug",
+        "masters": [
+	        "172.20.240.95:7002",
+	        "172.20.240.94:7002",
+	        "172.20.240.67:7002"
+        ],
+        "exporterPort": 9512,
+        "prof": "7013"
+   }
+

master/api_service.go

@@ -22,13 +22,14 @@ import (
 	"time"
 
 	"bytes"
+	"io/ioutil"
+	"strings"
+
 	"github.com/chubaofs/chubaofs/proto"
 	"github.com/chubaofs/chubaofs/util"
 	"github.com/chubaofs/chubaofs/util/cryptoutil"
 	"github.com/chubaofs/chubaofs/util/errors"
 	"github.com/chubaofs/chubaofs/util/log"
-	"io/ioutil"
-	"strings"
 )
 
 // ClusterView provides the view of a cluster.
@@ -903,8 +904,43 @@ func parseRequestToGetTaskResponse(r *http.Request) (tr *proto.AdminTask, err er
 	return
 }
 
-func parseRequestToGetVol(r *http.Request) (name, authKey string, err error) {
-	return parseVolNameAndAuthKey(r)
+func parseVolName(r *http.Request) (name string, err error) {
+	if err = r.ParseForm(); err != nil {
+		return
+	}
+	if name, err = extractName(r); err != nil {
+		return
+	}
+	return
+}
+
+type getVolParameter struct {
+	name                string
+	authKey             string
+	skipOwnerValidation bool
+}
+
+func parseGetVolParameter(r *http.Request) (p *getVolParameter, err error) {
+	p = &getVolParameter{}
+	skipOwnerValidationVal := r.Header.Get(proto.SkipOwnerValidation)
+	if len(skipOwnerValidationVal) > 0 {
+		if p.skipOwnerValidation, err = strconv.ParseBool(skipOwnerValidationVal); err != nil {
+			return
+		}
+	}
+	if p.name = r.FormValue(nameKey); p.name == "" {
+		err = keyNotFound(nameKey)
+		return
+	}
+	if !volNameRegexp.MatchString(p.name) {
+		err = errors.New("name can only be number and letters")
+		return
+	}
+	if p.authKey = r.FormValue(volAuthKey); !p.skipOwnerValidation && len(p.authKey) == 0 {
+		err = keyNotFound(volAuthKey)
+		return
+	}
+	return
 }
 
 func parseVolNameAndAuthKey(r *http.Request) (name, authKey string, err error) {
@@ -1333,24 +1369,23 @@ func (m *Server) getDataPartitions(w http.ResponseWriter, r *http.Request) {
 
 func (m *Server) getVol(w http.ResponseWriter, r *http.Request) {
 	var (
-		err          error
-		name         string
-		authKey      string
-		vol          *Vol
-		checkMessage string
-		jobj         proto.APIAccessReq
-		ticket       cryptoutil.Ticket
-		ts           int64
+		err     error
+		vol     *Vol
+		message string
+		jobj    proto.APIAccessReq
+		ticket  cryptoutil.Ticket
+		ts      int64
+		param   *getVolParameter
 	)
-	if name, authKey, err = parseRequestToGetVol(r); err != nil {
+	if param, err = parseGetVolParameter(r); err != nil {
 		sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
 		return
 	}
-	if vol, err = m.cluster.getVol(name); err != nil {
+	if vol, err = m.cluster.getVol(param.name); err != nil {
 		sendErrReply(w, r, newErrHTTPReply(proto.ErrVolNotExists))
 		return
 	}
-	if !matchKey(vol.Owner, authKey) {
+	if !param.skipOwnerValidation && !matchKey(vol.Owner, param.authKey) {
 		sendErrReply(w, r, newErrHTTPReply(proto.ErrVolAuthKeyNotMatch))
 		return
 	}
@@ -1360,7 +1395,7 @@ func (m *Server) getVol(w http.ResponseWriter, r *http.Request) {
 		viewCache = vol.getViewCache()
 	}
 	if vol.authenticate {
-		if jobj, ticket, ts, err = parseAndCheckTicket(r, m.cluster.MasterSecretKey, name); err != nil {
+		if jobj, ticket, ts, err = parseAndCheckTicket(r, m.cluster.MasterSecretKey, param.name); err != nil {
 			if err == proto.ErrExpiredTicket {
 				sendErrReply(w, r, newErrHTTPReply(err))
 				return
@@ -1368,15 +1403,11 @@ func (m *Server) getVol(w http.ResponseWriter, r *http.Request) {
 			sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeInvalidTicket, Msg: err.Error()})
 			return
 		}
-		if checkMessage, err = genCheckMessage(&jobj, ts, ticket.SessionKey.Key); err != nil {
+		if message, err = genRespMessage(viewCache, &jobj, ts, ticket.SessionKey.Key); err != nil {
 			sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeMasterAPIGenRespError, Msg: err.Error()})
 			return
 		}
-		resp := &proto.GetVolResponse{
-			VolViewCache: viewCache,
-			CheckMsg:     checkMessage,
-		}
-		sendOkReply(w, r, newSuccessHTTPReply(resp))
+		sendOkReply(w, r, newSuccessHTTPReply(message))
 	} else {
 		send(w, r, viewCache)
 	}
@@ -1565,23 +1596,25 @@ func extractTicketMess(req *proto.APIAccessReq, key []byte, volName string) (tic
 		err = fmt.Errorf("CheckAPIAccessCaps failed: %s", err.Error())
 		return
 	}
-	if err = proto.CheckVOLAccessCaps(&ticket, proto.VOLRsc, volName, proto.VOLAccess, proto.MasterNode); err != nil {
+	if err = proto.CheckVOLAccessCaps(&ticket, proto.OwnerVOLRsc, volName, proto.VOLAccess, proto.MasterNode); err != nil {
 		err = fmt.Errorf("CheckVOLAccessCaps failed: %s", err.Error())
 		return
 	}
 	return
 }
 
-func genCheckMessage(req *proto.APIAccessReq, ts int64, key []byte) (message string, err error) {
+func genRespMessage(data []byte, req *proto.APIAccessReq, ts int64, key []byte) (message string, err error) {
 	var (
 		jresp []byte
-		resp  proto.APIAccessResp
+		resp  proto.MasterAPIAccessResp
 	)
 
-	resp.Type = req.Type + 1
-	resp.ClientID = req.ClientID
-	resp.ServiceID = req.ServiceID
-	resp.Verifier = ts + 1 // increase ts by one for client verify server
+	resp.Data = data
+
+	resp.APIResp.Type = req.Type + 1
+	resp.APIResp.ClientID = req.ClientID
+	resp.APIResp.ServiceID = req.ServiceID
+	resp.APIResp.Verifier = ts + 1 // increase ts by one for client verify server
 
 	if jresp, err = json.Marshal(resp); err != nil {
 		err = fmt.Errorf("json marshal for response failed %s", err.Error())

master/api_service_test.go

@@ -67,7 +67,6 @@ func createDefaultMasterServerForTest() *Server {
 		"electionTick":6,
 		"logDir": "/export/chubaofs/Logs",
 		"logLevel":"DEBUG",
-		"warnLogDir": "/export/chubaofs/Logs",
 		"walDir":"/export/chubaofs/raft",
 		"storeDir":"/export/chubaofs/rocksdbstore",
 		"clusterName":"chubaofs"

master/cluster.go

@@ -1168,6 +1168,8 @@ func (c *Cluster) doCreateVol(name, owner string, dpSize, capacity uint64, dpRep
 		goto errHandler
 	}
 	vol = newVol(id, name, owner, dpSize, capacity, uint8(dpReplicaNum), defaultReplicaNum, followerRead, authenticate)
+	// refresh oss secure
+	vol.refreshOSSSecure()
 	if err = c.syncAddVol(vol); err != nil {
 		goto errHandler
 	}

master/cluster_test.go

@@ -19,7 +19,7 @@ func buildPanicVol() *Vol {
 	if err != nil {
 		return nil
 	}
-	vol := newVol(id, commonVol.Name, commonVol.Owner, commonVol.dataPartitionSize, commonVol.Capacity, defaultReplicaNum, defaultReplicaNum, false, false)
+	vol := newVol(id, commonVol.Name, commonVol.Owner, commonVol.dataPartitionSize, commonVol.Capacity, defaultReplicaNum, defaultReplicaNum, false)
 	vol.dataPartitions = nil
 	return vol
 }

master/data_partition_check.go

@@ -219,12 +219,12 @@ func (partition *DataPartition) missingReplicaAddress(dataPartitionSize uint64)
 	}
 
 	// go through all the hosts to find the missing replica
-	for _, addr := range partition.Hosts {
-		if _, ok := partition.hasReplica(addr); !ok {
+	for _, host := range partition.Hosts {
+		if _, ok := partition.hasReplica(host); !ok {
 			log.LogError(fmt.Sprintf("action[missingReplicaAddress],partitionID:%v lack replication:%v",
-				partition.PartitionID, addr))
+				partition.PartitionID, host))
 			err = proto.ErrMissingReplica
-			addr = addr
+			addr = host
 			break
 		}
 	}

master/http_server.go

@@ -64,7 +64,7 @@ func (m *Server) handleFunctions() {
 	http.Handle(proto.ClientMetaPartition, m.handlerWithInterceptor())
 	http.Handle(proto.GetDataNodeTaskResponse, m.handlerWithInterceptor())
 	http.Handle(proto.GetMetaNodeTaskResponse, m.handlerWithInterceptor())
-	http.Handle(proto.AdminCreateMP, m.handlerWithInterceptor())
+	http.Handle(proto.AdminCreateMetaPartition, m.handlerWithInterceptor())
 	http.Handle(proto.ClientVolStat, m.handlerWithInterceptor())
 	http.Handle(proto.AddRaftNode, m.handlerWithInterceptor())
 	http.Handle(proto.RemoveRaftNode, m.handlerWithInterceptor())
@@ -165,7 +165,7 @@ func (m *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		m.loadMetaPartition(w, r)
 	case proto.AdminDecommissionMetaPartition:
 		m.decommissionMetaPartition(w, r)
-	case proto.AdminCreateMP:
+	case proto.AdminCreateMetaPartition:
 		m.createMetaPartition(w, r)
 	case proto.AdminAddMetaReplica:
 		m.addMetaReplica(w, r)

master/master_manager_test.go

@@ -61,6 +61,7 @@ func TestRaft(t *testing.T) {
 }
 
 func snapshotTest(t *testing.T) {
+	var err error
 	mdSnapshot, err := server.cluster.fsm.Snapshot()
 	if err != nil {
 		t.Error(err)
@@ -68,7 +69,12 @@ func snapshotTest(t *testing.T) {
 	}
 	t.Logf("snapshot apply index[%v]\n", mdSnapshot.ApplyIndex())
 	s := &Server{}
-	dbStore := raftstore.NewRocksDBStore("/export/chubaofs/raft2", LRUCacheSize, WriteBufferSize)
+
+	var dbStore *raftstore.RocksDBStore
+	dbStore, err = raftstore.NewRocksDBStore("/export/chubaofs/raft2", LRUCacheSize, WriteBufferSize)
+	if err != nil {
+		t.Fatalf("ioen rocks db store fail cause: %v", err)
+	}
 	fsm := &MetadataFsm{
 		rs:    server.fsm.rs,
 		store: dbStore,

master/meta_partition_test.go

@@ -37,7 +37,7 @@ func createMetaPartition(vol *Vol, t *testing.T) {
 	var start uint64
 	start = mp.Start + defaultMetaPartitionInodeIDStep
 	reqURL := fmt.Sprintf("%v%v?name=%v&start=%v",
-		hostAddr, proto.AdminCreateMP, vol.Name, start)
+		hostAddr, proto.AdminCreateMetaPartition, vol.Name, start)
 	fmt.Println(reqURL)
 	process(reqURL, t)
 	if start < mp.MaxInodeID {

master/metadata_fsm_op.go

@@ -116,6 +116,13 @@ type volValue struct {
 	Owner             string
 	FollowerRead      bool
 	Authenticate      bool
+	OSSAccessKey      string
+	OSSSecretKey      string
+}
+
+func (v *volValue) Bytes() (raw []byte, err error) {
+	raw, err = json.Marshal(v)
+	return
 }
 
 func newVolValue(vol *Vol) (vv *volValue) {
@@ -130,10 +137,20 @@ func newVolValue(vol *Vol) (vv *volValue) {
 		Owner:             vol.Owner,
 		FollowerRead:      vol.FollowerRead,
 		Authenticate:      vol.authenticate,
+		OSSAccessKey:      vol.OSSAccessKey,
+		OSSSecretKey:      vol.OSSSecretKey,
 	}
 	return
 }
 
+func newVolValueFromBytes(raw []byte) (*volValue, error) {
+	vv := &volValue{}
+	if err := json.Unmarshal(raw, vv); err != nil {
+		return nil, err
+	}
+	return vv, nil
+}
+
 type dataNodeValue struct {
 	ID        uint64
 	NodeSetID uint64
@@ -519,12 +536,12 @@ func (c *Cluster) loadVols() (err error) {
 		return err
 	}
 	for _, value := range result {
-		vv := &volValue{}
-		if err = json.Unmarshal(value, vv); err != nil {
+		var vv *volValue
+		if vv, err = newVolValueFromBytes(value); err != nil {
 			err = fmt.Errorf("action[loadVols],value:%v,unmarshal err:%v", string(value), err)
 			return err
 		}
-		vol := newVol(vv.ID, vv.Name, vv.Owner, vv.DataPartitionSize, vv.Capacity, vv.DpReplicaNum, vv.ReplicaNum, vv.FollowerRead, vv.Authenticate)
+		vol := newVolFromVolValue(vv)
 		vol.Status = vv.Status
 		c.putVol(vol)
 		log.LogInfof("action[loadVols],vol[%v]", vol.Name)

master/server.go

@@ -101,7 +101,10 @@ func (m *Server) Start(cfg *config.Config) (err error) {
 		log.LogError(err)
 		return
 	}
-	m.rocksDBStore = raftstore.NewRocksDBStore(m.storeDir, LRUCacheSize, WriteBufferSize)
+	if m.rocksDBStore, err = raftstore.NewRocksDBStore(m.storeDir, LRUCacheSize, WriteBufferSize); err != nil {
+		return
+	}
+
 	if err = m.createRaftServer(); err != nil {
 		log.LogError(errors.Stack(err))
 		return

master/vol.go

@@ -29,6 +29,8 @@ type Vol struct {
 	ID                 uint64
 	Name               string
 	Owner              string
+	OSSAccessKey       string
+	OSSSecretKey       string
 	dpReplicaNum       uint8
 	mpReplicaNum       uint8
 	Status             uint8
@@ -76,6 +78,29 @@ func newVol(id uint64, name, owner string, dpSize, capacity uint64, dpReplicaNum
 	return
 }
 
+func newVolFromVolValue(vv *volValue) (vol *Vol) {
+	vol = newVol(
+		vv.ID,
+		vv.Name,
+		vv.Owner,
+		vv.DataPartitionSize,
+		vv.Capacity,
+		vv.DpReplicaNum,
+		vv.ReplicaNum,
+		vv.FollowerRead,
+		vv.Authenticate)
+	// overwrite oss secure
+	vol.OSSAccessKey, vol.OSSSecretKey = vv.OSSAccessKey, vv.OSSSecretKey
+	vol.Status = vv.Status
+	return vol
+}
+
+func (vol *Vol) refreshOSSSecure() (key, secret string) {
+	vol.OSSAccessKey = util.RandomString(16, util.Numeric|util.LowerLetter|util.UpperLetter)
+	vol.OSSSecretKey = util.RandomString(32, util.Numeric|util.LowerLetter|util.UpperLetter)
+	return vol.OSSAccessKey, vol.OSSSecretKey
+}
+
 func (vol *Vol) addMetaPartition(mp *MetaPartition) {
 	vol.mpsLock.Lock()
 	defer vol.mpsLock.Unlock()
@@ -372,6 +397,8 @@ func (vol *Vol) totalUsedSpace() uint64 {
 
 func (vol *Vol) updateViewCache(c *Cluster) {
 	view := proto.NewVolView(vol.Name, vol.Status, vol.FollowerRead)
+	view.SetOwner(vol.Owner)
+	view.SetOSSSecure(vol.OSSAccessKey, vol.OSSSecretKey)
 	mpViews := vol.getMetaPartitionsView()
 	view.MetaPartitions = mpViews
 	mpViewsReply := newSuccessHTTPReply(mpViews)

master/vol_test.go

@@ -219,8 +219,12 @@ func TestConcurrentReadWriteDataPartitionMap(t *testing.T) {
 	mp2.Status = proto.ReadOnly
 	vol.addMetaPartition(mp2)
 	vol.updateViewCache(server.cluster)
+	for id := 0; id < 30000; id++ {
+		dp := newDataPartition(uint64(id), 3, name, volID)
+		vol.dataPartitions.put(dp)
+	}
 	go func() {
-		var id uint64
+		var id uint64 = 30000
 		for {
 			id++
 			dp := newDataPartition(id, 3, name, volID)

metanode/api_handler.go

@@ -29,7 +29,7 @@ import (
 type APIResponse struct {
 	Code int         `json:"code"`
 	Msg  string      `json:"msg"`
-	Data interface{} `json:"data, omitempty"`
+	Data interface{} `json:"data,omitempty"`
 }
 
 // NewAPIResponse returns a new API response.

metanode/const.go

@@ -77,7 +77,7 @@ type (
 
 	// Client -> MetaNode
 	EvictInodeReq = proto.EvictInodeRequest
-	// Client -> MetaNOde
+	// Client -> MetaNode
 	SetattrRequest = proto.SetAttrRequest
 )
 
@@ -102,6 +102,11 @@ const (
 	opFSMInternalDelExtentFile
 	opFSMInternalDelExtentCursor
 	opExtentFileSnapshot
+	opFSMSetXAttr
+	opFSMRemoveXAttr
+	opFSMCreateMultipart
+	opFSMRemoveMultipart
+	opFSMAppendMultipart
 )
 
 var (

metanode/extend.go

@@ -0,0 +1,169 @@
+// Copyright 2018 The Chubao Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package metanode
+
+import (
+	"bytes"
+	"encoding/binary"
+	"sync"
+
+	"github.com/chubaofs/chubaofs/util/btree"
+)
+
+type Extend struct {
+	inode   uint64
+	dataMap map[string][]byte
+	mu      sync.RWMutex
+}
+
+func NewExtend(inode uint64) *Extend {
+	return &Extend{inode: inode, dataMap: make(map[string][]byte)}
+}
+
+func NewExtendFromBytes(raw []byte) (*Extend, error) {
+	var err error
+	var buffer = bytes.NewBuffer(raw)
+	// decode inode
+	var inode uint64
+	if inode, err = binary.ReadUvarint(buffer); err != nil {
+		return nil, err
+	}
+	var ext = NewExtend(inode)
+	// decode number of key-value pairs
+	var numKV uint64
+	if numKV, err = binary.ReadUvarint(buffer); err != nil {
+		return nil, err
+	}
+	var readBytes = func() ([]byte, error) {
+		var length uint64
+		if length, err = binary.ReadUvarint(buffer); err != nil {
+			return nil, err
+		}
+		var data = make([]byte, length)
+		if _, err = buffer.Read(data); err != nil {
+			return nil, err
+		}
+		return data, nil
+	}
+	for i := 0; i < int(numKV); i++ {
+		var k, v []byte
+		if k, err = readBytes(); err != nil {
+			return nil, err
+		}
+		if v, err = readBytes(); err != nil {
+			return nil, err
+		}
+		ext.Put(k, v)
+	}
+	return ext, nil
+}
+
+func (e *Extend) Less(than btree.Item) bool {
+	ext, is := than.(*Extend)
+	return is && e.inode < ext.inode
+}
+
+func (e *Extend) Put(key, value []byte) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.dataMap[string(key)] = value
+}
+
+func (e *Extend) Get(key []byte) (value []byte, exist bool) {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	value, exist = e.dataMap[string(key)]
+	return
+}
+
+func (e *Extend) Remove(key []byte) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	delete(e.dataMap, string(key))
+	return
+}
+
+func (e *Extend) Range(visitor func(key, value []byte) bool) {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	for k, v := range e.dataMap {
+		if !visitor([]byte(k), v) {
+			return
+		}
+	}
+}
+
+func (e *Extend) Merge(o *Extend, override bool) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	o.Range(func(key, value []byte) bool {
+		strKey := string(key)
+		if _, exist := e.dataMap[strKey]; override || !exist {
+			copied := make([]byte, len(value))
+			copy(copied, value)
+			e.dataMap[strKey] = copied
+		}
+		return true
+	})
+}
+
+func (e *Extend) Copy() btree.Item {
+	newExt := NewExtend(e.inode)
+	for k, v := range e.dataMap {
+		newExt.dataMap[k] = v
+	}
+	return newExt
+}
+
+func (e *Extend) Bytes() ([]byte, error) {
+	var err error
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	var n int
+	var tmp = make([]byte, binary.MaxVarintLen64)
+	var buffer = bytes.NewBuffer(nil)
+	// write inode with varint codec
+	n = binary.PutUvarint(tmp, e.inode)
+	if _, err = buffer.Write(tmp[:n]); err != nil {
+		return nil, err
+	}
+	// write number of key-value pairs
+	n = binary.PutUvarint(tmp, uint64(len(e.dataMap)))
+	if _, err = buffer.Write(tmp[:n]); err != nil {
+		return nil, err
+	}
+	// write key-value paris
+	var writeBytes = func(val []byte) error {
+		n = binary.PutUvarint(tmp, uint64(len(val)))
+		if _, err = buffer.Write(tmp[:n]); err != nil {
+			return err
+		}
+		if _, err = buffer.Write(val); err != nil {
+			return err
+		}
+		return nil
+	}
+	for k, v := range e.dataMap {
+		// key
+		if err = writeBytes([]byte(k)); err != nil {
+			return nil, err
+		}
+		// value
+		if err = writeBytes(v); err != nil {
+			return nil, err
+		}
+	}
+	return buffer.Bytes(), nil
+}

metanode/extend_test.go

@@ -0,0 +1,57 @@
+// Copyright 2018 The Chubao Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package metanode
+
+import (
+	"math/rand"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/chubaofs/chubaofs/util"
+)
+
+func TestExtend_Bytes(t *testing.T) {
+	var err error
+	const numSamples = 100
+
+	var random = rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	extends := make([]*Extend, numSamples)
+	for i := 0; i < numSamples; i++ {
+		extend := NewExtend(random.Uint64())
+		extend.Put([]byte("msg"), []byte(util.RandomString(16, util.Numeric|util.LowerLetter|util.UpperLetter)))
+		extends[i] = extend
+	}
+
+	outputs := make([][]byte, numSamples)
+	for i := 0; i < numSamples; i++ {
+		if outputs[i], err = extends[i].Bytes(); err != nil {
+			t.Fatalf("encode extend to bytes fail cause: %v", err)
+		}
+	}
+
+	// validate result
+	for i := 0; i < numSamples; i++ {
+		var e *Extend
+		if e, err = NewExtendFromBytes(outputs[i]); err != nil {
+			t.Fatalf("decode bytes to extend fail cause: %v", err)
+		}
+		if !reflect.DeepEqual(e, extends[i]) {
+			t.Fatalf("result mismatch")
+		}
+	}
+
+}

metanode/manager.go

@@ -122,6 +122,32 @@ func (m *metadataManager) HandleMetadataOperation(conn net.Conn, p *Packet,
 		err = m.opMetaPartitionTryToLeader(conn, p, remoteAddr)
 	case proto.OpMetaBatchInodeGet:
 		err = m.opMetaBatchInodeGet(conn, p, remoteAddr)
+	case proto.OpMetaDeleteInode:
+		err = m.opMetaDeleteInode(conn, p, remoteAddr)
+	case proto.OpMetaBatchExtentsAdd:
+		err = m.opMetaBatchExtentsAdd(conn, p, remoteAddr)
+	// operations for extend attributes
+	case proto.OpMetaSetXAttr:
+		err = m.opMetaSetXAttr(conn, p, remoteAddr)
+	case proto.OpMetaGetXAttr:
+		err = m.opMetaGetXAttr(conn, p, remoteAddr)
+	case proto.OpMetaBatchGetXAttr:
+		err = m.opMetaBatchGetXAttr(conn, p, remoteAddr)
+	case proto.OpMetaRemoveXAttr:
+		err = m.opMetaRemoveXAttr(conn, p, remoteAddr)
+	case proto.OpMetaListXAttr:
+		err = m.opMetaListXAttr(conn, p, remoteAddr)
+	// operations for multipart session
+	case proto.OpCreateMultipart:
+		err = m.opCreateMultipart(conn, p, remoteAddr)
+	case proto.OpListMultiparts:
+		err = m.opListMultipart(conn, p, remoteAddr)
+	case proto.OpRemoveMultipart:
+		err = m.opRemoveMultipart(conn, p, remoteAddr)
+	case proto.OpAddMultipartPart:
+		err = m.opAppendMultipart(conn, p, remoteAddr)
+	case proto.OpGetMultipart:
+		err = m.opGetMultipart(conn, p, remoteAddr)
 	default:
 		err = fmt.Errorf("%s unknown Opcode: %d, reqId: %d", remoteAddr,
 			p.Opcode, p.GetReqID())

metanode/manager_op.go

@@ -117,7 +117,7 @@ func (m *metadataManager) opCreateMetaPartition(conn net.Conn, p *Packet,
 			" struct: %s", err.Error())
 		return
 	}
-	log.LogDebugf("%s [opCreateMetaPartition] [remoteAddr=%s]accept a from"+
+	log.LogDebugf("[opCreateMetaPartition] [remoteAddr=%s]accept a from"+
 		" master message: %v", remoteAddr, adminTask)
 	// create a new meta partition.
 	if err = m.createPartition(req.PartitionID, req.VolName,
@@ -363,7 +363,7 @@ func (m *metadataManager) opMetaEvictInode(conn net.Conn, p *Packet,
 	if err != nil {
 		p.PacketErrorWithBody(proto.OpNotExistErr, nil)
 		m.respondToClient(conn, p)
-		err = errors.NewErrorf("[opMetaEvictInode] req: %s, resp: %v", req, err.Error())
+		err = errors.NewErrorf("[opMetaEvictInode] req: %v, resp: %v", req, err.Error())
 		return
 	}
 	if !m.serveProxy(conn, mp, p) {
@@ -371,7 +371,7 @@ func (m *metadataManager) opMetaEvictInode(conn net.Conn, p *Packet,
 	}
 
 	if err = mp.EvictInode(req, p); err != nil {
-		err = errors.NewErrorf("[opMetaEvictInode] req: %s, resp: %v", req, err.Error())
+		err = errors.NewErrorf("[opMetaEvictInode] req: %v, resp: %v", req, err.Error())
 	}
 	m.respondToClient(conn, p)
 	log.LogDebugf("%s [opMetaEvictInode] req: %d - %v, resp: %v, body: %s",
@@ -838,3 +838,255 @@ errDeal:
 	m.respondToClient(conn, p)
 	return
 }
+
+func (m *metadataManager) opMetaDeleteInode(conn net.Conn, p *Packet,
+	remoteAddr string) (err error) {
+	req := &proto.DeleteInodeRequest{}
+	if err = json.Unmarshal(p.Data, req); err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		_ = m.respondToClient(conn, p)
+		return
+	}
+	mp, err := m.getPartition(req.PartitionId)
+	if err != nil {
+		p.PacketErrorWithBody(proto.OpNotExistErr, []byte(err.Error()))
+		_ = m.respondToClient(conn, p)
+		return
+	}
+	if !m.serveProxy(conn, mp, p) {
+		return
+	}
+	err = mp.DeleteInode(req, p)
+	_ = m.respondToClient(conn, p)
+	log.LogDebugf("%s [opMetaDeleteInode] req: %d - %v, resp: %v, body: %s",
+		remoteAddr, p.GetReqID(), req, p.GetResultMsg(), p.Data)
+	return
+}
+
+func (m *metadataManager) opMetaSetXAttr(conn net.Conn, p *Packet, remoteAddr string) (err error) {
+	req := &proto.SetXAttrRequest{}
+	if err = json.Unmarshal(p.Data, req); err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		_ = m.respondToClient(conn, p)
+		return
+	}
+	mp, err := m.getPartition(req.PartitionId)
+	if err != nil {
+		p.PacketErrorWithBody(proto.OpNotExistErr, []byte(err.Error()))
+		_ = m.respondToClient(conn, p)
+		return
+	}
+	if !m.serveProxy(conn, mp, p) {
+		return
+	}
+	err = mp.SetXAttr(req, p)
+	_ = m.respondToClient(conn, p)
+	log.LogDebugf("%s [opMetaSetXAttr] req: %d - %v, resp: %v, body: %s",
+		remoteAddr, p.GetReqID(), req, p.GetResultMsg(), p.Data)
+	return
+}
+
+func (m *metadataManager) opMetaGetXAttr(conn net.Conn, p *Packet, remoteAddr string) (err error) {
+	req := &proto.GetXAttrRequest{}
+	if err = json.Unmarshal(p.Data, req); err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		_ = m.respondToClient(conn, p)
+		return
+	}
+	mp, err := m.getPartition(req.PartitionId)
+	if err != nil {
+		p.PacketErrorWithBody(proto.OpNotExistErr, []byte(err.Error()))
+		_ = m.respondToClient(conn, p)
+		return
+	}
+	if !m.serveProxy(conn, mp, p) {
+		return
+	}
+	err = mp.GetXAttr(req, p)
+	_ = m.respondToClient(conn, p)
+	log.LogDebugf("%s [opMetaGetXAttr] req: %d - %v, resp: %v, body: %s",
+		remoteAddr, p.GetReqID(), req, p.GetResultMsg(), p.Data)
+	return
+}
+
+func (m *metadataManager) opMetaBatchGetXAttr(conn net.Conn, p *Packet, remoteAddr string) (err error) {
+	req := &proto.BatchGetXAttrRequest{}
+	if err = json.Unmarshal(p.Data, req); err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		_ = m.respondToClient(conn, p)
+		return
+	}
+	mp, err := m.getPartition(req.PartitionId)
+	if err != nil {
+		p.PacketErrorWithBody(proto.OpNotExistErr, []byte(err.Error()))
+		_ = m.respondToClient(conn, p)
+		return
+	}
+	if !m.serveProxy(conn, mp, p) {
+		return
+	}
+	err = mp.BatchGetXAttr(req, p)
+	_ = m.respondToClient(conn, p)
+	log.LogDebugf("%s [opMetaBatchGetXAttr req: %d - %v, resp: %v, body: %s",
+		remoteAddr, p.GetReqID(), req, p.GetResultMsg(), p.Data)
+	return
+}
+
+func (m *metadataManager) opMetaRemoveXAttr(conn net.Conn, p *Packet, remoteAddr string) (err error) {
+	req := &proto.RemoveXAttrRequest{}
+	if err = json.Unmarshal(p.Data, req); err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		_ = m.respondToClient(conn, p)
+		return
+	}
+	mp, err := m.getPartition(req.PartitionId)
+	if err != nil {
+		p.PacketErrorWithBody(proto.OpNotExistErr, []byte(err.Error()))
+		_ = m.respondToClient(conn, p)
+		return
+	}
+	if !m.serveProxy(conn, mp, p) {
+		return
+	}
+	err = mp.RemoveXAttr(req, p)
+	_ = m.respondToClient(conn, p)
+	log.LogDebugf("%s [opMetaGetXAttr] req: %d - %v, resp: %v, body: %s",
+		remoteAddr, p.GetReqID(), req, p.GetResultMsg(), p.Data)
+	return
+}
+
+func (m *metadataManager) opMetaListXAttr(conn net.Conn, p *Packet, remoteAddr string) (err error) {
+	req := &proto.ListXAttrRequest{}
+	if err = json.Unmarshal(p.Data, req); err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		_ = m.respondToClient(conn, p)
+		return
+	}
+	mp, err := m.getPartition(req.PartitionId)
+	if err != nil {
+		p.PacketErrorWithBody(proto.OpNotExistErr, []byte(err.Error()))
+		_ = m.respondToClient(conn, p)
+		return
+	}
+	if !m.serveProxy(conn, mp, p) {
+		return
+	}
+	err = mp.ListXAttr(req, p)
+	_ = m.respondToClient(conn, p)
+	log.LogDebugf("%s [opMetaGetXAttr] req: %d - %v, resp: %v, body: %s",
+		remoteAddr, p.GetReqID(), req, p.GetResultMsg(), p.Data)
+	return
+}
+
+func (m *metadataManager) opMetaBatchExtentsAdd(conn net.Conn, p *Packet, remoteAddr string) (err error) {
+	req := &proto.AppendExtentKeysRequest{}
+	if err = json.Unmarshal(p.Data, req); err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		_ = m.respondToClient(conn, p)
+		return
+	}
+	mp, err := m.getPartition(req.PartitionId)
+	if err != nil {
+		p.PacketErrorWithBody(proto.OpNotExistErr, []byte(err.Error()))
+		_ = m.respondToClient(conn, p)
+		return
+	}
+	if !m.serveProxy(conn, mp, p) {
+		return
+	}
+	err = mp.BatchExtentAppend(req, p)
+	_ = m.respondToClient(conn, p)
+	log.LogDebugf("%s [opMetaBatchExtentsAdd] req: %d - %v, resp: %v, body: %s",
+		remoteAddr, p.GetReqID(), req, p.GetResultMsg(), p.Data)
+	return
+}
+
+func (m *metadataManager) opCreateMultipart(conn net.Conn, p *Packet, remote string) (err error) {
+	req := &proto.CreateMultipartRequest{}
+	if err = json.Unmarshal(p.Data, req); err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		_ = m.respondToClient(conn, p)
+		return
+	}
+	mp, err := m.getPartition(req.PartitionId)
+	if err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		_ = m.respondToClient(conn, p)
+		return
+	}
+	if !m.serveProxy(conn, mp, p) {
+		return
+	}
+	err = mp.CreateMultipart(req, p)
+	_ = m.respondToClient(conn, p)
+	return
+}
+
+func (m *metadataManager) opRemoveMultipart(conn net.Conn, p *Packet, remote string) (err error) {
+	req := &proto.RemoveMultipartRequest{}
+	if err = json.Unmarshal(p.Data, req); err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		_ = m.respondToClient(conn, p)
+		return
+	}
+	mp, err := m.getPartition(req.PartitionId)
+	if err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		_ = m.respondToClient(conn, p)
+		return
+	}
+	if !m.serveProxy(conn, mp, p) {
+		return
+	}
+	err = mp.RemoveMultipart(req, p)
+	_ = m.respondToClient(conn, p)
+	return
+}
+
+func (m *metadataManager) opGetMultipart(conn net.Conn, p *Packet, remote string) (err error) {
+	req := &proto.GetMultipartRequest{}
+	if err = json.Unmarshal(p.Data, req); err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		_ = m.respondToClient(conn, p)
+		return
+	}
+	mp, err := m.getPartition(req.PartitionId)
+	if err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		_ = m.respondToClient(conn, p)
+		return
+	}
+	if !m.serveProxy(conn, mp, p) {
+		return
+	}
+	err = mp.GetMultipart(req, p)
+	_ = m.respondToClient(conn, p)
+	return
+}
+
+func (m *metadataManager) opAppendMultipart(conn net.Conn, p *Packet, remote string) (err error) {
+	defer func() {
+		if err != nil {
+			p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		}
+		_ = m.respondToClient(conn, p)
+	}()
+	req := &proto.AddMultipartPartRequest{}
+	if err = json.Unmarshal(p.Data, req); err != nil {
+		return
+	}
+	mp, err := m.getPartition(req.PartitionId)
+	if err != nil {
+		return
+	}
+	if !m.serveProxy(conn, mp, p) {
+		return
+	}
+	err = mp.AppendMultipart(req, p)
+	return
+}
+
+func (m *metadataManager) opListMultipart(conn net.Conn, p *Packet, remote string) (err error) {
+	// TODO: implement method 'opListMultipart'
+	return
+}

metanode/manager_resp.go

@@ -15,20 +15,15 @@
 package metanode
 
 import (
-	"encoding/json"
+	"github.com/chubaofs/chubaofs/proto"
 	"net"
 
 	"github.com/chubaofs/chubaofs/util/errors"
 	"github.com/chubaofs/chubaofs/util/log"
 )
 
-const (
-	masterResponsePath = "/metaNode/response" // Method: 'POST',
-	// ContentType: 'application/json'
-)
-
 // Reply operation results to the master.
-func (m *metadataManager) respondToMaster(data interface{}) (err error) {
+func (m *metadataManager) respondToMaster(task *proto.AdminTask) (err error) {
 	// handle panic
 	defer func() {
 		if r := recover(); r != nil {
@@ -40,14 +35,7 @@ func (m *metadataManager) respondToMaster(data interface{}) (err error) {
 			}
 		}
 	}()
-
-	// process data and send reply though http specified remote address.
-	jsonBytes, err := json.Marshal(data)
-	if err != nil {
-		return
-	}
-	_, err = masterHelper.Request("POST", masterResponsePath, nil, jsonBytes)
-	if err != nil {
+	if err = masterClient.NodeAPI().ResponseMetaNodeTask(task); err != nil {
 		err = errors.Trace(err, "try respondToMaster failed")
 	}
 	return

metanode/metanode.go

@@ -15,7 +15,8 @@
 package metanode
 
 import (
-	"encoding/json"
+	"github.com/chubaofs/chubaofs/master"
+	masterSDK "github.com/chubaofs/chubaofs/sdk/master"
 	"os"
 	"strings"
 	"sync"
@@ -23,7 +24,6 @@ import (
 	"time"
 
 	"fmt"
-	"net/http"
 	"strconv"
 
 	"github.com/chubaofs/chubaofs/proto"
@@ -37,7 +37,7 @@ import (
 
 var (
 	clusterInfo    *proto.ClusterInfo
-	masterHelper   util.MasterHelper
+	masterClient   *masterSDK.MasterClient
 	configTotalMem uint64
 )
 
@@ -97,30 +97,20 @@ type MetaNodeInfo struct {
 }
 
 func (m *MetaNode) checkLocalPartitionMatchWithMaster() (err error) {
-	params := make(map[string]string)
-	params["addr"] = m.localAddr + ":" + m.listen
-	var data interface{}
+	var metaNodeInfo *master.MetaNode
 	for i := 0; i < 3; i++ {
-		data, err = masterHelper.Request(http.MethodGet, proto.GetMetaNode, params, nil)
-		if err != nil {
-			log.LogErrorf("checkLocalPartitionMatchWithMaster error %v", err)
+		if metaNodeInfo, err = masterClient.NodeAPI().GetMetaNode(fmt.Sprintf("%s:%s", m.localAddr, m.listen)); err != nil {
+			log.LogErrorf("checkLocalPartitionMatchWithMaster: get MetaNode info fail: err(%v)", err)
 			continue
 		}
 		break
 	}
 
-	minfo := new(MetaNodeInfo)
-	if err = json.Unmarshal(data.([]byte), minfo); err != nil {
-		err = fmt.Errorf("checkLocalPartitionMatchWithMaster jsonUnmarsh failed %v", err)
-		log.LogErrorf(err.Error())
-		return
-	}
-
-	if len(minfo.PersistenceMetaPartitions) == 0 {
+	if len(metaNodeInfo.PersistenceMetaPartitions) == 0 {
 		return
 	}
 	lackPartitions := make([]uint64, 0)
-	for _, partitionID := range minfo.PersistenceMetaPartitions {
+	for _, partitionID := range metaNodeInfo.PersistenceMetaPartitions {
 		_, err := m.metadataManager.GetPartition(partitionID)
 		if err != nil {
 			lackPartitions = append(lackPartitions, partitionID)
@@ -226,14 +216,12 @@ func (m *MetaNode) parseConfig(cfg *config.Config) (err error) {
 	log.LogInfof("[parseConfig] load raftHeartbeatPort[%v].", m.raftHeartbeatPort)
 	log.LogInfof("[parseConfig] load raftReplicatePort[%v].", m.raftReplicatePort)
 
-	addrs := cfg.GetArray(cfgMasterAddr)
-	if len(addrs) == 0 {
-		addrs = cfg.GetArray(cfgMasterAddrs)
-	}
-	masterHelper = util.NewMasterHelper()
+	addrs := cfg.GetArray(cfgMasterAddrs)
+	masters := make([]string, 0, len(addrs))
 	for _, addr := range addrs {
-		masterHelper.AddNode(addr.(string))
+		masters = append(masters, addr.(string))
 	}
+	masterClient = masterSDK.NewMasterClient(masters, false)
 	err = m.validConfig()
 	return
 }
@@ -249,7 +237,7 @@ func (m *MetaNode) validConfig() (err error) {
 	if m.raftDir == "" {
 		m.raftDir = defaultRaftDir
 	}
-	if len(masterHelper.Nodes()) == 0 {
+	if len(masterClient.Nodes()) == 0 {
 		err = errors.New("master address list is empty")
 		return
 	}
@@ -283,10 +271,10 @@ func (m *MetaNode) stopMetaManager() {
 
 func (m *MetaNode) register() (err error) {
 	step := 0
-	reqParam := make(map[string]string)
+	var nodeAddress string
 	for {
 		if step < 1 {
-			clusterInfo, err = getClusterInfo()
+			clusterInfo, err = getClientIP()
 			if err != nil {
 				log.LogErrorf("[register] %s", err.Error())
 				continue
@@ -295,28 +283,16 @@ func (m *MetaNode) register() (err error) {
 				m.localAddr = clusterInfo.Ip
 			}
 			m.clusterId = clusterInfo.Cluster
-			reqParam["addr"] = m.localAddr + ":" + m.listen
+			nodeAddress = m.localAddr + ":" + m.listen
 			step++
 		}
-		var respBody []byte
-		respBody, err = masterHelper.Request("POST", proto.AddMetaNode, reqParam, nil)
-		if err != nil {
-			log.LogErrorf("[register] %s", err.Error())
-			time.Sleep(3 * time.Second)
-			continue
-		}
-		nodeIDStr := strings.TrimSpace(string(respBody))
-		if nodeIDStr == "" {
-			log.LogErrorf("[register] master respond empty body")
-			time.Sleep(3 * time.Second)
-			continue
-		}
-		m.nodeId, err = strconv.ParseUint(nodeIDStr, 10, 64)
-		if err != nil {
-			log.LogErrorf("[register] parse to nodeID: %s", err.Error())
+		var nodeID uint64
+		if nodeID, err = masterClient.NodeAPI().AddMetaNode(nodeAddress); err != nil {
+			log.LogErrorf("register: register to master fail: address(%v) err(%s)", nodeAddress, err)
 			time.Sleep(3 * time.Second)
 			continue
 		}
+		m.nodeId = nodeID
 		return
 	}
 }
@@ -326,14 +302,7 @@ func NewServer() *MetaNode {
 	return &MetaNode{}
 }
 
-func getClusterInfo() (*proto.ClusterInfo, error) {
-	respBody, err := masterHelper.Request("GET", proto.AdminGetIP, nil, nil)
-	if err != nil {
-		return nil, err
-	}
-	cInfo := &proto.ClusterInfo{}
-	if err = json.Unmarshal(respBody, cInfo); err != nil {
-		return nil, err
-	}
-	return cInfo, nil
+func getClientIP() (ci *proto.ClusterInfo, err error) {
+	ci, err = masterClient.AdminAPI().GetClusterInfo()
+	return
 }

metanode/multipart.go

@@ -0,0 +1,337 @@
+// Copyright 2018 The Chubao Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package metanode
+
+import (
+	"bytes"
+	"encoding/binary"
+	"github.com/chubaofs/chubaofs/util/btree"
+	"sort"
+	"sync"
+	"time"
+)
+
+// Part defined necessary fields for multipart part management.
+type Part struct {
+	ID         uint16
+	UploadTime time.Time
+	MD5        string
+	Size       uint64
+	Inode      uint64
+}
+
+func (m Part) Bytes() ([]byte, error) {
+	var err error
+	var buffer = bytes.NewBuffer(nil)
+	var tmp = make([]byte, binary.MaxVarintLen64)
+	var n int
+	// ID
+	n = binary.PutUvarint(tmp, uint64(m.ID))
+	if _, err = buffer.Write(tmp[:n]); err != nil {
+		return nil, err
+	}
+	// upload time
+	n = binary.PutVarint(tmp, m.UploadTime.UnixNano())
+	if _, err = buffer.Write(tmp[:n]); err != nil {
+		return nil, err
+	}
+	// MD5
+	n = binary.PutUvarint(tmp, uint64(len(m.MD5)))
+	if _, err = buffer.Write(tmp[:n]); err != nil {
+		return nil, err
+	}
+	if _, err = buffer.WriteString(m.MD5); err != nil {
+		return nil, err
+	}
+	// size
+	n = binary.PutUvarint(tmp, m.Size)
+	if _, err = buffer.Write(tmp[:n]); err != nil {
+		return nil, err
+	}
+	// inode
+	n = binary.PutUvarint(tmp, m.Inode)
+	if _, err = buffer.Write(tmp[:n]); err != nil {
+		return nil, err
+	}
+	return buffer.Bytes(), nil
+}
+
+func PartFromBytes(raw []byte) *Part {
+	var offset, n int
+	// decode ID
+	var u64ID uint64
+	u64ID, n = binary.Uvarint(raw)
+	offset += n
+	// decode upload time
+	var uploadTimeI64 int64
+	uploadTimeI64, n = binary.Varint(raw[offset:])
+	offset += n
+	// decode MD5
+	var md5Len uint64
+	md5Len, n = binary.Uvarint(raw[offset:])
+	offset += n
+	var md5Content = string(raw[offset : offset+int(md5Len)])
+	offset += int(md5Len)
+	// decode size
+	var sizeU64 uint64
+	sizeU64, n = binary.Uvarint(raw[offset:])
+	offset += n
+	// decode inode
+	var inode uint64
+	inode, n = binary.Uvarint(raw[offset:])
+
+	var muPart = &Part{
+		ID:         uint16(u64ID),
+		UploadTime: time.Unix(0, uploadTimeI64),
+		MD5:        md5Content,
+		Size:       sizeU64,
+		Inode:      inode,
+	}
+	return muPart
+}
+
+type Parts []*Part
+
+func (m Parts) Len() int {
+	return len(m)
+}
+
+func (m Parts) Less(i, j int) bool {
+	return m[i].ID < m[j].ID
+}
+
+func (m Parts) Swap(i, j int) {
+	m[i], m[j] = m[j], m[i]
+}
+
+func (m Parts) Sort() {
+	sort.Sort(m)
+}
+
+func (m *Parts) Hash(part *Part) (has bool) {
+	i := sort.Search(len(*m), func(i int) bool {
+		return (*m)[i].ID >= part.ID
+	})
+	has = i < len(*m) && (*m)[i].ID == part.ID
+	return
+}
+
+func (m *Parts) Insert(part *Part, replace bool) (success bool) {
+	i := sort.Search(len(*m), func(i int) bool {
+		return (*m)[i].ID >= part.ID
+	})
+	if i < len(*m) && (*m)[i].ID == part.ID {
+		if replace {
+			(*m)[i] = part
+			return true
+		}
+		return false
+	}
+	*m = append(*m, part)
+	m.Sort()
+	return true
+}
+
+func (m *Parts) Remove(id uint16) {
+	i := sort.Search(len(*m), func(i int) bool {
+		return (*m)[i].ID >= id
+	})
+	if i < len(*m) && (*m)[i].ID == id {
+		if len(*m) > i+1 {
+			*m = append((*m)[:i], (*m)[i+1:]...)
+		} else {
+			*m = (*m)[:i]
+		}
+	}
+}
+
+func (m Parts) Search(id uint16) (part *Part, found bool) {
+	i := sort.Search(len(m), func(i int) bool {
+		return m[i].ID >= id
+	})
+	if i < len(m) && m[i].ID == id {
+		return m[i], true
+	}
+	return nil, false
+}
+
+func (m Parts) Bytes() ([]byte, error) {
+	var err error
+	var n int
+	var buffer = bytes.NewBuffer(nil)
+	var tmp = make([]byte, binary.MaxVarintLen64)
+	n = binary.PutUvarint(tmp, uint64(len(m)))
+	if _, err = buffer.Write(tmp[:n]); err != nil {
+		return nil, err
+	}
+	var marshaled []byte
+	for _, p := range m {
+		marshaled, err = p.Bytes()
+		if err != nil {
+			return nil, err
+		}
+		// write part length
+		n = binary.PutUvarint(tmp, uint64(len(marshaled)))
+		if _, err = buffer.Write(tmp[:n]); err != nil {
+			return nil, err
+		}
+		// write part bytes
+		if _, err = buffer.Write(marshaled); err != nil {
+			return nil, err
+		}
+	}
+	return buffer.Bytes(), nil
+}
+
+func PartsFromBytes(raw []byte) Parts {
+	var offset, n int
+	var numPartsU64 uint64
+	numPartsU64, n = binary.Uvarint(raw)
+	offset += n
+	var muParts = make([]*Part, int(numPartsU64))
+	for i := 0; i < int(numPartsU64); i++ {
+		var partLengthU64 uint64
+		partLengthU64, n = binary.Uvarint(raw[offset:])
+		offset += n
+		part := PartFromBytes(raw[offset : offset+int(partLengthU64)])
+		muParts[i] = part
+		offset += int(partLengthU64)
+	}
+	return muParts
+}
+
+// Multipart defined necessary fields for multipart session management.
+type Multipart struct {
+	// session fields
+	id       string
+	key      string
+	initTime time.Time
+	parts    Parts
+
+	mu sync.RWMutex
+}
+
+func (m *Multipart) Less(than btree.Item) bool {
+	thanMultipart, is := than.(*Multipart)
+	return is && m.id < thanMultipart.id
+}
+
+func (m *Multipart) Copy() btree.Item {
+	return &Multipart{
+		id:       m.id,
+		key:      m.key,
+		initTime: m.initTime,
+		parts:    append(Parts{}, m.parts...),
+	}
+}
+
+func (m *Multipart) ID() string {
+	return m.id
+}
+
+func (m *Multipart) InsertPart(part *Part, replace bool) (success bool) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.parts == nil {
+		m.parts = PartsFromBytes(nil)
+	}
+	success = m.parts.Insert(part, replace)
+	return
+}
+
+func (m *Multipart) Parts() []*Part {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return append([]*Part{}, m.parts...)
+}
+
+func (m *Multipart) Bytes() ([]byte, error) {
+	var n int
+	var buffer = bytes.NewBuffer(nil)
+	var err error
+	tmp := make([]byte, binary.MaxVarintLen64)
+	// marshal id
+	var marshalStr = func(src string) error {
+		n = binary.PutUvarint(tmp, uint64(len(src)))
+		if _, err = buffer.Write(tmp[:n]); err != nil {
+			return err
+		}
+		if _, err = buffer.WriteString(src); err != nil {
+			return err
+		}
+		return nil
+	}
+	// marshal id
+	if err = marshalStr(m.id); err != nil {
+		return nil, err
+	}
+	// marshal key
+	if err = marshalStr(m.key); err != nil {
+		return nil, err
+	}
+	// marshal init time
+	n = binary.PutVarint(tmp, m.initTime.UnixNano())
+	if _, err = buffer.Write(tmp[:n]); err != nil {
+		return nil, err
+	}
+	// marshal parts
+	var marshaledParts []byte
+	if marshaledParts, err = m.parts.Bytes(); err != nil {
+		return nil, err
+	}
+	n = binary.PutUvarint(tmp, uint64(len(marshaledParts)))
+	if _, err = buffer.Write(tmp[:n]); err != nil {
+		return nil, err
+	}
+	if _, err = buffer.Write(marshaledParts); err != nil {
+		return nil, err
+	}
+	return buffer.Bytes(), nil
+}
+
+func MultipartFromBytes(raw []byte) *Multipart {
+	var unmarshalStr = func(data []byte) (string, int) {
+		var n int
+		var lengthU64 uint64
+		lengthU64, n = binary.Uvarint(data)
+		return string(data[n : n+int(lengthU64)]), n + int(lengthU64)
+	}
+	var offset, n int
+	// decode id
+	var id string
+	id, n = unmarshalStr(raw)
+	offset += n
+	// decode key
+	var key string
+	key, n = unmarshalStr(raw[offset:])
+	offset += n
+	// decode init time
+	var initTimeI64 int64
+	initTimeI64, n = binary.Varint(raw[offset:])
+	offset += n
+	// decode parts
+	var partsLengthU64 uint64
+	partsLengthU64, n = binary.Uvarint(raw[offset:])
+	offset += n
+	var parts = PartsFromBytes(raw[offset : offset+int(partsLengthU64)])
+
+	var muSession = &Multipart{
+		id:       id,
+		key:      key,
+		initTime: time.Unix(0, initTimeI64),
+		parts:    parts,
+	}
+	return muSession
+}

metanode/multipart_test.go

@@ -0,0 +1,151 @@
+// Copyright 2018 The Chubao Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package metanode
+
+import (
+	"math/rand"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/chubaofs/chubaofs/util"
+)
+
+func TestMUPart_Bytes(t *testing.T) {
+	var (
+		id         uint16 = 1
+		uploadTime        = time.Now().Local()
+		md5               = util.RandomString(16, util.UpperLetter|util.Numeric)
+		size       uint64 = 65536
+		inode      uint64 = 12345
+		err        error
+	)
+	part1 := &Part{
+		ID:         id,
+		UploadTime: uploadTime,
+		MD5:        md5,
+		Size:       size,
+		Inode:      inode,
+	}
+	var partBytes []byte
+	if partBytes, err = part1.Bytes(); err != nil {
+		t.Fatalf("get bytes of part fail cause: %v", err)
+	}
+	part2 := PartFromBytes(partBytes)
+	if !reflect.DeepEqual(part1, part2) {
+		t.Fatalf("result mismatch:\n\tpart1:%v\n\tpart2:%v", part1, part2)
+	}
+	t.Logf("encoded length: %v", len(partBytes))
+}
+
+func TestMUParts_Bytes(t *testing.T) {
+	var err error
+	var random = rand.New(rand.NewSource(time.Now().UnixNano()))
+	var parts1 = PartsFromBytes(nil)
+	for i := 0; i < 100; i++ {
+		part := &Part{
+			ID:         uint16(i),
+			UploadTime: time.Now().Local(),
+			MD5:        util.RandomString(16, util.UpperLetter|util.Numeric),
+			Size:       random.Uint64(),
+			Inode:      random.Uint64(),
+		}
+		parts1.Insert(part, false)
+	}
+	var partsBytes []byte
+	partsBytes, err = parts1.Bytes()
+	if partsBytes, err = parts1.Bytes(); err != nil {
+		t.Fatalf("get bytes of part fail cause: %v", err)
+	}
+	parts2 := PartsFromBytes(partsBytes)
+	if !reflect.DeepEqual(parts1, parts2) {
+		t.Fatalf("result mismatch:\n\tpart1:%v\n\tpart2:%v", parts1, parts2)
+	}
+	t.Logf("encoded length: %v", len(partsBytes))
+}
+
+func TestMUParts_Modify(t *testing.T) {
+	var random = rand.New(rand.NewSource(time.Now().UnixNano()))
+	var parts = PartsFromBytes(nil)
+	for i := 0; i < 100; i++ {
+		part := &Part{
+			ID:         uint16(i),
+			UploadTime: time.Now().Local(),
+			MD5:        util.RandomString(16, util.UpperLetter|util.Numeric),
+			Size:       random.Uint64(),
+			Inode:      random.Uint64(),
+		}
+		parts.Insert(part, false)
+	}
+	if parts.Len() != 100 {
+		t.Fatalf("parts length mismatch: except 100 actual %v", parts.Len())
+	}
+	// validate before modify
+	if _, found := parts.Search(0); !found {
+		t.Fatalf("part id[0] not found before modify")
+	}
+	if _, found := parts.Search(50); !found {
+		t.Fatalf("part id[50] not found before modify")
+	}
+	if _, found := parts.Search(99); !found {
+		t.Fatalf("part id[99] not found before modify")
+	}
+	// modify
+	parts.Remove(0)
+	parts.Remove(50)
+	parts.Remove(99)
+	if parts.Len() != 97 {
+		t.Fatalf("parts length mismatch: expect 97 actual %v", parts.Len())
+	}
+	// validate after modify
+	if _, found := parts.Search(0); found {
+		t.Fatalf("part id[0] not found before modify")
+	}
+	if _, found := parts.Search(50); found {
+		t.Fatalf("part id[50] not found before modify")
+	}
+	if _, found := parts.Search(99); found {
+		t.Fatalf("part id[99] not found before modify")
+	}
+}
+
+func TestMUSession_Bytes(t *testing.T) {
+	var err error
+	var random = rand.New(rand.NewSource(time.Now().UnixNano()))
+	var session1 = MultipartFromBytes(nil)
+	for i := 0; i < 100; i++ {
+		id := uint16(i)
+		md5 := util.RandomString(16, util.UpperLetter|util.Numeric)
+		size := random.Uint64()
+		inode := random.Uint64()
+		session1.InsertPart(&Part{
+			ID:         id,
+			MD5:        md5,
+			Size:       size,
+			Inode:      inode,
+			UploadTime: time.Now().Local(),
+		}, false)
+	}
+	var sessionBytes []byte
+	sessionBytes, err = session1.Bytes()
+	if err != nil {
+		t.Fatalf("encode session to bytes fail caue: %v", err)
+	}
+	session2 := MultipartFromBytes(sessionBytes)
+	if !reflect.DeepEqual(session1, session2) {
+		t.Fatalf("result mismatch:\n\tsession1:%v\n\tsession2:%v", session1, session2)
+	}
+	t.Logf("encoded session length: %v", len(sessionBytes))
+}

metanode/partition.go

@@ -15,6 +15,7 @@
 package metanode
 
 import (
+	"bytes"
 	"encoding/json"
 	"sort"
 	"strconv"
@@ -113,6 +114,15 @@ type OpInode interface {
 	EvictInode(req *EvictInodeReq, p *Packet) (err error)
 	SetAttr(reqData []byte, p *Packet) (err error)
 	GetInodeTree() *BTree
+	DeleteInode(req *proto.DeleteInodeRequest, p *Packet) (err error)
+}
+
+type OpExtend interface {
+	SetXAttr(req *proto.SetXAttrRequest, p *Packet) (err error)
+	GetXAttr(req *proto.GetXAttrRequest, p *Packet) (err error)
+	BatchGetXAttr(req *proto.BatchGetXAttrRequest, p *Packet) (err error)
+	RemoveXAttr(req *proto.RemoveXAttrRequest, p *Packet) (err error)
+	ListXAttr(req *proto.ListXAttrRequest, p *Packet) (err error)
 }
 
 // OpDentry defines the interface for the dentry operations.
@@ -130,6 +140,14 @@ type OpExtent interface {
 	ExtentAppend(req *proto.AppendExtentKeyRequest, p *Packet) (err error)
 	ExtentsList(req *proto.GetExtentsRequest, p *Packet) (err error)
 	ExtentsTruncate(req *ExtentsTruncateReq, p *Packet) (err error)
+	BatchExtentAppend(req *proto.AppendExtentKeysRequest, p *Packet) (err error)
+}
+
+type OpMultipart interface {
+	GetMultipart(req *proto.GetMultipartRequest, p *Packet) (err error)
+	CreateMultipart(req *proto.CreateMultipartRequest, p *Packet) (err error)
+	AppendMultipart(req *proto.AddMultipartPartRequest, p *Packet) (err error)
+	RemoveMultipart(req *proto.RemoveMultipartRequest, p *Packet) (err error)
 }
 
 // OpMeta defines the interface for the metadata operations.
@@ -138,6 +156,8 @@ type OpMeta interface {
 	OpDentry
 	OpExtent
 	OpPartition
+	OpExtend
+	OpMultipart
 }
 
 // OpPartition defines the interface for the partition operations.
@@ -175,6 +195,8 @@ type metaPartition struct {
 	applyID       uint64 // Inode/Dentry max applyID, this index will be update after restoring from the dumped data.
 	dentryTree    *BTree
 	inodeTree     *BTree // btree for inodes
+	extendTree    *BTree // btree for inode extend (XAttr) management
+	multipartTree *BTree // collection for multipart management
 	raftPartition raftstore.Partition
 	stopC         chan bool
 	storeChan     chan *storeMsg
@@ -333,16 +355,18 @@ func (mp *metaPartition) getRaftPort() (heartbeat, replica int, err error) {
 // NewMetaPartition creates a new meta partition with the specified configuration.
 func NewMetaPartition(conf *MetaPartitionConfig, manager *metadataManager) MetaPartition {
 	mp := &metaPartition{
-		config:     conf,
-		dentryTree: NewBtree(),
-		inodeTree:  NewBtree(),
-		stopC:      make(chan bool),
-		storeChan:  make(chan *storeMsg, 5),
-		freeList:   newFreeList(),
-		extDelCh:   make(chan BtreeItem, 10000),
-		extReset:   make(chan struct{}),
-		vol:        NewVol(),
-		manager:    manager,
+		config:        conf,
+		dentryTree:    NewBtree(),
+		inodeTree:     NewBtree(),
+		extendTree:    NewBtree(),
+		multipartTree: NewBtree(),
+		stopC:         make(chan bool),
+		storeChan:     make(chan *storeMsg, 5),
+		freeList:      newFreeList(),
+		extDelCh:      make(chan BtreeItem, 10000),
+		extReset:      make(chan struct{}),
+		vol:           NewVol(),
+		manager:       manager,
 	}
 	return mp
 }
@@ -393,14 +417,20 @@ func (mp *metaPartition) load() (err error) {
 	if err = mp.loadMetadata(); err != nil {
 		return
 	}
-	loadSnapshotDir := path.Join(mp.config.RootDir, snapshotDir)
-	if err = mp.loadInode(loadSnapshotDir); err != nil {
+	snapshotPath := path.Join(mp.config.RootDir, snapshotDir)
+	if err = mp.loadInode(snapshotPath); err != nil {
 		return
 	}
-	if err = mp.loadDentry(loadSnapshotDir); err != nil {
+	if err = mp.loadDentry(snapshotPath); err != nil {
 		return
 	}
-	err = mp.loadApplyID(loadSnapshotDir)
+	if err = mp.loadExtend(snapshotPath); err != nil {
+		return
+	}
+	if err = mp.loadMultipart(snapshotPath); err != nil {
+		return
+	}
+	err = mp.loadApplyID(snapshotPath)
 	return
 }
 
@@ -421,21 +451,28 @@ func (mp *metaPartition) store(sm *storeMsg) (err error) {
 			os.RemoveAll(tmpDir)
 		}
 	}()
-	var (
-		inoCRC, denCRC uint32
-	)
-	if inoCRC, err = mp.storeInode(tmpDir, sm); err != nil {
-		return
+	var crcBuffer = bytes.NewBuffer(make([]byte, 0, 16))
+	var storeFuncs = []func(dir string, sm *storeMsg) (uint32, error){
+		mp.storeInode,
+		mp.storeDentry,
+		mp.storeExtend,
+		mp.storeMultipart,
 	}
-	if denCRC, err = mp.storeDentry(tmpDir, sm); err != nil {
-		return
+	for _, storeFunc := range storeFuncs {
+		var crc uint32
+		if crc, err = storeFunc(tmpDir, sm); err != nil {
+			return
+		}
+		if crcBuffer.Len() != 0 {
+			crcBuffer.WriteString(" ")
+		}
+		crcBuffer.WriteString(fmt.Sprintf("%d", crc))
 	}
 	if err = mp.storeApplyID(tmpDir, sm); err != nil {
 		return
 	}
 	// write crc to file
-	if err = ioutil.WriteFile(path.Join(tmpDir, SnapshotSign),
-		[]byte(fmt.Sprintf("%d %d", inoCRC, denCRC)), 0775); err != nil {
+	if err = ioutil.WriteFile(path.Join(tmpDir, SnapshotSign), crcBuffer.Bytes(), 0775); err != nil {
 		return
 	}
 	snapshotDir := path.Join(mp.config.RootDir, snapshotDir)
@@ -457,7 +494,7 @@ func (mp *metaPartition) store(sm *storeMsg) (err error) {
 	err = nil
 
 	if err = os.Rename(tmpDir, snapshotDir); err != nil {
-		os.Rename(backupDir, snapshotDir)
+		_ = os.Rename(backupDir, snapshotDir)
 		return
 	}
 	err = os.RemoveAll(backupDir)
@@ -589,9 +626,15 @@ func (mp *metaPartition) Reset() (err error) {
 	mp.dentryTree.Reset()
 	mp.config.Cursor = 0
 	mp.applyID = 0
-	// delete ino/dentry applyID file
-	mp.deleteApplyFile()
-	mp.deleteDentryFile()
-	mp.deleteInodeFile()
+
+	// remove files
+	filenames := []string{applyIDFile, dentryFile, inodeFile, extendFile, multipartFile}
+	for _, filename := range filenames {
+		filepath := path.Join(mp.config.RootDir, filename)
+		if err = os.Remove(filepath); err != nil {
+			return
+		}
+	}
+
 	return
 }

metanode/partition_free_list.go

@@ -15,7 +15,6 @@
 package metanode
 
 import (
-	"encoding/json"
 	"fmt"
 	"github.com/chubaofs/chubaofs/proto"
 	"github.com/chubaofs/chubaofs/util/errors"
@@ -52,25 +51,34 @@ func (mp *metaPartition) startFreeList() (err error) {
 
 func (mp *metaPartition) updateVolWorker() {
 	t := time.NewTicker(UpdateVolTicket)
-	reqURL := fmt.Sprintf("%s?name=%s", proto.ClientDataPartitions, mp.config.VolName)
+	var convert = func(view *proto.DataPartitionsView) *DataPartitionsView {
+		newView := &DataPartitionsView{
+			DataPartitions: make([]*DataPartition, len(view.DataPartitions)),
+		}
+		for i:=0; i<len(view.DataPartitions); i++ {
+			newView.DataPartitions[i] = &DataPartition{
+				PartitionID: view.DataPartitions[i].PartitionID,
+				Status: view.DataPartitions[i].Status,
+				Hosts: view.DataPartitions[i].Hosts,
+				ReplicaNum: view.DataPartitions[i].ReplicaNum,
+			}
+		}
+		return newView
+	}
 	for {
 		select {
 		case <-mp.stopC:
 			t.Stop()
 			return
 		case <-t.C:
-			respBody, err := masterHelper.Request("GET", reqURL, nil, nil)
+			volName := mp.config.VolName
+			dataView, err := masterClient.ClientAPI().GetDataPartitions(volName)
 			if err != nil {
-				log.LogErrorf("[updateVol] %s", err.Error())
+				log.LogErrorf("updateVolWorker: get data partitions view fail: volume(%v) err(%v)",
+					volName, err)
 				break
 			}
-
-			dataView := NewDataPartitionsView()
-			if err = json.Unmarshal(respBody, dataView); err != nil {
-				log.LogErrorf("[updateVol] %s", err.Error())
-				break
-			}
-			mp.vol.UpdatePartitions(dataView)
+			mp.vol.UpdatePartitions(convert(dataView))
 		}
 	}
 }

metanode/partition_fsm.go

@@ -21,15 +21,15 @@ import (
 	"io"
 	"sync/atomic"
 
+	"io/ioutil"
+	"os"
+	"path"
+
 	"github.com/chubaofs/chubaofs/proto"
 	"github.com/chubaofs/chubaofs/util/exporter"
 	"github.com/chubaofs/chubaofs/util/log"
 	"github.com/tiglabs/raft"
 	raftproto "github.com/tiglabs/raft/proto"
-	"io/ioutil"
-	"os"
-	"path"
-	"strings"
 )
 
 // Apply applies the given operational commands.
@@ -120,13 +120,16 @@ func (mp *metaPartition) Apply(command []byte, index uint64) (resp interface{},
 	case opFSMStoreTick:
 		inodeTree := mp.getInodeTree()
 		dentryTree := mp.getDentryTree()
+		extendTree := mp.extendTree.GetTree()
+		multipartTree := mp.multipartTree.GetTree()
 		msg := &storeMsg{
-			command:    opFSMStoreTick,
-			applyIndex: index,
-			inodeTree:  inodeTree,
-			dentryTree: dentryTree,
+			command:       opFSMStoreTick,
+			applyIndex:    index,
+			inodeTree:     inodeTree,
+			dentryTree:    dentryTree,
+			extendTree:    extendTree,
+			multipartTree: multipartTree,
 		}
-
 		mp.storeChan <- msg
 	case opFSMInternalDeleteInode:
 		err = mp.internalDelete(msg.V)
@@ -134,6 +137,30 @@ func (mp *metaPartition) Apply(command []byte, index uint64) (resp interface{},
 		err = mp.delOldExtentFile(msg.V)
 	case opFSMInternalDelExtentCursor:
 		err = mp.setExtentDeleteFileCursor(msg.V)
+	case opFSMSetXAttr:
+		var extend *Extend
+		if extend, err = NewExtendFromBytes(msg.V); err != nil {
+			return
+		}
+		err = mp.fsmSetXAttr(extend)
+	case opFSMRemoveXAttr:
+		var extend *Extend
+		if extend, err = NewExtendFromBytes(msg.V); err != nil {
+			return
+		}
+		err = mp.fsmRemoveXAttr(extend)
+	case opFSMCreateMultipart:
+		var multipart *Multipart
+		multipart = MultipartFromBytes(msg.V)
+		resp = mp.fsmCreateMultipart(multipart)
+	case opFSMRemoveMultipart:
+		var multipart *Multipart
+		multipart = MultipartFromBytes(msg.V)
+		resp = mp.fsmRemoveMultipart(multipart)
+	case opFSMAppendMultipart:
+		var multipart *Multipart
+		multipart = MultipartFromBytes(msg.V)
+		resp = mp.fsmAppendMultipart(multipart)
 	}
 	return
 }
@@ -175,37 +202,22 @@ func (mp *metaPartition) ApplyMemberChange(confChange *raftproto.ConfChange, ind
 }
 
 // Snapshot returns the snapshot of the current meta partition.
-func (mp *metaPartition) Snapshot() (raftproto.Snapshot, error) {
-	applyID := mp.applyID
-	ino := mp.getInodeTree()
-	dentry := mp.getDentryTree()
-	finfos, err := ioutil.ReadDir(mp.config.RootDir)
-	if err != nil {
-		return nil, err
-	}
-	var fileList []string
-	for _, in := range finfos {
-		if in.IsDir() {
-			continue
-		}
-		if strings.HasPrefix(in.Name(), prefixDelExtent) {
-			fileList = append(fileList, in.Name())
-		}
-	}
-	snapIter := NewMetaItemIterator(applyID, ino, dentry, mp.config.RootDir,
-		fileList)
-	return snapIter, nil
+func (mp *metaPartition) Snapshot() (snap raftproto.Snapshot, err error) {
+	snap, err = newMetaItemIterator(mp)
+	return
 }
 
 // ApplySnapshot applies the given snapshots.
 func (mp *metaPartition) ApplySnapshot(peers []raftproto.Peer, iter raftproto.SnapIterator) (err error) {
 	var (
-		data       []byte
-		index      int
-		appIndexID uint64
-		cursor     uint64
-		inodeTree  = NewBtree()
-		dentryTree = NewBtree()
+		data          []byte
+		index         int
+		appIndexID    uint64
+		cursor        uint64
+		inodeTree     = NewBtree()
+		dentryTree    = NewBtree()
+		extendTree    = NewBtree()
+		multipartTree = NewBtree()
 	)
 	defer func() {
 		if err == io.EOF {
@@ -216,16 +228,18 @@ func (mp *metaPartition) ApplySnapshot(peers []raftproto.Peer, iter raftproto.Sn
 			err = nil
 			// store message
 			mp.storeChan <- &storeMsg{
-				command:    opFSMStoreTick,
-				applyIndex: mp.applyID,
-				inodeTree:  mp.inodeTree,
-				dentryTree: mp.dentryTree,
+				command:       opFSMStoreTick,
+				applyIndex:    mp.applyID,
+				inodeTree:     mp.inodeTree,
+				dentryTree:    mp.dentryTree,
+				extendTree:    mp.extendTree,
+				multipartTree: mp.multipartTree,
 			}
 			mp.extReset <- struct{}{}
-			log.LogDebugf("[ApplySnapshot] successful.")
+			log.LogDebugf("ApplySnapshot: finish with EOF: partitionID(%v) applyID(%v)", mp.config.PartitionId, mp.applyID)
 			return
 		}
-		log.LogErrorf("[ApplySnapshot]: %s", err.Error())
+		log.LogErrorf("ApplySnapshot: stop with error: partitionID(%v) err(%v)", mp.config.PartitionId, err)
 	}()
 	for {
 		data, err = iter.Next()
@@ -252,23 +266,38 @@ func (mp *metaPartition) ApplySnapshot(peers []raftproto.Peer, iter raftproto.Sn
 				cursor = ino.Inode
 			}
 			inodeTree.ReplaceOrInsert(ino, true)
-			log.LogDebugf("action[ApplySnapshot] create inode[%v].", ino)
+			log.LogDebugf("ApplySnapshot: create inode: partitonID(%v) inode(%v).", mp.config.PartitionId, ino)
 		case opFSMCreateDentry:
 			dentry := &Dentry{}
-
-			// TODO Unhandled errors
-			dentry.UnmarshalKey(snap.K)
-			dentry.UnmarshalValue(snap.V)
+			if err = dentry.UnmarshalKey(snap.K); err != nil {
+				return
+			}
+			if err = dentry.UnmarshalValue(snap.V); err != nil {
+				return
+			}
 			dentryTree.ReplaceOrInsert(dentry, true)
-			log.LogDebugf("action[ApplySnapshot] create dentry[%v].", dentry)
+			log.LogDebugf("ApplySnapshot: create dentry: partitionID(%v) err(%v)", mp.config.PartitionId, dentry)
+		case opFSMSetXAttr:
+			var extend *Extend
+			if extend, err = NewExtendFromBytes(snap.V); err != nil {
+				return
+			}
+			extendTree.ReplaceOrInsert(extend, true)
+			log.LogDebugf("ApplySnapshot: set extend attributes: partitionID(%v) extend(%v)",
+				mp.config.PartitionId, extend)
+		case opFSMCreateMultipart:
+			var multipart = MultipartFromBytes(snap.V)
+			multipartTree.ReplaceOrInsert(multipart, true)
+			log.LogDebugf("ApplySnapshot: create multipart: partitoinID(%v) multipart(%v)", mp.config.PartitionId, multipart)
 		case opExtentFileSnapshot:
 			fileName := string(snap.K)
 			fileName = path.Join(mp.config.RootDir, fileName)
 			if err = ioutil.WriteFile(fileName, snap.V, 0644); err != nil {
-				log.LogErrorf("action[ApplySnapshot] SnapDeleteExtent[%v].",
-					err.Error())
+				log.LogErrorf("ApplySnapshot: write snap extent delete file fail: partitionID(%v) err(%v)",
+					mp.config.PartitionId, err)
 			}
-			log.LogDebugf("action[ApplySnapshot] SnapDeleteExtent[%v].", fileName)
+			log.LogDebugf("ApplySnapshot: write snap extent delete file: partitonID(%v) filename(%v).",
+				mp.config.PartitionId, fileName)
 		default:
 			err = fmt.Errorf("unknown op=%d", snap.Op)
 			return

metanode/partition_fsmop_extend.go

@@ -0,0 +1,46 @@
+// Copyright 2018 The Chubao Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package metanode
+
+type ExtendOpResult struct {
+	Status uint8
+	Extend *Extend
+}
+
+func (mp *metaPartition) fsmSetXAttr(extend *Extend) (err error) {
+	treeItem := mp.extendTree.Get(extend)
+	var e *Extend
+	if treeItem == nil {
+		e = NewExtend(extend.inode)
+		mp.extendTree.ReplaceOrInsert(e, true)
+	} else {
+		e = treeItem.(*Extend)
+	}
+	e.Merge(extend, true)
+	return
+}
+
+func (mp *metaPartition) fsmRemoveXAttr(extend *Extend) (err error) {
+	treeItem := mp.extendTree.Get(extend)
+	if treeItem == nil {
+		return
+	}
+	e := treeItem.(*Extend)
+	extend.Range(func(key, value []byte) bool {
+		e.Remove(key)
+		return true
+	})
+	return
+}

metanode/partition_fsmop_inode.go

@@ -17,9 +17,10 @@ package metanode
 import (
 	"bytes"
 	"encoding/binary"
+	"io"
+
 	"github.com/chubaofs/chubaofs/proto"
 	"github.com/chubaofs/chubaofs/util/log"
-	"io"
 )
 
 type InodeResponse struct {
@@ -150,7 +151,8 @@ func (mp *metaPartition) internalDelete(val []byte) (err error) {
 			}
 			return
 		}
-		log.LogDebugf("recive raftLeader free inode(%v)", ino.Inode)
+		log.LogDebugf("internalDelete: received internal delete: partitionID(%v) inode(%v)",
+			mp.config.PartitionId, ino.Inode)
 		mp.internalDeleteInode(ino)
 	}
 }
@@ -158,6 +160,7 @@ func (mp *metaPartition) internalDelete(val []byte) (err error) {
 func (mp *metaPartition) internalDeleteInode(ino *Inode) {
 	mp.inodeTree.Delete(ino)
 	mp.freeList.Remove(ino.Inode)
+	mp.extendTree.Delete(&Extend{inode: ino.Inode}) // Also delete extend attribute.
 	return
 }
 

metanode/partition_fsmop_multipart.go

@@ -0,0 +1,47 @@
+// Copyright 2018 The Chubao Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package metanode
+
+import "github.com/chubaofs/chubaofs/proto"
+
+func (mp *metaPartition) fsmCreateMultipart(multipart *Multipart) (status uint8) {
+	_, ok := mp.multipartTree.ReplaceOrInsert(multipart, false)
+	if !ok {
+		return proto.OpExistErr
+	}
+	return proto.OpOk
+}
+
+func (mp *metaPartition) fsmRemoveMultipart(multipart *Multipart) (status uint8) {
+	deletedItem := mp.multipartTree.Delete(multipart)
+	if deletedItem == nil {
+		return proto.OpNotExistErr
+	}
+	return proto.OpOk
+}
+
+func (mp *metaPartition) fsmAppendMultipart(multipart *Multipart) (status uint8) {
+	storedItem := mp.multipartTree.Get(multipart)
+	if storedItem == nil {
+		return proto.OpNotExistErr
+	}
+	storedMultipart := storedItem.(*Multipart)
+	for _, part := range multipart.Parts() {
+		if !storedMultipart.InsertPart(part, false) {
+			return proto.OpExistErr
+		}
+	}
+	return proto.OpOk
+}

metanode/partition_item.go

@@ -18,9 +18,14 @@ import (
 	"bytes"
 	"encoding/binary"
 	"encoding/json"
+	"fmt"
 	"io"
 	"io/ioutil"
+	"os"
 	"path"
+	"reflect"
+	"strings"
+	"sync"
 )
 
 // MetaItem defines the structure of the metadata operations.
@@ -111,34 +116,127 @@ func NewMetaItem(op uint32, key, value []byte) *MetaItem {
 	}
 }
 
-// MetaItemIterator defines the iterator of the MetaItem.
-type MetaItemIterator struct {
-	applyID     uint64
-	cur         int
-	curItem     BtreeItem
-	inoLen      int
-	inodeTree   *BTree
-	dentryLen   int
-	dentryTree  *BTree
-	fileRootDir string
-	fileList    []string
-	total       int
+type fileData struct {
+	filename string
+	data     []byte
 }
 
-// NewMetaItemIterator returns a new MetaItemIterator.
-func NewMetaItemIterator(applyID uint64, ino, den *BTree,
-	rootDir string, filelist []string) *MetaItemIterator {
-	si := new(MetaItemIterator)
-	si.applyID = applyID
-	si.inodeTree = ino
-	si.dentryTree = den
-	si.cur = 0
-	si.inoLen = ino.Len()
-	si.dentryLen = den.Len()
-	si.fileRootDir = rootDir
-	si.fileList = filelist
-	si.total = si.inoLen + si.dentryLen
-	return si
+// MetaItemIterator defines the iterator of the MetaItem.
+type MetaItemIterator struct {
+	fileRootDir   string
+	applyID       uint64
+	inodeTree     *BTree
+	dentryTree    *BTree
+	extendTree    *BTree
+	multipartTree *BTree
+
+	filenames []string
+
+	dataCh    chan interface{}
+	errorCh   chan error
+	err       error
+	closeCh   chan struct{}
+	closeOnce sync.Once
+}
+
+// newMetaItemIterator returns a new MetaItemIterator.
+func newMetaItemIterator(mp *metaPartition) (si *MetaItemIterator, err error) {
+	si = new(MetaItemIterator)
+	si.fileRootDir = mp.config.RootDir
+	si.applyID = mp.applyID
+	si.inodeTree = mp.inodeTree.GetTree()
+	si.dentryTree = mp.dentryTree.GetTree()
+	si.extendTree = mp.extendTree.GetTree()
+	si.multipartTree = mp.multipartTree.GetTree()
+	si.dataCh = make(chan interface{})
+	si.errorCh = make(chan error, 1)
+	si.closeCh = make(chan struct{})
+
+	// collect extend del files
+	var filenames = make([]string, 0)
+	var fileInfos []os.FileInfo
+	if fileInfos, err = ioutil.ReadDir(mp.config.RootDir); err != nil {
+		return
+	}
+
+	for _, fileInfo := range fileInfos {
+		if !fileInfo.IsDir() && strings.HasPrefix(fileInfo.Name(), prefixDelExtent) {
+			filenames = append(filenames, fileInfo.Name())
+		}
+	}
+	si.filenames = filenames
+
+	// start data producer
+	go func(iter *MetaItemIterator) {
+		defer func() {
+			close(iter.dataCh)
+			close(iter.errorCh)
+		}()
+		var produceItem = func(item interface{}) (success bool) {
+			select {
+			case iter.dataCh <- item:
+				return true
+			case <-iter.closeCh:
+				return false
+			}
+		}
+		var produceError = func(err error) {
+			select {
+			case iter.errorCh <- err:
+			default:
+			}
+		}
+		var checkClose = func() (closed bool) {
+			select {
+			case <-iter.closeCh:
+				return true
+			default:
+				return false
+			}
+		}
+		// process inodes
+		iter.inodeTree.Ascend(func(i BtreeItem) bool {
+			return produceItem(i)
+		})
+		if checkClose() {
+			return
+		}
+		// process dentries
+		iter.dentryTree.Ascend(func(i BtreeItem) bool {
+			return produceItem(i)
+		})
+		if checkClose() {
+			return
+		}
+		// process extends
+		iter.extendTree.Ascend(func(i BtreeItem) bool {
+			return produceItem(i)
+		})
+		if checkClose() {
+			return
+		}
+		// process multiparts
+		iter.multipartTree.Ascend(func(i BtreeItem) bool {
+			return produceItem(i)
+		})
+		if checkClose() {
+			return
+		}
+		// process extent del files
+		var err error
+		var raw []byte
+		for _, filename := range iter.filenames {
+			if raw, err = ioutil.ReadFile(path.Join(iter.fileRootDir, filename)); err != nil {
+				produceError(err)
+				return
+			}
+			if !produceItem(&fileData{filename: filename, data: raw}) {
+				return
+			}
+		}
+	}(si)
+
+	return
 }
 
 // ApplyIndex returns the applyID of the iterator.
@@ -148,72 +246,65 @@ func (si *MetaItemIterator) ApplyIndex() uint64 {
 
 // Close closes the iterator.
 func (si *MetaItemIterator) Close() {
-	si.cur = si.total + 1
+	si.closeOnce.Do(func() {
+		close(si.closeCh)
+	})
 	return
 }
 
 // Next returns the next item.
 func (si *MetaItemIterator) Next() (data []byte, err error) {
-	// TODO: Redesign iterator to improve performance. [Mervin]
-	// First Send ApplyIndex
-	if si.cur == 0 {
-		appIdBuf := make([]byte, 8)
-		binary.BigEndian.PutUint64(appIdBuf, si.applyID)
-		data = appIdBuf[:]
-		si.cur++
+
+	if si.err != nil {
+		err = si.err
+		return
+	}
+	var item interface{}
+	select {
+	case item = <-si.dataCh:
+		if item == nil {
+			err, si.err = io.EOF, io.EOF
+			si.Close()
+			return
+		}
+	case err = <-si.errorCh:
+		si.err = err
+		si.Close()
 		return
 	}
 
-	if si.cur <= si.inoLen {
-		si.inodeTree.AscendGreaterOrEqual(si.curItem, func(i BtreeItem) bool {
-			ino := i.(*Inode)
-			if si.curItem == ino {
-				return true
-			}
-			si.curItem = ino
-			snap := NewMetaItem(opFSMCreateInode, ino.MarshalKey(),
-				ino.MarshalValue())
-			data, err = snap.MarshalBinary()
-			si.cur++
-			return false
-		})
-		return
+	var snap *MetaItem
+	switch typedItem := item.(type) {
+	case *Inode:
+		snap = NewMetaItem(opFSMCreateInode, typedItem.MarshalKey(), typedItem.MarshalValue())
+	case *Dentry:
+		snap = NewMetaItem(opFSMCreateDentry, typedItem.MarshalKey(), typedItem.MarshalValue())
+	case *Extend:
+		var raw []byte
+		if raw, err = typedItem.Bytes(); err != nil {
+			si.err = err
+			si.Close()
+			return
+		}
+		snap = NewMetaItem(opFSMSetXAttr, nil, raw)
+	case *Multipart:
+		var raw []byte
+		if raw, err = typedItem.Bytes(); err != nil {
+			si.err = err
+			si.Close()
+			return
+		}
+		snap = NewMetaItem(opFSMCreateMultipart, nil, raw)
+	case *fileData:
+		snap = NewMetaItem(opExtentFileSnapshot, []byte(typedItem.filename), typedItem.data)
+	default:
+		panic(fmt.Sprintf("unknown item type: %v", reflect.TypeOf(item).Name()))
 	}
 
-	if si.cur == (si.inoLen + 1) {
-		si.curItem = nil
-	}
-
-	if si.cur <= si.total {
-		si.dentryTree.AscendGreaterOrEqual(si.curItem, func(i BtreeItem) bool {
-			dentry := i.(*Dentry)
-			if si.curItem == dentry {
-				return true
-			}
-			si.curItem = dentry
-			snap := NewMetaItem(opFSMCreateDentry, dentry.MarshalKey(),
-				dentry.MarshalValue())
-			data, err = snap.MarshalBinary()
-			si.cur++
-			return false
-		})
-	}
-	if len(si.fileList) == 0 {
-		err = io.EOF
-		data = nil
+	if data, err = snap.MarshalBinary(); err != nil {
+		si.err = err
+		si.Close()
 		return
 	}
-
-	fileName := si.fileList[0]
-	fileBody, err := ioutil.ReadFile(path.Join(si.fileRootDir, fileName))
-	if err != nil {
-		data = nil
-		return
-	}
-	snap := NewMetaItem(opExtentFileSnapshot, []byte(fileName), fileBody)
-	data, err = snap.MarshalBinary()
-	if err != nil {
-		si.fileList = si.fileList[1:]
-	}
 	return
 }

metanode/partition_op_extend.go

@@ -0,0 +1,131 @@
+// Copyright 2018 The Chubao Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package metanode
+
+import (
+	"encoding/json"
+	"github.com/chubaofs/chubaofs/proto"
+)
+
+func (mp *metaPartition) SetXAttr(req *proto.SetXAttrRequest, p *Packet) (err error) {
+	var extend = NewExtend(req.Inode)
+	extend.Put([]byte(req.Key), []byte(req.Value))
+	if _, err = mp.putExtend(opFSMSetXAttr, extend); err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		return
+	}
+	p.PacketOkReply()
+	return
+}
+
+func (mp *metaPartition) GetXAttr(req *proto.GetXAttrRequest, p *Packet) (err error) {
+	var response = &proto.GetXAttrResponse{
+		VolName:     req.VolName,
+		PartitionId: req.PartitionId,
+		Inode:       req.Inode,
+		Key:         req.Key,
+	}
+	treeItem := mp.extendTree.Get(NewExtend(req.Inode))
+	if treeItem != nil {
+		extend := treeItem.(*Extend)
+		if value, exist := extend.Get([]byte(req.Key)); exist {
+			response.Value = string(value)
+		}
+	}
+	var encoded []byte
+	encoded, err = json.Marshal(response)
+	if err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		return
+	}
+	p.PacketOkWithBody(encoded)
+	return
+}
+
+func (mp *metaPartition) BatchGetXAttr(req *proto.BatchGetXAttrRequest, p *Packet) (err error) {
+	var response = &proto.BatchGetXAttrResponse{
+		VolName:     req.VolName,
+		PartitionId: req.PartitionId,
+		XAttrs:      make([]*proto.XAttrInfo, 0, len(req.Inodes)),
+	}
+	for _, inode := range req.Inodes {
+		treeItem := mp.extendTree.Get(NewExtend(inode))
+		if treeItem != nil {
+			extend := treeItem.(*Extend)
+			info := &proto.XAttrInfo{
+				Inode:  inode,
+				XAttrs: make(map[string]string),
+			}
+			for _, key := range req.Keys {
+				if val, exist := extend.Get([]byte(key)); exist {
+					info.XAttrs[key] = string(val)
+				}
+			}
+			response.XAttrs = append(response.XAttrs, info)
+		}
+	}
+	var encoded []byte
+	if encoded, err = json.Marshal(response); err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		return
+	}
+	p.PacketOkWithBody(encoded)
+	return
+}
+
+func (mp *metaPartition) RemoveXAttr(req *proto.RemoveXAttrRequest, p *Packet) (err error) {
+	var extend = NewExtend(req.Inode)
+	extend.Put([]byte(req.Key), nil)
+	if _, err = mp.putExtend(opFSMRemoveXAttr, extend); err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		return
+	}
+	p.PacketOkReply()
+	return
+}
+
+func (mp *metaPartition) ListXAttr(req *proto.ListXAttrRequest, p *Packet) (err error) {
+	var response = &proto.ListXAttrResponse{
+		VolName:     req.VolName,
+		PartitionId: req.PartitionId,
+		Inode:       req.Inode,
+		XAttr:       make(map[string]string),
+	}
+	treeItem := mp.extendTree.Get(NewExtend(req.Inode))
+	if treeItem != nil {
+		extend := treeItem.(*Extend)
+		extend.Range(func(key, value []byte) bool {
+			response.XAttr[string(key)] = string(value)
+			return true
+		})
+	}
+	var encoded []byte
+	encoded, err = json.Marshal(response)
+	if err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		return
+	}
+	p.PacketOkWithBody(encoded)
+	return
+}
+
+func (mp *metaPartition) putExtend(op uint32, extend *Extend) (resp interface{}, err error) {
+	var marshaled []byte
+	if marshaled, err = extend.Bytes(); err != nil {
+		return
+	}
+	resp, err = mp.Put(op, marshaled)
+	return
+}

metanode/partition_op_extent.go

@@ -17,8 +17,9 @@ package metanode
 import (
 	"encoding/json"
 
-	"github.com/chubaofs/chubaofs/proto"
 	"os"
+
+	"github.com/chubaofs/chubaofs/proto"
 )
 
 // ExtentAppend appends an extent.
@@ -88,3 +89,30 @@ func (mp *metaPartition) ExtentsTruncate(req *ExtentsTruncateReq,
 	p.PacketErrorWithBody(msg.Status, nil)
 	return
 }
+
+func (mp *metaPartition) BatchExtentAppend(req *proto.AppendExtentKeysRequest, p *Packet) (err error) {
+	ino := NewInode(req.Inode, 0)
+	extents := req.Extents
+	for _, extent := range extents {
+		ino.Extents.Append(&proto.ExtentKey{
+			FileOffset:   extent.FileOffset,
+			PartitionId:  extent.PartitionId,
+			ExtentId:     extent.ExtentId,
+			ExtentOffset: extent.ExtentOffset,
+			Size:         extent.Size,
+			CRC:          extent.CRC,
+		})
+	}
+	val, err := ino.Marshal()
+	if err != nil {
+		p.PacketErrorWithBody(proto.OpErr, nil)
+		return
+	}
+	resp, err := mp.Put(opFSMExtentsAdd, val)
+	if err != nil {
+		p.PacketErrorWithBody(proto.OpAgain, []byte(err.Error()))
+		return
+	}
+	p.PacketErrorWithBody(resp.(uint8), nil)
+	return
+}

metanode/partition_op_inode.go

@@ -229,3 +229,19 @@ func (mp *metaPartition) SetAttr(reqData []byte, p *Packet) (err error) {
 func (mp *metaPartition) GetInodeTree() *BTree {
 	return mp.inodeTree.GetTree()
 }
+
+func (mp *metaPartition) DeleteInode(req *proto.DeleteInodeRequest, p *Packet) (err error) {
+	ino := NewInode(req.Inode, 0)
+	encoded, err := ino.Marshal()
+	if err != nil {
+		p.ResultCode = proto.OpErr
+		return
+	}
+	_, err = mp.Put(opFSMInternalDeleteInode, encoded)
+	if err != nil {
+		p.PacketErrorWithBody(proto.OpAgain, []byte(err.Error()))
+		return
+	}
+	p.PacketOkReply()
+	return
+}

metanode/partition_op_multipart.go

@@ -0,0 +1,216 @@
+// Copyright 2018 The Chubao Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package metanode
+
+import (
+	"encoding/json"
+	"strings"
+	"time"
+
+	"github.com/google/uuid"
+
+	"github.com/chubaofs/chubaofs/proto"
+)
+
+func (mp *metaPartition) GetMultipart(req *proto.GetMultipartRequest, p *Packet) (err error) {
+	item := mp.multipartTree.Get(&Multipart{id: req.MultipartId})
+	if item == nil {
+		p.PacketErrorWithBody(proto.OpNotExistErr, nil)
+		return
+	}
+	multipart := item.(*Multipart)
+	resp := &proto.GetMultipartResponse{
+		Info: &proto.MultipartInfo{
+			ID:       multipart.id,
+			Path:     multipart.key,
+			InitTime: multipart.initTime,
+			Parts:    make([]*proto.MultipartPartInfo, 0, len(multipart.parts)),
+		},
+	}
+	for _, part := range multipart.Parts() {
+		resp.Info.Parts = append(resp.Info.Parts, &proto.MultipartPartInfo{
+			ID:         part.ID,
+			Inode:      part.Inode,
+			MD5:        part.MD5,
+			Size:       part.Size,
+			UploadTime: part.UploadTime,
+		})
+	}
+	var reply []byte
+	if reply, err = json.Marshal(resp); err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		return
+	}
+	p.PacketOkWithBody(reply)
+	return
+}
+
+func (mp *metaPartition) AppendMultipart(req *proto.AddMultipartPartRequest, p *Packet) (err error) {
+	if req.Part == nil {
+		p.PacketOkReply()
+		return
+	}
+	item := mp.multipartTree.Get(&Multipart{id: req.MultipartId})
+	if item == nil {
+		p.PacketErrorWithBody(proto.OpNotExistErr, nil)
+		return
+	}
+	multipart := &Multipart{
+		id: req.MultipartId,
+		parts: Parts{
+			&Part{
+				ID:         req.Part.ID,
+				UploadTime: req.Part.UploadTime,
+				MD5:        req.Part.MD5,
+				Size:       req.Part.Size,
+				Inode:      req.Part.Inode,
+			},
+		},
+	}
+	var resp interface{}
+	if resp, err = mp.putMultipart(opFSMAppendMultipart, multipart); err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		return
+	}
+	status := resp.(uint8)
+	if status != proto.OpOk {
+		p.PacketErrorWithBody(status, nil)
+		return
+	}
+	p.PacketOkReply()
+	return
+}
+
+func (mp *metaPartition) RemoveMultipart(req *proto.RemoveMultipartRequest, p *Packet) (err error) {
+	multipart := &Multipart{
+		id: req.MultipartId,
+	}
+	var resp interface{}
+	if resp, err = mp.putMultipart(opFSMRemoveMultipart, multipart); err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		return
+	}
+	status := resp.(uint8)
+	if status != proto.OpOk {
+		p.PacketErrorWithBody(status, nil)
+		return
+	}
+	p.PacketOkReply()
+	return
+}
+
+func (mp *metaPartition) CreateMultipart(req *proto.CreateMultipartRequest, p *Packet) (err error) {
+	nextId := strings.ReplaceAll(uuid.New().String(), "-", "")
+	multipart := &Multipart{
+		id:       nextId,
+		key:      req.Path,
+		initTime: time.Now().Local(),
+	}
+	if _, err = mp.putMultipart(opFSMCreateMultipart, multipart); err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		return
+	}
+	resp := &proto.CreateMultipartResponse{
+		Info: &proto.MultipartInfo{
+			ID:   nextId,
+			Path: req.Path,
+		},
+	}
+	var reply []byte
+	if reply, err = json.Marshal(resp); err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		return
+	}
+	p.PacketOkWithBody(reply)
+	return
+}
+
+func (mp *metaPartition) ListMultipart(req *proto.ListMultipartRequest, p *Packet) (err error) {
+	max := int(req.Max)
+	marker := req.Marker
+	multipartIdMarker := req.MultipartIdMarker
+	prefix := req.Prefix
+	var matches = make([]*Multipart, 0, max)
+	var walkTreeFunc = func(i BtreeItem) bool {
+		multipart := i.(*Multipart)
+		// key marker is enabled
+		if len(marker) > 0 && marker < multipart.key {
+			// skip and continue
+			return true
+		}
+		// prefix is enabled
+		if len(prefix) > 0 && !strings.HasPrefix(multipart.key, prefix) {
+			// skip and continue
+			return true
+		}
+		matches = append(matches, multipart)
+		return !(len(matches) >= max)
+	}
+	if len(multipartIdMarker) > 0 {
+		mp.multipartTree.AscendGreaterOrEqual(&Multipart{id: multipartIdMarker}, walkTreeFunc)
+	} else {
+		mp.multipartTree.Ascend(walkTreeFunc)
+	}
+	multipartInfos := make([]*proto.MultipartInfo, len(matches))
+
+	var convertPartFunc = func(part *Part) *proto.MultipartPartInfo {
+		return &proto.MultipartPartInfo{
+			ID:         part.ID,
+			Inode:      part.Inode,
+			MD5:        part.MD5,
+			Size:       part.Size,
+			UploadTime: part.UploadTime,
+		}
+	}
+
+	var convertMultipartFunc = func(multipart *Multipart) *proto.MultipartInfo {
+		partInfos := make([]*proto.MultipartPartInfo, len(multipart.parts))
+		for i := 0; i < len(multipart.parts); i++ {
+			partInfos[i] = convertPartFunc(multipart.parts[i])
+		}
+		return &proto.MultipartInfo{
+			ID:       multipart.id,
+			Path:     multipart.key,
+			InitTime: multipart.initTime,
+			Parts:    partInfos,
+		}
+	}
+
+	for i := 0; i < len(matches); i++ {
+		multipartInfos[i] = convertMultipartFunc(matches[i])
+	}
+
+	resp := &proto.ListMultipartResponse{
+		Multiparts: multipartInfos,
+	}
+
+	var reply []byte
+	if reply, err = json.Marshal(resp); err != nil {
+		p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
+		return
+	}
+	p.PacketOkWithBody(reply)
+	return
+}
+
+// SendMultipart replicate specified multipart operation to raft.
+func (mp *metaPartition) putMultipart(op uint32, multipart *Multipart) (resp interface{}, err error) {
+	var encoded []byte
+	if encoded, err = multipart.Bytes(); err != nil {
+		return
+	}
+	resp, err = mp.Put(op, encoded)
+	return
+}

metanode/partition_store.go

@@ -19,8 +19,6 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"fmt"
-	"github.com/chubaofs/chubaofs/proto"
-	"github.com/chubaofs/chubaofs/util/errors"
 	"hash/crc32"
 	"io"
 	"io/ioutil"
@@ -28,6 +26,12 @@ import (
 	"path"
 	"strings"
 	"sync/atomic"
+
+	"github.com/chubaofs/chubaofs/util/log"
+
+	"github.com/chubaofs/chubaofs/proto"
+	"github.com/chubaofs/chubaofs/util/errors"
+	mmap "github.com/edsrzf/mmap-go"
 )
 
 const (
@@ -36,6 +40,8 @@ const (
 	snapshotBackup  = ".snapshot_backup"
 	inodeFile       = "inode"
 	dentryFile      = "dentry"
+	extendFile      = "extend"
+	multipartFile   = "multipart"
 	applyIDFile     = "apply"
 	SnapshotSign    = ".sign"
 	metadataFile    = "meta"
@@ -72,10 +78,20 @@ func (mp *metaPartition) loadMetadata() (err error) {
 	mp.config.End = mConf.End
 	mp.config.Peers = mConf.Peers
 	mp.config.Cursor = mp.config.Start
+
+	log.LogInfof("loadMetadata: load complete: partitionID(%v) volume(%v) range(%v,%v) cursor(%v)",
+		mp.config.PartitionId, mp.config.VolName, mp.config.Start, mp.config.End, mp.config.Cursor)
 	return
 }
 
 func (mp *metaPartition) loadInode(rootDir string) (err error) {
+	var numInodes uint64
+	defer func() {
+		if err == nil {
+			log.LogInfof("loadInode: load complete: partitonID(%v) volume(%v) numInodes(%v)",
+				mp.config.PartitionId, mp.config.VolName, numInodes)
+		}
+	}()
 	filename := path.Join(rootDir, inodeFile)
 	if _, err = os.Stat(filename); err != nil {
 		err = nil
@@ -124,11 +140,19 @@ func (mp *metaPartition) loadInode(rootDir string) (err error) {
 		if mp.config.Cursor < ino.Inode {
 			mp.config.Cursor = ino.Inode
 		}
+		numInodes += 1
 	}
 }
 
 // Load dentry from the dentry snapshot.
 func (mp *metaPartition) loadDentry(rootDir string) (err error) {
+	var numDentries uint64
+	defer func() {
+		if err == nil {
+			log.LogInfof("loadDentry: load complete: partitonID(%v) volume(%v) numDentries(%v)",
+				mp.config.PartitionId, mp.config.VolName, numDentries)
+		}
+	}()
 	filename := path.Join(rootDir, dentryFile)
 	if _, err = os.Stat(filename); err != nil {
 		err = nil
@@ -179,13 +203,98 @@ func (mp *metaPartition) loadDentry(rootDir string) (err error) {
 			return
 		}
 		if status := mp.fsmCreateDentry(dentry, true); status != proto.OpOk {
-			err = errors.NewErrorf("[loadDentry] createDentry dentry: %v, "+
-				"resp code: %d", status)
+			err = errors.NewErrorf("[loadDentry] createDentry dentry: %v, resp code: %d", dentry, status)
 			return
 		}
+		numDentries += 1
 	}
 }
 
+func (mp *metaPartition) loadExtend(rootDir string) error {
+	var err error
+	filename := path.Join(rootDir, extendFile)
+	if _, err = os.Stat(filename); err != nil {
+		return nil
+	}
+	fp, err := os.OpenFile(filename, os.O_RDONLY, 0644)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = fp.Close()
+	}()
+	var mem mmap.MMap
+	if mem, err = mmap.Map(fp, mmap.RDONLY, 0); err != nil {
+		return err
+	}
+	defer func() {
+		_ = mem.Unmap()
+	}()
+	var offset, n int
+	// read number of extends
+	var numExtends uint64
+	numExtends, n = binary.Uvarint(mem)
+	offset += n
+	for i := uint64(0); i < numExtends; i++ {
+		// read length
+		var numBytes uint64
+		numBytes, n = binary.Uvarint(mem[offset:])
+		offset += n
+		var extend *Extend
+		if extend, err = NewExtendFromBytes(mem[offset : offset+int(numBytes)]); err != nil {
+			return err
+		}
+		log.LogDebugf("loadExtend: new extend from bytes: partitionID（%v) volume(%v) inode(%v)",
+			mp.config.PartitionId, mp.config.VolName, extend.inode)
+		_ = mp.fsmSetXAttr(extend)
+		offset += int(numBytes)
+	}
+	log.LogInfof("loadExtend: load complete: partitionID(%v) volume(%v) numExtends(%v) filename(%v)",
+		mp.config.PartitionId, mp.config.VolName, numExtends, filename)
+	return nil
+}
+
+func (mp *metaPartition) loadMultipart(rootDir string) error {
+	var err error
+	filename := path.Join(rootDir, multipartFile)
+	if _, err = os.Stat(filename); err != nil {
+		return nil
+	}
+	fp, err := os.OpenFile(filename, os.O_RDONLY, 0644)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = fp.Close()
+	}()
+	var mem mmap.MMap
+	if mem, err = mmap.Map(fp, mmap.RDONLY, 0); err != nil {
+		return err
+	}
+	defer func() {
+		_ = mem.Unmap()
+	}()
+	var offset, n int
+	// read number of extends
+	var numMultiparts uint64
+	numMultiparts, n = binary.Uvarint(mem)
+	offset += n
+	for i := uint64(0); i < numMultiparts; i++ {
+		// read length
+		var numBytes uint64
+		numBytes, n = binary.Uvarint(mem[offset:])
+		offset += n
+		var multipart *Multipart
+		multipart = MultipartFromBytes(mem[offset : offset+int(numBytes)])
+		log.LogDebugf("loadMultipart: create multipart from bytes: partitionID（%v) multipartID(%v)", mp.config.PartitionId, multipart.id)
+		mp.fsmCreateMultipart(multipart)
+		offset += int(numBytes)
+	}
+	log.LogInfof("loadMultipart: load complete: partitionID(%v) numMultiparts(%v) filename(%v)",
+		mp.config.PartitionId, numMultiparts, filename)
+	return nil
+}
+
 func (mp *metaPartition) loadApplyID(rootDir string) (err error) {
 	filename := path.Join(rootDir, applyIDFile)
 	if _, err = os.Stat(filename); err != nil {
@@ -219,7 +328,8 @@ func (mp *metaPartition) loadApplyID(rootDir string) (err error) {
 	if cursor > atomic.LoadUint64(&mp.config.Cursor) {
 		atomic.StoreUint64(&mp.config.Cursor, cursor)
 	}
-
+	log.LogInfof("loadApplyID: load complete: partitionID(%v) volume(%v) applyID(%v) filename(%v)",
+		mp.config.PartitionId, mp.config.VolName, mp.applyID, filename)
 	return
 }
 
@@ -250,7 +360,11 @@ func (mp *metaPartition) persistMetadata() (err error) {
 	if _, err = fp.Write(data); err != nil {
 		return
 	}
-	err = os.Rename(filename, path.Join(mp.config.RootDir, metadataFile))
+	if err = os.Rename(filename, path.Join(mp.config.RootDir, metadataFile)); err != nil {
+		return
+	}
+	log.LogInfof("persistMetata: persist complete: partitionID(%v) volume(%v) range(%v,%v) cursor(%v)",
+		mp.config.PartitionId, mp.config.VolName, mp.config.Start, mp.config.End, mp.config.Cursor)
 	return
 }
 
@@ -268,6 +382,8 @@ func (mp *metaPartition) storeApplyID(rootDir string, sm *storeMsg) (err error)
 	if _, err = fp.WriteString(fmt.Sprintf("%d|%d", sm.applyIndex, atomic.LoadUint64(&mp.config.Cursor))); err != nil {
 		return
 	}
+	log.LogInfof("storeApplyID: store complete: partitionID(%v) volume(%v) applyID(%v)",
+		mp.config.PartitionId, mp.config.VolName, sm.applyIndex)
 	return
 }
 
@@ -310,6 +426,8 @@ func (mp *metaPartition) storeInode(rootDir string,
 		return true
 	})
 	crc = sign.Sum32()
+	log.LogInfof("storeInode: store complete: partitoinID(%v) volume(%) numInodes(%v) crc(%v)",
+		mp.config.PartitionId, mp.config.VolName, sm.inodeTree.Len(), crc)
 	return
 }
 
@@ -352,22 +470,137 @@ func (mp *metaPartition) storeDentry(rootDir string,
 		return true
 	})
 	crc = sign.Sum32()
+	log.LogInfof("storeDentry: store complete: partitoinID(%v) volume(%) numDentries(%v) crc(%v)",
+		mp.config.PartitionId, mp.config.VolName, sm.dentryTree.Len(), crc)
 	return
 }
 
-func (mp *metaPartition) deleteInodeFile() {
-	filename := path.Join(mp.config.RootDir, inodeFile)
-	// TODO Unhandled errors
-	os.Remove(filename)
-}
-func (mp *metaPartition) deleteDentryFile() {
-	filename := path.Join(mp.config.RootDir, dentryFile)
-	// TODO Unhandled errors
-	os.Remove(filename)
+func (mp *metaPartition) storeExtend(rootDir string, sm *storeMsg) (crc uint32, err error) {
+	var extendTree = sm.extendTree
+	var fp = path.Join(rootDir, extendFile)
+	var f *os.File
+	f, err = os.OpenFile(fp, os.O_RDWR|os.O_TRUNC|os.O_APPEND|os.O_CREATE, 0755)
+	if err != nil {
+		return
+	}
+	defer func() {
+		closeErr := f.Close()
+		if err == nil && closeErr != nil {
+			err = closeErr
+		}
+	}()
+	var writer = bufio.NewWriterSize(f, 4*1024*1024)
+	var crc32 = crc32.NewIEEE()
+	var varintTmp = make([]byte, binary.MaxVarintLen64)
+	var n int
+	// write number of extends
+	n = binary.PutUvarint(varintTmp, uint64(extendTree.Len()))
+	if _, err = writer.Write(varintTmp[:n]); err != nil {
+		return
+	}
+	if _, err = crc32.Write(varintTmp[:n]); err != nil {
+		return
+	}
+	extendTree.Ascend(func(i BtreeItem) bool {
+		e := i.(*Extend)
+		var raw []byte
+		if raw, err = e.Bytes(); err != nil {
+			return false
+		}
+		// write length
+		n = binary.PutUvarint(varintTmp, uint64(len(raw)))
+		if _, err = writer.Write(varintTmp[:n]); err != nil {
+			return false
+		}
+		if _, err = crc32.Write(varintTmp[:n]); err != nil {
+			return false
+		}
+		// write raw
+		if _, err = writer.Write(raw); err != nil {
+			return false
+		}
+		if _, err = crc32.Write(raw); err != nil {
+			return false
+		}
+		return true
+	})
+	if err != nil {
+		return
+	}
 
+	if err = writer.Flush(); err != nil {
+		return
+	}
+	if err = f.Sync(); err != nil {
+		return
+	}
+	crc = crc32.Sum32()
+	log.LogInfof("storeExtend: store complete: partitoinID(%v) volume(%) numExtends(%v) crc(%v)",
+		mp.config.PartitionId, mp.config.VolName, extendTree.Len(), crc)
+	return
 }
-func (mp *metaPartition) deleteApplyFile() {
-	filename := path.Join(mp.config.RootDir, applyIDFile)
-	// TODO Unhandled errors
-	os.Remove(filename)
+
+func (mp *metaPartition) storeMultipart(rootDir string, sm *storeMsg) (crc uint32, err error) {
+	var multipartTree = sm.multipartTree
+	var fp = path.Join(rootDir, multipartFile)
+	var f *os.File
+	f, err = os.OpenFile(fp, os.O_RDWR|os.O_TRUNC|os.O_APPEND|os.O_CREATE, 0755)
+	if err != nil {
+		return
+	}
+	defer func() {
+		closeErr := f.Close()
+		if err == nil && closeErr != nil {
+			err = closeErr
+		}
+	}()
+	var writer = bufio.NewWriterSize(f, 4*1024*1024)
+	var crc32 = crc32.NewIEEE()
+	var varintTmp = make([]byte, binary.MaxVarintLen64)
+	var n int
+	// write number of extends
+	n = binary.PutUvarint(varintTmp, uint64(multipartTree.Len()))
+	if _, err = writer.Write(varintTmp[:n]); err != nil {
+		return
+	}
+	if _, err = crc32.Write(varintTmp[:n]); err != nil {
+		return
+	}
+	multipartTree.Ascend(func(i BtreeItem) bool {
+		m := i.(*Multipart)
+		var raw []byte
+		if raw, err = m.Bytes(); err != nil {
+			return false
+		}
+		// write length
+		n = binary.PutUvarint(varintTmp, uint64(len(raw)))
+		if _, err = writer.Write(varintTmp[:n]); err != nil {
+			return false
+		}
+		if _, err = crc32.Write(varintTmp[:n]); err != nil {
+			return false
+		}
+		// write raw
+		if _, err = writer.Write(raw); err != nil {
+			return false
+		}
+		if _, err = crc32.Write(raw); err != nil {
+			return false
+		}
+		return true
+	})
+	if err != nil {
+		return
+	}
+
+	if err = writer.Flush(); err != nil {
+		return
+	}
+	if err = f.Sync(); err != nil {
+		return
+	}
+	crc = crc32.Sum32()
+	log.LogInfof("storeMultipart: store complete: partitoinID(%v) volume(%) numMultiparts(%v) crc(%v)",
+		mp.config.PartitionId, mp.config.VolName, multipartTree.Len(), crc)
+	return
 }

metanode/partition_store_ticket.go

@@ -23,10 +23,12 @@ import (
 )
 
 type storeMsg struct {
-	command    uint32
-	applyIndex uint64
-	inodeTree  *BTree
-	dentryTree *BTree
+	command       uint32
+	applyIndex    uint64
+	inodeTree     *BTree
+	dentryTree    *BTree
+	extendTree    *BTree
+	multipartTree *BTree
 }
 
 func (mp *metaPartition) startSchedule(curIndex uint64) {

objectnode/acl.go

@@ -0,0 +1,266 @@
+package objectnode
+
+// https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/dev/S3_ACLs_UsingACLs.html
+
+import (
+	"encoding/xml"
+	"errors"
+
+	"github.com/chubaofs/chubaofs/util/log"
+)
+
+const (
+	maxGrantCount  = 100 //ACL 可以拥有最多 100 个授权。
+	bucketRootPath = "/"
+)
+
+const (
+	//Permission Value
+	ReadPermission        Permission = "READ"
+	WritePermission                  = "WRITE"
+	ReadACPPermission                = "READ_ACP"
+	WriteACPPermission               = "WRITE_ACP"
+	FullControlPermission            = "FULL_CONTROL"
+)
+
+// https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/dev/acl-overview.html
+var (
+	aclBucketPermissionActions = map[Permission][]Action{
+		ReadPermission:     {ListBucketAction, ListBucketVersionsAction, ListBucketMultipartUploadsAction},
+		WritePermission:    {PutObjectAction, DeleteObjectAction},
+		ReadACPPermission:  {GetBucketAclAction},
+		WriteACPPermission: {PutBucketAclAction},
+		FullControlPermission: {
+			ListBucketAction, ListBucketVersionsAction, ListBucketMultipartUploadsAction,
+			PutObjectAction, DeleteObjectAction,
+			GetBucketAclAction,
+			PutBucketAclAction},
+	}
+	aclObjectPermissionActions = map[Permission][]Action{
+		ReadPermission:     {GetObjectAction, GetObjectVersionAction, GetObjectTorrentAction},
+		WritePermission:    {},
+		ReadACPPermission:  {GetObjectAclAction, GetObjectVersionAclAction},
+		WriteACPPermission: {PutObjectAclAction, PutObjectVersionAclAction},
+		FullControlPermission: {
+			GetObjectAction, GetObjectVersionAction, GetObjectTorrentAction,
+			GetObjectAclAction, GetObjectVersionAclAction,
+			PutObjectAclAction, PutObjectVersionAclAction},
+	}
+)
+
+type StandardACL string
+
+const (
+	PrivateACL                StandardACL = "private"
+	PublicReadACL                         = "public-read"
+	PubliceReadWriteACL                   = "public-read-write"
+	AwsExecReadACL                        = "aws-exec-read"
+	AuthenticatedReadACL                  = "authenticated-read"
+	BucketOwnerReadACL                    = "bucket-owner-read"
+	BucketOwnerFullControlACL             = "bucket-owner-full-control"
+	LogDeliveryWriteACL                   = "log-delivery-write"
+)
+
+type ResourceType string
+
+const (
+	bucketResource ResourceType = "bucket"
+	objectResource              = "object"
+)
+
+type AclRole = string
+
+const (
+	objectOwnerRole AclRole = "owner"
+	bucketOwnerRole         = "bucket-owner"
+	allUsersRole            = "AllUsers"
+	LogDeliveryRole         = "LogDelivery"
+)
+
+var (
+	aclPermissions = map[StandardACL]map[ResourceType]map[string][]Permission{
+		PrivateACL:                {"bucket": {"owner": {FullControlPermission}}, "object": {"owner": {FullControlPermission}}},
+		PublicReadACL:             {"bucket": {"owner": {FullControlPermission}, "AllUsers": {ReadPermission}}, "object": {"owner": {FullControlPermission}, "AllUsers": {ReadPermission}}},
+		PubliceReadWriteACL:       {"bucket": {"owner": {FullControlPermission}, "AllUsers": {ReadPermission, WritePermission}}, "object": {"owner": {FullControlPermission}, "AllUsers": {ReadPermission, WritePermission}}},
+		AwsExecReadACL:            {"bucket": {"owner": {FullControlPermission}}, "object": {"owner": {FullControlPermission}}},
+		AuthenticatedReadACL:      {"bucket": {"owner": {FullControlPermission}}, "object": {"owner": {FullControlPermission}}},
+		BucketOwnerReadACL:        {"object": {"owner": {FullControlPermission}, "bucket-owner": {ReadPermission}}},
+		BucketOwnerFullControlACL: {"object": {"owner": {FullControlPermission}, "bucket-owner": {FullControlPermission}}},
+		LogDeliveryWriteACL:       {"bucket": {"LogDelivery": {WriteACPPermission, ReadACPPermission}}},
+	}
+)
+
+//grant permission
+type Permission string
+
+// grantee
+type Grantee struct {
+	Xmlns        string `xmlns:si,attr,omitempty`
+	Xmlsi        string `xsi:type,attr,omitempty`
+	Id           string `xml:"ID,omitempty"`
+	URI          string `xml:"URI,omitempty"`
+	Type         string `xml:"Type,omitempty"`
+	DisplayName  string `xml:"DisplayName,omitempty"`
+	EmailAddress string `xml:"EmailAddress,omitempty"`
+}
+
+// grant
+type Grant struct {
+	Grantee    Grantee    `xml:"Grantee,omitempty"`
+	Permission Permission `xml:"Permission,omitempty"`
+}
+
+// access control list
+type AccessControlList struct {
+	Grants []Grant `xml:"Grant,omitempty"`
+}
+
+// owner
+type Owner struct {
+	Id          string `xml:"ID"`
+	DispalyName string `xml:"DisplayName"`
+}
+
+// access control policy
+type AccessControlPolicy struct {
+	Xmlns string            `xml:"xmlns:xsi,attr"`
+	Owner Owner             `xml:"Owner,omitempty"`
+	Acl   AccessControlList `xml:"AccessControlList,omitempty"`
+}
+
+func (acp *AccessControlPolicy) Validate(bucket string) (bool, error) {
+	for _, grant := range acp.Acl.Grants {
+		if !grant.Validate() {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+func (acp *AccessControlPolicy) IsAllowed(param *RequestParam) bool {
+	log.LogInfof("acl is allowed ?")
+	if len(acp.Acl.Grants) == 0 {
+		return true
+	}
+	for _, grant := range acp.Acl.Grants {
+		if grant.IsAllowed(param) {
+			return true
+		}
+	}
+	return false
+}
+
+var (
+	aclGrantKeyPermissionMap = map[string]Permission{
+		"x-amz-grant-full-control": FullControlPermission,
+		"x-amz-grant-read":         ReadPermission,
+		"x-amz-grant-read-acp":     ReadACPPermission,
+		"x-amz-grant-write":        WritePermission,
+		"x-amz-grant-write-acp":    WriteACPPermission,
+	}
+	aclRoleURIMap = map[string]string{
+		"AllUsers":    "http://acs.amazonaws.com/groups/global/AllUsers",
+		"LogDelivery": "http://acs.amazonaws.com/groups/s3/LogDelivery",
+	}
+)
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAcl.html
+func (acp *AccessControlPolicy) SetBucketStandardACL(param *RequestParam, acl string) {
+	sacl := StandardACL(acl)
+	var (
+		rolePermissionsMap map[string][]Permission
+		ok                 bool
+	)
+
+	if rolePermissionsMap, ok = aclPermissions[sacl]["bucket"]; !ok {
+		return
+	}
+	for role, permissions := range rolePermissionsMap {
+		grantee := Grantee{}
+		if uri, ok := aclRoleURIMap[role]; ok {
+			grantee.URI = uri
+		} else {
+			grantee.Id = param.account
+			grantee.DisplayName = param.account
+		}
+		for _, p := range permissions {
+			grant := Grant{
+				Grantee:    grantee,
+				Permission: p,
+			}
+			acp.Acl.Grants = append(acp.Acl.Grants, grant)
+		}
+	}
+}
+
+func (acp *AccessControlPolicy) SetBucketGrantACL(param *RequestParam, permission Permission) {
+	grantee := Grantee{
+		Id:          param.account,
+		DisplayName: param.account,
+	}
+	grant := Grant{
+		Grantee:    grantee,
+		Permission: permission,
+	}
+	acp.Acl.Grants = append(acp.Acl.Grants, grant)
+}
+
+func (acl *AccessControlPolicy) Marshal() ([]byte, error) {
+	data, err := xml.Marshal(acl)
+	if err != nil {
+		return nil, err
+	}
+	return append([]byte(xml.Header), data...), nil
+}
+
+func ParseACL(bytes []byte, bucket string) (*AccessControlPolicy, error) {
+	acl := &AccessControlPolicy{}
+	err2 := xml.Unmarshal(bytes, acl)
+	if err2 != nil {
+		return nil, err2
+	}
+
+	ok, err3 := acl.Validate(bucket)
+	if err3 != nil {
+		return nil, err3
+	}
+	if !ok {
+		return nil, errors.New("")
+	}
+
+	return acl, nil
+}
+
+func storeBucketACL(bytes []byte, vol *volume) (*AccessControlPolicy, error) {
+	store, err1 := vol.vm.GetStore()
+	if err1 != nil {
+		return nil, err1
+	}
+
+	acl, err3 := ParseACL(bytes, vol.name)
+	if err3 != nil {
+		return nil, err3
+	}
+
+	err4 := store.Put(vol.name, bucketRootPath, OSS_ACL_KEY, bytes)
+	if err4 != nil {
+		return nil, err4
+	}
+
+	vol.storeACL(acl)
+
+	return acl, nil
+}
+
+func (g Grant) Validate() bool {
+	return true
+}
+
+func (g *Grant) IsAllowed(param *RequestParam) bool {
+	if param.account != g.Grantee.Id {
+		return false
+	}
+	actions := aclBucketPermissionActions[g.Permission]
+	return IsIntersectionActions(actions, param.actions)
+}

objectnode/acl_handler.go

@@ -0,0 +1,154 @@
+package objectnode
+
+// https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/dev/acl-using-rest-api.html
+
+import (
+	"encoding/xml"
+	"errors"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/chubaofs/chubaofs/util/log"
+)
+
+const (
+	OSS_ACL_KEY      = "oss:acl"
+	XMLNS            = "http://www.w3.org/2001/XMLSchema-instance"
+	XMLXSI           = "CanonicalUser"
+	DEF_GRANTEE_TYPE = "CanonicalUser" //
+
+)
+
+var (
+	defaultGrant = Grant{
+		Grantee: Grantee{
+			Xmlns: XMLNS,
+			Xmlsi: XMLXSI,
+			Type:  DEF_GRANTEE_TYPE,
+		},
+		Permission: FullControlPermission,
+	}
+)
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketAcl.html
+func (o *ObjectNode) getBucketACLHandler(w http.ResponseWriter, r *http.Request) {
+	log.LogInfof("Get bucket acl")
+	var (
+		err error
+		ec  *ErrorCode
+	)
+	defer o.errorResponse(w, r, err, ec)
+
+	_, bucket, _, vol, err := o.parseRequestParams(r)
+	if bucket == "" {
+		ec = &NoSuchBucket
+		return
+	}
+
+	//om := vol.OSSMeta()
+	acl := vol.loadACL()
+	var aclData []byte
+	if acl != nil {
+		aclData, err = xml.Marshal(acl)
+		if err != nil {
+			ec = &InternalError
+			return
+		}
+
+	} else {
+		acl = &AccessControlPolicy{}
+		acl.Acl.Grants = append(acl.Acl.Grants, defaultGrant)
+		aclData, err = xml.Marshal(acl)
+		if err != nil {
+			ec = &InternalError
+			return
+		}
+	}
+
+	w.Write(aclData)
+
+	return
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAcl.html#API_PutBucketAcl_RequestSyntax
+//
+func (o *ObjectNode) putBucketACLHandler(w http.ResponseWriter, r *http.Request) {
+	var (
+		err error
+		ec  *ErrorCode
+	)
+	defer o.errorResponse(w, r, err, ec)
+
+	log.LogInfof("Put bucket acl")
+	_, bucket, _, vol, err1 := o.parseRequestParams(r)
+	if err1 != nil {
+		err = err1
+		return
+	}
+	if bucket == "" {
+		err = errors.New("")
+		ec = &NoSuchBucket
+		return
+	}
+
+	bytes, err2 := ioutil.ReadAll(r.Body)
+	if err2 != nil && err2 != io.EOF {
+		err = err2
+		return
+	}
+
+	acl, err3 := ParseACL(bytes, vol.name)
+	if err3 != nil {
+		err = err3
+		return
+	}
+	if acl == nil {
+		err = errors.New("")
+		return
+	}
+
+	//add standard acl request header
+	// https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/dev/acl-overview.html
+	p, err5 := o.parseRequestParam(r)
+	if err5 != nil {
+		err = err5
+		return
+	}
+	if standardAcls, found := r.Header["x-amz-acl"]; found {
+		acl.SetBucketStandardACL(p, standardAcls[0])
+	} else {
+		for grant, permission := range aclGrantKeyPermissionMap {
+			if _, found2 := r.Header[grant]; found2 {
+				acl.SetBucketGrantACL(p, permission)
+			}
+		}
+	}
+
+	newBytes, err6 := acl.Marshal()
+	if err6 != nil {
+		err = err6
+		return
+	}
+
+	// store bucket acl
+	_, err4 := storeBucketACL(newBytes, vol)
+	if err4 != nil {
+		err = err4
+		return
+	}
+
+	return
+}
+
+func (o *ObjectNode) getObjectACLHandler(w http.ResponseWriter, r *http.Request) {
+	//TODO: implement get object acl handler
+
+	return
+}
+
+func (o *ObjectNode) putObjectACLHandler(w http.ResponseWriter, r *http.Request) {
+	//TODO: implement get object acl handler
+
+	return
+}

objectnode/acl_test.go

@@ -0,0 +1,2 @@
+// Package s3 provides ...
+package objectnode

objectnode/api_handler.go

@@ -0,0 +1,129 @@
+// Copyright 2018 The ChubaoFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package objectnode
+
+import (
+	"errors"
+	"net/http"
+	"strings"
+
+	"github.com/gorilla/mux"
+
+	"github.com/chubaofs/chubaofs/util/log"
+)
+
+type RequestParam struct {
+	account  string
+	resource string
+	bucket   string
+	object   string
+	actions  []Action
+	sourceIP string
+	vol      *volume
+	condVals map[string][]string
+	isOwner  bool
+	vars     map[string]string
+}
+
+func (o *ObjectNode) parseRequestParam(r *http.Request) (*RequestParam, error) {
+	p := new(RequestParam)
+	p.vars = mux.Vars(r)
+	p.bucket = p.vars["bucket"]
+	p.object = p.vars["object"]
+	p.vol, _ = o.getVol(p.bucket)
+	p.sourceIP = getRequestIP(r)
+	p.condVals = getCondtionValues(r)
+	if len(p.bucket) > 0 {
+		p.resource = p.bucket
+		if len(p.object) > 0 {
+			if strings.HasPrefix(p.object, "/") {
+				p.resource = p.bucket + p.object
+			} else {
+				p.resource = p.bucket + "/" + p.object
+			}
+		}
+	}
+	auth := parseRequestAuthInfo(r)
+	if auth != nil && p.vol != nil {
+		accessKey, _ := p.vol.OSSSecure()
+		p.account = accessKey
+		if auth.accessKey == accessKey {
+			p.isOwner = true
+		}
+	}
+
+	return p, nil
+}
+
+//Deprecated:
+func (o *ObjectNode) parseRequestParams(r *http.Request) (vars map[string]string, bucket, object string, vl *volume, err error) {
+	vars = mux.Vars(r)
+	bucket = vars["bucket"]
+	object = vars["object"]
+	if bucket != "" {
+		if vm, ok := o.vm.(*volumeManager); ok {
+			vl, err = vm.loadVolume(bucket)
+			if err != nil {
+				log.LogErrorf("parseRequestParams: load volume fail, requestId(%o) bucket(%v) err(%v)",
+					RequestIDFromRequest(r), bucket, err)
+			}
+		} else {
+			log.LogErrorf("parseRequestParams: load volume fail, requestId(%o) bucket(%v) err(%v)",
+				RequestIDFromRequest(r), bucket, err)
+		}
+	}
+	return
+}
+
+func (o *ObjectNode) getVol(bucket string) (vol *volume, err error) {
+	if bucket == "" {
+		return nil, errors.New("bucket name is empty")
+	}
+	vm, ok := o.vm.(*volumeManager)
+	if !ok {
+		return nil, errors.New("volumeManger is invalid")
+	}
+
+	vol, err = vm.loadVolume(bucket)
+	if err != nil {
+		log.LogErrorf("parseRequestParams: load volume fail, bucket(%v) err(%v)", bucket, err)
+		return nil, err
+	}
+
+	return vol, nil
+}
+
+func (o *ObjectNode) errorResponse(w http.ResponseWriter, r *http.Request, err error, ec *ErrorCode) {
+	if err != nil || ec != nil {
+		if err != nil {
+			log.LogErrorf("Handler: parse request parameters fail, requestID(%o) err(%v)",
+				RequestIDFromRequest(r), err)
+		}
+		if ec == nil {
+			ec = &InternalError
+		}
+		_ = ec.ServeResponse(w, r)
+	}
+}
+
+func (o *ObjectNode) unsupportedOperationHandler(w http.ResponseWriter, r *http.Request) {
+	var err error
+	if err = UnsupportedOperation.ServeResponse(w, r); err != nil {
+		log.LogErrorf("unsupportedOperationHandler: serve response fail: requestID(%v) err(%v)",
+			RequestIDFromRequest(r), err)
+		ServeInternalStaticErrorResponse(w, r)
+	}
+	return
+}

objectnode/api_handler_bucket.go

@@ -0,0 +1,40 @@
+package objectnode
+
+import (
+	"net/http"
+
+	"github.com/chubaofs/chubaofs/util/log"
+)
+
+// Head bucket
+// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadBucket.html
+func (o *ObjectNode) headBucketHandler(w http.ResponseWriter, r *http.Request) {
+	// do nothing
+}
+
+// List buckets
+// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBuckets.html
+func (o *ObjectNode) listBucketsHandler(w http.ResponseWriter, r *http.Request) {
+	// TODO: implement 'listBucketsHandler'
+}
+
+// Get bucket location
+// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLocation.html
+func (o *ObjectNode) getBucketLocation(w http.ResponseWriter, r *http.Request) {
+	log.LogInfof("getBucketLocation: get bucket location: requestID(%v)", RequestIDFromRequest(r))
+	// TODO: implement 'getBucketLocation'
+	var output = &GetBucketLocationOutput{
+		LocationConstraint: o.region,
+	}
+	var marshaled []byte
+	var err error
+	if marshaled, err = MarshalXMLEntity(output); err != nil {
+		log.LogErrorf("getBucketLocation: marshal result fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+		ServeInternalStaticErrorResponse(w, r)
+		return
+	}
+	if _, err = w.Write(marshaled); err != nil {
+		log.LogErrorf("getBucketLocation: write response body fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+	}
+	return
+}

objectnode/api_handler_multipart.go

@@ -0,0 +1,373 @@
+// Copyright 2018 The ChubaoFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package objectnode
+
+import (
+	"net/http"
+	"strconv"
+
+	"github.com/chubaofs/chubaofs/util/log"
+)
+
+// Create multipart upload
+// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html
+func (o *ObjectNode) createMultipleUploadHandler(w http.ResponseWriter, r *http.Request) {
+	log.LogInfof("createMultipleUploadHandler: init multiple upload, requestID(%v) remote(%v)",
+		RequestIDFromRequest(r), r.RemoteAddr)
+
+	_, bucket, object, vl, err := o.parseRequestParams(r)
+	if err != nil {
+		log.LogErrorf("createMultipleUploadHandler: parse request parameters fail, requestID(%o) err(%v)",
+			RequestIDFromRequest(r), err)
+		_ = NoSuchBucket.ServeResponse(w, r)
+		return
+	}
+	uploadId, initErr := vl.InitMultipart(object)
+	if initErr != nil {
+		log.LogErrorf("createMultipleUploadHandler:  init multipart fail, requestID(%o) err(%v)",
+			RequestIDFromRequest(r), err)
+		_ = NoSuchBucket.ServeResponse(w, r)
+		return
+	}
+
+	initResult := InitMultipartResult{
+		Bucket:   bucket,
+		Key:      object,
+		UploadId: uploadId,
+	}
+
+	var bytes []byte
+	var marshalError error
+	if bytes, marshalError = MarshalXMLEntity(initResult); marshalError != nil {
+		log.LogErrorf("createMultipleUploadHandler: marshal result fail, requestID(%v) err(%v)",
+			RequestIDFromRequest(r), err)
+		_ = InternalError.ServeResponse(w, r)
+		return
+	}
+
+	// set response header
+	w.Header().Set(HeaderNameContentType, HeaderValueContentTypeXML)
+	w.Header().Set(HeaderNameContentLength, strconv.Itoa(len(bytes)))
+	if _, err = w.Write(bytes); err != nil {
+		log.LogErrorf("createMultipleUploadHandler: write response body fail, requestID(%v) err(%v)",
+			RequestIDFromRequest(r), err)
+	}
+	return
+}
+
+// Upload part
+// Uploads a part in a multipart upload.
+// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html .
+func (o *ObjectNode) uploadPartHandler(w http.ResponseWriter, r *http.Request) {
+	log.LogInfof("uploadPartHandler: upload part, requestID(%v) remote(%v)",
+		RequestIDFromRequest(r), r.RemoteAddr)
+	// check args
+	params, _, object, vl, err := o.parseRequestParams(r)
+	if err != nil {
+		log.LogErrorf("uploadPartHandler: parse request parameters fail, requestID(%o) err(%v)", RequestIDFromRequest(r), err)
+		_ = NoSuchBucket.ServeResponse(w, r)
+		return
+	}
+
+	//// get upload id and part number
+	uploadId := params[ParamUploadId]
+	partNumber := params[ParamPartNumber]
+	if uploadId == "" || partNumber == "" {
+		log.LogErrorf("uploadPartHandler: illegal uploadID or partNumber, requestID(%v)", RequestIDFromRequest(r))
+		_ = InvalidArgument.ServeResponse(w, r)
+		return
+	}
+
+	var partNumberInt uint64
+	if partNumberInt, err = strconv.ParseUint(partNumber, 10, 64); err != nil {
+		log.LogErrorf("uploadPartHandler: parse part number fail, requestID(%o) raw(%v) err(%v)",
+			RequestIDFromRequest(r), partNumber, err)
+		_ = InvalidArgument.ServeResponse(w, r)
+		return
+	}
+
+	// handle exception
+
+	var fsFileInfo *FSFileInfo
+	if fsFileInfo, err = vl.WritePart(object, uploadId, uint16(partNumberInt), r.Body); err != nil {
+		log.LogErrorf("uploadPartHandler: write part fail, requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+		_ = InternalError.ServeResponse(w, r)
+		return
+	}
+	log.LogDebugf("uploadPartHandler: write part, requestID(%v) fsFileInfo(%v)", RequestIDFromRequest(r), fsFileInfo)
+
+	// write header to response
+	w.Header().Set(HeaderNameContentLength, "0")
+	w.Header().Set(HeaderNameETag, fsFileInfo.ETag)
+	return
+}
+
+// List parts
+// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html
+func (o *ObjectNode) listPartsHandler(w http.ResponseWriter, r *http.Request) {
+	log.LogInfof("listPartsHandler: list parts, requestID(%v) remote(%v)", RequestIDFromRequest(r), r.RemoteAddr)
+
+	// check args
+	params, bucket, object, vl, err := o.parseRequestParams(r)
+	if err != nil {
+		log.LogErrorf("listPartsHandler: parse request parameters fail, requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+		_ = NoSuchBucket.ServeResponse(w, r)
+		return
+	}
+	//// get upload id and part number
+	uploadId := params[ParamUploadId]
+	maxParts := params[ParamMaxParts]
+	partNoMarker := params[ParamPartNoMarker]
+
+	var maxPartsInt uint64
+	var partNoMarkerInt uint64
+
+	if uploadId == "" {
+		log.LogErrorf("listPartsHandler: illegal update ID, requestID(%o) err(%v)", RequestIDFromRequest(r), err)
+		_ = InvalidArgument.ServeResponse(w, r)
+		return
+	}
+
+	if maxParts == "" {
+		maxPartsInt = MaxParts
+	} else {
+		maxPartsInt, err = strconv.ParseUint(maxParts, 10, 64)
+		if err != nil {
+			log.LogErrorf("listPartsHandler: parse max parts fail, requestID(%v) raw(%v) err(%v)", RequestIDFromRequest(r), maxParts, err)
+			_ = InvalidArgument.ServeResponse(w, r)
+			return
+		}
+		if maxPartsInt > MaxParts {
+			maxPartsInt = MaxParts
+		}
+	}
+	if partNoMarker != "" {
+		res, err := strconv.ParseUint(uploadId, 10, 64)
+		if err != nil {
+			log.LogErrorf("listPatsHandler: parse update ID fail, requestID(%v) raw(%v) err(%v)", RequestIDFromRequest(r), uploadId, err)
+			_ = InvalidArgument.ServeResponse(w, r)
+			return
+		}
+		partNoMarkerInt = res
+	}
+
+	fsParts, nextMarker, isTruncated, err := vl.ListParts(object, uploadId, maxPartsInt, partNoMarkerInt)
+	if err != nil {
+		log.LogErrorf("listPartsHandler: volume list parts fail, requestID(%v) uploadID(%v) maxParts(%v) partNoMarker(%v) err(%v)",
+			RequestIDFromRequest(r), uploadId, maxPartsInt, partNoMarkerInt, err)
+		_ = InternalError.ServeResponse(w, r)
+		return
+	}
+	log.LogDebugf("listPartsHandler: volume list parts, "+
+		"requestID(%v) uploadID(%v) maxParts(%v) partNoMarker(%v) numFSParts(%v) nextMarker(%v) isTruncated(%v)",
+		RequestIDFromRequest(r), uploadId, maxPartsInt, partNoMarkerInt, len(fsParts), nextMarker, isTruncated)
+
+	// get owner
+	accessKey, _ := vl.OSSSecure()
+	bucketOwner := NewBucketOwner(accessKey)
+
+	// get parts
+	parts := NewParts(fsParts)
+
+	listPartsResult := ListPartsResult{
+		Bucket:       bucket,
+		Key:          object,
+		UploadId:     uploadId,
+		StorageClass: StorageClassStandard,
+		NextMarker:   int(nextMarker),
+		MaxParts:     int(maxPartsInt),
+		IsTruncated:  isTruncated,
+		Parts:        parts,
+		Owner:        bucketOwner,
+	}
+
+	var bytes []byte
+	var marshalError error
+	if bytes, marshalError = MarshalXMLEntity(listPartsResult); marshalError != nil {
+		log.LogErrorf("listPartsHandler: marshal result fail, requestID(%o) err(%v)",
+			RequestIDFromRequest(r), err)
+		_ = InternalError.ServeResponse(w, r)
+		return
+	}
+
+	// set response header
+	w.Header().Set(HeaderNameContentType, HeaderValueContentTypeXML)
+	w.Header().Set(HeaderNameContentLength, strconv.Itoa(len(bytes)))
+	if _, err = w.Write(bytes); err != nil {
+		log.LogErrorf("listPartsHandler: write response body fail, requestID(%v) err(%v)",
+			RequestIDFromRequest(r), err)
+	}
+	return
+}
+
+// Complete multipart
+// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html
+func (o *ObjectNode) completeMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
+	log.LogInfof("completeMultipartUploadHandler: complete multiple upload, requestID(%v) remote(%v)", RequestIDFromRequest(r), r.RemoteAddr)
+
+	params, bucket, object, vl, err := o.parseRequestParams(r)
+	if err != nil {
+		log.LogErrorf("[UploadPart] request id [%o], Get volume failed cause : %o", r.URL, err)
+		_ = NoSuchBucket.ServeResponse(w, r)
+		return
+	}
+
+	// get upload id and part number
+	uploadId := params[ParamUploadId]
+	if uploadId == "" {
+		log.LogErrorf("[CompleteMultipartUpload] request id [%o], Upload id can not be empty", r.URL.String())
+		_ = InvalidArgument.ServeResponse(w, r)
+		return
+	}
+
+	fsFileInfo, err := vl.CompleteMultipart(object, uploadId)
+	if err != nil {
+		log.LogErrorf("completeMultipartUploadHandler: complete multipart fail, requestID(%o) uploadID(%v) err(%v)",
+			RequestIDFromRequest(r), uploadId, err)
+		_ = InternalError.ServeResponse(w, r)
+		return
+	}
+	log.LogDebugf("completeMultipartUploadHandler: complete multipart, requestID(%o) uploadID(%v) path(%v)",
+		RequestIDFromRequest(r), uploadId, object)
+
+	// write response
+	completeResult := CompleteMultipartResult{
+		Bucket: bucket,
+		Key:    object,
+		ETag:   fsFileInfo.ETag,
+	}
+
+	var bytes []byte
+	var marshalError error
+	if bytes, marshalError = MarshalXMLEntity(completeResult); marshalError != nil {
+		log.LogErrorf("completeMultipartUploadHandler: marshal result fail, requestID(%o) err(%v)", RequestIDFromRequest(r), marshalError)
+		_ = InternalError.ServeResponse(w, r)
+		return
+	}
+
+	// set response header
+	w.Header().Set(HeaderNameContentType, HeaderValueContentTypeXML)
+	w.Header().Set(HeaderNameContentLength, strconv.Itoa(len(bytes)))
+	if _, err = w.Write(bytes); err != nil {
+		log.LogErrorf("completeMultipartUploadHandler: write response body fail, requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+		return
+	}
+	return
+}
+
+// Abort multipart
+// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html .
+func (o *ObjectNode) abortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
+	log.LogInfof("abortMultipartUploadHandler: abort multiple upload, requestID(%v) remote(%v)", RequestIDFromRequest(r), r.RemoteAddr)
+
+	// check args
+	params, _, object, vl, err := o.parseRequestParams(r)
+	if err != nil {
+		log.LogErrorf("abortMultipartUploadHandler: parse request parameters fail, requestID(%o) err(%v)", RequestIDFromRequest(r), err)
+		_ = NoSuchBucket.ServeResponse(w, r)
+		return
+	}
+
+	uploadId := params["uploadId"]
+	//// Abort multipart upload
+	err = vl.AbortMultipart(object, uploadId)
+	if err != nil {
+		log.LogErrorf("abortMultipartUploadHandler: volume abort multipart fail, requestID(%o) uploadID(%v) err(%v)", RequestIDFromRequest(r), uploadId, err)
+		_ = InternalError.ServeResponse(w, r)
+		return
+	}
+	log.LogDebugf("abortMultipartUploadHandler: volume abort multipart, requestID(%o) uploadID(%v) path(%v)", RequestIDFromRequest(r), uploadId, object)
+	return
+}
+
+// List multipart uploads
+// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html
+func (o *ObjectNode) listMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
+	log.LogInfof("abortMultipartUploadHandler: list multipart uploads, requestID(%v) remote(%v)", RequestIDFromRequest(r), r.RemoteAddr)
+	// check args
+	params, bucket, _, vl, err := o.parseRequestParams(r)
+	if err != nil {
+		log.LogErrorf("abortMultipartUploadHandler: parse request parameters fail, requestID(%o) err(%v)", RequestIDFromRequest(r), err)
+		_ = NoSuchBucket.ServeResponse(w, r)
+	}
+
+	// get list uploads parameter
+	prefix := params[ParamPrefix]
+	keyMarker := params[ParamKeyMarker]
+	delimiter := params[ParamPartDelimiter]
+	maxUploads := params[ParamPartMaxUploads]
+	uploadIdMarker := params[ParamUploadIdMarker]
+
+	var maxUploadsInt uint64
+	if maxUploads == "" {
+		maxUploadsInt = MaxUploads
+	} else {
+		maxUploadsInt, err = strconv.ParseUint(maxUploads, 10, 64)
+		if err != nil {
+			log.LogErrorf("[ListMultipartUploads] request id [%o], Param max-uploads is invalid, failed info : %o", r.URL, err)
+			_ = InvalidArgument.ServeResponse(w, r)
+			return
+		}
+		if maxUploadsInt > MaxUploads {
+			maxUploadsInt = MaxUploads
+		}
+	}
+
+	fsUploads, nextKeyMarker, nextUploadIdMarker, IsTruncated, prefixes, err := vl.ListMultipartUploads(prefix, delimiter, keyMarker, uploadIdMarker, maxUploadsInt)
+	if err != nil {
+		log.LogErrorf("[ListMultipartUploads] request id [%o], List multipart uploads failed cause : %o", r.URL, err)
+		_ = NoSuchBucket.ServeResponse(w, r)
+		return
+	}
+
+	accessKey, _ := vl.OSSSecure()
+	uploads := NewUploads(fsUploads, accessKey)
+
+	var commonPrefixes = make([]*CommonPrefix, 0)
+	for _, prefix := range prefixes {
+		commonPrefix := &CommonPrefix{
+			Prefix: prefix,
+		}
+		commonPrefixes = append(commonPrefixes, commonPrefix)
+	}
+
+	listUploadsResult := ListUploadsResult{
+		Bucket:             bucket,
+		KeyMarker:          keyMarker,
+		UploadIdMarker:     uploadIdMarker,
+		NextKeyMarker:      nextKeyMarker,
+		NextUploadIdMarker: nextUploadIdMarker,
+		Delimiter:          delimiter,
+		Prefix:             prefix,
+		MaxUploads:         int(maxUploadsInt),
+		IsTruncated:        IsTruncated,
+		Uploads:            uploads,
+		CommonPrefixes:     commonPrefixes,
+	}
+
+	var bytes []byte
+	var marshalError error
+	if bytes, marshalError = MarshalXMLEntity(listUploadsResult); marshalError != nil {
+		log.LogErrorf("[ListMultipartUploads] request id [%o], Marshal list uploads result failed cause : %o", r.URL, err)
+		_ = InternalError.ServeResponse(w, r)
+		return
+	}
+
+	// set response header
+	w.Header().Set(HeaderNameContentType, HeaderValueContentTypeXML)
+	w.Header().Set(HeaderNameContentLength, strconv.Itoa(len(bytes)))
+	w.Write(bytes)
+	return
+}

objectnode/api_handler_object.go

@@ -0,0 +1,790 @@
+// Copyright 2018 The ChubaoFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package objectnode
+
+import (
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"sync"
+	"syscall"
+
+	"github.com/chubaofs/chubaofs/util/log"
+)
+
+var (
+	rangeRegexp = regexp.MustCompile("^bytes=(\\d)+-(\\d)*$")
+)
+
+// Get object
+// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html
+func (o *ObjectNode) getObjectHandler(w http.ResponseWriter, r *http.Request) {
+	// TODO: range read support
+	log.LogInfof("getObjectHandler: get object, requestID(%v) remote(%v)", RequestIDFromRequest(r), r.RemoteAddr)
+
+	_, _, object, vl, err := o.parseRequestParams(r)
+	if err != nil {
+		log.LogErrorf("getObjectHandler: parse request parameters fail, requestId(%o) err(%v)", RequestIDFromRequest(r), err)
+		_ = NoSuchBucket.ServeResponse(w, r)
+		return
+	}
+
+	// parse http range option
+	var rangeOpt = strings.TrimSpace(r.Header.Get(HeaderNameRange))
+	var rangeLower uint64
+	var rangeUpper uint64
+	var isRangeRead bool
+	if len(rangeOpt) > 0 && rangeRegexp.MatchString(rangeOpt) {
+
+		var hyphenIndex = strings.Index(rangeOpt, "-")
+		if hyphenIndex < 0 {
+			_ = InvalidArgument.ServeResponse(w, r)
+			return
+		}
+
+		var lowerPart = rangeOpt[len("bytes="):hyphenIndex]
+		var upperPart = ""
+		if hyphenIndex+1 < len(rangeOpt) {
+			upperPart = rangeOpt[hyphenIndex+1:]
+		}
+
+		if len(lowerPart) > 0 {
+			if rangeLower, err = strconv.ParseUint(lowerPart, 10, 64); err != nil {
+				log.LogErrorf("getObjectHandler: parse range lower fail: requestID(%v) rangeOpt(%v) err(%v)",
+					RequestIDFromRequest(r), rangeOpt, err)
+				ServeInternalStaticErrorResponse(w, r)
+				return
+			}
+		}
+		if len(upperPart) > 0 {
+			if rangeUpper, err = strconv.ParseUint(upperPart, 10, 64); err != nil {
+				log.LogErrorf("getObjectHandler: parse range upper fail: requestID(%v) rangeOpt(%v) err(%v)",
+					RequestIDFromRequest(r), rangeOpt, err)
+				ServeInternalStaticErrorResponse(w, r)
+				return
+			}
+		}
+		if rangeUpper > 0 && rangeUpper < rangeLower {
+			// upper enabled and lower than lower side
+			if err = InvalidArgument.ServeResponse(w, r); err != nil {
+				log.LogErrorf("getObjectHandler: serve response fail: requestID(%v) err(%v)",
+					RequestIDFromRequest(r), err)
+				return
+			}
+		}
+
+		isRangeRead = true
+		log.LogDebugf("getObjectHandler: parse range option: requestID(%v) rangeOpt(%v) rangeLower(%v) rangeUpper(%v)",
+			RequestIDFromRequest(r), rangeOpt, rangeLower, rangeUpper)
+	}
+
+	// get object meta
+	fileInfo, err := vl.FileInfo(object)
+	if err != nil {
+		log.LogErrorf("getObjectHandler: volume get file info fail, requestId(%o) err(%v)", RequestIDFromRequest(r), err)
+		_ = NoSuchKey.ServeResponse(w, r)
+		return
+	}
+
+	// validate and fix range
+	if isRangeRead && rangeUpper > uint64(fileInfo.Size) {
+		rangeUpper = uint64(fileInfo.Size)
+	}
+
+	// compute content length
+	var contentLength = uint64(fileInfo.Size)
+	if isRangeRead {
+		contentLength = rangeUpper - rangeLower
+	}
+
+	// set response header for GetObject
+	w.Header().Set(HeaderNameETag, fileInfo.ETag)
+	w.Header().Set(HeaderNameAcceptRange, HeaderValueAcceptRange)
+	w.Header().Set(HeaderNameLastModified, formatTimeRFC1123(fileInfo.ModifyTime))
+	w.Header().Set(HeaderNameContentType, HeaderValueTypeStream)
+	w.Header().Set(HeaderNameContentLength, strconv.FormatUint(contentLength, 10))
+
+	if isRangeRead {
+		w.Header().Set(HeaderNameContentRange, fmt.Sprintf("bytes %d-%d/%d", rangeLower, rangeUpper, fileInfo.Size))
+	}
+
+	// get object content
+	var offset = rangeLower
+	var size = uint64(fileInfo.Size)
+	if isRangeRead {
+		if rangeUpper == 0 {
+			size = uint64(fileInfo.Size) - rangeLower
+		} else {
+			size = rangeUpper - rangeLower
+		}
+	}
+	if err = vl.ReadFile(object, w, offset, size); err != nil {
+		log.LogErrorf("getObjectHandler: read from volume fail: requestId(%v) volume(%v) path(%v) offset(%v) size(%v) err(%v)",
+			RequestIDFromRequest(r), vl.name, object, offset, size, err)
+		_ = InternalError.ServeResponse(w, r)
+		return
+	}
+	log.LogDebugf("getObjectHandler: volume read file: requestID(%v) volume(%v) path(%v) offset(%v) size(%v)",
+		RequestIDFromRequest(r), vl.name, object, offset, size)
+	return
+}
+
+// Head object
+// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html
+func (o *ObjectNode) headObjectHandler(w http.ResponseWriter, r *http.Request) {
+	log.LogInfof("headObjectHandler: get object meta, requestID(%v) remote(%v)", RequestIDFromRequest(r), r.RemoteAddr)
+
+	// check args
+	_, _, object, vl, err := o.parseRequestParams(r)
+	log.LogInfof("headObjectHandler: parse request params result in header object handler, object : (%o), vl : (%o), err : (%o)", object, vl.name, err)
+	if err != nil {
+		log.LogErrorf("headObjectHandler: parse request parameters fail, requestId(%o) err(%v)", RequestIDFromRequest(r), err)
+		_ = NoSuchBucket.ServeResponse(w, r)
+		return
+	}
+
+	// get object meta
+	fileInfo, err := vl.FileInfo(object)
+	if err != nil && err == syscall.ENOENT {
+		log.LogErrorf("headObjectHandler: get file meta fail, requestId(%o), err(%v)", RequestIDFromRequest(r), err)
+		_ = NoSuchKey.ServeResponse(w, r)
+		return
+	}
+	if err != nil {
+		log.LogErrorf("headObjectHandler: get file meta fail, requestId(%o), err(%v)", RequestIDFromRequest(r), err)
+		_ = InternalError.ServeResponse(w, r)
+		return
+	}
+
+	// set response header
+	w.Header().Set(HeaderNameETag, fileInfo.ETag)
+	w.Header().Set(HeaderNameAcceptRange, HeaderValueAcceptRange)
+	w.Header().Set(HeaderNameContentType, HeaderValueTypeStream)
+	w.Header().Set(HeaderNameLastModified, formatTimeRFC1123(fileInfo.ModifyTime))
+	w.Header().Set(HeaderNameContentLength, strconv.Itoa(int(fileInfo.Size)))
+	w.Header().Set(HeaderNameContentMD5, EmptyContentMD5String)
+	return
+}
+
+// Delete objects (multiple objects)
+// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
+func (o *ObjectNode) deleteObjectsHandler(w http.ResponseWriter, r *http.Request) {
+	log.LogInfof("deleteObjectsHandler: delete multiple objects, requestID(%v) remote(%v)",
+		RequestIDFromRequest(r), r.RemoteAddr)
+	// check args
+	_, _, _, vl, err := o.parseRequestParams(r)
+
+	bytes, err := ioutil.ReadAll(r.Body)
+	if err != nil && err != io.EOF {
+		log.LogErrorf("deleteObjectsHandler: read request body fail: requestID(%o) err(%v)", RequestIDFromRequest(r), err)
+		_ = InternalError.ServeResponse(w, r)
+		return
+	}
+
+	deleteReq := DeleteRequest{}
+	err = UnmarshalXMLEntity(bytes, &deleteReq)
+	if err != nil {
+		log.LogErrorf("deleteObjectsHandler: unmarshal xml fail: requestID(%o) err(%v)",
+			RequestIDFromRequest(r), err)
+		_ = InvalidArgument.ServeResponse(w, r)
+		return
+	}
+
+	if len(deleteReq.Objects) <= 0 {
+		log.LogDebugf("deleteObjectsHandler: non objects found in request: requestID(%o)", RequestIDFromRequest(r))
+		_ = InvalidArgument.ServeResponse(w, r)
+		return
+	}
+
+	var wg sync.WaitGroup
+	deletesResult := DeletesResult{
+		DeletedObjects: make([]Deleted, 0),
+		DeletedErrors:  make([]Error, 0),
+	}
+	deletedObjectsCh := make(chan *Deleted, len(deleteReq.Objects))
+	deletedErrorsCh := make(chan *Error, len(deleteReq.Objects))
+
+	for _, object := range deleteReq.Objects {
+		wg.Add(1)
+		go func(obj Object) {
+			defer func() {
+				wg.Done()
+				if errDelete := recover(); errDelete != nil {
+					deletedError := Error{
+						Key:     obj.Key,
+						Code:    strconv.Itoa(InternalError.StatusCode),
+						Message: InternalError.ErrorMessage,
+					}
+					deletedErrorsCh <- &deletedError
+				}
+			}()
+
+			err = vl.DeleteFile(obj.Key)
+			if err != nil {
+			ossError := transferError(obj.Key, err)
+				deletedErrorsCh <- &ossError
+			} else {
+				deleted := Deleted{Key: obj.Key}
+				deletedObjectsCh <- &deleted
+				log.LogDebugf("deleteObjectsHandler: delete object: requestID(%v) key(%o)", RequestIDFromRequest(r),
+					deleted.Key)
+			}
+		}(object)
+	}
+
+	wg.Wait()
+	close(deletedObjectsCh)
+	close(deletedErrorsCh)
+
+	for {
+		deletedObject := <-deletedObjectsCh
+		if deletedObject == nil {
+			break
+		}
+		deletesResult.DeletedObjects = append(deletesResult.DeletedObjects, *deletedObject)
+	}
+	for {
+		deletedError := <-deletedErrorsCh
+		if deletedError == nil {
+			break
+		}
+		deletesResult.DeletedErrors = append(deletesResult.DeletedErrors, *deletedError)
+	}
+
+	log.LogDebugf("deleteObjectsHandler: delete objects: deletes(%v) errors(%v)", len(deletesResult.DeletedObjects), len(deletesResult.DeletedErrors))
+	var bytesRes []byte
+	var marshalError error
+	if bytesRes, marshalError = MarshalXMLEntity(deletesResult); marshalError != nil {
+		log.LogErrorf("deleteObjectsHandler: marshal xml entity fail: requestID(%o) err(%v)", RequestIDFromRequest(r), err)
+		if respErr := InternalError.ServeResponse(w, r); respErr != nil {
+			log.LogErrorf("deleteObjectsHandler: write response fail: requestID(%v) err(%v)", RequestIDFromRequest(r), respErr)
+			return
+		}
+		return
+	}
+
+	// set response header
+	w.Header().Set(HeaderNameContentType, HeaderValueContentTypeXML)
+	w.Header().Set(HeaderNameContentLength, strconv.Itoa(len(bytesRes)))
+	if _, err = w.Write(bytesRes); err != nil {
+		log.LogErrorf("deleteObjectsHandler: write response body fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+	}
+	return
+}
+
+func parseCopySourceInfo(r *http.Request) (sourceBucket, sourceObject string) {
+	var copySource = r.Header.Get(HeaderNameCopySource)
+	if strings.HasPrefix(copySource, "/") {
+		copySource = copySource[1:]
+	}
+	position := strings.Index(copySource, "/")
+	var bucket, object string
+	if position >= 0 {
+		bucket = copySource[:position]
+		if position+1 <= len(copySource) {
+			object = copySource[position+1:]
+		}
+	}
+	sourceBucket = bucket
+	sourceObject = object
+	return
+}
+
+// Copy object
+// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html .
+func (o *ObjectNode) copyObjectHandler(w http.ResponseWriter, r *http.Request) {
+	// check args
+	_, bucket, object, vl, err := o.parseRequestParams(r)
+	if err != nil {
+		log.LogErrorf("copyObjectHandler: parse request params fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+		_ = NoSuchBucket.ServeResponse(w, r)
+		return
+	}
+
+	sourceBucket, sourceObject := parseCopySourceInfo(r)
+	if bucket != sourceBucket {
+		log.LogDebugf("copyObjectHandler: source bucket is not same with bucket: requestID(%v) target(%v) source(%v)",
+			RequestIDFromRequest(r), bucket, sourceBucket)
+		_ = UnsupportedOperation.ServeResponse(w, r)
+		return
+	}
+
+	if sourceObject == object {
+		log.LogErrorf("copyObjectHandler: source object same with target object: requestID(%v) target(%v) source(%v)",
+			RequestIDFromRequest(r), object, sourceObject)
+		_ = InvalidArgument.ServeResponse(w, r)
+		return
+	}
+
+	// get object meta
+	fileInfo, err := vl.FileInfo(sourceObject)
+	if err != nil {
+		log.LogErrorf("copyObjectHandler: volume get file info fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+		_ = NoSuchKey.ServeResponse(w, r)
+		return
+	}
+
+	// get header
+	copyMatch := r.Header.Get(HeaderNameCopyMatch)
+	noneMatch := r.Header.Get(HeaderNameCopyNoneMatch)
+	modified := r.Header.Get(HeaderNameCopyModified)
+	unModified := r.Header.Get(HeaderNameCopyUnModified)
+
+	// response 412
+	if modified != "" {
+		fileModTime := fileInfo.ModifyTime
+		modifiedTime, err := parseTimeRFC1123(modified)
+		if err != nil {
+			log.LogErrorf("copyObjectHandler: parse RFC1123 time fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+			_ = InvalidArgument.ServeResponse(w, r)
+			return
+		}
+		if fileModTime.Before(modifiedTime) {
+			log.LogInfof("copyObjectHandler: file modified time not after than specified time: requestID(%v)", RequestIDFromRequest(r))
+			_ = PreconditionFailed.ServeResponse(w, r)
+			return
+		}
+	}
+	if unModified != "" {
+		fileModTime := fileInfo.ModifyTime
+		unmodifiedTime, err := parseTimeRFC1123(unModified)
+		if err != nil {
+			log.LogErrorf("copyObjectHandler: parse RFC1123 time fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+			_ = InvalidArgument.ServeResponse(w, r)
+			return
+		}
+		if fileModTime.After(unmodifiedTime) {
+			log.LogInfof("copyObjectHandler: file modified time not before than specified time: requestID(%v)", RequestIDFromRequest(r))
+			_ = PreconditionFailed.ServeResponse(w, r)
+			return
+		}
+	}
+	if copyMatch != "" && fileInfo.ETag != copyMatch {
+		log.LogInfof("copyObjectHandler: eTag mismatched with specified: requestID(%v)", RequestIDFromRequest(r))
+		_ = PreconditionFailed.ServeResponse(w, r)
+		return
+	}
+	if noneMatch != "" && fileInfo.ETag == noneMatch {
+		log.LogInfof("copyObjectHandler: eTag same with specified: requestID(%v)", RequestIDFromRequest(r))
+		_ = PreconditionFailed.ServeResponse(w, r)
+		return
+	}
+
+	fsFileInfo, err := vl.CopyFile(object, sourceObject)
+	if err != nil {
+		log.LogErrorf("copyObjectHandler: volume copy file fail: requestID(%o) volume(%v) source(%v) target(%v) err(%v)",
+			RequestIDFromRequest(r), vl.name, sourceObject, object, err)
+		_ = InternalError.ServeResponse(w, r)
+		return
+	}
+
+	copyResult := CopyResult{
+		ETag:         fsFileInfo.ETag,
+		LastModified: formatTimeISO(fsFileInfo.ModifyTime),
+	}
+
+	var bytes []byte
+	var marshalError error
+	if bytes, marshalError = MarshalXMLEntity(copyResult); marshalError != nil {
+		log.LogErrorf("copyObjectHandler: marshal xml entity fail: requestID(%o) err(%v)", RequestIDFromRequest(r), err)
+		_ = InternalError.ServeResponse(w, r)
+		return
+	}
+
+	// set response header
+	w.Header().Set(HeaderNameContentType, HeaderValueContentTypeXML)
+	w.Header().Set(HeaderNameContentLength, strconv.Itoa(len(bytes)))
+	_, _ = w.Write(bytes)
+
+	return
+}
+
+// List objects v1
+// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjects.html
+func (o *ObjectNode) getBucketV1Handler(w http.ResponseWriter, r *http.Request) {
+	// check args
+	_, bucket, _, vl, err := o.parseRequestParams(r)
+	if err != nil {
+		log.LogErrorf("getBucketV1Handler: parse request parameters fail, requestID(%o) err(%v)",
+			RequestIDFromRequest(r), err)
+		_ = NoSuchBucket.ServeResponse(w, r)
+		return
+	}
+
+	// get options
+	marker := r.URL.Query().Get(ParamMarker)
+	prefix := r.URL.Query().Get(ParamPrefix)
+	maxKeys := r.URL.Query().Get(ParamMaxKeys)
+	delimiter := r.URL.Query().Get(ParamPartDelimiter)
+
+	var maxKeysInt uint64
+	if maxKeys != "" {
+		maxKeysInt, err = strconv.ParseUint(maxKeys, 10, 16)
+		if err != nil {
+			log.LogErrorf("getBucketV1Handler: parse max key fail, requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+			_ = InvalidArgument.ServeResponse(w, r)
+			return
+		}
+		if maxKeysInt > MaxKeys {
+			maxKeysInt = MaxKeys
+		}
+	} else {
+		maxKeysInt = uint64(MaxKeys)
+	}
+
+	listBucketRequest := &ListBucketRequestV1{
+		prefix:    prefix,
+		delimiter: delimiter,
+		marker:    marker,
+		maxKeys:   maxKeysInt,
+	}
+
+	fsFileInfos, nextMarker, isTruncated, prefixes, err := vl.ListFilesV1(listBucketRequest)
+	if err != nil {
+		log.LogErrorf("getBucketV1Handler: list file fail, requestID(%o), err(%o)", r.URL, err)
+		_ = InvalidArgument.ServeResponse(w, r)
+		return
+	}
+
+	// get owner
+	aceesKey, _ := vl.OSSSecure()
+	bucketOwner := NewBucketOwner(aceesKey)
+	var contents = make([]*Content, 0)
+	if len(fsFileInfos) > 0 {
+		for _, fsFileInfo := range fsFileInfos {
+			content := &Content{
+				Key:          fsFileInfo.Path,
+				LastModified: formatTimeISO(fsFileInfo.ModifyTime),
+				ETag:         fsFileInfo.ETag,
+				Size:         int(fsFileInfo.Size),
+				StorageClass: StorageClassStandard,
+				Owner:        bucketOwner,
+			}
+			contents = append(contents, content)
+		}
+	}
+
+	var commonPrefixes = make([]*CommonPrefix, 0)
+	for _, prefix := range prefixes {
+		commonPrefix := &CommonPrefix{
+			Prefix: prefix,
+		}
+		commonPrefixes = append(commonPrefixes, commonPrefix)
+	}
+
+	listBucketResult := &ListBucketResult{
+		Bucket:         bucket,
+		Prefix:         prefix,
+		Marker:         marker,
+		MaxKeys:        int(maxKeysInt),
+		Delimiter:      delimiter,
+		IsTruncated:    isTruncated,
+		NextMarker:     nextMarker,
+		Contents:       contents,
+		CommonPrefixes: commonPrefixes,
+	}
+
+	var bytes []byte
+	var marshalError error
+	if bytes, marshalError = MarshalXMLEntity(listBucketResult); marshalError != nil {
+		log.LogErrorf("getBucketV1Handler: marshal result fail, requestID(%o) err(%v)", RequestIDFromRequest(r), err)
+		_ = InvalidArgument.ServeResponse(w, r)
+		return
+	}
+
+	// set response header
+	w.Header().Set(HeaderNameContentType, HeaderValueContentTypeXML)
+	w.Header().Set(HeaderNameContentLength, strconv.Itoa(len(bytes)))
+	w.Write(bytes)
+
+	return
+}
+
+// List objects version 2
+// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
+func (o *ObjectNode) getBucketV2Handler(w http.ResponseWriter, r *http.Request) {
+	log.LogInfof("getBucketV2Handler: get bucket, requestID(%v) remote(%v)", RequestIDFromRequest(r), r.RemoteAddr)
+
+	// check args
+	_, bucket, _, vl, err := o.parseRequestParams(r)
+	if err != nil {
+		log.LogErrorf("getBucketV2Handler: parse request parameters fail, requestID(%o) err(%v)", RequestIDFromRequest(r), err)
+		_ = InvalidArgument.ServeResponse(w, r)
+		return
+	}
+
+	// get options
+	prefix := r.URL.Query().Get(ParamPrefix)
+	maxKeys := r.URL.Query().Get(ParamMaxKeys)
+	delimiter := r.URL.Query().Get(ParamPartDelimiter)
+	contToken := r.URL.Query().Get(ParamContToken)
+	fetchOwner := r.URL.Query().Get(ParamFetchOwner)
+	startAfter := r.URL.Query().Get(ParamStartAfter)
+
+	var maxKeysInt uint64
+	if maxKeys != "" {
+		maxKeysInt, err = strconv.ParseUint(maxKeys, 10, 16)
+		if err != nil {
+			log.LogErrorf("getBucketV2Handler: parse max keys fail, requestID(%v) err(%v)",
+				RequestIDFromRequest(r), err)
+			_ = InvalidArgument.ServeResponse(w, r)
+			return
+		}
+		if maxKeysInt > MaxKeys {
+			maxKeysInt = MaxKeys
+		}
+	} else {
+		maxKeysInt = MaxKeys
+	}
+
+	var fetchOwnerBool bool
+	if fetchOwner != "" {
+		fetchOwnerBool, err = strconv.ParseBool(fetchOwner)
+		if err != nil {
+			log.LogErrorf("getBucketV2Handler: requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+			_ = InvalidArgument.ServeResponse(w, r)
+			return
+		}
+	} else {
+		fetchOwnerBool = false
+	}
+
+	request := &ListBucketRequestV2{
+		delimiter:  delimiter,
+		maxKeys:    maxKeysInt,
+		prefix:     prefix,
+		contToken:  contToken,
+		fetchOwner: fetchOwnerBool,
+		startAfter: startAfter,
+	}
+
+	fsFileInfos, keyCount, nextToken, isTruncated, prefixes, err := vl.ListFilesV2(request)
+	if err != nil {
+		log.LogErrorf("getBucketV2Handler: request id [%o], Get files list failed cause : %o", r.URL, err)
+		_ = InternalError.ServeResponse(w, r)
+		return
+	}
+	// get owner
+	var bucketOwner *BucketOwner
+	if fetchOwnerBool {
+		accessKey, _ := vl.OSSSecure()
+		bucketOwner = NewBucketOwner(accessKey)
+	}
+
+	var contents = make([]*Content, 0)
+	if len(fsFileInfos) > 0 {
+		for _, fsFileInfo := range fsFileInfos {
+			content := &Content{
+				Key:          fsFileInfo.Path,
+				LastModified: formatTimeISO(fsFileInfo.ModifyTime),
+				ETag:         fsFileInfo.ETag,
+				Size:         int(fsFileInfo.Size),
+				StorageClass: StorageClassStandard,
+				Owner:        bucketOwner,
+			}
+			contents = append(contents, content)
+		}
+	}
+
+	var commonPrefixes = make([]*CommonPrefix, 0)
+	for _, prefix := range prefixes {
+		commonPrefix := &CommonPrefix{
+			Prefix: prefix,
+		}
+		commonPrefixes = append(commonPrefixes, commonPrefix)
+	}
+
+	listBucketResult := ListBucketResultV2{
+		Name:           bucket,
+		Prefix:         prefix,
+		Token:          contToken,
+		NextToken:      nextToken,
+		KeyCount:       keyCount,
+		MaxKeys:        maxKeysInt,
+		Delimiter:      delimiter,
+		IsTruncated:    isTruncated,
+		Contents:       contents,
+		CommonPrefixes: commonPrefixes,
+	}
+
+	var bytes []byte
+	var marshalError error
+	if bytes, marshalError = MarshalXMLEntity(listBucketResult); marshalError != nil {
+		log.LogErrorf("getBucketV2Handler: marshal result fail, requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+		_ = InvalidArgument.ServeResponse(w, r)
+		return
+	}
+
+	// set response header
+	w.Header().Set(HeaderNameContentType, HeaderValueContentTypeXML)
+	w.Header().Set(HeaderNameContentLength, strconv.Itoa(len(bytes)))
+	if _, err = w.Write(bytes); err != nil {
+		log.LogErrorf("getBucketVeHandler: write response body fail, requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+	}
+	return
+}
+
+// Put object
+// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html
+func (o *ObjectNode) putObjectHandler(w http.ResponseWriter, r *http.Request) {
+	log.LogInfof("putObjectHandler: put object, requestID(%v) remote(%v)", RequestIDFromRequest(r), r.RemoteAddr)
+
+	_, bucket, object, vl, err := o.parseRequestParams(r)
+	if err != nil {
+		log.LogErrorf("putObjectHandler: parser request parameters fail, requestID(%o) err(%v)", RequestIDFromRequest(r), err)
+		_ = NoSuchBucket.ServeResponse(w, r)
+		return
+	}
+
+	// check args
+	if bucket == "" || object == "" {
+		log.LogErrorf("putObjectHandler: illegal bucket or object found: requestID(%o)", RequestIDFromRequest(r))
+		_ = InvalidArgument.ServeResponse(w, r)
+		return
+	}
+
+	// Get request MD5, if request MD5 is not empty, compute and verify it.
+	requestMD5 := r.Header.Get(HeaderNameContentMD5)
+
+	var checkMD5 bool
+	if len(requestMD5) > 0 {
+		checkMD5 = true
+	}
+
+	var multipartID string
+	if multipartID, err = vl.InitMultipart(object); err != nil {
+		log.LogErrorf("putObjectHandler: volume init multipart fail: requestID(%v) path(%v) err(%v)",
+			RequestIDFromRequest(r), object, err)
+		_ = InternalError.ServeResponse(w, r)
+		return
+	}
+	defer func() {
+		// rollback policy
+		if err != nil {
+			if abortErr := vl.AbortMultipart(object, multipartID); abortErr != nil {
+				log.LogErrorf("putObjectHandler: volume abort multipart fail: requestID(%v) path(%v) multipartID(%v) err(%v)",
+					RequestIDFromRequest(r), object, multipartID, err)
+			}
+		}
+	}()
+	const partID uint16 = 1
+	if _, err = vl.WritePart(object, multipartID, partID, r.Body); err != nil {
+		log.LogErrorf("putObjectHandler: volume write part fail: requestID(%v) path(%v) multipartID(%v) err(%v)",
+			RequestIDFromRequest(r), object, multipartID, err)
+		_ = InternalError.ServeResponse(w, r)
+		return
+	}
+	var fsFileInfo *FSFileInfo
+	if fsFileInfo, err = vl.CompleteMultipart(object, multipartID); err != nil {
+		log.LogErrorf("putObjectHandler: volume complete multipart fail: requestID(%v) path(%v) multipartID(%v) err(%v)",
+			RequestIDFromRequest(r), object, multipartID, err)
+		_ = InternalError.ServeResponse(w, r)
+		return
+	}
+
+	// validate content MD5 value
+	if strings.HasSuffix(requestMD5, "==") {
+		var decoded []byte
+		if decoded, err = base64.StdEncoding.DecodeString(requestMD5); err != nil {
+			log.LogErrorf("putObjectHandler: decode request MD5 value fail: requestID(%v) raw(%v) err(%v)",
+				RequestIDFromRequest(r), requestMD5, err)
+			_ = InternalError.ServeResponse(w, r)
+			return
+		}
+		requestMD5 = hex.EncodeToString(decoded)
+	}
+	// check content MD5
+	if checkMD5 && requestMD5 != fsFileInfo.ETag {
+		log.LogErrorf("putObjectHandler: MD5 validate fail: requestID(%v) requestMD5(%o) serverMD5(%o)",
+			r.URL.EscapedPath(), requestMD5, fsFileInfo.ETag)
+		_ = BadDigest.ServeResponse(w, r)
+		return
+	}
+
+	// set response header
+	w.Header().Set(HeaderNameETag, fsFileInfo.ETag)
+	w.Header().Set(HeaderNameContentLength, "0")
+	return
+}
+
+// Delete object
+// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html .
+func (o *ObjectNode) deleteObjectHandler(w http.ResponseWriter, r *http.Request) {
+	log.LogInfof("Delete object...")
+
+	_, _, object, vl, err := o.parseRequestParams(r)
+	if err != nil {
+		log.LogErrorf("[GetObject] request id [%o], Bucket or object can not be empty", r.URL)
+		_ = NoSuchBucket.ServeResponse(w, r)
+		return
+	}
+
+	err = vl.DeleteFile(object)
+	if err != nil {
+		log.LogErrorf("[DeleteFile] request id [%o], Delete object failed cause : %o", r.URL.String(), err)
+		_ = InternalError.ServeResponse(w, r)
+		return
+	}
+
+	return
+}
+
+// Get object tagging
+// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectTagging.html
+func (o *ObjectNode) getObjectTagging(w http.ResponseWriter, r *http.Request) {
+	// TODO: implement handler 'GetObjectTagging'
+	return
+}
+
+// Put object tagging
+// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectTagging.html
+func (o *ObjectNode) putObjectTagging(w http.ResponseWriter, r *http.Request) {
+	// TODO: implement handler 'PutObjectTagging'
+	return
+}
+
+// Delete object tagging
+// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjectTagging.html
+func (o *ObjectNode) deleteObjectTagging(w http.ResponseWriter, r *http.Request) {
+	// TODO: implement handler 'DeleteObjectTagging'
+	return
+}
+
+// Put object extend attribute (xattr)
+func (o *ObjectNode) putObjectXAttr(w http.ResponseWriter, r *http.Request) {
+	// TODO: implement 'putObjectXAttr'
+}
+
+// Get object extend attribute (xattr)
+func (o *ObjectNode) getObjectXAttr(w http.ResponseWriter, r *http.Request) {
+	// TODO: implement 'getObjectXAttr'
+}
+
+// Delete object extend attribute (xattr)
+func (o *ObjectNode) deleteObjectXAttr(w http.ResponseWriter, r *http.Request) {
+	// TODO: implement 'deleteObjectXAttr'
+}
+
+// List object xattrs
+func (o *ObjectNode) listObjectXAttrs(w http.ResponseWriter, r *http.Request) {
+	// TODO: implement 'listObjectXAttrs'
+}

objectnode/api_middleware.go

@@ -0,0 +1,125 @@
+// Package s3 provides ...
+package objectnode
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/chubaofs/chubaofs/util/log"
+
+	"github.com/google/uuid"
+	"github.com/gorilla/mux"
+)
+
+const (
+	ctxKeyRequestID = "ctx_request_id"
+)
+
+func RequestIDFromRequest(r *http.Request) (id string) {
+	return mux.Vars(r)[ctxKeyRequestID]
+}
+
+func (o *ObjectNode) traceMiddleware(next http.Handler) http.Handler {
+	var generateRequestID = func() (string, error) {
+		var uUID uuid.UUID
+		var err error
+		if uUID, err = uuid.NewRandom(); err != nil {
+			return "", err
+		}
+		return strings.ReplaceAll(uUID.String(), "-", ""), nil
+	}
+	var handlerFunc http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {
+
+		var err error
+		var requestID string
+		if requestID, err = generateRequestID(); err != nil {
+			log.LogErrorf("traceMiddleware: generate request ID fail, remote(%v) url(%v) err(%v)",
+				r.RemoteAddr, r.URL.String(), err)
+			_ = InternalError.ServeResponse(w, r)
+			return
+		}
+		mux.Vars(r)[ctxKeyRequestID] = requestID
+		w.Header().Set(HeaderNameRequestId, requestID)
+
+		var startTime = time.Now()
+
+		next.ServeHTTP(w, r)
+
+		var headerToString = func(header http.Header) string {
+			var sb = strings.Builder{}
+			for k := range header {
+				if sb.Len() != 0 {
+					sb.WriteString(",")
+				}
+				sb.WriteString(fmt.Sprintf("%o:[%o]", k, header.Get(k)))
+			}
+			return "{" + sb.String() + "}"
+		}
+
+		log.LogDebugf("traceMiddleware: trace request:\n"+
+			"  requestID(%v) host(%v) method(%v) url(%v)\n"+
+			"  header(%v)\n"+
+			"  remote(%v) cost(%v)",
+			requestID, r.Host, r.Method, r.URL.String(), headerToString(r.Header), getRequestIP(r), time.Since(startTime))
+
+	}
+	return handlerFunc
+}
+
+func (o *ObjectNode) authMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(
+		func(w http.ResponseWriter, r *http.Request) {
+			//  1. check auth type
+			if isSignaturedV4(r) {
+				if ok, _ := o.checkSignatureV4(r); !ok {
+					if err := AccessDenied.ServeResponse(w, r); err != nil {
+						log.LogErrorf("authMiddleware: serve access denied response fail, requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+					}
+					return
+				}
+			} else if isSignaturedV2(r) {
+				if ok, _ := o.checkSignatureV2(r); !ok {
+					if err := AccessDenied.ServeResponse(w, r); err != nil {
+						log.LogErrorf("authMiddleware: serve access denied response fail, requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+					}
+					return
+				}
+			} else if isPresignedSignaturedV2(r) {
+				if ok, _ := o.checkPresignedSignatureV2(r); !ok {
+					log.LogDebugf("authMiddleware: presigned v2 denied: requestID(%v)", RequestIDFromRequest(r))
+					if err := AccessDenied.ServeResponse(w, r); err != nil {
+						log.LogErrorf("authMiddleware: serve response fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+					}
+					return
+				}
+			} else if isPresignedSignaturedV4(r) {
+				if ok, _ := o.checkPresignedSignatureV4(r); !ok {
+					log.LogDebugf("authMiddleware: presigned v4 denied: requestID(%v)", RequestIDFromRequest(r))
+					if err := AccessDenied.ServeResponse(w, r); err != nil {
+						log.LogErrorf("authMiddleware: serve response fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+					}
+					return
+				}
+			} else {
+				if err := AccessDenied.ServeResponse(w, r); err != nil {
+					log.LogErrorf("authMiddleware: serve response fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+				}
+				return
+			}
+
+			next.ServeHTTP(w, r)
+		})
+}
+
+func (o *ObjectNode) contentMiddleware(next http.Handler) http.Handler {
+	var handlerFunc http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {
+		if len(r.Header) > 0 && len(r.Header.Get(http.CanonicalHeaderKey(HeaderNameDecodeContentLength))) > 0 {
+			r.Body = NewChunkedReader(r.Body)
+			log.LogDebugf("contentMiddleware: chunk reader inited: requestID(%v)", RequestIDFromRequest(r))
+		}
+		next.ServeHTTP(w, r)
+	}
+	return handlerFunc
+}

objectnode/auth.go

@@ -0,0 +1,51 @@
+// Package s3 provides ...
+package objectnode
+
+import "net/http"
+
+//https://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#ConstructingTheAuthenticationHeader
+
+type AuthType string
+
+const (
+	SignatrueV2 AuthType = "signature_v2"
+	SignatrueV4          = "signature_v4"
+	PresignedV2          = "presigned_v2"
+	PresignedV4          = "presigned_v4"
+)
+
+type RequestAuthInfo struct {
+	authType  AuthType
+	accessKey string
+}
+
+func parseRequestAuthInfo(r *http.Request) *RequestAuthInfo {
+	auth := new(RequestAuthInfo)
+	if isSignaturedV2(r) {
+		auth.authType = SignatrueV2
+		ai, _ := parseRequestAuthInfoV2(r)
+		if ai != nil {
+			auth.accessKey = ai.accessKeyId
+		}
+	} else if isSignaturedV4(r) {
+		auth.authType = SignatrueV4
+		ai, _ := parseRequestV4(r)
+		if ai != nil {
+			auth.accessKey = ai.Credential.AccessKey
+		}
+	} else if isPresignedSignaturedV2(r) {
+		auth.authType = PresignedV2
+		ai, _ := parsePresignedV2AuthInfo(r)
+		if ai != nil {
+			auth.accessKey = ai.accessKeyId
+		}
+	} else if isPresignedSignaturedV4(r) {
+		auth.authType = PresignedV4
+		ai, _ := parseRequestV4(r)
+		if ai != nil {
+			auth.accessKey = ai.Credential.AccessKey
+		}
+	}
+
+	return auth
+}

objectnode/auth_signature_v2.go

@@ -0,0 +1,394 @@
+// Package s3 provides ...
+package objectnode
+
+import (
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"errors"
+	"net/http"
+	"net/url"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/chubaofs/chubaofs/util"
+	"github.com/chubaofs/chubaofs/util/log"
+	"github.com/gorilla/mux"
+)
+
+//https://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#ConstructingTheAuthenticationHeader
+
+const (
+	RequestHeaderV2Authorization       = "Authorization"
+	RequestHeaderV2AuthorizationScheme = "AWS"
+	RequestHeaderV2XAmzDate            = "X-Amz-Date"
+)
+
+var PresignedSignatureV2Queries = []string{
+	"AWSAccessKeyId",
+	"Signature",
+}
+
+var SignatureV2WhiteQueries = map[string]struct{}{
+	"acl":                          struct{}{},
+	"delete":                       struct{}{},
+	"lifecycle":                    struct{}{},
+	"location":                     struct{}{},
+	"logging":                      struct{}{},
+	"notification":                 struct{}{},
+	"partNumber":                   struct{}{},
+	"policy":                       struct{}{},
+	"requestPayment":               struct{}{},
+	"response-cache-control":       struct{}{},
+	"response-content-disposition": struct{}{},
+	"response-content-encoding":    struct{}{},
+	"response-content-language":    struct{}{},
+	"response-content-type":        struct{}{},
+	"response-expires":             struct{}{},
+	"torrent":                      struct{}{},
+	"uploadId":                     struct{}{},
+	"uploads":                      struct{}{},
+	"versionId":                    struct{}{},
+	"versioning":                   struct{}{},
+	"versions":                     struct{}{},
+}
+
+//
+type requestAuthInfoV2 struct {
+	r           *http.Request
+	authType    AuthType
+	bucket      string
+	accessKeyId string
+	signature   string
+	expires     string
+}
+
+// http://127.0.0.1:33032/ltptest/b.txt
+//  ?AWSAccessKeyId=Yqnqp4v6q1fzNM2e
+//  &Expires=1573369185
+//  &Signature=GJCqOY0ahf1BdzJDjNnFWB7vfSc%3D
+//
+func parsePresignedV2AuthInfo(r *http.Request) (*requestAuthInfoV2, error) {
+	//
+	ai := new(requestAuthInfoV2)
+	uris := strings.SplitN(r.RequestURI, "?", 2)
+	if len(uris) < 2 {
+		log.LogInfof("checkPresignedSignatureV2 error, request url invalid %v ", r.RequestURI)
+		return nil, errors.New("uri is invalid")
+	}
+
+	vars := mux.Vars(r)
+	ai.accessKeyId = vars["accessKey"]
+	ai.signature = vars["signature"]
+	ai.expires = vars["expires"]
+
+	return ai, nil
+}
+
+// Authorization: AWS AWSAccessKeyId:Signature
+func parseRequestAuthInfoV2(r *http.Request) (ra *requestAuthInfoV2, err error) {
+	ra = &requestAuthInfoV2{r: r}
+
+	vars := mux.Vars(r)
+	ra.bucket = vars["bucket"]
+
+	authStr := r.Header.Get(RequestHeaderV2Authorization)
+	if authStr == "" {
+		err = errors.New("header not found authentication")
+		return nil, err
+	}
+
+	if !strings.HasPrefix(authStr, RequestHeaderV2AuthorizationScheme) {
+		return nil, errors.New("header has no prefix ")
+	}
+
+	credentialStr := util.SubString(authStr, len(RequestHeaderV2AuthorizationScheme), len(authStr))
+	credentialStr = strings.Trim(credentialStr, " ")
+	credentials := strings.Split(credentialStr, ":")
+	if len(credentials) < 2 {
+		err = errors.New("")
+		return nil, err
+	}
+
+	ra.accessKeyId = credentials[0]
+	ra.signature = credentials[1]
+
+	return
+}
+
+func isSignaturedV2(r *http.Request) bool {
+	_, ok1 := r.Header[HeaderNameAuthorization]
+	_, ok2 := r.Header["X-Amz-Content-Sha256"]
+	if ok1 && !ok2 {
+		log.LogDebugf("[handleHttpRestAPI] invalid request, has no authorization info, request id [%s]", r.URL.EscapedPath())
+		return true
+	}
+
+	return false
+}
+
+func isRequestQueryValid(queries url.Values, neededQueries []string) bool {
+	for _, q := range neededQueries {
+		k := strings.ToLower(q)
+		if _, ok := queries[k]; !ok {
+			return false
+		}
+	}
+	return true
+}
+
+// http://127.0.0.1:33032/ltptest/b.txt
+//  ?AWSAccessKeyId=Yqnqp4v6q1fzNM2e
+//  &Expires=1573369185
+//  &Signature=GJCqOY0ahf1BdzJDjNnFWB7vfSc%3D
+//
+func isPresignedSignaturedV2(r *http.Request) bool {
+	if u, err := url.Parse(strings.ToLower(r.URL.String())); err == nil {
+		return isRequestQueryValid(u.Query(), PresignedSignatureV2Queries)
+	}
+	return false
+}
+
+//
+func (o *ObjectNode) checkSignatureV2(r *http.Request) (bool, error) {
+	// parse v2 request header and query, and get reqSignature
+	authInfo, err := parseRequestAuthInfoV2(r)
+	if err != nil {
+		log.LogInfof("parseRequestAuthInfoV2 error: %v, %v", authInfo.r, err)
+		return false, err
+	}
+
+	v, err := o.vm.Volume(authInfo.bucket)
+	if err != nil {
+		log.LogInfof("load Volume error: %v, %v", authInfo.r, err)
+		return false, err
+	}
+	volAccessKey, volSecret := v.OSSSecure()
+
+	if authInfo.accessKeyId != volAccessKey {
+		log.LogInfof("load Volume error: %v, %v", authInfo.accessKeyId, volAccessKey)
+		return false, errors.New("")
+	}
+
+	// 2. calculate new signature
+	newSignature, err1 := calculateSignatureV2(authInfo, volSecret, o.domains)
+	if err1 != nil {
+		log.LogInfof("calculute SignatureV2 error: %v, %v", authInfo.r, err)
+		return false, err1
+	}
+
+	// 3. compare newSignatrue and reqSignature
+	if authInfo.signature == newSignature {
+		return true, nil
+	}
+	log.LogInfof("newSignature: %v, reqSignature: %v, %v", newSignature, authInfo.signature, authInfo.r)
+
+	return false, nil
+}
+
+func calculateSignatureV2(authInfo *requestAuthInfoV2, secretKey string, domains []string) (signature string, err error) {
+
+	//encodedResource := strings.Split(authInfo.r.RequestURI, "?")[0]
+	canonicalResource, err1 := getCanonicalizedResourceV2(authInfo.r, domains)
+	if err1 != nil {
+		return "", err1
+	}
+
+	canonicalResourceQuery := getCanonicalQueryV2(canonicalResource, authInfo.r.URL.Query().Encode())
+
+	date := authInfo.r.Header.Get("Date")
+	method := authInfo.r.Method
+	canonicalHeaders := canonicalizedAmzHeadersV2(authInfo.r.Header)
+	if len(canonicalHeaders) > 0 {
+		canonicalHeaders += "\n"
+	}
+	contentHash := authInfo.r.Header.Get(HeaderNameContentMD5)
+	contentEnc := authInfo.r.Header.Get(HeaderNameContentEnc)
+	stringToSign := strings.Join([]string{
+		method,
+		contentHash,
+		contentEnc,
+		date,
+		canonicalHeaders,
+	}, "\n")
+
+	stringToSign = stringToSign + canonicalResourceQuery
+
+	hm := hmac.New(sha1.New, []byte(secretKey))
+	hm.Write([]byte(stringToSign))
+
+	signature = base64.StdEncoding.EncodeToString(hm.Sum(nil))
+
+	return
+}
+
+//
+//
+func (o *ObjectNode) checkPresignedSignatureV2(r *http.Request) (bool, error) {
+	//
+	uris := strings.SplitN(r.RequestURI, "?", 2)
+	if len(uris) < 2 {
+		log.LogInfof("checkPresignedSignatureV2 error, request url invalid %v ", r.RequestURI)
+		return false, nil
+	}
+
+	params, _, _, vl, err := o.parseRequestParams(r)
+	if err != nil || vl == nil {
+		log.LogInfof("check PresignedSignatureV2 error: %v %v", err, vl)
+		return false, err
+	}
+	accessKey := params["accessKey"]
+	signature := params["signature"]
+	expires := params["expires"]
+	if accessKey == "" || signature == "" || expires == "" {
+		log.LogInfof("checkPresignedSignatureV2 params not valid: %v", params)
+		return false, nil
+	}
+
+	log.LogDebugf("checkPresignedSignatureV2: parse signature info: requestID(%v) url(%v) accessKey(%v) signature(%v) expires(%v)",
+		RequestIDFromRequest(r), r.URL.String(), accessKey, signature, expires)
+
+	//check access key
+	vlKey, secretKey := vl.OSSSecure()
+	if vlKey == "" || accessKey != vlKey {
+		return false, nil
+	}
+
+	// check expires
+	if ok, _ := checkExpires(expires); !ok {
+		log.LogDebugf("checkPresignedSignatureV2: signature expired: requestID(%v) expires(%v)", RequestIDFromRequest(r), expires)
+		return false, nil
+	}
+
+	//calculatePresignedSignature
+	uri := strings.Split(r.RequestURI, "?")[0]
+	canoncialResourceQuery := getCanonicalQueryV2(uri, r.URL.Query().Encode())
+	calSignature := calPresignedSignatureV2(r.Method, canoncialResourceQuery, expires, secretKey, r.Header)
+	if calSignature != signature {
+		log.LogDebugf("checkPresignedSignatureV2: invalid signature: requestID(%v) client(%v) server(%v)",
+			RequestIDFromRequest(r), signature, calSignature)
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func checkExpires(expires string) (ok bool, err error) {
+	expiresInt, err := strconv.ParseInt(expires, 10, 64)
+	if err != nil {
+		return false, err
+	}
+	now := time.Now().UTC().Unix()
+	if now < expiresInt {
+		log.LogInfof("checkPresignedSignatureV2 expired is out time %v, now: %v", expires, now)
+		return true, nil
+	}
+
+	return false, nil
+}
+
+func getCanonicalQueryV2(encodeResource string, encodeQuery string) string {
+	var canonicalQueries []string
+	items := strings.Split(encodeQuery, "&")
+	queries := make(map[string]string)
+	for _, item := range items {
+		k := item
+		v := ""
+		i := strings.Index(item, "=")
+		if i != -1 {
+			k = item[:i]
+			v = item[i+1:]
+		}
+		queries[k] = v
+	}
+
+	for k, v := range queries {
+		if _, ok := SignatureV2WhiteQueries[k]; !ok {
+			continue
+		}
+
+		query := k
+		if v != "" {
+			query = k + "=" + v
+		}
+		canonicalQueries = append(canonicalQueries, query)
+	}
+
+	sort.Strings(canonicalQueries)
+
+	canonicalQuery := strings.Join(canonicalQueries, "&")
+	if canonicalQuery != "" {
+		return encodeResource + "?" + canonicalQuery
+	}
+
+	if encodeResource == "/" {
+		return ""
+	}
+
+	return encodeResource
+}
+
+//
+func canonicalizedAmzHeadersV2(headers http.Header) string {
+	var keys []string
+	keyval := make(map[string]string)
+	for key := range headers {
+		lkey := strings.ToLower(key)
+		if !strings.HasPrefix(lkey, "x-amz-") {
+			continue
+		}
+		keys = append(keys, lkey)
+		keyval[lkey] = strings.Join(headers[key], ",")
+	}
+	sort.Strings(keys)
+	var canonicalHeaders []string
+	for _, key := range keys {
+		canonicalHeaders = append(canonicalHeaders, key+":"+keyval[key])
+	}
+	return strings.Join(canonicalHeaders, "\n")
+}
+
+//
+func calPresignedSignatureV2(method, canonicalQuery, expires, secretKey string, header http.Header) string {
+	date := expires
+	if date == "" {
+		date = header.Get(HeaderNameDate)
+	}
+	canonicalHeaders := canonicalizedAmzHeadersV2(header)
+	contentHash := header.Get(HeaderNameContentMD5)
+	contentEnc := header.Get(HeaderNameContentEnc)
+	stringToSign := strings.Join([]string{
+		method,
+		contentHash,
+		contentEnc,
+		date,
+		canonicalHeaders,
+	}, "\n") + canonicalQuery
+
+	hm := hmac.New(sha1.New, []byte(secretKey))
+	hm.Write([]byte(stringToSign))
+
+	return base64.StdEncoding.EncodeToString(hm.Sum(nil))
+}
+
+func getCanonicalizedResourceV2(r *http.Request, domains []string) (resource string, err error) {
+	path := strings.Split(r.RequestURI, "?")[0]
+	if len(domains) > 0 {
+		for _, d := range domains {
+			if !strings.HasSuffix(r.Host, "."+d) {
+				continue
+			}
+			vars := mux.Vars(r)
+			bucket := vars["bucket"]
+			resource = "/" + bucket + path
+			return
+		}
+
+	} else {
+		resource = path
+	}
+
+	return
+}

objectnode/auth_signature_v4.go

@@ -0,0 +1,577 @@
+package objectnode
+
+import (
+	"encoding/hex"
+	"errors"
+	"net/http"
+	"net/url"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/chubaofs/chubaofs/util"
+	"github.com/chubaofs/chubaofs/util/log"
+	"github.com/gorilla/mux"
+)
+
+const (
+	MaxPresignedExpires = 3 * 365 * 24 * 60 * 60 //10years
+	DateFormatISO8601   = "20060102T150405Z"     //"yyyyMMddTHHmmssZ"
+	MaxSkewTime         = 15 * time.Minute
+
+	XAmzContentSha256 = "X-Amz-Content-Sha256"
+	XAmzCredential    = "X-Amz-Credential"
+	XAmzSignature     = "X-Amz-Signature" //
+	XAmzSignedHeaders = "X-Amz-SignedHeaders"
+	XAmzAlgorithm     = "X-Amz-Algorithm"
+	XAmzDate          = "X-Amz-Date"
+	XAmzExpires       = "X-Amz-Expires"
+
+	SignatureV4Algorithm = "AWS4-HMAC-SHA256"
+	SignatureV4Request   = "aws4-request"
+	SignedHeaderHost     = "host"
+	UnsignedPayload      = "UNSIGNED-PAYLOAD"
+	PresignedV4QueryAuth = "Authorization"
+
+	credentialFlag    = "Credential="
+	signatureFlag     = "Signature="
+	signedHeadersFlag = "SignedHeaders="
+)
+
+var (
+	InvalidParamError = errors.New("invalid param")
+	MaxExpiresError   = errors.New("max expires value")
+)
+
+var PresignedSignatureV4Queries = []string{
+	XAmzCredential,
+	XAmzSignature,
+}
+
+var AuthSignatureV4Headers = []string{
+	PresignedV4QueryAuth,
+	XAmzContentSha256,
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
+// url: "127.0.0.1:9000/umptest/b.tinyExtents
+//      ?X-Amz-Algorithm=AWS4-HMAC-SHA256_CMD
+//      &X-Amz-Credential=<your-access-key-id>/20130721/us-east-1/s3/aws4_request
+//      &X-Amz-Date=20191111T112557Z
+//      &X-Amz-Expires=432000
+//      &X-Amz-SignedHeaders=host
+//      &X-Amz-Signature=55c588734f017b861c24cbd69c203c283aad566cfe4ee712b7a3c846e1de151a"
+//
+func isPresignedSignaturedV4(r *http.Request) bool {
+	if u, err := url.Parse(strings.ToLower(r.URL.String())); err == nil {
+		return isRequestQueryValid(u.Query(), PresignedSignatureV4Queries)
+	}
+	return false
+}
+
+// is request signature V4
+func isSignaturedV4(r *http.Request) bool {
+	for _, h := range AuthSignatureV4Headers {
+		_, ok := r.Header[h]
+		if !ok {
+			return false
+		}
+	}
+
+	return true
+}
+
+// check request signature valid
+func (o *ObjectNode) checkSignatureV4(r *http.Request) (bool, error) {
+	_, _, _, vl, _ := o.parseRequestParams(r)
+	if vl == nil {
+		log.LogInfof("[handleHttpRestAPI] invalid request, has no authorization info, request id [%o]", r.URL.EscapedPath())
+		return false, nil
+	}
+	_, secretKey := vl.OSSSecure()
+	req, err := parseRequestV4(r)
+	if err != nil {
+		return false, err
+	}
+	newSignature := calculateSignatureV4(r, o.region, secretKey, req.SignedHeaders)
+	if req.Signature != newSignature {
+		log.LogDebugf("checkSignatureV4: invalid signature: requestID(%v) client(%v) server(%v)",
+			RequestIDFromRequest(r), req.Signature, newSignature)
+		return false, nil
+	}
+
+	return true, nil
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
+// url: "127.0.0.1:9000/umptest/b.tinyExtents
+//      ?X-Amz-Algorithm=AWS4-HMAC-SHA256_CMD
+//      &X-Amz-Credential=<your-access-key-id>/20130721/us-east-1/s3/aws4_request
+//      &X-Amz-Date=20191111T112557Z
+//      &X-Amz-Expires=432000
+//      &X-Amz-SignedHeaders=host
+//      &X-Amz-Signature=55c588734f017b861c24cbd69c203c283aad566cfe4ee712b7a3c846e1de151a"
+//
+func (o *ObjectNode) checkPresignedSignatureV4(r *http.Request) (pass bool, err error) {
+	var req *signatureRequestV4
+	req, err = parseRequestV4(r)
+	if err != nil {
+		log.LogErrorf("checkPresignedSignatureV4: parse request fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+		return
+	}
+
+	//check req valid
+	var ok bool
+	if ok, err = req.isValid(); !ok {
+		log.LogErrorf("checkPresignedSignatureV4: request invalid: requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+		return
+	}
+
+	// check accessKey valid
+	var v Volume
+	v, err = o.vm.Volume(req.bucket)
+	if err != nil {
+		log.LogErrorf("checkPresignedSignatureV4: get volume fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+		return
+	}
+	//check accesskey
+	vaKey, secretKey := v.OSSSecure()
+	if req.Credential.AccessKey != vaKey {
+		log.LogInfof("checkPresignedSignatureV4: credential accessKey invalid: requestID(%v) requestAK(%v) volAK(%v)",
+			RequestIDFromRequest(r), req.Credential.AccessKey, vaKey)
+		err = errors.New("accesskey invalid")
+		return
+	}
+	// create canonicalRequest
+	var canonicalHeader http.Header
+	canonicalHeader, err = req.createCanonicalHeaderV4()
+	if err != nil {
+		log.LogErrorf("checkPresignedSignatureV4: create canonical header fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err)
+		return
+	}
+	canonicalHeaderStr := buildCanonicalHeaderString(r.Host, canonicalHeader, req.SignedHeaders)
+	headerNames := getCanonicalHeaderNames(req.SignedHeaders)
+	payload := UnsignedPayload
+	canonicalQuery := createCanonicalQueryV4(req)
+	canonicalRequestString := createCanonicalRequestString(r.Method, req.URI, canonicalQuery, canonicalHeaderStr, headerNames, payload)
+
+	log.LogDebugf("checkPresignedSignatureV4: middle data:\n"+
+		"  RequestID: %v\n"+
+		"  CanonicalRequest: %v",
+		RequestIDFromRequest(r),
+		canonicalRequestString)
+
+	// build signingKey
+	signingKey := buildSigningKey(SCHEME, secretKey, req.Credential.Date, req.Credential.Region, req.Credential.Service, req.Credential.Request)
+
+	// build stringToSign
+	scope := buildScope(req.Credential.Date, req.Credential.Region, req.Credential.Service, req.Credential.Request)
+	stringToSign := buildStringToSign(req.Algorithm, req.Timestamp, scope, canonicalRequestString)
+
+	//sign stringToSign with signingKey
+	newSignature := hex.EncodeToString(sign(stringToSign, signingKey))
+
+	//compare newSignature with request signature
+	pass = newSignature == req.Signature
+	return
+}
+
+type credential struct {
+	AccessKey string
+	Date      string
+	Region    string
+	Service   string //s3
+	Request   string
+}
+
+type signatureRequestV4 struct {
+	r             *http.Request
+	bucket        string
+	URI           string
+	Algorithm     string
+	Timestamp     string
+	Expires       string
+	Signature     string
+	SignedHeaders []string
+	Credential    credential
+}
+
+func (c *credential) GetScopeString() string {
+	return strings.Join([]string{
+		c.Date,
+		c.Region,
+		c.Service,
+		c.Request,
+	}, "/")
+}
+
+//get presignedReq query
+func (req *signatureRequestV4) Query() url.Values {
+	return req.r.URL.Query()
+}
+
+// get Timestamp
+func (req *signatureRequestV4) GetTimestamp() (time.Time, error) {
+	return time.Parse(DateFormatISO8601, req.Timestamp)
+}
+
+// get
+func (req *signatureRequestV4) GetExpires() (time.Duration, error) {
+	return time.ParseDuration(req.Expires + "s")
+}
+
+//
+func (req *signatureRequestV4) isValid() (bool, error) {
+	expires, err := req.GetExpires()
+	if err != nil {
+		return false, errors.New("expires is invalid ")
+	}
+	if expires < 0 {
+		return false, errors.New("expires < 0 ")
+	}
+	if expires.Seconds() > MaxPresignedExpires {
+		return false, errors.New("expires > MaxPresignedExpires ")
+	}
+	utcNow := time.Now().UTC()
+	ts, err1 := req.GetTimestamp()
+	if err1 != nil {
+		return false, errors.New("expires is invalid ")
+	}
+	if ts.After(utcNow.Add(MaxSkewTime)) {
+		return false, errors.New("req date invalid ")
+	}
+	if utcNow.Sub(ts) > expires {
+		return false, errors.New("expires time out ")
+	}
+
+	return true, nil
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html
+//
+//  Authorization: AWS4-HMAC-SHA256\nCredential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;range;x-amz-date,
+//  Signature=fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024
+//
+func (req *signatureRequestV4) parseRequestHeaderV4(r *http.Request) (err error) {
+	authorizationValue := r.Header.Get(HeaderNameAuthorization)
+	if authorizationValue != "" {
+		authorizationValue = strings.ReplaceAll(authorizationValue, " ", "")
+		credentialIndex := strings.Index(authorizationValue, credentialFlag)
+		algorithm := util.SubString(authorizationValue, len(HeaderNameAuthorization)+1, credentialIndex)
+		if algorithm != "" {
+			req.Algorithm = algorithm
+		}
+		credentialSignatureStr := util.SubString(authorizationValue, credentialIndex+len(credentialFlag), len(authorizationValue))
+		authorizationValues := strings.Split(credentialSignatureStr, ",")
+		if len(authorizationValues) < 2 {
+			log.LogInfof("decode signature error: %v  ", authorizationValue)
+			return errors.New("request header authorization parse error")
+		}
+
+		credentialStr := authorizationValues[0]
+		if credentialStr != "" {
+			req.parseCredential(credentialStr)
+		}
+		signedHeadersStr := authorizationValues[1]
+		signatureStr := authorizationValues[2]
+		req.Signature = util.SubString(signatureStr, len(signatureFlag), len(signatureStr))
+
+		signedHeadersStr = util.SubString(signedHeadersStr, len(signedHeadersFlag), len(signedHeadersStr))
+		req.SignedHeaders = strings.Split(signedHeadersStr, ";")
+	}
+	return
+}
+
+func (req *signatureRequestV4) parseCredential(credentialStr string) error {
+	credentialStr = strings.TrimPrefix(credentialStr, credentialFlag)
+	item := strings.Split(credentialStr, "/")
+	if len(item) < 5 {
+		return errors.New("x-amz-credential params invalid")
+	}
+	req.Credential = credential{
+		AccessKey: item[0],
+		Date:      item[1],
+		Region:    item[2],
+		Service:   item[3],
+		Request:   item[4],
+	}
+	return nil
+}
+
+func (req *signatureRequestV4) parseRequestQueryV4(r *http.Request) (err error) {
+	q := r.URL.Query()
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	if bucket != "" {
+		req.bucket = bucket
+	}
+	algorithm := q.Get(XAmzAlgorithm)
+	if algorithm != "" {
+		req.Algorithm = algorithm
+	}
+	timestamp := q.Get(XAmzDate)
+	if timestamp != "" {
+		req.Timestamp = timestamp
+		if _, err = req.GetTimestamp(); err != nil {
+			return err
+		}
+	}
+	expires := q.Get(XAmzExpires)
+	if expires != "" {
+		req.Expires = expires
+		if _, err = req.GetExpires(); err != nil {
+			return err
+		}
+	}
+	credentialStr := q.Get(XAmzCredential)
+	if credentialStr != "" {
+		req.parseCredential(credentialStr)
+	}
+	signatrue := q.Get(XAmzSignature)
+	if signatrue != "" {
+		req.Signature = signatrue
+	}
+	signedHeaders := q.Get(XAmzSignedHeaders)
+	if signedHeaders != "" {
+		req.SignedHeaders = strings.Split(signedHeaders, ";")
+	}
+	return
+}
+
+// getPresignedV4 encodeQuery
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
+// url: "127.0.0.1:9000/umptest/b.tinyExtents
+//      ?X-Amz-Algorithm=AWS4-HMAC-SHA256_CMD
+//      &X-Amz-Credential=<your-access-key-id>/20130721/us-east-1/s3/aws4_request
+//      &X-Amz-Date=20191111T112557Z
+//      &X-Amz-Expires=432000
+//      &X-Amz-SignedHeaders=host
+//      &X-Amz-Signature=55c588734f017b861c24cbd69c203c283aad566cfe4ee712b7a3c846e1de151a"
+func parseRequestV4(r *http.Request) (req *signatureRequestV4, err error) {
+	req = &signatureRequestV4{}
+	req.r = r
+
+	uri := strings.Split(r.RequestURI, "?")[0]
+	if uri != "" {
+		req.URI = uri
+	} else {
+		req.URI = "/"
+	}
+
+	//parseHeader
+	err = req.parseRequestHeaderV4(r)
+	if err != nil {
+		return
+	}
+
+	//parseQuery
+	err = req.parseRequestQueryV4(r)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+// create canonical query not contain X-Amz-Signature query
+func createCanonicalQueryV4(req *signatureRequestV4) string {
+	newQuery := make(url.Values)
+	newQuery.Set(XAmzAlgorithm, req.Algorithm)
+	newQuery.Set(XAmzExpires, req.Expires)
+	newQuery.Set(XAmzDate, req.Timestamp)
+	newQuery.Set(XAmzSignedHeaders, req.Query().Get(XAmzSignedHeaders))
+	newQuery.Set(XAmzCredential, req.Query().Get(XAmzCredential))
+	hashContent := req.Query().Get(XAmzContentSha256)
+	if hashContent != "" {
+		newQuery.Set(XAmzContentSha256, hashContent)
+	} else {
+	}
+
+	for k, v := range req.Query() {
+		key := strings.ToLower(k)
+		if strings.Contains(key, "x-amz-meta-") {
+			newQuery.Set(k, v[0])
+			continue
+		}
+		if strings.Contains(key, "x-amz-server-side-") {
+			newQuery.Set(k, v[0])
+		}
+		if strings.HasPrefix(key, "x-amz") {
+			continue
+		}
+		newQuery[k] = v
+	}
+
+	return newQuery.Encode()
+}
+
+func buildSigningKey(scheme, secret, date, region, service, terminator string) []byte {
+	secretKey := []byte(scheme + secret)
+	dateKey := sign(date, secretKey)
+	dateRegionKey := sign(region, dateKey)
+	dateRegionServiceKey := sign(service, dateRegionKey)
+	signingKey := sign(terminator, dateRegionServiceKey)
+	return signingKey
+}
+
+func getContentHash(headers http.Header) (contentHash string) {
+	for headerName := range headers {
+		if strings.ToLower(headerName) == strings.ToLower(HeaderNameContentHash) {
+			contentHash = headers.Get(headerName)
+			break
+		}
+	}
+	return
+}
+
+func getEncodeQuery(r *http.Request) string {
+	return r.URL.Query().Encode()
+}
+
+// calculete signature v4
+func calculateSignatureV4(r *http.Request, region, secretKey string, signedHeaders []string) string {
+	headers := r.Header
+
+	// get request start time in ISO8601 type
+	canonicalHeaderString := buildCanonicalHeaderString(r.Host, headers, signedHeaders)
+	headerNames := getCanonicalHeaderNames(signedHeaders)
+	contentHash := getContentHash(headers)
+	encodeQuery := getEncodeQuery(r)
+	canonicalURI := getCanonicalURI(r)
+	canonicalRequest := createCanonicalRequestString(r.Method, canonicalURI, encodeQuery, canonicalHeaderString, headerNames, contentHash)
+
+	dateStamp := getCurrentDateStamp()
+	timestamp := getStartTime(headers)
+	signingKey := buildSigningKey(SCHEME, secretKey, dateStamp, region, SERVICE, TERMINATOR)
+	scope := buildScope(dateStamp, region, SERVICE, TERMINATOR)
+	stringToSign := buildStringToSign(SignatureV4Algorithm, timestamp, scope, canonicalRequest)
+	signature := sign(stringToSign, signingKey)
+
+	log.LogDebugf("calculateSignatureV4: middle data:\n"+
+		"  RequestID: %v\n"+
+		"  CanonicalRequest: %v",
+		RequestIDFromRequest(r),
+		canonicalRequest)
+	return hex.EncodeToString(signature)
+}
+
+func getCanonicalURI(r *http.Request) string {
+	return strings.Split(r.URL.RequestURI(), "?")[0]
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+func createCanonicalRequestString(httpMethod, canonicalURI, encodeQuery, canonicalHeaders, headerName, hashedPayload string) string {
+	canonicalQueryString := strings.Replace(encodeQuery, "+", "%20", -1)
+	return strings.Join([]string{
+		httpMethod,
+		canonicalURI,
+		canonicalQueryString,
+		canonicalHeaders,
+		headerName,
+		hashedPayload,
+	}, "\n")
+}
+
+// get headerNames string from signedHeaders
+//  1. sort headers
+//  2. lower
+//  3. join with ;
+func getCanonicalHeaderNames(signedHeaders []string) (headerName string) {
+	sort.Strings(signedHeaders)
+
+	lowerSignedHeaders := make([]string, 0)
+	for _, headerName := range signedHeaders {
+		lowerSignedHeaders = append(lowerSignedHeaders, headerName)
+	}
+	headerName = strings.Join(lowerSignedHeaders, ";")
+	return
+}
+
+func contains(items []string, key string) bool {
+	for _, s := range items {
+		if s == key {
+			return true
+		}
+	}
+	return false
+}
+
+// canonical header
+func (req *signatureRequestV4) createCanonicalHeaderV4() (canonicalHeader http.Header, err error) {
+	if !contains(req.SignedHeaders, SignedHeaderHost) {
+		return nil, errors.New("signedHeaders not contain host")
+	}
+	canonicalHeader = make(http.Header)
+	reqHeader := req.r.Header
+	reqQuery := req.r.URL.Query()
+	for _, header := range req.SignedHeaders {
+		vals, ok := reqHeader[http.CanonicalHeaderKey(header)] //
+		if !ok {
+			vals, ok = reqQuery[header] //
+		}
+		if ok {
+			for _, val := range vals {
+				canonicalHeader.Add(header, val)
+			}
+			continue
+		}
+		switch strings.ToLower(header) {
+		case "content-length":
+			canonicalHeader.Set(header, strconv.FormatInt(req.r.ContentLength, 10))
+		case "expect":
+			canonicalHeader.Set(header, "100-continue")
+		case "transfer-encoding":
+			canonicalHeader.Set(header, req.r.Host)
+		default:
+			return nil, nil
+		}
+	}
+
+	return
+}
+
+// filter request headers by signedHeaders then gen join to a string
+func buildCanonicalHeaderString(host string, headers http.Header, signedHeaders []string) (headerString string) {
+	// copy a new http header from headers, because net/http package remove host header when parsing request
+	newHeaders := make(http.Header)
+	for n := range headers {
+		newHeaders.Add(n, headers.Get(n))
+	}
+	newHeaders.Add(HeaderNameHost, host)
+
+	canonicalHeaders := make([]string, 0)
+	sort.Strings(signedHeaders)
+	for _, signedHeaderName := range signedHeaders {
+		vals, ok := newHeaders[(http.CanonicalHeaderKey(signedHeaderName))]
+		if ok {
+			sb := strings.Builder{}
+			sb.WriteString(strings.ToLower(signedHeaderName))
+			sb.WriteString(":")
+			sb.WriteString(strings.Join(vals, ","))
+			canonicalHeaders = append(canonicalHeaders, sb.String())
+		}
+	}
+	headerString = strings.Join(canonicalHeaders, "\n") + "\n"
+	return
+}
+
+// build scope string to sign
+func buildScope(date, region, service, request string) string {
+	return strings.Join([]string{
+		date,
+		region,
+		service,
+		request,
+	}, "/")
+}
+
+// build string to sign
+func buildStringToSign(algorithm, timeStamp, scope, canonicalRequestString string) string {
+	return strings.Join([]string{
+		algorithm,
+		timeStamp,
+		scope,
+		calcHash(canonicalRequestString),
+	}, "\n")
+}

objectnode/auth_signature_v4_test.go

@@ -0,0 +1 @@
+package objectnode

objectnode/chunk.go

@@ -0,0 +1,188 @@
+// Copyright 2018 The ChubaoFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package objectnode
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+)
+
+var (
+	ErrUnexpectedChunkEnd  = errors.New("unexpected chunk end")
+	ErrUnexpectedChunkSize = errors.New("unexpected chunk size")
+)
+
+type chunkedReader struct {
+	readCloser io.ReadCloser
+	chunkSize  int
+	pos        int
+	bof        bool
+	eof        bool
+}
+
+func (r *chunkedReader) Close() error {
+	return r.readCloser.Close()
+}
+
+func (r *chunkedReader) Read(p []byte) (n int, err error) {
+
+	if r.eof {
+		n = 0
+		err = io.EOF
+		return
+	}
+
+	if r.pos >= r.chunkSize {
+		if err = r.nextChunk(); err != nil {
+			return
+		}
+		if r.eof {
+			n = 0
+			err = io.EOF
+			return
+		}
+	}
+	length := len(p)
+	if r.chunkSize-r.pos < length {
+		length = r.chunkSize - r.pos
+	}
+	if n, err = r.readCloser.Read(p[:length]); err != nil {
+		return
+	}
+	r.pos += n
+	return
+}
+
+func (r *chunkedReader) nextChunk() (err error) {
+	if !r.bof {
+		if err = r.readCRLF(); err != nil {
+			return
+		}
+	}
+	if r.chunkSize, err = r.readChunkSize(); err != nil {
+		return
+	}
+	r.bof = false
+	r.pos = 0
+	if r.chunkSize == 0 {
+		r.eof = true
+	}
+	return
+}
+
+func (r *chunkedReader) readCRLF() (err error) {
+
+	var tmp = make([]byte, 2)
+	if _, err = r.readCloser.Read(tmp); err != nil {
+		return
+	}
+	if cr, lf := int(tmp[0]), int(tmp[1]); (cr != '\r') || (lf != '\n') {
+		err = fmt.Errorf("CRLF expected and end of chunk: %v/%v", cr, lf)
+		return
+	}
+	return
+}
+
+func (r *chunkedReader) readByteToInt() (i int, err error) {
+	var tmp = make([]byte, 1)
+	if _, err = r.readCloser.Read(tmp); err != nil {
+		return
+	}
+	i = int(tmp[0])
+	return
+}
+
+func (r *chunkedReader) readChunkSize() (size int, err error) {
+	var buf = make([]byte, 0)
+	var state int
+	for state != -1 {
+		var b int
+		if b, err = r.readByteToInt(); err != nil {
+			return
+		}
+		if b == -1 {
+			err = ErrUnexpectedChunkEnd
+			return
+		}
+		switch state {
+		case 0:
+			switch b {
+			case '\r':
+				state = 1
+				break
+
+			case '"':
+				state = 2
+
+			default:
+				buf = append(buf, byte(b))
+			}
+			break
+
+		case 1:
+			if b == '\n' {
+				state = -1
+			} else {
+				err = ErrUnexpectedChunkSize
+				return
+			}
+			break
+
+		case 2:
+			switch b {
+			case '\\':
+				if b, err = r.readByteToInt(); err != nil {
+					return
+				}
+				buf = append(buf, byte(b))
+				break
+
+			case '"':
+				state = 0
+
+			default:
+				buf = append(buf, byte(b))
+			}
+			break
+
+		default:
+			panic("assertion fail")
+		}
+	}
+	var str = string(buf)
+	if sepIndex := strings.Index(str, ";"); sepIndex > 0 {
+		str = str[:sepIndex]
+	}
+	str = strings.TrimSpace(str)
+
+	var result int64
+	if result, err = strconv.ParseInt(str, 16, 64); err != nil {
+		return
+	}
+	size = int(result)
+	return
+}
+
+func NewChunkedReader(reader io.ReadCloser) io.ReadCloser {
+	return &chunkedReader{
+		readCloser: reader,
+		pos:        0,
+		bof:        true,
+		eof:        false,
+	}
+}

objectnode/chunk_test.go

@@ -0,0 +1,65 @@
+// Copyright 2018 The ChubaoFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package objectnode
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"net/http/httputil"
+	"testing"
+)
+
+type readCloser struct {
+	io.Reader
+}
+
+func (r *readCloser) Read(p []byte) (n int, err error) {
+	return r.Reader.Read(p)
+}
+
+func (readCloser) Close() error {
+	return nil
+}
+
+func wrapReader(reader io.Reader) io.ReadCloser {
+	return &readCloser{
+		Reader: reader,
+	}
+}
+
+func TestChunkedReader_ReadChunked(t *testing.T) {
+	var err error
+	plain := `Hello world.`
+	buffer := new(bytes.Buffer)
+	chunkedWriter := httputil.NewChunkedWriter(buffer)
+	if _, err = chunkedWriter.Write([]byte(plain)); err != nil {
+		t.Fatal(err)
+	}
+	_ = chunkedWriter.Close()
+	chunked := string(buffer.Bytes())
+	t.Logf("plain:\n%v", plain)
+	t.Logf("chunked:\n%v", chunked)
+
+	reader := NewChunkedReader(wrapReader(bytes.NewReader([]byte(chunked))))
+	result, err := ioutil.ReadAll(reader)
+	if err != nil {
+		t.Fatalf("read from chunked readCloser fail: err(%v)", err)
+	}
+	if string(result) != plain {
+		t.Fatalf("result mismatch:\nexpect(%v)\nactual(%v)", plain, string(result))
+	}
+	t.Logf("result: %v", string(result))
+}

objectnode/const.go

@@ -0,0 +1,98 @@
+// Copyright 2018 The ChubaoFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package objectnode
+
+type OSSOperation string
+
+const (
+	HeaderNameServer        = "server"
+	HeaderNameHost          = "Host"
+	HeaderNameLastModified  = "Last-Modified"
+	HeaderNameETag          = "ETag"
+	HeaderNameDate          = "Date"
+	HeaderNameContentMD5    = "content-md5"
+	HeaderNameContentEnc    = "content-encoding"
+	HeaderNameContentType   = "Content-Type"
+	HeaderNameContentLength = "Content-Length"
+	HeaderNameContentRange  = "Content-Range"
+	HeaderNameAuthorization = "Authorization"
+	HeaderNameAcceptRange   = "Accept-Ranges"
+	HeaderNameRange         = "Range"
+
+	HeaderNameStartDate           = "x-amz-date"
+	HeaderNameRequestId           = "x-amz-request-id"
+	HeaderNameContentHash         = "X-Amz-Content-SHA256"
+	HeaderNameCopySource          = "X-Amz-Copy-Source"
+	HeaderNameCopyMatch           = "x-amz-copy-source-if-match"
+	HeaderNameCopyNoneMatch       = "x-amz-copy-source-if-none-match"
+	HeaderNameCopyModified        = "x-amz-copy-source-if-modified-since"
+	HeaderNameCopyUnModified      = "x-amz-copy-source-if-unmodified-since"
+	HeaderNameDecodeContentLength = "X-Amz-Decoded-Content-Length"
+)
+
+const (
+	HeaderValueServer         = "ChubaoFS"
+	HeaderValueAcceptRange    = "bytes"
+	HeaderValueTypeStream     = "application/octet-stream"
+	HeaderValueContentTypeXML = "application/xml"
+)
+
+const (
+	SubObjectDelete    = "delete"
+	SubMultipartUpload = "uploads"
+)
+
+const (
+	ParamUploadId   = "uploadId"
+	ParamPartNumber = "partNumber"
+	ParamKeyMarker  = "key-marker"
+	ParamMarker     = "marker"
+	ParamPrefix     = "prefix"
+	ParamContToken  = "continuation-token"
+	ParamFetchOwner = "fetch-owner"
+	ParamMaxKeys    = "max-keys"
+	ParamStartAfter = "start-after"
+
+	ParamMaxParts       = "max-parts"
+	ParamUploadIdMarker = "upload-id-​marker"
+	ParamPartNoMarker   = "part-number-marker"
+	ParamPartMaxUploads = "max-uploads"
+	ParamPartDelimiter  = "delimiter"
+)
+
+const (
+	MaxKeys    = 1000
+	MaxParts   = 1000
+	MaxUploads = 1000
+)
+
+const (
+	StorageClassStandard = "Standard"
+)
+
+// XAttr keys for ObjectNode compatible feature
+const (
+	XAttrKeyOSSETag    = "oss:etag"
+	XAttrKeyOSSTagging = "oss:tg"
+	XAttrKeyOSSPolicy  = "oss:ply"
+)
+
+const (
+	AMZTimeFormat = "2006-01-02T15:04:05Z"
+)
+
+const (
+	EmptyContentMD5String = "d41d8cd98f00b204e9800998ecf8427e"
+)

objectnode/fs.go

@@ -0,0 +1,95 @@
+package objectnode
+
+import (
+	"io"
+	"os"
+	"sort"
+	"time"
+
+	"github.com/chubaofs/chubaofs/proto"
+)
+
+type VolumeManager interface {
+	Volume(volName string) (Volume, error)
+	Release(volName string)
+	GetStore() (Store, error)
+	InitStore(s Store)
+	Close()
+}
+
+type FSFileInfo struct {
+	Path       string
+	Size       int64
+	Mode       os.FileMode
+	ModifyTime time.Time
+	ETag       string
+	Inode      uint64
+}
+
+type Prefixes []string
+
+type PrefixMap map[string]struct{}
+
+func (m PrefixMap) AddPrefix(prefix string) {
+	m[prefix] = struct{}{}
+}
+
+func (m PrefixMap) Prefixes() Prefixes {
+	s := make([]string, 0, len(m))
+	for prefix := range m {
+		s = append(s, prefix)
+	}
+	sort.Strings(s)
+	return s
+}
+
+type FSUpload struct {
+	Key          string
+	UploadId     string
+	StorageClass string
+	Initiated    string
+}
+
+type FSPart struct {
+	PartNumber   int
+	LastModified string
+	ETag         string
+	Size         int
+}
+
+type Volume interface {
+	OSSSecure() (accessKey, secretKey string)
+	OSSMeta() *OSSMeta
+
+	// ListFiles return an FileInfo slice of specified volume, like read dir for hole volume.
+	// The result will be ordered by full path.
+	ListFilesV1(request *ListBucketRequestV1) ([]*FSFileInfo, string, bool, []string, error)
+
+	ListFilesV2(request *ListBucketRequestV2) ([]*FSFileInfo, uint64, string, bool, []string, error)
+
+	// PutObject create file in specified volume with specified path.
+	WriteFile(path string, reader io.Reader) (*FSFileInfo, error)
+
+	// DeleteFile delete specified file from specified volume. If target is not exists then returns error.
+	DeleteFile(path string) error
+
+	FileInfo(path string) (*FSFileInfo, error)
+
+	// operation about multipart uploads
+	InitMultipart(path string) (multipartID string, err error)
+	WritePart(path, multipartID string, partId uint16, reader io.Reader) (*FSFileInfo, error)
+	ListParts(path, multipartID string, maxParts, partNumberMarker uint64) ([]*FSPart, uint64, bool, error)
+	CompleteMultipart(path, multipartID string) (*FSFileInfo, error)
+	AbortMultipart(path, multipartID string) error
+	ListMultipartUploads(prefix, delimiter, keyMarker, uploadIdMarker string, maxUploads uint64) ([]*FSUpload, string, string, bool, []string, error)
+
+	ReadFile(path string, writer io.Writer, offset, size uint64) error
+
+	CopyFile(path, sourcePath string) (*FSFileInfo, error)
+
+	SetXAttr(path string, key string, data []byte) error
+	GetXAttr(path string, key string) (*proto.XAttrInfo, error)
+	DeleteXAttr(path string, key string) error
+
+	Close() error
+}

objectnode/fs_manager.go

@@ -0,0 +1,89 @@
+package objectnode
+
+import (
+	"errors"
+	"sync"
+
+	"github.com/chubaofs/chubaofs/util/log"
+)
+
+type volumeManager struct {
+	masters   []string
+	volumes   map[string]*volume // volume key -> vol
+	volMu     sync.RWMutex
+	store     Store
+	closeOnce sync.Once
+}
+
+func (m *volumeManager) Release(volName string) {
+	panic("implement me")
+}
+
+func (m *volumeManager) ReleaseAll() {
+	panic("implement me")
+}
+
+func (m *volumeManager) Volume(volName string) (Volume, error) {
+	return m.loadVolume(volName)
+}
+
+func (m *volumeManager) loadVolume(volName string) (*volume, error) {
+	var err error
+	var volume *volume
+	var exist bool
+	m.volMu.RLock()
+	volume, exist = m.volumes[volName]
+	m.volMu.RUnlock()
+	if !exist {
+		m.volMu.Lock()
+		volume, exist = m.volumes[volName]
+		if exist {
+			m.volMu.Unlock()
+			return volume, nil
+		}
+		if volume, err = newVolume(m.masters, volName); err != nil {
+			m.volMu.Unlock()
+			return nil, err
+		}
+		ak, sk := volume.OSSSecure()
+		log.LogDebugf("[loadVolume] load volume: Name[%v] AccessKey[%v] SecretKey[%v]", volName, ak, sk)
+		m.volumes[volName] = volume
+		volume.vm = m
+		m.volMu.Unlock()
+
+		volume.loadOSSMeta()
+	}
+
+	return volume, nil
+}
+
+// Release all
+func (m *volumeManager) Close() {
+	m.volMu.Lock()
+	defer m.volMu.Unlock()
+	for volKey, vol := range m.volumes {
+		_ = vol.Close()
+		log.LogDebugf("release volume %v", volKey)
+	}
+	m.volumes = make(map[string]*volume)
+}
+
+func (m *volumeManager) InitStore(s Store) {
+	s.Init(m)
+	m.store = s
+}
+
+func (m *volumeManager) GetStore() (Store, error) {
+	if m.store == nil {
+		return nil, errors.New("store not init")
+	}
+	return m.store, nil
+}
+
+func NewVolumeManager(masters []string) VolumeManager {
+	vc := &volumeManager{
+		volumes: make(map[string]*volume),
+		masters: masters,
+	}
+	return vc
+}

objectnode/fs_store.go

@@ -0,0 +1,13 @@
+package objectnode
+
+type MetaStore interface {
+}
+
+// MetaStore
+type Store interface {
+	Init(vm *volumeManager)
+	Put(ns, obj, key string, data []byte) error
+	Get(ns, obj, key string) (data []byte, err error)
+	List(ns, obj string) (data [][]byte, err error)
+	Delete(ns, obj, key string) error
+}

objectnode/fs_store_authnode.go

@@ -0,0 +1,33 @@
+package objectnode
+
+type authnodeStore struct {
+	vm *volumeManager //vol *volume
+}
+
+func (s *authnodeStore) Init(vm *volumeManager) {
+	s.vm = vm
+	//TODO: init authnode store
+}
+
+func (s *authnodeStore) Get(vol, path, key string) (val []byte, err error) {
+
+	return
+}
+
+func (s *authnodeStore) Put(vol, path, key string, data []byte) (err error) {
+	//TODO: implement authonode store put method
+
+	return nil
+}
+
+func (s *authnodeStore) Delete(vol, path, key string) (err error) {
+	//TODO: implement authonode store put method
+
+	return
+}
+
+func (s *authnodeStore) List(vol, path string) (data [][]byte, err error) {
+	//TODO: implement authonode store list method
+
+	return
+}

objectnode/fs_store_object.go

@@ -0,0 +1,36 @@
+package objectnode
+
+import (
+	"github.com/chubaofs/chubaofs/util/log"
+)
+
+const (
+	META_OSS_VOLUME = ".oss_meta"
+)
+
+type objectStore struct {
+	vm *volumeManager
+}
+
+func (s *objectStore) Init(vm *volumeManager) {
+	s.vm = vm
+	//TODO: init meta dir
+}
+
+func (s *objectStore) Put(vol, obj, key string, data []byte) (err error) {
+	log.LogInfo("put object store")
+
+	return
+}
+
+func (s *objectStore) Get(vol, obj, key string) (data []byte, err error) {
+	return
+}
+
+func (s *objectStore) Delete(vol, obj, key string) (err error) {
+	return
+}
+
+func (s *objectStore) List(vol, obj string) (data [][]byte, err error) {
+	return
+}

objectnode/fs_store_xattr.go

@@ -0,0 +1,89 @@
+package objectnode
+
+import (
+	"strings"
+
+	"github.com/chubaofs/chubaofs/proto"
+
+	"github.com/chubaofs/chubaofs/util/log"
+)
+
+const (
+	volumeRootInode = uint64(1)
+)
+
+type xattrStore struct {
+	vm *volumeManager //vol *volume
+}
+
+func (s *xattrStore) Init(vm *volumeManager) {
+	s.vm = vm
+}
+
+func (s *xattrStore) getInode(vol, path string) (*volume, uint64, error) {
+	v, err := s.vm.loadVolume(vol)
+	if err != nil {
+		return nil, 0, err
+	}
+	inode := volumeRootInode
+	if path != "" && path != "/" {
+		items := strings.Split(path, "/")
+		for _, item := range items {
+			if item == "" {
+				continue
+			}
+			inode, _, err = v.mw.Lookup_ll(inode, item)
+			if err != nil {
+				return v, inode, err
+			}
+		}
+	}
+	return v, inode, nil
+}
+
+func (s *xattrStore) Put(vol, path, key string, data []byte) (err error) {
+	v, err1 := s.vm.loadVolume(vol)
+	if err1 != nil {
+		err = err1
+		return
+	}
+	err = v.SetXAttr(path, key, data)
+	if err != nil {
+		log.LogErrorf("policy: %v, %v", key, data)
+	}
+	return
+}
+
+func (s *xattrStore) Get(vol, path, key string) (val []byte, err error) {
+	var v *volume
+	v, err = s.vm.loadVolume(vol)
+	if err != nil {
+		return
+	}
+
+	var xattrInfo *proto.XAttrInfo
+	if xattrInfo, err = v.GetXAttr(path, key); err != nil {
+		return
+	}
+	if xattrInfo == nil {
+		return
+	}
+
+	var strVal string
+	strVal = xattrInfo.XAttrs[key]
+	if len(strVal) > 0 {
+		val = []byte(strVal)
+		return
+	}
+	return
+}
+
+func (s *xattrStore) Delete(vol, obj, key string) (err error) {
+
+	return
+}
+
+func (s *xattrStore) List(vol, obj string) (data [][]byte, err error) {
+
+	return
+}

objectnode/fs_stream.go

@@ -0,0 +1,81 @@
+package objectnode
+
+import (
+	"context"
+	"io"
+	"sync"
+)
+
+type Stream struct {
+	mu     *sync.Mutex
+	cond   *sync.Cond
+	buffer []byte
+	closed bool
+	ctx    context.Context
+}
+
+func (s *Stream) Write(p []byte) (n int, err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	select {
+	case <-s.ctx.Done():
+		return 0, s.ctx.Err()
+	default:
+	}
+	if s.closed {
+		return 0, io.EOF
+	}
+	s.buffer = append(s.buffer, p...)
+	s.cond.L.Lock()
+	s.cond.Broadcast()
+	s.cond.L.Unlock()
+	return len(p), nil
+}
+
+func (s *Stream) Close() error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.closed = true
+	s.cond.L.Lock()
+	s.cond.Broadcast()
+	s.cond.L.Unlock()
+	return nil
+}
+
+func (s *Stream) Read(p []byte) (n int, err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	select {
+	case <-s.ctx.Done():
+		return 0, s.ctx.Err()
+	default:
+	}
+	if len(s.buffer) == 0 && s.closed {
+		return 0, io.EOF
+	}
+	if len(s.buffer) == 0 {
+		s.mu.Unlock()
+		s.cond.L.Lock()
+		s.cond.Wait()
+		s.cond.L.Unlock()
+		s.mu.Lock()
+		if len(s.buffer) == 0 && s.closed {
+			return 0, io.EOF
+		}
+	}
+	var readN = len(p)
+	if len(s.buffer) < readN {
+		readN = len(s.buffer)
+	}
+	copy(p, s.buffer[:readN])
+	s.buffer = s.buffer[readN:]
+	return readN, nil
+}
+
+func NewStream(ctx context.Context) *Stream {
+	stream := &Stream{}
+	stream.mu = new(sync.Mutex)
+	stream.cond = sync.NewCond(new(sync.Mutex))
+	stream.ctx = ctx
+	return stream
+}

objectnode/fs_stream_test.go

@@ -0,0 +1,88 @@
+package objectnode
+
+import (
+	"context"
+	"io"
+	"reflect"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestNewStream_modify(t *testing.T) {
+	ctx, _ := context.WithCancel(context.Background())
+	stream := NewStream(ctx)
+	var src = []byte("1234567890")
+
+	// write to stream
+	var err error
+	if _, err = stream.Write(src); err != nil {
+		t.Fatalf("write fail casue: %v", err)
+	}
+	_ = stream.Close()
+	// modify source
+	for i := 0; i < len(src); i++ {
+		src[i] = 0
+	}
+	// read from stream
+	var actual = make([]byte, len(src))
+	if _, err = stream.Read(actual); err != nil && err != io.EOF {
+		t.Fatalf("read fail cause: %v", err)
+	}
+	if string(actual) != "1234567890" {
+		t.Fatalf("result mismatch")
+	}
+}
+
+func TestStream_rw(t *testing.T) {
+	ctx, _ := context.WithCancel(context.Background())
+	stream := NewStream(ctx)
+	var src = []byte("1234567890")
+
+	asyncWG := new(sync.WaitGroup)
+
+	// async write
+	asyncWG.Add(1)
+	go func() {
+		defer asyncWG.Done()
+		var err error
+		var n int
+		for _, b := range src {
+			n, err = stream.Write([]byte{b})
+			if err != nil {
+				t.Fatalf("write fail cause: %v", err)
+			}
+			if n != 1 {
+				t.Fatalf("write bytes count mismatch: expect 10 actual %v", n)
+			}
+			time.Sleep(100 * time.Millisecond)
+		}
+		_ = stream.Close()
+	}()
+
+	// async read
+	asyncWG.Add(1)
+	go func() {
+		defer asyncWG.Done()
+		var err error
+		var n int
+		tmp := make([]byte, 3)
+		result := make([]byte, 0)
+		for {
+			n, err = stream.Read(tmp)
+			if err != nil && err != io.EOF {
+				t.Fatalf("read fail cause: %v", err)
+			}
+			if err == io.EOF {
+				break
+			}
+			t.Logf("read: %v", tmp[:n])
+			result = append(result, tmp[:n]...)
+		}
+		if !reflect.DeepEqual(result, src) {
+			t.Fatalf("result mismatch: expect %v actual %v", src, result)
+		}
+	}()
+
+	asyncWG.Wait()
+}

objectnode/fs_volume_test.go

@@ -0,0 +1 @@
+package objectnode