build: add blobstore module access

Signed-off-by: slasher <mcq.sejust@gmail.com>

blobstore/.github/workflows/codeql.yml

@@ -0,0 +1,52 @@
+name: CodeQL
+
+on:
+  push:
+    branches:
+      - master
+      - main
+      - release-*
+    tags:
+      - v*
+  pull_request:
+    types: [opened, synchronize, reopened]
+    branches:
+      - master
+      - main
+      - release-*
+
+jobs:
+  CodeQL-Build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ["go"]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v1
+        with:
+          languages: ${{ matrix.language }}
+
+      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v1
+
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 https://git.io/JvXDl
+
+      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+      #    and modify them (or add more) to build your code if your project
+      #    uses a compiled language
+
+      #- run: |
+      #   make bootstrap
+      #   make release
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v1

blobstore/.github/workflows/format.yml

@@ -0,0 +1,37 @@
+name: Format
+
+on:
+  push:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  GolangFormat:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go Version 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.17
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Make rocksdb includes
+        run: |
+          wget -O deps.tar.gz http://ocs-upload-cn.heytapcs.com/deps.tar.gz
+          tar -zxf deps.tar.gz
+          mv deps .deps
+          source env.sh
+          echo "CGO_CFLAGS=${CGO_CFLAGS}" >> $GITHUB_ENV
+          echo "CGO_LDFLAGS=${CGO_LDFLAGS}" >> $GITHUB_ENV
+
+      - name: Check vet
+        run: |
+          go vet -trimpath ./...
+
+      - name: Go code format with gofumpt
+        run: |
+          go install mvdan.cc/gofumpt@v0.2.1
+          gofumpt -l -w .
+          git diff --exit-code

blobstore/.github/workflows/lint.yml

@@ -0,0 +1,56 @@
+name: Lint
+
+on:
+  push:
+    branches:
+      - master
+      - main
+      - release-*
+    tags:
+      - v*
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+permissions:
+  contents: read
+
+jobs:
+  GolangCI-Lint:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Make rocksdb includes
+        run: |
+          wget -O deps.tar.gz http://ocs-upload-cn.heytapcs.com/deps.tar.gz
+          tar -zxf deps.tar.gz
+          mv deps .deps
+          source env.sh
+          echo "CGO_CFLAGS=${CGO_CFLAGS}" >> $GITHUB_ENV
+          echo "CGO_LDFLAGS=${CGO_LDFLAGS}" >> $GITHUB_ENV
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v2.5.2
+        with:
+          # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
+          version: v1.43.0
+
+          # Optional: working directory, useful for monorepos
+          # working-directory: somedir
+
+          # Optional: golangci-lint command line arguments.
+          # args: --issues-exit-code=0 -D errcheck -D typecheck --exclude SA1019
+          args: --issues-exit-code=1 -D errcheck -E bodyclose ./...
+
+          # Optional: show only new issues if it's a pull request. The default value is `false`.
+          # only-new-issues: true
+
+          # Optional: if set to true then the action will use pre-installed Go.
+          # skip-go-installation: true
+
+          # Optional: if set to true then the action don't cache or restore ~/go/pkg.
+          # skip-pkg-cache: true
+
+          # Optional: if set to true then the action don't cache or restore ~/.cache/go-build.
+          # skip-build-cache: true

blobstore/.github/workflows/test.yml

@@ -0,0 +1,49 @@
+name: Test
+
+on:
+  push:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  GolangTest:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        # os: [windows-latest, ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Set up Go Version 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.17
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Make rocksdb includes
+        run: |
+          wget -O deps.tar.gz http://ocs-upload-cn.heytapcs.com/deps.tar.gz
+          tar -zxf deps.tar.gz
+          mv deps .deps
+          source env.sh
+          echo "CGO_CFLAGS=${CGO_CFLAGS}" >> $GITHUB_ENV
+          echo "CGO_LDFLAGS=${CGO_LDFLAGS}" >> $GITHUB_ENV
+
+      - name: Check and Verify Go modules
+        run: |
+          go mod tidy
+          git diff --exit-code go.mod go.sum
+          go mod verify
+
+      - name: Go test with coverage
+        run: |
+          go test -trimpath -covermode=count --coverprofile coverage.txt ./...
+        env:
+          JENKINS_TEST: TRUE
+
+      - name: Upload codecov
+        uses: codecov/codecov-action@v2
+        with:
+          files: ./coverage.txt

blobstore/.gitignore

@@ -0,0 +1,12 @@
+.vscode
+.idea/
+.DS_Store
+.version
+.cache
+.deps/
+bin/
+*/*/*/run/
+node_modules/
+package.json
+package-lock.json
+!.gitkeep

blobstore/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

blobstore/Makefile

@@ -0,0 +1,68 @@
+# Copyright 2022 The CubeFS Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+PROJECTDIR=$(shell pwd)
+OS=$(shell uname -s)
+BINDIR=$(PROJECTDIR)/bin
+GCFLAGS=all=-trimpath=$(PROJECTDIR)
+ASMFLAGS=all=-trimpath=$(PROJECTDIR)
+BRANCHNAME=${BUILD_BRANCH}
+COMMITID=${GIT_COMMIT}
+
+ifeq ($(BRANCHNAME),)
+	BRANCHNAME=$(shell git symbolic-ref --short -q HEAD)
+	COMMITID=$(shell git rev-parse --short HEAD)
+endif
+
+LDFLAGS=-w -s
+ifneq ($(BRANCHNAME),)
+    LDFLAGS+= -X "github.com/cubefs/cubefs/blobstore/util/version.version=$(BRANCHNAME)/$(COMMITID)"
+endif
+
+BUILD=go build -v -gcflags=$(GCFLAGS) -asmflags=$(ASMFLAGS) -ldflags='$(LDFLAGS)' -o $(BINDIR)
+INSTALL=CGO_ENABLED=0 $(BUILD)
+CGOINSTALL=CGO_ENABLED=1 $(BUILD)
+CMDDIR=github.com/cubefs/cubefs/blobstore/cmd
+TARGETS=blobnode cm access scheduler tinker proxy cli
+
+.PHONY: clean all $(TARGETS)
+all:$(TARGETS)
+
+cm:
+	@echo "building clustermgr"
+	@$(CGOINSTALL) $(CMDDIR)/clustermgr
+
+blobnode:
+	@echo "building blobnode"
+	@$(CGOINSTALL) $(CMDDIR)/blobnode
+
+access:
+	@echo "building access"
+	@$(INSTALL) $(CMDDIR)/access
+
+scheduler:
+	@echo "building scheduler"
+	@$(INSTALL) $(CMDDIR)/scheduler
+
+proxy:
+	@echo "building proxy"
+	@$(INSTALL) $(CMDDIR)/proxy
+
+cli:
+	@echo "building cli"
+	@$(CGOINSTALL) $(PROJECTDIR)/cli/cli
+
+clean:
+	@go clean -i ./...
+	@rm -f $(BINDIR)/*

blobstore/README.md

@@ -0,0 +1,40 @@
+# BlobStore
+- [Overview](#overview)
+- [Documents](#documents)
+- [Build BlobStore](#build-blobstore)
+- [Deploy BlobStore](#deploy-blobstore)
+- [Manage BlobStore](#manage-blobstore)
+- [License](#license)
+
+## Overview
+BlobStore is a highly reliable,highly available and  ultra-large scale  distributed storage system. The system adopts Reed-Solomon code, which provides higher data durability with less storage cost than use three copies  backup technology, and supports multiple erasure code modes multiple availability zones,and optimizes for small file to meet the storage needs of different scenarios.
+Some key features of BlobStore include:
+- ultra-large scale
+- high reliability
+- flexible deployment
+- low cost
+
+
+## Documents
+
+English version: https://cubefs.readthedocs.io/en/latest/
+
+Chinese version: https://cubefs.readthedocs.io/zh_CN/latest/
+
+## Build BlobStore
+
+```
+$ git clone http://github.com/cubefs/cubefs.git
+$ cd cubefs/blobstore
+$ source env.sh
+$ ./build.sh
+```
+
+## Deploy BlobStore
+For more details please refer to [documentation](https://cubefs.readthedocs.io/en/latest/user-guide/blobstore.html).
+
+## Manage BlobStore
+For more details please refer to [documentation](https://cubefs.readthedocs.io/en/latest/admin-api/blobstore/blobnode.html).
+
+## License
+BlobStore is licensed under the Apache License, Version 2.0. For detail see LICENSE and NOTICE.

blobstore/access/access_mock_test.go

@@ -0,0 +1,217 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/cubefs/cubefs/blobstore/access (interfaces: StreamHandler,Limiter)
+
+// Package access is a generated GoMock package.
+package access
+
+import (
+	context "context"
+	io "io"
+	reflect "reflect"
+
+	access0 "github.com/cubefs/cubefs/blobstore/api/access"
+	codemode "github.com/cubefs/cubefs/blobstore/common/codemode"
+	proto "github.com/cubefs/cubefs/blobstore/common/proto"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockStreamHandler is a mock of StreamHandler interface.
+type MockStreamHandler struct {
+	ctrl     *gomock.Controller
+	recorder *MockStreamHandlerMockRecorder
+}
+
+// MockStreamHandlerMockRecorder is the mock recorder for MockStreamHandler.
+type MockStreamHandlerMockRecorder struct {
+	mock *MockStreamHandler
+}
+
+// NewMockStreamHandler creates a new mock instance.
+func NewMockStreamHandler(ctrl *gomock.Controller) *MockStreamHandler {
+	mock := &MockStreamHandler{ctrl: ctrl}
+	mock.recorder = &MockStreamHandlerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockStreamHandler) EXPECT() *MockStreamHandlerMockRecorder {
+	return m.recorder
+}
+
+// Admin mocks base method.
+func (m *MockStreamHandler) Admin() interface{} {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Admin")
+	ret0, _ := ret[0].(interface{})
+	return ret0
+}
+
+// Admin indicates an expected call of Admin.
+func (mr *MockStreamHandlerMockRecorder) Admin() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Admin", reflect.TypeOf((*MockStreamHandler)(nil).Admin))
+}
+
+// Alloc mocks base method.
+func (m *MockStreamHandler) Alloc(arg0 context.Context, arg1 uint64, arg2 uint32, arg3 proto.ClusterID, arg4 codemode.CodeMode) (*access0.Location, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Alloc", arg0, arg1, arg2, arg3, arg4)
+	ret0, _ := ret[0].(*access0.Location)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Alloc indicates an expected call of Alloc.
+func (mr *MockStreamHandlerMockRecorder) Alloc(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Alloc", reflect.TypeOf((*MockStreamHandler)(nil).Alloc), arg0, arg1, arg2, arg3, arg4)
+}
+
+// Delete mocks base method.
+func (m *MockStreamHandler) Delete(arg0 context.Context, arg1 *access0.Location) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Delete", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Delete indicates an expected call of Delete.
+func (mr *MockStreamHandlerMockRecorder) Delete(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockStreamHandler)(nil).Delete), arg0, arg1)
+}
+
+// Get mocks base method.
+func (m *MockStreamHandler) Get(arg0 context.Context, arg1 io.Writer, arg2 access0.Location, arg3, arg4 uint64) (func() error, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2, arg3, arg4)
+	ret0, _ := ret[0].(func() error)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Get indicates an expected call of Get.
+func (mr *MockStreamHandlerMockRecorder) Get(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockStreamHandler)(nil).Get), arg0, arg1, arg2, arg3, arg4)
+}
+
+// Put mocks base method.
+func (m *MockStreamHandler) Put(arg0 context.Context, arg1 io.Reader, arg2 int64, arg3 access0.HasherMap) (*access0.Location, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Put", arg0, arg1, arg2, arg3)
+	ret0, _ := ret[0].(*access0.Location)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Put indicates an expected call of Put.
+func (mr *MockStreamHandlerMockRecorder) Put(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockStreamHandler)(nil).Put), arg0, arg1, arg2, arg3)
+}
+
+// PutAt mocks base method.
+func (m *MockStreamHandler) PutAt(arg0 context.Context, arg1 io.Reader, arg2 proto.ClusterID, arg3 proto.Vid, arg4 proto.BlobID, arg5 int64, arg6 access0.HasherMap) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "PutAt", arg0, arg1, arg2, arg3, arg4, arg5, arg6)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// PutAt indicates an expected call of PutAt.
+func (mr *MockStreamHandlerMockRecorder) PutAt(arg0, arg1, arg2, arg3, arg4, arg5, arg6 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutAt", reflect.TypeOf((*MockStreamHandler)(nil).PutAt), arg0, arg1, arg2, arg3, arg4, arg5, arg6)
+}
+
+// MockLimiter is a mock of Limiter interface.
+type MockLimiter struct {
+	ctrl     *gomock.Controller
+	recorder *MockLimiterMockRecorder
+}
+
+// MockLimiterMockRecorder is the mock recorder for MockLimiter.
+type MockLimiterMockRecorder struct {
+	mock *MockLimiter
+}
+
+// NewMockLimiter creates a new mock instance.
+func NewMockLimiter(ctrl *gomock.Controller) *MockLimiter {
+	mock := &MockLimiter{ctrl: ctrl}
+	mock.recorder = &MockLimiterMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockLimiter) EXPECT() *MockLimiterMockRecorder {
+	return m.recorder
+}
+
+// Acquire mocks base method.
+func (m *MockLimiter) Acquire(arg0 string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Acquire", arg0)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Acquire indicates an expected call of Acquire.
+func (mr *MockLimiterMockRecorder) Acquire(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Acquire", reflect.TypeOf((*MockLimiter)(nil).Acquire), arg0)
+}
+
+// Reader mocks base method.
+func (m *MockLimiter) Reader(arg0 context.Context, arg1 io.Reader) io.Reader {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Reader", arg0, arg1)
+	ret0, _ := ret[0].(io.Reader)
+	return ret0
+}
+
+// Reader indicates an expected call of Reader.
+func (mr *MockLimiterMockRecorder) Reader(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reader", reflect.TypeOf((*MockLimiter)(nil).Reader), arg0, arg1)
+}
+
+// Release mocks base method.
+func (m *MockLimiter) Release(arg0 string) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "Release", arg0)
+}
+
+// Release indicates an expected call of Release.
+func (mr *MockLimiterMockRecorder) Release(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Release", reflect.TypeOf((*MockLimiter)(nil).Release), arg0)
+}
+
+// Status mocks base method.
+func (m *MockLimiter) Status() Status {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Status")
+	ret0, _ := ret[0].(Status)
+	return ret0
+}
+
+// Status indicates an expected call of Status.
+func (mr *MockLimiterMockRecorder) Status() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Status", reflect.TypeOf((*MockLimiter)(nil).Status))
+}
+
+// Writer mocks base method.
+func (m *MockLimiter) Writer(arg0 context.Context, arg1 io.Writer) io.Writer {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Writer", arg0, arg1)
+	ret0, _ := ret[0].(io.Writer)
+	return ret0
+}
+
+// Writer indicates an expected call of Writer.
+func (mr *MockLimiterMockRecorder) Writer(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Writer", reflect.TypeOf((*MockLimiter)(nil).Writer), arg0, arg1)
+}

blobstore/access/codemode.go

@@ -0,0 +1,45 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"fmt"
+
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+)
+
+// CodeModePair codemode with pair of tactic and policy
+type CodeModePair struct {
+	Policy codemode.Policy
+	Tactic codemode.Tactic
+}
+
+// CodeModePairs map of CodeModePair
+type CodeModePairs map[codemode.CodeMode]CodeModePair
+
+// SelectCodeMode select codemode by size
+func (c CodeModePairs) SelectCodeMode(size int64) codemode.CodeMode {
+	for codeMode, pair := range c {
+		policy := pair.Policy
+		if !policy.Enable {
+			continue
+		}
+		if size >= policy.MinSize && size <= policy.MaxSize {
+			return codeMode
+		}
+	}
+
+	panic(fmt.Sprintf("no codemode policy to be selected by size %d, %+v", size, c))
+}

blobstore/access/codemode_test.go

@@ -0,0 +1,71 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/access"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+)
+
+func TestAccessStreamCodeModePairs(t *testing.T) {
+	m := access.CodeModePairs{
+		codemode.EC6P6: access.CodeModePair{
+			Policy: codemode.Policy{
+				ModeName: codemode.EC6P6.Name(),
+				MinSize:  1 << 10,
+				MaxSize:  1 << 20,
+				Enable:   true,
+			},
+			Tactic: codemode.EC6P6.Tactic(),
+		},
+		codemode.EC6P10L2: access.CodeModePair{
+			Policy: codemode.Policy{
+				ModeName: codemode.EC6P10L2.Name(),
+				MinSize:  1 << 30,
+				MaxSize:  1 << 40,
+				Enable:   true,
+			},
+			Tactic: codemode.EC6P10L2.Tactic(),
+		},
+	}
+
+	cases := []struct {
+		size    int64
+		isPanic bool
+		mode    codemode.CodeMode
+	}{
+		{-1, true, 0},
+		{0, true, 0},
+		{1 << 8, true, 0},
+		{1 << 10, false, codemode.EC6P6},
+		{1 << 14, false, codemode.EC6P6},
+		{1 << 20, false, codemode.EC6P6},
+		{1 << 25, true, 0},
+		{1 << 30, false, codemode.EC6P10L2},
+		{1 << 40, false, codemode.EC6P10L2},
+		{1 << 50, true, 0},
+	}
+	for _, cs := range cases {
+		if cs.isPanic {
+			require.Panics(t, func() { m.SelectCodeMode(cs.size) })
+		} else {
+			require.Equal(t, cs.mode, m.SelectCodeMode(cs.size))
+		}
+	}
+}

blobstore/access/config_defaulter.go

@@ -0,0 +1,75 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+const (
+	defaultMaxBlobSize uint32 = 1 << 22 // 4MB
+
+	defaultDiskPunishIntervalS    int = 60
+	defaultServicePunishIntervalS int = 60
+	defaultAllocRetryTimes        int = 3
+	defaultAllocRetryIntervalMS   int = 100
+	defaultEncoderConcurrency     int = 1000
+	defaultMinReadShardsX         int = 1
+
+	// client timeout ms
+	defaultTimeoutClusterMgr int64 = 1000 * 3
+	defaultTimeoutProxy      int64 = 1000 * 5
+	defaultTimeoutBlobnode   int64 = 1000 * 5
+
+	defaultAllocatorErrorPercentThreshold  int = 50
+	defaultAllocatorMaxConcurrentRequests  int = 10240
+	defaultAllocatorRequestVolumeThreshold int = 100
+	defaultAllocatorSleepWindow            int = 2 * 1000
+	defaultAllocatorTimeout                int = 30 * 1000
+	defaultBlobnodeErrorPercentThreshold   int = 80
+	defaultBlobnodeMaxConcurrentRequests   int = 102400
+	defaultBlobnodeRequestVolumeThreshold  int = 1000
+	defaultBlobnodeSleepWindow             int = 5 * 1000
+	defaultBlobnodeTimeout                 int = 600 * 1000
+)
+
+// ec buffer
+// |     blobsize      |      EC15P12      |       EC6P6       |     EC16P20L2     |     EC6P10L2      |       EC3P3       |
+// |      1(1 B)       |   55296(54 KiB)   |   24576(24 KiB)   |   77824(76 KiB)   |   36864(36 KiB)   |   12288(12 KiB)   |
+// |   2048(2.0 KiB)   |   55296(54 KiB)   |   24576(24 KiB)   |   77824(76 KiB)   |   36864(36 KiB)   |   12288(12 KiB)   |
+// |   12288(12 KiB)   |   55296(54 KiB)   |   24576(24 KiB)   |   77824(76 KiB)   |   36864(36 KiB)   |   24576(24 KiB)   |
+// |   65536(64 KiB)   |  117990(115 KiB)  |  131076(128 KiB)  |  155648(152 KiB)  |  196614(192 KiB)  |  131076(128 KiB)  |
+// |  524288(512 KiB)  |  943731(922 KiB)  | 1048584(1.0 MiB)  | 1245184(1.2 MiB)  | 1572876(1.5 MiB)  | 1048578(1.0 MiB)  |
+// | 1048576(1.0 MiB)  | 1887462(1.8 MiB)  | 2097156(2.0 MiB)  | 2490368(2.4 MiB)  | 3145734(3.0 MiB)  | 2097156(2.0 MiB)  |
+// | 2097152(2.0 MiB)  | 3774897(3.6 MiB)  | 4194312(4.0 MiB)  | 4980736(4.8 MiB)  | 6291468(6.0 MiB)  | 4194306(4.0 MiB)  |
+// | 4194304(4.0 MiB)  | 7549767(7.2 MiB)  | 8388612(8.0 MiB)  | 9961472(9.5 MiB)  | 12582918(12 MiB)  | 8388612(8.0 MiB)  |
+// | 8388608(8.0 MiB)  | 15099507(14 MiB)  | 16777224(16 MiB)  | 19922944(19 MiB)  | 25165836(24 MiB)  | 16777218(16 MiB)  |
+// | 16777216(16 MiB)  | 30199014(29 MiB)  | 33554436(32 MiB)  | 39845888(38 MiB)  | 50331654(48 MiB)  | 33554436(32 MiB)  |
+
+const (
+	_kib = 1 << 10
+	_mib = 1 << 20
+)
+
+func getDefaultMempoolSize() map[int]int {
+	return map[int]int{
+		_kib * 2:   -1, // 2KiB for aligned ranged object
+		_kib * 16:  -1,
+		_kib * 128: -1,
+		_kib * 512: -1,
+		_mib * 1:   -1,
+		_mib * 2:   -1,
+		_mib * 4:   -1,
+		_mib * 8:   -1,
+		_mib * 16:  -1,
+		_mib * 32:  -1,
+	}
+}

blobstore/access/controller/cluster.go

@@ -0,0 +1,376 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package controller
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"math/rand"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/hashicorp/consul/api"
+
+	cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/redis"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/util/errors"
+	"github.com/cubefs/cubefs/blobstore/util/log"
+)
+
+// TODO: how to stop service of one cluster???
+
+// AlgChoose algorithm of choose cluster
+type AlgChoose uint32
+
+const (
+	minAlg AlgChoose = iota
+	// AlgAvailable available capacity and some random alloc
+	AlgAvailable
+	// AlgRoundRobin alloc cluster round robin
+	AlgRoundRobin
+	// AlgRandom completely random alloc
+	AlgRandom
+	maxAlg
+)
+
+// IsValid returns valid algorithm or not.
+func (alg AlgChoose) IsValid() bool {
+	return alg > minAlg && alg < maxAlg
+}
+
+func (alg AlgChoose) String() string {
+	switch alg {
+	case AlgAvailable:
+		return "Available"
+	case AlgRandom:
+		return "Random"
+	default:
+		return "Unknow"
+	}
+}
+
+// errors
+var (
+	ErrNoSuchCluster    = errors.New("controller: no such cluster")
+	ErrInvalidChooseAlg = errors.New("controller: invalid cluster chosen algorithm")
+)
+
+// ClusterController controller of clusters in one region
+type ClusterController interface {
+	// Region returns region in configuration
+	Region() string
+	// All returns all cluster info in this region
+	All() []*cmapi.ClusterInfo
+	// ChooseOne returns a available cluster to upload
+	ChooseOne() (*cmapi.ClusterInfo, error)
+	// GetServiceController return ServiceController in specified cluster
+	GetServiceController(clusterID proto.ClusterID) (ServiceController, error)
+	// GetVolumeGetter return VolumeGetter in specified cluster
+	GetVolumeGetter(clusterID proto.ClusterID) (VolumeGetter, error)
+	// GetConfig get specified config of key from cluster manager
+	GetConfig(ctx context.Context, key string) (string, error)
+	// ChangeChooseAlg change alloc algorithm
+	ChangeChooseAlg(alg AlgChoose) error
+}
+
+// ClusterConfig cluster config
+//
+// Region and RegionMagic are paired,
+// magic cannot change if one region was deployed.
+type ClusterConfig struct {
+	IDC               string              `json:"-"`
+	Region            string              `json:"region"`
+	RegionMagic       string              `json:"region_magic"`
+	ClusterReloadSecs int                 `json:"cluster_reload_secs"`
+	ServiceReloadSecs int                 `json:"service_reload_secs"`
+	CMClientConfig    cmapi.Config        `json:"clustermgr_client_config"`
+	RedisClientConfig redis.ClusterConfig `json:"redis_client_config"`
+
+	ServicePunishThreshold      uint32 `json:"service_punish_threshold"`
+	ServicePunishValidIntervalS int    `json:"service_punish_valid_interval_s"`
+}
+
+type cluster struct {
+	clusterInfo *cmapi.ClusterInfo
+	client      *cmapi.Client
+}
+
+type clusterMap map[proto.ClusterID]*cluster
+
+type clusterQueue []*cmapi.ClusterInfo
+
+type clusterControllerImpl struct {
+	region          string
+	kvClient        *api.Client
+	allocAlg        uint32
+	totalAvailable  int64        // available space of all clusters
+	clusters        atomic.Value // all clusters
+	available       atomic.Value // available clusters
+	serviceMgrs     sync.Map
+	volumeGetters   sync.Map
+	roundRobinCount uint64 // a count for round robin
+
+	config ClusterConfig
+}
+
+// NewClusterController returns a cluster controller
+func NewClusterController(cfg *ClusterConfig, kvClient *api.Client) (ClusterController, error) {
+	controller := &clusterControllerImpl{
+		region:   cfg.Region,
+		kvClient: kvClient,
+		config:   *cfg,
+	}
+	atomic.StoreUint32(&controller.allocAlg, uint32(AlgAvailable))
+
+	if err := controller.load(); err != nil {
+		return nil, errors.Base(err, "load cluster failed")
+	}
+
+	if cfg.ClusterReloadSecs <= 0 {
+		cfg.ClusterReloadSecs = 3
+	}
+	tick := time.NewTicker(time.Duration(cfg.ClusterReloadSecs) * time.Second)
+	go func() {
+		defer tick.Stop()
+		for range tick.C {
+			if err := controller.load(); err != nil {
+				log.Warn("load timer error", err)
+			}
+		}
+	}()
+	return controller, nil
+}
+
+func (c *clusterControllerImpl) load() error {
+	span := trace.SpanFromContextSafe(context.Background())
+
+	path := cmapi.GetConsulClusterPath(c.region)
+	span.Debug("to list consul path", path)
+
+	pairs, _, err := c.kvClient.KV().List(path, nil)
+	if err != nil {
+		return err
+	}
+	span.Debugf("found %d clusters", len(pairs))
+
+	allClusters := make(clusterMap)
+	available := make([]*cmapi.ClusterInfo, 0, len(pairs))
+	totalAvailable := int64(0)
+	for _, pair := range pairs {
+		clusterInfo := &cmapi.ClusterInfo{}
+		err := json.Unmarshal(pair.Value, clusterInfo)
+		if err != nil {
+			span.Warnf("decode failed, raw:%s, error:%s", string(pair.Value), err.Error())
+			continue
+		}
+
+		clusterKey := filepath.Base(pair.Key)
+		span.Debug("found cluster", clusterKey)
+
+		clusterID, err := strconv.Atoi(clusterKey)
+		if err != nil {
+			span.Warn("invalid cluster id", clusterKey, err)
+			continue
+		}
+		if clusterInfo.ClusterID != proto.ClusterID(clusterID) {
+			span.Warn("mismatch cluster id", clusterInfo.ClusterID, clusterID)
+			continue
+		}
+
+		allClusters[proto.ClusterID(clusterID)] = &cluster{clusterInfo: clusterInfo}
+		if !clusterInfo.Readonly && clusterInfo.Available > 0 {
+			available = append(available, clusterInfo)
+			totalAvailable += clusterInfo.Available
+		} else {
+			span.Debug("readonly or no available cluster", clusterID)
+		}
+	}
+
+	sort.Slice(available, func(i, j int) bool {
+		return available[i].Capacity < available[j].Capacity
+	})
+
+	newClusters := make([]*cmapi.ClusterInfo, 0, len(allClusters))
+	for clusterID := range allClusters {
+		if _, ok := c.serviceMgrs.Load(clusterID); !ok {
+			newClusters = append(newClusters, allClusters[clusterID].clusterInfo)
+		}
+	}
+
+	for _, newCluster := range newClusters {
+		conf := c.config.CMClientConfig
+		conf.Hosts = newCluster.Nodes
+		cmCli := cmapi.New(&conf)
+
+		clusterID := newCluster.ClusterID
+		allClusters[clusterID].client = cmCli
+
+		removeThisCluster := func() {
+			delete(allClusters, clusterID)
+			if !newCluster.Readonly && newCluster.Available > 0 {
+				totalAvailable -= newCluster.Available
+				for j := range available {
+					if available[j].ClusterID == clusterID {
+						available = append(available[:j], available[j+1:]...)
+						break
+					}
+				}
+			}
+		}
+
+		serviceController, err := NewServiceController(ServiceConfig{
+			ClusterID:                   clusterID,
+			IDC:                         c.config.IDC,
+			ReloadSec:                   c.config.ServiceReloadSecs,
+			ServicePunishThreshold:      c.config.ServicePunishThreshold,
+			ServicePunishValidIntervalS: c.config.ServicePunishValidIntervalS,
+		}, cmCli)
+		if err != nil {
+			removeThisCluster()
+			span.Warn("new service manager failed", clusterID, err)
+			continue
+		}
+
+		var redisCli *redis.ClusterClient
+		if len(c.config.RedisClientConfig.Addrs) > 0 {
+			redisCli = redis.NewClusterClient(&c.config.RedisClientConfig)
+		}
+		volumeGetter, err := NewVolumeGetter(clusterID, cmCli, redisCli, -1)
+		if err != nil {
+			removeThisCluster()
+			span.Warn("new volume getter failed", clusterID, err)
+			continue
+		}
+
+		c.serviceMgrs.Store(clusterID, serviceController)
+		c.volumeGetters.Store(clusterID, volumeGetter)
+
+		span.Debug("loaded new cluster", clusterID)
+	}
+
+	c.clusters.Store(allClusters)
+	c.available.Store(clusterQueue(available))
+	atomic.StoreInt64(&c.totalAvailable, totalAvailable)
+
+	span.Infof("loaded %d clusters, and %d available, total available space %.3fTB",
+		len(allClusters), len(available), float64(totalAvailable)/(1<<40))
+	return nil
+}
+
+func (c *clusterControllerImpl) Region() string {
+	return c.region
+}
+
+func (c *clusterControllerImpl) All() []*cmapi.ClusterInfo {
+	allClusters := c.clusters.Load().(clusterMap)
+
+	ret := make([]*cmapi.ClusterInfo, 0, len(allClusters))
+	for _, clusterInfo := range allClusters {
+		ret = append(ret, clusterInfo.clusterInfo)
+	}
+
+	return ret
+}
+
+func (c *clusterControllerImpl) ChooseOne() (*cmapi.ClusterInfo, error) {
+	alg := AlgChoose(atomic.LoadUint32(&c.allocAlg))
+
+	switch alg {
+	case AlgAvailable:
+		totalAvailable := atomic.LoadInt64(&c.totalAvailable)
+		if totalAvailable <= 0 {
+			return nil, fmt.Errorf("no available space %d", totalAvailable)
+		}
+		randValue := rand.Int63n(totalAvailable)
+		available := c.available.Load().(clusterQueue)
+		for _, cluster := range available {
+			if cluster.Available >= randValue {
+				return cluster, nil
+			}
+			randValue -= cluster.Available
+		}
+
+	case AlgRoundRobin:
+		available := c.available.Load().(clusterQueue)
+		if length := uint64(len(available)); length > 0 {
+			count := atomic.AddUint64(&c.roundRobinCount, 1)
+			return available[count%length], nil
+		}
+
+	case AlgRandom:
+		available := c.available.Load().(clusterQueue)
+		if length := int64(len(available)); length > 0 {
+			return available[rand.Int63()%length], nil
+		}
+
+	default:
+		return nil, fmt.Errorf("not implemented algorithm %s(%d)", alg.String(), alg)
+	}
+
+	return nil, fmt.Errorf("no available cluster by %s", alg.String())
+}
+
+func (c *clusterControllerImpl) ChangeChooseAlg(alg AlgChoose) error {
+	if !alg.IsValid() {
+		return ErrInvalidChooseAlg
+	}
+
+	atomic.StoreUint32(&c.allocAlg, uint32(alg))
+	return nil
+}
+
+func (c *clusterControllerImpl) GetServiceController(clusterID proto.ClusterID) (ServiceController, error) {
+	if serviceController, exist := c.serviceMgrs.Load(clusterID); exist {
+		if controller, ok := serviceController.(ServiceController); ok {
+			return controller, nil
+		}
+		return nil, fmt.Errorf("not service controller for %d", clusterID)
+	}
+	return nil, fmt.Errorf("no service controller of %d", clusterID)
+}
+
+func (c *clusterControllerImpl) GetVolumeGetter(clusterID proto.ClusterID) (VolumeGetter, error) {
+	if volumeGetter, exist := c.volumeGetters.Load(clusterID); exist {
+		if getter, ok := volumeGetter.(VolumeGetter); ok {
+			return getter, nil
+		}
+		return nil, fmt.Errorf("not volume getter for %d", clusterID)
+	}
+	return nil, fmt.Errorf("no volume getter for %d", clusterID)
+}
+
+func (c *clusterControllerImpl) GetConfig(ctx context.Context, key string) (ret string, err error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	allClusters := c.clusters.Load().(clusterMap)
+	if len(allClusters) == 0 {
+		return "", ErrNoSuchCluster
+	}
+
+	for _, cluster := range allClusters {
+		if ret, err = cluster.client.GetConfig(ctx, key); err == nil {
+			return
+		}
+		span.Warnf("get config[%s] from cluster[%d] failed, err: %v", key, cluster.clusterInfo.ClusterID, err)
+	}
+	return
+}

blobstore/access/controller/cluster_test.go

@@ -0,0 +1,341 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package controller_test
+
+import (
+	"context"
+	"encoding/json"
+	"math/rand"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/consul/api"
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/access/controller"
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/redis"
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+	"github.com/cubefs/cubefs/blobstore/util/log"
+)
+
+var (
+	hostAddr string
+	consulKV api.KVPairs = nil
+
+	stableCluster = &atomic.Value{}
+
+	cc, cc0, cc1, cc2, cc3, cc19 controller.ClusterController
+)
+
+func initCluster() {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", consul)
+	mux.HandleFunc("/service/get", serviceGet)
+
+	testServer := httptest.NewServer(mux)
+	hostAddr = testServer.URL
+
+	cluster1 := clustermgr.ClusterInfo{
+		Region:    region,
+		ClusterID: 1,
+		Capacity:  1 << 50,
+		Available: 1 << 45,
+		Readonly:  false,
+		Nodes:     []string{hostAddr, hostAddr},
+	}
+	data1, _ := json.Marshal(cluster1)
+
+	cluster2 := clustermgr.ClusterInfo{
+		Region:    region,
+		ClusterID: 2,
+		Capacity:  1 << 50,
+		Available: 1 << 45,
+		Readonly:  true,
+		Nodes:     []string{hostAddr, hostAddr},
+	}
+	data2, _ := json.Marshal(cluster2)
+
+	cluster3 := clustermgr.ClusterInfo{
+		Region:    region,
+		ClusterID: 3,
+		Capacity:  1 << 50,
+		Available: -1024,
+		Readonly:  false,
+		Nodes:     []string{hostAddr, hostAddr},
+	}
+	data3, _ := json.Marshal(cluster3)
+
+	cluster9 := clustermgr.ClusterInfo{
+		Region:    region,
+		ClusterID: 9,
+		Capacity:  1 << 50,
+		Available: 1 << 40,
+		Readonly:  false,
+		Nodes:     []string{hostAddr, hostAddr},
+	}
+	data9, _ := json.Marshal(cluster9)
+
+	consulKV = api.KVPairs{
+		&api.KVPair{Key: "1", Value: data1},
+		&api.KVPair{Key: "2", Value: data2},
+		&api.KVPair{Key: "3", Value: data3},
+		&api.KVPair{Key: "9", Value: data9},
+		&api.KVPair{Key: "4", Value: data1},
+		&api.KVPair{Key: "5", Value: []byte("{invalid json")},
+		&api.KVPair{Key: "cannot-parse-key", Value: []byte("{}")},
+	}
+
+	time.Sleep(time.Millisecond * 200)
+	initCC()
+}
+
+func consul(w http.ResponseWriter, req *http.Request) {
+	val := stableCluster.Load()
+	if val != nil {
+		if b := val.([]byte); b != nil {
+			w.Write(b)
+			return
+		}
+	}
+
+	data, _ := json.Marshal(consulKV)
+	w.Write(data)
+}
+
+func serviceGet(w http.ResponseWriter, req *http.Request) {
+	val := stableCluster.Load()
+	if val != nil {
+		if b := val.([]byte); b != nil {
+			w.Write([]byte("{}"))
+			return
+		}
+	}
+
+	if rand.Int31()%2 == 0 {
+		w.Write([]byte("{cannot unmarshal"))
+	} else {
+		w.Write([]byte("{}"))
+	}
+}
+
+func initCC() {
+	defer func() {
+		var data []byte
+		stableCluster.Store(data)
+	}()
+
+	count := 0
+	for cc == nil || cc0 == nil || cc1 == nil {
+		c := newCC()
+		if cc == nil {
+			cc = c
+		}
+
+		clusters := c.All()
+		switch len(clusters) {
+		case 0:
+			cc0 = c
+		case 1:
+			switch clusters[0].ClusterID {
+			case 1:
+				cc1 = c
+			case 2:
+				cc2 = c
+			case 3:
+				cc3 = c
+			}
+		}
+
+		count++
+	}
+	log.Debug("init clusters run:", count)
+
+	// init cluster 2
+	if cc2 == nil {
+		var clusters api.KVPairs
+		clusters = append(clusters, consulKV[1])
+		data, _ := json.Marshal(clusters)
+		stableCluster.Store(data)
+		cc2 = newCC()
+	}
+	// init cluster 3
+	if cc3 == nil {
+		var clusters api.KVPairs
+		clusters = append(clusters, consulKV[2])
+		data, _ := json.Marshal(clusters)
+		stableCluster.Store(data)
+		cc3 = newCC()
+	}
+
+	// init cluster 1 and 9
+	if cc19 == nil {
+		var clusters api.KVPairs
+		clusters = append(clusters, consulKV[0])
+		clusters = append(clusters, consulKV[3])
+		data, _ := json.Marshal(clusters)
+		stableCluster.Store(data)
+		cc19 = newCC()
+	}
+}
+
+func newCC() controller.ClusterController {
+	cfg := controller.ClusterConfig{
+		Region:            region,
+		ClusterReloadSecs: 0,
+		CMClientConfig: clustermgr.Config{
+			LbConfig: rpc.LbConfig{
+				Hosts: []string{"http://localhost"},
+			},
+		},
+		RedisClientConfig: redis.ClusterConfig{
+			Addrs: []string{redismr.Addr()},
+		},
+	}
+	if rand.Int31()%2 == 0 {
+		cfg.RedisClientConfig.Addrs = nil
+	}
+	conf := api.DefaultConfig()
+	conf.Address = hostAddr
+	conf.Transport = &http.Transport{
+		MaxIdleConns:    1000,
+		IdleConnTimeout: time.Minute,
+	}
+	conf.Transport.DialContext = (&net.Dialer{KeepAlive: time.Minute}).DialContext
+
+	client, _ := api.NewClient(conf)
+	cc, err := controller.NewClusterController(&cfg, client)
+	if err != nil {
+		panic(err)
+	}
+	return cc
+}
+
+func TestAccessClusterNew(t *testing.T) {
+	require.Equal(t, region, cc.Region())
+}
+
+func TestAccessClusterAll(t *testing.T) {
+	for range [5]struct{}{} {
+		require.LessOrEqual(t, len(newCC().All()), 4)
+	}
+}
+
+func TestAccessClusterChooseOne(t *testing.T) {
+	{
+		cluster, err := cc0.ChooseOne()
+		require.Error(t, err)
+		require.Equal(t, (*clustermgr.ClusterInfo)(nil), cluster)
+	}
+	{
+		_, err := cc3.ChooseOne()
+		require.Error(t, err)
+
+		_, err = cc2.ChooseOne()
+		require.Error(t, err)
+
+		cc2.ChangeChooseAlg(controller.AlgRoundRobin)
+		_, err = cc2.ChooseOne()
+		require.Error(t, err)
+	}
+	{
+		cluster, err := cc1.ChooseOne()
+		require.NoError(t, err)
+		require.NotNil(t, cluster)
+
+		for _, alg := range []controller.AlgChoose{
+			controller.AlgAvailable,
+			controller.AlgRoundRobin,
+			controller.AlgRandom,
+		} {
+			cc1.ChangeChooseAlg(alg)
+			for range [100]struct{}{} {
+				clusterx, err := cc1.ChooseOne()
+				require.NoError(t, err)
+				require.Equal(t, cluster, clusterx)
+			}
+		}
+	}
+}
+
+func TestAccessClusterGetHandler(t *testing.T) {
+	{
+		service, err := cc2.GetServiceController(1)
+		require.Error(t, err)
+		require.Equal(t, nil, service)
+
+		getter, err := cc2.GetVolumeGetter(1)
+		require.Error(t, err)
+		require.Equal(t, nil, getter)
+
+		_, err = cc2.GetConfig(context.TODO(), "key")
+		require.Error(t, err)
+	}
+	{
+		service, err := cc1.GetServiceController(1)
+		require.NoError(t, err)
+		require.NotNil(t, service)
+
+		getter, err := cc1.GetVolumeGetter(1)
+		require.NoError(t, err)
+		require.NotNil(t, getter)
+
+		_, err = cc1.GetConfig(context.TODO(), "key")
+		require.Error(t, err)
+	}
+}
+
+func TestAccessClusterChangeChooseAlg(t *testing.T) {
+	cases := []struct {
+		alg controller.AlgChoose
+		err error
+	}{
+		{0, controller.ErrInvalidChooseAlg},
+		{controller.AlgAvailable, nil},
+		{controller.AlgRoundRobin, nil},
+		{controller.AlgRandom, nil},
+		{1024, controller.ErrInvalidChooseAlg},
+	}
+	for _, cs := range cases {
+		err := cc.ChangeChooseAlg(cs.alg)
+		require.Equal(t, err, cs.err)
+	}
+}
+
+func TestAccessClusterChooseBalance(t *testing.T) {
+	cc := cc19
+	for _, alg := range []controller.AlgChoose{
+		controller.AlgAvailable,
+		controller.AlgRoundRobin,
+		controller.AlgRandom,
+	} {
+		err := cc.ChangeChooseAlg(alg)
+		require.NoError(t, err)
+
+		m := make(map[proto.ClusterID]int, 2)
+		for range [10000]struct{}{} {
+			cluster, err := cc.ChooseOne()
+			require.NoError(t, err)
+			m[cluster.ClusterID]++
+		}
+
+		t.Logf("balance with algorithm %s: %+v", alg, m)
+	}
+}

blobstore/access/controller/metric.go

@@ -0,0 +1,33 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package controller
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var cacheMetric = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Namespace: "blobstore",
+		Subsystem: "access",
+		Name:      "cache_hit_rate",
+		Help:      "cache hit rate",
+	},
+	[]string{"cluster", "service", "status"},
+)
+
+func init() {
+	prometheus.MustRegister(cacheMetric)
+}

blobstore/access/controller/mock_test.go

@@ -0,0 +1,157 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package controller_test
+
+import (
+	"context"
+	"math/rand"
+	"testing"
+	"time"
+
+	"github.com/alicebob/miniredis/v2"
+	"github.com/golang/mock/gomock"
+
+	bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
+	cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/redis"
+	"github.com/cubefs/cubefs/blobstore/testing/mocks"
+	"github.com/cubefs/cubefs/blobstore/util/errors"
+	"github.com/cubefs/cubefs/blobstore/util/log"
+)
+
+const (
+	idc    = "test-idc"
+	region = "test_region"
+)
+
+var (
+	vid404      = proto.Vid(404)
+	errNotFound = errors.New("not found")
+
+	redismr  *miniredis.Miniredis
+	rediscli *redis.ClusterClient
+	cmcli    cmapi.APIAccess
+
+	dataCalled  map[proto.Vid]int
+	dataNodes   map[string]cmapi.ServiceInfo
+	dataVolumes map[proto.Vid]cmapi.VolumeInfo
+	dataDisks   map[proto.DiskID]bnapi.DiskInfo
+)
+
+func init() {
+	rand.Seed(int64(time.Now().Nanosecond()))
+	log.SetOutputLevel(log.Lfatal)
+
+	dataCalled = map[proto.Vid]int{1: 0, 9: 0}
+
+	dataVolumes = make(map[proto.Vid]cmapi.VolumeInfo, 4)
+	dataVolumes[proto.Vid(1)] = cmapi.VolumeInfo{
+		VolumeInfoBase: cmapi.VolumeInfoBase{
+			Vid:      1,
+			CodeMode: codemode.EC6P10L2,
+		},
+		Units: []cmapi.Unit{
+			{Vuid: 1011, DiskID: 1021, Host: "1031"},
+			{Vuid: 1012, DiskID: 1022, Host: "1032"},
+		},
+	}
+	dataVolumes[proto.Vid(9)] = cmapi.VolumeInfo{
+		VolumeInfoBase: cmapi.VolumeInfoBase{
+			Vid:      9,
+			CodeMode: codemode.EC16P20L2,
+		},
+		Units: []cmapi.Unit{
+			{Vuid: 9011, DiskID: 9021, Host: "9031"},
+			{Vuid: 9012, DiskID: 9022, Host: "9032"},
+		},
+	}
+	dataVolumes[vid404] = cmapi.VolumeInfo{VolumeInfoBase: cmapi.VolumeInfoBase{Vid: vid404}}
+
+	dataNodes = make(map[string]cmapi.ServiceInfo)
+	dataNodes[proto.ServiceNameProxy] = cmapi.ServiceInfo{
+		Nodes: []cmapi.ServiceNode{
+			{
+				ClusterID: 1,
+				Name:      proto.ServiceNameProxy,
+				Host:      "proxy-1",
+				Idc:       idc,
+			},
+			{
+				ClusterID: 1,
+				Name:      proto.ServiceNameProxy,
+				Host:      "proxy-2",
+				Idc:       idc,
+			},
+		},
+	}
+
+	dataDisks = make(map[proto.DiskID]bnapi.DiskInfo)
+	dataDisks[10001] = bnapi.DiskInfo{
+		ClusterID: 1,
+		Idc:       idc,
+		Host:      "blobnode-1",
+		DiskHeartBeatInfo: bnapi.DiskHeartBeatInfo{
+			DiskID: 10001,
+		},
+	}
+	dataDisks[10002] = bnapi.DiskInfo{
+		ClusterID: 1,
+		Idc:       idc,
+		Host:      "blobnode-2",
+		DiskHeartBeatInfo: bnapi.DiskHeartBeatInfo{
+			DiskID: 10002,
+		},
+	}
+
+	redismr, _ = miniredis.Run()
+	rediscli = redis.NewClusterClient(&redis.ClusterConfig{
+		Addrs: []string{redismr.Addr()},
+	})
+
+	ctr := gomock.NewController(&testing.T{})
+	cli := mocks.NewMockClientAPI(ctr)
+	cli.EXPECT().GetConfig(gomock.Any(), gomock.Any()).AnyTimes().Return("abc", nil)
+	cli.EXPECT().GetService(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
+		func(ctx context.Context, args cmapi.GetServiceArgs) (cmapi.ServiceInfo, error) {
+			if val, ok := dataNodes[args.Name]; ok {
+				return val, nil
+			}
+			return cmapi.ServiceInfo{}, errNotFound
+		})
+	cli.EXPECT().GetVolumeInfo(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
+		func(ctx context.Context, args *cmapi.GetVolumeArgs) (*cmapi.VolumeInfo, error) {
+			if val, ok := dataVolumes[args.Vid]; ok {
+				dataCalled[args.Vid]++
+				if args.Vid == vid404 {
+					return nil, errcode.ErrVolumeNotExist
+				}
+				return &val, nil
+			}
+			return nil, errNotFound
+		})
+	cli.EXPECT().DiskInfo(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
+		func(ctx context.Context, id proto.DiskID) (*bnapi.DiskInfo, error) {
+			if val, ok := dataDisks[id]; ok {
+				return &val, nil
+			}
+			return nil, errNotFound
+		})
+	cmcli = cli
+
+	initCluster()
+}

blobstore/access/controller/service.go

@@ -0,0 +1,335 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package controller
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"golang.org/x/sync/singleflight"
+
+	"github.com/cubefs/cubefs/blobstore/api/blobnode"
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/util/defaulter"
+	"github.com/cubefs/cubefs/blobstore/util/errors"
+	"github.com/cubefs/cubefs/blobstore/util/log"
+)
+
+const (
+	_diskHostServicePrefix = "diskhost"
+
+	// default service punish check valid interval
+	defaultServicePinishValidIntervalS int = 30
+	// default service punish check threshold
+	defaultServicePinishThreshold uint32 = 3
+)
+
+// HostIDC item of host with idc
+type HostIDC struct {
+	Host     string
+	IDC      string
+	Punished bool
+}
+
+// ServiceController support for both data node discovery and normal service discovery
+type ServiceController interface {
+	// GetServiceHost return an available service host
+	GetServiceHost(ctx context.Context, name string) (host string, err error)
+	// GetServiceHosts return all available service random sorted hosts
+	GetServiceHosts(ctx context.Context, name string) (hosts []string, err error)
+	// GetDiskHost return an disk's related data node host
+	GetDiskHost(ctx context.Context, diskID proto.DiskID) (hostIDC *HostIDC, err error)
+	// PunishService will punish an service host for an punishTimeSec interval
+	PunishService(ctx context.Context, service, host string, punishTimeSec int)
+	// PunishServiceWithThreshold will punish an service host for
+	// an punishTimeSec interval if service failed times satisfied with threshold during some interval time
+	PunishServiceWithThreshold(ctx context.Context, service, host string, punishTimeSec int)
+	// PunishDisk will punish a disk host for an punishTimeSec interval
+	PunishDisk(ctx context.Context, diskID proto.DiskID, punishTimeSec int)
+	// PunishDiskWithThreshold will punish a disk host for
+	// an punishTimeSec interval if disk host failed times satisfied with threshold
+	PunishDiskWithThreshold(ctx context.Context, diskID proto.DiskID, punishTimeSec int)
+}
+
+type (
+	serviceList []*hostItem
+	serviceMap  map[string]*atomic.Value
+)
+
+// hostItem represent a service or host item info
+type hostItem struct {
+	host string
+	idc  string
+
+	// punish time record the punish end time unix of host item
+	punishTimeUnix int64
+	// modify time record the last modify time unix of host item
+	lastModifyTime int64
+	// failedTimes record the service host failed times during some interval
+	failedTimes uint32
+}
+
+func (h *hostItem) isPunish() bool {
+	return time.Since(time.Unix(atomic.LoadInt64(&h.punishTimeUnix), 0)) < 0
+}
+
+// ServiceConfig service config
+type ServiceConfig struct {
+	ClusterID                   proto.ClusterID
+	IDC                         string
+	ReloadSec                   int
+	ServicePunishThreshold      uint32
+	ServicePunishValidIntervalS int
+}
+
+type serviceControllerImpl struct {
+	// allServices hold all disk/service host map, use for quickly find out
+	allServices  sync.Map
+	serviceHosts serviceMap
+
+	group        singleflight.Group
+	serviceLocks map[string]*sync.RWMutex
+	cmClient     clustermgr.APIAccess
+
+	config ServiceConfig
+}
+
+// NewServiceController returns a service controller
+func NewServiceController(cfg ServiceConfig, cmCli clustermgr.APIAccess) (ServiceController, error) {
+	defaulter.Equal(&cfg.ServicePunishThreshold, defaultServicePinishThreshold)
+	defaulter.LessOrEqual(&cfg.ServicePunishValidIntervalS, defaultServicePinishValidIntervalS)
+
+	controller := &serviceControllerImpl{
+		serviceHosts: serviceMap{
+			proto.ServiceNameProxy: &atomic.Value{},
+		},
+		cmClient: cmCli,
+		serviceLocks: map[string]*sync.RWMutex{
+			proto.ServiceNameProxy: {},
+		},
+		config: cfg,
+	}
+
+	err := controller.load(cfg.ClusterID, cfg.IDC)
+	if err != nil {
+		return nil, errors.Base(err, "load service failed")
+	}
+
+	if cfg.ReloadSec <= 0 {
+		cfg.ReloadSec = 10
+	}
+	tick := time.NewTicker(time.Duration(cfg.ReloadSec) * time.Second)
+	go func() {
+		defer tick.Stop()
+		for range tick.C {
+			if err := controller.load(cfg.ClusterID, cfg.IDC); err != nil {
+				log.Warn("load timer error", err)
+			}
+		}
+	}()
+	return controller, nil
+}
+
+// load initial all service and service hosts
+func (s *serviceControllerImpl) load(cid proto.ClusterID, idc string) error {
+	span, ctx := trace.StartSpanFromContext(context.Background(), "access_cluster_service")
+	span.Debug("service loader for cluster:", cid)
+
+	serviceName := proto.ServiceNameProxy
+	service, err := s.cmClient.GetService(ctx, clustermgr.GetServiceArgs{Name: serviceName})
+	if err != nil {
+		span.Warn("get service from cluster manager failed", err)
+		return err
+	}
+
+	span.Debugf("found %d server nodes of %s in the cluster", len(service.Nodes), serviceName)
+	hostItems := make(serviceList, 0, len(service.Nodes))
+	for _, node := range service.Nodes {
+		if node.Idc != idc {
+			continue
+		}
+		hostItems = append(hostItems, &hostItem{idc: node.Idc, host: node.Host})
+	}
+	if len(hostItems) > 0 {
+		for _, item := range hostItems {
+			s.allServices.Store(serviceName+item.host, item)
+			span.Debugf("store node %+v", item)
+		}
+		s.serviceHosts[serviceName].Store(hostItems)
+	}
+	return nil
+}
+
+// GetServiceHost return an available service host
+func (s *serviceControllerImpl) GetServiceHost(ctx context.Context, name string) (host string, err error) {
+	serviceList, ok := s.serviceHosts[name].Load().(serviceList)
+	if !ok {
+		return "", errors.Newf("not found host of %s", name)
+	}
+
+	lock := s.getServiceLock(name)
+	idx := 0
+
+RETRY:
+
+	lock.RLock()
+	length := len(serviceList)
+	if length == 0 {
+		lock.RUnlock()
+		return "", errors.Newf("no any host of %s", name)
+	}
+	idx = rand.Intn(length)
+
+	item := serviceList[idx]
+	lock.RUnlock()
+
+	if !item.isPunish() {
+		return item.host, nil
+	}
+
+	lock.Lock()
+	// double check
+	v := serviceList[idx]
+	// if serviceList[idx] still equal to item, then remove it
+	if v == item {
+		serviceList = append(serviceList[:idx], serviceList[idx+1:]...)
+	}
+	s.serviceHosts[name].Store(serviceList)
+	lock.Unlock()
+
+	goto RETRY
+}
+
+// GetServiceHosts return all available random-sorted hosts of service
+func (s *serviceControllerImpl) GetServiceHosts(ctx context.Context, name string) (hosts []string, err error) {
+	serviceList, ok := s.serviceHosts[name].Load().(serviceList)
+	if !ok {
+		return nil, errors.Newf("not found host of %s", name)
+	}
+
+	lock := s.getServiceLock(name)
+
+	lock.RLock()
+	length := len(serviceList)
+	if length == 0 {
+		lock.RUnlock()
+		return nil, errors.Newf("no any host of %s", name)
+	}
+
+	hosts = make([]string, 0, length)
+	for _, item := range serviceList {
+		if !item.isPunish() {
+			hosts = append(hosts, item.host)
+		}
+	}
+	lock.RUnlock()
+
+	if len(hosts) == 0 {
+		return nil, errors.Newf("no available host of %s", name)
+	}
+
+	rand.Shuffle(len(hosts), func(i, j int) {
+		hosts[i], hosts[j] = hosts[j], hosts[i]
+	})
+	return hosts, nil
+}
+
+// GetDiskHost return an disk's related data node host
+func (s *serviceControllerImpl) GetDiskHost(ctx context.Context, diskID proto.DiskID) (*HostIDC, error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	v, ok := s.allServices.Load(_diskHostServicePrefix + (diskID.ToString()))
+	if ok {
+		item := v.(*hostItem)
+		return &HostIDC{
+			Host:     item.host,
+			IDC:      item.idc,
+			Punished: item.isPunish(),
+		}, nil
+	}
+	ret, err, _ := s.group.Do("get-diskinfo-"+diskID.ToString(), func() (interface{}, error) {
+		diskInfo, err := s.cmClient.DiskInfo(ctx, diskID)
+		if err != nil {
+			return nil, err
+		}
+		return diskInfo, nil
+	})
+	if err != nil {
+		span.Error("can't get disk host from clustermgr", err)
+		return nil, errors.Base(err, "get disk info", diskID)
+	}
+	diskInfo := ret.(*blobnode.DiskInfo)
+
+	item := &hostItem{host: diskInfo.Host, idc: diskInfo.Idc}
+	s.allServices.Store(_diskHostServicePrefix+(diskInfo.DiskID.ToString()), item)
+	return &HostIDC{
+		Host:     item.host,
+		IDC:      item.idc,
+		Punished: item.isPunish(),
+	}, nil
+}
+
+// PunishService will punish an service host for an punishTimeSec interval
+func (s *serviceControllerImpl) PunishService(ctx context.Context, service, host string, punishTimeSec int) {
+	v, ok := s.allServices.Load(service + host)
+	if !ok {
+		panic(fmt.Sprintf("can't find host in all services map, %s-%s", service, host))
+	}
+	item := v.(*hostItem)
+
+	// atomic set item's punish time unix
+	atomic.StoreInt64(&item.punishTimeUnix, time.Now().Add(time.Duration(punishTimeSec)*time.Second).Unix())
+}
+
+// PunishDisk will punish a disk host for an punishTimeSec interval
+func (s *serviceControllerImpl) PunishDisk(ctx context.Context, diskID proto.DiskID, punishTimeSec int) {
+	s.PunishService(ctx, _diskHostServicePrefix, diskID.ToString(), punishTimeSec)
+}
+
+// PunishDiskWithThreshold will punish a disk host for
+// an punishTimeSec interval if disk host failed times satisfied with threshold
+func (s *serviceControllerImpl) PunishDiskWithThreshold(ctx context.Context, diskID proto.DiskID, punishTimeSec int) {
+	s.PunishServiceWithThreshold(ctx, _diskHostServicePrefix, diskID.ToString(), punishTimeSec)
+}
+
+// PunishServiceWithThreshold will punish an service host for
+// an punishTimeSec interval if service failed times satisfied with threshold
+func (s *serviceControllerImpl) PunishServiceWithThreshold(ctx context.Context, service, host string, punishTimeSec int) {
+	v, ok := s.allServices.Load(service + host)
+	if !ok {
+		panic(fmt.Sprintf("can't can host in all services map, %s-%s", service, host))
+	}
+	item := v.(*hostItem)
+	new := atomic.AddUint32(&item.failedTimes, 1)
+	// failedTimes larger than threshold, then check the lastModifyTime
+	if new >= s.config.ServicePunishThreshold {
+		if time.Since(time.Unix(atomic.LoadInt64(&item.lastModifyTime), 0)) < time.Duration(s.config.ServicePunishValidIntervalS)*time.Second {
+			s.PunishService(ctx, service, host, punishTimeSec)
+			return
+		}
+		atomic.AddUint32(&item.failedTimes, -(new - 1))
+	}
+	atomic.StoreInt64(&item.lastModifyTime, time.Now().Unix())
+}
+
+func (s *serviceControllerImpl) getServiceLock(name string) *sync.RWMutex {
+	return s.serviceLocks[name]
+}

blobstore/access/controller/service_test.go

@@ -0,0 +1,243 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package controller_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/access/controller"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+)
+
+var serviceName = proto.ServiceNameProxy
+
+type hostSet map[string]struct{}
+
+func (set hostSet) Keys() []string {
+	keys := make([]string, 0, len(set))
+	for k := range set {
+		keys = append(keys, k)
+	}
+	return keys
+}
+
+var _, serviceCtx = trace.StartSpanFromContext(context.Background(), "TestAccessService")
+
+func TestAccessServiceNew(t *testing.T) {
+	{
+		sc, err := controller.NewServiceController(
+			controller.ServiceConfig{
+				ClusterID: 0,
+				IDC:       idc,
+			}, cmcli)
+		require.NoError(t, err)
+
+		_, err = sc.GetServiceHost(serviceCtx, serviceName)
+		require.NoError(t, err)
+	}
+	{
+		sc, err := controller.NewServiceController(
+			controller.ServiceConfig{
+				ClusterID: 0,
+				IDC:       idc + "x",
+				ReloadSec: 1,
+			}, cmcli)
+		require.NoError(t, err)
+
+		_, err = sc.GetServiceHost(serviceCtx, serviceName)
+		require.Error(t, err)
+
+		time.Sleep(time.Second * 2)
+	}
+}
+
+func TestAccessServiceGetServiceHost(t *testing.T) {
+	sc, err := controller.NewServiceController(
+		controller.ServiceConfig{
+			ClusterID: 0,
+			IDC:       idc,
+			ReloadSec: 1,
+		}, cmcli)
+	require.NoError(t, err)
+
+	keys := make(hostSet)
+	for ii := 0; ii < 100; ii++ {
+		host, err := sc.GetServiceHost(serviceCtx, serviceName)
+		require.NoError(t, err)
+		keys[host] = struct{}{}
+	}
+	require.ElementsMatch(t, keys.Keys(), []string{"proxy-1", "proxy-2"})
+
+	hosts, err := sc.GetServiceHosts(serviceCtx, serviceName)
+	require.NoError(t, err)
+	require.ElementsMatch(t, []string{"proxy-1", "proxy-2"}, hosts)
+}
+
+func TestAccessServicePunishService(t *testing.T) {
+	sc, err := controller.NewServiceController(
+		controller.ServiceConfig{
+			ClusterID: 0,
+			IDC:       idc,
+			ReloadSec: 1,
+		}, cmcli)
+	require.NoError(t, err)
+
+	{
+		keys := make(hostSet)
+		for ii := 0; ii < 100; ii++ {
+			host, err := sc.GetServiceHost(serviceCtx, serviceName)
+			require.NoError(t, err)
+			keys[host] = struct{}{}
+		}
+		require.ElementsMatch(t, keys.Keys(), []string{"proxy-1", "proxy-2"})
+	}
+
+	sc.PunishService(serviceCtx, serviceName, "proxy-2", 2)
+	for ii := 0; ii < 100; ii++ {
+		host, err := sc.GetServiceHost(serviceCtx, serviceName)
+		require.NoError(t, err)
+		require.True(t, host == "proxy-1")
+	}
+
+	time.Sleep(time.Second * 2)
+	{
+		keys := make(hostSet)
+		for ii := 0; ii < 100; ii++ {
+			host, err := sc.GetServiceHost(serviceCtx, serviceName)
+			require.NoError(t, err)
+			keys[host] = struct{}{}
+		}
+		require.ElementsMatch(t, keys.Keys(), []string{"proxy-1", "proxy-2"})
+	}
+}
+
+func TestAccessServicePunishServiceWithThreshold(t *testing.T) {
+	threshold := uint32(5)
+	sc, err := controller.NewServiceController(
+		controller.ServiceConfig{
+			ClusterID:                   0,
+			IDC:                         idc,
+			ReloadSec:                   1,
+			ServicePunishThreshold:      threshold,
+			ServicePunishValidIntervalS: 2,
+		}, cmcli)
+	require.NoError(t, err)
+
+	{
+		keys := make(hostSet)
+		for ii := 0; ii < 100; ii++ {
+			host, err := sc.GetServiceHost(serviceCtx, serviceName)
+			require.NoError(t, err)
+			keys[host] = struct{}{}
+		}
+		require.ElementsMatch(t, keys.Keys(), []string{"proxy-1", "proxy-2"})
+	}
+
+	// not punish
+	for ii := threshold; ii > 1; ii-- {
+		sc.PunishServiceWithThreshold(serviceCtx, serviceName, "proxy-1", 2)
+	}
+	{
+		keys := make(hostSet)
+		for ii := 0; ii < 100; ii++ {
+			host, err := sc.GetServiceHost(serviceCtx, serviceName)
+			require.NoError(t, err)
+			keys[host] = struct{}{}
+		}
+		require.ElementsMatch(t, keys.Keys(), []string{"proxy-1", "proxy-2"})
+	}
+
+	// punished
+	sc.PunishServiceWithThreshold(serviceCtx, serviceName, "proxy-1", 2)
+	for ii := 0; ii < 100; ii++ {
+		host, err := sc.GetServiceHost(serviceCtx, serviceName)
+		require.NoError(t, err)
+		require.True(t, host == "proxy-2")
+	}
+
+	time.Sleep(time.Second * 2)
+	{
+		keys := make(hostSet)
+		for ii := 0; ii < 100; ii++ {
+			host, err := sc.GetServiceHost(serviceCtx, serviceName)
+			require.NoError(t, err)
+			keys[host] = struct{}{}
+		}
+		require.ElementsMatch(t, keys.Keys(), []string{"proxy-1", "proxy-2"})
+	}
+}
+
+func TestAccessServiceGetDiskHost(t *testing.T) {
+	sc, err := controller.NewServiceController(
+		controller.ServiceConfig{
+			ClusterID: 0,
+			IDC:       idc,
+			ReloadSec: 1,
+		}, cmcli)
+	require.NoError(t, err)
+
+	{
+		host, err := sc.GetDiskHost(serviceCtx, proto.DiskID(10001))
+		require.NoError(t, err)
+		require.True(t, host.Host == "blobnode-1")
+	}
+	{
+		_, err := sc.GetDiskHost(serviceCtx, proto.DiskID(10000))
+		require.Error(t, err)
+	}
+}
+
+func TestAccessServicePunishDisk(t *testing.T) {
+	sc, err := controller.NewServiceController(
+		controller.ServiceConfig{
+			ClusterID: 0,
+			IDC:       idc,
+			ReloadSec: 1,
+		}, cmcli)
+	require.NoError(t, err)
+
+	{
+		host, err := sc.GetDiskHost(serviceCtx, proto.DiskID(10001))
+		require.NoError(t, err)
+		require.True(t, host.Host == "blobnode-1")
+		require.False(t, host.Punished)
+	}
+	sc.PunishDisk(serviceCtx, 10001, 1)
+	{
+		host, err := sc.GetDiskHost(serviceCtx, proto.DiskID(10001))
+		require.NoError(t, err)
+		require.True(t, host.Host == "blobnode-1")
+		require.True(t, host.Punished)
+	}
+	time.Sleep(time.Second)
+	{
+		host, err := sc.GetDiskHost(serviceCtx, proto.DiskID(10001))
+		require.NoError(t, err)
+		require.True(t, host.Host == "blobnode-1")
+		require.False(t, host.Punished)
+	}
+	sc.PunishDiskWithThreshold(serviceCtx, 10001, 1)
+	{
+		host, err := sc.GetDiskHost(serviceCtx, proto.DiskID(10001))
+		require.NoError(t, err)
+		require.True(t, host.Host == "blobnode-1")
+		require.False(t, host.Punished)
+	}
+}

blobstore/access/controller/volume.go

@@ -0,0 +1,326 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package controller
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"time"
+
+	"golang.org/x/sync/singleflight"
+
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/memcache"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/redis"
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/util/errors"
+	"github.com/cubefs/cubefs/blobstore/util/retry"
+)
+
+const (
+	_defaultCacheSize       = 1 << 17
+	_defaultCacheExpiration = int64(2 * time.Minute)
+)
+
+// Unit alias of clustermgr.Unit
+type Unit = clustermgr.Unit
+
+// VolumePhy volume physical info
+//     Vid, CodeMode and Units are from cluster mgr
+//     IsPunish is cached in memory
+//     Timestamp is cached in redis to clear outdate info
+type VolumePhy struct {
+	Vid       proto.Vid         `json:"vid"`
+	CodeMode  codemode.CodeMode `json:"codemode"`
+	IsPunish  bool              `json:"-"`
+	Timestamp int64             `json:"timestamp"`
+	Units     []Unit            `json:"units"`
+}
+
+// VolumeGetter getter of volume physical location
+//
+// ctx: context with trace or something
+//
+// isCache: is false means reading from cluster then updating redis and memcache
+//     otherwise reading from memcache -> redis -> cluster
+type VolumeGetter interface {
+	// Get returns volume physical location of vid
+	Get(ctx context.Context, vid proto.Vid, isCache bool) *VolumePhy
+	// Punish punish vid with interval seconds
+	Punish(ctx context.Context, vid proto.Vid, punishIntervalS int)
+}
+type cvid uint64
+
+var (
+	fmtKeyGroupVolume = func(id cvid) string { return fmt.Sprintf("get-volume-%d", id) }
+	fmtKeyRedisVolume = func(cid proto.ClusterID, vid proto.Vid) string {
+		return fmt.Sprintf("access/volume/%d/%d", cid, vid)
+	}
+)
+
+func addCVid(cid proto.ClusterID, vid proto.Vid) cvid {
+	return cvid((uint64(cid) << 32) | uint64(vid))
+}
+
+// volumePhyCacher local k-v cache for (Vid, VolumePhy)
+type volumePhyCacher interface {
+	Get(key cvid) *VolumePhy
+	Set(key cvid, value VolumePhy)
+}
+
+type volumeMemCache struct {
+	cache *memcache.MemCache
+}
+
+var _ volumePhyCacher = (*volumeMemCache)(nil)
+
+// Get implements volumePhyCacher.Get
+func (vc *volumeMemCache) Get(key cvid) *VolumePhy {
+	value := vc.cache.Get(key)
+	if value == nil {
+		return nil
+	}
+
+	phy, ok := value.(VolumePhy)
+	if !ok {
+		return nil
+	}
+
+	return &phy
+}
+
+// Set implements volumePhyCacher.Set
+func (vc *volumeMemCache) Set(key cvid, value VolumePhy) {
+	vc.cache.Set(key, value)
+}
+
+type volumeGetterImpl struct {
+	ctx context.Context
+	cid proto.ClusterID
+
+	volumeMemCache volumePhyCacher
+	memExpiration  int64
+	punishCache    *memcache.MemCache
+
+	redisClient *redis.ClusterClient
+	cmClient    clustermgr.APIAccess
+
+	singleRun *singleflight.Group
+}
+
+// NewVolumeGetter new a volume getter
+//   memExpiration expiration of memcache, 0 means no expiration
+func NewVolumeGetter(clusterID proto.ClusterID, cmCli clustermgr.APIAccess, redisCli *redis.ClusterClient,
+	memExpiration time.Duration) (VolumeGetter, error) {
+	_, ctx := trace.StartSpanFromContext(context.Background(), "")
+
+	expiration := int64(memExpiration)
+	if expiration < 0 {
+		expiration = _defaultCacheExpiration
+	}
+
+	mc, err := memcache.NewMemCache(ctx, _defaultCacheSize)
+	if err != nil {
+		return nil, err
+	}
+	punishCache, err := memcache.NewMemCache(ctx, 1024)
+	if err != nil {
+		return nil, err
+	}
+
+	getter := &volumeGetterImpl{
+		ctx:            ctx,
+		cid:            clusterID,
+		volumeMemCache: &volumeMemCache{cache: mc},
+		memExpiration:  expiration,
+		punishCache:    punishCache,
+		redisClient:    redisCli,
+		cmClient:       cmCli,
+		singleRun:      new(singleflight.Group),
+	}
+
+	return getter, nil
+}
+
+// Get implements interface VolumeGetter
+//     1.top level cache from local
+//     2.second level cache from redis cluster
+func (v *volumeGetterImpl) Get(ctx context.Context, vid proto.Vid, isCache bool) (phy *VolumePhy) {
+	span := trace.SpanFromContextSafe(ctx)
+	cid := v.cid.ToString()
+	id := addCVid(v.cid, vid)
+
+	// check if volume punish
+	defer func() {
+		if phy == nil {
+			return
+		}
+
+		val := v.punishCache.Get(id)
+		if val == nil {
+			return
+		}
+
+		if time.Since(time.Unix(val.(int64), 0)) < 0 {
+			phy.IsPunish = true
+		} else {
+			v.punishCache.Set(id, nil)
+		}
+	}()
+
+	// if isCache, reading from memcache -> redis -> cluster
+	if isCache {
+		if phy = v.getFromLocalCache(ctx, id); phy != nil {
+			if v.memExpiration == 0 {
+				if phy.Timestamp < 0 {
+					phy = nil
+				}
+				cacheMetric.WithLabelValues(cid, "memcache", "hit").Inc()
+				return
+			}
+
+			now := time.Now().UnixNano()
+			// exist volume and not expired
+			if phy.Timestamp > 0 && now < phy.Timestamp+v.memExpiration {
+				span.Debug("got from memcache volume", cid, vid)
+				cacheMetric.WithLabelValues(cid, "memcache", "hit").Inc()
+				return
+			}
+
+			// not exist volume and not expired, return nil
+			if phy.Timestamp < 0 && now < -phy.Timestamp+v.memExpiration {
+				span.Debug("got from memcache not exist volume", cid, vid)
+				cacheMetric.WithLabelValues(cid, "memcache", "hit").Inc()
+				phy = nil
+				return
+			}
+
+			cacheMetric.WithLabelValues(cid, "memcache", "expired").Inc()
+		}
+		cacheMetric.WithLabelValues(cid, "memcache", "miss").Inc()
+
+		if phy = v.getFromRedis(ctx, fmtKeyRedisVolume(v.cid, vid)); phy != nil {
+			span.Debug("got from redis volume", cid, vid)
+			cacheMetric.WithLabelValues(cid, "redis", "hit").Inc()
+
+			phy.Timestamp = time.Now().UnixNano()
+			v.setToLocalCache(ctx, id, phy)
+			return
+		}
+		cacheMetric.WithLabelValues(cid, "redis", "miss").Inc()
+	}
+
+	val, err, _ := v.singleRun.Do(fmtKeyGroupVolume(id), func() (interface{}, error) {
+		return v.getFromClusterAndUpdate(ctx, vid)
+	})
+	if err != nil {
+		cacheMetric.WithLabelValues(cid, "clustermgr", "miss").Inc()
+		span.Error("get volume location from clustermgr failed", errors.Detail(err))
+		return
+	}
+	cacheMetric.WithLabelValues(cid, "clustermgr", "hit").Inc()
+
+	phy, ok := val.(*VolumePhy)
+	if !ok {
+		span.Errorf("%s has bad value in singleflight group %+v", fmtKeyGroupVolume(id), val)
+		return
+	}
+	span.Debug("got from cluster volume", cid, vid)
+	return
+}
+
+func (v *volumeGetterImpl) Punish(ctx context.Context, vid proto.Vid, punishIntervalS int) {
+	v.punishCache.Set(addCVid(v.cid, vid),
+		time.Now().Add(time.Duration(punishIntervalS)*time.Second).Unix())
+}
+
+func (v *volumeGetterImpl) setToLocalCache(ctx context.Context, id cvid, phy *VolumePhy) error {
+	v.volumeMemCache.Set(id, *phy)
+	return nil
+}
+
+func (v *volumeGetterImpl) getFromLocalCache(ctx context.Context, id cvid) *VolumePhy {
+	return v.volumeMemCache.Get(id)
+}
+
+// expiration random 30 - 60 minutes
+func (v *volumeGetterImpl) setToRedis(ctx context.Context, key string, phy *VolumePhy) error {
+	if v.redisClient == nil {
+		return nil
+	}
+	expiration := rand.Intn(30) + 30
+	return v.redisClient.Set(ctx, key, phy, time.Minute*time.Duration(expiration))
+}
+
+func (v *volumeGetterImpl) getFromRedis(ctx context.Context, key string) *VolumePhy {
+	if v.redisClient == nil {
+		return nil
+	}
+	value := &VolumePhy{}
+	if err := v.redisClient.Get(ctx, key, value); err != nil {
+		span := trace.SpanFromContextSafe(ctx)
+		span.Info(key, err)
+		return nil
+	}
+	return value
+}
+
+func (v *volumeGetterImpl) getFromClusterAndUpdate(ctx context.Context, vid proto.Vid) (*VolumePhy, error) {
+	span := trace.SpanFromContextSafe(ctx)
+	var (
+		vInfo *clustermgr.VolumeInfo
+		err   error
+	)
+
+	id := addCVid(v.cid, vid)
+	if err = retry.ExponentialBackoff(3, 100).On(func() error {
+		if vInfo, err = v.cmClient.GetVolumeInfo(ctx, &clustermgr.GetVolumeArgs{Vid: vid}); err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		// set local cache as nil(Timestamp is negative) when volume not exist
+		if rpc.DetectStatusCode(err) == errcode.CodeVolumeNotExist {
+			phy := &VolumePhy{
+				Vid:       vid,
+				Timestamp: -time.Now().UnixNano(),
+			}
+			span.Infof("to update memcache on not exist volume(%d-%d) %+v", v.cid, vid, phy)
+			v.setToLocalCache(ctx, id, phy)
+		}
+		return nil, errors.Base(err, "get volume from clustermgr", v.cid, vid)
+	}
+
+	phy := &VolumePhy{
+		Vid:       vInfo.Vid,
+		CodeMode:  vInfo.CodeMode,
+		Timestamp: time.Now().UnixNano(),
+		Units:     make([]Unit, len(vInfo.Units)),
+	}
+	copy(phy.Units, vInfo.Units[:])
+
+	span.Debugf("to update memcache and redis volume(%d-%d) %+v", v.cid, vid, phy)
+	v.setToLocalCache(ctx, id, phy)
+	if err := v.setToRedis(ctx, fmtKeyRedisVolume(v.cid, vid), phy); err != nil {
+		span.Warn("set volume location into redis failed", err)
+	}
+
+	return phy, nil
+}

blobstore/access/controller/volume_test.go

@@ -0,0 +1,239 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package controller_test
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/access/controller"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/redis"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+)
+
+var fmtKeyRedisVolume = func(cid proto.ClusterID, vid proto.Vid) string {
+	return fmt.Sprintf("access/volume/%d/%d", cid, vid)
+}
+
+func TestAccessVolumeGetterNew(t *testing.T) {
+	_, ctx := trace.StartSpanFromContext(context.Background(), "TestAccessVolumeGetterNew")
+
+	getter, err := controller.NewVolumeGetter(1, cmcli, rediscli, time.Millisecond*200)
+	require.Nil(t, err)
+	info := getter.Get(ctx, proto.Vid(0), true)
+	require.Nil(t, info)
+
+	id := proto.Vid(1)
+	dataCalled[id] = 0
+	info = getter.Get(ctx, id, true)
+	require.NotNil(t, info)
+	require.Equal(t, 1, dataCalled[id])
+	require.Equal(t, proto.Vuid(1011), info.Units[0].Vuid)
+
+	info = getter.Get(ctx, id, true)
+	require.Equal(t, 1, dataCalled[id])
+	require.Equal(t, proto.Vuid(1012), info.Units[1].Vuid)
+
+	// delete redis cache
+	rediscli.Del(context.TODO(), fmtKeyRedisVolume(1, id))
+	getter.Get(ctx, id, true)
+	require.Equal(t, 1, dataCalled[id])
+
+	time.Sleep(time.Millisecond * 250)
+	getter.Get(ctx, id, true)
+	require.Equal(t, 2, dataCalled[id])
+
+	getter.Get(ctx, id, false)
+	require.Equal(t, 3, dataCalled[id])
+
+	rediscli.Del(context.TODO(), fmtKeyRedisVolume(1, id))
+	id = proto.Vid(9)
+	info = getter.Get(ctx, id, true)
+	require.NotNil(t, info)
+	require.Equal(t, 1, dataCalled[id])
+
+	time.Sleep(time.Millisecond * 250)
+	info = getter.Get(ctx, id, true)
+	require.NotNil(t, info)
+	require.Equal(t, 1, dataCalled[id])
+
+	info = getter.Get(ctx, proto.Vid(10), false)
+	require.Nil(t, info)
+}
+
+func TestAccessVolumeGetterNotExistVolume(t *testing.T) {
+	_, ctx := trace.StartSpanFromContext(context.Background(), "TestAccessVolumeGetterNotExistVolume")
+
+	getter, err := controller.NewVolumeGetter(0xfe, cmcli, rediscli, time.Millisecond*500)
+	require.Nil(t, err)
+	info := getter.Get(ctx, proto.Vid(0), true)
+	require.Nil(t, info)
+
+	id := proto.Vid(1)
+	dataCalled[id] = 0
+	info = getter.Get(ctx, id, true)
+	require.NotNil(t, info)
+	require.Equal(t, 1, dataCalled[id])
+	require.Equal(t, proto.Vuid(1011), info.Units[0].Vuid)
+
+	id = vid404
+	dataCalled[id] = 0
+	info = getter.Get(ctx, id, true)
+	require.Nil(t, info)
+	require.Equal(t, 3, dataCalled[id])
+	require.ErrorIs(t, redis.ErrNotFound, rediscli.Get(ctx, fmtKeyRedisVolume(0xfe, id), nil))
+
+	var wg sync.WaitGroup
+	wg.Add(10)
+	for ii := 0; ii < 10; ii++ {
+		go func() {
+			getter.Get(ctx, id, true)
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+	require.Equal(t, 3, dataCalled[id])
+
+	time.Sleep(time.Millisecond * 500)
+	getter.Get(ctx, id, true)
+	require.Equal(t, 6, dataCalled[id])
+	require.ErrorIs(t, redis.ErrNotFound, rediscli.Get(ctx, fmtKeyRedisVolume(0xfe, id), nil))
+
+	getter, err = controller.NewVolumeGetter(0xee, cmcli, rediscli, 0)
+	require.NoError(t, err)
+	id = vid404
+	dataCalled[id] = 0
+	for ii := 0; ii < 10; ii++ {
+		getter.Get(ctx, id, true)
+		require.Equal(t, 3, dataCalled[id])
+		time.Sleep(time.Millisecond * 100)
+	}
+}
+
+func TestAccessVolumeGetterExpiration(t *testing.T) {
+	_, ctx := trace.StartSpanFromContext(context.Background(), "TestAccessVolumeGetterExpiration")
+	mockRedis := redis.NewClusterClient(&redis.ClusterConfig{
+		Addrs: []string{"127.0.0.1:5678"},
+	})
+
+	getter, err := controller.NewVolumeGetter(1, cmcli, mockRedis, time.Millisecond*200)
+	require.Nil(t, err)
+
+	info := getter.Get(ctx, proto.Vid(0), true)
+	require.Nil(t, info)
+
+	id := proto.Vid(1)
+	dataCalled[id] = 0
+	info = getter.Get(ctx, id, true)
+	require.NotNil(t, info)
+	require.Equal(t, 1, dataCalled[id])
+
+	info = getter.Get(ctx, id, true)
+	require.NotNil(t, info)
+	require.Equal(t, 1, dataCalled[id])
+
+	// expiration
+	time.Sleep(time.Millisecond * 250)
+	info = getter.Get(ctx, id, true)
+	require.NotNil(t, info)
+	require.Equal(t, 2, dataCalled[id])
+
+	info = getter.Get(ctx, id, false)
+	require.NotNil(t, info)
+	require.Equal(t, 3, dataCalled[id])
+}
+
+func TestAccessVolumeGetterBrokenRedis(t *testing.T) {
+	_, ctx := trace.StartSpanFromContext(context.Background(), "TestAccessVolumeGetterBrokenRedis")
+	mockRedis := redis.NewClusterClient(&redis.ClusterConfig{
+		Addrs: []string{"127.0.0.1:5678"},
+	})
+
+	getter, err := controller.NewVolumeGetter(1, cmcli, mockRedis, -1)
+	require.Nil(t, err)
+
+	info := getter.Get(ctx, proto.Vid(0), true)
+	require.Nil(t, info)
+
+	id := proto.Vid(1)
+	dataCalled[id] = 0
+	info = getter.Get(ctx, id, true)
+	require.NotNil(t, info)
+	require.Equal(t, 1, dataCalled[id])
+	require.Equal(t, proto.Vuid(1011), info.Units[0].Vuid)
+
+	info = getter.Get(ctx, id, true)
+	require.NotNil(t, info)
+	require.Equal(t, 1, dataCalled[id])
+	require.Equal(t, proto.Vuid(1012), info.Units[1].Vuid)
+}
+
+func TestAccessVolumeGetterNoRedis(t *testing.T) {
+	_, ctx := trace.StartSpanFromContext(context.Background(), "TestAccessVolumeGetterNoRedis")
+
+	getter, err := controller.NewVolumeGetter(1, cmcli, nil, -1)
+	require.Nil(t, err)
+
+	info := getter.Get(ctx, proto.Vid(0), true)
+	require.Nil(t, info)
+
+	id := proto.Vid(1)
+	dataCalled[id] = 0
+	info = getter.Get(ctx, id, true)
+	require.NotNil(t, info)
+	require.Equal(t, 1, dataCalled[id])
+	require.Equal(t, proto.Vuid(1011), info.Units[0].Vuid)
+
+	info = getter.Get(ctx, id, true)
+	require.NotNil(t, info)
+	require.Equal(t, 1, dataCalled[id])
+	require.Equal(t, proto.Vuid(1012), info.Units[1].Vuid)
+}
+
+func TestAccessVolumePunish(t *testing.T) {
+	_, ctx := trace.StartSpanFromContext(context.Background(), "TestAccessVolumePunish")
+
+	getter, err := controller.NewVolumeGetter(1, cmcli, rediscli, 0)
+	require.Nil(t, err)
+
+	info := getter.Get(ctx, proto.Vid(0), true)
+	require.Nil(t, info)
+
+	getter.Punish(ctx, proto.Vid(0), 10)
+	info = getter.Get(ctx, proto.Vid(0), true)
+	require.Nil(t, info)
+
+	id := proto.Vid(1)
+	info = getter.Get(ctx, id, true)
+	require.NotNil(t, info)
+	require.Equal(t, proto.Vuid(1011), info.Units[0].Vuid)
+	require.False(t, info.IsPunish)
+
+	getter.Punish(ctx, id, 1)
+	info = getter.Get(ctx, id, true)
+	require.NotNil(t, info)
+	require.True(t, info.IsPunish)
+
+	time.Sleep(time.Second)
+	info = getter.Get(ctx, id, true)
+	require.NotNil(t, info)
+	require.False(t, info.IsPunish)
+}

blobstore/access/controller_mock_test.go

@@ -0,0 +1,305 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/cubefs/cubefs/blobstore/access/controller (interfaces: ClusterController,ServiceController,VolumeGetter)
+
+// Package access is a generated GoMock package.
+package access
+
+import (
+	context "context"
+	reflect "reflect"
+
+	controller "github.com/cubefs/cubefs/blobstore/access/controller"
+	clustermgr "github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	proto "github.com/cubefs/cubefs/blobstore/common/proto"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockClusterController is a mock of ClusterController interface.
+type MockClusterController struct {
+	ctrl     *gomock.Controller
+	recorder *MockClusterControllerMockRecorder
+}
+
+// MockClusterControllerMockRecorder is the mock recorder for MockClusterController.
+type MockClusterControllerMockRecorder struct {
+	mock *MockClusterController
+}
+
+// NewMockClusterController creates a new mock instance.
+func NewMockClusterController(ctrl *gomock.Controller) *MockClusterController {
+	mock := &MockClusterController{ctrl: ctrl}
+	mock.recorder = &MockClusterControllerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockClusterController) EXPECT() *MockClusterControllerMockRecorder {
+	return m.recorder
+}
+
+// All mocks base method.
+func (m *MockClusterController) All() []*clustermgr.ClusterInfo {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "All")
+	ret0, _ := ret[0].([]*clustermgr.ClusterInfo)
+	return ret0
+}
+
+// All indicates an expected call of All.
+func (mr *MockClusterControllerMockRecorder) All() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "All", reflect.TypeOf((*MockClusterController)(nil).All))
+}
+
+// ChangeChooseAlg mocks base method.
+func (m *MockClusterController) ChangeChooseAlg(arg0 controller.AlgChoose) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ChangeChooseAlg", arg0)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// ChangeChooseAlg indicates an expected call of ChangeChooseAlg.
+func (mr *MockClusterControllerMockRecorder) ChangeChooseAlg(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChangeChooseAlg", reflect.TypeOf((*MockClusterController)(nil).ChangeChooseAlg), arg0)
+}
+
+// ChooseOne mocks base method.
+func (m *MockClusterController) ChooseOne() (*clustermgr.ClusterInfo, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ChooseOne")
+	ret0, _ := ret[0].(*clustermgr.ClusterInfo)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ChooseOne indicates an expected call of ChooseOne.
+func (mr *MockClusterControllerMockRecorder) ChooseOne() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChooseOne", reflect.TypeOf((*MockClusterController)(nil).ChooseOne))
+}
+
+// GetConfig mocks base method.
+func (m *MockClusterController) GetConfig(arg0 context.Context, arg1 string) (string, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetConfig", arg0, arg1)
+	ret0, _ := ret[0].(string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetConfig indicates an expected call of GetConfig.
+func (mr *MockClusterControllerMockRecorder) GetConfig(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfig", reflect.TypeOf((*MockClusterController)(nil).GetConfig), arg0, arg1)
+}
+
+// GetServiceController mocks base method.
+func (m *MockClusterController) GetServiceController(arg0 proto.ClusterID) (controller.ServiceController, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetServiceController", arg0)
+	ret0, _ := ret[0].(controller.ServiceController)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetServiceController indicates an expected call of GetServiceController.
+func (mr *MockClusterControllerMockRecorder) GetServiceController(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetServiceController", reflect.TypeOf((*MockClusterController)(nil).GetServiceController), arg0)
+}
+
+// GetVolumeGetter mocks base method.
+func (m *MockClusterController) GetVolumeGetter(arg0 proto.ClusterID) (controller.VolumeGetter, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetVolumeGetter", arg0)
+	ret0, _ := ret[0].(controller.VolumeGetter)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetVolumeGetter indicates an expected call of GetVolumeGetter.
+func (mr *MockClusterControllerMockRecorder) GetVolumeGetter(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVolumeGetter", reflect.TypeOf((*MockClusterController)(nil).GetVolumeGetter), arg0)
+}
+
+// Region mocks base method.
+func (m *MockClusterController) Region() string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Region")
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// Region indicates an expected call of Region.
+func (mr *MockClusterControllerMockRecorder) Region() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Region", reflect.TypeOf((*MockClusterController)(nil).Region))
+}
+
+// MockServiceController is a mock of ServiceController interface.
+type MockServiceController struct {
+	ctrl     *gomock.Controller
+	recorder *MockServiceControllerMockRecorder
+}
+
+// MockServiceControllerMockRecorder is the mock recorder for MockServiceController.
+type MockServiceControllerMockRecorder struct {
+	mock *MockServiceController
+}
+
+// NewMockServiceController creates a new mock instance.
+func NewMockServiceController(ctrl *gomock.Controller) *MockServiceController {
+	mock := &MockServiceController{ctrl: ctrl}
+	mock.recorder = &MockServiceControllerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockServiceController) EXPECT() *MockServiceControllerMockRecorder {
+	return m.recorder
+}
+
+// GetDiskHost mocks base method.
+func (m *MockServiceController) GetDiskHost(arg0 context.Context, arg1 proto.DiskID) (*controller.HostIDC, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetDiskHost", arg0, arg1)
+	ret0, _ := ret[0].(*controller.HostIDC)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetDiskHost indicates an expected call of GetDiskHost.
+func (mr *MockServiceControllerMockRecorder) GetDiskHost(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDiskHost", reflect.TypeOf((*MockServiceController)(nil).GetDiskHost), arg0, arg1)
+}
+
+// GetServiceHost mocks base method.
+func (m *MockServiceController) GetServiceHost(arg0 context.Context, arg1 string) (string, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetServiceHost", arg0, arg1)
+	ret0, _ := ret[0].(string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetServiceHost indicates an expected call of GetServiceHost.
+func (mr *MockServiceControllerMockRecorder) GetServiceHost(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetServiceHost", reflect.TypeOf((*MockServiceController)(nil).GetServiceHost), arg0, arg1)
+}
+
+// GetServiceHosts mocks base method.
+func (m *MockServiceController) GetServiceHosts(arg0 context.Context, arg1 string) ([]string, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetServiceHosts", arg0, arg1)
+	ret0, _ := ret[0].([]string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetServiceHosts indicates an expected call of GetServiceHosts.
+func (mr *MockServiceControllerMockRecorder) GetServiceHosts(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetServiceHosts", reflect.TypeOf((*MockServiceController)(nil).GetServiceHosts), arg0, arg1)
+}
+
+// PunishDisk mocks base method.
+func (m *MockServiceController) PunishDisk(arg0 context.Context, arg1 proto.DiskID, arg2 int) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "PunishDisk", arg0, arg1, arg2)
+}
+
+// PunishDisk indicates an expected call of PunishDisk.
+func (mr *MockServiceControllerMockRecorder) PunishDisk(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PunishDisk", reflect.TypeOf((*MockServiceController)(nil).PunishDisk), arg0, arg1, arg2)
+}
+
+// PunishDiskWithThreshold mocks base method.
+func (m *MockServiceController) PunishDiskWithThreshold(arg0 context.Context, arg1 proto.DiskID, arg2 int) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "PunishDiskWithThreshold", arg0, arg1, arg2)
+}
+
+// PunishDiskWithThreshold indicates an expected call of PunishDiskWithThreshold.
+func (mr *MockServiceControllerMockRecorder) PunishDiskWithThreshold(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PunishDiskWithThreshold", reflect.TypeOf((*MockServiceController)(nil).PunishDiskWithThreshold), arg0, arg1, arg2)
+}
+
+// PunishService mocks base method.
+func (m *MockServiceController) PunishService(arg0 context.Context, arg1, arg2 string, arg3 int) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "PunishService", arg0, arg1, arg2, arg3)
+}
+
+// PunishService indicates an expected call of PunishService.
+func (mr *MockServiceControllerMockRecorder) PunishService(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PunishService", reflect.TypeOf((*MockServiceController)(nil).PunishService), arg0, arg1, arg2, arg3)
+}
+
+// PunishServiceWithThreshold mocks base method.
+func (m *MockServiceController) PunishServiceWithThreshold(arg0 context.Context, arg1, arg2 string, arg3 int) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "PunishServiceWithThreshold", arg0, arg1, arg2, arg3)
+}
+
+// PunishServiceWithThreshold indicates an expected call of PunishServiceWithThreshold.
+func (mr *MockServiceControllerMockRecorder) PunishServiceWithThreshold(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PunishServiceWithThreshold", reflect.TypeOf((*MockServiceController)(nil).PunishServiceWithThreshold), arg0, arg1, arg2, arg3)
+}
+
+// MockVolumeGetter is a mock of VolumeGetter interface.
+type MockVolumeGetter struct {
+	ctrl     *gomock.Controller
+	recorder *MockVolumeGetterMockRecorder
+}
+
+// MockVolumeGetterMockRecorder is the mock recorder for MockVolumeGetter.
+type MockVolumeGetterMockRecorder struct {
+	mock *MockVolumeGetter
+}
+
+// NewMockVolumeGetter creates a new mock instance.
+func NewMockVolumeGetter(ctrl *gomock.Controller) *MockVolumeGetter {
+	mock := &MockVolumeGetter{ctrl: ctrl}
+	mock.recorder = &MockVolumeGetterMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockVolumeGetter) EXPECT() *MockVolumeGetterMockRecorder {
+	return m.recorder
+}
+
+// Get mocks base method.
+func (m *MockVolumeGetter) Get(arg0 context.Context, arg1 proto.Vid, arg2 bool) *controller.VolumePhy {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2)
+	ret0, _ := ret[0].(*controller.VolumePhy)
+	return ret0
+}
+
+// Get indicates an expected call of Get.
+func (mr *MockVolumeGetterMockRecorder) Get(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockVolumeGetter)(nil).Get), arg0, arg1, arg2)
+}
+
+// Punish mocks base method.
+func (m *MockVolumeGetter) Punish(arg0 context.Context, arg1 proto.Vid, arg2 int) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "Punish", arg0, arg1, arg2)
+}
+
+// Punish indicates an expected call of Punish.
+func (mr *MockVolumeGetterMockRecorder) Punish(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Punish", reflect.TypeOf((*MockVolumeGetter)(nil).Punish), arg0, arg1, arg2)
+}

blobstore/access/limiter.go

@@ -0,0 +1,238 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"context"
+	"io"
+	"time"
+
+	"golang.org/x/time/rate"
+
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/util/limit"
+	"github.com/cubefs/cubefs/blobstore/util/limit/count"
+)
+
+const (
+	_tagLimitedR = "limitedr"
+	_tagLimitedW = "limitedw"
+)
+
+// Limiter rps and bps limiter
+type Limiter interface {
+	// Acquire acquire with one request per second
+	Acquire(name string) error
+	// Release release of one request per second
+	Release(name string)
+
+	// Reader return io.Reader with bandwidth rate limit
+	Reader(ctx context.Context, r io.Reader) io.Reader
+	// Writer return io.Writer with bandwidth rate limit
+	Writer(ctx context.Context, w io.Writer) io.Writer
+
+	// Status returns running status
+	// TODO: calculate rate limit wait concurrent
+	Status() Status
+}
+
+// LimitConfig configuration of limiter
+type LimitConfig struct {
+	NameRps    map[string]int `json:"name_rps"`    // request with name n/s
+	ReaderMBps int            `json:"reader_mbps"` // read with MB/s
+	WriterMBps int            `json:"writer_mbps"` // write with MB/s
+}
+
+// Status running status
+type Status struct {
+	Config    LimitConfig    `json:"config"`     // configuration status
+	Running   map[string]int `json:"running"`    // running request
+	ReadWait  int            `json:"read_wait"`  // wait reading duration
+	WriteWait int            `json:"write_wait"` // wait writing duration
+}
+
+// Reader limited reader
+type Reader struct {
+	ctx        context.Context
+	rate       *rate.Limiter
+	underlying io.Reader
+}
+
+var _ io.Reader = &Reader{}
+
+func (r *Reader) Read(p []byte) (n int, err error) {
+	n, err = r.underlying.Read(p)
+
+	now := time.Now()
+	reserve := r.rate.ReserveN(now, n)
+
+	// Wait if necessary
+	delay := reserve.DelayFrom(now)
+	if delay == 0 {
+		return
+	}
+	t := time.NewTimer(delay)
+	defer t.Stop()
+
+	span := trace.SpanFromContextSafe(r.ctx)
+	// for access PUT request is Read from client
+	span.SetTag(_tagLimitedW, delay.Milliseconds())
+
+	select {
+	case <-t.C:
+		// We can proceed.
+		return
+	case <-r.ctx.Done():
+		// Context was canceled before we could proceed.  Cancel the
+		// reservation, which may permit other events to proceed sooner.
+		reserve.Cancel()
+		err = r.ctx.Err()
+		return
+	}
+}
+
+// Writer limited writer
+type Writer struct {
+	ctx        context.Context
+	rate       *rate.Limiter
+	underlying io.Writer
+}
+
+var _ io.Writer = &Writer{}
+
+func (w *Writer) Write(p []byte) (n int, err error) {
+	n, err = w.underlying.Write(p)
+
+	now := time.Now()
+	reserve := w.rate.ReserveN(now, n)
+
+	// Wait if necessary
+	delay := reserve.DelayFrom(now)
+	if delay == 0 {
+		return
+	}
+	t := time.NewTimer(delay)
+	defer t.Stop()
+
+	span := trace.SpanFromContextSafe(w.ctx)
+	// for access GET request is Write to client
+	span.SetTag(_tagLimitedR, delay.Milliseconds())
+
+	select {
+	case <-t.C:
+		// We can proceed.
+		return
+	case <-w.ctx.Done():
+		// Context was canceled before we could proceed.  Cancel the
+		// reservation, which may permit other events to proceed sooner.
+		reserve.Cancel()
+		err = w.ctx.Err()
+		return
+	}
+}
+
+type limiter struct {
+	config     LimitConfig
+	limiters   map[string]limit.Limiter
+	rateReader *rate.Limiter
+	rateWriter *rate.Limiter
+}
+
+// NewLimiter returns a Limiter
+func NewLimiter(cfg LimitConfig) Limiter {
+	mb := 1 << 20
+	lim := &limiter{
+		config:   cfg,
+		limiters: make(map[string]limit.Limiter, len(cfg.NameRps)),
+	}
+
+	for name, rps := range cfg.NameRps {
+		if rps > 0 {
+			lim.limiters[name] = count.New(rps)
+		}
+	}
+
+	if cfg.ReaderMBps > 0 {
+		lim.rateReader = rate.NewLimiter(rate.Limit(cfg.ReaderMBps*mb), 2*cfg.ReaderMBps*mb)
+	}
+	if cfg.WriterMBps > 0 {
+		lim.rateWriter = rate.NewLimiter(rate.Limit(cfg.WriterMBps*mb), 2*cfg.WriterMBps*mb)
+	}
+
+	return lim
+}
+
+func (lim *limiter) Acquire(name string) error {
+	if l := lim.limiters[name]; l != nil {
+		return l.Acquire()
+	}
+	return nil
+}
+
+func (lim *limiter) Release(name string) {
+	if l := lim.limiters[name]; l != nil {
+		l.Release()
+	}
+}
+
+func (lim *limiter) Reader(ctx context.Context, r io.Reader) io.Reader {
+	if lim.rateReader != nil {
+		return &Reader{
+			ctx:        ctx,
+			rate:       lim.rateReader,
+			underlying: r,
+		}
+	}
+	return r
+}
+
+func (lim *limiter) Writer(ctx context.Context, w io.Writer) io.Writer {
+	if lim.rateWriter != nil {
+		return &Writer{
+			ctx:        ctx,
+			rate:       lim.rateWriter,
+			underlying: w,
+		}
+	}
+	return w
+}
+
+func (lim *limiter) Status() Status {
+	st := Status{
+		Config: lim.config,
+	}
+
+	st.Running = make(map[string]int, len(lim.limiters))
+	for name, nl := range lim.limiters {
+		st.Running[name] = nl.Running()
+	}
+
+	st.ReadWait = rateWait(lim.rateReader)
+	st.WriteWait = rateWait(lim.rateWriter)
+
+	return st
+}
+
+// rateWait get duration of waiting half of limit
+func rateWait(r *rate.Limiter) int {
+	if r == nil {
+		return 0
+	}
+	now := time.Now()
+	reserve := r.ReserveN(now, int(r.Limit())/2)
+	duration := reserve.DelayFrom(now)
+	reserve.Cancel()
+	return int(duration.Milliseconds())
+}

blobstore/access/limiter_test.go

@@ -0,0 +1,302 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"context"
+	"io"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/time/rate"
+
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/util/limit"
+)
+
+type (
+	limitReader struct {
+		size int
+		read int
+	}
+	limitWriter struct{}
+)
+
+func (r *limitReader) Read(p []byte) (n int, err error) {
+	if r.read >= r.size {
+		return 0, io.EOF
+	}
+
+	b := make([]byte, 1<<20)
+	n = copy(p, b)
+	r.read += n
+	return
+}
+
+func (w *limitWriter) Write(p []byte) (n int, err error) {
+	b := make([]byte, 1<<20)
+	n = 0
+	for len(p) > 0 {
+		nn := copy(p, b)
+		n += nn
+		p = p[nn:]
+	}
+	return
+}
+
+func TestAccessLimitReader(t *testing.T) {
+	ctx, cancel := context.WithCancel(ctxWithName("TestAccessLimitReader")())
+	r := &Reader{
+		ctx:        ctx,
+		rate:       rate.NewLimiter(rate.Limit(1<<20), 1<<20),
+		underlying: &limitReader{size: 1 << 24},
+	}
+	errCh := make(chan error)
+	go func() {
+		buf := make([]byte, 1<<24)
+		_, err := io.ReadFull(r, buf)
+		errCh <- err
+	}()
+
+	time.Sleep(time.Second)
+	cancel()
+
+	err := <-errCh
+	require.Error(t, err)
+}
+
+func TestAccessLimitWriter(t *testing.T) {
+	ctx, cancel := context.WithCancel(ctxWithName("TestAccessLimitWriter")())
+	w := &Writer{
+		ctx:        ctx,
+		rate:       rate.NewLimiter(rate.Limit(1<<20), 1<<20),
+		underlying: &limitWriter{},
+	}
+	errCh := make(chan error)
+	go func() {
+		_, err := w.Write(make([]byte, 1<<24))
+		errCh <- err
+	}()
+
+	time.Sleep(time.Second)
+	cancel()
+
+	err := <-errCh
+	require.Error(t, err)
+}
+
+func TestAccessLimiterBase(t *testing.T) {
+	nameGet := "get"
+	namePut := "put"
+	cfg := LimitConfig{
+		NameRps: map[string]int{
+			nameGet: 1,
+			namePut: 1,
+		},
+		ReaderMBps: 1,
+		WriterMBps: 1,
+	}
+	l := NewLimiter(cfg)
+
+	{
+		for range [100]struct{}{} {
+			err := l.Acquire("")
+			assert.NoError(t, err)
+		}
+		l.Release("")
+	}
+	{
+		err := l.Acquire(nameGet)
+		assert.NoError(t, err)
+		err = l.Acquire(nameGet)
+		assert.Equal(t, limit.ErrLimited, err)
+		l.Release(nameGet)
+		err = l.Acquire(nameGet)
+		assert.NoError(t, err)
+		l.Release(nameGet)
+	}
+	{
+		err := l.Acquire(nameGet)
+		assert.NoError(t, err)
+		err = l.Acquire(namePut)
+		assert.NoError(t, err)
+
+		err = l.Acquire(nameGet)
+		assert.Equal(t, limit.ErrLimited, err)
+		err = l.Acquire(namePut)
+		assert.Equal(t, limit.ErrLimited, err)
+
+		l.Release(nameGet)
+		l.Release(namePut)
+	}
+
+	{
+		ctx := ctxWithName("TestAccessLimiterBase")()
+		rbuff := &limitReader{size: 1 << 8}
+		r := l.Reader(ctx, rbuff)
+
+		wbuff := &limitWriter{}
+		w := l.Writer(ctx, wbuff)
+		buf := make([]byte, 1<<20)
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func() {
+			io.CopyBuffer(w, r, buf)
+			wg.Done()
+		}()
+		wg.Wait()
+
+		span := trace.SpanFromContextSafe(ctx)
+		tags := span.Tags()
+		require.Empty(t, tags, span.TraceID(), tags)
+	}
+	{
+		ctx, cancel := context.WithCancel(ctxWithName("TestAccessLimiterBase")())
+		rbuff := &limitReader{size: 1 << 30}
+		r := l.Reader(ctx, rbuff)
+
+		wbuff := &limitWriter{}
+		w := l.Writer(ctx, wbuff)
+
+		var wg sync.WaitGroup
+		wg.Add(1)
+		closeCh := make(chan struct{})
+		go func() {
+			defer wg.Done()
+
+			for {
+				select {
+				case <-closeCh:
+					return
+				default:
+				}
+
+				_, err := io.CopyN(w, r, 1<<20)
+				if err != nil {
+					return
+				}
+			}
+		}()
+
+		time.Sleep(time.Second)
+		cancel()
+		close(closeCh)
+		wg.Wait()
+
+		span := trace.SpanFromContextSafe(ctx)
+		tags := span.Tags()
+		require.NotEmpty(t, tags, span.TraceID(), tags)
+	}
+}
+
+func TestAccessLimiterNoop(t *testing.T) {
+	l := NewLimiter(LimitConfig{
+		NameRps:    nil,
+		ReaderMBps: 0,
+		WriterMBps: 0,
+	})
+
+	name := "noop"
+	err := l.Acquire(name)
+	assert.NoError(t, err)
+	err = l.Acquire(name)
+	assert.NoError(t, err)
+	l.Release(name)
+	l.Release(name)
+
+	ctx := ctxWithName("TestAccessLimiterNoop")()
+	rbuff := &limitReader{size: 1 << 10}
+	r := l.Reader(ctx, rbuff)
+
+	wbuff := &limitWriter{}
+	w := l.Writer(ctx, wbuff)
+	buf := make([]byte, 1<<5)
+	ch := make(chan struct{})
+	go func() {
+		io.CopyBuffer(w, r, buf)
+		close(ch)
+	}()
+
+	<-ch
+	span := trace.SpanFromContextSafe(ctx)
+	tags := span.Tags()
+	require.Empty(t, tags, span.TraceID(), tags)
+}
+
+func TestAccessLimiterStatus(t *testing.T) {
+	{
+		l := NewLimiter(LimitConfig{
+			NameRps:    nil,
+			ReaderMBps: 0,
+			WriterMBps: 0,
+		})
+		for range [100]struct{}{} {
+			l.Acquire("foo")
+		}
+		t.Logf("%+v\n", l.Status())
+	}
+	{
+		ctx := ctxWithName("TestAccessLimiterStatus")()
+		name := "foo"
+		l := NewLimiter(LimitConfig{
+			NameRps:    map[string]int{name: 10},
+			ReaderMBps: 2,
+			WriterMBps: 10,
+		})
+
+		ch := make(chan struct{})
+		for range [7]struct{}{} {
+			go func() {
+				l.Acquire(name)
+				<-ch
+				l.Release(name)
+			}()
+		}
+
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func() {
+			rbuff := &limitReader{size: 1 << 24}
+			r := l.Reader(ctx, rbuff)
+			io.CopyBuffer(&limitWriter{}, r, make([]byte, 1<<20))
+			wg.Done()
+		}()
+
+		w := l.Writer(ctx, &limitWriter{})
+		wg.Add(8)
+		for range [8]struct{}{} {
+			go func() {
+				defer wg.Done()
+				for {
+					select {
+					case <-ch:
+						return
+					default:
+					}
+					w.Write(make([]byte, 1<<20))
+				}
+			}()
+		}
+
+		time.Sleep(time.Second * 3)
+		t.Logf("%+v\n", l.Status())
+
+		close(ch)
+		wg.Wait()
+	}
+}

blobstore/access/metric.go

@@ -0,0 +1,54 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+var unhealthMetric = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Namespace: "blobstore",
+		Subsystem: "access",
+		Name:      "unhealth",
+		Help:      "unhealth action on access",
+	},
+	[]string{"cluster", "action", "module", "host", "reason"},
+)
+
+var downloadMetric = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Namespace: "blobstore",
+		Subsystem: "access",
+		Name:      "download",
+		Help:      "download way on access",
+	},
+	[]string{"cluster", "way", "reason"},
+)
+
+func init() {
+	prometheus.MustRegister(unhealthMetric)
+	prometheus.MustRegister(downloadMetric)
+}
+
+func reportUnhealth(cid proto.ClusterID, action, module, host, reason string) {
+	unhealthMetric.WithLabelValues(cid.ToString(), action, module, host, reason).Inc()
+}
+
+func reportDownload(cid proto.ClusterID, way, reason string) {
+	downloadMetric.WithLabelValues(cid.ToString(), way, reason).Inc()
+}

blobstore/access/server.go

@@ -0,0 +1,681 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"crypto/sha1"
+	"fmt"
+	"net/http"
+	"strconv"
+	"sync"
+	"time"
+
+	consulapi "github.com/hashicorp/consul/api"
+
+	"github.com/cubefs/cubefs/blobstore/access/controller"
+	"github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/cmd"
+	"github.com/cubefs/cubefs/blobstore/common/consul"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/profile"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/resourcepool"
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/common/uptoken"
+	"github.com/cubefs/cubefs/blobstore/util/closer"
+	"github.com/cubefs/cubefs/blobstore/util/errors"
+	"github.com/cubefs/cubefs/blobstore/util/log"
+)
+
+const (
+	limitNameAlloc  = "alloc"
+	limitNamePut    = "put"
+	limitNamePutAt  = "putat"
+	limitNameGet    = "get"
+	limitNameDelete = "delete"
+	limitNameSign   = "sign"
+)
+
+const (
+	_tokenExpiration = time.Hour * 12
+)
+
+var (
+	// tokenSecretKeys alloc token with the first secret key always,
+	// so that you can change the secret key.
+	//
+	// parse-1: insert a new key at the first index,
+	// parse-2: delete the old key at the last index after _tokenExpiration duration.
+	tokenSecretKeys = [...][20]byte{
+		{0x5f, 0x00, 0x88, 0x96, 0x00, 0xa1, 0xfe, 0x1b},
+		{0xff, 0x1f, 0x2f, 0x4f, 0x7f, 0xaf, 0xef, 0xff},
+	}
+	_initTokenSecret sync.Once
+)
+
+func initTokenSecret(b []byte) {
+	_initTokenSecret.Do(func() {
+		for idx := range tokenSecretKeys {
+			copy(tokenSecretKeys[idx][7:], b)
+		}
+	})
+}
+
+func initWithRegionMagic(regionMagic string) {
+	if regionMagic == "" {
+		log.Warn("no region magic setting, using default secret keys for checksum")
+		return
+	}
+
+	log.Info("using magic secret keys for checksum with:", regionMagic)
+	b := sha1.Sum([]byte(regionMagic))
+	initTokenSecret(b[:8])
+	initLocationSecret(b[:8])
+}
+
+type accessStatus struct {
+	Limit Status              `json:"limit"`
+	Pool  resourcepool.Status `json:"pool"`
+
+	Config   StreamConfig                            `json:"config"`
+	Clusters []*clustermgr.ClusterInfo               `json:"clusters"`
+	Services map[proto.ClusterID]map[string][]string `json:"services"`
+}
+
+// Config service configs
+type Config struct {
+	cmd.Config
+	ConsulAgentAddr string        `json:"consul_agent_addr"`
+	ServiceRegister consul.Config `json:"service_register"`
+	Stream          StreamConfig  `json:"stream"`
+	Limit           LimitConfig   `json:"limit"`
+}
+
+// Service rpc service
+type Service struct {
+	config        Config
+	streamHandler StreamHandler
+	limiter       Limiter
+	closer        closer.Closer
+}
+
+// New returns an access service
+func New(cfg Config) *Service {
+	consulConf := consulapi.DefaultConfig()
+	consulConf.Address = cfg.ConsulAgentAddr
+
+	client, err := consulapi.NewClient(consulConf)
+	if err != nil {
+		log.Fatalf("new consul client failed, err: %v", err)
+	}
+
+	// add region magic checksum to the secret keys
+	initWithRegionMagic(cfg.Stream.ClusterConfig.RegionMagic)
+
+	cl := closer.New()
+	return &Service{
+		config:        cfg,
+		streamHandler: NewStreamHandler(&cfg.Stream, client, cl.Done()),
+		limiter:       NewLimiter(cfg.Limit),
+		closer:        cl,
+	}
+}
+
+// Close close server
+func (s *Service) Close() {
+	s.closer.Close()
+}
+
+// RegisterService register service to rpc
+func (s *Service) RegisterService() {
+	_, err := consul.ServiceRegister(s.config.BindAddr, &s.config.ServiceRegister)
+	if err != nil {
+		log.Fatalf("service register failed, err: %v", err)
+	}
+}
+
+// RegisterAdminHandler register admin handler to profile
+func (s *Service) RegisterAdminHandler() {
+	profile.HandleFunc(http.MethodGet, "/access/status", func(c *rpc.Context) {
+		var admin *streamAdmin
+		if sa := s.streamHandler.Admin(); sa != nil {
+			if ad, ok := sa.(*streamAdmin); ok {
+				admin = ad
+			}
+		}
+		if admin == nil {
+			c.RespondStatus(http.StatusServiceUnavailable)
+			return
+		}
+
+		ctx := c.Request.Context()
+		span := trace.SpanFromContextSafe(ctx)
+
+		status := new(accessStatus)
+		status.Limit = s.limiter.Status()
+		status.Pool = admin.memPool.Status()
+		status.Config = admin.config
+		status.Clusters = admin.controller.All()
+		status.Services = make(map[proto.ClusterID]map[string][]string, len(status.Clusters))
+
+		for _, cluster := range status.Clusters {
+			service, err := admin.controller.GetServiceController(cluster.ClusterID)
+			if err != nil {
+				span.Warn(err.Error())
+				continue
+			}
+
+			svrs := make(map[string][]string, 1)
+			svrName := proto.ServiceNameProxy
+			if hosts, err := service.GetServiceHosts(ctx, svrName); err == nil {
+				svrs[svrName] = hosts
+			} else {
+				span.Warn(err.Error())
+			}
+			status.Services[cluster.ClusterID] = svrs
+		}
+		c.RespondJSON(status)
+	})
+
+	profile.HandleFunc(http.MethodPost, "/access/stream/controller/alg/:alg", func(c *rpc.Context) {
+		algInt, err := strconv.Atoi(c.Param.ByName("alg"))
+		if err != nil {
+			c.RespondWith(http.StatusBadRequest, "", []byte(err.Error()))
+			return
+		}
+		if algInt < 0 {
+			c.RespondWith(http.StatusBadRequest, "", []byte("invalid algorithm"))
+			return
+		}
+
+		alg := controller.AlgChoose(algInt)
+		if sa := s.streamHandler.Admin(); sa != nil {
+			if admin, ok := sa.(*streamAdmin); ok {
+				if err := admin.controller.ChangeChooseAlg(alg); err != nil {
+					c.RespondWith(http.StatusForbidden, "", []byte(err.Error()))
+					return
+				}
+
+				span := trace.SpanFromContextSafe(c.Request.Context())
+				span.Warnf("change cluster choose algorithm to (%d %s)", alg, alg.String())
+				c.Respond()
+				return
+			}
+		}
+
+		c.RespondStatus(http.StatusServiceUnavailable)
+	}, rpc.OptArgsURI())
+}
+
+// Limit rps controller
+func (s *Service) Limit(c *rpc.Context) {
+	name := ""
+	switch c.Request.URL.Path {
+	case "/alloc":
+		name = limitNameAlloc
+	case "/put":
+		name = limitNamePut
+	case "/putat":
+		name = limitNamePutAt
+	case "/get":
+		name = limitNameGet
+	case "/delete":
+		name = limitNameDelete
+	case "/sign":
+		name = limitNameSign
+	}
+	if name == "" {
+		return
+	}
+
+	if err := s.limiter.Acquire(name); err != nil {
+		span := trace.SpanFromContextSafe(c.Request.Context())
+		span.Info("access concurrent limited", name, err)
+		c.AbortWithError(errcode.ErrAccessLimited)
+		return
+	}
+	defer s.limiter.Release(name)
+	c.Next()
+}
+
+// Put one object
+func (s *Service) Put(c *rpc.Context) {
+	args := new(access.PutArgs)
+	if err := c.ParseArgs(args); err != nil {
+		c.RespondError(err)
+		return
+	}
+
+	ctx := c.Request.Context()
+	span := trace.SpanFromContextSafe(ctx)
+
+	span.Debugf("accept /put request args:%+v", args)
+	if !args.IsValid() {
+		span.Debugf("invalid args:%+v", args)
+		c.RespondError(errcode.ErrIllegalArguments)
+		return
+	}
+
+	hashSumMap := args.Hashes.ToHashSumMap()
+	hasherMap := make(access.HasherMap, len(hashSumMap))
+	// make hashser
+	for alg := range hashSumMap {
+		hasherMap[alg] = alg.ToHasher()
+	}
+
+	rc := s.limiter.Reader(ctx, c.Request.Body)
+	loc, err := s.streamHandler.Put(ctx, rc, args.Size, hasherMap)
+	if err != nil {
+		span.Error("stream put failed", errors.Detail(err))
+		c.RespondError(httpError(err))
+		return
+	}
+
+	// hasher sum
+	for alg, hasher := range hasherMap {
+		hashSumMap[alg] = hasher.Sum(nil)
+	}
+
+	if err := fillCrc(loc); err != nil {
+		span.Error("stream put fill location crc", err)
+		c.RespondError(httpError(err))
+		return
+	}
+
+	c.RespondJSON(access.PutResp{
+		Location:   *loc,
+		HashSumMap: hashSumMap,
+	})
+	span.Infof("done /put request location:%+v hash:%+v", loc, hashSumMap.All())
+}
+
+// PutAt put one blob
+func (s *Service) PutAt(c *rpc.Context) {
+	args := new(access.PutAtArgs)
+	if err := c.ParseArgs(args); err != nil {
+		c.RespondError(err)
+		return
+	}
+
+	ctx := c.Request.Context()
+	span := trace.SpanFromContextSafe(ctx)
+
+	span.Debugf("accept /putat request args:%+v", args)
+	if !args.IsValid() {
+		span.Debugf("invalid args:%+v", args)
+		c.RespondError(errcode.ErrIllegalArguments)
+		return
+	}
+
+	valid := false
+	for _, secretKey := range tokenSecretKeys {
+		token := uptoken.DecodeToken(args.Token)
+		if token.IsValid(args.ClusterID, args.Vid, args.Blobid, uint32(args.Size), secretKey[:]) {
+			valid = true
+			break
+		}
+	}
+	if !valid {
+		span.Debugf("invalid token:%s", args.Token)
+		c.RespondError(errcode.ErrIllegalArguments)
+		return
+	}
+
+	hashSumMap := args.Hashes.ToHashSumMap()
+	hasherMap := make(access.HasherMap, len(hashSumMap))
+	// make hashser
+	for alg := range hashSumMap {
+		hasherMap[alg] = alg.ToHasher()
+	}
+
+	rc := s.limiter.Reader(ctx, c.Request.Body)
+	err := s.streamHandler.PutAt(ctx, rc, args.ClusterID, args.Vid, args.Blobid, args.Size, hasherMap)
+	if err != nil {
+		span.Error("stream putat failed", errors.Detail(err))
+		c.RespondError(httpError(err))
+		return
+	}
+
+	// hasher sum
+	for alg, hasher := range hasherMap {
+		hashSumMap[alg] = hasher.Sum(nil)
+	}
+
+	c.RespondJSON(access.PutAtResp{HashSumMap: hashSumMap})
+	span.Infof("done /putat request hash:%+v", hashSumMap.All())
+}
+
+// Alloc alloc one location
+func (s *Service) Alloc(c *rpc.Context) {
+	args := new(access.AllocArgs)
+	if err := c.ParseArgs(args); err != nil {
+		c.RespondError(err)
+		return
+	}
+
+	ctx := c.Request.Context()
+	span := trace.SpanFromContextSafe(ctx)
+
+	span.Debugf("accept /alloc request args:%+v", args)
+	if !args.IsValid() {
+		span.Debugf("invalid args:%+v", args)
+		c.RespondError(errcode.ErrIllegalArguments)
+		return
+	}
+
+	location, err := s.streamHandler.Alloc(ctx, args.Size, args.BlobSize, args.AssignClusterID, args.CodeMode)
+	if err != nil {
+		span.Error("stream alloc failed", errors.Detail(err))
+		c.RespondError(httpError(err))
+		return
+	}
+
+	if err := fillCrc(location); err != nil {
+		span.Error("stream alloc fill location crc", err)
+		c.RespondError(httpError(err))
+		return
+	}
+
+	resp := access.AllocResp{
+		Location: *location,
+		Tokens:   genTokens(location),
+	}
+	c.RespondJSON(resp)
+	span.Infof("done /alloc request resp:%+v", resp)
+}
+
+// Get read file
+func (s *Service) Get(c *rpc.Context) {
+	args := new(access.GetArgs)
+	if err := c.ParseArgs(args); err != nil {
+		c.RespondError(err)
+		return
+	}
+
+	ctx := c.Request.Context()
+	span := trace.SpanFromContextSafe(ctx)
+
+	span.Debugf("accept /get request args:%+v", args)
+	if !args.IsValid() || !verifyCrc(&args.Location) {
+		span.Debugf("invalid args:%+v", args)
+		c.RespondError(errcode.ErrIllegalArguments)
+		return
+	}
+
+	w := c.Writer
+	writer := s.limiter.Writer(ctx, w)
+	transfer, err := s.streamHandler.Get(ctx, writer, args.Location, args.ReadSize, args.Offset)
+	if err != nil {
+		span.Error("stream get prepare failed", errors.Detail(err))
+		c.RespondError(httpError(err))
+		return
+	}
+
+	w.Header().Set(rpc.HeaderContentType, rpc.MIMEStream)
+	w.Header().Set(rpc.HeaderContentLength, strconv.FormatInt(int64(args.ReadSize), 10))
+	if args.ReadSize > 0 && args.ReadSize != args.Location.Size {
+		w.Header().Set(rpc.HeaderContentRange, fmt.Sprintf("bytes %d-%d/%d",
+			args.Offset, args.Offset+args.ReadSize-1, args.Location.Size))
+		c.RespondStatus(http.StatusPartialContent)
+	} else {
+		c.RespondStatus(http.StatusOK)
+	}
+
+	// flush headers to client firstly
+	c.Flush()
+
+	err = transfer()
+	if err != nil {
+		span.Error("stream get transfer failed", errors.Detail(err))
+		return
+	}
+	span.Info("done /get request")
+}
+
+// Delete  all blobs in this location
+func (s *Service) Delete(c *rpc.Context) {
+	args := new(access.DeleteArgs)
+	if err := c.ParseArgs(args); err != nil {
+		c.RespondError(err)
+		return
+	}
+
+	ctx := c.Request.Context()
+	span := trace.SpanFromContextSafe(ctx)
+
+	var err error
+	var resp access.DeleteResp
+	defer func() {
+		if err != nil {
+			c.RespondError(httpError(err))
+			return
+		}
+
+		if len(resp.FailedLocations) > 0 {
+			span.Errorf("failed locations N %d of %d", len(resp.FailedLocations), len(args.Locations))
+			// must return 2xx even if has failed locations,
+			// cos rpc read body only on 2xx.
+			// TODO: return other http status code
+			c.RespondStatusData(http.StatusIMUsed, resp)
+			return
+		}
+
+		c.RespondJSON(resp)
+	}()
+
+	if !args.IsValid() {
+		err = errcode.ErrIllegalArguments
+		return
+	}
+	span.Debugf("accept /delete request args: locations %d", len(args.Locations))
+	defer span.Info("done /delete request")
+
+	clusterBlobsN := make(map[proto.ClusterID]int, 4)
+	for _, loc := range args.Locations {
+		if !verifyCrc(&loc) {
+			span.Infof("invalid crc %+v", loc)
+			err = errcode.ErrIllegalArguments
+			return
+		}
+		clusterBlobsN[loc.ClusterID] += len(loc.Blobs)
+	}
+
+	if len(args.Locations) == 1 {
+		loc := args.Locations[0]
+		if err := s.streamHandler.Delete(ctx, &loc); err != nil {
+			span.Error("stream delete failed", errors.Detail(err))
+			resp.FailedLocations = []access.Location{loc}
+		}
+		return
+	}
+
+	// merge the same cluster locations to one delete message,
+	// anyone of this cluster failed, all locations mark failure,
+	//
+	// a min delete message about 10-20 bytes,
+	// max delete locations is 1024, one location is max to 5G,
+	// merged message max size about 40MB.
+
+	merged := make(map[proto.ClusterID][]access.SliceInfo, len(clusterBlobsN))
+	for id, n := range clusterBlobsN {
+		merged[id] = make([]access.SliceInfo, 0, n)
+	}
+	for _, loc := range args.Locations {
+		merged[loc.ClusterID] = append(merged[loc.ClusterID], loc.Blobs...)
+	}
+
+	var wg sync.WaitGroup
+	failedCh := make(chan proto.ClusterID, 1)
+	done := make(chan struct{})
+	go func() {
+		for id := range failedCh {
+			if resp.FailedLocations == nil {
+				resp.FailedLocations = make([]access.Location, 0, len(args.Locations))
+			}
+			for _, loc := range args.Locations {
+				if loc.ClusterID == id {
+					resp.FailedLocations = append(resp.FailedLocations, loc)
+				}
+			}
+		}
+		close(done)
+	}()
+
+	wg.Add(len(merged))
+	for id := range merged {
+		go func(id proto.ClusterID) {
+			if err := s.streamHandler.Delete(ctx, &access.Location{
+				ClusterID: id,
+				BlobSize:  1,
+				Blobs:     merged[id],
+			}); err != nil {
+				span.Error("stream delete failed", id, errors.Detail(err))
+				failedCh <- id
+			}
+			wg.Done()
+		}(id)
+	}
+
+	wg.Wait()
+	close(failedCh)
+	<-done
+}
+
+// DeleteBlob delete one blob
+func (s *Service) DeleteBlob(c *rpc.Context) {
+	args := new(access.DeleteBlobArgs)
+	if err := c.ParseArgs(args); err != nil {
+		c.RespondError(err)
+		return
+	}
+
+	ctx := c.Request.Context()
+	span := trace.SpanFromContextSafe(ctx)
+
+	span.Debugf("accept /deleteblob request args:%+v", args)
+	if !args.IsValid() {
+		span.Debugf("invalid args:%+v", args)
+		c.RespondError(errcode.ErrIllegalArguments)
+		return
+	}
+
+	valid := false
+	for _, secretKey := range tokenSecretKeys {
+		token := uptoken.DecodeToken(args.Token)
+		if token.IsValid(args.ClusterID, args.Vid, args.Blobid, uint32(args.Size), secretKey[:]) {
+			valid = true
+			break
+		}
+	}
+	if !valid {
+		span.Debugf("invalid token:%s", args.Token)
+		c.RespondError(errcode.ErrIllegalArguments)
+		return
+	}
+
+	if err := s.streamHandler.Delete(ctx, &access.Location{
+		ClusterID: args.ClusterID,
+		BlobSize:  1,
+		Blobs: []access.SliceInfo{{
+			MinBid: args.Blobid,
+			Vid:    args.Vid,
+			Count:  1,
+		}},
+	}); err != nil {
+		span.Error("stream delete blob failed", errors.Detail(err))
+		c.RespondError(httpError(err))
+		return
+	}
+
+	c.Respond()
+	span.Info("done /deleteblob request")
+}
+
+// Sign generate crc with locations
+func (s *Service) Sign(c *rpc.Context) {
+	args := new(access.SignArgs)
+	if err := c.ParseArgs(args); err != nil {
+		c.RespondError(err)
+		return
+	}
+
+	ctx := c.Request.Context()
+	span := trace.SpanFromContextSafe(ctx)
+
+	if !args.IsValid() {
+		c.RespondError(errcode.ErrIllegalArguments)
+		return
+	}
+	span.Debugf("accept /sign request args: %+v", args)
+
+	loc := args.Location
+	crcOld := loc.Crc
+	if err := signCrc(&loc, args.Locations); err != nil {
+		span.Error("stream sign failed", errors.Detail(err))
+		c.RespondError(errcode.ErrIllegalArguments)
+		return
+	}
+
+	c.RespondJSON(access.SignResp{Location: loc})
+	span.Infof("done /sign request crc %d -> %d, resp:%+v", crcOld, loc.Crc, loc)
+}
+
+func httpError(err error) error {
+	if e, ok := err.(rpc.HTTPError); ok {
+		return e
+	}
+	if e, ok := err.(*errors.Error); ok {
+		return rpc.NewError(http.StatusInternalServerError, "ServerError", e.Cause())
+	}
+	return errcode.ErrUnexpected
+}
+
+// genTokens generate tokens
+// 1. Returns 0 token if has no blobs.
+// 2. Returns 1 token if file size less than blobsize.
+// 3. Returns len(blobs) tokens if size divided by blobsize.
+// 4. Otherwise returns len(blobs)+1 tokens, the last token
+//    will be used by the last blob, even if the last slice blobs' size
+//    less than blobsize.
+// 5. Each segment blob has its specified token include the last blob.
+func genTokens(location *access.Location) []string {
+	tokens := make([]string, 0, len(location.Blobs)+1)
+
+	hasMultiBlobs := location.Size >= uint64(location.BlobSize)
+	lastSize := uint32(location.Size % uint64(location.BlobSize))
+	for idx, blob := range location.Blobs {
+		// returns one token if size < blobsize
+		if hasMultiBlobs {
+			count := blob.Count
+			if idx == len(location.Blobs)-1 && lastSize > 0 {
+				count--
+			}
+			tokens = append(tokens, uptoken.EncodeToken(uptoken.NewUploadToken(location.ClusterID,
+				blob.Vid, blob.MinBid, count,
+				location.BlobSize, _tokenExpiration, tokenSecretKeys[0][:])))
+		}
+
+		// token of the last blob
+		if idx == len(location.Blobs)-1 && lastSize > 0 {
+			tokens = append(tokens, uptoken.EncodeToken(uptoken.NewUploadToken(location.ClusterID,
+				blob.Vid, blob.MinBid+proto.BlobID(blob.Count)-1, 1,
+				lastSize, _tokenExpiration, tokenSecretKeys[0][:])))
+		}
+	}
+
+	return tokens
+}

blobstore/access/server_location.go

@@ -0,0 +1,127 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"fmt"
+	"hash/crc32"
+	"sync"
+
+	"github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/util/bytespool"
+)
+
+const (
+	// DO NOT CHANGE IT.
+	_crcPoly = uint32(0x59c8943c)
+)
+
+var (
+	// DO NOT CHANGE IT.
+	_crcTable    = crc32.MakeTable(_crcPoly)
+	_crcMagicKey = [20]byte{
+		0x52, 0xe, 0x53, 0x53, 0x81,
+		0x1f, 0x51, 0xb7, 0xa4, 0x72,
+		0x10, 0x33, 0x64, 0xa7, 0x3a,
+		0x10, 0x19, 0xbc, 0x60, 0x7,
+	}
+	_initLocationSecret sync.Once
+)
+
+func initLocationSecret(b []byte) {
+	_initLocationSecret.Do(func() {
+		copy(_crcMagicKey[7:], b)
+	})
+}
+
+func calcCrc(loc *access.Location) (uint32, error) {
+	crcWriter := crc32.New(_crcTable)
+
+	buf := bytespool.Alloc(1024)
+	defer bytespool.Free(buf)
+
+	n := loc.Encode2(buf)
+	if n < 4 {
+		return 0, fmt.Errorf("no enough bytes(%d) fill into buf", n)
+	}
+
+	if _, err := crcWriter.Write(_crcMagicKey[:]); err != nil {
+		return 0, fmt.Errorf("fill crc %s", err.Error())
+	}
+	if _, err := crcWriter.Write(buf[4:n]); err != nil {
+		return 0, fmt.Errorf("fill crc %s", err.Error())
+	}
+
+	return crcWriter.Sum32(), nil
+}
+
+func fillCrc(loc *access.Location) error {
+	crc, err := calcCrc(loc)
+	if err != nil {
+		return err
+	}
+	loc.Crc = crc
+	return nil
+}
+
+func verifyCrc(loc *access.Location) bool {
+	crc, err := calcCrc(loc)
+	if err != nil {
+		return false
+	}
+	return loc.Crc == crc
+}
+
+func signCrc(loc *access.Location, locs []access.Location) error {
+	first := locs[0]
+	bids := make(map[proto.BlobID]struct{}, 64)
+
+	if loc.ClusterID != first.ClusterID ||
+		loc.CodeMode != first.CodeMode ||
+		loc.BlobSize != first.BlobSize {
+		return fmt.Errorf("not equal in constant field")
+	}
+
+	for _, l := range locs {
+		if !verifyCrc(&l) {
+			return fmt.Errorf("not equal in crc %d", l.Crc)
+		}
+
+		// assert
+		if l.ClusterID != first.ClusterID ||
+			l.CodeMode != first.CodeMode ||
+			l.BlobSize != first.BlobSize {
+			return fmt.Errorf("not equal in constant field")
+		}
+
+		for _, blob := range l.Blobs {
+			for c := 0; c < int(blob.Count); c++ {
+				bids[blob.MinBid+proto.BlobID(c)] = struct{}{}
+			}
+		}
+	}
+
+	for _, blob := range loc.Blobs {
+		for c := 0; c < int(blob.Count); c++ {
+			bid := blob.MinBid + proto.BlobID(c)
+			if _, ok := bids[bid]; !ok {
+				return fmt.Errorf("not equal in blob_id(%d)", bid)
+			}
+		}
+	}
+
+	return fillCrc(loc)
+}

blobstore/access/server_location_test.go

@@ -0,0 +1,203 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"fmt"
+	"hash/crc32"
+	"math"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/util/bytespool"
+)
+
+var (
+	testMaxBlob = access.SliceInfo{
+		MinBid: proto.BlobID(math.MaxUint64),
+		Vid:    proto.Vid(math.MaxInt32),
+		Count:  math.MaxUint32,
+	}
+	testMaxLoc = access.Location{
+		ClusterID: proto.ClusterID(math.MaxUint32),
+		CodeMode:  codemode.CodeMode(math.MaxInt8),
+		Size:      math.MaxUint64,
+		BlobSize:  math.MaxUint32,
+		Crc:       math.MaxUint32,
+	}
+
+	testMinBlob = access.SliceInfo{}
+	testMinLoc  = access.Location{}
+)
+
+func TestAccessServiceLocationCrc(t *testing.T) {
+	{
+		_, err := calcCrc(nil)
+		require.Error(t, err)
+	}
+	{
+		crc, err := calcCrc(&testMinLoc)
+		require.NoError(t, err)
+		require.Equal(t, uint32(0x8a7370cc), crc)
+	}
+	{
+		crc, err := calcCrc(&testMaxLoc)
+		require.NoError(t, err)
+		require.Equal(t, uint32(0xda55150a), crc)
+	}
+	{
+		loc := testMinLoc.Copy()
+		loc.Size = 1 << 30
+
+		err := fillCrc(&loc)
+		require.NoError(t, err)
+		require.Equal(t, uint32(0x9e17bc9e), loc.Crc)
+	}
+	{
+		loc := testMinLoc.Copy()
+		loc.Size = 1 << 30
+		require.False(t, verifyCrc(&loc))
+
+		loc.Crc = 0x9e17bc9e
+		require.True(t, verifyCrc(&loc))
+	}
+}
+
+func TestAccessServiceLocationSecret(t *testing.T) {
+	secret := make([]byte, len(_crcMagicKey))
+	copy(secret, _crcMagicKey[:])
+	defer func() {
+		copy(_crcMagicKey[:], secret)
+	}()
+	{
+		initLocationSecret([]byte{0x34, 0x45, 0x18, 0x4f})
+		crc, err := calcCrc(&testMinLoc)
+		require.NoError(t, err)
+		require.NotEqual(t, uint32(0x8a7370cc), crc)
+		require.Equal(t, uint32(0xdbe8df90), crc)
+	}
+	{
+		// init once
+		initLocationSecret([]byte{0x1, 0x2, 0x3, 0x4})
+		crc, err := calcCrc(&testMinLoc)
+		require.NoError(t, err)
+		require.Equal(t, uint32(0xdbe8df90), crc)
+	}
+}
+
+func TestAccessServiceLocationSignCrc(t *testing.T) {
+	loc := &access.Location{
+		ClusterID: 1,
+		CodeMode:  1,
+		Size:      1023,
+		BlobSize:  6,
+		Crc:       0,
+		Blobs: []access.SliceInfo{{
+			MinBid: 11,
+			Vid:    199,
+			Count:  10,
+		}},
+	}
+	fillCrc(loc)
+	require.True(t, verifyCrc(loc))
+
+	{
+		loc1, loc2 := loc.Copy(), loc.Copy()
+		require.NoError(t, signCrc(loc, []access.Location{loc1, loc2}))
+	}
+	{
+		loc1, loc2 := loc.Copy(), loc.Copy()
+		loc1.BlobSize = 100
+		fillCrc(&loc1)
+		require.Error(t, signCrc(loc, []access.Location{loc1, loc2}))
+	}
+	{
+		loc1, loc2 := loc.Copy(), loc.Copy()
+		loc2.Crc = 0
+		require.Error(t, signCrc(loc, []access.Location{loc1, loc2}))
+	}
+	{
+		loc1, loc2 := loc.Copy(), loc.Copy()
+		loc2.ClusterID = 2
+		fillCrc(&loc2)
+		require.Error(t, signCrc(loc, []access.Location{loc1, loc2}))
+	}
+	{
+		loc1, loc2 := loc.Copy(), loc.Copy()
+		loc2.CodeMode = 100
+		fillCrc(&loc2)
+		require.Error(t, signCrc(loc, []access.Location{loc1, loc2}))
+	}
+	{
+		loc1, loc2 := loc.Copy(), loc.Copy()
+		loc1.Blobs = nil
+		loc2.Blobs[0].Count = 5
+		fillCrc(&loc1)
+		fillCrc(&loc2)
+		require.Error(t, signCrc(loc, []access.Location{loc1, loc2}))
+	}
+}
+
+func calcCrcWithoutMagic(loc *access.Location) (uint32, error) {
+	crcWriter := crc32.New(_crcTable)
+
+	buf := bytespool.Alloc(1024)
+	defer bytespool.Free(buf)
+
+	n := loc.Encode2(buf)
+	crcWriter.Write(buf[4:n])
+
+	return crcWriter.Sum32(), nil
+}
+
+func benchmarkCrc(b *testing.B, key string,
+	location access.Location, blob access.SliceInfo,
+	run func(loc *access.Location) (uint32, error)) {
+	cases := []int{0, 2, 4, 8, 16, 32}
+	for _, l := range cases {
+		b.ResetTimer()
+		b.Run(fmt.Sprintf(key+"-%d", l), func(b *testing.B) {
+			loc := location.Copy()
+			loc.Blobs = make([]access.SliceInfo, l)
+			for idx := range loc.Blobs {
+				loc.Blobs[idx] = blob
+			}
+			b.ResetTimer()
+			for ii := 0; ii <= b.N; ii++ {
+				run(&loc)
+			}
+		})
+	}
+}
+
+func BenchmarkAccessServerCrcWithMagicMin(b *testing.B) {
+	benchmarkCrc(b, "min-with-magic", testMinLoc, testMinBlob, calcCrc)
+}
+
+func BenchmarkAccessServerCrcWithoutMagicMin(b *testing.B) {
+	benchmarkCrc(b, "min-without-magic", testMinLoc, testMinBlob, calcCrcWithoutMagic)
+}
+
+func BenchmarkAccessServerCrcWithMagicMax(b *testing.B) {
+	benchmarkCrc(b, "max-with-magic", testMaxLoc, testMaxBlob, calcCrc)
+}
+
+func BenchmarkAccessServerCrcWithoutMagicMax(b *testing.B) {
+	benchmarkCrc(b, "max-without-magic", testMaxLoc, testMaxBlob, calcCrcWithoutMagic)
+}

blobstore/access/server_test.go

@@ -0,0 +1,706 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+	"github.com/cubefs/cubefs/blobstore/common/uptoken"
+)
+
+var (
+	ctx              = context.Background()
+	_blobSize uint32 = 1 << 20
+	location         = &access.Location{
+		ClusterID: 1,
+		CodeMode:  1,
+		BlobSize:  _blobSize,
+		Crc:       0,
+		Blobs: []access.SliceInfo{{
+			MinBid: 111,
+			Vid:    1111,
+			Count:  11,
+		}},
+	}
+
+	testServer *httptest.Server
+	once       sync.Once
+)
+
+func runMockService(s *Service) string {
+	once.Do(func() {
+		testServer = httptest.NewServer(NewHandler(s))
+	})
+	return testServer.URL
+}
+
+func newService() *Service {
+	ctr := gomock.NewController(&testing.T{})
+	s := NewMockStreamHandler(ctr)
+
+	s.EXPECT().Alloc(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
+		func(ctx context.Context, size uint64, blobSize uint32,
+			assignClusterID proto.ClusterID, codeMode codemode.CodeMode) (*access.Location, error) {
+			if size < 1024 {
+				return nil, errors.New("fake alloc location")
+			}
+			loc := location.Copy()
+			loc.Size = uint64(size)
+			fillCrc(&loc)
+			return &loc, nil
+		})
+
+	s.EXPECT().PutAt(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(),
+		gomock.Any()).AnyTimes().DoAndReturn(
+		func(ctx context.Context, rc io.Reader,
+			clusterID proto.ClusterID, vid proto.Vid, bid proto.BlobID, size int64,
+			hasherMap access.HasherMap) error {
+			if size < 1024 {
+				return errcode.ErrAccessLimited
+			}
+			return nil
+		})
+
+	s.EXPECT().Put(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
+		func(ctx context.Context, rc io.Reader, size int64, hasherMap access.HasherMap) (*access.Location, error) {
+			if size < 1024 {
+				return nil, errors.New("fake put nil body")
+			}
+			loc := location.Copy()
+			loc.Size = uint64(size)
+			fillCrc(&loc)
+			return &loc, nil
+		})
+
+	s.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
+		func(ctx context.Context, w io.Writer, location access.Location, readSize, offset uint64) (func() error, error) {
+			if readSize < 1024 {
+				return nil, errors.New("fake get nil body")
+			}
+			return func() error { return nil }, nil
+		})
+	s.EXPECT().Delete(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
+		func(ctx context.Context, location *access.Location) error {
+			if location.ClusterID >= 10 {
+				return errors.New("fake delete error with cluster")
+			} else if location.ClusterID == 1 && location.Crc > 0 && location.Size < 1024 {
+				return errors.New("fake delete error")
+			}
+			return nil
+		})
+
+	return &Service{
+		streamHandler: s,
+		limiter: NewLimiter(LimitConfig{
+			NameRps: map[string]int{
+				limitNameAlloc: 2,
+			},
+			ReaderMBps: 0,
+			WriterMBps: 0,
+		}),
+	}
+}
+
+func newClient() rpc.Client {
+	return rpc.NewClient(&rpc.Config{})
+}
+
+func TestAccessServiceNew(t *testing.T) {
+	runMockService(newService())
+}
+
+func TestAccessServiceAlloc(t *testing.T) {
+	host := runMockService(newService())
+	cli := newClient()
+
+	url := func() string {
+		return fmt.Sprintf("%s/alloc", host)
+	}
+	args := access.AllocArgs{
+		Size:            0,
+		BlobSize:        0,
+		AssignClusterID: 0,
+		CodeMode:        0,
+	}
+	{
+		resp := &access.AllocResp{}
+		err := cli.PostWith(ctx, url(), resp, args)
+		assertErrorCode(t, 400, err)
+	}
+	{
+		args.Size = 1023
+		resp := &access.AllocResp{}
+		err := cli.PostWith(ctx, url(), resp, args)
+		assertErrorCode(t, 500, err)
+	}
+	{
+		args.Size = 1024
+		resp := &access.AllocResp{}
+		err := cli.PostWith(ctx, url(), resp, args)
+		require.NoError(t, err)
+		require.Equal(t, uint64(1024), resp.Location.Size)
+	}
+	{
+		args.Size = uint64(_blobSize)
+		resp := &access.AllocResp{}
+		err := cli.PostWith(ctx, url(), resp, args)
+		require.NoError(t, err)
+		require.Equal(t, uint64(_blobSize), resp.Location.Size)
+	}
+	{
+		args.Size = uint64(_blobSize) + 1
+		resp := &access.AllocResp{}
+		err := cli.PostWith(ctx, url(), resp, args)
+		require.NoError(t, err)
+		require.Equal(t, uint64(_blobSize)+1, resp.Location.Size)
+	}
+}
+
+func TestAccessServicePutAt(t *testing.T) {
+	host := runMockService(newService())
+	cli := newClient()
+
+	url := func(size int64, token string) string {
+		return fmt.Sprintf("%s/putat?clusterid=1&volumeid=1111&blobid=111&size=%d&hashes=14&token=%s",
+			host, size, token)
+	}
+
+	for _, method := range []string{http.MethodPut, http.MethodPost} {
+		args := access.PutArgs{
+			Size: 0,
+		}
+		{
+			buf := make([]byte, args.Size)
+			req, _ := http.NewRequest(method, url(args.Size, ""), bytes.NewReader(buf))
+			resp := &access.PutAtResp{}
+			err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode())
+			assertErrorCode(t, 400, err)
+		}
+		{
+			args.Size = 1023
+			buf := make([]byte, args.Size)
+			req, _ := http.NewRequest(method, url(args.Size, ""), bytes.NewReader(buf))
+			resp := &access.PutAtResp{}
+			err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode())
+			assertErrorCode(t, 400, err)
+		}
+		{
+			args.Size = 1023
+			buf := make([]byte, args.Size)
+			resp := &access.PutAtResp{}
+			req, _ := http.NewRequest(method, url(args.Size, "c1fdcecaacbfafd86f0b00"), bytes.NewReader(buf))
+			err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode())
+			assertErrorCode(t, 552, err)
+		}
+		{
+			args.Size = 1024
+			buf := make([]byte, args.Size)
+			req, _ := http.NewRequest(method, url(args.Size, "8238436d05ecf2366f0b00"), bytes.NewReader(buf))
+			resp := &access.PutAtResp{}
+			err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode())
+			require.NoError(t, err)
+
+			req, _ = http.NewRequest(method, url(args.Size, "1238436d05ecf2366f0b00"), bytes.NewReader(buf))
+			err = cli.DoWith(ctx, req, resp, rpc.WithCrcEncode())
+			assertErrorCode(t, 400, err)
+		}
+	}
+}
+
+func TestAccessServicePut(t *testing.T) {
+	host := runMockService(newService())
+	cli := newClient()
+
+	url := func(size int64, hashes access.HashAlgorithm) string {
+		return fmt.Sprintf("%s/put?size=%d&hashes=%d", host, size, hashes)
+	}
+
+	for _, method := range []string{http.MethodPut, http.MethodPost} {
+		args := access.PutArgs{
+			Size:   0,
+			Hashes: 14,
+			Body:   nil,
+		}
+		{
+			req, _ := http.NewRequest(method, fmt.Sprintf("%s/put?size=size", host), args.Body)
+			resp := &access.PutResp{}
+			err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode())
+			assertErrorCode(t, 400, err)
+		}
+		{
+			req, _ := http.NewRequest(method, url(args.Size, args.Hashes), args.Body)
+			resp := &access.PutResp{}
+			err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode())
+			assertErrorCode(t, 400, err)
+		}
+		{
+			args.Body = bytes.NewReader(make([]byte, 1023))
+			req, _ := http.NewRequest(method, url(1023, args.Hashes), args.Body)
+			resp := &access.PutResp{}
+			err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode())
+			assertErrorCode(t, 500, err)
+		}
+		{
+			args.Body = bytes.NewReader(make([]byte, 1024))
+			req, _ := http.NewRequest(method, url(1024, args.Hashes), args.Body)
+			resp := &access.PutResp{}
+			err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode())
+			require.NoError(t, err)
+			require.Equal(t, uint64(1024), resp.Location.Size)
+		}
+	}
+}
+
+func TestAccessServiceGet(t *testing.T) {
+	host := runMockService(newService())
+	cli := newClient()
+
+	url := func() string {
+		return fmt.Sprintf("%s/get", host)
+	}
+	args := access.GetArgs{
+		Location: location.Copy(),
+		Offset:   0,
+		ReadSize: 0,
+	}
+	{
+		args.ReadSize = 10
+		resp, err := cli.Post(ctx, url(), args)
+		require.NoError(t, err)
+		resp.Body.Close()
+		require.Equal(t, 400, resp.StatusCode, resp.Status)
+	}
+	{
+		args.Location.Size = 1023
+		args.ReadSize = 1023
+		fillCrc(&args.Location)
+		resp, err := cli.Post(ctx, url(), args)
+		require.NoError(t, err)
+		resp.Body.Close()
+		require.Equal(t, 500, resp.StatusCode, resp.Status)
+	}
+	{
+		args.Location.Size = 1024
+		args.ReadSize = 1024
+		fillCrc(&args.Location)
+		resp, err := cli.Post(ctx, url(), args)
+		require.NoError(t, err)
+		resp.Body.Close()
+		require.Equal(t, 200, resp.StatusCode, resp.Status)
+	}
+	{
+		args.Location.Size = 10240
+		args.Offset = 1000
+		args.ReadSize = 1024
+		fillCrc(&args.Location)
+		resp, err := cli.Post(ctx, url(), args)
+		require.NoError(t, err)
+		resp.Body.Close()
+		require.Equal(t, 206, resp.StatusCode, resp.Status)
+	}
+}
+
+func TestAccessServiceDelete(t *testing.T) {
+	host := runMockService(newService())
+	cli := newClient()
+
+	url := fmt.Sprintf("%s/delete", host)
+	deleteRequest := func(args interface{}) (code int, ret access.DeleteResp, err error) {
+		resp, err := cli.Post(ctx, url, args)
+		if err != nil {
+			return
+		}
+		defer resp.Body.Close()
+
+		code = resp.StatusCode
+		if code/100 == 2 {
+			size, _ := strconv.Atoi(resp.Header.Get("Content-Length"))
+			buf := make([]byte, size)
+			_, err = io.ReadFull(resp.Body, buf)
+			if err != nil {
+				return
+			}
+			if err = json.Unmarshal(buf, &ret); err != nil {
+				return
+			}
+		}
+		if code >= 400 {
+			err = rpc.NewError(code, "Code", fmt.Errorf("httpcode: %d", code))
+			return
+		}
+		return
+	}
+
+	args := access.DeleteArgs{
+		Locations: []access.Location{location.Copy()},
+	}
+	{
+		code, _, err := deleteRequest(access.DeleteArgs{})
+		require.Error(t, err)
+		require.Equal(t, 400, code)
+	}
+	{
+		code, _, err := deleteRequest(args)
+		require.Error(t, err)
+		require.Equal(t, 400, code)
+	}
+	{
+		fillCrc(&args.Locations[0])
+		code, resp, err := deleteRequest(args)
+		require.NoError(t, err)
+		require.Equal(t, 226, code)
+		require.Equal(t, args.Locations[0], resp.FailedLocations[0])
+	}
+	{
+		loc := &args.Locations[0]
+		loc.Size = 1024
+		fillCrc(loc)
+		code, _, err := deleteRequest(args)
+		require.NoError(t, err)
+		require.Equal(t, 200, code)
+	}
+	{
+		loc := location.Copy()
+		loc.Size = 1024
+		fillCrc(&loc)
+		locs := make([]access.Location, access.MaxDeleteLocations)
+		for idx := range locs {
+			locs[idx] = loc
+		}
+		code, resp, err := deleteRequest(access.DeleteArgs{Locations: locs})
+		require.NoError(t, err)
+		require.Equal(t, 200, code)
+		require.Equal(t, 0, len(resp.FailedLocations))
+	}
+	{
+		loc := location.Copy()
+		loc.Size = 1024
+		fillCrc(&loc)
+		locs := make([]access.Location, access.MaxDeleteLocations+1)
+		for idx := range locs {
+			locs[idx] = loc
+		}
+		code, _, err := deleteRequest(access.DeleteArgs{Locations: locs})
+		require.Error(t, err)
+		require.Equal(t, 400, code)
+	}
+	{
+		loc := location.Copy()
+		loc.Size = 1024
+		loc.ClusterID = proto.ClusterID(11)
+		fillCrc(&loc)
+		code, resp, err := deleteRequest(access.DeleteArgs{Locations: []access.Location{loc}})
+		require.NoError(t, err)
+		require.Equal(t, 226, code)
+		require.Equal(t, 1, len(resp.FailedLocations))
+		require.Equal(t, proto.ClusterID(11), resp.FailedLocations[0].ClusterID)
+	}
+	{
+		locs := make([]access.Location, access.MaxDeleteLocations)
+		for idx := range locs {
+			loc := location.Copy()
+			loc.Size = 1024
+			loc.ClusterID = proto.ClusterID(idx % 11)
+			fillCrc(&loc)
+			locs[idx] = loc
+		}
+		code, resp, err := deleteRequest(access.DeleteArgs{Locations: locs})
+		require.NoError(t, err)
+		require.Equal(t, 226, code)
+		require.Equal(t, 93, len(resp.FailedLocations))
+	}
+}
+
+func TestAccessServiceDeleteBlob(t *testing.T) {
+	host := runMockService(newService())
+	cli := newClient()
+
+	url := func(size int64, token string) string {
+		return fmt.Sprintf("%s/deleteblob?clusterid=1&volumeid=1111&blobid=111&size=%d&token=%s",
+			host, size, token)
+	}
+
+	method := http.MethodDelete
+	args := access.PutArgs{
+		Size: 0,
+	}
+	{
+		req, _ := http.NewRequest(method, url(args.Size, "xxx"), nil)
+		err := cli.DoWith(ctx, req, nil)
+		assertErrorCode(t, 400, err)
+	}
+	{
+		req, _ := http.NewRequest(method, url(args.Size, ""), nil)
+		err := cli.DoWith(ctx, req, nil)
+		assertErrorCode(t, 400, err)
+	}
+	{
+		args.Size = 1023
+		req, _ := http.NewRequest(method, url(args.Size, ""), nil)
+		err := cli.DoWith(ctx, req, nil)
+		assertErrorCode(t, 400, err)
+	}
+	{
+		args.Size = 1023
+		req, _ := http.NewRequest(method, url(args.Size, "xxx"), nil)
+		err := cli.DoWith(ctx, req, nil)
+		assertErrorCode(t, 400, err)
+	}
+	{
+		args.Size = 1023
+		req, _ := http.NewRequest(method, url(args.Size, "c1fdcecaacbfafd86f0b00"), nil)
+		err := cli.DoWith(ctx, req, nil)
+		require.NoError(t, err)
+	}
+
+	{
+		url := func() string {
+			return fmt.Sprintf("%s/deleteblob?clusterid=11&volumeid=1111&blobid=111&size=%d&token=%s",
+				host, 1024, "f034db4503d5dc3f6f0100")
+		}
+		req, _ := http.NewRequest(method, url(), nil)
+		err := cli.DoWith(ctx, req, nil)
+		assertErrorCode(t, 500, err)
+	}
+}
+
+func TestAccessServiceSign(t *testing.T) {
+	host := runMockService(newService())
+	cli := newClient()
+
+	url := func() string {
+		return fmt.Sprintf("%s/sign", host)
+	}
+	args := access.SignArgs{
+		Locations: []access.Location{location.Copy()},
+		Location:  location.Copy(),
+	}
+	{
+		resp := &access.SignResp{}
+		err := cli.PostWith(ctx, url(), resp, access.SignArgs{})
+		assertErrorCode(t, 400, err)
+	}
+	{
+		resp := &access.SignResp{}
+		err := cli.PostWith(ctx, url(), resp, args)
+		assertErrorCode(t, 400, err)
+	}
+	{
+		fillCrc(&args.Locations[0])
+		resp := &access.SignResp{}
+		err := cli.PostWith(ctx, url(), resp, args)
+		require.NoError(t, err)
+	}
+}
+
+func assertErrorCode(t *testing.T, code int, err error) {
+	require.Error(t, err)
+	codeActual := rpc.DetectStatusCode(err)
+	require.Equal(t, code, codeActual, err.Error())
+}
+
+func TestAccessServiceTokens(t *testing.T) {
+	skey := tokenSecretKeys[0][:]
+	checker := func(loc *access.Location, tokens []string) {
+		if loc.Size == 0 {
+			require.Equal(t, 0, len(tokens))
+			return
+		}
+
+		hasMultiBlobs := loc.Size >= uint64(loc.BlobSize)
+		lastSize := uint32(loc.Size % uint64(loc.BlobSize))
+		if !hasMultiBlobs {
+			require.Equal(t, 1, len(tokens))
+
+			token := uptoken.DecodeToken(tokens[0])
+			blob := loc.Blobs[0]
+			for bid := blob.MinBid - 100; bid < blob.MinBid+100; bid++ {
+				require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
+			}
+			require.True(t, token.IsValid(loc.ClusterID, blob.Vid, blob.MinBid, lastSize, skey))
+			return
+		}
+
+		if lastSize == 0 {
+			require.Equal(t, len(loc.Blobs), len(tokens))
+			for idx, blob := range loc.Blobs {
+				token := uptoken.DecodeToken(tokens[idx])
+				for ii := uint32(0); ii < 100; ii++ {
+					bid := blob.MinBid - proto.BlobID(ii) - 1
+					require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
+					bid = blob.MinBid + proto.BlobID(blob.Count+ii)
+					require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
+				}
+				for ii := uint32(0); ii < blob.Count; ii++ {
+					bid := blob.MinBid + proto.BlobID(ii)
+					require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
+				}
+			}
+			return
+		}
+
+		require.Equal(t, len(loc.Blobs)+1, len(tokens))
+		for ii := 0; ii < len(loc.Blobs)-1; ii++ {
+			token := uptoken.DecodeToken(tokens[ii])
+			blob := loc.Blobs[ii]
+			for ii := uint32(0); ii < blob.Count; ii++ {
+				bid := blob.MinBid + proto.BlobID(ii)
+				require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
+			}
+		}
+
+		token := uptoken.DecodeToken(tokens[len(loc.Blobs)-1])
+		blob := loc.Blobs[len(loc.Blobs)-1]
+		for ii := uint32(0); ii < 100; ii++ {
+			bid := blob.MinBid - proto.BlobID(ii) - 1
+			require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
+			bid = blob.MinBid + proto.BlobID(blob.Count+ii) - 1
+			require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
+		}
+		for ii := uint32(0); ii < blob.Count-1; ii++ {
+			bid := blob.MinBid + proto.BlobID(ii)
+			require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
+		}
+
+		token = uptoken.DecodeToken(tokens[len(loc.Blobs)])
+		lastbid := blob.MinBid + proto.BlobID(blob.Count) - 1
+		require.True(t, token.IsValid(loc.ClusterID, blob.Vid, lastbid, lastSize, skey))
+	}
+
+	{
+		loc := &access.Location{
+			Size:     0,
+			BlobSize: 333,
+			Blobs:    []access.SliceInfo{},
+		}
+		checker(loc, genTokens(loc))
+	}
+	{
+		loc := &access.Location{
+			Size:     1,
+			BlobSize: 1024,
+			Blobs: []access.SliceInfo{
+				{MinBid: 100, Vid: 1000, Count: 1},
+			},
+		}
+		checker(loc, genTokens(loc))
+	}
+	{
+		loc := &access.Location{
+			Size:     1024,
+			BlobSize: 1024,
+			Blobs: []access.SliceInfo{
+				{MinBid: 100, Vid: 1000, Count: 1},
+			},
+		}
+		checker(loc, genTokens(loc))
+	}
+	{
+		loc := &access.Location{
+			Size:     1025,
+			BlobSize: 1024,
+			Blobs: []access.SliceInfo{
+				{MinBid: 100, Vid: 1000, Count: 2},
+			},
+		}
+		checker(loc, genTokens(loc))
+	}
+	{
+		loc := &access.Location{
+			Size:     2048,
+			BlobSize: 1024,
+			Blobs: []access.SliceInfo{
+				{MinBid: 100, Vid: 1000, Count: 2},
+			},
+		}
+		checker(loc, genTokens(loc))
+	}
+	{
+		loc := &access.Location{
+			Size:     10240,
+			BlobSize: 1024,
+			Blobs: []access.SliceInfo{
+				{MinBid: 100, Vid: 1000, Count: 4},
+				{MinBid: 200, Vid: 1000, Count: 6},
+			},
+		}
+		checker(loc, genTokens(loc))
+	}
+	{
+		loc := &access.Location{
+			Size:     1025,
+			BlobSize: 1024,
+			Blobs: []access.SliceInfo{
+				{MinBid: 100, Vid: 1000, Count: 1},
+				{MinBid: 200, Vid: 1000, Count: 1},
+			},
+		}
+		checker(loc, genTokens(loc))
+	}
+	{
+		loc := &access.Location{
+			Size:     10242,
+			BlobSize: 1024,
+			Blobs: []access.SliceInfo{
+				{MinBid: 100, Vid: 1000, Count: 5},
+				{MinBid: 200, Vid: 1000, Count: 6},
+			},
+		}
+		checker(loc, genTokens(loc))
+	}
+}
+
+func TestAccessServiceLimited(t *testing.T) {
+	host := runMockService(newService())
+	cli := newClient()
+
+	url := func() string {
+		return fmt.Sprintf("%s/alloc", host)
+	}
+	args := access.AllocArgs{Size: 1024}
+	var wg sync.WaitGroup
+	wg.Add(100)
+	for i := 0; i < 100; i++ {
+		go func() {
+			defer wg.Done()
+			resp := &access.AllocResp{}
+			err := cli.PostWith(ctx, url(), resp, args)
+			if err != nil {
+				assertErrorCode(t, errcode.CodeAccessLimited, err)
+			} else {
+				require.Equal(t, uint64(1024), resp.Location.Size)
+			}
+		}()
+	}
+	wg.Wait()
+}

blobstore/access/service.go

@@ -0,0 +1,104 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/cmd"
+	"github.com/cubefs/cubefs/blobstore/common/config"
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+)
+
+var (
+	gConfig  Config
+	gService *Service
+)
+
+func init() {
+	mod := &cmd.Module{
+		Name:       "ACCESS",
+		InitConfig: initConfig,
+		SetUp:      setUp,
+		TearDown:   tearDown,
+	}
+	cmd.RegisterGracefulModule(mod)
+}
+
+func initConfig(args []string) (*cmd.Config, error) {
+	config.Init("f", "", "access.conf")
+	if err := config.Load(&gConfig); err != nil {
+		return nil, err
+	}
+	return &gConfig.Config, nil
+}
+
+func setUp() (*rpc.Router, []rpc.ProgressHandler) {
+	gService = New(gConfig)
+	// register all self functions of service
+	gService.RegisterService()
+	gService.RegisterAdminHandler()
+	return NewHandler(gService), nil
+}
+
+func tearDown() {
+	gService.Close()
+}
+
+// NewHandler returns app server handler
+func NewHandler(service *Service) *rpc.Router {
+	rpc.RegisterArgsParser(&access.PutArgs{}, "json")
+	rpc.RegisterArgsParser(&access.PutAtArgs{}, "json")
+	rpc.RegisterArgsParser(&access.DeleteBlobArgs{}, "json")
+
+	rpc.Use(service.Limit)
+
+	// POST /put?size={size}&hashes={hashes}
+	// request  body:  DataStream
+	// response body:  json
+	rpc.POST("/put", service.Put, rpc.OptArgsQuery())
+	// PUT /put?size={size}&hashes={hashes}
+	rpc.PUT("/put", service.Put, rpc.OptArgsQuery())
+
+	// POST /putat?clusterid={clusterid}&volumeid={volumeid}&blobid={blobid}&size={size}&hashes={hashes}&token={token}
+	// request  body:  DataStream
+	// response body:  json
+	rpc.POST("/putat", service.PutAt, rpc.OptArgsQuery())
+	// PUT /putat?clusterid={clusterid}&volumeid={volumeid}&blobid={blobid}&size={size}&hashes={hashes}&token={token}
+	rpc.PUT("/putat", service.PutAt, rpc.OptArgsQuery())
+
+	// POST /alloc
+	// request  body:  json
+	// response body:  json
+	rpc.POST("/alloc", service.Alloc, rpc.OptArgsBody())
+
+	// POST /get
+	// request  body:  json
+	// response body:  DataStream
+	rpc.POST("/get", service.Get, rpc.OptArgsBody())
+
+	// POST /delete
+	// request  body:  json
+	// response body:  json
+	rpc.POST("/delete", service.Delete, rpc.OptArgsBody())
+	// DELETE /deleteblob
+	rpc.DELETE("/deleteblob", service.DeleteBlob, rpc.OptArgsQuery())
+
+	// POST /sign
+	// request  body:  json
+	// response body:  json
+	rpc.POST("/sign", service.Sign, rpc.OptArgsBody())
+
+	return rpc.DefaultRouter
+}

blobstore/access/stream.go

@@ -0,0 +1,467 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/afex/hystrix-go/hystrix"
+	"github.com/hashicorp/consul/api"
+
+	"github.com/cubefs/cubefs/blobstore/access/controller"
+	"github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/api/blobnode"
+	"github.com/cubefs/cubefs/blobstore/api/proxy"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	"github.com/cubefs/cubefs/blobstore/common/ec"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/resourcepool"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/util/defaulter"
+	"github.com/cubefs/cubefs/blobstore/util/errors"
+	"github.com/cubefs/cubefs/blobstore/util/log"
+	"github.com/cubefs/cubefs/blobstore/util/retry"
+)
+
+const (
+	defaultMaxObjectSize int64 = 5 * (1 << 30) // 5GB
+
+	// hystrix command define
+	allocCommand = "alloc"
+	rwCommand    = "rw"
+
+	serviceProxy = proto.ServiceNameProxy
+)
+
+// StreamHandler stream http handler
+type StreamHandler interface {
+	// Alloc access interface /alloc
+	//     required: size, file size
+	//     optional: blobSize > 0, alloc with blobSize
+	//               assignClusterID > 0, assign to alloc in this cluster certainly
+	//               codeMode > 0, alloc in this codemode
+	//     return: a location of file
+	Alloc(ctx context.Context, size uint64, blobSize uint32,
+		assignClusterID proto.ClusterID, codeMode codemode.CodeMode) (*access.Location, error)
+
+	// PutAt access interface /putat, put one blob
+	//     required: rc file reader
+	//     required: clusterID VolumeID BlobID
+	//     required: size, one blob size
+	//     optional: hasherMap, computing hash
+	PutAt(ctx context.Context, rc io.Reader,
+		clusterID proto.ClusterID, vid proto.Vid, bid proto.BlobID, size int64, hasherMap access.HasherMap) error
+
+	// Put put one object
+	//     required: size, file size
+	//     optional: hasher map to calculate hash.Hash
+	Put(ctx context.Context, rc io.Reader, size int64, hasherMap access.HasherMap) (*access.Location, error)
+
+	// Get read file
+	//     required: location, readSize
+	//     optional: offset(default is 0)
+	//
+	//     first return value is data transfer to copy data after argument checking
+	//
+	//  Read data shards firstly, if blob size is small or read few bytes
+	//  then ec reconstruct-read, try to reconstruct from N+X to N+M
+	//
+	//  sorted N+X is, such as we use mode EC6P10L2, X=2 and Read from idc=2
+	//  shards like this
+	//              data N 6        |    parity M 10     | local L 2
+	//        d1  d2  d3  d4  d5  d6  p1 .. p5  p6 .. p10  l1  l2
+	//   idc   1   1   1   2   2   2     1         2        1   2
+	//
+	//sorted  d4  d5  d6  p6 .. p10  d1  d2  d3  p1 .. p5
+	//read-1 [d4                p10]
+	//read-2 [d4                p10  d1]
+	//read-3 [d4                p10  d1  d2]
+	//...
+	//read-9 [d4                                       p5]
+	//failed
+	Get(ctx context.Context, w io.Writer, location access.Location, readSize, offset uint64) (func() error, error)
+
+	// Delete delete all blobs in this location
+	Delete(ctx context.Context, location *access.Location) error
+
+	// Admin returns internal admin interface.
+	Admin() interface{}
+}
+
+type streamAdmin struct {
+	config     StreamConfig
+	memPool    *resourcepool.MemPool
+	controller controller.ClusterController
+}
+
+// StreamConfig access stream handler config
+type StreamConfig struct {
+	IDC string `json:"idc"`
+
+	MaxBlobSize                uint32 `json:"max_blob_size"`
+	DiskPunishIntervalS        int    `json:"disk_punish_interval_s"`
+	DiskTimeoutPunishIntervalS int    `json:"disk_timeout_punish_interval_s"`
+	ServicePunishIntervalS     int    `json:"service_punish_interval_s"`
+	AllocRetryTimes            int    `json:"alloc_retry_times"`
+	AllocRetryIntervalMS       int    `json:"alloc_retry_interval_ms"`
+	EncoderEnableVerify        bool   `json:"encoder_enableverify"`
+	EncoderConcurrency         int    `json:"encoder_concurrency"`
+	MinReadShardsX             int    `json:"min_read_shards_x"`
+	ShardCrcDisabled           bool   `json:"shard_crc_disabled"`
+
+	MemPoolSizeClasses map[int]int `json:"mem_pool_size_classes"`
+
+	// CodeModesPutQuorums
+	// just for one AZ is down, cant write quorum in all AZs
+	CodeModesPutQuorums map[codemode.CodeMode]int `json:"code_mode_put_quorums"`
+
+	ClusterConfig  controller.ClusterConfig `json:"cluster_config"`
+	BlobnodeConfig blobnode.Config          `json:"blobnode_config"`
+	ProxyConfig    proxy.Config             `json:"proxy_config"`
+
+	// hystrix command config
+	AllocCommandConfig hystrix.CommandConfig `json:"alloc_command_config"`
+	RWCommandConfig    hystrix.CommandConfig `json:"rw_command_config"`
+}
+
+// discard unhealthy volume
+type discardVid struct {
+	cid      proto.ClusterID
+	codeMode codemode.CodeMode
+	vid      proto.Vid
+}
+
+type blobIdent struct {
+	cid proto.ClusterID
+	vid proto.Vid
+	bid proto.BlobID
+}
+
+func (id *blobIdent) String() string {
+	return fmt.Sprintf("blob(%d %d %d)", id.cid, id.vid, id.bid)
+}
+
+// Handler stream handler
+type Handler struct {
+	memPool           *resourcepool.MemPool
+	encoder           map[codemode.CodeMode]ec.Encoder
+	clusterController controller.ClusterController
+
+	blobnodeClient blobnode.StorageAPI
+	proxyClient    proxy.Client
+
+	allCodeModes  CodeModePairs
+	maxObjectSize int64
+
+	discardVidChan chan discardVid
+	stopCh         <-chan struct{}
+
+	StreamConfig
+}
+
+func confCheck(cfg *StreamConfig) {
+	if cfg.IDC == "" {
+		log.Fatal("idc config can not be null")
+	}
+	cfg.ClusterConfig.IDC = cfg.IDC
+
+	if len(cfg.MemPoolSizeClasses) == 0 {
+		cfg.MemPoolSizeClasses = getDefaultMempoolSize()
+	}
+
+	for mode, quorum := range cfg.CodeModesPutQuorums {
+		tactic := mode.Tactic()
+		if quorum < tactic.N+tactic.L+1 || quorum > mode.GetShardNum() {
+			log.Fatalf("invalid put quorum(%d) in codemode(%d): %+v", quorum, mode, tactic)
+		}
+	}
+
+	defaulter.Equal(&cfg.MaxBlobSize, defaultMaxBlobSize)
+	defaulter.LessOrEqual(&cfg.DiskPunishIntervalS, defaultDiskPunishIntervalS)
+	defaulter.LessOrEqual(&cfg.DiskTimeoutPunishIntervalS, defaultDiskPunishIntervalS/10)
+	defaulter.LessOrEqual(&cfg.ServicePunishIntervalS, defaultServicePunishIntervalS)
+	defaulter.LessOrEqual(&cfg.AllocRetryTimes, defaultAllocRetryTimes)
+	if cfg.AllocRetryIntervalMS <= 100 {
+		cfg.AllocRetryIntervalMS = defaultAllocRetryIntervalMS
+	}
+	defaulter.LessOrEqual(&cfg.EncoderConcurrency, defaultEncoderConcurrency)
+	defaulter.LessOrEqual(&cfg.MinReadShardsX, defaultMinReadShardsX)
+
+	defaulter.LessOrEqual(&cfg.ClusterConfig.CMClientConfig.Config.ClientTimeoutMs, defaultTimeoutClusterMgr)
+	defaulter.LessOrEqual(&cfg.BlobnodeConfig.ClientTimeoutMs, defaultTimeoutBlobnode)
+	defaulter.LessOrEqual(&cfg.ProxyConfig.ClientTimeoutMs, defaultTimeoutProxy)
+
+	hc := cfg.AllocCommandConfig
+	defaulter.LessOrEqual(&hc.Timeout, defaultAllocatorTimeout)
+	defaulter.LessOrEqual(&hc.MaxConcurrentRequests, defaultAllocatorMaxConcurrentRequests)
+	defaulter.LessOrEqual(&hc.RequestVolumeThreshold, defaultAllocatorRequestVolumeThreshold)
+	defaulter.LessOrEqual(&hc.SleepWindow, defaultAllocatorSleepWindow)
+	defaulter.LessOrEqual(&hc.ErrorPercentThreshold, defaultAllocatorErrorPercentThreshold)
+	cfg.AllocCommandConfig = hc
+
+	hc = cfg.RWCommandConfig
+	defaulter.LessOrEqual(&hc.Timeout, defaultBlobnodeTimeout)
+	defaulter.LessOrEqual(&hc.MaxConcurrentRequests, defaultBlobnodeMaxConcurrentRequests)
+	defaulter.LessOrEqual(&hc.RequestVolumeThreshold, defaultBlobnodeRequestVolumeThreshold)
+	defaulter.LessOrEqual(&hc.SleepWindow, defaultBlobnodeSleepWindow)
+	defaulter.LessOrEqual(&hc.ErrorPercentThreshold, defaultBlobnodeErrorPercentThreshold)
+	cfg.RWCommandConfig = hc
+}
+
+// NewStreamHandler returns a stream handler
+func NewStreamHandler(cfg *StreamConfig, kvClient *api.Client, stopCh <-chan struct{}) StreamHandler {
+	confCheck(cfg)
+
+	clusterController, err := controller.NewClusterController(&cfg.ClusterConfig, kvClient)
+	if err != nil {
+		log.Fatalf("new cluster controller failed, err: %v", err)
+	}
+
+	handler := &Handler{
+		memPool:           resourcepool.NewMemPool(cfg.MemPoolSizeClasses),
+		clusterController: clusterController,
+
+		blobnodeClient: blobnode.New(&cfg.BlobnodeConfig),
+		proxyClient:    proxy.New(&cfg.ProxyConfig),
+
+		maxObjectSize: defaultMaxObjectSize,
+		StreamConfig:  *cfg,
+	}
+
+	rawCodeModePolicies, err := handler.clusterController.GetConfig(context.Background(), proto.CodeModeConfigKey)
+	if err != nil {
+		log.Fatal("get codemode policy from cluster manager failed, err: ", err)
+	}
+	codeModePolicies := make([]codemode.Policy, 0)
+	err = json.Unmarshal([]byte(rawCodeModePolicies), &codeModePolicies)
+	if err != nil {
+		log.Fatal("json decode codemode policy failed, err: ", err)
+	}
+	if len(codeModePolicies) <= 0 {
+		log.Fatal("invalid codemode policy raw: ", rawCodeModePolicies)
+	}
+
+	allCodeModes := make(CodeModePairs)
+	encoders := make(map[codemode.CodeMode]ec.Encoder)
+	maxSize := int64(0)
+	for _, policy := range codeModePolicies {
+		if policy.MaxSize > maxSize {
+			maxSize = policy.MaxSize
+		}
+		codeMode := policy.ModeName.GetCodeMode()
+		tactic := codeMode.Tactic()
+		allCodeModes[codeMode] = CodeModePair{
+			Policy: policy,
+			Tactic: tactic,
+		}
+		encoder, err := ec.NewEncoder(ec.Config{
+			CodeMode:     tactic,
+			EnableVerify: cfg.EncoderEnableVerify,
+			Concurrency:  cfg.EncoderConcurrency,
+		})
+		if err != nil {
+			log.Fatalf("new encoder failed, err: %v", err)
+		}
+		encoders[codeMode] = encoder
+	}
+	handler.allCodeModes = allCodeModes
+	handler.encoder = encoders
+	if maxSize < handler.maxObjectSize {
+		handler.maxObjectSize = maxSize
+	}
+
+	hystrix.ConfigureCommand(allocCommand, cfg.AllocCommandConfig)
+	hystrix.ConfigureCommand(rwCommand, cfg.RWCommandConfig)
+
+	handler.discardVidChan = make(chan discardVid, 8)
+	handler.stopCh = stopCh
+	handler.loopDiscardVids()
+	return handler
+}
+
+// Delete delete all blobs in this location
+func (h *Handler) Delete(ctx context.Context, location *access.Location) error {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Debugf("to delete %+v", location)
+	return h.clearGarbage(ctx, location)
+}
+
+// Admin returns internal admin interface.
+func (h *Handler) Admin() interface{} {
+	return &streamAdmin{
+		config:     h.StreamConfig,
+		memPool:    h.memPool,
+		controller: h.clusterController,
+	}
+}
+
+func (h *Handler) sendRepairMsgBg(ctx context.Context, blob blobIdent, badIdxes []uint8) {
+	go func() {
+		h.sendRepairMsg(ctx, blob, badIdxes)
+	}()
+}
+
+func (h *Handler) sendRepairMsg(ctx context.Context, blob blobIdent, badIdxes []uint8) {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Infof("to repair %s indexes(%+v)", blob.String(), badIdxes)
+
+	clusterID := blob.cid
+	serviceController, err := h.clusterController.GetServiceController(clusterID)
+	if err != nil {
+		span.Error(errors.Detail(err))
+		return
+	}
+	reportUnhealth(clusterID, "repair.msg", "-", "-", "-")
+
+	repairArgs := &proxy.ShardRepairArgs{
+		ClusterID: clusterID,
+		Bid:       blob.bid,
+		Vid:       blob.vid,
+		BadIdxes:  badIdxes[:],
+		Reason:    "access-repair",
+	}
+
+	if err := retry.Timed(3, 200).On(func() error {
+		host, err := serviceController.GetServiceHost(ctx, serviceProxy)
+		if err != nil {
+			span.Warn(err)
+			return err
+		}
+		err = h.proxyClient.SendShardRepairMsg(ctx, host, repairArgs)
+		if err != nil {
+			span.Warnf("send to %s repair message(%+v) %s", host, repairArgs, err.Error())
+			serviceController.PunishServiceWithThreshold(ctx, serviceProxy, host, h.ServicePunishIntervalS)
+			reportUnhealth(clusterID, "punish", serviceProxy, host, "failed")
+			err = errors.Base(err, host)
+		}
+		return err
+	}); err != nil {
+		reportUnhealth(clusterID, "repair.msg", serviceProxy, "-", "failed")
+		span.Errorf("send repair message(%+v) failed %s", repairArgs, errors.Detail(err))
+		return
+	}
+
+	span.Infof("send repair message(%+v)", repairArgs)
+}
+
+func (h *Handler) clearGarbage(ctx context.Context, location *access.Location) error {
+	span := trace.SpanFromContextSafe(ctx)
+	serviceController, err := h.clusterController.GetServiceController(location.ClusterID)
+	if err != nil {
+		span.Error(errors.Detail(err))
+		return errors.Base(err, "clear location:", *location)
+	}
+
+	blobs := location.Spread()
+	deleteArgs := &proxy.DeleteArgs{
+		ClusterID: location.ClusterID,
+		Blobs:     make([]proxy.BlobDelete, 0, len(blobs)),
+	}
+
+	for _, blob := range blobs {
+		deleteArgs.Blobs = append(deleteArgs.Blobs, proxy.BlobDelete{
+			Bid: blob.Bid,
+			Vid: blob.Vid,
+		})
+	}
+
+	var logMsg interface{} = location
+	if len(deleteArgs.Blobs) <= 20 {
+		logMsg = deleteArgs
+	}
+	if err := retry.Timed(3, 200).On(func() error {
+		host, err := serviceController.GetServiceHost(ctx, serviceProxy)
+		if err != nil {
+			span.Warn(err)
+			return err
+		}
+		err = h.proxyClient.SendDeleteMsg(ctx, host, deleteArgs)
+		if err != nil {
+			span.Warnf("send to %s delete message(%+v) %s", host, logMsg, err.Error())
+			serviceController.PunishServiceWithThreshold(ctx, serviceProxy, host, h.ServicePunishIntervalS)
+			reportUnhealth(location.ClusterID, "punish", serviceProxy, host, "failed")
+			err = errors.Base(err, host)
+		}
+		return err
+	}); err != nil {
+		reportUnhealth(location.ClusterID, "delete.msg", serviceProxy, "-", "failed")
+		span.Errorf("send delete message(%+v) failed %s", logMsg, errors.Detail(err))
+		return errors.Base(err, "send delete message:", logMsg)
+	}
+
+	span.Infof("send delete message(%+v)", logMsg)
+	return nil
+}
+
+// getVolume get volume info
+func (h *Handler) getVolume(ctx context.Context, clusterID proto.ClusterID, vid proto.Vid, isCache bool) (*controller.VolumePhy, error) {
+	volumeGetter, err := h.clusterController.GetVolumeGetter(clusterID)
+	if err != nil {
+		return nil, err
+	}
+
+	volume := volumeGetter.Get(ctx, vid, isCache)
+	if volume == nil {
+		return nil, errors.Newf("not found volume of (%d %d)", clusterID, vid)
+	}
+
+	return volume, nil
+}
+
+func (h *Handler) punishVolume(ctx context.Context, clusterID proto.ClusterID, vid proto.Vid, host, reason string) {
+	reportUnhealth(clusterID, "punish", "volume", host, reason)
+	if volumeGetter, err := h.clusterController.GetVolumeGetter(clusterID); err == nil {
+		volumeGetter.Punish(ctx, vid, h.DiskPunishIntervalS)
+	}
+}
+
+func (h *Handler) punishDisk(ctx context.Context, clusterID proto.ClusterID, diskID proto.DiskID, host, reason string) {
+	reportUnhealth(clusterID, "punish", "disk", host, reason)
+	if serviceController, err := h.clusterController.GetServiceController(clusterID); err == nil {
+		serviceController.PunishDisk(ctx, diskID, h.DiskPunishIntervalS)
+	}
+}
+
+func (h *Handler) punishDiskWith(ctx context.Context, clusterID proto.ClusterID, diskID proto.DiskID, host, reason string) {
+	reportUnhealth(clusterID, "punish", "diskwith", host, reason)
+	if serviceController, err := h.clusterController.GetServiceController(clusterID); err == nil {
+		serviceController.PunishDiskWithThreshold(ctx, diskID, h.DiskTimeoutPunishIntervalS)
+	}
+}
+
+// blobCount blobSize > 0 is certain
+func blobCount(size uint64, blobSize uint32) uint64 {
+	return (size + uint64(blobSize) - 1) / uint64(blobSize)
+}
+
+func minU64(a, b uint64) uint64 {
+	if a <= b {
+		return a
+	}
+	return b
+}
+
+func errorTimeout(err error) bool {
+	msg := err.Error()
+	return strings.Contains(msg, "Timeout") || strings.Contains(msg, "timeout")
+}
+
+func errorConnectionRefused(err error) bool {
+	return strings.Contains(err.Error(), "connection refused")
+}

blobstore/access/stream_alloc.go

@@ -0,0 +1,206 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"context"
+	"sync/atomic"
+
+	"github.com/afex/hystrix-go/hystrix"
+	"github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/api/proxy"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/util/errors"
+	"github.com/cubefs/cubefs/blobstore/util/retry"
+)
+
+var errAllocatePunishedVolume = errors.New("allocate punished volume")
+
+// Alloc access interface /alloc
+//     required: size, file size
+//     optional: blobSize > 0, alloc with blobSize
+//               assignClusterID > 0, assign to alloc in this cluster certainly
+//               codeMode > 0, alloc in this codemode
+//     return: a location of file
+func (h *Handler) Alloc(ctx context.Context, size uint64, blobSize uint32,
+	assignClusterID proto.ClusterID, codeMode codemode.CodeMode) (*access.Location, error) {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Debugf("alloc request with size:%d blobsize:%d cluster:%d codemode:%d",
+		size, blobSize, assignClusterID, codeMode)
+
+	if int64(size) > h.maxObjectSize {
+		span.Info("exceed max object size", h.maxObjectSize)
+		return nil, errcode.ErrAccessExceedSize
+	}
+
+	if blobSize == 0 {
+		blobSize = atomic.LoadUint32(&h.MaxBlobSize)
+		span.Debugf("fill blobsize:%d", blobSize)
+	}
+
+	if codeMode == 0 {
+		codeMode = h.allCodeModes.SelectCodeMode(int64(size))
+		span.Debugf("select codemode:%d", codeMode)
+	}
+	if !codeMode.IsValid() {
+		span.Infof("invalid codemode:%d", codeMode)
+		return nil, errcode.ErrIllegalArguments
+	}
+
+	clusterID, blobs, err := h.allocFromAllocatorWithHystrix(ctx, codeMode, size, blobSize, assignClusterID)
+	if err != nil {
+		span.Error("alloc from proxy", errors.Detail(err))
+		return nil, err
+	}
+	span.Debugf("allocated from %d %+v", clusterID, blobs)
+
+	location := &access.Location{
+		ClusterID: clusterID,
+		CodeMode:  codeMode,
+		Size:      size,
+		BlobSize:  blobSize,
+		Blobs:     blobs,
+	}
+	span.Debugf("alloc ok %+v", location)
+	return location, nil
+}
+
+func (h *Handler) allocFromAllocatorWithHystrix(ctx context.Context, codeMode codemode.CodeMode, size uint64, blobSize uint32,
+	clusterID proto.ClusterID) (cid proto.ClusterID, bidRets []access.SliceInfo, err error) {
+	err = hystrix.Do(allocCommand, func() error {
+		cid, bidRets, err = h.allocFromAllocator(ctx, codeMode, size, blobSize, clusterID)
+		return err
+	}, nil)
+	return
+}
+
+func (h *Handler) allocFromAllocator(ctx context.Context, codeMode codemode.CodeMode, size uint64, blobSize uint32,
+	clusterID proto.ClusterID) (proto.ClusterID, []access.SliceInfo, error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	if blobSize == 0 {
+		blobSize = atomic.LoadUint32(&h.MaxBlobSize)
+	}
+	if clusterID == 0 {
+		clusterChosen, err := h.clusterController.ChooseOne()
+		if err != nil {
+			return 0, nil, err
+		}
+		clusterID = clusterChosen.ClusterID
+	}
+
+	args := proxy.AllocVolsArgs{
+		Fsize:    size,
+		CodeMode: codeMode,
+		BidCount: blobCount(size, blobSize),
+	}
+
+	var allocRets []proxy.AllocRet
+	var allocHost string
+	hostsSet := make(map[string]struct{}, 1)
+	if err := retry.ExponentialBackoff(h.AllocRetryTimes, uint32(h.AllocRetryIntervalMS)).On(func() error {
+		serviceController, err := h.clusterController.GetServiceController(clusterID)
+		if err != nil {
+			span.Warn(err)
+			return errors.Info(err, "get service controller", clusterID)
+		}
+
+		var host string
+		for range [10]struct{}{} {
+			host, err = serviceController.GetServiceHost(ctx, serviceProxy)
+			if err != nil {
+				span.Warn(err)
+				return errors.Info(err, "get proxy host", clusterID)
+			}
+			if _, ok := hostsSet[host]; ok {
+				continue
+			}
+			hostsSet[host] = struct{}{}
+			break
+		}
+		allocHost = host
+
+		allocRets, err = h.proxyClient.VolumeAlloc(ctx, host, &args)
+		if err != nil {
+			if errorTimeout(err) || errorConnectionRefused(err) {
+				span.Info("punish unreachable proxy host:", host)
+				reportUnhealth(clusterID, "punish", serviceProxy, host, "Timeout")
+				serviceController.PunishServiceWithThreshold(ctx, serviceProxy, host, h.ServicePunishIntervalS)
+			}
+			span.Warn(host, err)
+			return errors.Base(err, "alloc from proxy", host)
+		}
+
+		// filter punished volume in allocating progress
+		for _, ret := range allocRets {
+			vInfo, err := h.getVolume(ctx, clusterID, ret.Vid, true)
+			if err != nil {
+				span.Warn(err)
+				return err
+			}
+			if vInfo.IsPunish {
+				// return err and retry allocate
+				err = errAllocatePunishedVolume
+				args.Excludes = append(args.Excludes, vInfo.Vid)
+				span.Warn("next retry exclude vid:", vInfo.Vid, err)
+				return err
+			}
+		}
+
+		return nil
+	}); err != nil {
+		if err != errAllocatePunishedVolume {
+			reportUnhealth(clusterID, "allocate", "-", "-", "failed")
+			return 0, nil, err
+		}
+		// still write to storage if allocating punished volume
+		reportUnhealth(clusterID, "allocate", "-", "-", "punished")
+	}
+
+	// cache vid in which allocator
+	for _, ret := range allocRets {
+		setCacheVidHost(clusterID, ret.Vid, allocHost)
+	}
+
+	blobN := blobCount(size, blobSize)
+	blobs := make([]access.SliceInfo, 0, blobN)
+	for _, bidRet := range allocRets {
+		if blobN <= 0 {
+			break
+		}
+
+		count := minU64(blobN, uint64(bidRet.BidEnd)-uint64(bidRet.BidStart)+1)
+		blobN -= count
+
+		blobs = append(blobs, access.SliceInfo{
+			MinBid: bidRet.BidStart,
+			Vid:    bidRet.Vid,
+			Count:  uint32(count),
+		})
+	}
+	if blobN > 0 {
+		return 0, nil, errors.New("no enough blob ids from allocator")
+	}
+
+	if uint32(len(blobs)) > access.MaxLocationBlobs {
+		span.Errorf("alloc exceed max blobs %d>%d", len(blobs), access.MaxLocationBlobs)
+		return 0, nil, errors.New("alloc exceed max blobs of location")
+	}
+
+	return clusterID, blobs, nil
+}

blobstore/access/stream_alloc_test.go

@@ -0,0 +1,70 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+)
+
+func TestAccessStreamAllocBase(t *testing.T) {
+	ctx := ctxWithName("TestAccessStreamAllocBase")
+	// 4M blobsize
+	{
+		loc, err := streamer.Alloc(ctx(), 1<<30, 0, 0, 0)
+		require.NoError(t, err)
+		require.Equal(t, clusterID, loc.ClusterID)
+		require.Equal(t, codemode.EC6P6, loc.CodeMode)
+		require.Equal(t, uint64(1<<30), loc.Size)
+		require.Equal(t, uint32(1<<22), loc.BlobSize)
+		require.Equal(t, 2, len(loc.Blobs))
+		require.Equal(t, uint32(1), loc.Blobs[0].Count)
+		require.Equal(t, uint32((1<<8)-1), loc.Blobs[1].Count)
+	}
+	{
+		loc, err := streamer.Alloc(ctx(), (1<<30)+1, 0, 0, 0)
+		require.NoError(t, err)
+		require.Equal(t, 2, len(loc.Blobs))
+		require.Equal(t, uint32(1), loc.Blobs[0].Count)
+		require.Equal(t, uint32(1<<8), loc.Blobs[1].Count)
+	}
+	// 1M blobsize
+	{
+		loc, err := streamer.Alloc(ctx(), 1<<30, 1<<20, 0, 0)
+		require.NoError(t, err)
+		require.Equal(t, 2, len(loc.Blobs))
+		require.Equal(t, uint32(1), loc.Blobs[0].Count)
+		require.Equal(t, uint32((1<<10)-1), loc.Blobs[1].Count)
+	}
+	// max size + 1
+	{
+		_, err := streamer.Alloc(ctx(), uint64(defaultMaxObjectSize+1), 1<<20, 0, 0)
+		require.EqualError(t, errcode.ErrAccessExceedSize, err.Error())
+	}
+
+	{
+		// wait for service manager to reload
+		defer func() {
+			time.Sleep(time.Second * 2)
+		}()
+		_, err := streamer.Alloc(ctx(), allocTimeoutSize+1, 0, 0, 0)
+		require.Error(t, err)
+	}
+}

blobstore/access/stream_get.go

@@ -0,0 +1,823 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math/rand"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/afex/hystrix-go/hystrix"
+
+	"github.com/cubefs/cubefs/blobstore/access/controller"
+	"github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/api/blobnode"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	"github.com/cubefs/cubefs/blobstore/common/ec"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/util/errors"
+	"github.com/cubefs/cubefs/blobstore/util/retry"
+)
+
+var (
+	errNeedReconstructRead = errors.New("need to reconstruct read")
+	errCanceledReadShard   = errors.New("canceled read shard")
+	errPunishedDisk        = errors.New("punished disk")
+)
+
+type blobGetArgs struct {
+	Vid      proto.Vid
+	Bid      proto.BlobID
+	BlobSize uint64
+	Offset   uint64
+	ReadSize uint64
+}
+
+type shardData struct {
+	index  int
+	status bool
+	buffer []byte
+}
+
+type sortedVuid struct {
+	index  int
+	vuid   proto.Vuid
+	diskID proto.DiskID
+	host   string
+}
+
+type pipeBuffer struct {
+	err    error
+	blob   blobGetArgs
+	shards [][]byte
+}
+
+// Get read file
+//     required: location, readSize
+//     optional: offset(default is 0)
+//
+//     first return value is data transfer to copy data after argument checking
+//
+//  Read data shards firstly, if blob size is small or read few bytes
+//  then ec reconstruct-read, try to reconstruct from N+X to N+M
+//
+//  sorted N+X is, such as we use mode EC6P10L2, X=2 and Read from idc=2
+//  shards like this
+//              data N 6        |    parity M 10     | local L 2
+//        d1  d2  d3  d4  d5  d6  p1 .. p5  p6 .. p10  l1  l2
+//   idc   1   1   1   2   2   2     1         2        1   2
+//
+//sorted  d4  d5  d6  p6 .. p10  d1  d2  d3  p1 .. p5
+//read-1 [d4                p10]
+//read-2 [d4                p10  d1]
+//read-3 [d4                p10  d1  d2]
+//...
+//read-9 [d4                                       p5]
+//failed
+func (h *Handler) Get(ctx context.Context, w io.Writer, location access.Location, readSize, offset uint64) (func() error, error) {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Debugf("get request cluster:%d size:%d offset:%d", location.ClusterID, readSize, offset)
+
+	blobs, err := genLocationBlobs(&location, readSize, offset)
+	if err != nil {
+		span.Info("illegal argument", err)
+		return func() error { return nil }, errcode.ErrIllegalArguments
+	}
+	if len(blobs) == 0 {
+		return func() error { return nil }, nil
+	}
+
+	clusterID := location.ClusterID
+	var serviceController controller.ServiceController
+	if err = retry.Timed(3, 200).On(func() error {
+		sc, err := h.clusterController.GetServiceController(clusterID)
+		if err != nil {
+			return err
+		}
+		serviceController = sc
+		return nil
+	}); err != nil {
+		span.Error("get service", errors.Detail(err))
+		return func() error { return nil }, err
+	}
+
+	return func() error {
+		getTime := new(timeReadWrite)
+		defer func() {
+			span.AppendRPCTrackLog([]string{getTime.String()})
+		}()
+
+		// try to read data shard only,
+		//   if blobsize is small: all data is in the first shard, cos shards aligned by MinShardSize.
+		//   read few bytes: read bytes less than quarter of blobsize, like Range:[0-1].
+		if len(blobs) == 1 {
+			blob := blobs[0]
+			sizes, _ := ec.GetBufferSizes(int(blob.BlobSize), location.CodeMode.Tactic())
+			if int(blob.BlobSize) <= sizes.ShardSize || blob.ReadSize < blob.BlobSize/4 {
+				span.Debugf("read data shard only readsize:%d blobsize:%d shardsize:%d",
+					blob.ReadSize, blob.BlobSize, sizes.ShardSize)
+
+				reportDownload(clusterID, "Range", "-")
+				err := h.getDataShardOnly(ctx, getTime, w, serviceController, clusterID, blob)
+				if err != errNeedReconstructRead {
+					if err != nil {
+						span.Error("read data shard only", err)
+						reportDownload(clusterID, "Range", err.Error())
+					}
+					return err
+				}
+				reportDownload(clusterID, "Range", err.Error())
+				span.Info("read data shard only failed", err)
+			}
+		}
+
+		// data stream flow:
+		// client <--copy-- pipeline <--swap-- readBlob <--copy-- blobnode
+		//
+		// Alloc N+M shard buffers here, and release after written to client.
+		// Replace not-empty buffers in readBlob, need release old-buffers in that function.
+		closeCh := make(chan struct{})
+		pipeline := func() <-chan pipeBuffer {
+			ch := make(chan pipeBuffer, 1)
+			go func() {
+				defer close(ch)
+
+				var (
+					blobVolume  *controller.VolumePhy
+					sortedVuids []sortedVuid
+				)
+				for _, blob := range blobs {
+					var err error
+					if blobVolume == nil || blobVolume.Vid != blob.Vid {
+						blobVolume, err = h.getVolume(ctx, clusterID, blob.Vid, true)
+						if err != nil {
+							span.Error("get volume", err)
+							ch <- pipeBuffer{err: err}
+							return
+						}
+
+						tactic := blobVolume.CodeMode.Tactic()
+						// do not use local shards
+						sortedVuids = genSortedVuidByIDC(ctx, serviceController, h.IDC, blobVolume.Units[:tactic.N+tactic.M])
+						span.Debugf("to read blob(%d %d %d) with read-shard-x:%d active-shard-n:%d of data-n:%d party-n:%d",
+							clusterID, blob.Vid, blob.Bid, h.MinReadShardsX, len(sortedVuids), tactic.N, tactic.M)
+						if len(sortedVuids) < tactic.N {
+							err = fmt.Errorf("broken blob(%d %d %d)", clusterID, blob.Vid, blob.Bid)
+							span.Error(err)
+							ch <- pipeBuffer{err: err}
+							return
+						}
+					}
+
+					codeMode := blobVolume.CodeMode
+					tactic := codeMode.Tactic()
+					sizes, _ := ec.GetBufferSizes(int(blob.BlobSize), tactic)
+					shardSize := sizes.ShardSize
+
+					st := time.Now()
+					shards := make([][]byte, tactic.N+tactic.M)
+					for ii := range shards {
+						buf, _ := h.memPool.Alloc(shardSize)
+						shards[ii] = buf
+					}
+					getTime.IncA(time.Since(st))
+
+					err = h.readOneBlob(ctx, getTime, serviceController, clusterID,
+						blobVolume.Vid, codeMode, blob, sortedVuids, shards)
+					if err != nil {
+						span.Error("read one blob", blob.Bid, err)
+						for _, buf := range shards {
+							h.memPool.Put(buf)
+						}
+						ch <- pipeBuffer{err: err}
+						return
+					}
+
+					select {
+					case <-closeCh:
+						for _, buf := range shards {
+							h.memPool.Put(buf)
+						}
+						return
+					case ch <- pipeBuffer{blob: blob, shards: shards}:
+					}
+				}
+			}()
+
+			return ch
+		}()
+
+		var err error
+		reportDownload(clusterID, "EC", "-")
+		for line := range pipeline {
+			if line.err != nil {
+				err = line.err
+				break
+			}
+
+			startWrite := time.Now()
+
+			idx := 0
+			off := line.blob.Offset
+			toReadSize := line.blob.ReadSize
+			for toReadSize > 0 {
+				buf := line.shards[idx]
+				l := uint64(len(buf))
+				if off >= l {
+					idx++
+					off -= l
+					continue
+				}
+
+				toRead := minU64(toReadSize, l-off)
+				if _, e := w.Write(buf[off : off+toRead]); e != nil {
+					err = errors.Info(e, "write to response")
+					break
+				}
+				idx++
+				off = 0
+				toReadSize -= toRead
+			}
+
+			getTime.IncW(time.Since(startWrite))
+
+			for _, buf := range line.shards {
+				h.memPool.Put(buf)
+			}
+			if err != nil {
+				close(closeCh)
+				break
+			}
+		}
+
+		// release buffer in pipeline if fail to write client
+		go func() {
+			for line := range pipeline {
+				for _, buf := range line.shards {
+					h.memPool.Put(buf)
+				}
+			}
+		}()
+
+		if err != nil {
+			span.Error("get request error", err)
+			return err
+		}
+		return nil
+	}, nil
+}
+
+// 1. try to min-read shards bytes
+// 2. if failed try to read next shard to reconstruct
+// 3. write the the right offset bytes to writer
+func (h *Handler) readOneBlob(ctx context.Context, getTime *timeReadWrite,
+	serviceController controller.ServiceController,
+	clusterID proto.ClusterID, vid proto.Vid, codeMode codemode.CodeMode,
+	blob blobGetArgs, sortedVuids []sortedVuid, shards [][]byte) error {
+	span := trace.SpanFromContextSafe(ctx)
+
+	tactic := codeMode.Tactic()
+	sizes, err := ec.GetBufferSizes(int(blob.BlobSize), tactic)
+	if err != nil {
+		return err
+	}
+	empties := emptyDataShardIndexes(sizes)
+
+	dataN, dataParityN := tactic.N, tactic.N+tactic.M
+	minShardsRead := dataN + h.MinReadShardsX
+	if minShardsRead > len(sortedVuids) {
+		minShardsRead = len(sortedVuids)
+	}
+	shardSize := len(shards[0])
+
+	stopChan := make(chan struct{})
+	nextChan := make(chan struct{}, len(sortedVuids))
+
+	shardPipe := func() <-chan shardData {
+		ch := make(chan shardData)
+		go func() {
+			wg := new(sync.WaitGroup)
+			defer func() {
+				wg.Wait()
+				close(ch)
+			}()
+
+			for _, vuid := range sortedVuids[:minShardsRead] {
+				if _, ok := empties[vuid.index]; !ok {
+					wg.Add(1)
+					go func(vuid sortedVuid) {
+						ch <- h.readOneShard(ctx, serviceController, clusterID, vid,
+							shardSize, blob, vuid, stopChan)
+						wg.Done()
+					}(vuid)
+				}
+			}
+
+			for _, vuid := range sortedVuids[minShardsRead:] {
+				if _, ok := empties[vuid.index]; ok {
+					continue
+				}
+
+				select {
+				case <-stopChan:
+					return
+				case <-nextChan:
+				}
+
+				wg.Add(1)
+				go func(vuid sortedVuid) {
+					ch <- h.readOneShard(ctx, serviceController, clusterID, vid,
+						shardSize, blob, vuid, stopChan)
+					wg.Done()
+				}(vuid)
+			}
+		}()
+
+		return ch
+	}()
+
+	received := make(map[int]bool, minShardsRead)
+	for idx := range empties {
+		received[idx] = true
+		h.memPool.Zero(shards[idx])
+	}
+
+	startRead := time.Now()
+	reconstructed := false
+	for shard := range shardPipe {
+		// swap shard buffer
+		if shard.status {
+			buf := shards[shard.index]
+			shards[shard.index] = shard.buffer
+			h.memPool.Put(buf)
+		}
+
+		received[shard.index] = shard.status
+		if len(received) < dataN {
+			continue
+		}
+
+		// bad data index
+		badIdx := make([]int, 0, 8)
+		for i := 0; i < dataN; i++ {
+			if succ, ok := received[i]; !ok || !succ {
+				badIdx = append(badIdx, i)
+			}
+		}
+		if len(badIdx) == 0 {
+			reconstructed = true
+			close(stopChan)
+			break
+		}
+
+		// update bad parity index
+		for i := dataN; i < dataParityN; i++ {
+			if succ, ok := received[i]; !ok || !succ {
+				badIdx = append(badIdx, i)
+			}
+		}
+
+		badShards := 0
+		for _, succ := range received {
+			if !succ {
+				badShards++
+			}
+		}
+		// it will not wait all the shards, cos has no enough shards to reconstruct
+		if badShards > dataParityN-dataN {
+			span.Infof("bid(%d) bad(%d) has no enough to reconstruct", blob.Bid, badShards)
+			close(stopChan)
+			break
+		}
+
+		// has bad shards, but have enough shards to reconstruct
+		if len(received) >= dataN+badShards {
+			span.Debugf("bid(%d) ready to ec reconstruct data", blob.Bid)
+			err := h.encoder[codeMode].ReconstructData(shards, badIdx)
+			if err == nil {
+				reconstructed = true
+				close(stopChan)
+				break
+			}
+			span.Infof("bid(%d) ec reconstruct data error:%s", blob.Bid, err.Error())
+		}
+
+		if len(received) >= len(sortedVuids) {
+			close(stopChan)
+			break
+		}
+		nextChan <- struct{}{}
+	}
+	getTime.IncR(time.Since(startRead))
+
+	// release buffer of delayed shards
+	go func() {
+		for shard := range shardPipe {
+			if shard.status {
+				h.memPool.Put(shard.buffer)
+			}
+		}
+	}()
+
+	if reconstructed {
+		return nil
+	}
+	return fmt.Errorf("broken blob(%d %d %d)", clusterID, blob.Vid, blob.Bid)
+}
+
+func (h *Handler) readOneShard(ctx context.Context, serviceController controller.ServiceController,
+	clusterID proto.ClusterID, vid proto.Vid, shardSize int,
+	blob blobGetArgs, vuid sortedVuid, stopChan <-chan struct{}) shardData {
+	span := trace.SpanFromContextSafe(ctx)
+	shardResult := shardData{
+		index:  vuid.index,
+		status: false,
+	}
+
+	args := blobnode.RangeGetShardArgs{
+		GetShardArgs: blobnode.GetShardArgs{
+			DiskID: vuid.diskID,
+			Vuid:   vuid.vuid,
+			Bid:    blob.Bid,
+		},
+		Offset: 0,
+		Size:   int64(shardSize),
+	}
+
+	var (
+		err  error
+		body io.ReadCloser
+	)
+	if hErr := hystrix.Do(rwCommand, func() error {
+		body, err = h.getOneShardFromHost(ctx, serviceController, vuid.host, vuid.diskID, args,
+			vuid.index, clusterID, vid, 3, stopChan)
+		if err != nil && (errorTimeout(err) || rpc.DetectStatusCode(err) == errcode.CodeOverload) {
+			return err
+		}
+		return nil
+	}, nil); hErr != nil {
+		span.Warnf("hystrix: read blob(%d %d %d) on blobnode(%d %d %s) ecidx(%d): %s",
+			clusterID, vid, blob.Bid, vuid.vuid, vuid.diskID, vuid.host, vuid.index, hErr.Error())
+		return shardResult
+	}
+
+	if err != nil {
+		if err == errPunishedDisk || err == errCanceledReadShard {
+			span.Debugf("read blob(%d %d %d) on blobnode(%d %d %s) ecidx(%d): %s",
+				clusterID, vid, blob.Bid, vuid.vuid, vuid.diskID, vuid.host, vuid.index, err.Error())
+			return shardResult
+		}
+		span.Warnf("read blob(%d %d %d) on blobnode(%d %d %s) ecidx(%d): %s",
+			clusterID, vid, blob.Bid, vuid.vuid, vuid.diskID, vuid.host, vuid.index, errors.Detail(err))
+		return shardResult
+	}
+	defer body.Close()
+
+	buf, err := h.memPool.Alloc(shardSize)
+	if err != nil {
+		span.Warn(err)
+		return shardResult
+	}
+
+	_, err = io.ReadFull(body, buf)
+	if err != nil {
+		h.memPool.Put(buf)
+		span.Warnf("read blob(%d %d %d) on blobnode(%d %d %s) ecidx(%d): %s",
+			clusterID, vid, blob.Bid,
+			vuid.vuid, vuid.diskID, vuid.host, vuid.index, err.Error())
+		return shardResult
+	}
+
+	shardResult.status = true
+	shardResult.buffer = buf
+	return shardResult
+}
+
+func (h *Handler) getDataShardOnly(ctx context.Context, getTime *timeReadWrite,
+	w io.Writer, serviceController controller.ServiceController,
+	clusterID proto.ClusterID, blob blobGetArgs) error {
+	span := trace.SpanFromContextSafe(ctx)
+	if blob.ReadSize == 0 {
+		return nil
+	}
+
+	blobVolume, err := h.getVolume(ctx, clusterID, blob.Vid, true)
+	if err != nil {
+		return err
+	}
+	tactic := blobVolume.CodeMode.Tactic()
+
+	from, to := int(blob.Offset), int(blob.Offset+blob.ReadSize)
+	buffer, err := ec.NewRangeBuffer(int(blob.BlobSize), from, to, tactic, h.memPool)
+	if err != nil {
+		return err
+	}
+	defer buffer.Release()
+
+	shardSize := buffer.ShardSize
+	firstShardIdx := int(blob.Offset) / shardSize
+	shardOffset := int(blob.Offset) % shardSize
+
+	startRead := time.Now()
+	remainSize := blob.ReadSize
+	bufOffset := 0
+	for i, shard := range blobVolume.Units[firstShardIdx:tactic.N] {
+		if remainSize <= 0 {
+			break
+		}
+
+		toReadSize := minU64(remainSize, uint64(shardSize-shardOffset))
+		args := blobnode.RangeGetShardArgs{
+			GetShardArgs: blobnode.GetShardArgs{
+				DiskID: shard.DiskID,
+				Vuid:   shard.Vuid,
+				Bid:    blob.Bid,
+			},
+			Offset: int64(shardOffset),
+			Size:   int64(toReadSize),
+		}
+
+		body, err := h.getOneShardFromHost(ctx, serviceController, shard.Host, shard.DiskID, args,
+			firstShardIdx+i, clusterID, blob.Vid, 1, nil)
+		if err != nil {
+			span.Warnf("read blob(%d %d %d) on blobnode(%d %d %s) ecidx(%d): %s",
+				clusterID, blob.Vid, blob.Bid,
+				shard.Vuid, shard.DiskID, shard.Host, firstShardIdx+i, errors.Detail(err))
+			return errNeedReconstructRead
+		}
+		defer body.Close()
+
+		buf := buffer.DataBuf[bufOffset : bufOffset+int(toReadSize)]
+		_, err = io.ReadFull(body, buf)
+		if err != nil {
+			span.Warn(err)
+			return errNeedReconstructRead
+		}
+
+		// reset next shard offset
+		shardOffset = 0
+		remainSize -= toReadSize
+		bufOffset += int(toReadSize)
+	}
+	getTime.IncR(time.Since(startRead))
+
+	if remainSize > 0 {
+		return fmt.Errorf("no enough data to read %d", remainSize)
+	}
+
+	startWrite := time.Now()
+	if _, err := w.Write(buffer.DataBuf[:int(blob.ReadSize)]); err != nil {
+		getTime.IncW(time.Since(startWrite))
+		return errors.Info(err, "write to response")
+	}
+	getTime.IncW(time.Since(startWrite))
+
+	return nil
+}
+
+// getOneShardFromHost get body of one shard
+func (h *Handler) getOneShardFromHost(ctx context.Context, serviceController controller.ServiceController,
+	host string, diskID proto.DiskID, args blobnode.RangeGetShardArgs, // get shard param with host diskid
+	index int, clusterID proto.ClusterID, vid proto.Vid, // param to update volume cache
+	attempts int, cancelChan <-chan struct{}, // do not retry again if cancelChan was closed
+) (io.ReadCloser, error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	// skip punished disk
+	if diskHost, err := serviceController.GetDiskHost(ctx, diskID); err != nil {
+		return nil, err
+	} else if diskHost.Punished {
+		return nil, errPunishedDisk
+	}
+
+	var (
+		rbody io.ReadCloser
+		rerr  error
+	)
+	rerr = retry.ExponentialBackoff(attempts, 200).RuptOn(func() (bool, error) {
+		if cancelChan != nil {
+			select {
+			case <-cancelChan:
+				return true, errCanceledReadShard
+			default:
+			}
+		}
+
+		// new child span to get from blobnode, we should finish it here.
+		spanChild, ctxChild := trace.StartSpanFromContextWithTraceID(
+			context.Background(), "GetFromBlobnode", span.TraceID())
+		defer spanChild.Finish()
+
+		body, _, err := h.blobnodeClient.RangeGetShard(ctxChild, host, &args)
+		if err == nil {
+			rbody = body
+			return true, nil
+		}
+
+		code := rpc.DetectStatusCode(err)
+		switch code {
+		case errcode.CodeOverload:
+			return true, err
+
+		// EIO and Readonly error, then we need to punish disk in local and no need to retry
+		case errcode.CodeDiskBroken, errcode.CodeVUIDReadonly:
+			h.punishDisk(ctx, clusterID, diskID, host, "BrokenOrRO")
+			span.Infof("punish disk:%d on:%s cos:blobnode/%d", diskID, host, code)
+			return true, fmt.Errorf("punished disk (%d %s)", diskID, host)
+
+		// vuid not found means the reflection between vuid and diskID has change,
+		// should refresh the blob volume cache
+		case errcode.CodeDiskNotFound, errcode.CodeVuidNotFound:
+			span.Infof("volume info outdated disk %d on host %s", diskID, host)
+
+			latestVolume, e := h.getVolume(ctx, clusterID, vid, false)
+			if e != nil {
+				span.Warnf("update volume info with no cache %d %d err: %s", clusterID, vid, e)
+				return false, err
+			}
+			newUnit := latestVolume.Units[index]
+
+			newDiskID := newUnit.DiskID
+			if newDiskID != diskID {
+				hi, e := serviceController.GetDiskHost(ctx, newDiskID)
+				if e == nil && !hi.Punished {
+					span.Infof("update disk %d %d %d -> %d", clusterID, vid, diskID, newDiskID)
+
+					host = hi.Host
+					diskID = newDiskID
+					args.GetShardArgs.DiskID = diskID
+					args.GetShardArgs.Vuid = newUnit.Vuid
+					return false, err
+				}
+			}
+
+			h.punishDiskWith(ctx, clusterID, diskID, host, "NotFound")
+			span.Debugf("punish threshold disk:%d cos:blobnode/%d", diskID, code)
+		}
+
+		// do not retry on timeout then punish threshold this disk
+		if errorTimeout(err) {
+			h.punishDiskWith(ctx, clusterID, diskID, host, "Timeout")
+			return true, err
+		}
+		span.Debugf("read from disk:%d blobnode/%s", diskID, err.Error())
+
+		err = errors.Base(err, fmt.Sprintf("get shard on (%d %s)", diskID, host))
+		return false, err
+	})
+
+	return rbody, rerr
+}
+
+func genLocationBlobs(location *access.Location, readSize uint64, offset uint64) ([]blobGetArgs, error) {
+	if offset+readSize > location.Size {
+		return nil, fmt.Errorf("FileSize:%d ReadSize:%d Offset:%d", location.Size, readSize, offset)
+	}
+
+	blobSize := uint64(location.BlobSize)
+	if blobSize <= 0 {
+		return nil, fmt.Errorf("BlobSize:%d", blobSize)
+	}
+
+	remainSize := readSize
+	firstBlobIdx := offset / blobSize
+	blobOffset := offset % blobSize
+
+	idx := uint64(0)
+	blobs := make([]blobGetArgs, 0, 1+(readSize+blobOffset)/blobSize)
+	for _, blob := range location.Blobs {
+		currBlobID := blob.MinBid
+
+		for ii := uint32(0); ii < blob.Count; ii++ {
+			if remainSize <= 0 {
+				return blobs, nil
+			}
+
+			if idx >= firstBlobIdx {
+				toReadSize := minU64(remainSize, blobSize-blobOffset)
+				if toReadSize > 0 {
+					blobs = append(blobs, blobGetArgs{
+						Vid:      blob.Vid,
+						Bid:      currBlobID,
+						BlobSize: minU64(location.Size-idx*blobSize, blobSize), // update the last blob size
+						Offset:   blobOffset,
+						ReadSize: toReadSize,
+					})
+				}
+
+				// reset next blob offset
+				blobOffset = 0
+				remainSize -= toReadSize
+			}
+
+			currBlobID++
+			idx++
+		}
+	}
+
+	if remainSize > 0 {
+		return nil, fmt.Errorf("no enough data to read %d", remainSize)
+	}
+
+	return blobs, nil
+}
+
+func genSortedVuidByIDC(ctx context.Context, serviceController controller.ServiceController, idc string,
+	vuidPhys []controller.Unit) []sortedVuid {
+	span := trace.SpanFromContextSafe(ctx)
+
+	vuids := make([]sortedVuid, 0, len(vuidPhys))
+	sortMap := make(map[int][]sortedVuid)
+
+	for idx, phy := range vuidPhys {
+		var hostIDC *controller.HostIDC
+		if err := retry.ExponentialBackoff(2, 100).On(func() error {
+			hi, e := serviceController.GetDiskHost(context.Background(), phy.DiskID)
+			if e != nil {
+				return e
+			}
+			hostIDC = hi
+			return nil
+		}); err != nil {
+			span.Warnf("no host of disk(%d %d) %s", phy.Vuid, phy.DiskID, err.Error())
+			continue
+		}
+
+		dis := distance(idc, hostIDC.IDC, hostIDC.Punished)
+		if _, ok := sortMap[dis]; !ok {
+			sortMap[dis] = make([]sortedVuid, 0, 8)
+		}
+		sortMap[dis] = append(sortMap[dis], sortedVuid{
+			index:  idx,
+			vuid:   phy.Vuid,
+			diskID: phy.DiskID,
+			host:   phy.Host,
+		})
+	}
+
+	keys := make([]int, 0, len(sortMap))
+	for dis := range sortMap {
+		keys = append(keys, dis)
+	}
+	sort.Ints(keys)
+
+	for _, dis := range keys {
+		ids := sortMap[dis]
+		rand.Shuffle(len(ids), func(i, j int) {
+			ids[i], ids[j] = ids[j], ids[i]
+		})
+		vuids = append(vuids, ids...)
+		if dis > 1 {
+			span.Debugf("distance: %d punished vuids: %+v", dis, ids)
+		}
+	}
+
+	return vuids
+}
+
+func distance(idc1, idc2 string, punished bool) int {
+	if punished {
+		if idc1 == idc2 {
+			return 2
+		}
+		return 3
+	}
+	if idc1 == idc2 {
+		return 0
+	}
+	return 1
+}
+
+func emptyDataShardIndexes(sizes ec.BufferSizes) map[int]struct{} {
+	firstEmptyIdx := (sizes.DataSize + sizes.ShardSize - 1) / sizes.ShardSize
+	n := sizes.ECDataSize / sizes.ShardSize
+	if firstEmptyIdx >= n {
+		return make(map[int]struct{})
+	}
+
+	set := make(map[int]struct{}, n-firstEmptyIdx)
+	for i := firstEmptyIdx; i < n; i++ {
+		set[i] = struct{}{}
+	}
+
+	return set
+}

blobstore/access/stream_get_test.go

@@ -0,0 +1,566 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"bytes"
+	"crypto/rand"
+	mrand "math/rand"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+func TestAccessStreamGetBase(t *testing.T) {
+	ctx := ctxWithName("TestAccessStreamGetBase")
+	// error
+	{
+		dataShards.clean()
+		data := []byte("x")
+		loc, err := streamer.Put(ctx(), bytes.NewReader(data), int64(len(data)), nil)
+		require.NoError(t, err)
+
+		buff := bytes.NewBuffer(nil)
+		_, err = streamer.Get(ctx(), buff, *loc, 1, 1)
+		require.NotNil(t, err)
+		_, err = streamer.Get(ctx(), buff, *loc, 2, 0)
+		require.NotNil(t, err)
+	}
+	// 1 byte
+	{
+		dataShards.clean()
+		data := []byte("x")
+		loc, err := streamer.Put(ctx(), bytes.NewReader(data), int64(len(data)), nil)
+		require.NoError(t, err)
+
+		buff := bytes.NewBuffer(nil)
+		transfer, err := streamer.Get(ctx(), buff, *loc, 1, 0)
+		require.NoError(t, err)
+		err = transfer()
+		require.NoError(t, err)
+		require.True(t, dataEqual(data, buff.Bytes()))
+
+		_, err = streamer.Get(ctx(), nil, *loc, 0, 1)
+		require.NoError(t, err)
+	}
+
+	vuidController.SetBNRealError(true)
+	defer func() {
+		vuidController.SetBNRealError(false)
+	}()
+
+	cases := []struct {
+		size int
+	}{
+		{12},
+		{1 << 12},
+		{(1 << 13) + 777},
+		{1 << 22},
+		{(1 << 22) + 1},
+		{(1 << 22) + 1023},
+		{1 << 23},
+		{(1 << 23) + 1025},
+	}
+	for _, cs := range cases {
+		dataShards.clean()
+		size := cs.size
+		data := make([]byte, size)
+		rand.Read(data)
+		loc, err := streamer.Put(ctx(), bytes.NewReader(data), int64(size), nil)
+		require.NoError(t, err)
+
+		buff := bytes.NewBuffer(nil)
+		transfer, err := streamer.Get(ctx(), buff, *loc, uint64(size), 0)
+		require.NoError(t, err)
+		err = transfer()
+		require.NoError(t, err)
+		require.True(t, dataEqual(data, buff.Bytes()))
+
+		// time wait the punished services
+		time.Sleep(time.Second * time.Duration(punishServiceS))
+	}
+
+	dataShards.clean()
+}
+
+func TestAccessStreamGetBroken(t *testing.T) {
+	ctx := ctxWithName("TestAccessStreamGetBroken")
+	defer func() {
+		dataShards.clean()
+		for ii := 0; ii < len(allID); ii++ {
+			vuidController.Unbreak(proto.Vuid(allID[ii]))
+		}
+		vuidController.Break(1005)
+	}()
+
+	dataShards.clean()
+	tactic := codemode.EC6P6.Tactic()
+	size := tactic.N * tactic.MinShardSize
+	data := make([]byte, size)
+	rand.Read(data)
+	// time wait the punished services
+	time.Sleep(time.Second * time.Duration(punishServiceS))
+	loc, err := streamer.Put(ctx(), bytes.NewReader(data), int64(size), nil)
+	require.NoError(t, err)
+
+	cases := []struct {
+		id       proto.Vuid
+		hasError bool
+	}{
+		{1001, false},
+		{1002, false},
+		{1003, false},
+		{1004, false},
+		{1005, false},
+		{1006, false},
+		{1007, true},
+		{1008, true},
+	}
+
+	for _, cs := range cases {
+		vuidController.Break(cs.id)
+
+		buff := bytes.NewBuffer(nil)
+		transfer, err := streamer.Get(ctx(), buff, *loc, uint64(size), 0)
+		require.Nil(t, err)
+		err = transfer()
+		if cs.hasError {
+			require.NotNil(t, err)
+		} else {
+			require.NoError(t, err)
+			require.True(t, dataEqual(data, buff.Bytes()))
+		}
+	}
+}
+
+func TestAccessStreamGetOffset(t *testing.T) {
+	ctx := ctxWithName("TestAccessStreamGetOffset")
+	vuidController.Unbreak(1005)
+	defer vuidController.Break(1005)
+
+	cases := []struct {
+		size     int64
+		offset   uint64
+		readSize uint64
+	}{
+		{12, 0, 12},
+		{12, 1, 11},
+		{12, 1, 1},
+		{12, 1, 4},
+		{12, 1, 7},
+		{(1 << 22) + 1, 0, 1 << 22},
+		{(1 << 22) + 1, 1 << 20, 1 << 20},
+		{(1 << 22) + 1, 1 << 22, 1},
+		{(1 << 22) + 1023, 0, 1},
+		{(1 << 22) + 1023, 1 << 22, 1022},
+		{(1 << 22) + 1023, 1 << 22, 1023},
+		{12192823, 6799138, 908019},
+	}
+	for _, cs := range cases {
+		dataShards.clean()
+		size := cs.size
+		data := make([]byte, size)
+		rand.Read(data)
+		loc, err := streamer.Put(ctx(), bytes.NewReader(data), size, nil)
+		require.NoError(t, err)
+
+		buff := bytes.NewBuffer(nil)
+		transfer, _ := streamer.Get(ctx(), buff, *loc, cs.readSize, cs.offset)
+		err = transfer()
+		require.NoError(t, err)
+		require.True(t, dataEqual(data[cs.offset:cs.offset+cs.readSize], buff.Bytes()))
+	}
+
+	dataShards.clean()
+}
+
+func TestAccessStreamGetShardTimeout(t *testing.T) {
+	ctx := ctxWithName("TestAccessStreamGetShardTimeout")
+	dataShards.clean()
+	vuidController.Unbreak(1005)
+	defer func() {
+		vuidController.Break(1005)
+		dataShards.clean()
+	}()
+
+	size := 1 << 22
+	buff := make([]byte, size)
+	rand.Read(buff)
+	loc, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil)
+	require.NoError(t, err)
+
+	// no delay when blocking one shard, cos MinReadShardsX = 1
+	vuidController.Block(1001)
+	defer func() {
+		vuidController.Unblock(1001)
+	}()
+	{
+		startTime := time.Now()
+		transfer, _ := streamer.Get(ctx(), bytes.NewBuffer(nil), *loc, uint64(size), 0)
+		err := transfer()
+		require.NoError(t, err)
+
+		duration := time.Since(startTime)
+		require.GreaterOrEqual(t, vuidController.duration, duration, "greater duration: ", duration)
+	}
+
+	// delay one duration when blocking two shard, cos MinReadShardsX = 1
+	vuidController.Block(1002)
+	defer func() {
+		vuidController.Unblock(1002)
+	}()
+	{
+		startTime := time.Now()
+		transfer, _ := streamer.Get(ctx(), bytes.NewBuffer(nil), *loc, uint64(size), 0)
+		err = transfer()
+		require.NoError(t, err)
+
+		duration := time.Since(startTime)
+		minDuration := vuidController.duration
+		maxDuration := minDuration + minDuration/2
+		t.Log(duration, minDuration, maxDuration)
+		require.LessOrEqual(t, minDuration, duration, "less duration: ", duration)
+		require.GreaterOrEqual(t, maxDuration, duration, "greater duration: ", duration)
+	}
+}
+
+func TestAccessStreamGetShardBroken(t *testing.T) {
+	ctx := ctxWithName("TestAccessStreamGetShardBroken")
+	dataShards.clean()
+	vuidController.Unbreak(1005)
+	defer func() {
+		vuidController.Break(1005)
+		dataShards.clean()
+	}()
+
+	size := 1 << 22
+	buff := make([]byte, size)
+	rand.Read(buff)
+	loc, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil)
+	require.NoError(t, err)
+
+	// no delay when blocking one shard, cos MinReadShardsX = 1
+	// it will not wait the blocking shard, cos has no enough shards to reconstruct
+	vuidController.Block(1001)
+	for _, id := range allID[1:] {
+		vuidController.Break(proto.Vuid(id))
+	}
+	defer func() {
+		vuidController.Unblock(1001)
+		for _, id := range allID[1:] {
+			vuidController.Unbreak(proto.Vuid(id))
+		}
+	}()
+	{
+		startTime := time.Now()
+		transfer, err := streamer.Get(ctx(), bytes.NewBuffer(nil), *loc, uint64(size), 0)
+		require.NoError(t, err)
+		err = transfer()
+		require.Error(t, err)
+
+		duration := time.Since(startTime)
+		require.GreaterOrEqual(t, vuidController.duration, duration, "greater duration: ", duration)
+	}
+}
+
+func TestAccessStreamGetLocalIDC(t *testing.T) {
+	ctx := ctxWithName("TestAccessStreamGetLocalIDC")
+	dataShards.clean()
+	vuidController.Unbreak(1005)
+	defer func() {
+		vuidController.Break(1005)
+		dataShards.clean()
+	}()
+
+	size := 1 << 22
+	buff := make([]byte, size)
+	rand.Read(buff)
+	loc, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil)
+	require.NoError(t, err)
+
+	// no delay when blocking other idc all shards
+	func() {
+		for _, id := range idcOtherID {
+			vuidController.Block(proto.Vuid(id))
+		}
+		defer func() {
+			for _, id := range idcOtherID {
+				vuidController.Unblock(proto.Vuid(id))
+			}
+		}()
+		startTime := time.Now()
+		transfer, _ := streamer.Get(ctx(), bytes.NewBuffer(nil), *loc, uint64(size), 0)
+		err = transfer()
+		require.NoError(t, err)
+
+		duration := time.Since(startTime)
+		require.GreaterOrEqual(t, vuidController.duration, duration, "greater duration: ", duration)
+	}()
+
+	// max delay one duration when other local has two blocked shard, cos MinReadShardsX = 1
+	vuidController.Break(proto.Vuid(idcID[0]))
+	defer func() {
+		vuidController.Unbreak(proto.Vuid(idcID[0]))
+	}()
+	func() {
+		vuidController.Block(proto.Vuid(idcOtherID[0]))
+		vuidController.Block(proto.Vuid(idcOtherID[1]))
+		defer func() {
+			vuidController.Unblock(proto.Vuid(idcOtherID[0]))
+			vuidController.Unblock(proto.Vuid(idcOtherID[1]))
+		}()
+		startTime := time.Now()
+		transfer, _ := streamer.Get(ctx(), bytes.NewBuffer(nil), *loc, uint64(size), 0)
+		err = transfer()
+		require.NoError(t, err)
+
+		duration := time.Since(startTime)
+		if duration < vuidController.duration {
+			return
+		}
+		minDuration := vuidController.duration
+		maxDuration := minDuration + minDuration/2
+		t.Log(duration, minDuration, maxDuration)
+		require.LessOrEqual(t, minDuration, duration, "less duration: ", duration)
+		require.GreaterOrEqual(t, maxDuration, duration, "greater duration: ", duration)
+	}()
+}
+
+func TestAccessStreamGetAligned(t *testing.T) {
+	ctx := ctxWithName("TestAccessStreamGetAligned")
+	defer func() {
+		dataShards.clean()
+		for ii := 0; ii < len(allID); ii++ {
+			vuidController.Unbreak(proto.Vuid(allID[ii]))
+		}
+		vuidController.Break(1005)
+	}()
+	vuidController.Unbreak(1005)
+
+	shardSize := codemode.EC6P6.Tactic().MinShardSize
+	cases := []struct {
+		size       int
+		goodShards int
+		succ       bool
+	}{
+		{1, 0, false},
+		{1, 1, true},
+		{1, 2, true},
+
+		{shardSize - 1, 1, true},
+		{shardSize, 1, true},
+		{shardSize + 1, 1, false},
+		{shardSize + 1, 2, true},
+
+		{shardSize * 2, 1, false},
+		{shardSize * 2, 2, true},
+		{shardSize * 3, 2, false},
+		{shardSize * 3, 3, true},
+		{shardSize * 4, 3, false},
+		{shardSize * 4, 4, true},
+		{shardSize * 5, 4, false},
+		{shardSize * 5, 5, true},
+		{shardSize * 6, 5, false},
+		{shardSize * 6, 6, true},
+	}
+
+	randomGoodShards := func(n int) {
+		for ii := 0; ii < len(allID); ii++ {
+			vuidController.Break(proto.Vuid(allID[ii]))
+		}
+
+		shards := make([]int, 0, len(allID))
+		shards = append(shards, allID[6:]...)
+		shards = append(shards, allID[:n]...)
+
+		mrand.Shuffle(len(shards), func(i, j int) {
+			shards[i], shards[j] = shards[j], shards[i]
+		})
+		for ii := 0; ii < n; ii++ {
+			vuidController.Unbreak(proto.Vuid(shards[ii]))
+		}
+	}
+
+	for _, cs := range cases {
+		dataShards.clean()
+
+		data := make([]byte, cs.size)
+		rand.Read(data)
+		loc, err := streamer.Put(ctx(), bytes.NewReader(data), int64(cs.size), nil)
+		require.NoError(t, err)
+
+		// cos put shards asynchronously, should wait all shard written
+		time.Sleep(200 * time.Millisecond)
+
+		randomGoodShards(cs.goodShards)
+
+		buff := bytes.NewBuffer(nil)
+		transfer, err := streamer.Get(ctx(), buff, *loc, uint64(cs.size), 0)
+		require.NoError(t, err)
+
+		err = transfer()
+		if cs.succ {
+			require.NoError(t, err)
+			require.True(t, dataEqual(data, buff.Bytes()))
+		} else {
+			require.Error(t, err)
+		}
+
+		for ii := 0; ii < len(allID); ii++ {
+			vuidController.Unbreak(proto.Vuid(allID[ii]))
+		}
+	}
+}
+
+func TestAccessStreamGenLocationBlobs(t *testing.T) {
+	firstSliceStart := proto.BlobID(100)
+	secondSliceStart := proto.BlobID(200)
+
+	loc := access.Location{
+		ClusterID: 0,
+		Size:      1024*4 + 37 + 1024*2, // 5 fine blobs and 2 missing blobs
+		BlobSize:  1024,
+		Blobs: []access.SliceInfo{
+			{
+				MinBid: firstSliceStart,
+				Vid:    proto.Vid(1001),
+				Count:  3,
+			},
+			{
+				MinBid: secondSliceStart,
+				Vid:    proto.Vid(2001),
+				Count:  2,
+			},
+		},
+	}
+
+	type blobArgs = blobGetArgs
+	cases := []struct {
+		readSize, offset uint64
+		err              bool
+		checker          func([]blobArgs) bool
+	}{
+		{1024 * 7, 0, true, func(blobs []blobArgs) bool { return blobs == nil }},
+		{1024 * 6, 38, true, func(blobs []blobArgs) bool { return blobs == nil }},
+		{1024 * 5, 38, true, func(blobs []blobArgs) bool { return blobs == nil }},
+
+		{0, 0, false, func(blobs []blobArgs) bool {
+			return len(blobs) == 0
+		}},
+		{1, 0, false, func(blobs []blobArgs) bool {
+			return len(blobs) == 1 && blobs[0].Bid == firstSliceStart
+		}},
+		{1024, 0, false, func(blobs []blobArgs) bool {
+			return len(blobs) == 1 && blobs[0].Bid == firstSliceStart
+		}},
+		{1025, 0, false, func(blobs []blobArgs) bool {
+			return len(blobs) == 2 && blobs[1].Bid == firstSliceStart+1
+		}},
+		{1024*4 + 37, 0, false, func(blobs []blobArgs) bool {
+			return len(blobs) == 5
+		}},
+
+		{0, 1024, false, func(blobs []blobArgs) bool {
+			return len(blobs) == 0
+		}},
+		{1, 1023, false, func(blobs []blobArgs) bool {
+			return len(blobs) == 1 && blobs[0].Bid == firstSliceStart
+		}},
+		{1, 1024, false, func(blobs []blobArgs) bool {
+			return len(blobs) == 1 && blobs[0].Bid == firstSliceStart+1
+		}},
+		{1024, 1024, false, func(blobs []blobArgs) bool {
+			return len(blobs) == 1 && blobs[0].Bid == firstSliceStart+1
+		}},
+		{1024 * 2, 1024 + 1, false, func(blobs []blobArgs) bool {
+			return len(blobs) == 3 && blobs[0].Bid == firstSliceStart+1 &&
+				blobs[0].Offset == 1 && blobs[0].ReadSize == 1023 &&
+				blobs[1].Offset == 0 && blobs[1].ReadSize == 1024 &&
+				blobs[2].Offset == 0 && blobs[2].ReadSize == 1
+		}},
+
+		{1, 1024 * 4, false, func(blobs []blobArgs) bool {
+			return len(blobs) == 1 && blobs[0].Bid == secondSliceStart+1 &&
+				blobs[0].Offset == 0 && blobs[0].ReadSize == 1
+		}},
+		{1, 1024*4 + 1, false, func(blobs []blobArgs) bool {
+			return len(blobs) == 1 && blobs[0].Bid == secondSliceStart+1 &&
+				blobs[0].Offset == 1 && blobs[0].ReadSize == 1
+		}},
+		{36, 1024*4 + 1, false, func(blobs []blobArgs) bool {
+			return len(blobs) == 1 && blobs[0].Bid == secondSliceStart+1 &&
+				blobs[0].Offset == 1 && blobs[0].ReadSize == 36
+		}},
+	}
+
+	for _, cs := range cases {
+		blobs, err := genLocationBlobs(&loc, cs.readSize, cs.offset)
+		if cs.err {
+			require.True(t, err != nil)
+		} else {
+			require.True(t, err == nil)
+		}
+		require.True(t, cs.checker(blobs))
+	}
+}
+
+type writer struct {
+	buf []byte
+}
+
+func (w *writer) Write(p []byte) (n int, err error) {
+	copy(w.buf, p)
+	return len(p), nil
+}
+
+func BenchmarkAccessStreamGet(b *testing.B) {
+	ctx := ctxWithName("BenchmarkAccessStreamGet")()
+	vuidController.Unbreak(1005)
+	defer func() {
+		vuidController.Break(1005)
+		dataShards.clean()
+	}()
+
+	cases := []struct {
+		name string
+		size int
+	}{
+		{"1B", 1},
+		{"1KB", 1 << 10},
+		{"1MB", 1 << 20},
+		{"4MB", 1 << 22},
+		{"8MB", 1 << 23},
+	}
+
+	w := &writer{buf: make([]byte, 1<<23)}
+	for _, cs := range cases {
+		b.ResetTimer()
+		b.Run(cs.name, func(b *testing.B) {
+			loc, err := streamer.Put(ctx, newReader(cs.size), int64(cs.size), nil)
+			require.NoError(b, err)
+
+			b.ResetTimer()
+			for ii := 0; ii <= b.N; ii++ {
+				transfer, _ := streamer.Get(ctx, w, *loc, uint64(cs.size), 0)
+				transfer()
+			}
+		})
+	}
+}

blobstore/access/stream_loop.go

@@ -0,0 +1,121 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"context"
+	"time"
+
+	"github.com/cubefs/cubefs/blobstore/api/proxy"
+	"github.com/cubefs/cubefs/blobstore/common/memcache"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/util/errors"
+)
+
+type (
+	vidHostKey struct {
+		cid proto.ClusterID
+		vid proto.Vid
+	}
+)
+
+// cacheVidAllocator memory cache of vid in allocator host
+var cacheVidAllocator *memcache.MemCache
+
+func init() {
+	mc, err := memcache.NewMemCache(context.Background(), 1<<15)
+	if err != nil {
+		panic(err)
+	}
+	cacheVidAllocator = mc
+}
+
+func setCacheVidHost(cid proto.ClusterID, vid proto.Vid, host string) {
+	cacheVidAllocator.Set(vidHostKey{cid: cid, vid: vid}, host)
+}
+
+func getCacheVidHost(cid proto.ClusterID, vid proto.Vid) (string, error) {
+	val := cacheVidAllocator.Get(vidHostKey{cid: cid, vid: vid})
+	if val == nil {
+		return "", errors.Newf("not found host of (%d %d)", cid, vid)
+	}
+	host, ok := val.(string)
+	if !ok {
+		return "", errors.Newf("not string host of (%d %d)", cid, vid)
+	}
+	return host, nil
+}
+
+func (h *Handler) loopDiscardVids() {
+	go func() {
+		cache := make(map[discardVid]struct{})
+
+		duration := time.Second * 3
+		timer := time.NewTimer(duration)
+		defer timer.Stop()
+
+		for {
+			flush := false
+			select {
+			case <-h.stopCh:
+				return
+			case dv := <-h.discardVidChan:
+				cache[dv] = struct{}{}
+				if len(cache) >= 8 {
+					flush = true
+				}
+			case <-timer.C:
+				if len(cache) > 0 {
+					flush = true
+				}
+				timer.Reset(duration)
+			}
+
+			if !flush {
+				continue
+			}
+
+			for dv := range cache {
+				go func(dv discardVid) {
+					h.tryDiscardVidOnAllocator(dv.cid, dv.vid, proxy.AllocVolsArgs{
+						Fsize:    1,
+						BidCount: 1,
+						CodeMode: dv.codeMode,
+						Discards: []proto.Vid{dv.vid},
+					})
+				}(dv)
+				delete(cache, dv)
+			}
+		}
+	}()
+}
+
+func (h *Handler) tryDiscardVidOnAllocator(cid proto.ClusterID, vid proto.Vid, args proxy.AllocVolsArgs) {
+	span, ctx := trace.StartSpanFromContext(context.Background(), "")
+
+	host, err := getCacheVidHost(cid, vid)
+	if err != nil {
+		span.Warnf("to discard vids %+v : %v", args, err)
+		return
+	}
+
+	span.Infof("to post on %s discard vids %+v", host, args)
+
+	_, err = h.proxyClient.VolumeAlloc(ctx, host, &args)
+	if err != nil {
+		span.Warnf("post on %s discard vids %+v : %v", host, args, err)
+	}
+}

blobstore/access/stream_mock_test.go

@@ -0,0 +1,551 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+// github.com/cubefs/cubefs/blobstore/access/... module access interfaces
+//go:generate mockgen -destination=./controller_mock_test.go -package=access -mock_names ClusterController=MockClusterController,ServiceController=MockServiceController,VolumeGetter=MockVolumeGetter github.com/cubefs/cubefs/blobstore/access/controller ClusterController,ServiceController,VolumeGetter
+//go:generate mockgen -destination=./access_mock_test.go -package=access -mock_names StreamHandler=MockStreamHandler,Limiter=MockLimiter github.com/cubefs/cubefs/blobstore/access StreamHandler,Limiter
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"hash/crc32"
+	"io"
+	"io/ioutil"
+	"math"
+	"math/rand"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/afex/hystrix-go/hystrix"
+	"github.com/alicebob/miniredis/v2"
+	"github.com/golang/mock/gomock"
+
+	"github.com/cubefs/cubefs/blobstore/access/controller"
+	"github.com/cubefs/cubefs/blobstore/api/blobnode"
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/api/proxy"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	"github.com/cubefs/cubefs/blobstore/common/ec"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/redis"
+	"github.com/cubefs/cubefs/blobstore/common/resourcepool"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/testing/mocks"
+	"github.com/cubefs/cubefs/blobstore/util/log"
+)
+
+var (
+	errNotFound     = errors.New("not found")
+	errAllocTimeout = errors.New("alloc timeout")
+
+	allocTimeoutSize uint64 = 1 << 40
+	punishServiceS          = 1
+
+	idc        = "test-idc"
+	idcOther   = "test-idc-other"
+	allID      = []int{1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012}
+	idcID      = []int{1001, 1002, 1003, 1007, 1008, 1009}
+	idcOtherID = []int{1004, 1005, 1006, 1010, 1011, 1012}
+
+	clusterID = proto.ClusterID(1)
+	volumeID  = proto.Vid(1)
+	blobSize  = 1 << 22
+
+	streamer *Handler
+
+	memPool     *resourcepool.MemPool
+	encoder     map[codemode.CodeMode]ec.Encoder
+	proxyClient proxy.Client
+
+	allCodeModes CodeModePairs
+
+	redismr  *miniredis.Miniredis
+	rediscli *redis.ClusterClient
+
+	cmcli             clustermgr.APIAccess
+	volumeGetter      controller.VolumeGetter
+	serviceController controller.ServiceController
+	cc                controller.ClusterController
+
+	clusterInfo *clustermgr.ClusterInfo
+	dataVolume  *clustermgr.VolumeInfo
+	dataAllocs  []proxy.AllocRet
+	dataNodes   map[string]clustermgr.ServiceInfo
+	dataDisks   map[proto.DiskID]blobnode.DiskInfo
+	dataShards  *shardsData
+
+	vuidController *vuidControl
+
+	putErrors = []errcode.Error{
+		errcode.ErrDiskBroken, errcode.ErrReadonlyVUID,
+		errcode.ErrChunkNoSpace,
+		errcode.ErrNoSuchDisk, errcode.ErrNoSuchVuid,
+	}
+	getErrors = []errcode.Error{
+		errcode.ErrOverload,
+		errcode.ErrDiskBroken, errcode.ErrReadonlyVUID,
+		errcode.ErrNoSuchDisk, errcode.ErrNoSuchVuid,
+	}
+)
+
+type shardKey struct {
+	Vuid proto.Vuid
+	Bid  proto.BlobID
+}
+
+type shardsData struct {
+	mutex sync.RWMutex
+	data  map[shardKey][]byte
+}
+
+func (d *shardsData) clean() {
+	d.mutex.Lock()
+	for key := range d.data {
+		d.data[key] = d.data[key][:0]
+	}
+	d.mutex.Unlock()
+}
+
+func (d *shardsData) get(vuid proto.Vuid, bid proto.BlobID) []byte {
+	key := shardKey{Vuid: vuid, Bid: bid}
+	d.mutex.RLock()
+	data := d.data[key]
+	buff := make([]byte, len(data))
+	copy(buff, data)
+	d.mutex.RUnlock()
+	return buff
+}
+
+func (d *shardsData) set(vuid proto.Vuid, bid proto.BlobID, b []byte) {
+	key := shardKey{Vuid: vuid, Bid: bid}
+	d.mutex.Lock()
+	old := d.data[key]
+	if cap(old) <= len(b) {
+		d.data[key] = make([]byte, len(b))
+	} else {
+		d.data[key] = old[:len(b)]
+	}
+	copy(d.data[key], b)
+	d.mutex.Unlock()
+}
+
+type vuidControl struct {
+	mutex    sync.Mutex
+	broken   map[proto.Vuid]bool
+	blocked  map[proto.Vuid]bool
+	block    func()
+	duration time.Duration
+
+	isBNRealError bool // is return blobnode real error
+}
+
+func (c *vuidControl) Break(id proto.Vuid) {
+	c.mutex.Lock()
+	c.broken[id] = true
+	c.mutex.Unlock()
+}
+
+func (c *vuidControl) Unbreak(id proto.Vuid) {
+	c.mutex.Lock()
+	delete(c.broken, id)
+	c.mutex.Unlock()
+}
+
+func (c *vuidControl) Isbroken(id proto.Vuid) bool {
+	c.mutex.Lock()
+	v, ok := c.broken[id]
+	c.mutex.Unlock()
+	return ok && v
+}
+
+func (c *vuidControl) Block(id proto.Vuid) {
+	c.mutex.Lock()
+	c.blocked[id] = true
+	c.mutex.Unlock()
+}
+
+func (c *vuidControl) Unblock(id proto.Vuid) {
+	c.mutex.Lock()
+	delete(c.blocked, id)
+	c.mutex.Unlock()
+}
+
+func (c *vuidControl) Isblocked(id proto.Vuid) bool {
+	c.mutex.Lock()
+	v, ok := c.blocked[id]
+	c.mutex.Unlock()
+	return ok && v
+}
+
+func (c *vuidControl) SetBNRealError(b bool) {
+	c.mutex.Lock()
+	c.isBNRealError = b
+	c.mutex.Unlock()
+}
+
+func (c *vuidControl) IsBNRealError() bool {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+	return c.isBNRealError
+}
+
+func randBlobnodeRealError(errors []errcode.Error) error {
+	n := rand.Intn(1024) % len(errors)
+	return errors[n]
+}
+
+var storageAPIRangeGetShard = func(ctx context.Context, host string, args *blobnode.RangeGetShardArgs) (
+	body io.ReadCloser, shardCrc uint32, err error) {
+	if vuidController.Isbroken(args.Vuid) {
+		err = errors.New("get shard fake error")
+		if vuidController.IsBNRealError() {
+			err = randBlobnodeRealError(getErrors)
+		}
+		return
+	}
+	if vuidController.Isblocked(args.Vuid) {
+		vuidController.block()
+		if rand.Intn(2) == 0 {
+			err = errors.New("get shard timeout")
+		} else {
+			err = errors.New("get shard Timeout")
+		}
+		return
+	}
+
+	buff := dataShards.get(args.Vuid, args.Bid)
+	if len(buff) == 0 {
+		return nil, 0, errNotFound
+	}
+	if len(buff) < int(args.Offset+args.Size) {
+		err = errors.New("get shard concurrently")
+		return
+	}
+
+	buff = buff[int(args.Offset):int(args.Offset+args.Size)]
+	shardCrc = crc32.ChecksumIEEE(buff)
+	body = ioutil.NopCloser(bytes.NewReader(buff))
+	return
+}
+
+var storageAPIPutShard = func(ctx context.Context, host string, args *blobnode.PutShardArgs) (
+	crc uint32, err error) {
+	if vuidController.Isbroken(args.Vuid) {
+		err = errors.New("put shard fake error")
+		if vuidController.IsBNRealError() {
+			err = randBlobnodeRealError(putErrors)
+		}
+		return
+	}
+	if vuidController.Isblocked(args.Vuid) {
+		vuidController.block()
+		err = errors.New("put shard timeout")
+		return
+	}
+
+	buffer, _ := memPool.Alloc(int(args.Size))
+	defer memPool.Put(buffer)
+
+	buffer = buffer[:int(args.Size)]
+	_, err = io.ReadFull(args.Body, buffer)
+	if err != nil {
+		return
+	}
+
+	crc = crc32.ChecksumIEEE(buffer)
+	dataShards.set(args.Vuid, args.Bid, buffer)
+	return
+}
+
+func initMockData() {
+	dataAllocs = make([]proxy.AllocRet, 2)
+	dataAllocs[0] = proxy.AllocRet{
+		BidStart: 10000,
+		BidEnd:   10000,
+		Vid:      volumeID,
+	}
+	dataAllocs[1] = proxy.AllocRet{
+		BidStart: 20000,
+		BidEnd:   50000,
+		Vid:      volumeID,
+	}
+
+	dataVolume = &clustermgr.VolumeInfo{
+		VolumeInfoBase: clustermgr.VolumeInfoBase{
+			Vid:      volumeID,
+			CodeMode: codemode.EC6P6,
+		},
+		Units: func() (units []clustermgr.Unit) {
+			for _, id := range allID {
+				units = append(units, clustermgr.Unit{
+					Vuid:   proto.Vuid(id),
+					DiskID: proto.DiskID(id),
+					Host:   strconv.Itoa(id),
+				})
+			}
+			return
+		}(),
+	}
+
+	proxyNodes := make([]clustermgr.ServiceNode, 32)
+	for idx := range proxyNodes {
+		proxyNodes[idx] = clustermgr.ServiceNode{
+			ClusterID: 1,
+			Name:      serviceProxy,
+			Host:      fmt.Sprintf("proxy-%d", idx),
+			Idc:       idc,
+		}
+	}
+
+	dataNodes = make(map[string]clustermgr.ServiceInfo)
+	dataNodes[serviceProxy] = clustermgr.ServiceInfo{
+		Nodes: proxyNodes,
+	}
+
+	dataDisks = make(map[proto.DiskID]blobnode.DiskInfo)
+	for _, id := range idcID {
+		dataDisks[proto.DiskID(id)] = blobnode.DiskInfo{
+			ClusterID: clusterID, Idc: idc, Host: strconv.Itoa(id),
+			DiskHeartBeatInfo: blobnode.DiskHeartBeatInfo{DiskID: proto.DiskID(id)},
+		}
+	}
+	for _, id := range idcOtherID {
+		dataDisks[proto.DiskID(id)] = blobnode.DiskInfo{
+			ClusterID: clusterID, Idc: idcOther, Host: strconv.Itoa(id),
+			DiskHeartBeatInfo: blobnode.DiskHeartBeatInfo{DiskID: proto.DiskID(id)},
+		}
+	}
+
+	dataShards = &shardsData{
+		data: make(map[shardKey][]byte, len(allID)),
+	}
+	dataShards.clean()
+
+	ctr := gomock.NewController(&testing.T{})
+	cli := mocks.NewMockClientAPI(ctr)
+	cli.EXPECT().GetService(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
+		func(ctx context.Context, args clustermgr.GetServiceArgs) (clustermgr.ServiceInfo, error) {
+			if val, ok := dataNodes[args.Name]; ok {
+				return val, nil
+			}
+			return clustermgr.ServiceInfo{}, errNotFound
+		})
+	cli.EXPECT().GetVolumeInfo(gomock.Any(), gomock.Any()).AnyTimes().Return(dataVolume, nil)
+	cli.EXPECT().DiskInfo(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
+		func(ctx context.Context, id proto.DiskID) (*blobnode.DiskInfo, error) {
+			if val, ok := dataDisks[id]; ok {
+				return &val, nil
+			}
+			return nil, errNotFound
+		})
+	cmcli = cli
+
+	redismr, _ = miniredis.Run()
+	if rand.Int()%2 == 0 {
+		rediscli = redis.NewClusterClient(&redis.ClusterConfig{
+			Addrs: []string{redismr.Addr()},
+		})
+	}
+	clusterInfo = &clustermgr.ClusterInfo{
+		Region:    "test-region",
+		ClusterID: clusterID,
+		Nodes:     []string{"node-1", "node-2", "node-3"},
+	}
+	serviceController, _ = controller.NewServiceController(
+		controller.ServiceConfig{
+			ClusterID: clusterID,
+			IDC:       idc,
+			ReloadSec: 1000,
+		}, cmcli)
+	volumeGetter, _ = controller.NewVolumeGetter(clusterID, cmcli, rediscli, 0)
+
+	ctr = gomock.NewController(&testing.T{})
+	c := NewMockClusterController(ctr)
+	c.EXPECT().Region().AnyTimes().Return("test-region")
+	c.EXPECT().ChooseOne().AnyTimes().Return(clusterInfo, nil)
+	c.EXPECT().GetServiceController(gomock.Any()).AnyTimes().Return(serviceController, nil)
+	c.EXPECT().GetVolumeGetter(gomock.Any()).AnyTimes().Return(volumeGetter, nil)
+	c.EXPECT().ChangeChooseAlg(gomock.Any()).AnyTimes().DoAndReturn(
+		func(alg controller.AlgChoose) error {
+			if alg < 10 {
+				return nil
+			}
+			return controller.ErrInvalidChooseAlg
+		})
+	cc = c
+
+	ctr = gomock.NewController(&testing.T{})
+	allocCli := mocks.NewMockProxyClient(ctr)
+	allocCli.EXPECT().SendDeleteMsg(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
+	allocCli.EXPECT().SendShardRepairMsg(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
+	allocCli.EXPECT().VolumeAlloc(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
+		func(ctx context.Context, host string, args *proxy.AllocVolsArgs) ([]proxy.AllocRet, error) {
+			if args.Fsize > allocTimeoutSize {
+				return nil, errAllocTimeout
+			}
+			return dataAllocs, nil
+		})
+	proxyClient = allocCli
+}
+
+func initPool() {
+	memPool = resourcepool.NewMemPool(getDefaultMempoolSize())
+}
+
+func initEncoder() {
+	coderEC6P6, _ := ec.NewEncoder(ec.Config{
+		CodeMode:     codemode.EC6P6.Tactic(),
+		EnableVerify: true,
+	})
+	coderEC6P10L2, _ := ec.NewEncoder(ec.Config{
+		CodeMode:     codemode.EC6P10L2.Tactic(),
+		EnableVerify: true,
+	})
+	coderEC15P12, _ := ec.NewEncoder(ec.Config{
+		CodeMode:     codemode.EC15P12.Tactic(),
+		EnableVerify: true,
+	})
+	coderEC16P20L2, _ := ec.NewEncoder(ec.Config{
+		CodeMode:     codemode.EC16P20L2.Tactic(),
+		EnableVerify: true,
+	})
+	encoder = map[codemode.CodeMode]ec.Encoder{
+		codemode.EC6P6:     coderEC6P6,
+		codemode.EC6P10L2:  coderEC6P10L2,
+		codemode.EC15P12:   coderEC15P12,
+		codemode.EC16P20L2: coderEC16P20L2,
+	}
+}
+
+func initEC() {
+	allCodeModes = CodeModePairs{
+		codemode.EC6P6: CodeModePair{
+			Policy: codemode.Policy{
+				ModeName: codemode.EC6P6.Name(),
+				MaxSize:  math.MaxInt64,
+				Enable:   true,
+			},
+			Tactic: codemode.EC6P6.Tactic(),
+		},
+		codemode.EC6P10L2: CodeModePair{
+			Policy: codemode.Policy{
+				ModeName: codemode.EC6P10L2.Name(),
+				MaxSize:  -1,
+			},
+			Tactic: codemode.EC6P10L2.Tactic(),
+		},
+		codemode.EC15P12: CodeModePair{
+			Policy: codemode.Policy{
+				ModeName: codemode.EC15P12.Name(),
+				MaxSize:  -1,
+			},
+			Tactic: codemode.EC15P12.Tactic(),
+		},
+		codemode.EC16P20L2: CodeModePair{
+			Policy: codemode.Policy{
+				ModeName: codemode.EC16P20L2.Name(),
+				MaxSize:  -1,
+			},
+			Tactic: codemode.EC16P20L2.Tactic(),
+		},
+	}
+}
+
+func initController() {
+	vuidController = &vuidControl{
+		broken:  make(map[proto.Vuid]bool),
+		blocked: make(map[proto.Vuid]bool),
+		block: func() {
+			time.Sleep(2 * time.Second)
+		},
+		duration:      2 * time.Second,
+		isBNRealError: false,
+	}
+	// initialized broken 1005
+	vuidController.Break(1005)
+}
+
+func newMockStorageAPI() blobnode.StorageAPI {
+	ctr := gomock.NewController(&testing.T{})
+	api := mocks.NewMockStorageAPI(ctr)
+	api.EXPECT().RangeGetShard(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().
+		DoAndReturn(storageAPIRangeGetShard)
+	api.EXPECT().PutShard(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().
+		DoAndReturn(storageAPIPutShard)
+	return api
+}
+
+func init() {
+	rand.Seed(int64(time.Now().Nanosecond()))
+	log.SetOutputLevel(log.Lfatal)
+
+	hystrix.ConfigureCommand(rwCommand, hystrix.CommandConfig{
+		Timeout:               9000,
+		MaxConcurrentRequests: 9000,
+		ErrorPercentThreshold: 90,
+	})
+
+	initPool()
+	initEncoder()
+	initEC()
+	initMockData()
+	initController()
+
+	streamer = &Handler{
+		memPool:           memPool,
+		encoder:           encoder,
+		clusterController: cc,
+
+		blobnodeClient: newMockStorageAPI(),
+		proxyClient:    proxyClient,
+
+		allCodeModes:  allCodeModes,
+		maxObjectSize: defaultMaxObjectSize,
+		StreamConfig: StreamConfig{
+			IDC:                    idc,
+			MaxBlobSize:            uint32(blobSize), // 4M
+			DiskPunishIntervalS:    punishServiceS,
+			ServicePunishIntervalS: punishServiceS,
+			AllocRetryTimes:        3,
+			AllocRetryIntervalMS:   3000,
+			MinReadShardsX:         defaultMinReadShardsX,
+		},
+		discardVidChan: make(chan discardVid, 8),
+		stopCh:         make(chan struct{}),
+	}
+	streamer.loopDiscardVids()
+}
+
+func ctxWithName(funcName string) func() context.Context {
+	return func() context.Context {
+		_, ctx := trace.StartSpanFromContextWithTraceID(context.Background(), funcName, funcName)
+		return ctx
+	}
+}
+
+func getBufSizes(size int) ec.BufferSizes {
+	sizes, _ := ec.GetBufferSizes(size, codemode.EC6P6.Tactic())
+	return sizes
+}
+
+func dataEqual(exp, act []byte) bool {
+	return crc32.ChecksumIEEE(exp) == crc32.ChecksumIEEE(act)
+}

blobstore/access/stream_put.go

@@ -0,0 +1,442 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"hash/crc32"
+	"io"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/afex/hystrix-go/hystrix"
+
+	"github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/api/blobnode"
+	"github.com/cubefs/cubefs/blobstore/common/ec"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/util/errors"
+	"github.com/cubefs/cubefs/blobstore/util/retry"
+)
+
+// TODO: To Be Continue
+//  put empty shard to blobnode if file has been aligned.
+
+// Put put one object
+//     required: size, file size
+//     optional: hasher map to calculate hash.Hash
+func (h *Handler) Put(ctx context.Context, rc io.Reader, size int64,
+	hasherMap access.HasherMap) (*access.Location, error) {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Debugf("put request size:%d hashes:b(%b)", size, hasherMap.ToHashAlgorithm())
+
+	if size <= 0 {
+		return nil, errcode.ErrIllegalArguments
+	}
+	if size > h.maxObjectSize {
+		span.Info("exceed max object size", h.maxObjectSize)
+		return nil, errcode.ErrAccessExceedSize
+	}
+
+	// 1.make hasher
+	if len(hasherMap) > 0 {
+		rc = io.TeeReader(rc, hasherMap.ToWriter())
+	}
+
+	// 2.choose cluster and alloc volume from allocator
+	selectedCodeMode := h.allCodeModes.SelectCodeMode(size)
+	span.Debugf("select codemode %d", selectedCodeMode)
+
+	blobSize := atomic.LoadUint32(&h.MaxBlobSize)
+	clusterID, blobs, err := h.allocFromAllocatorWithHystrix(ctx, selectedCodeMode, uint64(size), blobSize, 0)
+	if err != nil {
+		span.Error("alloc failed", errors.Detail(err))
+		return nil, err
+	}
+	span.Debugf("allocated from %d %+v", clusterID, blobs)
+
+	// 3.read body and split, alloc from mem pool;ec encode and put into data node
+	limitReader := io.LimitReader(rc, int64(size))
+	location := &access.Location{
+		ClusterID: clusterID,
+		CodeMode:  selectedCodeMode,
+		Size:      uint64(size),
+		BlobSize:  blobSize,
+		Blobs:     blobs,
+	}
+
+	uploadSucc := false
+	defer func() {
+		if !uploadSucc {
+			span.Infof("put failed clean location %+v", location)
+			if err := h.clearGarbage(ctx, location); err != nil {
+				span.Warn(errors.Detail(err))
+			}
+		}
+	}()
+
+	var buffer *ec.Buffer
+	putTime := new(timeReadWrite)
+	defer func() {
+		// release ec buffer which have not takeover
+		buffer.Release()
+		span.AppendRPCTrackLog([]string{putTime.String()})
+	}()
+
+	// concurrent buffer in per request
+	const concurrence = 4
+	ready := make(chan struct{}, concurrence)
+	for range [concurrence]struct{}{} {
+		ready <- struct{}{}
+	}
+
+	encoder := h.encoder[selectedCodeMode]
+	tactic := selectedCodeMode.Tactic()
+	for _, blob := range location.Spread() {
+		vid, bid, bsize := blob.Vid, blob.Bid, int(blob.Size)
+
+		// new an empty ec buffer for per blob
+		var err error
+		st := time.Now()
+		buffer, err = ec.NewBuffer(bsize, tactic, h.memPool)
+		putTime.IncA(time.Since(st))
+		if err != nil {
+			return nil, err
+		}
+
+		readBuff := buffer.DataBuf[:bsize]
+		shards, err := encoder.Split(buffer.ECDataBuf)
+		if err != nil {
+			return nil, err
+		}
+
+		startRead := time.Now()
+		n, err := io.ReadFull(limitReader, readBuff)
+		putTime.IncR(time.Since(startRead))
+		if err != nil && err != io.EOF {
+			span.Infof("read blob data failed want:%d read:%d %s", bsize, n, err.Error())
+			return nil, errcode.ErrAccessReadRequestBody
+		}
+		if n != bsize {
+			span.Infof("read blob less data want:%d but:%d", bsize, n)
+			return nil, errcode.ErrAccessReadRequestBody
+		}
+
+		// ec encode
+		if err = encoder.Encode(shards); err != nil {
+			return nil, err
+		}
+
+		blobident := blobIdent{clusterID, vid, bid}
+		span.Debug("to write", blobident)
+
+		// takeover the buffer, release to pool in function writeToBlobnodes
+		takeoverBuffer := buffer
+		buffer = nil
+		<-ready
+		startWrite := time.Now()
+		err = h.writeToBlobnodesWithHystrix(ctx, blobident, shards, func() {
+			takeoverBuffer.Release()
+			ready <- struct{}{}
+		})
+		putTime.IncW(time.Since(startWrite))
+		if err != nil {
+			return nil, errors.Info(err, "write to blobnode failed")
+		}
+	}
+
+	uploadSucc = true
+	return location, nil
+}
+
+func (h *Handler) writeToBlobnodesWithHystrix(ctx context.Context,
+	blob blobIdent, shards [][]byte, callback func()) error {
+	safe := make(chan struct{}, 1)
+	err := hystrix.Do(rwCommand, func() error {
+		safe <- struct{}{}
+		return h.writeToBlobnodes(ctx, blob, shards, callback)
+	}, nil)
+
+	select {
+	case <-safe:
+	default:
+		callback() // callback if fused by hystrix
+	}
+	return err
+}
+
+type shardPutStatus struct {
+	index  int
+	status bool
+}
+
+// writeToBlobnodes write shards to blobnodes.
+// takeover ec buffer release by callback.
+// return if had quorum successful shards, then wait all shards in background.
+func (h *Handler) writeToBlobnodes(ctx context.Context,
+	blob blobIdent, shards [][]byte, callback func()) (err error) {
+	span := trace.SpanFromContextSafe(ctx)
+	clusterID, vid, bid := blob.cid, blob.vid, blob.bid
+
+	wg := &sync.WaitGroup{}
+	defer func() {
+		// waiting all shards done in background
+		go func() {
+			wg.Wait()
+			callback()
+		}()
+	}()
+
+	volume, err := h.getVolume(ctx, clusterID, vid, true)
+	if err != nil {
+		return
+	}
+	serviceController, err := h.clusterController.GetServiceController(clusterID)
+	if err != nil {
+		return
+	}
+
+	statusCh := make(chan shardPutStatus, len(volume.Units))
+	tactic := volume.CodeMode.Tactic()
+	putQuorum := uint32(tactic.PutQuorum)
+	if num, ok := h.CodeModesPutQuorums[volume.CodeMode]; ok && num <= tactic.N+tactic.M {
+		putQuorum = uint32(num)
+	}
+
+	// writtenNum ONLY apply on data and partiy shards
+	// TODO: count N and M in each AZ,
+	//    decision ec data is recoverable or not.
+	maxWrittenIndex := tactic.N + tactic.M
+	writtenNum := uint32(0)
+
+	wg.Add(len(volume.Units))
+	for i, unitI := range volume.Units {
+		index, unit := i, unitI
+
+		go func() {
+			status := shardPutStatus{index: index}
+			defer func() {
+				statusCh <- status
+				wg.Done()
+			}()
+
+			diskID := unit.DiskID
+			args := &blobnode.PutShardArgs{
+				DiskID: diskID,
+				Vuid:   unit.Vuid,
+				Bid:    bid,
+				Size:   int64(len(shards[index])),
+				Type:   blobnode.NormalIO,
+			}
+
+			crcDisabled := h.ShardCrcDisabled
+			var crcOrigin uint32
+			if !crcDisabled {
+				crcOrigin = crc32.ChecksumIEEE(shards[index])
+			}
+
+			// new child span to write to blobnode, we should finish it here.
+			spanChild, ctxChild := trace.StartSpanFromContextWithTraceID(
+				context.Background(), "WriteToBlobnode", span.TraceID())
+			defer spanChild.Finish()
+
+		RETRY:
+			hostInfo, err := serviceController.GetDiskHost(ctxChild, diskID)
+			if err != nil {
+				span.Error("get disk host failed", errors.Detail(err))
+				return
+			}
+			// punished disk, ignore and return
+			if hostInfo.Punished {
+				span.Infof("ignore punished disk(%d %s) uvid(%d) ecidx(%02d) in idc(%s)",
+					diskID, hostInfo.Host, unit.Vuid, index, hostInfo.IDC)
+				return
+			}
+			host := hostInfo.Host
+
+			var (
+				writeErr  error
+				needRetry bool
+				crc       uint32
+			)
+			writeErr = retry.ExponentialBackoff(3, 200).RuptOn(func() (bool, error) {
+				args.Body = bytes.NewReader(shards[index])
+
+				crc, err = h.blobnodeClient.PutShard(ctxChild, host, args)
+				if err == nil {
+					if !crcDisabled && crc != crcOrigin {
+						return false, fmt.Errorf("crc mismatch 0x%x != 0x%x", crc, crcOrigin)
+					}
+
+					needRetry = false
+					return true, nil
+				}
+
+				code := rpc.DetectStatusCode(err)
+				switch code {
+				case errcode.CodeDiskBroken, errcode.CodeDiskNotFound,
+					errcode.CodeChunkNoSpace, errcode.CodeVUIDReadonly:
+					h.discardVidChan <- discardVid{
+						cid:      clusterID,
+						codeMode: volume.CodeMode,
+						vid:      vid,
+					}
+				}
+
+				switch code {
+				// EIO and Readonly error, then we need to punish disk in local and no necessary to retry
+				case errcode.CodeDiskBroken, errcode.CodeVUIDReadonly:
+					h.punishVolume(ctx, clusterID, vid, host, "BrokenOrRO")
+					h.punishDisk(ctx, clusterID, diskID, host, "BrokenOrRO")
+					span.Infof("punish disk:%d volume:%d cos:blobnode/%d", diskID, vid, code)
+					return true, err
+
+				// chunk no space, we should punish this volume
+				case errcode.CodeChunkNoSpace:
+					h.punishVolume(ctx, clusterID, vid, host, "NoSpace")
+					span.Infof("punish volume:%d cos:blobnode/%d", vid, code)
+					return true, err
+
+				// vuid not found means the reflection between vuid and diskID has change, we should refresh the volume
+				// disk not found means disk has been repaired or offline
+				case errcode.CodeDiskNotFound, errcode.CodeVuidNotFound:
+					latestVolume, e := h.getVolume(ctx, clusterID, vid, false)
+					if e != nil {
+						return true, errors.Base(err, "get volume with no cache failed").Detail(e)
+					}
+
+					newUnit := latestVolume.Units[index]
+					if diskID != newUnit.DiskID {
+						diskID = newUnit.DiskID
+						unit = newUnit
+						args.DiskID = newUnit.DiskID
+						args.Vuid = newUnit.Vuid
+
+						needRetry = true
+						return true, err
+					}
+
+					h.punishVolume(ctx, clusterID, vid, host, "NotFound")
+					h.punishDisk(ctx, clusterID, diskID, host, "NotFound")
+					span.Infof("punish disk:%d volume:%d cos:blobnode/%d", diskID, vid, code)
+					return true, err
+				}
+
+				// in timeout case and writtenNum is not satisfied with putQuorum, then should retry
+				if errorTimeout(err) && atomic.LoadUint32(&writtenNum) < putQuorum {
+					h.punishDiskWith(ctx, clusterID, diskID, host, "Timeout")
+					span.Info("connect timeout, need to punish threshold disk", diskID, host)
+					return false, err
+				}
+
+				// others, do not retry this round
+				return true, err
+			})
+
+			if needRetry {
+				goto RETRY
+			}
+			if writeErr != nil {
+				span.Warnf("write %s on blobnode(%d %d %s) ecidx(%02d): %s",
+					blob.String(), args.Vuid, args.DiskID, hostInfo.Host, index, errors.Detail(writeErr))
+				return
+			}
+
+			if index < maxWrittenIndex {
+				atomic.AddUint32(&writtenNum, 1)
+			}
+			status.status = true
+		}()
+	}
+
+	received := make(map[int]shardPutStatus, len(volume.Units))
+	for len(received) < len(volume.Units) && atomic.LoadUint32(&writtenNum) < putQuorum {
+		st := <-statusCh
+		received[st.index] = st
+	}
+
+	writeDone := make(chan struct{}, 1)
+	// write unaccomplished shard to repair queue
+	go func(writeDone <-chan struct{}) {
+		for len(received) < len(volume.Units) {
+			st := <-statusCh
+			received[st.index] = st
+		}
+
+		if _, ok := <-writeDone; !ok {
+			return
+		}
+
+		badIdxes := make([]uint8, 0)
+		for idx := range volume.Units {
+			if st, ok := received[idx]; ok && st.status {
+				continue
+			}
+			badIdxes = append(badIdxes, uint8(idx))
+		}
+		if len(badIdxes) > 0 {
+			h.sendRepairMsgBg(ctx, blob, badIdxes)
+		}
+	}(writeDone)
+
+	// return if had quorum successful shards
+	if atomic.LoadUint32(&writtenNum) >= putQuorum {
+		writeDone <- struct{}{}
+		return
+	}
+
+	// It tolerate one az was down when we have 3 or more azs.
+	// But MUST make sure others azs data is all completed,
+	// And all data in the down az are failed.
+	if tactic.AZCount >= 3 {
+		allFine := 0
+		allDown := 0
+
+		for _, azIndexes := range tactic.GetECLayoutByAZ() {
+			azFine := true
+			azDown := true
+			for _, idx := range azIndexes {
+				if st, ok := received[idx]; !ok || !st.status {
+					azFine = false
+				} else {
+					azDown = false
+				}
+			}
+			if azFine {
+				allFine++
+			}
+			if azDown {
+				allDown++
+			}
+		}
+
+		span.Debugf("tolerate-multi-az-write (az-fine:%d az-down:%d az-all:%d)", allFine, allDown, tactic.AZCount)
+		if allFine == tactic.AZCount-1 && allDown == 1 {
+			span.Warnf("tolerate-multi-az-write (az-fine:%d az-down:%d az-all:%d) of %s",
+				allFine, allDown, tactic.AZCount, blob.String())
+			writeDone <- struct{}{}
+			return
+		}
+	}
+
+	close(writeDone)
+	err = fmt.Errorf("quorum write failed (%d < %d) of %s", writtenNum, putQuorum, blob.String())
+	return
+}

blobstore/access/stream_put_test.go

@@ -0,0 +1,349 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"bytes"
+	"crypto/md5"
+	"crypto/rand"
+	"crypto/sha1"
+	"crypto/sha256"
+	"encoding/hex"
+	"hash/crc32"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/api/access"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+func TestAccessStreamPutBase(t *testing.T) {
+	ctx := ctxWithName("TestAccessStreamPutBase")
+	vuidController.SetBNRealError(true)
+	defer func() {
+		vuidController.SetBNRealError(false)
+	}()
+
+	// 0
+	{
+		size := 0
+		_, err := streamer.Put(ctx(), newReader(size), int64(size), nil)
+		require.Error(t, err)
+	}
+	// 1 byte
+	{
+		size := 1
+		loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil)
+		require.NoError(t, err)
+		require.Equal(t, 1, len(loc.Blobs))
+		require.Equal(t, uint32(1), loc.Blobs[0].Count)
+		// time wait the punished services
+		time.Sleep(time.Second * time.Duration(punishServiceS))
+	}
+	// <4M
+	{
+		size := 1 << 18
+		loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil)
+		require.NoError(t, err)
+		require.Equal(t, 1, len(loc.Blobs))
+		require.Equal(t, uint32(1), loc.Blobs[0].Count)
+		time.Sleep(time.Second * time.Duration(punishServiceS))
+	}
+	// 8M + 1k
+	{
+		size := (1 << 23) + 1024
+		loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil)
+		require.NoError(t, err)
+		require.Equal(t, 2, len(loc.Blobs))
+		require.Equal(t, uint32(2), loc.Blobs[1].Count)
+		time.Sleep(time.Second * time.Duration(punishServiceS))
+	}
+	// max size + 1
+	{
+		size := defaultMaxObjectSize + 1
+		_, err := streamer.Put(ctx(), nil, int64(size), nil)
+		require.EqualError(t, errcode.ErrAccessExceedSize, err.Error())
+	}
+
+	dataShards.clean()
+}
+
+func TestAccessStreamPutSum(t *testing.T) {
+	ctx := ctxWithName("TestAccessStreamPutSum")
+	sumChecker := func(data []byte) {
+		dataShards.clean()
+
+		hasherMap := access.HasherMap{
+			access.HashAlgDummy:  access.HashAlgDummy.ToHasher(),
+			access.HashAlgCRC32:  access.HashAlgCRC32.ToHasher(),
+			access.HashAlgMD5:    access.HashAlgMD5.ToHasher(),
+			access.HashAlgSHA1:   access.HashAlgSHA1.ToHasher(),
+			access.HashAlgSHA256: access.HashAlgSHA256.ToHasher(),
+		}
+		hashSumMap := make(access.HashSumMap, len(hasherMap))
+
+		_, err := streamer.Put(ctx(), bytes.NewReader(data), int64(len(data)), hasherMap)
+		require.NoError(t, err)
+		for alg, hasher := range hasherMap {
+			hashSumMap[alg] = hasher.Sum(nil)
+		}
+
+		bytesMD5 := md5.Sum(data)
+		bytesSHA1 := sha1.Sum(data)
+		bytesSHA256 := sha256.Sum256(data)
+
+		require.Equal(t, hashSumMap.GetSumVal(access.HashAlgCRC32), crc32.ChecksumIEEE(data))
+		require.Equal(t, hashSumMap.GetSumVal(access.HashAlgMD5), sum2Str(bytesMD5[:]))
+		require.Equal(t, hashSumMap.GetSumVal(access.HashAlgSHA1), sum2Str(bytesSHA1[:]))
+		require.Equal(t, hashSumMap.GetSumVal(access.HashAlgSHA256), sum2Str(bytesSHA256[:]))
+	}
+
+	shardsChecher := func(shardSize int, buff []byte, bid proto.BlobID) {
+		require.Equal(t, buff[:shardSize], dataShards.get(1001, bid))
+		require.Equal(t, buff[shardSize:shardSize*2], dataShards.get(1002, bid))
+		require.Equal(t, buff[shardSize*2:shardSize*3], dataShards.get(1003, bid))
+		require.Equal(t, buff[shardSize*3:shardSize*4], dataShards.get(1004, bid))
+		require.Equal(t, 0, len(dataShards.get(1005, 10000)))
+		require.Equal(t, buff[shardSize*5:shardSize*6], dataShards.get(1006, bid))
+		require.NotEqual(t, ([]byte)(nil), dataShards.get(1007, bid))
+		require.Equal(t, shardSize, len(dataShards.get(1007, bid)))
+		require.Equal(t, shardSize, len(dataShards.get(1012, bid)))
+	}
+
+	// 1 byte
+	{
+		data := []byte("x")
+		sumChecker(data)
+
+		shardSize := getBufSizes(len(data)).ShardSize
+		zeroShard := make([]byte, shardSize)
+		shard1 := make([]byte, shardSize)
+		copy(shard1, data)
+
+		require.Equal(t, shard1, dataShards.get(1001, 10000))
+		require.Equal(t, zeroShard, dataShards.get(1002, 10000))
+		require.Equal(t, zeroShard, dataShards.get(1004, 10000))
+		require.Equal(t, 0, len(dataShards.get(1005, 10000)))
+		require.NotEqual(t, ([]byte)(nil), dataShards.get(1007, 10000))
+	}
+	// 7 bytes
+	{
+		data := []byte{0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6}
+		sumChecker(data)
+
+		sizes := getBufSizes(len(data))
+		buff := make([]byte, sizes.ECDataSize)
+		copy(buff, data)
+		shardsChecher(sizes.ShardSize, buff, 10000)
+	}
+	// 4K + 511B, aligned with MinShardSize
+	{
+		size := (1 << 12) + 511
+		data := make([]byte, size)
+		rand.Read(data)
+		sumChecker(data)
+
+		sizes := getBufSizes(len(data))
+		buff := make([]byte, sizes.ECDataSize)
+		copy(buff, data)
+		shardsChecher(sizes.ShardSize, buff, 10000)
+	}
+	// 4M
+	{
+		size := 1 << 22
+		data := make([]byte, size)
+		rand.Read(data)
+		sumChecker(data)
+
+		sizes := getBufSizes(len(data))
+		buff := make([]byte, sizes.ECDataSize)
+		copy(buff, data)
+		shardsChecher(sizes.ShardSize, buff, 10000)
+	}
+	// 4M + 1B
+	{
+		size := (1 << 22) + 1
+		data := make([]byte, size)
+		rand.Read(data)
+		sumChecker(data)
+
+		sizes := getBufSizes(blobSize)
+		buff := make([]byte, sizes.ECDataSize)
+		copy(buff, data[:blobSize])
+		shardsChecher(sizes.ShardSize, buff, 10000)
+
+		data = data[blobSize:]
+		sizes = getBufSizes(len(data))
+		buff = make([]byte, sizes.ECDataSize)
+		copy(buff, data)
+		shardsChecher(sizes.ShardSize, buff, 20000)
+	}
+	// 6M + 1K + 7B
+	{
+		size := (1<<20)*6 + 1024 + 7
+		data := make([]byte, size)
+		rand.Read(data)
+		sumChecker(data)
+
+		sizes := getBufSizes(blobSize)
+		buff := make([]byte, sizes.ECDataSize)
+		copy(buff, data[:blobSize])
+		shardsChecher(sizes.ShardSize, buff, 10000)
+
+		data = data[blobSize:]
+		sizes = getBufSizes(len(data))
+		buff = make([]byte, sizes.ECDataSize)
+		copy(buff, data)
+		shardsChecher(sizes.ShardSize, buff, 20000)
+	}
+
+	dataShards.clean()
+}
+
+func TestAccessStreamPutShardTimeout(t *testing.T) {
+	ctx := ctxWithName("TestAccessStreamPutShardTimeout")
+	dataShards.clean()
+	vuidController.Block(1001)
+	vuidController.Unbreak(1005)
+	defer func() {
+		vuidController.Unblock(1001)
+		vuidController.Unblock(1002)
+		vuidController.Break(1005)
+		dataShards.clean()
+	}()
+
+	size := 1 << 22
+	buff := make([]byte, size)
+	rand.Read(buff)
+	startTime := time.Now()
+	loc, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil)
+	require.NoError(t, err)
+
+	// response immediately if had quorum shards
+	duration := time.Since(startTime)
+	require.GreaterOrEqual(t, time.Second/2, duration, "greater duration: ", duration)
+
+	transfer, err := streamer.Get(ctx(), bytes.NewBuffer(nil), *loc, uint64(size), 0)
+	require.NoError(t, err)
+	err = transfer()
+	require.NoError(t, err)
+
+	// wait all shards if no quorum shards
+	vuidController.Block(1002)
+	{
+		startTime := time.Now()
+		_, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil)
+		require.Error(t, err)
+
+		duration := time.Since(startTime)
+		minDuration := vuidController.duration * 3 // retry ExponentialBackoff(3, 200)
+		maxDuration := minDuration + minDuration/2
+		t.Log(duration, minDuration, maxDuration)
+		require.LessOrEqual(t, minDuration, duration, "less duration: ", duration)
+		require.GreaterOrEqual(t, maxDuration, duration, "greater duration: ", duration)
+	}
+}
+
+func TestAccessStreamPutQuorum(t *testing.T) {
+	ctx := ctxWithName("TestAccessStreamPutQuorum")
+	defer func() {
+		dataShards.clean()
+		for ii := 0; ii < len(allID); ii++ {
+			vuidController.Unbreak(proto.Vuid(allID[ii]))
+		}
+		vuidController.Break(1005)
+	}()
+
+	dataShards.clean()
+	vuidController.Unbreak(1005)
+	cases := []struct {
+		ids      []proto.Vuid
+		hasError bool
+	}{
+		// indexes by AZ
+		// [[0 1 6 7] [2 3 8 9] [4 5 10 11]]
+		{[]proto.Vuid{1001}, false},
+		{[]proto.Vuid{1001, 1002}, true},              // az-0 instable
+		{[]proto.Vuid{1001, 1002, 1007, 1008}, false}, // az-0 down
+		{[]proto.Vuid{1003}, false},
+		{[]proto.Vuid{1003, 1010}, true},              // az-1 instable
+		{[]proto.Vuid{1003, 1004, 1009, 1010}, false}, // az-1 down
+		{[]proto.Vuid{1005}, false},
+		{[]proto.Vuid{1005, 1011}, true},              // az-2 instable
+		{[]proto.Vuid{1005, 1006, 1011, 1012}, false}, // az-2 down
+		{[]proto.Vuid{1001, 1011}, true},
+		{[]proto.Vuid{1006, 1010}, true},
+		{[]proto.Vuid{1008, 1010, 1012}, true},
+	}
+
+	size := 1 << 22
+	buff := make([]byte, size)
+	rand.Read(buff)
+	for _, cs := range cases {
+		for _, id := range cs.ids {
+			vuidController.Break(id)
+		}
+
+		_, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil)
+		if cs.hasError {
+			require.NotNil(t, err)
+		} else {
+			require.NoError(t, err)
+		}
+
+		time.Sleep(time.Second * time.Duration(punishServiceS))
+		for _, id := range cs.ids {
+			vuidController.Unbreak(id)
+		}
+	}
+}
+
+func BenchmarkAccessStreamPut(b *testing.B) {
+	ctx := ctxWithName("BenchmarkAccessStreamPut")()
+	vuidController.Unbreak(1005)
+	defer func() {
+		vuidController.Break(1005)
+		dataShards.clean()
+	}()
+
+	cases := []struct {
+		name string
+		size int
+	}{
+		{"1B", 1},
+		{"1KB", 1 << 10},
+		{"1MB", 1 << 20},
+		{"4MB", 1 << 22},
+		{"8MB", 1 << 23},
+	}
+
+	buff := make([]byte, 1<<23)
+	rand.Read(buff)
+	for _, cs := range cases {
+		b.ResetTimer()
+		b.Run(cs.name, func(b *testing.B) {
+			for ii := 0; ii <= b.N; ii++ {
+				streamer.Put(ctx, bytes.NewReader(buff[:cs.size]), int64(cs.size), nil)
+			}
+		})
+	}
+}
+
+func sum2Str(b []byte) string {
+	return hex.EncodeToString(b[:])
+}

blobstore/access/stream_putat.go

@@ -0,0 +1,102 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"context"
+	"io"
+	"time"
+
+	"github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/common/ec"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+)
+
+// PutAt access interface /putat, put one blob
+//     required: rc file reader
+//     required: clusterID VolumeID BlobID
+//     required: size, one blob size
+//     optional: hasherMap, computing hash
+func (h *Handler) PutAt(ctx context.Context, rc io.Reader,
+	clusterID proto.ClusterID, vid proto.Vid, bid proto.BlobID, size int64,
+	hasherMap access.HasherMap) error {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Debugf("putat request cluster:%d vid:%d bid:%d size:%d hashes:b(%b)",
+		clusterID, vid, bid, size, hasherMap.ToHashAlgorithm())
+
+	if len(hasherMap) > 0 {
+		rc = io.TeeReader(rc, hasherMap.ToWriter())
+	}
+
+	volume, err := h.getVolume(ctx, clusterID, vid, true)
+	if err != nil {
+		return err
+	}
+	encoder := h.encoder[volume.CodeMode]
+
+	readSize := int(size)
+	st := time.Now()
+	buffer, err := ec.NewBuffer(readSize, volume.CodeMode.Tactic(), h.memPool)
+	if err != nil {
+		return err
+	}
+
+	putTime := new(timeReadWrite)
+	putTime.IncA(time.Since(st))
+	defer func() {
+		buffer.Release()
+		span.AppendRPCTrackLog([]string{putTime.String()})
+	}()
+
+	shards, err := encoder.Split(buffer.ECDataBuf)
+	if err != nil {
+		return err
+	}
+
+	startRead := time.Now()
+	n, err := io.ReadFull(rc, buffer.DataBuf)
+	putTime.IncR(time.Since(startRead))
+	if err != nil && err != io.EOF {
+		span.Info("read blob data from request body", err)
+		return errcode.ErrAccessReadRequestBody
+	}
+	if n != readSize {
+		span.Infof("want to read %d, but %d", readSize, n)
+		return errcode.ErrAccessReadRequestBody
+	}
+
+	if err = encoder.Encode(shards); err != nil {
+		return err
+	}
+
+	blobident := blobIdent{clusterID, vid, bid}
+	span.Debug("to write", blobident)
+
+	takeoverBuffer := buffer
+	buffer = nil
+	startWrite := time.Now()
+	err = h.writeToBlobnodesWithHystrix(ctx, blobident, shards, func() {
+		takeoverBuffer.Release()
+	})
+	putTime.IncW(time.Since(startWrite))
+	if err != nil {
+		return err
+	}
+
+	span.Debugf("putat done cluster:%d vid:%d bid:%d size:%d", clusterID, vid, bid, size)
+	return nil
+}

blobstore/access/stream_putat_test.go

@@ -0,0 +1,97 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"bytes"
+	"crypto/rand"
+	"hash/crc32"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/api/access"
+)
+
+func TestAccessStreamPutAtBase(t *testing.T) {
+	ctx := ctxWithName("TestAccessStreamPutAtBase")
+	sumChecker := func(size int, data []byte) {
+		dataShards.clean()
+
+		hasherMap := access.HasherMap{
+			access.HashAlgCRC32: access.HashAlgCRC32.ToHasher(),
+		}
+		hashSumMap := make(access.HashSumMap, len(hasherMap))
+		err := streamer.PutAt(ctx(), bytes.NewReader(data), clusterID, 1, 10000, int64(size), hasherMap)
+		require.Nil(t, err)
+		for alg, hasher := range hasherMap {
+			hashSumMap[alg] = hasher.Sum(nil)
+		}
+		require.Equal(t, hashSumMap.GetSumVal(access.HashAlgCRC32), crc32.ChecksumIEEE(data[:size]))
+	}
+	// 0
+	{
+		size := 0
+		err := streamer.PutAt(ctx(), newReader(size), clusterID, 1, 10000, int64(size), nil)
+		require.NotNil(t, err)
+	}
+	// 1 byte
+	{
+		size := 1
+		data := make([]byte, size)
+		rand.Read(data)
+		sumChecker(size, data)
+
+		shardSize := getBufSizes(len(data)).ShardSize
+		zeroShard := make([]byte, shardSize)
+		shard1 := make([]byte, shardSize)
+		copy(shard1, data)
+
+		require.Equal(t, shard1, dataShards.get(1001, 10000))
+		require.Equal(t, zeroShard, dataShards.get(1002, 10000))
+		require.Equal(t, zeroShard, dataShards.get(1004, 10000))
+		require.Equal(t, 0, len(dataShards.get(1005, 10000)))
+		require.NotEqual(t, ([]byte)(nil), dataShards.get(1007, 10000))
+	}
+	// 4M per shard
+	{
+		size := (1 << 22) * 6
+		data := make([]byte, size)
+		rand.Read(data)
+		sumChecker(size, data)
+
+		require.Equal(t, 1<<22, len(dataShards.get(1001, 10000)))
+	}
+	// 4M + 1B, put 1B
+	{
+		size := (1 << 22) + 1
+		data := make([]byte, size)
+		rand.Read(data)
+		sumChecker(1, data)
+
+		shardSize := getBufSizes(1).ShardSize
+		require.Equal(t, shardSize, len(dataShards.get(1001, 10000)))
+	}
+	// 4M, error
+	{
+		dataShards.clean()
+		size := 1 << 22
+		err := streamer.PutAt(ctx(), newReader(size), clusterID, 1, 10000, int64(size)+1, nil)
+		require.NotNil(t, err)
+		require.Equal(t, 0, len(dataShards.get(1001, 10000)))
+	}
+
+	dataShards.clean()
+}

blobstore/access/stream_test.go

@@ -0,0 +1,96 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"bytes"
+	"crypto/rand"
+	"io"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/access/controller"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+)
+
+func newReader(size int) io.Reader {
+	buff := make([]byte, size)
+	rand.Read(buff)
+	return bytes.NewReader(buff)
+}
+
+func TestAccessStreamConfig(t *testing.T) {
+	cfg := StreamConfig{
+		IDC:                idcOther,
+		MemPoolSizeClasses: map[int]int{1024: 1},
+		CodeModesPutQuorums: map[codemode.CodeMode]int{
+			codemode.EC15P12:  16,
+			codemode.EC6P10L2: 18,
+		},
+	}
+	confCheck(&cfg)
+
+	require.Equal(t, idcOther, cfg.IDC)
+	require.Equal(t, map[int]int{1024: 1}, cfg.MemPoolSizeClasses)
+	require.Equal(t, defaultDiskPunishIntervalS, cfg.DiskPunishIntervalS)
+}
+
+func TestAccessStreamNew(t *testing.T) {
+	require.Equal(t, idc, streamer.IDC)
+
+	require.Panics(t, func() {
+		NewStreamHandler(&StreamConfig{IDC: "idc"}, nil, nil)
+	})
+}
+
+func TestAccessStreamDelete(t *testing.T) {
+	ctx := ctxWithName("TestAccessStreamDelete")
+	size := 1 << 18
+	loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil)
+	require.NoError(t, err)
+
+	err = streamer.Delete(ctx(), loc)
+	require.NoError(t, err)
+
+	dataShards.clean()
+}
+
+func TestAccessStreamAdmin(t *testing.T) {
+	{
+		handler := Handler{}
+		sa := handler.Admin()
+		require.NotNil(t, sa)
+
+		admin := sa.(*streamAdmin)
+		require.Nil(t, admin.memPool)
+		require.Nil(t, admin.controller)
+	}
+	{
+		sa := streamer.Admin()
+		require.NotNil(t, sa)
+
+		admin := sa.(*streamAdmin)
+		require.NotNil(t, admin.memPool)
+		t.Log("mempool status:", admin.memPool.Status())
+
+		ctr := admin.controller
+		require.NotNil(t, ctr)
+		t.Log("region:", ctr.Region())
+		require.Error(t, ctr.ChangeChooseAlg(controller.AlgChoose(100)))
+		require.NoError(t, ctr.ChangeChooseAlg(controller.AlgRandom))
+		require.NoError(t, ctr.ChangeChooseAlg(controller.AlgAvailable))
+	}
+}

blobstore/access/stream_time.go

@@ -0,0 +1,47 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"fmt"
+	"sync/atomic"
+	"time"
+)
+
+type timeReadWrite struct {
+	a int64 // alloc buffer
+	r int64 // PUT: read from client,  GET: read from blobnode
+	w int64 // PUT: write to blobnode, GET: write to client
+}
+
+func (t *timeReadWrite) IncA(dur time.Duration) {
+	atomic.AddInt64(&t.a, int64(dur))
+}
+
+func (t *timeReadWrite) IncR(dur time.Duration) {
+	atomic.AddInt64(&t.r, int64(dur))
+}
+
+func (t *timeReadWrite) IncW(dur time.Duration) {
+	atomic.AddInt64(&t.w, int64(dur))
+}
+
+// String within milliseconds
+func (t *timeReadWrite) String() string {
+	a := atomic.LoadInt64(&t.a) / 1e6
+	r := atomic.LoadInt64(&t.r) / 1e6
+	w := atomic.LoadInt64(&t.w) / 1e6
+	return fmt.Sprintf("a_%d_r_%d_w_%d", a, r, w)
+}

blobstore/api/access/client.go

@@ -0,0 +1,809 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/hashicorp/consul/api"
+	"gopkg.in/natefinch/lumberjack.v2"
+
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/resourcepool"
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/util/defaulter"
+	"github.com/cubefs/cubefs/blobstore/util/log"
+	"github.com/cubefs/cubefs/blobstore/util/selector"
+	"github.com/cubefs/cubefs/blobstore/util/task"
+)
+
+const (
+	defaultMaxSizePutOnce    int64 = 1 << 28 // 256MB
+	defaultMaxPartRetry      int   = 3
+	defaultMaxHostRetry      int   = 3
+	defaultPartConcurrence   int   = 4
+	defaultServiceIntervalMs int64 = 5000
+	defaultServiceName             = "access"
+
+	_cacheBufferPutOnce = 1 << 23 // 8M
+)
+
+// RPCConnectMode self-defined rpc client connection config setting
+type RPCConnectMode uint8
+
+// timeout: [short - - - - - - - - -> long]
+//       quick --> general --> default --> slow --> nolimit
+// speed: 40MB -->  20MB   -->  10MB   --> 4MB  --> nolimit
+const (
+	DefaultConnMode RPCConnectMode = iota
+	QuickConnMode
+	GeneralConnMode
+	SlowConnMode
+	NoLimitConnMode
+)
+
+func (mode RPCConnectMode) getConfig(speed float64, timeout, baseTimeout int64) rpc.Config {
+	getSpeed := func(defaultVal float64) float64 {
+		if speed > 0 {
+			return speed
+		}
+		return defaultVal
+	}
+	getBaseTimeout := func(defaultVal int64) int64 {
+		if baseTimeout > 0 {
+			return baseTimeout
+		}
+		return defaultVal
+	}
+	getTimeout := func(speed float64) int64 {
+		if timeout > 0 {
+			return timeout
+		}
+		return 5 * (1 << 30) * 1e3 / int64(speed*(1<<20))
+	}
+
+	config := rpc.Config{
+		// the whole request and response timeout
+		ClientTimeoutMs:   getTimeout(getSpeed(10)),
+		BodyBandwidthMBPs: getSpeed(10),
+		BodyBaseTimeoutMs: getBaseTimeout(30 * 1000),
+		Tc: rpc.TransportConfig{
+			// dial timeout
+			DialTimeoutMs: 5 * 1000,
+			// response header timeout after send the request
+			ResponseHeaderTimeoutMs: 5 * 1000,
+			// IdleConnTimeout is the maximum amount of time an idle
+			// (keep-alive) connection will remain idle before closing
+			// itself.Zero means no limit.
+			IdleConnTimeoutMs: 30 * 1000,
+
+			MaxIdleConns:        0,
+			MaxConnsPerHost:     2048,
+			MaxIdleConnsPerHost: 1024,
+			DisableCompression:  true,
+		},
+	}
+
+	switch mode {
+	case QuickConnMode:
+		config.ClientTimeoutMs = getTimeout(getSpeed(40))
+		config.BodyBandwidthMBPs = getSpeed(40)
+		config.BodyBaseTimeoutMs = getBaseTimeout(3 * 1000)
+		config.Tc.DialTimeoutMs = 2 * 1000
+		config.Tc.ResponseHeaderTimeoutMs = 2 * 1000
+		config.Tc.IdleConnTimeoutMs = 10 * 1000
+	case GeneralConnMode:
+		config.ClientTimeoutMs = getTimeout(getSpeed(20))
+		config.BodyBandwidthMBPs = getSpeed(20)
+		config.BodyBaseTimeoutMs = getBaseTimeout(10 * 1000)
+		config.Tc.DialTimeoutMs = 3 * 1000
+		config.Tc.ResponseHeaderTimeoutMs = 3 * 1000
+		config.Tc.IdleConnTimeoutMs = 30 * 1000
+	case SlowConnMode:
+		config.ClientTimeoutMs = getTimeout(getSpeed(4))
+		config.BodyBandwidthMBPs = getSpeed(4)
+		config.BodyBaseTimeoutMs = getBaseTimeout(120 * 1000)
+		config.Tc.DialTimeoutMs = 10 * 1000
+		config.Tc.ResponseHeaderTimeoutMs = 10 * 1000
+		config.Tc.IdleConnTimeoutMs = 60 * 1000
+	case NoLimitConnMode:
+		config.ClientTimeoutMs = 0
+		config.BodyBandwidthMBPs = getSpeed(0)
+		config.BodyBaseTimeoutMs = getBaseTimeout(0)
+		config.Tc.DialTimeoutMs = 0
+		config.Tc.ResponseHeaderTimeoutMs = 0
+		config.Tc.IdleConnTimeoutMs = 600 * 1000
+	default:
+	}
+
+	return config
+}
+
+// Config access client config
+type Config struct {
+	// ConnMode rpc connection timeout setting
+	ConnMode RPCConnectMode
+	// ClientTimeoutMs the whole request and response timeout
+	ClientTimeoutMs int64
+	// BodyBandwidthMBPs reading body timeout, request or response
+	//   timeout = ContentLength/BodyBandwidthMBPs + BodyBaseTimeoutMs
+	BodyBandwidthMBPs float64
+	// BodyBaseTimeoutMs base timeout for read body
+	BodyBaseTimeoutMs int64
+
+	// Consul is consul config for discovering service
+	Consul ConsulConfig
+	// ServiceIntervalMs is interval ms for discovering service
+	ServiceIntervalMs int64
+	// PriorityAddrs priority addrs of access service when retry
+	PriorityAddrs []string
+	// MaxSizePutOnce max size using once-put object interface
+	MaxSizePutOnce int64
+	// MaxPartRetry max retry times when putting one part, 0 means forever
+	MaxPartRetry int
+	// MaxHostRetry max retry hosts of access service
+	MaxHostRetry int
+	// PartConcurrence concurrence of put parts
+	PartConcurrence int
+
+	// RPCConfig user-defined rpc config
+	// All connections will use the config if it's not nil
+	// ConnMode will be ignored if rpc config is setting
+	RPCConfig *rpc.Config
+
+	// LogLevel client output logging level.
+	LogLevel log.Level
+
+	// Logger trace all logging to the logger if setting.
+	// It is an io.WriteCloser that writes to the specified filename.
+	// YOU should CLOSE it after you do not use the client anymore.
+	Logger *Logger
+}
+
+// ConsulConfig alias of consul api.Config
+// Fixup: client and sdk using the same config type
+type ConsulConfig = api.Config
+
+// Logger alias of lumberjack Logger
+// See more at: https://github.com/natefinch/lumberjack
+type Logger = lumberjack.Logger
+
+// client access rpc client
+type client struct {
+	config       Config
+	consulClient *api.Client
+	selector     selector.Selector
+	rpcClient    rpc.Client
+}
+
+// API access api for s3
+// To trace request id, the ctx is better WithRequestID(ctx, rid).
+type API interface {
+	// Put object once if size is not greater than MaxSizePutOnce, otherwise put blobs one by one.
+	// return a location and map of hash summary bytes you excepted.
+	Put(ctx context.Context, args *PutArgs) (location Location, hashSumMap HashSumMap, err error)
+	// Get object, range is supported.
+	Get(ctx context.Context, args *GetArgs) (body io.ReadCloser, err error)
+	// Delete all blobs in these locations.
+	// return failed locations which have yet been deleted if error is not nil.
+	Delete(ctx context.Context, args *DeleteArgs) (failedLocations []Location, err error)
+}
+
+var _ API = (*client)(nil)
+
+type hadReader struct {
+	isRead bool
+	reader io.Reader
+}
+
+// Read the io.Reader had been read
+func (r *hadReader) Read(p []byte) (n int, err error) {
+	r.isRead = true
+	return r.reader.Read(p)
+}
+
+type noopBody struct{}
+
+var _ io.ReadCloser = (*noopBody)(nil)
+
+func (rc noopBody) Read(p []byte) (n int, err error) { return 0, io.EOF }
+func (rc noopBody) Close() error                     { return nil }
+
+var memPool *resourcepool.MemPool
+
+func init() {
+	memPool = resourcepool.NewMemPool(map[int]int{
+		1 << 12: -1,
+		1 << 14: -1,
+		1 << 18: -1,
+		1 << 20: -1,
+		1 << 22: -1,
+		1 << 23: -1,
+		1 << 24: -1,
+	})
+}
+
+// New returns an access API
+func New(cfg Config) (API, error) {
+	defaulter.LessOrEqual(&cfg.MaxSizePutOnce, defaultMaxSizePutOnce)
+	defaulter.Less(&cfg.MaxPartRetry, defaultMaxPartRetry)
+	defaulter.LessOrEqual(&cfg.MaxHostRetry, defaultMaxHostRetry)
+	defaulter.LessOrEqual(&cfg.PartConcurrence, defaultPartConcurrence)
+	if cfg.ServiceIntervalMs < 500 {
+		cfg.ServiceIntervalMs = defaultServiceIntervalMs
+	}
+
+	var rpcClient rpc.Client
+	if cfg.RPCConfig != nil {
+		rpcClient = rpc.NewClient(cfg.RPCConfig)
+	} else {
+		rpcConfig := cfg.ConnMode.getConfig(cfg.BodyBandwidthMBPs,
+			cfg.ClientTimeoutMs, cfg.BodyBaseTimeoutMs)
+		rpcClient = rpc.NewClient(&rpcConfig)
+	}
+
+	log.SetOutputLevel(cfg.LogLevel)
+	if cfg.Logger != nil {
+		log.SetOutput(cfg.Logger)
+	}
+
+	consulConfig := api.Config(cfg.Consul)
+	consulClient, err := api.NewClient(&consulConfig)
+	if err != nil {
+		return nil, errcode.ErrAccessServiceDiscovery
+	}
+
+	first := true
+	serviceName := defaultServiceName
+	hostGetter := func() ([]string, error) {
+		if first && len(cfg.PriorityAddrs) > 0 {
+			hosts := make([]string, len(cfg.PriorityAddrs))
+			copy(hosts, cfg.PriorityAddrs[:])
+			first = false
+			return hosts, nil
+		}
+
+		services, _, err := consulClient.Health().Service(serviceName, "", true, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		hosts := make([]string, 0, len(services))
+		for _, s := range services {
+			address := s.Service.Address
+			if address == "" {
+				address = s.Node.Address
+			}
+			hosts = append(hosts, fmt.Sprintf("http://%s:%d", address, s.Service.Port))
+		}
+
+		if len(hosts) == 0 {
+			return nil, fmt.Errorf("unavailable service")
+		}
+
+		return hosts, nil
+	}
+
+	hostSelector, err := selector.NewSelector(cfg.ServiceIntervalMs, hostGetter)
+	if err != nil {
+		return nil, errcode.ErrAccessServiceDiscovery
+	}
+
+	return &client{
+		config:       cfg,
+		consulClient: consulClient,
+		selector:     hostSelector,
+		rpcClient:    rpcClient,
+	}, nil
+}
+
+func (c *client) Put(ctx context.Context, args *PutArgs) (location Location, hashSumMap HashSumMap, err error) {
+	if args.Size == 0 {
+		hashSumMap := args.Hashes.ToHashSumMap()
+		for alg := range hashSumMap {
+			hashSumMap[alg] = alg.ToHasher().Sum(nil)
+		}
+		return Location{Blobs: make([]SliceInfo, 0)}, hashSumMap, nil
+	}
+
+	ctx = withReqidContext(ctx)
+	if args.Size <= c.config.MaxSizePutOnce {
+		return c.putObject(ctx, args)
+	}
+	return c.putParts(ctx, args)
+}
+
+func (c *client) putObject(ctx context.Context, args *PutArgs) (location Location, hashSumMap HashSumMap, err error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	var (
+		cached bool
+		buffer []byte
+		reader *hadReader
+	)
+	if args.Size > 0 && args.Size <= _cacheBufferPutOnce {
+		cached = true
+		buffer, _ = memPool.Alloc(int(args.Size))
+		buffer = buffer[:args.Size]
+		defer memPool.Put(buffer)
+
+		_, err := io.ReadFull(args.Body, buffer)
+		if err != nil {
+			span.Error("read buffer from request", err)
+			return Location{}, nil, errcode.ErrAccessReadRequestBody
+		}
+	} else {
+		reader = &hadReader{
+			isRead: false,
+			reader: args.Body,
+		}
+	}
+
+	err = c.tryN(ctx, c.config.MaxHostRetry, func(host string) error {
+		var body io.Reader
+		if cached {
+			body = bytes.NewReader(buffer)
+		} else {
+			if reader.isRead {
+				span.Info("retry on other access host which had been read body")
+				return errcode.ErrAccessReadConflictBody
+			}
+			body = reader
+		}
+
+		urlStr := fmt.Sprintf("%s/put?size=%d&hashes=%d", host, args.Size, args.Hashes)
+		req, e := http.NewRequest(http.MethodPut, urlStr, body)
+		if e != nil {
+			return e
+		}
+
+		resp := &PutResp{}
+		e = c.rpcClient.DoWith(ctx, req, resp, rpc.WithCrcEncode())
+		if e == nil {
+			location = resp.Location
+			hashSumMap = resp.HashSumMap
+		}
+		return e
+	})
+	return
+}
+
+type blobPart struct {
+	cid   proto.ClusterID
+	vid   proto.Vid
+	bid   proto.BlobID
+	size  int
+	index int
+	token string
+	buf   []byte
+}
+
+func (c *client) putPartsBatch(ctx context.Context, parts []blobPart) error {
+	tasks := make([]func() error, 0, len(parts))
+	for _, part := range parts {
+		part := part
+		tasks = append(tasks, func() error {
+			return c.tryN(ctx, c.config.MaxHostRetry, func(host string) error {
+				urlStr := fmt.Sprintf("%s/putat?clusterid=%d&volumeid=%d&blobid=%d&size=%d&hashes=%d&token=%s",
+					host, part.cid, part.vid, part.bid, part.size, 0, part.token)
+				req, err := http.NewRequest(http.MethodPut, urlStr, bytes.NewReader(part.buf))
+				if err != nil {
+					return err
+				}
+				resp := &PutAtResp{}
+				return c.rpcClient.DoWith(ctx, req, resp, rpc.WithCrcEncode())
+			})
+		})
+	}
+
+	if err := task.Run(context.Background(), tasks...); err != nil {
+		for _, part := range parts {
+			part := part
+			// asynchronously delete blob
+			go func() {
+				c.tryN(ctx, c.config.MaxHostRetry, func(host string) error {
+					urlStr := fmt.Sprintf("%s/deleteblob?clusterid=%d&volumeid=%d&blobid=%d&size=%d&token=%s",
+						host, part.cid, part.vid, part.bid, part.size, part.token)
+					req, err := http.NewRequest(http.MethodDelete, urlStr, nil)
+					if err != nil {
+						return err
+					}
+					return c.rpcClient.DoWith(ctx, req, nil)
+				})
+			}()
+		}
+		return err
+	}
+	return nil
+}
+
+func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSumMap, error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	hashSumMap := args.Hashes.ToHashSumMap()
+	hasherMap := make(HasherMap, len(hashSumMap))
+	for alg := range hashSumMap {
+		hasherMap[alg] = alg.ToHasher()
+	}
+
+	reqBody := args.Body
+	if len(hasherMap) > 0 {
+		reqBody = io.TeeReader(args.Body, hasherMap.ToWriter())
+	}
+
+	var (
+		loc    Location
+		tokens []string
+	)
+
+	signArgs := SignArgs{}
+	success := false
+	defer func() {
+		if success {
+			return
+		}
+
+		locations := signArgs.Locations[:]
+		if len(locations) > 1 {
+			signArgs.Location = loc.Copy()
+			signResp := &SignResp{}
+			if err := c.tryN(ctx, c.config.MaxHostRetry, func(host string) error {
+				return c.rpcClient.PostWith(ctx, fmt.Sprintf("%s/sign", host), signResp, signArgs)
+			}); err == nil {
+				locations = []Location{signResp.Location.Copy()}
+			}
+		}
+		if len(locations) > 0 {
+			if _, err := c.Delete(ctx, &DeleteArgs{Locations: locations}); err != nil {
+				span.Warnf("clean location '%+v' failed %s", locations, err.Error())
+			}
+		}
+	}()
+
+	// alloc
+	err := c.tryN(ctx, c.config.MaxHostRetry, func(host string) error {
+		allocResp := &AllocResp{}
+		if err := c.rpcClient.PostWith(ctx, fmt.Sprintf("%s/alloc", host), allocResp, AllocArgs{
+			Size: uint64(args.Size),
+		}); err != nil {
+			return err
+		}
+		loc = allocResp.Location
+		tokens = allocResp.Tokens
+		return nil
+	})
+	if err != nil {
+		return loc, nil, err
+	}
+	signArgs.Locations = append(signArgs.Locations, loc.Copy())
+
+	// buffer pipeline
+	closeCh := make(chan struct{})
+	bufferPipe := func(size, blobSize int) <-chan []byte {
+		ch := make(chan []byte, c.config.PartConcurrence-1)
+		go func() {
+			for size > 0 {
+				toread := blobSize
+				if toread > size {
+					toread = size
+				}
+
+				buf, _ := memPool.Alloc(toread)
+				buf = buf[:toread]
+				_, err := io.ReadFull(reqBody, buf)
+				if err != nil {
+					span.Error("read buffer from request", err)
+					memPool.Put(buf)
+					close(ch)
+					return
+				}
+
+				select {
+				case <-closeCh:
+					memPool.Put(buf)
+					close(ch)
+					return
+				case ch <- buf:
+				}
+
+				size -= toread
+			}
+			close(ch)
+		}()
+		return ch
+	}(int(loc.Size), int(loc.BlobSize))
+
+	defer func() {
+		// waiting pipeline close if has error
+		for buf := range bufferPipe {
+			if len(buf) > 0 {
+				memPool.Put(buf)
+			}
+		}
+	}()
+
+	releaseBuffer := func(parts []blobPart) {
+		for _, part := range parts {
+			memPool.Put(part.buf)
+		}
+	}
+
+	currBlobIdx := 0
+	currBlobCount := uint32(0)
+	remainSize := loc.Size
+	restPartsLoc := loc
+
+	index := -1
+	readSize := 0
+	for readSize < int(loc.Size) {
+		parts := make([]blobPart, 0, c.config.PartConcurrence)
+
+		// waiting at least one blob
+		buf, ok := <-bufferPipe
+		if !ok && readSize < int(loc.Size) {
+			return Location{}, nil, errcode.ErrAccessReadRequestBody
+		}
+		index++
+		readSize += len(buf)
+		parts = append(parts, blobPart{size: len(buf), index: index, buf: buf})
+
+		more := true
+		for more && len(parts) < c.config.PartConcurrence {
+			select {
+			case buf, ok := <-bufferPipe:
+				if !ok {
+					if readSize < int(loc.Size) {
+						releaseBuffer(parts)
+						return Location{}, nil, errcode.ErrAccessReadRequestBody
+					}
+					more = false
+				} else {
+					index++
+					readSize += len(buf)
+					parts = append(parts, blobPart{size: len(buf), index: index, buf: buf})
+				}
+			default:
+				more = false
+			}
+		}
+
+		tryTimes := c.config.MaxPartRetry
+		for {
+			if uint32(len(loc.Blobs)) > MaxLocationBlobs {
+				releaseBuffer(parts)
+				close(closeCh)
+				return Location{}, nil, errcode.ErrUnexpected
+			}
+
+			// feed new params
+			currIdx := currBlobIdx
+			currCount := currBlobCount
+			for i := range parts {
+				token := tokens[currIdx]
+				if restPartsLoc.Size > uint64(loc.BlobSize) && parts[i].size < int(loc.BlobSize) {
+					token = tokens[currIdx+1]
+				}
+				parts[i].token = token
+				parts[i].cid = loc.ClusterID
+				parts[i].vid = loc.Blobs[currIdx].Vid
+				parts[i].bid = loc.Blobs[currIdx].MinBid + proto.BlobID(currCount)
+
+				currCount++
+				if loc.Blobs[currIdx].Count == currCount {
+					currIdx++
+					currCount = 0
+				}
+			}
+
+			err := c.putPartsBatch(ctx, parts)
+			if err == nil {
+				for _, part := range parts {
+					remainSize -= uint64(part.size)
+					currBlobCount++
+					// next blobs
+					if loc.Blobs[currBlobIdx].Count == currBlobCount {
+						currBlobIdx++
+						currBlobCount = 0
+					}
+				}
+
+				break
+			}
+			span.Warn("putat parts", err)
+
+			if tryTimes > 0 { // has retry setting
+				if tryTimes == 1 {
+					releaseBuffer(parts)
+					close(closeCh)
+					span.Error("exceed the max retry limit", c.config.MaxPartRetry)
+					return Location{}, nil, errcode.ErrUnexpected
+				}
+				tryTimes--
+			}
+
+			var restPartsResp *AllocResp
+			// alloc the rest parts
+			err = c.tryN(ctx, c.config.MaxHostRetry, func(host string) error {
+				resp := &AllocResp{}
+				if err := c.rpcClient.PostWith(ctx, fmt.Sprintf("%s/alloc", host), resp, AllocArgs{
+					Size:            remainSize,
+					BlobSize:        loc.BlobSize,
+					CodeMode:        loc.CodeMode,
+					AssignClusterID: loc.ClusterID,
+				}); err != nil {
+					return err
+				}
+				if len(resp.Location.Blobs) > 0 {
+					if newVid := resp.Location.Blobs[0].Vid; newVid == loc.Blobs[currBlobIdx].Vid {
+						return fmt.Errorf("alloc the same vid %d", newVid)
+					}
+				}
+				restPartsResp = resp
+				return nil
+			})
+			if err != nil {
+				releaseBuffer(parts)
+				close(closeCh)
+				span.Error("alloc another parts to put", err)
+				return Location{}, nil, errcode.ErrUnexpected
+			}
+
+			restPartsLoc = restPartsResp.Location
+			signArgs.Locations = append(signArgs.Locations, restPartsLoc.Copy())
+
+			if currBlobCount > 0 {
+				loc.Blobs[currBlobIdx].Count = currBlobCount
+				currBlobIdx++
+			}
+			loc.Blobs = append(loc.Blobs[:currBlobIdx], restPartsLoc.Blobs...)
+			tokens = append(tokens[:currBlobIdx], restPartsResp.Tokens...)
+
+			currBlobCount = 0
+		}
+
+		releaseBuffer(parts)
+	}
+
+	if len(signArgs.Locations) > 1 {
+		signArgs.Location = loc.Copy()
+		// sign
+		err = c.tryN(ctx, c.config.MaxHostRetry, func(host string) error {
+			signResp := &SignResp{}
+			if err := c.rpcClient.PostWith(ctx, fmt.Sprintf("%s/sign", host), signResp, signArgs); err != nil {
+				return err
+			}
+			loc = signResp.Location
+			return nil
+		})
+		if err != nil {
+			span.Error("sign location with crc", err)
+			return Location{}, nil, errcode.ErrUnexpected
+		}
+	}
+
+	for alg, hasher := range hasherMap {
+		hashSumMap[alg] = hasher.Sum(nil)
+	}
+	success = true
+	return loc, hashSumMap, nil
+}
+
+func (c *client) Get(ctx context.Context, args *GetArgs) (body io.ReadCloser, err error) {
+	if !args.IsValid() {
+		return nil, errcode.ErrIllegalArguments
+	}
+
+	ctx = withReqidContext(ctx)
+	if args.Location.Size == 0 || args.ReadSize == 0 {
+		return noopBody{}, nil
+	}
+
+	err = c.tryN(ctx, c.config.MaxHostRetry, func(host string) error {
+		gotBody, e := func() (io.ReadCloser, error) {
+			resp, e := c.rpcClient.Post(ctx, fmt.Sprintf("%s/get", host), args)
+			if e != nil {
+				return nil, e
+			}
+			if resp.StatusCode >= 400 {
+				return nil, rpc.NewError(resp.StatusCode, "StatusCode", fmt.Errorf("code: %d", resp.StatusCode))
+			}
+			return resp.Body, nil
+		}()
+		if e != nil {
+			return e
+		}
+		body = gotBody
+		return nil
+	})
+	return
+}
+
+func (c *client) Delete(ctx context.Context, args *DeleteArgs) ([]Location, error) {
+	if !args.IsValid() {
+		if args == nil {
+			return nil, errcode.ErrIllegalArguments
+		}
+		return args.Locations, errcode.ErrIllegalArguments
+	}
+
+	ctx = withReqidContext(ctx)
+	locations := make([]Location, 0, len(args.Locations))
+	for _, loc := range args.Locations {
+		if loc.Size > 0 {
+			locations = append(locations, loc.Copy())
+		}
+	}
+	if len(locations) == 0 {
+		return nil, nil
+	}
+
+	err := c.tryN(ctx, c.config.MaxHostRetry, func(host string) error {
+		// access response 2xx even if there has failed locations
+		deleteResp := &DeleteResp{}
+		if err := c.rpcClient.PostWith(ctx, fmt.Sprintf("%s/delete", host), deleteResp,
+			DeleteArgs{Locations: locations}); err != nil && rpc.DetectStatusCode(err) != http.StatusIMUsed {
+			return err
+		}
+		if len(deleteResp.FailedLocations) > 0 {
+			locations = deleteResp.FailedLocations[:]
+			return errcode.ErrUnexpected
+		}
+		return nil
+	})
+	if err == nil {
+		return nil, nil
+	}
+	return locations, err
+}
+
+func (c *client) tryN(ctx context.Context, n int, connector func(string) error) error {
+	span := trace.SpanFromContextSafe(ctx)
+
+	hs := c.selector.GetRandomN(n)
+	hosts := append(c.config.PriorityAddrs[:], hs...)
+	if len(hosts) == 0 {
+		return errcode.ErrAccessServiceDiscovery
+	}
+
+	var host string
+	for _, host = range hosts {
+		err := connector(host)
+		if err == nil {
+			return nil
+		}
+
+		// has connected access node
+		if httpErr, ok := err.(rpc.HTTPError); ok {
+			// 500 need to retry next host
+			if httpErr.StatusCode() == http.StatusInternalServerError {
+				span.Warn("httpcode 500 need to try next, failed on", host, err)
+				continue
+			}
+			return err
+		}
+
+		// cannot connected most probably
+		span.Warn("failed on", host, err)
+	}
+
+	span.Error("try all hosts failed, recently on", host)
+	return errcode.ErrUnexpected
+}

blobstore/api/access/client_reqid.go

@@ -0,0 +1,74 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"context"
+
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+)
+
+type ctxKey uint8
+
+const (
+	_operationName = "access_client"
+)
+
+const (
+	_ ctxKey = iota
+	reqidKey
+)
+
+// WithRequestID trace request id in full life of the request
+// The second parameter rid could be the one of type below:
+//     a string,
+//     an interface { String() string },
+//     an interface { TraceID() string },
+//     an interface { RequestID() string },
+func WithRequestID(ctx context.Context, rid interface{}) context.Context {
+	return context.WithValue(ctx, reqidKey, rid)
+}
+
+func reqidFromContext(ctx context.Context) (string, bool) {
+	val := ctx.Value(reqidKey)
+	if val == nil {
+		return "", false
+	}
+	if rid, ok := val.(string); ok {
+		return rid, true
+	}
+	if rid, ok := val.(interface{ String() string }); ok {
+		return rid.String(), true
+	}
+	if rid, ok := val.(interface{ TraceID() string }); ok {
+		return rid.TraceID(), true
+	}
+	if rid, ok := val.(interface{ RequestID() string }); ok {
+		return rid.RequestID(), true
+	}
+	return "", false
+}
+
+func withReqidContext(ctx context.Context) context.Context {
+	if rid, ok := reqidFromContext(ctx); ok {
+		_, ctx := trace.StartSpanFromContextWithTraceID(ctx, _operationName, rid)
+		return ctx
+	}
+	if span := trace.SpanFromContext(ctx); span != nil {
+		return ctx
+	}
+	_, ctx = trace.StartSpanFromContext(ctx, _operationName)
+	return ctx
+}

blobstore/api/access/client_test.go

@@ -0,0 +1,936 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access_test
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"fmt"
+	"hash/crc32"
+	"io"
+	"io/ioutil"
+	mrand "math/rand"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/consul/api"
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/api/access"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/common/uptoken"
+	"github.com/cubefs/cubefs/blobstore/util/bytespool"
+	"github.com/cubefs/cubefs/blobstore/util/log"
+)
+
+const (
+	blobSize = 1 << 20
+)
+
+type dataCacheT struct {
+	mu   sync.Mutex
+	data map[proto.BlobID][]byte
+}
+
+func (c *dataCacheT) clean() {
+	c.mu.Lock()
+	c.data = make(map[proto.BlobID][]byte, len(c.data))
+	c.mu.Unlock()
+}
+
+func (c *dataCacheT) put(bid proto.BlobID, b []byte) {
+	c.mu.Lock()
+	c.data[bid] = b
+	c.mu.Unlock()
+}
+
+func (c *dataCacheT) get(bid proto.BlobID) []byte {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.data[bid]
+}
+
+var (
+	mockServer *httptest.Server
+	client     access.API
+	dataCache  *dataCacheT
+	services   []*api.ServiceEntry
+	hostsApply []*api.ServiceEntry
+	tokenAlloc = []byte("token")
+	tokenPutat = []byte("token")
+
+	partRandBroken = false
+)
+
+func init() {
+	log.SetOutputLevel(log.Lfatal)
+
+	t := time.Now()
+	services = make([]*api.ServiceEntry, 0, 2)
+	services = append(services, &api.ServiceEntry{
+		Node: &api.Node{
+			ID: "unreachable",
+		},
+		Service: &api.AgentService{
+			Service: "access",
+			Address: "127.0.0.1",
+			Port:    9997,
+		},
+	})
+
+	dataCache = &dataCacheT{}
+	dataCache.clean()
+
+	rpc.RegisterArgsParser(&access.PutArgs{}, "json")
+	rpc.RegisterArgsParser(&access.PutAtArgs{}, "json")
+
+	handler := rpc.New()
+	handler.Handle(http.MethodGet, "/v1/health/service/access", handleService)
+	handler.Handle(http.MethodPost, "/alloc", handleAlloc, rpc.OptArgsBody())
+	handler.Handle(http.MethodPut, "/put", handlePut, rpc.OptArgsQuery())
+	handler.Handle(http.MethodPut, "/putat", handlePutAt, rpc.OptArgsQuery())
+	handler.Handle(http.MethodPost, "/get", handleGet, rpc.OptArgsBody())
+	handler.Handle(http.MethodPost, "/delete", handleDelete, rpc.OptArgsBody())
+	handler.Handle(http.MethodPost, "/sign", handleSign, rpc.OptArgsBody())
+	handler.Router.NotFound = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	})
+	mockServer = httptest.NewServer(handler)
+
+	u := strings.Split(mockServer.URL[7:], ":")
+	port, _ := strconv.Atoi(u[1])
+	services = append(services, &api.ServiceEntry{
+		Node: &api.Node{
+			ID: "mockServer",
+		},
+		Service: &api.AgentService{
+			Service: "access",
+			Address: u[0],
+			Port:    port,
+		},
+	})
+	hostsApply = services
+
+	cfg := access.Config{}
+	cfg.Consul.Address = mockServer.URL[7:] // strip http://
+	cfg.PriorityAddrs = []string{mockServer.URL}
+	cfg.ConnMode = access.QuickConnMode
+	cfg.MaxSizePutOnce = 1 << 20
+	cfg.PartConcurrence = 2
+	cfg.LogLevel = log.Lfatal
+	cli, err := access.New(cfg)
+	if err != nil {
+		panic(err)
+	}
+	client = cli
+
+	mrand.Seed(int64(time.Since(t)))
+}
+
+func handleService(c *rpc.Context) {
+	c.RespondJSON(hostsApply)
+}
+
+func handleAlloc(c *rpc.Context) {
+	args := new(access.AllocArgs)
+	if err := c.ParseArgs(args); err != nil {
+		c.RespondError(err)
+		return
+	}
+
+	loc := access.Location{
+		ClusterID: 1,
+		Size:      args.Size,
+		BlobSize:  blobSize,
+		Blobs: []access.SliceInfo{
+			{
+				MinBid: proto.BlobID(mrand.Int()),
+				Vid:    proto.Vid(mrand.Int()),
+				Count:  uint32((args.Size + blobSize - 1) / blobSize),
+			},
+		},
+	}
+	// split to two blobs if large enough
+	if loc.Blobs[0].Count > 2 {
+		loc.Blobs[0].Count = 2
+		loc.Blobs = append(loc.Blobs, []access.SliceInfo{
+			{
+				MinBid: proto.BlobID(mrand.Int()),
+				Vid:    proto.Vid(mrand.Int()),
+				Count:  uint32((args.Size - 2*blobSize + blobSize - 1) / blobSize),
+			},
+		}...)
+	}
+	// alloc the rest parts
+	if args.AssignClusterID > 0 {
+		loc.Blobs = []access.SliceInfo{
+			{
+				MinBid: proto.BlobID(mrand.Int()),
+				Vid:    proto.Vid(mrand.Int()),
+				Count:  uint32((args.Size + blobSize - 1) / blobSize),
+			},
+		}
+	}
+
+	tokens := make([]string, 0, len(loc.Blobs)+1)
+
+	hasMultiBlobs := loc.Size >= uint64(loc.BlobSize)
+	lastSize := uint32(loc.Size % uint64(loc.BlobSize))
+	for idx, blob := range loc.Blobs {
+		// returns one token if size < blobsize
+		if hasMultiBlobs {
+			count := blob.Count
+			if idx == len(loc.Blobs)-1 && lastSize > 0 {
+				count--
+			}
+			tokens = append(tokens, uptoken.EncodeToken(uptoken.NewUploadToken(loc.ClusterID,
+				blob.Vid, blob.MinBid, count,
+				loc.BlobSize, 0, tokenAlloc[:])))
+		}
+
+		// token of the last blob
+		if idx == len(loc.Blobs)-1 && lastSize > 0 {
+			tokens = append(tokens, uptoken.EncodeToken(uptoken.NewUploadToken(loc.ClusterID,
+				blob.Vid, blob.MinBid+proto.BlobID(blob.Count)-1, 1,
+				lastSize, 0, tokenAlloc[:])))
+		}
+	}
+
+	fillCrc(&loc)
+	c.RespondJSON(access.AllocResp{
+		Location: loc,
+		Tokens:   tokens,
+	})
+}
+
+func handlePut(c *rpc.Context) {
+	args := new(access.PutArgs)
+	if err := c.ParseArgs(args); err != nil {
+		c.RespondError(err)
+		return
+	}
+
+	// just for testing timeout
+	if args.Size < 0 {
+		time.Sleep(30 * time.Second)
+		c.RespondStatus(http.StatusForbidden)
+		return
+	}
+
+	buf := make([]byte, args.Size)
+	io.ReadFull(c.Request.Body, buf)
+	dataCache.put(0, buf)
+
+	hashSumMap := args.Hashes.ToHashSumMap()
+	for alg := range hashSumMap {
+		hasher := alg.ToHasher()
+		hasher.Write(buf)
+		hashSumMap[alg] = hasher.Sum(nil)
+	}
+
+	loc := access.Location{Size: uint64(args.Size)}
+	fillCrc(&loc)
+	c.RespondJSON(access.PutResp{
+		Location:   loc,
+		HashSumMap: hashSumMap,
+	})
+}
+
+func handlePutAt(c *rpc.Context) {
+	args := new(access.PutAtArgs)
+	if err := c.ParseArgs(args); err != nil {
+		c.RespondError(err)
+		return
+	}
+
+	if partRandBroken && args.Blobid%3 == 0 { // random broken
+		c.RespondStatus(http.StatusForbidden)
+		return
+	}
+
+	token := uptoken.DecodeToken(args.Token)
+	if !token.IsValid(args.ClusterID, args.Vid, args.Blobid, uint32(args.Size), tokenPutat[:]) {
+		c.RespondStatus(http.StatusForbidden)
+		return
+	}
+
+	blobBuf := make([]byte, args.Size)
+	io.ReadFull(c.Request.Body, blobBuf)
+
+	hashSumMap := args.Hashes.ToHashSumMap()
+	for alg := range hashSumMap {
+		hasher := alg.ToHasher()
+		hasher.Write(blobBuf)
+		hashSumMap[alg] = hasher.Sum(nil)
+	}
+
+	dataCache.put(args.Blobid, blobBuf)
+	c.RespondJSON(access.PutAtResp{HashSumMap: hashSumMap})
+}
+
+func handleGet(c *rpc.Context) {
+	args := new(access.GetArgs)
+	if err := c.ParseArgs(args); err != nil {
+		c.RespondError(err)
+		return
+	}
+
+	if !verifyCrc(&args.Location) {
+		c.RespondStatus(http.StatusForbidden)
+		return
+	}
+
+	c.Writer.Header().Set(rpc.HeaderContentLength, strconv.Itoa(int(args.ReadSize)))
+	c.RespondStatus(http.StatusOK)
+	if buf := dataCache.get(0); len(buf) > 0 {
+		c.Writer.Write(buf)
+	} else {
+		for _, blob := range args.Location.Spread() {
+			c.Writer.Write(dataCache.get(blob.Bid))
+		}
+	}
+}
+
+func handleDelete(c *rpc.Context) {
+	args := new(access.DeleteArgs)
+	if err := c.ParseArgs(args); err != nil {
+		c.RespondError(err)
+		return
+	}
+
+	if !args.IsValid() {
+		c.RespondStatus(http.StatusBadRequest)
+		return
+	}
+	for _, loc := range args.Locations {
+		if !verifyCrc(&loc) {
+			c.RespondStatus(http.StatusBadRequest)
+			return
+		}
+	}
+
+	if len(args.Locations) > 0 && len(args.Locations)%2 == 0 {
+		locs := args.Locations[:]
+		c.RespondStatusData(http.StatusIMUsed, access.DeleteResp{FailedLocations: locs})
+		return
+	}
+	c.RespondJSON(access.DeleteResp{})
+}
+
+func handleSign(c *rpc.Context) {
+	args := new(access.SignArgs)
+	if err := c.ParseArgs(args); err != nil {
+		c.RespondError(err)
+		return
+	}
+
+	if err := signCrc(&args.Location, args.Locations); err != nil {
+		c.RespondStatus(http.StatusForbidden)
+		return
+	}
+	c.RespondJSON(access.SignResp{Location: args.Location})
+}
+
+func calcCrc(loc *access.Location) (uint32, error) {
+	crcWriter := crc32.New(crc32.IEEETable)
+
+	buf := bytespool.Alloc(1024)
+	defer bytespool.Free(buf)
+
+	n := loc.Encode2(buf)
+	if n < 4 {
+		return 0, fmt.Errorf("no enough bytes(%d) fill into buf", n)
+	}
+
+	if _, err := crcWriter.Write(buf[4:n]); err != nil {
+		return 0, fmt.Errorf("fill crc %s", err.Error())
+	}
+
+	return crcWriter.Sum32(), nil
+}
+
+func fillCrc(loc *access.Location) error {
+	crc, err := calcCrc(loc)
+	if err != nil {
+		return err
+	}
+	loc.Crc = crc
+	return nil
+}
+
+func verifyCrc(loc *access.Location) bool {
+	crc, err := calcCrc(loc)
+	if err != nil {
+		return false
+	}
+	return loc.Crc == crc
+}
+
+func signCrc(loc *access.Location, locs []access.Location) error {
+	first := locs[0]
+	bids := make(map[proto.BlobID]struct{}, 64)
+
+	if loc.ClusterID != first.ClusterID ||
+		loc.CodeMode != first.CodeMode ||
+		loc.BlobSize != first.BlobSize {
+		return fmt.Errorf("not equal in constant field")
+	}
+
+	for _, l := range locs {
+		if !verifyCrc(&l) {
+			return fmt.Errorf("not equal in crc %d", l.Crc)
+		}
+
+		// assert
+		if l.ClusterID != first.ClusterID ||
+			l.CodeMode != first.CodeMode ||
+			l.BlobSize != first.BlobSize {
+			return fmt.Errorf("not equal in constant field")
+		}
+
+		for _, blob := range l.Blobs {
+			for c := 0; c < int(blob.Count); c++ {
+				bids[blob.MinBid+proto.BlobID(c)] = struct{}{}
+			}
+		}
+	}
+
+	for _, blob := range loc.Blobs {
+		for c := 0; c < int(blob.Count); c++ {
+			bid := blob.MinBid + proto.BlobID(c)
+			if _, ok := bids[bid]; !ok {
+				return fmt.Errorf("not equal in blob_id(%d)", bid)
+			}
+		}
+	}
+
+	return fillCrc(loc)
+}
+
+type stringid struct{ id string }
+
+func (s stringid) String() string { return s.id }
+
+type traceid struct{ id string }
+
+func (t traceid) TraceID() string { return t.id }
+
+type requestid struct{ id string }
+
+func (r requestid) RequestID() string { return r.id }
+
+func randCtx() context.Context {
+	ctx := context.Background()
+
+	switch mrand.Int31() % 7 {
+	case 0:
+		return ctx
+	case 1:
+		return access.WithRequestID(ctx, nil)
+	case 2:
+		return access.WithRequestID(ctx, "TestAccessClient-string")
+	case 3:
+		return access.WithRequestID(ctx, stringid{"TestAccessClient-String"})
+	case 4:
+		return access.WithRequestID(ctx, traceid{"TestAccessClient-TraceID"})
+	case 5:
+		return access.WithRequestID(ctx, requestid{"TestAccessClient-RequestID"})
+	case 6:
+		return access.WithRequestID(ctx, struct{}{})
+	default:
+	}
+
+	_, ctx = trace.StartSpanFromContext(ctx, "TestAccessClient")
+	return ctx
+}
+
+func TestAccessClientConnectionMode(t *testing.T) {
+	cases := []struct {
+		mode access.RPCConnectMode
+		size int64
+	}{
+		{0, 0},
+		{0, -1},
+		{access.QuickConnMode, 1 << 10},
+		{access.GeneralConnMode, 1 << 10},
+		{access.SlowConnMode, 1 << 10},
+		{access.NoLimitConnMode, 1 << 10},
+		{access.DefaultConnMode, 1 << 10},
+		{100, 1 << 10},
+		{0xff, 100},
+	}
+
+	for _, cs := range cases {
+		cfg := access.Config{}
+		cfg.Consul.Address = mockServer.URL[7:]
+		cfg.MaxSizePutOnce = cs.size
+		cfg.ConnMode = cs.mode
+		cfg.LogLevel = log.Lfatal
+		cli, err := access.New(cfg)
+		require.NoError(t, err)
+
+		if cs.size <= 0 {
+			continue
+		}
+		loc := access.Location{Size: uint64(mrand.Int63n(cs.size))}
+		fillCrc(&loc)
+		_, err = cli.Delete(randCtx(), &access.DeleteArgs{
+			Locations: []access.Location{loc},
+		})
+		require.NoError(t, err)
+	}
+}
+
+func TestAccessClientPutGet(t *testing.T) {
+	cases := []struct {
+		size int
+	}{
+		{0},
+		{1},
+		{1 << 6},
+		{1 << 10},
+		{1 << 19},
+		{(1 << 19) + 1},
+		{1 << 20},
+	}
+	for _, cs := range cases {
+		buff := make([]byte, cs.size)
+		rand.Read(buff)
+		crcExpected := crc32.ChecksumIEEE(buff)
+		args := access.PutArgs{
+			Size:   int64(cs.size),
+			Hashes: access.HashAlgCRC32,
+			Body:   bytes.NewBuffer(buff),
+		}
+
+		loc, hashSumMap, err := client.Put(randCtx(), &args)
+		crc, _ := hashSumMap.GetSum(access.HashAlgCRC32)
+		require.NoError(t, err)
+		require.Equal(t, crcExpected, crc)
+
+		body, err := client.Get(randCtx(), &access.GetArgs{Location: loc, ReadSize: uint64(cs.size)})
+		require.NoError(t, err)
+		defer body.Close()
+
+		io.ReadFull(body, buff)
+		require.Equal(t, crcExpected, crc32.ChecksumIEEE(buff))
+	}
+}
+
+func TestAccessClientPutAtBase(t *testing.T) {
+	cfg := access.Config{}
+	cfg.Consul.Address = mockServer.URL[7:]
+	cfg.MaxSizePutOnce = 1 << 20
+	cfg.PartConcurrence = 2
+	cfg.LogLevel = log.Lfatal
+	client, err := access.New(cfg)
+	require.NoError(t, err)
+
+	cases := []struct {
+		size int
+	}{
+		{0},
+		{1},
+		{1 << 10},
+		{(1 << 19) + 1},
+		{1 << 20},
+		{1<<20 + 1023},
+		{1<<23 + 1023},
+		{1 << 24},
+	}
+	for _, cs := range cases {
+		dataCache.clean()
+
+		buff := make([]byte, cs.size)
+		rand.Read(buff)
+		crcExpected := crc32.ChecksumIEEE(buff)
+		args := access.PutArgs{
+			Size:   int64(cs.size),
+			Hashes: access.HashAlgCRC32,
+			Body:   bytes.NewBuffer(buff),
+		}
+
+		loc, hashSumMap, err := client.Put(randCtx(), &args)
+		crc, _ := hashSumMap.GetSum(access.HashAlgCRC32)
+		require.NoError(t, err)
+		require.Equal(t, crcExpected, crc)
+
+		body, err := client.Get(randCtx(), &access.GetArgs{Location: loc, ReadSize: uint64(cs.size)})
+		require.NoError(t, err)
+		defer body.Close()
+
+		io.ReadFull(body, buff)
+		require.Equal(t, crcExpected, crc32.ChecksumIEEE(buff))
+	}
+}
+
+func TestAccessClientPutAtMerge(t *testing.T) {
+	cfg := access.Config{}
+	cfg.Consul.Address = mockServer.URL[7:]
+	cfg.MaxSizePutOnce = 1 << 20
+	cfg.PartConcurrence = 2
+	cfg.LogLevel = log.Lfatal
+	client, err := access.New(cfg)
+	require.NoError(t, err)
+
+	partRandBroken = true
+	defer func() {
+		partRandBroken = false
+	}()
+	dataCache.clean()
+
+	cases := []struct {
+		size int
+	}{
+		{1<<21 + 77777},
+		{1<<21 + 77777},
+		{1<<21 + 77777},
+		{1 << 22},
+		{1 << 22},
+		{1 << 22},
+	}
+	for _, cs := range cases {
+		dataCache.clean()
+
+		size := cs.size
+		buff := make([]byte, size)
+		rand.Read(buff)
+		crcExpected := crc32.ChecksumIEEE(buff)
+		args := access.PutArgs{
+			Size:   int64(size),
+			Hashes: access.HashAlgCRC32,
+			Body:   bytes.NewBuffer(buff),
+		}
+
+		loc, hashSumMap, err := client.Put(randCtx(), &args)
+		crc, _ := hashSumMap.GetSum(access.HashAlgCRC32)
+		require.NoError(t, err)
+		require.Equal(t, crcExpected, crc)
+
+		body, err := client.Get(randCtx(), &access.GetArgs{Location: loc, ReadSize: uint64(size)})
+		require.NoError(t, err)
+		defer body.Close()
+
+		io.ReadFull(body, buff)
+		require.Equal(t, crcExpected, crc32.ChecksumIEEE(buff))
+	}
+}
+
+func TestAccessClientPutMaxBlobsLength(t *testing.T) {
+	cfg := access.Config{}
+	cfg.Consul.Address = mockServer.URL[7:]
+	cfg.MaxSizePutOnce = 1 << 20
+	cfg.PartConcurrence = 2
+	cfg.LogLevel = log.Lfatal
+	client, err := access.New(cfg)
+	require.NoError(t, err)
+
+	partRandBroken = true
+	defer func() {
+		partRandBroken = false
+	}()
+
+	cases := []struct {
+		size int
+		err  error
+	}{
+		{1 << 20, nil},
+		{(1 << 21) + 1023, nil},
+		{1 << 25, errcode.ErrUnexpected},
+	}
+	for _, cs := range cases {
+		dataCache.clean()
+
+		buff := make([]byte, cs.size)
+		rand.Read(buff)
+		args := access.PutArgs{
+			Size:   int64(cs.size),
+			Hashes: access.HashAlgDummy,
+			Body:   bytes.NewBuffer(buff),
+		}
+
+		_, _, err := client.Put(randCtx(), &args)
+		require.ErrorIs(t, cs.err, err)
+	}
+}
+
+func linearTimeoutMs(baseSec, size, speedMBps float64) int64 {
+	const alignMs = 500
+	spentSec := size / 1024.0 / 1024.0 / speedMBps
+	timeoutMs := int64((baseSec + spentSec) * 1000)
+	if timeoutMs < 0 {
+		timeoutMs = 0
+	}
+	if ms := timeoutMs % alignMs; ms > 0 {
+		timeoutMs += (alignMs - ms)
+	}
+	return timeoutMs
+}
+
+func TestAccessClientPutTimeout(t *testing.T) {
+	cfg := access.Config{}
+	cfg.Consul.Address = mockServer.URL[7:]
+	cfg.LogLevel = log.Lfatal
+
+	mb := int(1 << 20)
+	ms := time.Millisecond
+
+	cases := []struct {
+		mode  access.RPCConnectMode
+		size  int
+		minMs time.Duration
+		maxMs time.Duration
+	}{
+		// QuickConnMode 3s + size / 40, dial and response 2s
+		{access.QuickConnMode, mb * -119, ms * 0, ms * 600},
+		{access.QuickConnMode, mb * -100, ms * 500, ms * 1000},
+		{access.QuickConnMode, mb * -80, ms * 1000, ms * 1500},
+		{access.QuickConnMode, mb * -1, ms * 2000, ms * 2500},
+
+		// DefaultConnMode 30s + size / 10, dial and response 5s
+		{access.DefaultConnMode, mb * -299, ms * 0, ms * 600},
+		{access.DefaultConnMode, mb * -280, ms * 2000, ms * 2500},
+		{access.DefaultConnMode, mb * -270, ms * 3000, ms * 3500},
+		{access.DefaultConnMode, mb * -1, ms * 5000, ms * 5500},
+	}
+	for _, cs := range cases {
+		cfg.ConnMode = cs.mode
+		switch cs.mode {
+		case access.QuickConnMode:
+			cfg.ClientTimeoutMs = linearTimeoutMs(3, float64(cs.size), 40)
+		case access.DefaultConnMode:
+			cfg.ClientTimeoutMs = linearTimeoutMs(30, float64(cs.size), 10)
+		}
+		client, err := access.New(cfg)
+		require.NoError(t, err)
+
+		buff := make([]byte, 0)
+		args := access.PutArgs{
+			Size: int64(cs.size),
+			Body: bytes.NewBuffer(buff),
+		}
+
+		startTime := time.Now()
+		_, _, err = client.Put(randCtx(), &args)
+		require.Error(t, err)
+		duration := time.Since(startTime)
+		require.GreaterOrEqual(t, cs.maxMs, duration, "greater duration: ", duration)
+		require.LessOrEqual(t, cs.minMs, duration, "less duration: ", duration)
+	}
+}
+
+func TestAccessClientDelete(t *testing.T) {
+	{
+		locs, err := client.Delete(randCtx(), nil)
+		require.Nil(t, locs)
+		require.ErrorIs(t, errcode.ErrIllegalArguments, err)
+	}
+	{
+		locs, err := client.Delete(randCtx(), &access.DeleteArgs{})
+		require.Nil(t, locs)
+		require.ErrorIs(t, errcode.ErrIllegalArguments, err)
+	}
+	{
+		locs, err := client.Delete(randCtx(), &access.DeleteArgs{
+			Locations: make([]access.Location, 1),
+		})
+		require.Nil(t, locs)
+		require.NoError(t, err)
+	}
+	{
+		args := &access.DeleteArgs{
+			Locations: make([]access.Location, 1000),
+		}
+		_, err := client.Delete(randCtx(), args)
+		require.NoError(t, err)
+	}
+	{
+		args := &access.DeleteArgs{
+			Locations: make([]access.Location, access.MaxDeleteLocations+1),
+		}
+		locs, err := client.Delete(randCtx(), args)
+		require.Equal(t, args.Locations, locs)
+		require.ErrorIs(t, errcode.ErrIllegalArguments, err)
+	}
+	{
+		loc := access.Location{Size: 100, Blobs: make([]access.SliceInfo, 0)}
+		fillCrc(&loc)
+		args := &access.DeleteArgs{
+			Locations: make([]access.Location, 0, access.MaxDeleteLocations),
+		}
+		for i := 1; i < access.MaxDeleteLocations/10; i++ {
+			args.Locations = append(args.Locations, loc)
+			locs, err := client.Delete(randCtx(), args)
+			if i%2 == 0 {
+				require.Equal(t, args.Locations, locs)
+				require.Equal(t, errcode.ErrUnexpected, err)
+			} else {
+				require.NoError(t, err)
+			}
+		}
+	}
+}
+
+func TestAccessClientRequestBody(t *testing.T) {
+	cfg := access.Config{}
+	cfg.Consul.Address = mockServer.URL[7:]
+	cfg.MaxSizePutOnce = 1 << 20
+	cfg.PartConcurrence = 2
+	cfg.LogLevel = log.Lfatal
+	cfg.PriorityAddrs = []string{mockServer.URL}
+	client, err := access.New(cfg)
+	require.NoError(t, err)
+
+	cases := []struct {
+		size int
+	}{
+		{0},
+		{1},
+		{1 << 10},
+		{(1 << 19) + 1},
+		{1 << 20},
+		{1<<20 + 1023},
+		{1<<23 + 1023},
+		{1 << 24},
+	}
+	for _, cs := range cases {
+		dataCache.clean()
+
+		buff := make([]byte, cs.size)
+		rand.Read(buff)
+		args := access.PutArgs{
+			Size: int64(cs.size) + 1,
+			Body: bytes.NewBuffer(buff),
+		}
+
+		_, _, err := client.Put(randCtx(), &args)
+		if cs.size < int(cfg.MaxSizePutOnce) {
+			if cs.size <= 1<<22 {
+				require.Equal(t, errcode.ErrAccessReadRequestBody, err)
+			} else {
+				require.Equal(t, errcode.ErrAccessReadConflictBody, err)
+			}
+		} else {
+			require.Equal(t, errcode.ErrAccessReadRequestBody, err)
+		}
+	}
+}
+
+func TestAccessClientPutAtToken(t *testing.T) {
+	tokenKey := tokenPutat
+	defer func() {
+		tokenPutat = tokenKey
+	}()
+
+	cfg := access.Config{}
+	cfg.Consul.Address = mockServer.URL[7:]
+	cfg.MaxSizePutOnce = 1 << 20
+	cfg.PartConcurrence = 1
+	cfg.MaxPartRetry = -1
+	cfg.LogLevel = log.Lfatal
+	client, err := access.New(cfg)
+	require.NoError(t, err)
+
+	cases := []struct {
+		keyLen int
+	}{
+		{0},
+		{1},
+		{20},
+		{100},
+	}
+	buff := make([]byte, 1<<21)
+	rand.Read(buff)
+	for _, cs := range cases {
+		tokenPutat = make([]byte, cs.keyLen)
+		rand.Read(tokenPutat)
+		args := access.PutArgs{
+			Size: int64(1 << 21),
+			Body: bytes.NewBuffer(buff),
+		}
+		_, _, err := client.Put(randCtx(), &args)
+		require.ErrorIs(t, errcode.ErrUnexpected, err)
+	}
+}
+
+func TestAccessClientRPCConfig(t *testing.T) {
+	cfg := access.Config{}
+	cfg.RPCConfig = &rpc.Config{}
+	cfg.LogLevel = log.Lfatal
+	cfg.Consul.Address = mockServer.URL[7:]
+	cfg.PriorityAddrs = []string{mockServer.URL}
+	client, err := access.New(cfg)
+	require.NoError(t, err)
+	cases := []struct {
+		size int
+	}{
+		{0},
+		{1},
+		{1 << 6},
+		{1 << 10},
+	}
+	for _, cs := range cases {
+		buff := make([]byte, cs.size)
+		rand.Read(buff)
+		args := access.PutArgs{
+			Size: int64(cs.size),
+			Body: bytes.NewBuffer(buff),
+		}
+		loc, _, err := client.Put(randCtx(), &args)
+		require.NoError(t, err)
+		_, err = client.Get(randCtx(), &access.GetArgs{Location: loc, ReadSize: uint64(cs.size)})
+		require.NoError(t, err)
+	}
+}
+
+func TestAccessClientLogger(t *testing.T) {
+	file, err := ioutil.TempFile(os.TempDir(), "TestAccessClientLogger")
+	require.NoError(t, err)
+	require.NoError(t, file.Close())
+	defer func() {
+		os.Remove(file.Name())
+	}()
+
+	cfg := access.Config{
+		LogLevel: log.Lfatal,
+		Logger: &access.Logger{
+			Filename: file.Name(),
+		},
+	}
+	cfg.Consul.Address = mockServer.URL[7:]
+	client, err := access.New(cfg)
+	require.NoError(t, err)
+	defer func() {
+		cfg.Logger.Close()
+	}()
+
+	size := 1024
+	args := access.PutArgs{
+		Size: int64(size),
+		Body: bytes.NewBuffer(make([]byte, size-1)),
+	}
+	_, _, err = client.Put(randCtx(), &args)
+	require.Error(t, err)
+}

blobstore/api/access/proto.go

@@ -0,0 +1,632 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"crypto/md5"
+	"crypto/sha1"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"hash"
+	"hash/crc32"
+	"io"
+
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+// HashAlgorithm hash.Hash algorithm when uploading data
+type HashAlgorithm uint8
+
+// defined hash algorithm
+const (
+	HashAlgDummy  HashAlgorithm = 1 << iota
+	HashAlgCRC32                // crc32 with IEEE
+	HashAlgMD5                  // md5
+	HashAlgSHA1                 // sha1
+	HashAlgSHA256               // sha256
+)
+
+const (
+	// HashSize dummy hash size
+	HashSize = 0
+
+	// MaxLocationBlobs max blobs length in Location
+	MaxLocationBlobs uint32 = 4
+	// MaxDeleteLocations max locations of delete request
+	MaxDeleteLocations int = 1024
+	// MaxBlobSize max blob size for allocation
+	MaxBlobSize uint32 = 1 << 25 // 32MB
+)
+
+type dummyHash struct{}
+
+var _ hash.Hash = (*dummyHash)(nil)
+
+// implements hash.Hash
+func (d dummyHash) Write(p []byte) (n int, err error) { return len(p), nil }
+func (d dummyHash) Sum(b []byte) []byte               { return []byte{} }
+func (d dummyHash) Reset()                            {}
+func (d dummyHash) Size() int                         { return 0 }
+func (d dummyHash) BlockSize() int                    { return 0 }
+
+// ToHasher returns a new hash.Hash computing checksum
+// the value of algorithm should be one of HashAlg*
+func (alg HashAlgorithm) ToHasher() hash.Hash {
+	switch alg {
+	case HashAlgCRC32:
+		return crc32.NewIEEE()
+	case HashAlgMD5:
+		return md5.New()
+	case HashAlgSHA1:
+		return sha1.New()
+	case HashAlgSHA256:
+		return sha256.New()
+	default:
+		return dummyHash{}
+	}
+}
+
+// ToHashSumMap returns a new HashSumMap, decode from rpc url argument
+func (alg HashAlgorithm) ToHashSumMap() HashSumMap {
+	h := make(HashSumMap)
+	for _, a := range []HashAlgorithm{
+		HashAlgDummy,
+		HashAlgCRC32,
+		HashAlgMD5,
+		HashAlgSHA1,
+		HashAlgSHA256,
+	} {
+		if alg&a == a {
+			h[a] = nil
+		}
+	}
+	return h
+}
+
+// HasherMap map hasher of HashAlgorithm
+type HasherMap map[HashAlgorithm]hash.Hash
+
+// ToHashAlgorithm returns HashAlgorithm
+func (h HasherMap) ToHashAlgorithm() HashAlgorithm {
+	alg := HashAlgorithm(0)
+	for k := range h {
+		alg |= k
+	}
+	return alg
+}
+
+// ToWriter returns io writer
+func (h HasherMap) ToWriter() io.Writer {
+	writers := make([]io.Writer, 0, len(h))
+	for _, hasher := range h {
+		writers = append(writers, hasher)
+	}
+	return io.MultiWriter(writers...)
+}
+
+// HashSumMap save checksum in rpc calls
+type HashSumMap map[HashAlgorithm][]byte
+
+// GetSum get checksum value and ok via HashAlgorithm
+//   HashAlgDummy  returns nil, bool
+//   HashAlgCRC32  returns uint32, bool
+//   HashAlgMD5    returns string(32), bool
+//   HashAlgSHA1   returns string(40), bool
+//   HashAlgSHA256 returns string(64), bool
+func (h HashSumMap) GetSum(key HashAlgorithm) (interface{}, bool) {
+	b, ok := h[key]
+	if !ok {
+		return nil, false
+	}
+
+	switch key {
+	case HashAlgCRC32:
+		if len(b) != crc32.Size {
+			return nil, false
+		}
+		return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24, true
+	case HashAlgMD5:
+		if len(b) != md5.Size {
+			return nil, false
+		}
+		return hex.EncodeToString(b[:]), true
+	case HashAlgSHA1:
+		if len(b) != sha1.Size {
+			return nil, false
+		}
+		return hex.EncodeToString(b[:]), true
+	case HashAlgSHA256:
+		if len(b) != sha256.Size {
+			return nil, false
+		}
+		return hex.EncodeToString(b[:]), true
+	default:
+		if len(b) != HashSize {
+			return nil, false
+		}
+		return nil, true
+	}
+}
+
+// GetSumVal get checksum only value via HashAlgorithm
+func (h HashSumMap) GetSumVal(key HashAlgorithm) interface{} {
+	val, _ := h.GetSum(key)
+	return val
+}
+
+// ToHashAlgorithm returns HashAlgorithm, encode to rpc url argument
+func (h HashSumMap) ToHashAlgorithm() HashAlgorithm {
+	alg := HashAlgorithm(0)
+	for k := range h {
+		alg |= k
+	}
+	return alg
+}
+
+// All returns readable checksum
+func (h HashSumMap) All() map[string]interface{} {
+	m := make(map[string]interface{})
+	for a, name := range map[HashAlgorithm]string{
+		HashAlgCRC32:  "crc32",
+		HashAlgMD5:    "md5",
+		HashAlgSHA1:   "sha1",
+		HashAlgSHA256: "sha256",
+	} {
+		if val, ok := h.GetSum(a); ok {
+			m[name] = val
+		}
+	}
+	return m
+}
+
+// Location file location, 4 + 1 + 8 + 4 + 4 + len*16 bytes
+// |                                        |
+// |   ClusterID(4)    |    CodeMode(1)     |
+// |                Size(8)                 |
+// |   BlobSize(4)     |      Crc(4)        |
+// |           len*SliceInfo(16)            |
+//
+// ClusterID which cluster file is in
+// CodeMode is ec encode mode, see defined in "common/lib/codemode"
+// Size is file size
+// BlobSize is every blob's size but the last one which's size=(Size mod BlobSize)
+// Crc is the checksum, change anything of the location, crc will mismatch
+// Blobs all blob information
+type Location struct {
+	_         [0]byte
+	ClusterID proto.ClusterID   `json:"cluster_id"`
+	CodeMode  codemode.CodeMode `json:"code_mode"`
+	Size      uint64            `json:"size"`
+	BlobSize  uint32            `json:"blob_size"`
+	Crc       uint32            `json:"crc"`
+	Blobs     []SliceInfo       `json:"blobs"`
+}
+
+// SliceInfo blobs info, 8 + 4 + 4 bytes
+//
+// MinBid is the first blob id
+// Vid is which volume all blobs in
+// Count is num of consecutive blob ids, count=1 just has one blob
+//
+// blob ids = [MinBid, MinBid+count)
+type SliceInfo struct {
+	_      [0]byte
+	MinBid proto.BlobID `json:"min_bid"`
+	Vid    proto.Vid    `json:"vid"`
+	Count  uint32       `json:"count"`
+}
+
+// Blob is one piece of data in a location
+//
+// Bid is the blob id
+// Vid is which volume the blob in
+// Size is real size of the blob
+type Blob struct {
+	Bid  proto.BlobID
+	Vid  proto.Vid
+	Size uint32
+}
+
+// Copy returns a new same Location
+func (loc *Location) Copy() Location {
+	dst := Location{
+		ClusterID: loc.ClusterID,
+		CodeMode:  loc.CodeMode,
+		Size:      loc.Size,
+		BlobSize:  loc.BlobSize,
+		Crc:       loc.Crc,
+		Blobs:     make([]SliceInfo, len(loc.Blobs)),
+	}
+	copy(dst.Blobs, loc.Blobs)
+	return dst
+}
+
+// Encode transfer Location to slice byte
+// Returns the buf created by me
+//  (n) means max-n bytes
+//  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+//  |  field  | crc | clusterid  | codemode |    size     |  blobsize  |
+//  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+//  | n-bytes |  4  | uvarint(5) |    1     | uvarint(10) | uvarint(5) |
+//  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+//          25   +  (5){len(blobs)}   +   len(Blobs) * 20
+//  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+//  |  blobs  | minbid | vid | count |           ...                   |
+//  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+//  | n-bytes |  (10)  | (5) |  (5)  | (20) | (20) |       ...         |
+//  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+func (loc *Location) Encode() []byte {
+	if loc == nil {
+		return nil
+	}
+	n := 25 + 5 + len(loc.Blobs)*20
+	buf := make([]byte, n)
+	n = loc.Encode2(buf)
+	return buf[:n]
+}
+
+// Encode2 transfer Location to the buf, the buf reuse by yourself
+// Returns the number of bytes read
+// If the buffer is too small, Encode2 will panic
+func (loc *Location) Encode2(buf []byte) int {
+	if loc == nil {
+		return 0
+	}
+
+	n := 0
+	binary.BigEndian.PutUint32(buf[n:], loc.Crc)
+	n += 4
+	n += binary.PutUvarint(buf[n:], uint64(loc.ClusterID))
+	buf[n] = byte(loc.CodeMode)
+	n++
+	n += binary.PutUvarint(buf[n:], uint64(loc.Size))
+	n += binary.PutUvarint(buf[n:], uint64(loc.BlobSize))
+
+	n += binary.PutUvarint(buf[n:], uint64(len(loc.Blobs)))
+	for _, blob := range loc.Blobs {
+		n += binary.PutUvarint(buf[n:], uint64(blob.MinBid))
+		n += binary.PutUvarint(buf[n:], uint64(blob.Vid))
+		n += binary.PutUvarint(buf[n:], uint64(blob.Count))
+	}
+
+	return n
+}
+
+// Decode parse location from buf
+// Returns the number of bytes read
+// Error is not nil when parsing failed
+func (loc *Location) Decode(buf []byte) (int, error) {
+	if loc == nil {
+		return 0, fmt.Errorf("location receiver is nil")
+	}
+
+	location, n, err := DecodeLocation(buf)
+	if err != nil {
+		return n, err
+	}
+
+	*loc = location
+	return n, nil
+}
+
+// ToString transfer location to hex string
+func (loc *Location) ToString() string {
+	return loc.HexString()
+}
+
+// HexString transfer location to hex string
+func (loc *Location) HexString() string {
+	return hex.EncodeToString(loc.Encode())
+}
+
+// Base64String transfer location to base64 string
+func (loc *Location) Base64String() string {
+	return base64.StdEncoding.EncodeToString(loc.Encode())
+}
+
+// Spread location blobs to slice
+func (loc *Location) Spread() []Blob {
+	count := 0
+	for _, blob := range loc.Blobs {
+		count += int(blob.Count)
+	}
+
+	blobs := make([]Blob, 0, count)
+	for _, blob := range loc.Blobs {
+		for offset := uint32(0); offset < blob.Count; offset++ {
+			blobs = append(blobs, Blob{
+				Bid:  blob.MinBid + proto.BlobID(offset),
+				Vid:  blob.Vid,
+				Size: loc.BlobSize,
+			})
+		}
+	}
+	if len(blobs) > 0 && loc.BlobSize > 0 {
+		if lastSize := loc.Size % uint64(loc.BlobSize); lastSize > 0 {
+			blobs[len(blobs)-1].Size = uint32(lastSize)
+		}
+	}
+	return blobs
+}
+
+// DecodeLocation parse location from buf
+// Returns Location and the number of bytes read
+// Error is not nil when parsing failed
+func DecodeLocation(buf []byte) (Location, int, error) {
+	var (
+		loc Location
+		n   int
+
+		val uint64
+		nn  int
+	)
+	next := func() (uint64, int) {
+		val, nn := binary.Uvarint(buf)
+		if nn <= 0 {
+			return 0, nn
+		}
+		n += nn
+		buf = buf[nn:]
+		return val, nn
+	}
+
+	if len(buf) < 4 {
+		return loc, n, fmt.Errorf("bytes crc %d", len(buf))
+	}
+	loc.Crc = binary.BigEndian.Uint32(buf)
+	n += 4
+	buf = buf[4:]
+
+	if val, nn = next(); nn <= 0 {
+		return loc, n, fmt.Errorf("bytes cluster_id %d", nn)
+	}
+	loc.ClusterID = proto.ClusterID(val)
+
+	if len(buf) < 1 {
+		return loc, n, fmt.Errorf("bytes codemode %d", len(buf))
+	}
+	loc.CodeMode = codemode.CodeMode(buf[0])
+	n++
+	buf = buf[1:]
+
+	if val, nn = next(); nn <= 0 {
+		return loc, n, fmt.Errorf("bytes size %d", nn)
+	}
+	loc.Size = val
+
+	if val, nn = next(); nn <= 0 {
+		return loc, n, fmt.Errorf("bytes blob_size %d", nn)
+	}
+	loc.BlobSize = uint32(val)
+
+	if val, nn = next(); nn <= 0 {
+		return loc, n, fmt.Errorf("bytes length blobs %d", nn)
+	}
+	length := int(val)
+
+	if length > 0 {
+		loc.Blobs = make([]SliceInfo, 0, length)
+	}
+	for index := 0; index < length; index++ {
+		var blob SliceInfo
+		if val, nn = next(); nn <= 0 {
+			return loc, n, fmt.Errorf("bytes %dth-blob min_bid %d", index, nn)
+		}
+		blob.MinBid = proto.BlobID(val)
+
+		if val, nn = next(); nn <= 0 {
+			return loc, n, fmt.Errorf("bytes %dth-blob vid %d", index, nn)
+		}
+		blob.Vid = proto.Vid(val)
+
+		if val, nn = next(); nn <= 0 {
+			return loc, n, fmt.Errorf("bytes %dth-blob count %d", index, nn)
+		}
+		blob.Count = uint32(val)
+
+		loc.Blobs = append(loc.Blobs, blob)
+	}
+
+	return loc, n, nil
+}
+
+// DecodeLocationFrom decode location from hex string
+func DecodeLocationFrom(s string) (Location, error) {
+	return DecodeLocationFromHex(s)
+}
+
+// DecodeLocationFromHex decode location from hex string
+func DecodeLocationFromHex(s string) (Location, error) {
+	var loc Location
+	src, err := hex.DecodeString(s)
+	if err != nil {
+		return loc, err
+	}
+	_, err = loc.Decode(src)
+	if err != nil {
+		return loc, err
+	}
+	return loc, nil
+}
+
+// DecodeLocationFromBase64 decode location from base64 string
+func DecodeLocationFromBase64(s string) (Location, error) {
+	var loc Location
+	src, err := base64.StdEncoding.DecodeString(s)
+	if err != nil {
+		return loc, err
+	}
+	_, err = loc.Decode(src)
+	if err != nil {
+		return loc, err
+	}
+	return loc, nil
+}
+
+// PutArgs for service /put
+// Hashes means how to calculate check sum,
+// HashAlgCRC32 | HashAlgMD5 equal 2 + 4 = 6
+type PutArgs struct {
+	Size   int64         `json:"size"`
+	Hashes HashAlgorithm `json:"hashes,omitempty"`
+	Body   io.Reader     `json:"-"`
+}
+
+// IsValid is valid put args
+func (args *PutArgs) IsValid() bool {
+	if args == nil {
+		return false
+	}
+	return args.Size > 0
+}
+
+// PutResp put response result
+type PutResp struct {
+	Location   Location   `json:"location"`
+	HashSumMap HashSumMap `json:"hashsum"`
+}
+
+// PutAtArgs for service /putat
+type PutAtArgs struct {
+	ClusterID proto.ClusterID `json:"clusterid"`
+	Vid       proto.Vid       `json:"volumeid"`
+	Blobid    proto.BlobID    `json:"blobid"`
+	Size      int64           `json:"size"`
+	Hashes    HashAlgorithm   `json:"hashes,omitempty"`
+	Token     string          `json:"token"`
+	Body      io.Reader       `json:"-"`
+}
+
+// IsValid is valid putat args
+func (args *PutAtArgs) IsValid() bool {
+	if args == nil {
+		return false
+	}
+	return args.ClusterID > proto.ClusterID(0) &&
+		args.Vid > proto.Vid(0) &&
+		args.Blobid > proto.BlobID(0) &&
+		args.Size > 0
+}
+
+// PutAtResp putat response result
+type PutAtResp struct {
+	HashSumMap HashSumMap `json:"hashsum"`
+}
+
+// AllocArgs for service /alloc
+type AllocArgs struct {
+	Size            uint64            `json:"size"`
+	BlobSize        uint32            `json:"blob_size"`
+	AssignClusterID proto.ClusterID   `json:"assign_cluster_id"`
+	CodeMode        codemode.CodeMode `json:"code_mode"`
+}
+
+// IsValid is valid alloc args
+func (args *AllocArgs) IsValid() bool {
+	if args == nil {
+		return false
+	}
+	if args.AssignClusterID > 0 {
+		return args.Size > 0 && args.BlobSize > 0 && args.BlobSize <= MaxBlobSize &&
+			args.CodeMode.IsValid()
+	}
+	return args.Size > 0 && args.BlobSize <= MaxBlobSize
+}
+
+// AllocResp alloc response result with tokens
+// if size mod blobsize == 0, length of tokens equal length of location blobs
+// otherwise additional token for the last blob uploading
+type AllocResp struct {
+	Location Location `json:"location"`
+	Tokens   []string `json:"tokens"`
+}
+
+// GetArgs for service /get
+type GetArgs struct {
+	Location Location `json:"location"`
+	Offset   uint64   `json:"offset"`
+	ReadSize uint64   `json:"read_size"`
+}
+
+// IsValid is valid get args
+func (args *GetArgs) IsValid() bool {
+	if args == nil {
+		return false
+	}
+	return args.Offset+args.ReadSize <= args.Location.Size
+}
+
+// DeleteArgs for service /delete
+type DeleteArgs struct {
+	Locations []Location `json:"locations"`
+}
+
+// IsValid is valid delete args
+func (args *DeleteArgs) IsValid() bool {
+	if args == nil {
+		return false
+	}
+	return len(args.Locations) > 0 && len(args.Locations) <= MaxDeleteLocations
+}
+
+// DeleteResp delete response with failed locations
+type DeleteResp struct {
+	FailedLocations []Location `json:"failed_locations,omitempty"`
+}
+
+// DeleteBlobArgs for service /deleteblob
+type DeleteBlobArgs struct {
+	ClusterID proto.ClusterID `json:"clusterid"`
+	Vid       proto.Vid       `json:"volumeid"`
+	Blobid    proto.BlobID    `json:"blobid"`
+	Size      int64           `json:"size"`
+	Token     string          `json:"token"`
+}
+
+// IsValid is valid delete blob args
+func (args *DeleteBlobArgs) IsValid() bool {
+	if args == nil {
+		return false
+	}
+	return args.ClusterID > proto.ClusterID(0) &&
+		args.Vid > proto.Vid(0) &&
+		args.Blobid > proto.BlobID(0) &&
+		args.Size > 0
+}
+
+// SignArgs for service /sign
+// Locations are signed location getting from /alloc
+// Location is to be signed location which merged by yourself
+type SignArgs struct {
+	Locations []Location `json:"locations"`
+	Location  Location   `json:"location"`
+}
+
+// IsValid is valid sign args
+func (args *SignArgs) IsValid() bool {
+	if args == nil {
+		return false
+	}
+	return len(args.Locations) > 0
+}
+
+// SignResp sign response location with crc
+type SignResp struct {
+	Location Location `json:"location"`
+}

blobstore/api/access/proto_test.go

@@ -0,0 +1,566 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access_test
+
+import (
+	"crypto/md5"
+	"crypto/rand"
+	"crypto/sha1"
+	"crypto/sha256"
+	"hash/crc32"
+	"math"
+	mrand "math/rand"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+func TestHashAlgorithm(t *testing.T) {
+	cases := []struct {
+		alg    access.HashAlgorithm
+		size   int
+		sumLen int
+	}{
+		{0, 0, access.HashSize},
+		{0b0, 1 << 10, access.HashSize},
+		{access.HashAlgorithm(0), 1 << 14, access.HashSize},
+
+		{1, 0, access.HashSize},
+		{0b1, 1 << 10, access.HashSize},
+		{access.HashAlgDummy, 1 << 14, access.HashSize},
+
+		{2, 0, crc32.Size},
+		{0b10, 1 << 10, crc32.Size},
+		{access.HashAlgCRC32, 1 << 14, crc32.Size},
+
+		{4, 0, md5.Size},
+		{0b100, 1 << 10, md5.Size},
+		{access.HashAlgMD5, 1 << 14, md5.Size},
+
+		{8, 0, sha1.Size},
+		{0b1000, 1 << 10, sha1.Size},
+		{access.HashAlgSHA1, 1 << 14, sha1.Size},
+
+		{16, 0, sha256.Size},
+		{0b10000, 1 << 10, sha256.Size},
+		{access.HashAlgSHA256, 1 << 14, sha256.Size},
+
+		{100, 0, access.HashSize},
+		{0b1111111, 1 << 10, access.HashSize},
+		{access.HashAlgDummy, 1 << 14, access.HashSize},
+	}
+	for _, cs := range cases {
+		buffer := make([]byte, cs.size)
+		rand.Read(buffer)
+
+		hasher := cs.alg.ToHasher()
+		_, err := hasher.Write(buffer)
+		require.NoError(t, err)
+		require.Equal(t, cs.sumLen, len(hasher.Sum(nil)))
+	}
+}
+
+func TestHasherMapWriter(t *testing.T) {
+	hasherMap := access.HasherMap{
+		access.HashAlgCRC32: access.HashAlgCRC32.ToHasher(),
+		access.HashAlgMD5:   access.HashAlgMD5.ToHasher(),
+		access.HashAlgSHA1:  access.HashAlgSHA1.ToHasher(),
+	}
+	writer := hasherMap.ToWriter()
+	writer.Write([]byte("foo x bar"))
+	require.Equal(t, []byte{0x5a, 0xc7, 0x36, 0x25}, hasherMap[access.HashAlgCRC32].Sum(nil))
+	require.Equal(t, []byte{
+		0x7c, 0xb4, 0x1e, 0x1a, 0x38, 0xad, 0x8b, 0xd9,
+		0x5a, 0x49, 0x2d, 0xb6, 0x29, 0x30, 0x62, 0x8a,
+	}, hasherMap[access.HashAlgMD5].Sum(nil))
+	require.Equal(t, []byte{
+		0x75, 0xe8, 0xe5, 0x0, 0x5b, 0xaa, 0x5c, 0xd, 0x40, 0x44, 0x1c, 0x42,
+		0xe5, 0xf6, 0xb4, 0x9a, 0xf, 0x80, 0x80, 0x11,
+	}, hasherMap[access.HashAlgSHA1].Sum(nil))
+}
+
+func TestHashSumMapGetSum(t *testing.T) {
+	crc32Hash := crc32.NewIEEE()
+	crc32Hash.Write([]byte("crc32"))
+	bytesCRC32 := crc32Hash.Sum(nil)
+	bytesMD5 := md5.Sum([]byte("md5"))
+	bytesSHA1 := sha1.Sum([]byte("sha1"))
+	bytesSHA256 := sha256.Sum256([]byte("sha256"))
+
+	cases := []struct {
+		key string
+		m   access.HashSumMap
+		alg access.HashAlgorithm
+		val interface{}
+		ok  bool
+	}{
+		{
+			"dummy",
+			access.HashSumMap{},
+			access.HashAlgDummy, nil, false,
+		},
+		{
+			"dummy",
+			access.HashSumMap{access.HashAlgDummy: []byte{0x11}},
+			access.HashAlgDummy, nil, false,
+		},
+		{
+			"dummy",
+			access.HashSumMap{access.HashAlgDummy: []byte{}},
+			access.HashAlgDummy, nil, true,
+		},
+
+		{
+			"crc32",
+			access.HashSumMap{access.HashAlgCRC32: []byte{}},
+			access.HashAlgCRC32, nil, false,
+		},
+		{
+			"crc32",
+			access.HashSumMap{access.HashAlgCRC32: []byte{}},
+			access.HashAlgCRC32, nil, false,
+		},
+		{
+			"crc32",
+			access.HashSumMap{access.HashAlgCRC32: bytesCRC32},
+			access.HashAlgCRC32, crc32.ChecksumIEEE([]byte("crc32")), true,
+		},
+
+		{
+			"md5",
+			access.HashSumMap{access.HashAlgMD5: []byte{}},
+			access.HashAlgMD5, nil, false,
+		},
+		{
+			"md5",
+			access.HashSumMap{access.HashAlgMD5: []byte{0x11}},
+			access.HashAlgMD5, nil, false,
+		},
+		{
+			"md5",
+			access.HashSumMap{access.HashAlgMD5: bytesMD5[:]},
+			access.HashAlgMD5, "1bc29b36f623ba82aaf6724fd3b16718", true,
+		},
+
+		{
+			"sha1",
+			access.HashSumMap{access.HashAlgSHA1: []byte{}},
+			access.HashAlgSHA1, nil, false,
+		},
+		{
+			"sha1",
+			access.HashSumMap{access.HashAlgSHA1: []byte{0x00, 0xff}},
+			access.HashAlgSHA1, nil, false,
+		},
+		{
+			"sha1",
+			access.HashSumMap{access.HashAlgSHA1: bytesSHA1[:]},
+			access.HashAlgSHA1, "415ab40ae9b7cc4e66d6769cb2c08106e8293b48", true,
+		},
+
+		{
+			"sha256",
+			access.HashSumMap{},
+			access.HashAlgSHA256, nil, false,
+		},
+		{
+			"sha256",
+			access.HashSumMap{access.HashAlgSHA256: nil},
+			access.HashAlgSHA256, nil, false,
+		},
+		{
+			"sha256",
+			access.HashSumMap{access.HashAlgSHA256: bytesSHA256[:]},
+			access.HashAlgSHA256,
+			"5d5b09f6dcb2d53a5fffc60c4ac0d55fabdf556069d6631545f42aa6e3500f2e", true,
+		},
+
+		{
+			"multi",
+			access.HashSumMap{access.HashAlgSHA1: bytesSHA1[:]},
+			access.HashAlgSHA256, nil, false,
+		},
+		{
+			"multi",
+			access.HashSumMap{access.HashAlgSHA1: bytesSHA1[:]},
+			access.HashAlgMD5, nil, false,
+		},
+	}
+
+	for _, cs := range cases {
+		val, ok := cs.m.GetSum(cs.alg)
+		require.Equal(t, cs.ok, ok, cs.key)
+		require.Equal(t, cs.val, val, cs.key)
+
+		val = cs.m.GetSumVal(cs.alg)
+		require.Equal(t, cs.val, val, cs.key)
+
+		hasherMap := make(access.HasherMap)
+		for alg := range cs.m {
+			hasherMap[alg] = nil
+		}
+		require.Equal(t, cs.m.ToHashAlgorithm(), hasherMap.ToHashAlgorithm())
+
+		cs.m.All()
+	}
+}
+
+func TestHashAlgorithm2HashSumMap(t *testing.T) {
+	cases := []struct {
+		m access.HashSumMap
+		h access.HashAlgorithm
+	}{
+		{access.HashSumMap{}, 0},
+		{access.HashSumMap{
+			access.HashAlgDummy: nil,
+		}, 0b1},
+		{access.HashSumMap{
+			access.HashAlgDummy:  nil,
+			access.HashAlgCRC32:  nil,
+			access.HashAlgMD5:    nil,
+			access.HashAlgSHA1:   nil,
+			access.HashAlgSHA256: nil,
+		}, 0b11111},
+		{access.HashSumMap{
+			access.HashAlgDummy:  nil,
+			access.HashAlgCRC32:  nil,
+			access.HashAlgMD5:    nil,
+			access.HashAlgSHA1:   nil,
+			access.HashAlgSHA256: nil,
+		}, 0b00011111},
+		{access.HashSumMap{
+			access.HashAlgDummy:  nil,
+			access.HashAlgSHA1:   nil,
+			access.HashAlgSHA256: nil,
+		}, 0b11001},
+	}
+	for _, cs := range cases {
+		h := cs.m.ToHashAlgorithm()
+		require.Equal(t, cs.h, h)
+		m := h.ToHashSumMap()
+		require.Equal(t, cs.m, m)
+	}
+}
+
+func TestLocationEncodeDecodeNil(t *testing.T) {
+	var loc *access.Location
+	require.Nil(t, loc.Encode())
+	require.Equal(t, 0, loc.Encode2(nil))
+	require.Equal(t, "", loc.ToString())
+	require.Equal(t, "", loc.HexString())
+	require.Equal(t, "", loc.Base64String())
+
+	n, err := loc.Decode(nil)
+	require.Error(t, err)
+	require.Equal(t, 0, n)
+
+	locx, n, err := access.DecodeLocation(nil)
+	require.Error(t, err)
+	require.Equal(t, 0, n)
+	require.Equal(t, access.Location{}, locx)
+
+	locx, err = access.DecodeLocationFrom("")
+	require.Error(t, err)
+	require.Equal(t, access.Location{}, locx)
+
+	locx, err = access.DecodeLocationFrom("xxx")
+	require.Error(t, err)
+	require.Equal(t, access.Location{}, locx)
+
+	locx, err = access.DecodeLocationFromHex("xxx")
+	require.Error(t, err)
+	require.Equal(t, access.Location{}, locx)
+
+	locx, err = access.DecodeLocationFromBase64("xxx")
+	require.Error(t, err)
+	require.Equal(t, access.Location{}, locx)
+}
+
+func TestLocationEncodeDecode(t *testing.T) {
+	for ii := 0; ii < 100; ii++ {
+		loc := &access.Location{
+			ClusterID: proto.ClusterID(mrand.Uint32()),
+			CodeMode:  codemode.CodeMode(mrand.Intn(0xff)),
+			Size:      mrand.Uint64(),
+			BlobSize:  mrand.Uint32(),
+			Crc:       mrand.Uint32(),
+		}
+
+		num := mrand.Intn(5)
+		for i := 0; i < num; i++ {
+			loc.Blobs = append(loc.Blobs, access.SliceInfo{
+				MinBid: proto.BlobID(mrand.Uint64()),
+				Vid:    proto.Vid(mrand.Uint32()),
+				Count:  mrand.Uint32(),
+			})
+		}
+
+		buf := loc.Encode()
+		bufx := make([]byte, len(buf))
+		n := loc.Encode2(bufx)
+		require.Equal(t, len(buf), n)
+		require.Equal(t, buf, bufx)
+
+		require.Panics(t, func() { loc.Encode2(nil) })
+		require.Panics(t, func() { loc.Encode2(bufx[:3]) })
+		require.Panics(t, func() { loc.Encode2(bufx[:n/2]) })
+		require.Panics(t, func() { loc.Encode2(bufx[:n-1]) })
+
+		locx := access.Location{}
+		locx.Decode(bufx)
+		require.Equal(t, loc.ToString(), locx.ToString())
+		require.Equal(t, loc.HexString(), locx.HexString())
+		require.Equal(t, loc.Base64String(), locx.Base64String())
+
+		str := loc.ToString()
+		locx, err := access.DecodeLocationFrom(str)
+		require.NoError(t, err)
+		require.Equal(t, *loc, locx)
+
+		str = loc.HexString()
+		locx, err = access.DecodeLocationFromHex(str)
+		require.NoError(t, err)
+		require.Equal(t, *loc, locx)
+
+		str = loc.Base64String()
+		locx, err = access.DecodeLocationFromBase64(str)
+		require.NoError(t, err)
+		require.Equal(t, *loc, locx)
+	}
+}
+
+func TestLocationDecodeError(t *testing.T) {
+	loc := &access.Location{
+		ClusterID: proto.ClusterID(math.MaxUint32),
+		CodeMode:  codemode.CodeMode(math.MaxInt8),
+		Size:      math.MaxUint64,
+		BlobSize:  math.MaxUint32,
+		Crc:       math.MaxUint32,
+	}
+
+	buf := loc.Encode()
+	require.Equal(t, 25+1, len(buf))
+	for _, n := range []int{3, 8, 9, 19, 24} {
+		_, _, err := access.DecodeLocation(buf[:n])
+		require.Error(t, err)
+		t.Log(err)
+	}
+
+	loc.Blobs = append(loc.Blobs, access.SliceInfo{
+		MinBid: proto.BlobID(math.MaxUint64),
+		Vid:    proto.Vid(math.MaxUint32),
+		Count:  math.MaxUint32,
+	})
+
+	buf = loc.Encode()
+	require.Equal(t, 25+1+20, len(buf))
+	for _, n := range []int{25, 35, 40, 45} {
+		_, _, err := access.DecodeLocation(buf[:n])
+		require.Error(t, err)
+		t.Log(err)
+	}
+}
+
+func TestLocationSpread(t *testing.T) {
+	{
+		var loc access.Location
+		blobs := loc.Spread()
+		require.NotNil(t, blobs)
+		require.Equal(t, 0, len(blobs))
+	}
+	{
+		loc := &access.Location{
+			Size:     10,
+			BlobSize: 1 << 22,
+			Blobs: []access.SliceInfo{{
+				MinBid: 100,
+				Vid:    4,
+				Count:  1,
+			}},
+		}
+		blobs := loc.Spread()
+		require.Equal(t, 1, len(blobs))
+		require.Equal(t, proto.BlobID(100), blobs[0].Bid)
+		require.Equal(t, proto.Vid(4), blobs[0].Vid)
+		require.Equal(t, uint32(10), blobs[0].Size)
+	}
+	{
+		loc := &access.Location{
+			Size:     (1 << 22) + 10,
+			BlobSize: 1 << 22,
+			Blobs: []access.SliceInfo{{
+				MinBid: 100,
+				Vid:    4,
+				Count:  2,
+			}},
+		}
+		blobs := loc.Spread()
+		require.Equal(t, 2, len(blobs))
+		require.Equal(t, proto.BlobID(100), blobs[0].Bid)
+		require.Equal(t, proto.Vid(4), blobs[0].Vid)
+		require.Equal(t, uint32(1<<22), blobs[0].Size)
+		require.Equal(t, proto.BlobID(101), blobs[1].Bid)
+		require.Equal(t, proto.Vid(4), blobs[1].Vid)
+		require.Equal(t, uint32(10), blobs[1].Size)
+	}
+	{
+		loc := &access.Location{
+			Size:     1 << 23,
+			BlobSize: 1 << 22,
+			Blobs: []access.SliceInfo{{
+				MinBid: 100,
+				Vid:    4,
+				Count:  1,
+			}, {
+				MinBid: 200,
+				Vid:    4,
+				Count:  1,
+			}},
+		}
+		blobs := loc.Spread()
+		require.Equal(t, 2, len(blobs))
+		require.Equal(t, proto.BlobID(100), blobs[0].Bid)
+		require.Equal(t, proto.Vid(4), blobs[0].Vid)
+		require.Equal(t, uint32(1<<22), blobs[0].Size)
+		require.Equal(t, proto.BlobID(200), blobs[1].Bid)
+		require.Equal(t, proto.Vid(4), blobs[1].Vid)
+		require.Equal(t, uint32(1<<22), blobs[1].Size)
+	}
+}
+
+func TestPutArgs(t *testing.T) {
+	cases := []struct {
+		size  int64
+		valid bool
+	}{
+		{-(1 << 20), false},
+		{-1, false},
+		{0, false},
+		{1, true},
+		{100, true},
+		{1 << 32, true},
+	}
+	for _, cs := range cases {
+		args := access.PutArgs{Size: cs.size}
+		require.Equal(t, cs.valid, args.IsValid())
+	}
+}
+
+func TestPutAtArgs(t *testing.T) {
+	cases := []struct {
+		cid   proto.ClusterID
+		vid   proto.Vid
+		bid   proto.BlobID
+		size  int64
+		valid bool
+	}{
+		{0, 0, 0, -1, false},
+		{0, 0, 0, 0, false},
+		{0, 0, 0, 1, false},
+		{1, 0, 0, 1, false},
+		{1, 1, 0, 1, false},
+		{1, 1, 1, 1, true},
+	}
+	for _, cs := range cases {
+		args := access.PutAtArgs{
+			ClusterID: cs.cid,
+			Vid:       cs.vid,
+			Blobid:    cs.bid,
+			Size:      cs.size,
+		}
+		require.Equal(t, cs.valid, args.IsValid())
+	}
+}
+
+func TestAllocArgs(t *testing.T) {
+	cases := []struct {
+		size     uint64
+		cid      proto.ClusterID
+		blobSize uint32
+		cm       codemode.CodeMode
+		valid    bool
+	}{
+		{0, 0, 0, 0, false},
+		{0, 0, 0, 1, false},
+		{1, 0, 0, 0, true},
+		{1, 1, 0, 0, false},
+		{1, 1, 1, 0, false},
+		{1, 1, 1, 1, true},
+		{1, 1, 1, 3, true},
+		{1, 1, 1, 0xff, false},
+		{1, 1, 0, 2, false},
+		{1, 0, access.MaxBlobSize, 0, true},
+		{1, 1, access.MaxBlobSize, 2, true},
+		{1, 0, access.MaxBlobSize + 1, 0, false},
+		{1, 1, access.MaxBlobSize + 1, 2, false},
+	}
+	for _, cs := range cases {
+		args := access.AllocArgs{
+			Size:            cs.size,
+			BlobSize:        cs.blobSize,
+			CodeMode:        cs.cm,
+			AssignClusterID: cs.cid,
+		}
+		require.Equal(t, cs.valid, args.IsValid())
+	}
+}
+
+func TestGetArgs(t *testing.T) {
+	args := access.GetArgs{}
+	require.True(t, args.IsValid())
+}
+
+func TestDeleteArgs(t *testing.T) {
+	args := access.DeleteArgs{}
+	require.False(t, args.IsValid())
+	require.False(t, (*access.DeleteArgs)(nil).IsValid())
+	args.Locations = []access.Location{{}}
+	require.True(t, args.IsValid())
+	args.Locations = make([]access.Location, access.MaxDeleteLocations)
+	require.True(t, args.IsValid())
+	args.Locations = make([]access.Location, access.MaxDeleteLocations+1)
+	require.False(t, args.IsValid())
+}
+
+func TestDeleteBlobArgs(t *testing.T) {
+	args := access.DeleteBlobArgs{}
+	require.False(t, args.IsValid())
+	require.False(t, (*access.DeleteBlobArgs)(nil).IsValid())
+	args.ClusterID = 1
+	require.False(t, args.IsValid())
+	args.Blobid = 1
+	require.False(t, args.IsValid())
+	args.Vid = 1
+	require.False(t, args.IsValid())
+	args.Size = 1
+	require.True(t, args.IsValid())
+}
+
+func TestSignArgs(t *testing.T) {
+	args := access.SignArgs{}
+	require.False(t, args.IsValid())
+	require.False(t, (*access.DeleteArgs)(nil).IsValid())
+	args.Locations = []access.Location{{}}
+	require.True(t, args.IsValid())
+}
+
+func init() {
+	mrand.Seed(time.Now().UnixNano())
+}

blobstore/blobstore.go

@@ -0,0 +1,27 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package blobstore
+
+// go vet
+//go:generate go vet ./...
+
+// code formats with 'gofumpt' at version v0.2.1
+// go install mvdan.cc/gofumpt@v0.2.1
+//go:generate gofumpt -l -w .
+//go:generate git diff --exit-code
+
+// code golangci lint with 'golangci-lint' version v1.43.0
+// go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.43.0
+//go:generate golangci-lint run --issues-exit-code=1 -D errcheck -E bodyclose ./...

blobstore/cli/access/access.go

@@ -0,0 +1,157 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/desertbit/grumble"
+	"github.com/hashicorp/consul/api"
+
+	"github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+	"github.com/cubefs/cubefs/blobstore/cli/common/cfmt"
+	"github.com/cubefs/cubefs/blobstore/cli/common/flags"
+	"github.com/cubefs/cubefs/blobstore/cli/config"
+)
+
+func newAccessClient() (access.API, error) {
+	return access.New(access.Config{
+		ConnMode:       access.RPCConnectMode(config.AccessConnMode()),
+		Consul:         access.ConsulConfig{Address: config.AccessConsulAddr()},
+		PriorityAddrs:  config.AccessPriorityAddrs(),
+		MaxSizePutOnce: config.AccessMaxSizePutOnce(),
+		MaxPartRetry:   config.AccessMaxPartRetry(),
+		MaxHostRetry:   config.AccessMaxHostRetry(),
+
+		ServiceIntervalMs: config.AccessServiceIntervalMs(),
+	})
+}
+
+func newConsulClient() (*api.Client, error) {
+	conf := api.DefaultConfig()
+	conf.Address = config.AccessConsulAddr()
+	return api.NewClient(conf)
+}
+
+func readLocation(f grumble.FlagMap) (loc access.Location, err error) {
+	if f.String("location") != "" {
+		loc, err = cfmt.ParseLocation(f.String("location"))
+		return
+	}
+
+	filepath := f.String("locationpath")
+	if filepath == "" {
+		err = fmt.Errorf("no location and locationpath setting")
+		return
+	}
+
+	file, err := os.Open(filepath)
+	if err != nil {
+		return
+	}
+	defer file.Close()
+
+	err = common.NewDecoder(file).Decode(&loc)
+	return
+}
+
+func accessFlags(f *grumble.Flags) {
+	// readable
+	flags.VerboseRegister(f)
+
+	// location
+	f.String("p", "locationpath", "", "location file path")
+	f.String("l", "location", "", "location string by [json|hex|base64]")
+}
+
+// Register register access
+func Register(app *grumble.App) {
+	accessCommand := &grumble.Command{
+		Name:     "access",
+		Help:     "access tools",
+		LongHelp: "blobstore access api tools",
+		Run: func(c *grumble.Context) error {
+			cli, err := newConsulClient()
+			if err != nil {
+				return err
+			}
+			services, _, err := cli.Health().Service("access", "", true,
+				&api.QueryOptions{RequireConsistent: true})
+			if err != nil {
+				return err
+			}
+			fmt.Println("discovery access services on")
+			fmt.Println(common.Readable(services))
+			fmt.Println()
+			return nil
+		},
+	}
+	app.AddCommand(accessCommand)
+
+	accessCommand.AddCommand(&grumble.Command{
+		Name: "put",
+		Help: "put file",
+		Run:  putFile,
+		Flags: func(f *grumble.Flags) {
+			accessFlags(f)
+			f.String("d", "data", "", "raw data body")
+			f.String("f", "filepath", "", "put file path")
+			f.Int64("", "size", 0, "put file size, 0 means file size")
+			f.Uint("", "hashes", 0, "put file hashes")
+		},
+	})
+	accessCommand.AddCommand(&grumble.Command{
+		Name: "get",
+		Help: "get file",
+		Run:  getFile,
+		Flags: func(f *grumble.Flags) {
+			accessFlags(f)
+			f.String("f", "filepath", "", "save file path")
+			f.Uint64("", "offset", 0, "get file offset")
+			f.Uint64("", "readsize", 0, "get file read size, 0 means file size")
+		},
+	})
+	accessCommand.AddCommand(&grumble.Command{
+		Name: "del",
+		Help: "del file",
+		Run:  delFile,
+		Flags: func(f *grumble.Flags) {
+			accessFlags(f)
+		},
+	})
+	accessCommand.AddCommand(&grumble.Command{
+		Name:     "cluster",
+		Help:     "show cluster",
+		LongHelp: "show cluster in [region]",
+		Run:      showClusters,
+		Args: func(a *grumble.Args) {
+			a.String("region", "show cluster of region", grumble.Default(""))
+		},
+	})
+	accessCommand.AddCommand(&grumble.Command{
+		Name:     "ec",
+		Help:     "show ec buffer size",
+		LongHelp: "show ec buffer size with [blobsize]",
+		Run:      showECbuffer,
+		Args: func(a *grumble.Args) {
+			a.Int("blobsize", "show ec buffer with blobsize", grumble.Default(0))
+		},
+		Flags: func(f *grumble.Flags) {
+			f.IntL("codemode", 0, "on special codemode")
+		},
+	})
+}

blobstore/cli/access/cluster.go

@@ -0,0 +1,94 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/desertbit/grumble"
+	"github.com/dustin/go-humanize"
+	"github.com/fatih/color"
+
+	cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+	"github.com/cubefs/cubefs/blobstore/cli/common/cfmt"
+)
+
+func showClusters(c *grumble.Context) error {
+	cli, err := newConsulClient()
+	if err != nil {
+		return err
+	}
+
+	var regions []string
+	if region := c.Args.String("region"); region != "" {
+		regions = []string{region}
+	} else {
+		pairs, _, err := cli.KV().List("/ebs/", nil)
+		if err != nil {
+			return err
+		}
+		for _, pair := range pairs {
+			paths := strings.Split(strings.Trim(pair.Key, "/"), "/")
+			region := paths[1]
+			has := false
+			for _, r := range regions {
+				if r == region {
+					has = true
+					break
+				}
+			}
+			if !has {
+				regions = append(regions, region)
+			}
+		}
+		fmt.Printf("found %s regions\n\n", color.GreenString("%d", len(regions)))
+	}
+
+	for _, region := range regions {
+		fmt.Println("to list region:", color.RedString("%s", region))
+		path := cmapi.GetConsulClusterPath(region)
+		pairs, _, err := cli.KV().List(path, nil)
+		if err != nil {
+			fmt.Println("\terror:", err)
+			continue
+		}
+
+		capacity := int64(1)
+		available := int64(1)
+		for _, pair := range pairs {
+			clusterInfo := &cmapi.ClusterInfo{}
+			err := common.Unmarshal(pair.Value, clusterInfo)
+			if err != nil {
+				fmt.Println("\terror:", err)
+				continue
+			}
+
+			capacity += clusterInfo.Capacity
+			available += clusterInfo.Available
+
+			clusterInfo.Capacity++
+			clusterInfo.Available++
+			fmt.Println(cfmt.ClusterInfoJoin(clusterInfo, "\t"))
+			fmt.Println()
+		}
+		fmt.Printf("    space in region: %s (%s / %s)\n", region,
+			common.ColorizeInt64(-available, capacity).Sprint(humanize.IBytes(uint64(available))),
+			humanize.IBytes(uint64(capacity)))
+	}
+
+	return nil
+}

blobstore/cli/access/del.go

@@ -0,0 +1,61 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"fmt"
+
+	"github.com/desertbit/grumble"
+
+	"github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+	"github.com/cubefs/cubefs/blobstore/cli/common/cfmt"
+	"github.com/cubefs/cubefs/blobstore/cli/common/flags"
+	"github.com/cubefs/cubefs/blobstore/cli/config"
+)
+
+func delFile(c *grumble.Context) error {
+	client, err := newAccessClient()
+	if err != nil {
+		return err
+	}
+
+	location, err := readLocation(c.Flags)
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("del location json    : %s\n", common.RawString(location))
+	if flags.Verbose(c.Flags) || config.Verbose() {
+		fmt.Println("del location readable:")
+		fmt.Println(cfmt.LocationJoin(&location, "\t"))
+	} else {
+		fmt.Printf("del location verbose : %+v\n", location)
+	}
+
+	if !common.Confirm("to delete?") {
+		return nil
+	}
+	deleteArgs := &access.DeleteArgs{
+		Locations: []access.Location{location},
+	}
+	_, err = client.Delete(common.CmdContext(), deleteArgs)
+	if err != nil {
+		return err
+	}
+
+	fmt.Println("DELETE OK!")
+	return nil
+}

blobstore/cli/access/ec.go

@@ -0,0 +1,135 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/desertbit/grumble"
+	"github.com/dustin/go-humanize"
+	"github.com/fatih/color"
+
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	"github.com/cubefs/cubefs/blobstore/common/ec"
+)
+
+func showECbuffer(c *grumble.Context) error {
+	kb := 1 << 10
+	mb := 1 << 20
+	perPage := 7
+
+	blobSize := c.Args.Int("blobsize")
+	if blobSize > 64*mb {
+		return fmt.Errorf("too large blobsize %d", blobSize)
+	}
+
+	var modes, all []codemode.CodeMode
+	all = codemode.GetAllCodeModes()
+	mode := c.Flags.Int("codemode")
+	for _, m := range all {
+		if m == codemode.CodeMode(mode) {
+			modes = []codemode.CodeMode{m}
+			break
+		}
+	}
+	if len(modes) == 0 {
+		modes = all
+	}
+
+	for len(modes) > 0 {
+		length := len(modes)
+		hasNextPage := length > perPage
+		if length > perPage {
+			length = perPage
+		}
+
+		showModes := modes[:length]
+		showEcbufferSize(-1, common.Optimal, showModes)
+		if blobSize > 0 {
+			showEcbufferSize(blobSize, common.Loaded, showModes)
+			if hasNextPage {
+				fmt.Println()
+			}
+			modes = modes[length:]
+			continue
+		}
+
+		alterColor := common.NewAlternateColor(3)
+		for _, size := range []int{
+			1,
+			kb * 2,
+			kb * 12,
+			kb * 64,
+			kb * 512,
+			mb * 1,
+			mb * 2,
+			mb * 4,
+			mb * 8,
+			mb * 16,
+		} {
+			showEcbufferSize(size, alterColor.Next(), showModes)
+		}
+		if hasNextPage {
+			fmt.Println()
+		}
+		modes = modes[length:]
+	}
+
+	return nil
+}
+
+func showEcbufferSize(size int, colorFmt *color.Color, modes []codemode.CodeMode) {
+	if size == -1 {
+		colorFmt.Printf("|%s|", center("blobsize"))
+		for _, mode := range modes {
+			colorFmt.Printf("%s|", center(mode.String()))
+		}
+		fmt.Println()
+		return
+	}
+
+	colorFmt.Printf("|%s|", center(size2Str(size)))
+	for _, mode := range modes {
+		sizes, _ := ec.GetBufferSizes(size, mode.Tactic())
+		colorFmt.Printf("%s|", center(size2Str(sizes.ECSize)))
+	}
+	fmt.Println()
+
+	colorFmt.Printf("|%s|", center("shard size"))
+	for _, mode := range modes {
+		sizes, _ := ec.GetBufferSizes(size, mode.Tactic())
+		colorFmt.Printf("%s|", center(size2Str(sizes.ShardSize)))
+	}
+	fmt.Println()
+}
+
+func center(s string) string {
+	all := 19
+	l := (all - len(s)) / 2
+	if l < 0 {
+		l = 0
+	}
+	r := all - len(s) - l
+	if r < 0 {
+		r = 0
+	}
+	return fmt.Sprintf("%s%s%s", strings.Repeat(" ", l), s, strings.Repeat(" ", r))
+}
+
+func size2Str(size int) string {
+	return fmt.Sprintf("%d(%s)", size, humanize.IBytes(uint64(size)))
+}

blobstore/cli/access/get.go

@@ -0,0 +1,110 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"time"
+
+	"github.com/desertbit/grumble"
+
+	"github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+	"github.com/cubefs/cubefs/blobstore/cli/common/cfmt"
+	"github.com/cubefs/cubefs/blobstore/cli/common/flags"
+	"github.com/cubefs/cubefs/blobstore/cli/config"
+)
+
+func getFile(c *grumble.Context) error {
+	client, err := newAccessClient()
+	if err != nil {
+		return err
+	}
+
+	location, err := readLocation(c.Flags)
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("get location json    : %s\n", common.RawString(location))
+	if flags.Verbose(c.Flags) || config.Verbose() {
+		fmt.Println("get location readable:")
+		fmt.Println(cfmt.LocationJoin(&location, "\t"))
+	} else {
+		fmt.Printf("get location verbose : %+v\n", location)
+	}
+
+	size := c.Flags.Uint64("readsize")
+	if size == 0 {
+		size = location.Size - c.Flags.Uint64("offset")
+	}
+
+	r, err := client.Get(common.CmdContext(), &access.GetArgs{
+		Location: location,
+		Offset:   c.Flags.Uint64("offset"),
+		ReadSize: size,
+	})
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+
+	reader := common.NewPReader(int(size), r)
+	defer reader.Close()
+
+	reader.LineBar(50)
+
+	filepath := c.Flags.String("filepath")
+	if filepath == "" {
+		var w io.Writer
+		buffer := bytes.NewBuffer(nil)
+		// > 4K
+		if size > 1<<12 {
+			fmt.Printf("data is too long %d > %d\n", size, 1<<12)
+			w = ioutil.Discard
+		} else {
+			w = buffer
+		}
+
+		_, err = io.CopyN(w, reader, int64(size))
+		if err != nil {
+			return fmt.Errorf("downloading %s", err.Error())
+		}
+
+		data := buffer.Bytes()
+		if len(data) > 0 {
+			time.Sleep(10 * time.Millisecond)
+			fmt.Printf("raw data %d: '%s'\n", len(data), string(data))
+		}
+		return nil
+	}
+
+	file, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
+	if err != nil {
+		return fmt.Errorf("open file %s : %s", filepath, err.Error())
+	}
+	defer file.Close()
+
+	_, err = io.CopyN(file, reader, int64(size))
+	if err != nil {
+		return fmt.Errorf("downloading to %s : %s", filepath, err.Error())
+	}
+
+	return nil
+}

blobstore/cli/access/put.go

@@ -0,0 +1,102 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package access
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+
+	"github.com/desertbit/grumble"
+
+	"github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+	"github.com/cubefs/cubefs/blobstore/cli/common/cfmt"
+	"github.com/cubefs/cubefs/blobstore/cli/common/flags"
+	"github.com/cubefs/cubefs/blobstore/cli/config"
+)
+
+func putFile(c *grumble.Context) error {
+	client, err := newAccessClient()
+	if err != nil {
+		return err
+	}
+
+	var (
+		reader *common.PReader
+		size   int64
+	)
+	raw := c.Flags.String("data")
+	if len(raw) > 0 {
+		data := []byte(raw)
+		size = int64(len(data))
+		reader = common.NewPReader(int(size), bytes.NewReader(data))
+
+	} else {
+		filepath := c.Flags.String("filepath")
+		if filepath == "" {
+			return fmt.Errorf("no filepath setting")
+		}
+
+		file, err := os.Open(c.Flags.String("filepath"))
+		if err != nil {
+			return fmt.Errorf("open file %s : %s", filepath, err.Error())
+		}
+		defer file.Close()
+
+		size = c.Flags.Int64("size")
+		if size == 0 {
+			st, _ := os.Stat(filepath)
+			size = st.Size()
+		}
+		reader = common.NewPReader(int(size), file)
+	}
+	defer reader.Close()
+
+	reader.LineBar(50)
+
+	putHashes := access.HashAlgorithm(c.Flags.Uint("hashes"))
+	fmt.Printf("to upload size:%d hash:b(%b)\n", size, putHashes)
+
+	location, hashes, err := client.Put(common.CmdContext(), &access.PutArgs{
+		Size:   size,
+		Hashes: putHashes,
+		Body:   reader,
+	})
+	if err != nil {
+		return err
+	}
+
+	fmt.Println(cfmt.HashSumMapJoin(hashes, "\t"))
+	fmt.Printf("put location json    : %s\n", common.RawString(location))
+	if flags.Verbose(c.Flags) || config.Verbose() {
+		fmt.Println("put location readable:")
+		fmt.Println(cfmt.LocationJoin(&location, "\t"))
+	} else {
+		fmt.Printf("put location verbose : %+v\n", location)
+	}
+
+	locPath := c.Flags.String("locationpath")
+	if locPath != "" {
+		f, err := os.OpenFile(locPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
+		if err != nil {
+			return fmt.Errorf("open file %s : %s", locPath, err.Error())
+		}
+		defer f.Close()
+		return common.NewEncoder(f).Encode(location)
+	}
+
+	return nil
+}

blobstore/cli/app.go

@@ -0,0 +1,107 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package cli
+
+import (
+	"os"
+	"os/user"
+	"time"
+
+	"github.com/desertbit/grumble"
+	"github.com/fatih/color"
+
+	"github.com/cubefs/cubefs/blobstore/cli/access"
+	"github.com/cubefs/cubefs/blobstore/cli/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+	"github.com/cubefs/cubefs/blobstore/cli/common/flags"
+	"github.com/cubefs/cubefs/blobstore/cli/config"
+)
+
+// App blobstore command app
+var App = grumble.New(&grumble.Config{
+	Name:                  "blobstore manager",
+	Description:           "A command manager of blobstore",
+	HistoryFile:           "/tmp/.blobstore_cli.history",
+	HistoryLimit:          10000,
+	ErrorColor:            color.New(color.FgRed, color.Bold, color.Faint),
+	HelpHeadlineColor:     color.New(color.FgGreen),
+	HelpHeadlineUnderline: false,
+	HelpSubCommands:       true,
+	Prompt:                "BS $> ",
+	PromptColor:           color.New(color.FgBlue, color.Bold),
+	PromptRuntime: func() func() string {
+		username := "NO-USER"
+		hostname := "NO-HOSTNAME"
+		if user, err := user.Current(); err == nil {
+			username = user.Username
+		}
+		if host, _ := os.Hostname(); host != "" {
+			hostname = host
+		}
+
+		return func() string {
+			now := time.Now()
+			mins := (now.Hour() * 60) + now.Minute()
+			curr := mins * 100 / (60 * 24)
+			tStr := now.Format("01-02 15:04:05.000")
+			return color.New().Sprintf("%s %s%s %s@%s %s ",
+				color.New(color.FgBlue, color.Italic).Sprint("BS"),
+				color.New(color.FgMagenta, color.Faint).Sprintf("[%s]", common.BoldBar(curr)),
+				color.New(color.FgMagenta, color.Faint).Sprintf("[%s]", tStr),
+				color.New(color.FgGreen, color.Bold).Sprint(username),
+				color.New(color.FgYellow, color.Bold).Sprint(hostname),
+				color.New(color.FgBlack, color.BgHiYellow).Sprint("  $> "),
+			)
+		}
+	}(),
+	Flags: func(f *grumble.Flags) {
+		flags.ConfigRegister(f)
+		flags.VerboseRegister(f)
+		flags.VverboseRegister(f)
+	},
+})
+
+func init() {
+	App.OnInit(func(a *grumble.App, fm grumble.FlagMap) error {
+		if path := flags.Config(fm); path != "" {
+			config.LoadConfig(path)
+		}
+		if flags.Verbose(fm) {
+			config.Set("Flag-Verbose", true)
+		}
+		if flags.Vverbose(fm) {
+			config.Set("Flag-Vverbose", true)
+		}
+		return nil
+	})
+
+	App.SetPrintASCIILogo(func(a *grumble.App) {
+		a.Println(` _______ _______ _______ _______ _______ _______ _______ _______ _______     _______ _______ _______ `)
+		a.Println(`|\     /|\     /|\     /|\     /|\     /|\     /|\     /|\     /|\     /|   |\     /|\     /|\     /|`)
+		a.Println(`| +---+ | +---+ | +---+ | +---+ | +---+ | +---+ | +---+ | +---+ | +---+ |   | +---+ | +---+ | +---+ |`)
+		a.Println(`| |   | | |   | | |   | | |   | | |   | | |   | | |   | | |   | | |   | |   | |   | | |   | | |   | |`)
+		a.Println(`| |b  | | |l  | | |o  | | |b  | | |s  | | |t  | | |o  | | |r  | | |e  | |   | |c  | | |l  | | |i  | |`)
+		a.Println(`| +---+ | +---+ | +---+ | +---+ | +---+ | +---+ | +---+ | +---+ | +---+ |   | +---+ | +---+ | +---+ |`)
+		a.Println(`|/_____\|/_____\|/_____\|/_____\|/_____\|/_____\|/_____\|/_____\|/_____\|   |/_____\|/_____\|/_____\|`)
+		a.Println()
+	})
+
+	registerHistory(App)
+	registerConfig(App)
+	registerUtil(App)
+
+	access.Register(App)
+	clustermgr.Register(App)
+}

blobstore/cli/cli/cli.conf

@@ -0,0 +1,25 @@
+{
+    "access": {
+        "conn_mode": 4,
+        "consul_addr": "http://127.0.0.1:8500",
+        "max_host_retry": 0,
+        "max_part_retry": 0,
+        "max_size_put_once": 0,
+        "priority_addrs": [
+            "http://localhost:9500",
+            "http://127.0.0.1:9500"
+        ],
+        "service_interval_ms": 0
+    },
+    "redis_addrs": [
+        "127.0.0.1:6379"
+    ],
+    "redis_user": "",
+    "redis_pass": "",
+    "cm_addrs": [
+        "http://localhost:9998",
+        "http://127.0.0.1:9998"
+    ],
+    "verbose": false,
+    "vverbose": false
+}

blobstore/cli/cli/main.go

@@ -0,0 +1,25 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package main
+
+import (
+	"github.com/desertbit/grumble"
+
+	"github.com/cubefs/cubefs/blobstore/cli"
+)
+
+func main() {
+	grumble.Main(cli.App)
+}

blobstore/cli/clustermgr/clustermgr.go

@@ -0,0 +1,107 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package clustermgr
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/desertbit/grumble"
+
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+	"github.com/cubefs/cubefs/blobstore/cli/config"
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+	"github.com/cubefs/cubefs/blobstore/common/rpc/auth"
+)
+
+func newCMClient(secret string, hosts ...string) *clustermgr.Client {
+	if len(hosts) == 0 {
+		hosts = config.ClusterMgrAddrs()
+	}
+	if secret == "" {
+		secret = config.ClusterMgrSecret()
+	}
+	enableAuth := false
+	if secret != "" {
+		enableAuth = true
+	}
+	return clustermgr.New(&clustermgr.Config{
+		LbConfig: rpc.LbConfig{
+			Hosts: hosts,
+			Config: rpc.Config{
+				Tc: rpc.TransportConfig{
+					Auth: auth.Config{
+						EnableAuth: enableAuth,
+						Secret:     secret,
+					},
+				},
+			},
+		},
+	})
+}
+
+func clusterFlags(f *grumble.Flags) {
+	f.StringL("host", "", "specific clustermgr host")
+	f.StringL("secret", "", "specific clustermgr secret")
+}
+
+func specificHosts(f grumble.FlagMap) []string {
+	var hosts []string
+	for _, host := range strings.Split(f.String("host"), ",") {
+		host = strings.TrimSpace(host)
+		if host == "" {
+			continue
+		}
+		if !strings.HasPrefix(host, "http") {
+			host = "http://" + host
+		}
+		hosts = append(hosts, host)
+	}
+	return hosts
+}
+
+// Register register cm
+func Register(app *grumble.App) {
+	cmCommand := &grumble.Command{
+		Name: "cm",
+		Help: "cluster manager tools",
+	}
+	app.AddCommand(cmCommand)
+
+	addCmdConfig(cmCommand)
+	addCmdService(cmCommand)
+	addCmdWalParse(cmCommand)
+	addCmdVolume(cmCommand)
+	addCmdListAllDB(cmCommand)
+	addCmdDisk(cmCommand)
+
+	cmCommand.AddCommand(&grumble.Command{
+		Name: "stat",
+		Help: "show stat of clustermgr",
+		Flags: func(f *grumble.Flags) {
+			clusterFlags(f)
+		},
+		Run: func(c *grumble.Context) error {
+			cli := newCMClient(c.Flags.String("secret"), specificHosts(c.Flags)...)
+			stat, err := cli.Stat(common.CmdContext())
+			if err != nil {
+				return err
+			}
+			fmt.Println(common.Readable(stat))
+			return nil
+		},
+	})
+}

blobstore/cli/clustermgr/config.go

@@ -0,0 +1,151 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package clustermgr
+
+import (
+	"fmt"
+	"net/http"
+	"sort"
+
+	"github.com/desertbit/grumble"
+
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+	"github.com/cubefs/cubefs/blobstore/cli/common/flags"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+)
+
+func cmdGetConfig(c *grumble.Context) error {
+	cli, ctx := newCMClient(c.Flags.String("secret"), specificHosts(c.Flags)...), common.CmdContext()
+
+	verbose := flags.Verbose(c.Flags)
+	key := c.Args.String("key")
+	if key != "" {
+		value, err := cli.GetConfig(ctx, key)
+		if err != nil {
+			return err
+		}
+		showConfig(key, value, verbose)
+		return nil
+	}
+
+	all, err := cli.ListConfig(ctx)
+	if err != nil {
+		return err
+	}
+	keys := make([]string, 0, len(all.Configs))
+	for key := range all.Configs {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+
+	for _, key := range keys {
+		showConfig(key, all.Configs[key], verbose)
+	}
+	return nil
+}
+
+func showConfig(key, value string, verbose bool) {
+	if verbose && key == "code_mode" {
+		policies := make([]codemode.Policy, 0)
+		if common.Unmarshal([]byte(value), &policies) == nil {
+			value = "\n" + common.Readable(policies)
+		}
+	}
+	fmt.Println("    ", key, ":", value)
+}
+
+func addCmdConfig(cmd *grumble.Command) {
+	configCommand := &grumble.Command{
+		Name:     "config",
+		Help:     "config tools",
+		LongHelp: "config tools for clustermgr",
+	}
+	cmd.AddCommand(configCommand)
+
+	configCommand.AddCommand(&grumble.Command{
+		Name: "get",
+		Help: "show config of [key]",
+		Run:  cmdGetConfig,
+		Args: func(a *grumble.Args) {
+			a.String("key", "config key", grumble.Default(""))
+		},
+		Flags: func(f *grumble.Flags) {
+			flags.VerboseRegister(f)
+			clusterFlags(f)
+		},
+	})
+	configCommand.AddCommand(&grumble.Command{
+		Name: "set",
+		Help: "set config of key",
+		Args: func(a *grumble.Args) {
+			a.String("key", "config key")
+			a.String("value", "config value")
+		},
+		Flags: func(f *grumble.Flags) {
+			clusterFlags(f)
+		},
+		Run: func(c *grumble.Context) error {
+			key := c.Args.String("key")
+			value := c.Args.String("value")
+
+			cli, ctx := newCMClient(c.Flags.String("secret"), specificHosts(c.Flags)...), common.CmdContext()
+			oldV, err := cli.GetConfig(ctx, key)
+			if err != nil {
+				if rpc.DetectStatusCode(err) != http.StatusNotFound ||
+					!common.Confirm("to set new key: "+common.Loaded.Sprint(key)+" ?") {
+					return err
+				}
+			}
+			showConfig(key, oldV, true)
+
+			if common.Confirm(fmt.Sprintf(
+				"to set key: `%s` from `%s` --> `%s` ?", common.Loaded.Sprint(key),
+				common.Danger.Sprint(oldV), common.Normal.Sprint(value))) {
+				return cli.SetConfig(ctx, &clustermgr.ConfigSetArgs{
+					Key:   key,
+					Value: value,
+				})
+			}
+			return nil
+		},
+	})
+	configCommand.AddCommand(&grumble.Command{
+		Name: "del",
+		Help: "del config of key",
+		Args: func(a *grumble.Args) {
+			a.String("key", "config key")
+		},
+		Flags: func(f *grumble.Flags) {
+			clusterFlags(f)
+		},
+		Run: func(c *grumble.Context) error {
+			key := c.Args.String("key")
+
+			cli, ctx := newCMClient(c.Flags.String("secret"), specificHosts(c.Flags)...), common.CmdContext()
+			oldV, err := cli.GetConfig(ctx, key)
+			if err != nil {
+				return err
+			}
+			showConfig(key, oldV, true)
+
+			if common.Confirm("to del key: " + common.Loaded.Sprint(key) + " ?") {
+				return cli.DeleteConfig(ctx, key)
+			}
+			return nil
+		},
+	})
+}

blobstore/cli/clustermgr/diskmgr.go

@@ -0,0 +1,203 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package clustermgr
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/desertbit/grumble"
+
+	"github.com/cubefs/cubefs/blobstore/api/blobnode"
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+	"github.com/cubefs/cubefs/blobstore/cli/common/args"
+	"github.com/cubefs/cubefs/blobstore/cli/common/cfmt"
+	"github.com/cubefs/cubefs/blobstore/cli/common/flags"
+	"github.com/cubefs/cubefs/blobstore/cli/config"
+	"github.com/cubefs/cubefs/blobstore/clustermgr/persistence/normaldb"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/util/errors"
+)
+
+func addCmdDisk(cmd *grumble.Command) {
+	command := &grumble.Command{
+		Name:     "disk",
+		Help:     "disk tools",
+		LongHelp: "disk tools for clustermgr",
+	}
+	cmd.AddCommand(command)
+
+	command.AddCommand(&grumble.Command{
+		Name: "get",
+		Help: "show disk <diskid>",
+		Run:  cmdGetDisk,
+		Args: func(a *grumble.Args) {
+			args.DiskIDRegister(a)
+		},
+		Flags: func(f *grumble.Flags) {
+			flags.VerboseRegister(f)
+			clusterFlags(f)
+		},
+	})
+
+	command.AddCommand(&grumble.Command{
+		Name: "listDisk",
+		Help: "show disks",
+		Run:  cmdListDisks,
+		Flags: func(f *grumble.Flags) {
+			flags.VverboseRegister(f)
+			flags.VerboseRegister(f)
+			clusterFlags(f)
+
+			f.UintL("status", 0, "list disk status")
+			f.Int64L("marker", 0, "list disk marker")
+			f.IntL("count", 0, "list disk count")
+		},
+	})
+
+	command.AddCommand(&grumble.Command{
+		Name: "updateDisk",
+		Help: "update disk info in db",
+		Run:  cmdUpdateDisk,
+		Args: func(a *grumble.Args) {
+			args.DiskIDRegister(a)
+			a.String("dbPath", "normal db path")
+			a.String("diskInfo", "modify disk info data")
+		},
+		Flags: func(f *grumble.Flags) {
+			flags.VerboseRegister(f)
+			clusterFlags(f)
+		},
+	})
+}
+
+func cmdGetDisk(c *grumble.Context) error {
+	cmClient := newCMClient(c.Flags.String("secret"), specificHosts(c.Flags)...)
+	ctx := common.CmdContext()
+	disk, err := cmClient.DiskInfo(ctx, args.DiskID(c.Args))
+	if err != nil {
+		return err
+	}
+	if config.Verbose() || flags.Verbose(c.Flags) {
+		fmt.Println(cfmt.DiskInfoJoinV(disk, ""))
+	} else {
+		fmt.Println(disk)
+	}
+	return nil
+}
+
+func cmdListDisks(c *grumble.Context) error {
+	cmClient := newCMClient(c.Flags.String("secret"), specificHosts(c.Flags)...)
+	ctx := common.CmdContext()
+
+	args := &clustermgr.ListOptionArgs{
+		Status: proto.DiskStatus(c.Flags.Uint("status")),
+		Marker: proto.DiskID(c.Flags.Int64("marker")),
+		Count:  c.Flags.Int("count"),
+	}
+	if args.Marker <= proto.InvalidDiskID {
+		args.Marker = proto.DiskID(1)
+	}
+
+	verbose := config.Verbose() || flags.Verbose(c.Flags)
+	vv := flags.Vverbose(c.Flags)
+	next := true
+	num := 0
+	ac := common.NewAlternateColor(3)
+	for next && args.Marker > proto.InvalidDiskID {
+		disks, err := cmClient.ListDisk(ctx, args)
+		if err != nil {
+			return err
+		}
+
+		for _, disk := range disks.Disks {
+			num++
+			if verbose || vv {
+				fmt.Printf("%4d. %s\n", num, strings.Repeat("- ", 60))
+				if vv {
+					ac.Next().Println(cfmt.DiskInfoJoinV(disk, "  "))
+				} else {
+					ac.Next().Println(cfmt.DiskInfoJoin(disk, "  "))
+				}
+			} else {
+				ac.Next().Printf("%4d. %v\n", num, disk)
+			}
+		}
+
+		if disks.Marker == proto.InvalidDiskID || len(disks.Disks) < args.Count {
+			next = false
+		} else {
+			args.Marker = disks.Marker
+			fmt.Println()
+			next = common.Confirm("list next page?")
+		}
+	}
+	return nil
+}
+
+func cmdUpdateDisk(c *grumble.Context) error {
+	diskid := args.DiskID(c.Args)
+	dbPath := c.Args.String("dbPath")
+	data := c.Args.String("diskInfo")
+	if diskid <= 0 || dbPath == "" || data == "" {
+		return errors.New("invalid common args")
+	}
+	diskInfo := &blobnode.DiskInfo{}
+	err := json.Unmarshal([]byte(data), diskInfo)
+	if err != nil {
+		return err
+	}
+	db, err := openNormalDB(dbPath, false)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+	tbl, err := openDiskTable(db)
+	if err != nil {
+		return err
+	}
+	diskRec, err := tbl.GetDisk(proto.DiskID(diskid))
+	if err != nil {
+		return err
+	}
+	if diskInfo.MaxChunkCnt > 0 {
+		diskRec.MaxChunkCnt = diskInfo.MaxChunkCnt
+	}
+	if diskInfo.FreeChunkCnt > 0 {
+		diskRec.FreeChunkCnt = diskInfo.FreeChunkCnt
+	}
+	if diskInfo.UsedChunkCnt > 0 {
+		diskRec.UsedChunkCnt = diskInfo.UsedChunkCnt
+	}
+	if diskInfo.Status > 0 {
+		diskRec.Status = diskInfo.Status
+	}
+
+	if !common.Confirm("to change?\n") {
+		return nil
+	}
+
+	return tbl.AddDisk(diskRec)
+}
+
+func openDiskTable(db *normaldb.NormalDB) (*normaldb.DiskTable, error) {
+	tbl, err := normaldb.OpenDiskTable(db, true)
+	if err != nil {
+		return nil, err
+	}
+	return tbl, nil
+}

blobstore/cli/clustermgr/list_all.go

@@ -0,0 +1,246 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package clustermgr
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/cubefs/cubefs/blobstore/common/kvstore"
+
+	"github.com/desertbit/grumble"
+
+	"github.com/cubefs/cubefs/blobstore/clustermgr/persistence/normaldb"
+	"github.com/cubefs/cubefs/blobstore/clustermgr/persistence/volumedb"
+)
+
+type volInfo struct {
+	volumedb.VolumeRecord
+	units []volumedb.VolumeUnitRecord
+	token volumedb.TokenRecord
+}
+
+type serviceNode struct {
+	ClusterID uint64    `json:"cluster_id"`
+	Name      string    `json:"moduleName"`
+	Host      string    `json:"host"`
+	Idc       string    `json:"idc"`
+	Timeout   int       `json:"timeout"`
+	Expires   time.Time `json:"expires"`
+}
+
+func addCmdListAllDB(cmd *grumble.Command) {
+	command := &grumble.Command{
+		Name:     "listAllDB",
+		Help:     "list all db tools",
+		LongHelp: "list all db tools for clustermgr",
+		Run:      cmdListAllDB,
+		Args: func(a *grumble.Args) {
+			a.String("volumeDBPath", "wal log filename")
+			a.String("normalDBPath", "wal log path")
+		},
+	}
+	cmd.AddCommand(command)
+}
+
+func cmdListAllDB(c *grumble.Context) error {
+	volumeDBPath := c.Args.String("volumeDBPath")
+	normalDBPath := c.Args.String("normalDBPath")
+	if volumeDBPath == "" || normalDBPath == "" {
+		return errors.New("invalid command arguments")
+	}
+
+	volumeDB, err := openVolumeDB(volumeDBPath, false)
+	if err != nil {
+		return err
+	}
+	defer volumeDB.Close()
+	normalDB, err := openNormalDB(normalDBPath, false)
+	if err != nil {
+		return err
+	}
+	defer normalDB.Close()
+
+	volumeTbl, err := volumedb.OpenVolumeTable(volumeDB)
+	if err != nil {
+		return err
+	}
+	fmt.Println("list volumes: ")
+	err = listAllVolumes(volumeTbl)
+	if err != nil {
+		return err
+	}
+	fmt.Println()
+
+	diskTbl, err := normaldb.OpenDiskTable(normalDB, true)
+	if err != nil {
+		return err
+	}
+	fmt.Println("list disk: ")
+	err = listAllDisks(diskTbl)
+	if err != nil {
+		return err
+	}
+	fmt.Println()
+
+	configTbl, err := normaldb.OpenConfigTable(normalDB)
+	if err != nil {
+		return err
+	}
+	fmt.Println("list config: ")
+	err = listAllConfigs(configTbl)
+	if err != nil {
+		return err
+	}
+	fmt.Println()
+
+	dropTbl, err := normaldb.OpenDroppedDiskTable(normalDB)
+	if err != nil {
+		return err
+	}
+	fmt.Println("list dropping disk: ")
+	err = listAllDroppingDisks(dropTbl)
+	if err != nil {
+		return err
+	}
+	fmt.Println()
+
+	scopeTbl, err := normaldb.OpenScopeTable(normalDB)
+	if err != nil {
+		return err
+	}
+	fmt.Println("list scope: ")
+	err = listAllScopes(scopeTbl)
+	if err != nil {
+		return err
+	}
+	fmt.Println()
+
+	serviceTbl := normaldb.OpenServiceTable(normalDB)
+	if err != nil {
+		return err
+	}
+	fmt.Println("list service: ")
+	err = listAllServices(serviceTbl)
+	if err != nil {
+		return err
+	}
+	fmt.Println()
+
+	return nil
+}
+
+func openNormalDB(path string, readonly bool) (*normaldb.NormalDB, error) {
+	db, err := normaldb.OpenNormalDB(path, false, func(option *kvstore.RocksDBOption) {
+		option.ReadOnly = readonly
+	})
+	if err != nil {
+		return nil, fmt.Errorf("open db failed, err: %s", err.Error())
+	}
+	return db, nil
+}
+
+func listAllVolumes(volumeTbl *volumedb.VolumeTable) error {
+	return volumeTbl.RangeVolumeRecord(func(volRecord *volumedb.VolumeRecord) error {
+		volInfo := volInfo{
+			VolumeRecord: *volRecord,
+			units:        make([]volumedb.VolumeUnitRecord, 0),
+		}
+
+		for _, vuidPrefix := range volRecord.VuidPrefixs {
+			unitRecord, err := volumeTbl.GetVolumeUnit(vuidPrefix)
+			if err != nil {
+				return fmt.Errorf("get volume unit failed: %s", err.Error())
+			}
+			volInfo.units = append(volInfo.units, *unitRecord)
+		}
+
+		token, err := volumeTbl.GetToken(volRecord.Vid)
+		if err != nil {
+			return fmt.Errorf("get volume token failed: %s", err.Error())
+		}
+		if token != nil {
+			volInfo.token = *token
+		}
+
+		data, err := json.Marshal(volInfo)
+		if err != nil {
+			return fmt.Errorf("json marshal failed, err: %s", err.Error())
+		}
+		fmt.Println(string(data))
+		return nil
+	})
+}
+
+func listAllDisks(tbl *normaldb.DiskTable) error {
+	list, err := tbl.GetAllDisks()
+	if err != nil {
+		return fmt.Errorf("list disk failed, err: %s", err.Error())
+	}
+	for i := range list {
+		data, err := json.Marshal(list[i])
+		if err != nil {
+			return fmt.Errorf("json marshal failed, err: %s", err.Error())
+		}
+		fmt.Println(string(data))
+	}
+	return nil
+}
+
+func listAllConfigs(tbl *normaldb.ConfigTable) error {
+	list, err := tbl.List()
+	if err != nil {
+		return fmt.Errorf("list disk failed, err: %s", err.Error())
+	}
+	fmt.Println(list)
+	return nil
+}
+
+func listAllDroppingDisks(tbl *normaldb.DroppedDiskTable) error {
+	list, err := tbl.GetAllDroppingDisk()
+	if err != nil {
+		return fmt.Errorf("list dropping disk failed, err: %s", err.Error())
+	}
+	fmt.Println(list)
+	return nil
+}
+
+func listAllScopes(tbl *normaldb.ScopeTable) error {
+	scopes, err := tbl.Load()
+	if err != nil {
+		return fmt.Errorf("list scopes failed, err: %s", err.Error())
+	}
+	fmt.Println(scopes)
+	return nil
+}
+
+func listAllServices(tbl *normaldb.ServiceTable) error {
+	var retErr error
+	err := tbl.Range(func(key, val []byte) bool {
+		node := &serviceNode{}
+		if err := json.Unmarshal(val, &node); err != nil {
+			retErr = fmt.Errorf("json unmarshal one service from db failed, err: %s", err.Error())
+			return false
+		}
+		fmt.Println(string(val))
+		return true
+	})
+	if err != nil {
+		retErr = err
+	}
+	return retErr
+}

blobstore/cli/clustermgr/raft_wal.go

@@ -0,0 +1,172 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package clustermgr
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"hash/crc32"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+
+	"github.com/desertbit/grumble"
+	pb "go.etcd.io/etcd/raft/v3/raftpb"
+
+	"github.com/cubefs/cubefs/blobstore/clustermgr/base"
+	"github.com/cubefs/cubefs/blobstore/clustermgr/volumemgr"
+)
+
+var logger *log.Logger
+
+func addCmdWalParse(cmd *grumble.Command) {
+	command := &grumble.Command{
+		Name:     "wal",
+		Help:     "wal tools",
+		LongHelp: "wal tools for clustermgr",
+	}
+	cmd.AddCommand(command)
+
+	command.AddCommand(&grumble.Command{
+		Name: "parse",
+		Help: "parse wal file or path",
+		Run:  cmdWalParse,
+		Args: func(a *grumble.Args) {
+			a.String("filename", "wal log filename", grumble.Default(""))
+			a.String("path", "wal log path", grumble.Default(""))
+			a.String("out", "pase result output", grumble.Default(""))
+		},
+	})
+}
+
+func cmdWalParse(c *grumble.Context) error {
+	filename := c.Args.String("filename")
+	path := c.Args.String("path")
+	out := c.Args.String("out")
+
+	if filename == "" && path == "" {
+		return errors.New("wal log file path can't be null")
+	}
+	if out != "" {
+		f, err := os.OpenFile(out, os.O_CREATE|os.O_APPEND, 0o755)
+		if err != nil {
+			return err
+		}
+		logger = log.New(f, "", log.LstdFlags)
+	}
+
+	if filename != "" {
+		parse(filename)
+		return nil
+	}
+	fis, err := ioutil.ReadDir(path)
+	if err != nil {
+		return err
+	}
+	for i := range fis {
+		if fis[i].IsDir() {
+			continue
+		}
+		parse(path + fis[i].Name())
+	}
+	return nil
+}
+
+func parse(filename string) {
+	f, err := os.Open(filename)
+	if err != nil {
+		printf("open %s error: %v", filename, err)
+		return
+	}
+
+	u64Buf := make([]byte, 8)
+	var offset int64
+	for {
+		n, err := io.ReadFull(f, u64Buf)
+		if err != nil {
+			if err != io.EOF {
+				printf("read data len offset=%d error: %v\n", offset, err)
+			}
+			break
+		}
+		offset = offset + int64(n)
+		dataLen := binary.BigEndian.Uint64(u64Buf)
+		crc := crc32.NewIEEE()
+		tr := io.TeeReader(f, crc)
+		data := make([]byte, dataLen+4)
+		n, err = io.ReadFull(tr, data[0:int(dataLen)])
+		if err != nil {
+			printf("read data offset=%d error: %v\n", offset, err)
+			break
+		}
+		offset = offset + int64(n)
+		n, err = io.ReadFull(f, data[dataLen:])
+		if err != nil {
+			printf("read data crc offset=%d error: %v\n", offset, err)
+			break
+		}
+		offset = offset + int64(n)
+
+		// decode crc
+		recType := data[0]
+		checksum := binary.BigEndian.Uint32(data[dataLen:])
+		data = data[1:dataLen]
+		// checksum
+		if checksum != crc.Sum32() {
+			printf("error checksum=%d\n", checksum)
+			break
+		}
+
+		if recType != 1 {
+			fmt.Println("type: ", recType)
+			break
+		}
+
+		var entry pb.Entry
+		if err = entry.Unmarshal(data); err != nil {
+			printf("unmarshal error: %v\n", err)
+			break
+		}
+		if entry.Type == pb.EntryConfChange {
+			var cc pb.ConfChange
+			cc.Unmarshal(entry.Data)
+			printf("term=%d index=%d ConfChange=%s\n", entry.Term, entry.Index, cc.String())
+		} else {
+			if len(entry.Data) == 0 {
+				printf("term=%d index=%d\n", entry.Term, entry.Index)
+			} else {
+				data := entry.Data[8:]
+				proposeInfo := base.DecodeProposeInfo(data)
+				if proposeInfo.Module == "VolumeMgr" && proposeInfo.OperType == volumemgr.OperTypeChunkReport {
+					printf("term=%d index=%d module=%s opertype=%d data=%v \n",
+						entry.Term, entry.Index, proposeInfo.Module, proposeInfo.OperType, proposeInfo.Data)
+				} else {
+					printf("term=%d index=%d module=%s opertype=%d data=%s \n",
+						entry.Term, entry.Index, proposeInfo.Module, proposeInfo.OperType, string(proposeInfo.Data))
+				}
+			}
+		}
+	}
+}
+
+func printf(format string, opts ...interface{}) {
+	if logger != nil {
+		logger.Printf(format, opts...)
+	} else {
+		fmt.Printf(format, opts...)
+	}
+}

blobstore/cli/clustermgr/service.go

@@ -0,0 +1,68 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package clustermgr
+
+import (
+	"fmt"
+
+	"github.com/desertbit/grumble"
+
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+func cmdGetService(c *grumble.Context) error {
+	cli, ctx := newCMClient(c.Flags.String("secret"), specificHosts(c.Flags)...), common.CmdContext()
+
+	names := []string{
+		proto.ServiceNameProxy,
+		proto.ServiceNameBlobNode,
+	}
+	name := c.Args.String("name")
+	if name != "" {
+		names = []string{name}
+	}
+
+	for _, name := range names {
+		fmt.Println("Service of", name, ":")
+		info, err := cli.GetService(ctx, clustermgr.GetServiceArgs{
+			Name: name,
+		})
+		if err != nil {
+			return err
+		}
+		fmt.Println(common.Readable(info))
+		fmt.Println()
+	}
+
+	return nil
+}
+
+func addCmdService(cmd *grumble.Command) {
+	serviceCommand := &grumble.Command{
+		Name:     "service",
+		Help:     "service tools",
+		LongHelp: "service tools for clustermgr [name]",
+		Run:      cmdGetService,
+		Args: func(a *grumble.Args) {
+			a.String("name", "service name", grumble.Default(""))
+		},
+		Flags: func(f *grumble.Flags) {
+			clusterFlags(f)
+		},
+	}
+	cmd.AddCommand(serviceCommand)
+}

blobstore/cli/clustermgr/volumemgr.go

@@ -0,0 +1,251 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package clustermgr
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/desertbit/grumble"
+
+	"github.com/cubefs/cubefs/blobstore/api/blobnode"
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+	"github.com/cubefs/cubefs/blobstore/cli/common/args"
+	"github.com/cubefs/cubefs/blobstore/cli/common/cfmt"
+	"github.com/cubefs/cubefs/blobstore/cli/common/flags"
+	"github.com/cubefs/cubefs/blobstore/clustermgr/persistence/volumedb"
+	"github.com/cubefs/cubefs/blobstore/common/kvstore"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/util/task"
+)
+
+func addCmdVolume(cmd *grumble.Command) {
+	command := &grumble.Command{
+		Name:     "volume",
+		Help:     "volume tools",
+		LongHelp: "volume tools for clustermgr",
+	}
+	cmd.AddCommand(command)
+
+	command.AddCommand(&grumble.Command{
+		Name: "listVolumeUnits",
+		Help: "show volume units",
+		Run:  cmdListVolumeUnits,
+		Args: func(a *grumble.Args) {
+			args.VidRegister(a)
+		},
+		Flags: func(f *grumble.Flags) {
+			flags.VerboseRegister(f)
+			clusterFlags(f)
+		},
+	})
+
+	command.AddCommand(&grumble.Command{
+		Name: "updateVolume",
+		Help: "update volume info",
+		Run:  cmdUpdateVolume,
+		Args: func(a *grumble.Args) {
+			args.VidRegister(a)
+			a.String("dbPath", "volume db path")
+			a.String("volumeInfo", "modify volume info, "+
+				"only support for codemode/status/total/used/free")
+		},
+		Flags: func(f *grumble.Flags) {
+			flags.VerboseRegister(f)
+			clusterFlags(f)
+		},
+	})
+
+	command.AddCommand(&grumble.Command{
+		Name: "updateVolumeUnit",
+		Help: "update volume unit",
+		Run:  cmdUpdateVolumeUnit,
+		Args: func(a *grumble.Args) {
+			args.VuidRegister(a)
+			a.String("dbPath", "volume db path")
+			a.String("unit", "modify volume unit, "+
+				"only support for diskid/epoch/nexEpoch/compacting")
+		},
+		Flags: func(f *grumble.Flags) {
+			flags.VerboseRegister(f)
+			clusterFlags(f)
+		},
+	})
+}
+
+func cmdUpdateVolume(c *grumble.Context) error {
+	vid := args.Vid(c.Args)
+	dbPath := c.Args.String("dbPath")
+	data := c.Args.String("volumeInfo")
+	if vid == 0 || dbPath == "" || data == "" {
+		return errors.New("invalid command arguments")
+	}
+
+	modifyInfo := &clustermgr.VolumeInfo{}
+	err := json.Unmarshal([]byte(data), modifyInfo)
+	if err != nil {
+		return err
+	}
+	db, err := openVolumeDB(dbPath, false)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+	tbl, err := openVolumeTable(db)
+	if err != nil {
+		return err
+	}
+	srcInfo, err := tbl.GetVolume(proto.Vid(vid))
+	if err != nil {
+		return err
+	}
+
+	if modifyInfo.Status.IsValid() {
+		srcInfo.Status = modifyInfo.Status
+	}
+	if modifyInfo.CodeMode.IsValid() {
+		srcInfo.CodeMode = modifyInfo.CodeMode
+	}
+	if modifyInfo.Total > 0 {
+		srcInfo.Total = modifyInfo.Total
+	}
+	if modifyInfo.Used > 0 {
+		srcInfo.Used = modifyInfo.Used
+	}
+	if modifyInfo.Free > 0 {
+		srcInfo.Free = modifyInfo.Free
+	}
+
+	if !common.Confirm("to change?\n" + cfmt.VolumeInfoJoin(modifyInfo, "")) {
+		return nil
+	}
+	return tbl.PutVolumeRecord(srcInfo)
+}
+
+func cmdListVolumeUnits(c *grumble.Context) error {
+	cmClient := newCMClient(c.Flags.String("secret"), specificHosts(c.Flags)...)
+	ctx := common.CmdContext()
+	volumeInfo, err := cmClient.GetVolumeInfo(ctx,
+		&clustermgr.GetVolumeArgs{Vid: args.Vid(c.Args)})
+	if err != nil {
+		return err
+	}
+
+	fmt.Println("volume info:")
+	fmt.Println(cfmt.VolumeInfoJoin(volumeInfo, "\t"))
+
+	verbose := flags.Verbose(c.Flags)
+
+	dnConfig := blobnode.Config{}
+	dnConfig.ClientTimeoutMs = 3000
+	blobnodeCli := blobnode.New(&dnConfig)
+
+	n := len(volumeInfo.Units)
+	loader := common.Loader(n)
+
+	taskArgs := make([]interface{}, n)
+	for i, arg := range volumeInfo.Units {
+		taskArgs[i] = arg
+	}
+	chunkInfos := make([]string, n)
+	task.C(func(i int, taskArgs interface{}) {
+		unit := taskArgs.(clustermgr.Unit)
+		info, err := blobnodeCli.StatChunk(ctx, unit.Host,
+			&blobnode.StatChunkArgs{
+				DiskID: unit.DiskID,
+				Vuid:   unit.Vuid,
+			})
+		if err != nil {
+			chunkInfos[i] = fmt.Sprintf("ERROR: %s %d %s", unit.Host, unit.Vuid, err.Error())
+		} else if verbose {
+			chunkInfos[i] = cfmt.ChunkInfoJoin(info, "\t")
+		} else {
+			chunkInfos[i] = fmt.Sprint(info)
+		}
+		loader <- 1
+	}, taskArgs)
+	time.Sleep(100 * time.Millisecond)
+
+	fmt.Println("chunks:")
+	for i, info := range chunkInfos {
+		if verbose {
+			fmt.Println("chunk info:", i)
+		}
+		fmt.Println(info)
+	}
+
+	return nil
+}
+
+func cmdUpdateVolumeUnit(c *grumble.Context) error {
+	vuid := args.Vuid(c.Args)
+	dbPath := c.Args.String("dbPath")
+	data := c.Args.String("unitInfo")
+	if vuid == 0 || dbPath == "" || data == "" {
+		return errors.New("invalid command arguments")
+	}
+
+	modifyInfo := &clustermgr.AdminUpdateUnitArgs{}
+	err := json.Unmarshal([]byte(data), modifyInfo)
+	if err != nil {
+		return err
+	}
+	db, err := openVolumeDB(dbPath, false)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+	tbl, err := openVolumeTable(db)
+	if err != nil {
+		return err
+	}
+	srcInfo, err := tbl.GetVolumeUnit(vuid.VuidPrefix())
+	if err != nil {
+		return err
+	}
+	if modifyInfo.DiskID > 0 {
+		srcInfo.DiskID = modifyInfo.DiskID
+	}
+	if proto.IsValidEpoch(modifyInfo.Epoch) {
+		srcInfo.Epoch = modifyInfo.Epoch
+	}
+	if proto.IsValidEpoch(modifyInfo.NextEpoch) {
+		srcInfo.NextEpoch = modifyInfo.NextEpoch
+	}
+	srcInfo.Compacting = modifyInfo.Compacting
+
+	return tbl.PutVolumeUnit(vuid.VuidPrefix(), srcInfo)
+}
+
+func openVolumeDB(path string, readonly bool) (*volumedb.VolumeDB, error) {
+	db, err := volumedb.Open(path, false, func(option *kvstore.RocksDBOption) {
+		option.ReadOnly = readonly
+	})
+	if err != nil {
+		return nil, fmt.Errorf("open db failed, err: %s", err.Error())
+	}
+	return db, nil
+}
+
+func openVolumeTable(db *volumedb.VolumeDB) (*volumedb.VolumeTable, error) {
+	tbl, err := volumedb.OpenVolumeTable(db)
+	if err != nil {
+		return nil, fmt.Errorf("open volume table failed, err: %s", err.Error())
+	}
+	return tbl, nil
+}

blobstore/cli/common/args/args.go

@@ -0,0 +1,56 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package args
+
+import (
+	"github.com/desertbit/grumble"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+// Line lines
+func Line(a *grumble.Args) {
+	a.Int("line", "show lines at the tail", grumble.Default(40))
+}
+
+// VidRegister vid
+func VidRegister(a *grumble.Args, opts ...grumble.ArgOption) {
+	a.Uint64("vid", "volume id", opts...)
+}
+
+// Vid returns vid
+func Vid(a grumble.ArgMap) proto.Vid {
+	return proto.Vid(a.Uint64("vid"))
+}
+
+// DiskIDRegister disk id
+func DiskIDRegister(a *grumble.Args, opts ...grumble.ArgOption) {
+	a.Uint64("diskID", "disk id", opts...)
+}
+
+// DiskID returns disk id
+func DiskID(a grumble.ArgMap) proto.DiskID {
+	return proto.DiskID(a.Uint64("diskID"))
+}
+
+// VuidRegister vuid
+func VuidRegister(a *grumble.Args, opts ...grumble.ArgOption) {
+	a.Uint64("vuid", "vuid", opts...)
+}
+
+// Vuid returns vuid
+func Vuid(a grumble.ArgMap) proto.Vuid {
+	return proto.Vuid(a.Uint64("vuid"))
+}

blobstore/cli/common/cfmt/access.go

@@ -0,0 +1,96 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package cfmt
+
+import (
+	"fmt"
+
+	"github.com/dustin/go-humanize"
+
+	"github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+)
+
+// ParseLocation parst location from json or string
+func ParseLocation(jsonORstr string) (access.Location, error) {
+	var loc access.Location
+	var err error
+	if err = common.Unmarshal([]byte(jsonORstr), &loc); err == nil {
+		return loc, nil
+	}
+	if loc, err = access.DecodeLocationFromHex(jsonORstr); err == nil {
+		return loc, nil
+	}
+	if loc, err = access.DecodeLocationFromBase64(jsonORstr); err == nil {
+		return loc, nil
+	}
+	return loc, fmt.Errorf("invalid (%s) %s", jsonORstr, err.Error())
+}
+
+// LocationJoin join line into string
+func LocationJoin(loc *access.Location, prefix string) string {
+	return joinWithPrefix(prefix, LocationF(loc))
+}
+
+// LocationF fmt pointer of Location
+func LocationF(loc *access.Location) (vals []string) {
+	if loc == nil {
+		return []string{" <nil> "}
+	}
+	vals = make([]string, 0, 8)
+	vals = append(vals, []string{
+		fmt.Sprintf("Crc        : %-12d (0x%x)", loc.Crc, loc.Crc),
+		fmt.Sprintf("ClusterID  : %d", loc.ClusterID),
+		fmt.Sprintf("CodeMode   : %-12d (%s)", loc.CodeMode, loc.CodeMode.String()),
+		fmt.Sprintf("Size       : %-12d (%s)", loc.Size, humanize.IBytes(loc.Size)),
+		fmt.Sprintf("BlobSize   : %-12d (%s)", loc.BlobSize, humanize.IBytes(uint64(loc.BlobSize))),
+		fmt.Sprintf("Blobs: (%d) [", len(loc.Blobs)),
+	}...)
+	for idx, blob := range loc.Blobs {
+		vals = append(vals, fmt.Sprintf(" >:%3d| MinBid: %-20d Vid: %-10d Count: %-10d",
+			idx, blob.MinBid, blob.Vid, blob.Count))
+	}
+	vals = append(vals, "]")
+	vals = append(vals, fmt.Sprintf("--> Encode: %d of %d bytes", len(loc.Encode()), 21+16*len(loc.Blobs)))
+	vals = append(vals, fmt.Sprintf("--> Hex   : %s", loc.HexString()))
+	vals = append(vals, fmt.Sprintf("--> Base64: %s", loc.Base64String()))
+	return
+}
+
+// HashSumMapJoin join line into string
+func HashSumMapJoin(hashes access.HashSumMap, prefix string) string {
+	return joinWithPrefix(prefix, HashSumMapF(hashes))
+}
+
+// HashSumMapF fmt HashSumMap
+func HashSumMapF(hashes access.HashSumMap) (vals []string) {
+	if v, ok := hashes.GetSum(access.HashAlgCRC32); ok {
+		val := v.(uint32)
+		vals = append(vals, fmt.Sprintf("CRC32 : %d (%x)", val, val))
+	}
+	if v, ok := hashes.GetSum(access.HashAlgMD5); ok {
+		val := v.(string)
+		vals = append(vals, fmt.Sprintf("MD5   : %s", val))
+	}
+	if v, ok := hashes.GetSum(access.HashAlgSHA1); ok {
+		val := v.(string)
+		vals = append(vals, fmt.Sprintf("SHA1  : %s", val))
+	}
+	if v, ok := hashes.GetSum(access.HashAlgSHA256); ok {
+		val := v.(string)
+		vals = append(vals, fmt.Sprintf("SHA256: %s", val))
+	}
+	return
+}

blobstore/cli/common/cfmt/access_test.go

@@ -0,0 +1,113 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package cfmt_test
+
+import (
+	"crypto/rand"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+	"github.com/cubefs/cubefs/blobstore/cli/common/cfmt"
+)
+
+func TestParseLocation(t *testing.T) {
+	loc := access.Location{
+		ClusterID: 1,
+		CodeMode:  3,
+		Size:      19213422425,
+		BlobSize:  1 << 22,
+		Crc:       1 << 31,
+		Blobs: []access.SliceInfo{
+			{MinBid: 0x199, Vid: 100020, Count: 0},
+			{MinBid: 10, Vid: 20, Count: 300},
+			{MinBid: 14225224, Vid: 0xffffffff, Count: 1 << 31},
+		},
+	}
+
+	locx, err := cfmt.ParseLocation("")
+	require.Error(t, err)
+
+	_, err = cfmt.ParseLocation("xxxx")
+	require.Error(t, err)
+
+	locx, err = cfmt.ParseLocation("{}")
+	require.NoError(t, err)
+	require.Equal(t, access.Location{}, locx)
+
+	locx, err = cfmt.ParseLocation(loc.ToString())
+	require.NoError(t, err)
+	require.Equal(t, loc, locx)
+
+	locx, err = cfmt.ParseLocation(loc.Base64String())
+	require.NoError(t, err)
+	require.Equal(t, loc, locx)
+
+	b, _ := common.Marshal(loc)
+	locx, err = cfmt.ParseLocation(string(b))
+	require.NoError(t, err)
+	require.Equal(t, loc, locx)
+}
+
+func TestLocation(t *testing.T) {
+	loc := access.Location{
+		ClusterID: 1,
+		CodeMode:  3,
+		Size:      19213422425,
+		BlobSize:  1 << 22,
+		Crc:       1 << 31,
+		Blobs: []access.SliceInfo{
+			{MinBid: 0x199, Vid: 100020, Count: 0},
+			{MinBid: 10, Vid: 20, Count: 300},
+			{MinBid: 14225224, Vid: 0xffffffff, Count: 1 << 31},
+		},
+	}
+	printLine()
+	for _, line := range cfmt.LocationF(&loc) {
+		fmt.Println(line)
+	}
+	printLine()
+	fmt.Println(cfmt.LocationJoin(&loc, "\t-->\t"))
+	printLine()
+	fmt.Println(cfmt.LocationJoin(nil, "\t--> "))
+	printLine()
+}
+
+func TestHashSumMap(t *testing.T) {
+	hashes := make(access.HashSumMap)
+	algs := access.HashAlgorithm(0xff - 1)
+	for alg := range algs.ToHashSumMap() {
+		hasher := alg.ToHasher()
+		buf := make([]byte, 1024)
+		rand.Read(buf)
+		hasher.Write(buf)
+		hashes[alg] = hasher.Sum(nil)
+	}
+	printLine()
+	for _, printLine := range cfmt.HashSumMapF(hashes) {
+		fmt.Println(printLine)
+	}
+	printLine()
+	fmt.Println(cfmt.HashSumMapJoin(hashes, "xxx | --> "))
+	printLine()
+}
+
+func printLine() {
+	fmt.Println(strings.Repeat("-", 100))
+}

blobstore/cli/common/cfmt/blobnode.go

@@ -0,0 +1,132 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package cfmt
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/dustin/go-humanize"
+	"github.com/fatih/color"
+
+	"github.com/cubefs/cubefs/blobstore/api/blobnode"
+)
+
+var chunkStatus2Str = map[blobnode.ChunkStatus]string{
+	blobnode.ChunkStatusNormal:   "normal",
+	blobnode.ChunkStatusReadOnly: "readonly",
+	blobnode.ChunkStatusRelease:  "release",
+}
+
+// ChunkidF chunk id
+func ChunkidF(id blobnode.ChunkId) string {
+	t := id.UnixTime()
+	c := color.New(color.Faint, color.Italic)
+	return fmt.Sprintf("%s (V:%20d T:%20d %s)", id.String(), id.VolumeUnitId(),
+		t, c.Sprint(time.Unix(0, int64(t)).Format("2006-01-02 15:04:05.000")),
+	)
+}
+
+// ChunkInfoJoin chunk info
+func ChunkInfoJoin(info *blobnode.ChunkInfo, prefix string) string {
+	return joinWithPrefix(prefix, ChunkInfoF(info))
+}
+
+// ChunkInfoF chunk info
+func ChunkInfoF(info *blobnode.ChunkInfo) []string {
+	if info == nil {
+		return []string{" <nil> "}
+	}
+	return []string{
+		fmt.Sprintf("ID     : %s", ChunkidF(info.Id)),
+		fmt.Sprintf("Vuid   : %s", VuidCF(info.Vuid)),
+		fmt.Sprintf("DiskID : %-10d", info.DiskID),
+		fmt.Sprintf("Status : %d           (%s)", info.Status, chunkStatus2Str[info.Status]),
+		fmt.Sprintf("Compact: %v", info.Compacting),
+		fmt.Sprintf("Size   : %-24s", humanIBytes(info.Size)),
+		fmt.Sprintf("Total  : %-24s Used: %-24s Free: %-24s",
+			humanIBytes(info.Total), humanIBytes(info.Used), humanIBytes(info.Free)),
+	}
+}
+
+// DiskHeartBeatInfoJoin disk heartbeat info
+func DiskHeartBeatInfoJoin(info *blobnode.DiskHeartBeatInfo, prefix string) string {
+	return joinWithPrefix(prefix, DiskHeartBeatInfoF(info))
+}
+
+// DiskHeartBeatInfoF disk heartbeat info
+func DiskHeartBeatInfoF(info *blobnode.DiskHeartBeatInfo) []string {
+	if info == nil {
+		return []string{" <nil> "}
+	}
+	return []string{
+		fmt.Sprintf("DiskID: %-12d | MaxN: %-8d | UsedN: %-8d | FreeN: %-8d",
+			info.DiskID, info.MaxChunkCnt, info.UsedChunkCnt, info.FreeChunkCnt),
+		fmt.Sprintf("Size  : %-24s | Used: %-24s | Free: %-24s",
+			humanIBytes(info.Size), humanIBytes(info.Used), humanIBytes(info.Free)),
+	}
+}
+
+// DiskInfoJoin disk info
+func DiskInfoJoin(info *blobnode.DiskInfo, prefix string) string {
+	return joinWithPrefix(prefix, DiskInfoF(info))
+}
+
+// DiskInfoF disk info
+func DiskInfoF(info *blobnode.DiskInfo) []string {
+	if info == nil {
+		return []string{" <nil> "}
+	}
+	return append(DiskHeartBeatInfoF(&info.DiskHeartBeatInfo),
+		[]string{
+			fmt.Sprintf("ClusterID : %-4d | Readonly: %-6v | IDC: %-12s | Rack: %s",
+				info.ClusterID, info.Readonly, info.Idc, info.Rack),
+			fmt.Sprintf("Status  : %-10s(%d) | Host: %-30s | Path: %s",
+				info.Status, info.Status, info.Host, info.Path),
+			fmt.Sprintf("CreateAt: %s (%s) | LastUpdateAt: %s (%s)",
+				info.CreateAt.Format(time.RFC822), humanize.Time(info.CreateAt),
+				info.LastUpdateAt.Format(time.RFC822), humanize.Time(info.LastUpdateAt)),
+		}...)
+}
+
+// DiskInfoJoinV disk info verbose
+func DiskInfoJoinV(info *blobnode.DiskInfo, prefix string) string {
+	return joinWithPrefix(prefix, DiskInfoFV(info))
+}
+
+// DiskInfoFV disk info
+func DiskInfoFV(info *blobnode.DiskInfo) []string {
+	if info == nil {
+		return []string{" <nil> "}
+	}
+	return []string{
+		fmt.Sprint("DiskID   : ", info.DiskID),
+		fmt.Sprint("Readonly : ", info.Readonly),
+		fmt.Sprintf("Status   : %s(%d) ", info.Status, info.Status),
+		fmt.Sprintf("Chunk    : MaxN: %-18d | UsedN: %-23d | FreeN: %-24d",
+			info.MaxChunkCnt, info.UsedChunkCnt, info.FreeChunkCnt),
+		fmt.Sprintf("Size     : %-24s | Used: %-24s | Free: %-24s",
+			humanIBytes(info.Size), humanIBytes(info.Used), humanIBytes(info.Free)),
+		fmt.Sprint("ClusterID: ", info.ClusterID),
+		fmt.Sprint("IDC      : ", info.Idc),
+		fmt.Sprint("Rack     : ", info.Rack),
+		fmt.Sprint("Host     : ", info.Host),
+		fmt.Sprint("Path     : ", info.Path),
+		fmt.Sprintf("CreateAt : %s (%s)",
+			info.CreateAt.Format(time.RFC822), humanize.Time(info.CreateAt)),
+		fmt.Sprintf("UpdateAt : %s (%s)",
+			info.LastUpdateAt.Format(time.RFC822), humanize.Time(info.LastUpdateAt)),
+	}
+}

blobstore/cli/common/cfmt/blobnode_test.go

@@ -0,0 +1,110 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package cfmt_test
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/cubefs/cubefs/blobstore/api/blobnode"
+	"github.com/cubefs/cubefs/blobstore/cli/common/cfmt"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+func TestChunkInfo(t *testing.T) {
+	val := blobnode.ChunkInfo{
+		Id:         blobnode.NewChunkId(4191518780817409),
+		Vuid:       4191518780817409,
+		DiskID:     0xff,
+		Total:      1 << 35,
+		Used:       1 << 34,
+		Free:       100,
+		Size:       1 << 32,
+		Status:     blobnode.ChunkStatusReadOnly,
+		Compacting: false,
+	}
+	printLine()
+	for _, line := range cfmt.ChunkInfoF(&val) {
+		fmt.Println(line)
+	}
+	printLine()
+	fmt.Println(cfmt.ChunkInfoJoin(&val, "\t--> "))
+	printLine()
+	fmt.Println(cfmt.ChunkInfoJoin(nil, "\t--> "))
+	printLine()
+}
+
+func TestDiskHeartBeatInfo(t *testing.T) {
+	val := blobnode.DiskHeartBeatInfo{
+		DiskID:       12342522,
+		Used:         10220055555,
+		Free:         9483500,
+		Size:         10 * (1 << 40),
+		MaxChunkCnt:  9999,
+		FreeChunkCnt: 880,
+		UsedChunkCnt: 82,
+	}
+	printLine()
+	for _, line := range cfmt.DiskHeartBeatInfoF(&val) {
+		fmt.Println(line)
+	}
+	printLine()
+	fmt.Println(cfmt.DiskHeartBeatInfoJoin(&val, "\t--> "))
+	printLine()
+	fmt.Println(cfmt.DiskHeartBeatInfoJoin(nil, "\t--> "))
+	printLine()
+}
+
+func TestDiskInfo(t *testing.T) {
+	val := blobnode.DiskInfo{
+		ClusterID:    0xf2,
+		Idc:          "idc--xx",
+		Rack:         "rack-10",
+		Host:         "http://255.255.255.255:99999",
+		Path:         "/path/to/disk",
+		Status:       proto.DiskStatusRepaired,
+		Readonly:     false,
+		CreateAt:     time.Now().Add(-time.Hour * 1020),
+		LastUpdateAt: time.Now().Add(-time.Hour * 224),
+		DiskHeartBeatInfo: blobnode.DiskHeartBeatInfo{
+			DiskID:       12342522,
+			Used:         10220055555,
+			Free:         9483500,
+			Size:         10 * (1 << 40),
+			MaxChunkCnt:  9999,
+			FreeChunkCnt: 880,
+			UsedChunkCnt: 82,
+		},
+	}
+	printLine()
+	for _, line := range cfmt.DiskInfoF(&val) {
+		fmt.Println(line)
+	}
+	printLine()
+	fmt.Println(cfmt.DiskInfoJoin(&val, "\t--> "))
+	printLine()
+	fmt.Println(cfmt.DiskInfoJoin(nil, "\t--> "))
+	printLine()
+
+	for _, line := range cfmt.DiskInfoFV(&val) {
+		fmt.Println(line)
+	}
+	printLine()
+	fmt.Println(cfmt.DiskInfoJoinV(&val, "\t--> "))
+	printLine()
+	fmt.Println(cfmt.DiskInfoJoinV(nil, "\t--> "))
+	printLine()
+}

blobstore/cli/common/cfmt/cfmt.go

@@ -0,0 +1,43 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+// Package cfmt provides fmt string for all struct in blobstore
+//
+// function *F make the pointer struct value to []string
+// function *Join []string into string with profix each line, sep is '\n'
+//
+package cfmt
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/dustin/go-humanize"
+)
+
+func joinWithPrefix(prefix string, vals []string) string {
+	if len(prefix) > 0 {
+		for idx := range vals {
+			vals[idx] = prefix + vals[idx]
+		}
+	}
+	return strings.Join(vals, "\n")
+}
+
+func humanIBytes(size interface{}) string {
+	str := fmt.Sprintf("%v", size)
+	s, _ := strconv.Atoi(str)
+	return fmt.Sprintf("%s (%s)", str, humanize.IBytes(uint64(s)))
+}

blobstore/cli/common/cfmt/cluster.go

@@ -0,0 +1,83 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package cfmt
+
+import (
+	"fmt"
+
+	"github.com/dustin/go-humanize"
+
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+)
+
+// VolumeInfoJoin volume info
+func VolumeInfoJoin(vol *clustermgr.VolumeInfo, prefix string) string {
+	return joinWithPrefix(prefix, VolumeInfoF(vol))
+}
+
+// VolumeInfoF volume info
+func VolumeInfoF(vol *clustermgr.VolumeInfo) []string {
+	if vol == nil {
+		return []string{" <nil> "}
+	}
+	usedC := common.ColorizeUint64(vol.Used, vol.Total)
+	freeC := common.ColorizeUint64Free(vol.Free, vol.Total)
+	vals := make([]string, 0, 32)
+	vals = append(vals, []string{
+		fmt.Sprintf("Vid        : %d", vol.Vid),
+		fmt.Sprintf("CodeMode   : %-12d (%s)", vol.CodeMode, vol.CodeMode.String()),
+		fmt.Sprintf("Status     : %-12d (%s)", vol.Status, vol.Status.String()),
+		fmt.Sprintf("HealthScore: %d", vol.HealthScore),
+		fmt.Sprintf("Total      : %-16d (%s)", vol.Total, humanize.IBytes(vol.Total)),
+		fmt.Sprintf("Free       : %-16d (%s)", vol.Free, freeC.Sprint(humanize.IBytes(vol.Free))),
+		fmt.Sprintf("Used       : %-16d (%s)", vol.Used, usedC.Sprint(humanize.IBytes(vol.Used))),
+		fmt.Sprintf("Uints: (%d) [", len(vol.Units)),
+	}...)
+	alterColor := common.NewAlternateColor(3)
+	for idx, unit := range vol.Units {
+		vals = append(vals, alterColor.Next().Sprintf(" >:%3d| Vuid: %s | DiskID: %-10d | Host: %s",
+			idx, VuidF(unit.Vuid), unit.DiskID, unit.Host))
+	}
+	vals = append(vals, "]")
+	return vals
+}
+
+// ClusterInfoJoin cluster info
+func ClusterInfoJoin(info *clustermgr.ClusterInfo, prefix string) string {
+	return joinWithPrefix(prefix, ClusterInfoF(info))
+}
+
+// ClusterInfoF cluster info
+func ClusterInfoF(info *clustermgr.ClusterInfo) []string {
+	if info == nil {
+		return []string{" <nil> "}
+	}
+	avaiC := common.ColorizeInt64(-info.Available, info.Capacity)
+	vals := make([]string, 0, 8)
+	vals = append(vals, []string{
+		fmt.Sprintf("Region    : %s", info.Region),
+		fmt.Sprintf("ClusterID : %d", info.ClusterID),
+		fmt.Sprintf("Readonly  : %v", info.Readonly),
+		fmt.Sprintf("Capacity  : %-16d (%s)", info.Capacity, humanize.IBytes(uint64(info.Capacity))),
+		fmt.Sprintf("Available : %-16d (%s)", info.Available, avaiC.Sprint(humanize.IBytes(uint64(info.Available)))),
+		fmt.Sprintf("Nodes: (%d) [", len(info.Nodes)),
+	}...)
+	for _, node := range info.Nodes {
+		vals = append(vals, fmt.Sprintf("  - %s", node))
+	}
+	vals = append(vals, "]")
+	return vals
+}

blobstore/cli/common/cfmt/cluster_test.go

@@ -0,0 +1,76 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package cfmt_test
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/cli/common/cfmt"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+func TestVolumeInfo(t *testing.T) {
+	val := clustermgr.VolumeInfo{
+		VolumeInfoBase: clustermgr.VolumeInfoBase{
+			Vid:         101313,
+			CodeMode:    2,
+			Status:      2,
+			HealthScore: -1,
+			Total:       1 << 50,
+			Free:        1 << 32,
+			Used:        1 << 45,
+		},
+		Units: make([]clustermgr.Unit, 32),
+	}
+	for i := 0; i < len(val.Units); i++ {
+		val.Units[i] = clustermgr.Unit{
+			Vuid:   proto.Vuid(rand.Uint64()),
+			DiskID: proto.DiskID(rand.Uint32()),
+			Host:   fmt.Sprintf("host-%d", i),
+		}
+	}
+	printLine()
+	for _, line := range cfmt.VolumeInfoF(&val) {
+		fmt.Println(line)
+	}
+	printLine()
+	fmt.Println(cfmt.VolumeInfoJoin(&val, "\t--> "))
+	printLine()
+	fmt.Println(cfmt.VolumeInfoJoin(nil, "\t--> "))
+	printLine()
+}
+
+func TestClusterInfo(t *testing.T) {
+	val := clustermgr.ClusterInfo{
+		Region:    "foo-region",
+		ClusterID: 0xffff,
+		Capacity:  1 << 40,
+		Available: 1 << 38,
+		Readonly:  false,
+		Nodes:     []string{"node-1", "node-2", "node-xxx"},
+	}
+	printLine()
+	for _, line := range cfmt.ClusterInfoF(&val) {
+		fmt.Println(line)
+	}
+	printLine()
+	fmt.Println(cfmt.ClusterInfoJoin(&val, "\t--> "))
+	printLine()
+	fmt.Println(cfmt.ClusterInfoJoin(nil, "\t--> "))
+	printLine()
+}

blobstore/cli/common/cfmt/proto.go

@@ -0,0 +1,40 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package cfmt
+
+import (
+	"fmt"
+
+	"github.com/fatih/color"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+// VuidF vuid fmt
+func VuidF(vuid proto.Vuid) string {
+	return fmt.Sprintf("%-20d (V:%10d I:%3d E:%10d)", vuid,
+		vuid.Vid(), vuid.Index(), vuid.Epoch(),
+	)
+}
+
+// VuidCF vuid fmt with color
+func VuidCF(vuid proto.Vuid) string {
+	c := color.New(color.Faint, color.Italic)
+	return fmt.Sprintf("%-20d (%s%10d %s%3d %s%10d)", vuid,
+		c.Sprint("V:"), vuid.Vid(),
+		c.Sprint("I:"), vuid.Index(),
+		c.Sprint("E:"), vuid.Epoch(),
+	)
+}

blobstore/cli/common/color.go

@@ -0,0 +1,144 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package common
+
+import (
+	"github.com/fatih/color"
+)
+
+// colorize defined
+var (
+	Optimal = color.New(color.FgHiWhite)
+	Normal  = color.New(color.FgHiGreen)
+	Loaded  = color.New(color.FgHiBlue)
+	Warn    = color.New(color.FgHiYellow)
+	Danger  = color.New(color.FgHiRed)
+	Dead    = color.New(color.FgBlack, color.BgCyan)
+
+	alternateColor = []*color.Color{
+		color.New(color.FgHiWhite),
+		color.New(color.FgWhite, color.Faint),
+		color.New(color.FgHiBlue),
+		color.New(color.FgBlue, color.Faint),
+		color.New(color.FgHiGreen),
+		color.New(color.FgGreen, color.Faint),
+	}
+	maxAlternateColor = len(alternateColor)
+)
+
+// AlternateColor alternate color formatter
+type AlternateColor struct {
+	index  int
+	n      int
+	colors []*color.Color
+}
+
+// NewAlternateColor returns an alternate color formatter
+func NewAlternateColor(n int) *AlternateColor {
+	if n <= 0 {
+		n = 1
+	}
+	if n > maxAlternateColor {
+		n = maxAlternateColor
+	}
+	return &AlternateColor{
+		index:  0,
+		n:      n,
+		colors: alternateColor[:n],
+	}
+}
+
+// Next returns next color
+func (a *AlternateColor) Next() *color.Color {
+	c := a.colors[a.index]
+	a.index = (a.index + 1) % a.n
+	return c
+}
+
+// Colorize by int [-100, 100] percent
+// Danger --> Optimal --> Danger
+func Colorize(percent int) *color.Color {
+	if percent < -100 {
+		percent = -100
+	}
+	if percent > 100 {
+		percent = 100
+	}
+	if percent < 0 {
+		percent = 100 + percent
+	}
+
+	switch {
+	case percent < 20:
+		return Optimal
+	case percent < 40:
+		return Normal
+	case percent < 60:
+		return Loaded
+	case percent < 85:
+		return Warn
+	case percent < 97:
+		return Danger
+	default:
+		return Dead
+	}
+}
+
+// ColorizeFloat color by float64 [-1.0, 1.0] ratio
+func ColorizeFloat(ratio float64) *color.Color {
+	return Colorize(int(ratio * 100))
+}
+
+func percentIfFree(free bool, percent int) int {
+	if free && percent == 0 {
+		return -1
+	}
+	return percent
+}
+
+// ColorizeInt by int
+func ColorizeInt(used, total int) *color.Color {
+	return Colorize(percentIfFree(used < 0, used*100/total))
+}
+
+// ColorizeInt32 by int32
+func ColorizeInt32(used, total int32) *color.Color {
+	return Colorize(percentIfFree(used < 0, int(used*100/total)))
+}
+
+// ColorizeInt64 by int64
+func ColorizeInt64(used, total int64) *color.Color {
+	return Colorize(percentIfFree(used < 0, int(used*100/total)))
+}
+
+// ColorizeUint32 by uint32
+func ColorizeUint32(used, total uint32) *color.Color {
+	return Colorize(int(used * 100 / total))
+}
+
+// ColorizeUint64 by uint64
+func ColorizeUint64(used, total uint64) *color.Color {
+	return Colorize(int(used * 100 / total))
+}
+
+// ColorizeUint32Free free by uint32
+func ColorizeUint32Free(free, total uint32) *color.Color {
+	return Colorize(percentIfFree(true, -int(free*100/total)))
+}
+
+// ColorizeUint64Free free by uint64
+func ColorizeUint64Free(free, total uint64) *color.Color {
+	return Colorize(percentIfFree(true, -int(free*100/total)))
+}

blobstore/cli/common/color_test.go

@@ -0,0 +1,101 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package common_test
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+	"time"
+
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+)
+
+func TestCmdCommonColorAlternate(t *testing.T) {
+	lines := []string{
+		"first-line",
+		"second-line",
+		"3th-line",
+		"4th-line",
+		"5th-line",
+		"6th-line",
+		"7th-line",
+	}
+	for _, n := range []int{-1, 0, 1, 2, 3, 4, 5, 6, 10} {
+		alterColor := common.NewAlternateColor(n)
+		fmt.Printf("%3d : ", n)
+		for _, line := range lines {
+			alterColor.Next().Print(line + " ")
+		}
+		fmt.Println()
+	}
+}
+
+func TestCmdCommonColorRatio(t *testing.T) {
+	cases := []struct {
+		rand   int
+		offset int
+		s      string
+	}{
+		{1000, -1100, "White [Optimal]"},
+		{20, 0, "white [Optimal]"},
+		{20, 20, "green [Normal]"},
+		{20, 40, "blue [Loaded]"},
+		{25, 60, "yellow [Warn]"},
+		{12, 85, "red [Danger]"},
+		{3, 97, "highlight [Dead]"},
+		{1000, 100, "highlight [Dead]"},
+	}
+
+	rand.Seed(time.Now().Unix())
+	for _, cs := range cases {
+		for i := 0; i < 10; i++ {
+			percent := rand.Intn(cs.rand) + cs.offset
+			c := common.Colorize(percent)
+			c.Printf("used(%d) %s ", percent, cs.s)
+			if cs.rand < 1000 {
+				c = common.Colorize(percent - 100)
+				c.Printf("| free(%d) %s", percent-100, cs.s)
+			}
+			fmt.Println()
+		}
+		fmt.Println()
+	}
+
+	for i := 0; i < 10; i++ {
+		f := rand.Float64()
+		c := common.ColorizeFloat(f)
+		c.Printf("used(%.3f) unknow ", f)
+		c = common.ColorizeFloat(-f)
+		c.Printf("free(%.3f) unknow ", -f)
+		fmt.Println()
+	}
+	fmt.Println()
+
+	common.ColorizeInt(0, 100).Printf("ColorizeInt --> used(%d/%d) white\n", 0, 100)
+	common.ColorizeInt(-1, 1000).Printf("ColorizeInt --> free(%d/%d) dead", -1, 10000)
+	fmt.Println()
+
+	common.ColorizeInt(80, 100).Printf("ColorizeInt --> used(%d/%d) yellow\n", 80, 100)
+	common.ColorizeInt(-80, 100).Printf("ColorizeInt --> free(%d/%d) green\n", 80, 100)
+	common.ColorizeInt32(80, 100).Printf("ColorizeInt32 --> used(%d/%d) yellow\n", 80, 100)
+	common.ColorizeInt32(-80, 100).Printf("ColorizeInt32 --> free(%d/%d) green\n", 80, 100)
+	common.ColorizeInt64(80, 100).Printf("ColorizeInt64 --> used(%d/%d) yellow\n", 80, 100)
+	common.ColorizeInt64(-80, 100).Printf("ColorizeInt64 --> free(%d/%d) green\n", 80, 100)
+	common.ColorizeUint32(80, 100).Printf("ColorizeUint32 --> used(%d/%d) yellow\n", 80, 100)
+	common.ColorizeUint32Free(80, 100).Printf("ColorizeUint32Free --> free(%d/%d) green\n", 80, 100)
+	common.ColorizeUint64(80, 100).Printf("ColorizeUint64 --> used(%d/%d) yellow\n", 80, 100)
+	common.ColorizeUint64Free(80, 100).Printf("ColorizeUint64Free --> free(%d/%d) green\n", 80, 100)
+}

blobstore/cli/common/common.go

@@ -0,0 +1,55 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package common
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+)
+
+// CmdContext returns context with special reqest id tag
+func CmdContext() context.Context {
+	_, ctx := trace.StartSpanFromContext(context.Background(), "from-cmd")
+	return ctx
+}
+
+// Confirm ask for confirmation
+// case insensitive 'y', 'yes', means yes
+// case insensitive 'n', 'no', means no
+// otherwise ask again
+func Confirm(s string) bool {
+	for {
+		fmt.Printf("%s [Y / N]: ", s)
+
+		var response string
+		_, err := fmt.Scanln(&response)
+		if err != nil {
+			fmt.Println("Error:", err)
+			return false
+		}
+
+		response = strings.ToLower(strings.TrimSpace(response))
+		switch response {
+		case "y", "yes":
+			return true
+		case "n", "no":
+			return false
+		default:
+		}
+	}
+}

blobstore/cli/common/common_test.go

@@ -0,0 +1,36 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package common_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+)
+
+func TestCmdCommonContext(t *testing.T) {
+	for ii := 0; ii < 10; ii++ {
+		span := trace.SpanFromContext(common.CmdContext())
+		t.Log(span)
+		span.Info("infoooooooooooooooooo bar")
+	}
+}
+
+func TestCmdCommonConfirm(t *testing.T) {
+	require.False(t, common.Confirm("test confirm?"))
+}

blobstore/cli/common/flags/flags.go

@@ -0,0 +1,57 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package flags
+
+import "github.com/desertbit/grumble"
+
+// VerboseRegister enable verbose mode
+func VerboseRegister(f *grumble.Flags) {
+	f.Bool("v", "verbose", false, "enable verbose mode")
+}
+
+// Verbose enable verbose mode
+func Verbose(f grumble.FlagMap) bool {
+	return f.Bool("verbose")
+}
+
+// VverboseRegister enable verbose verbose mode
+func VverboseRegister(f *grumble.Flags) {
+	f.BoolL("vv", false, "enable verbose verbose mode")
+}
+
+// Vverbose enable verbose verbose mode
+func Vverbose(f grumble.FlagMap) bool {
+	return f.Bool("vv")
+}
+
+// ConfigRegister config path
+func ConfigRegister(f *grumble.Flags) {
+	f.String("c", "config", "", "config path")
+}
+
+// Config config path
+func Config(f grumble.FlagMap) string {
+	return f.String("config")
+}
+
+// FilePathRegister file path, relative path is ok
+func FilePathRegister(f *grumble.Flags) {
+	f.String("f", "filepath", "", "file path")
+}
+
+// FilePath file path, relative path is ok
+func FilePath(f grumble.FlagMap) string {
+	return f.String("filepath")
+}

blobstore/cli/common/json.go

@@ -0,0 +1,56 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package common
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+)
+
+// Marshal alias of json.Marshal
+func Marshal(v interface{}) ([]byte, error) {
+	return json.Marshal(v)
+}
+
+// NewEncoder alias of json.NewEncoder
+func NewEncoder(w io.Writer) *json.Encoder {
+	return json.NewEncoder(w)
+}
+
+// Unmarshal alias of json.Unmarshal
+func Unmarshal(data []byte, val interface{}) error {
+	return json.Unmarshal(data, val)
+}
+
+// NewDecoder alias of json.NewDecoder
+func NewDecoder(r io.Reader) *json.Decoder {
+	return json.NewDecoder(r)
+}
+
+// RawString returns json string
+func RawString(v interface{}) string {
+	data, _ := Marshal(v)
+	return string(data)
+}
+
+// Readable print value of json
+func Readable(v interface{}) string {
+	data, err := json.MarshalIndent(v, "", "    ")
+	if err != nil {
+		return fmt.Sprint("Error: ", err)
+	}
+	return string(data)
+}

blobstore/cli/common/json_test.go

@@ -0,0 +1,54 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package common_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+)
+
+type typeT struct {
+	Int    int            `json:"int"`
+	Bool   bool           `json:"bool"`
+	String string         `json:"string"`
+	Slice  []float32      `json:"slice"`
+	Map    map[int]string `json:"map"`
+}
+
+func TestCmdCommonJSONBase(t *testing.T) {
+	value := typeT{
+		Int:    10,
+		Bool:   true,
+		String: "test json",
+		Slice:  []float32{0.02, 1.111},
+		Map:    map[int]string{-1: "-1", 100: "100"},
+	}
+
+	data, err := common.Marshal(value)
+	require.NoError(t, err)
+
+	var val typeT
+	err = common.Unmarshal(data, &val)
+	require.NoError(t, err)
+
+	require.Equal(t, value, val)
+
+	fmt.Println("Raw      JSON:", common.RawString(value))
+	fmt.Println("Readable JSON:", common.Readable(value))
+}

blobstore/cli/common/progress.go

@@ -0,0 +1,229 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package common
+
+import (
+	"fmt"
+	"io"
+	"strings"
+	"sync"
+
+	"github.com/fatih/color"
+)
+
+var boldRunes = []rune(" ▏▎▍▌▋▋▊▉█")
+
+// BoldBar returns bold progress bar like:
+// [███▍      ]
+//
+// curr is [0 - 100]
+func BoldBar(curr int) string {
+	if curr == 100 {
+		return "██████████"
+	}
+	return fmt.Sprintf("%s%*c", strings.Repeat("█", curr/10), curr/10-10, boldRunes[curr%10])
+}
+
+// LineBar returns line progress bar like:
+// [===============================================>  ]
+//
+// curr is [0 - 100]
+// width is the length of bar
+func LineBar(curr, width int) string {
+	n := 100 / width
+	if n <= 0 {
+		n = 1
+	}
+
+	if curr == 100 {
+		return strings.Repeat("=", 100/n)
+	}
+	return fmt.Sprintf("%s>%s", strings.Repeat("=", curr/n), strings.Repeat(" ", (100-curr-1)/n))
+}
+
+// Loader loader bar
+func Loader(all int) chan<- int {
+	ch := make(chan int)
+
+	go func() {
+		loaded := 0
+		for {
+			n, ok := <-ch
+			if !ok {
+				fmt.Println()
+				return
+			}
+			loaded += n
+			curr := loaded * 100 / all
+			fmt.Printf("\rLoading: [%s] %s/%s", BoldBar(curr),
+				color.GreenString("%-5d", loaded),
+				color.RedString("%5d", all))
+			if loaded >= all {
+				fmt.Println()
+				return
+			}
+		}
+	}()
+
+	return ch
+}
+
+// PReader progress bar for io.Reader
+// example:
+// reader := NewPReader(1024, r)
+// defer reader.Close()
+// reader.LineBar()
+//
+// // balabala ...
+type PReader struct {
+	total int
+	r     io.Reader
+
+	readCh  chan int
+	closeCh chan struct{}
+	once    sync.Once
+}
+
+// NewPReader returns a progress reader
+func NewPReader(total int, r io.Reader) *PReader {
+	return &PReader{
+		total: total,
+		r:     io.LimitReader(r, int64(total)),
+
+		readCh:  make(chan int, 1),
+		closeCh: make(chan struct{}),
+	}
+}
+
+func (r *PReader) Read(p []byte) (n int, err error) {
+	n, err = r.r.Read(p)
+	if n > 0 {
+		r.readCh <- n
+	}
+	return
+}
+
+func (r *PReader) bar(width int, printFunc func(curr int)) {
+	go func() {
+		read := 0
+		for {
+			select {
+			case n := <-r.readCh:
+				read += n
+				curr := read * 100 / r.total
+				printFunc(curr)
+				if read >= r.total {
+					fmt.Println()
+					return
+				}
+			case <-r.closeCh:
+				fmt.Println()
+				return
+			}
+		}
+	}()
+}
+
+// LineBar line bar
+func (r *PReader) LineBar(width int) {
+	r.bar(width, func(curr int) {
+		fmt.Printf("\rReading: [%s] %s%%", LineBar(curr, width), color.GreenString("%3d", curr))
+	})
+}
+
+// BoldBar bold bar
+func (r *PReader) BoldBar() {
+	r.bar(0, func(curr int) {
+		fmt.Printf("\rReading: [%s] %s%%", BoldBar(curr), color.GreenString("%3d", curr))
+	})
+}
+
+// Close the bar
+func (r *PReader) Close() {
+	r.once.Do(func() {
+		close(r.closeCh)
+	})
+}
+
+// PWriter progress bar for io.Writer
+type PWriter struct {
+	total int
+	w     io.Writer
+
+	writeCh chan int
+	closeCh chan struct{}
+	once    sync.Once
+}
+
+// NewPWriter returns a progress writer
+func NewPWriter(total int, w io.Writer) *PWriter {
+	return &PWriter{
+		total: total,
+		w:     w,
+
+		writeCh: make(chan int, 1),
+		closeCh: make(chan struct{}),
+	}
+}
+
+func (w *PWriter) Write(p []byte) (n int, err error) {
+	n, err = w.w.Write(p)
+	if n > 0 {
+		w.writeCh <- n
+	}
+	return
+}
+
+func (w *PWriter) bar(width int, printFunc func(curr int)) {
+	go func() {
+		written := 0
+		for {
+			select {
+			case n := <-w.writeCh:
+				written += n
+				curr := written * 100 / w.total
+				printFunc(curr)
+				if written >= w.total {
+					fmt.Println()
+					return
+				}
+			case <-w.closeCh:
+				fmt.Println()
+				return
+			}
+		}
+	}()
+}
+
+// LineBar line bar
+func (w *PWriter) LineBar(width int) {
+	w.bar(width, func(curr int) {
+		fmt.Printf("\rWriting: [%s] %s%%", LineBar(curr, width), color.GreenString("%3d", curr))
+	})
+}
+
+// BoldBar bold bar
+func (w *PWriter) BoldBar() {
+	w.bar(0, func(curr int) {
+		fmt.Printf("\rWriting: [%s] %s%%", BoldBar(curr), color.GreenString("%3d", curr))
+	})
+}
+
+// Close the bar
+func (w *PWriter) Close() {
+	w.once.Do(func() {
+		close(w.closeCh)
+	})
+}

blobstore/cli/common/progress_test.go

@@ -0,0 +1,168 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package common_test
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+	"time"
+
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+)
+
+func TestCmdCommonProgressBoldBar(t *testing.T) {
+	for curr := 0; curr <= 100; curr++ {
+		fmt.Printf("\r[%s] %d%%", common.BoldBar(curr), curr)
+		time.Sleep(20 * time.Millisecond)
+	}
+	fmt.Println()
+}
+
+func TestCmdCommonProgressLineBar(t *testing.T) {
+	for curr := 0; curr <= 100; curr++ {
+		fmt.Printf("\r[%s] %d%%", common.LineBar(curr, 10), curr)
+		time.Sleep(10 * time.Millisecond)
+	}
+	fmt.Println()
+
+	for curr := 0; curr <= 100; curr++ {
+		fmt.Printf("\r[%s] %d%%", common.LineBar(curr, 50), curr)
+		time.Sleep(10 * time.Millisecond)
+	}
+	fmt.Println()
+
+	for curr := 0; curr <= 100; curr++ {
+		fmt.Printf("\r[%s] %d%%", common.LineBar(curr, 100), curr)
+		time.Sleep(10 * time.Millisecond)
+	}
+	fmt.Println()
+}
+
+func TestCmdCommonProgressLoader(t *testing.T) {
+	alls := []int{
+		1,
+		7,
+		17,
+		20,
+		100,
+		19999,
+	}
+
+	for _, all := range alls {
+		ch := common.Loader(all)
+		n := (all + 9) / 10
+		loaded := 0
+		for ii := 0; ii < 10; ii++ {
+			if loaded >= all {
+				break
+			}
+			if loaded+n > all {
+				ch <- (all - loaded)
+			} else {
+				ch <- n
+			}
+			loaded += n
+			time.Sleep(100 * time.Millisecond)
+		}
+		close(ch)
+	}
+}
+
+type mockReader struct{}
+
+func (r *mockReader) Read(p []byte) (n int, err error) {
+	time.Sleep(10 * time.Millisecond)
+	return len(p), nil
+}
+
+func TestCmdCommonProgressReader(t *testing.T) {
+	sizes := []int{
+		1,
+		1 << 10,
+		10250,
+		1 << 20,
+		1 << 30,
+	}
+
+	rand.Seed(time.Now().Unix())
+	for _, size := range sizes {
+		reader := common.NewPReader(size, &mockReader{})
+		reader.LineBar((int(rand.Int31n(9)+1) * 10))
+		l := (size / 100) + 1
+		// 102 times
+		for ii := 0; ii <= 101; ii++ {
+			_, _ = reader.Read(make([]byte, l))
+		}
+		reader.Close()
+	}
+
+	for _, size := range sizes {
+		reader := common.NewPReader(size, &mockReader{})
+		reader.BoldBar()
+		l := (size / 100) + 1
+		for ii := 0; ii <= 101; ii++ {
+			_, _ = reader.Read(make([]byte, l))
+		}
+		reader.Close()
+	}
+}
+
+type mockWriter struct {
+	total   int
+	written int
+}
+
+func (w *mockWriter) Write(p []byte) (n int, err error) {
+	if w.written >= w.total {
+		return 0, nil
+	}
+	w.written += len(p)
+	time.Sleep(10 * time.Millisecond)
+	return len(p), nil
+}
+
+func TestCmdCommonProgressWriter(t *testing.T) {
+	sizes := []int{
+		1,
+		1 << 10,
+		10250,
+		1 << 20,
+		1 << 30,
+	}
+
+	rand.Seed(time.Now().Unix())
+	for _, size := range sizes {
+		writer := common.NewPWriter(size, &mockWriter{total: size})
+		writer.LineBar((int(rand.Int31n(9)+1) * 10))
+		l := (size / 100) + 1
+		// 102 times
+		for ii := 0; ii <= 101; ii++ {
+			_, _ = writer.Write(make([]byte, l))
+		}
+
+		writer.Close()
+	}
+
+	for _, size := range sizes {
+		writer := common.NewPWriter(size, &mockWriter{total: size})
+		writer.BoldBar()
+		l := (size / 100) + 1
+		for ii := 0; ii <= 101; ii++ {
+			_, _ = writer.Write(make([]byte, l))
+		}
+		writer.Close()
+	}
+}

blobstore/cli/config.go

@@ -0,0 +1,99 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package cli
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/desertbit/grumble"
+	"github.com/fatih/color"
+
+	"github.com/cubefs/cubefs/blobstore/cli/config"
+)
+
+func cmdConfigGet(c *grumble.Context) error {
+	fmt.Println("configs:")
+
+	all := config.All()
+	keys := c.Args.StringList("keys")
+	if len(keys) == 0 {
+		keys = make([]string, 0, len(all))
+		for key := range all {
+			keys = append(keys, key)
+		}
+	}
+	sort.Strings(keys)
+
+	for _, key := range keys {
+		fmt.Printf("\t%s --> %s\n", color.RedString("%-30s", key), color.GreenString("%+v", all[key]))
+	}
+	return nil
+}
+
+func registerConfig(app *grumble.App) {
+	configCommand := &grumble.Command{
+		Name: "config",
+		Help: "manager memory cache of config",
+	}
+	app.AddCommand(configCommand)
+
+	configCommand.AddCommand(&grumble.Command{
+		Name: "type",
+		Help: "print type in cache",
+		Run: func(c *grumble.Context) error {
+			config.PrintType()
+			return nil
+		},
+	})
+	configCommand.AddCommand(&grumble.Command{
+		Name:     "get",
+		Help:     "get config in cache",
+		LongHelp: "show all caches if no args",
+		Args: func(a *grumble.Args) {
+			a.StringList("keys", "config keys", grumble.Default([]string{}), grumble.Max(100))
+		},
+		Run: cmdConfigGet,
+	})
+	configCommand.AddCommand(&grumble.Command{
+		Name:     "set",
+		Help:     "set config to cache",
+		LongHelp: "set config to cache, value concat with ',' if []string",
+		Args: func(a *grumble.Args) {
+			a.String("key", "config key")
+			a.String("value", "config value, concat with ',' if []string")
+		},
+		Run: func(c *grumble.Context) error {
+			key, val := c.Args.String("key"), c.Args.String("value")
+			fmt.Println("To set Key:", color.RedString("%s", key), "Value:", color.GreenString("%s", val))
+			config.SetFrom(key, val)
+			return nil
+		},
+	})
+	configCommand.AddCommand(&grumble.Command{
+		Name: "del",
+		Help: "del config of keys",
+		Args: func(a *grumble.Args) {
+			a.StringList("keys", "config keys", grumble.Default([]string{}), grumble.Max(100))
+		},
+		Run: func(c *grumble.Context) error {
+			for _, key := range c.Args.StringList("keys") {
+				fmt.Println("To del Key:", color.RedString("%s", key))
+				config.Del(key)
+			}
+			return nil
+		},
+	})
+}

blobstore/cli/config/cache.go

@@ -0,0 +1,82 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package config
+
+var cacher = make(map[string]interface{})
+
+// All returns all cache
+func All() map[string]interface{} {
+	return cacher
+}
+
+// Set cache setter
+func Set(key string, value interface{}) {
+	cacher[key] = value
+}
+
+// SetFrom value from string
+func SetFrom(key, value string) {
+	if valuer, ok := keyValuer[key]; ok {
+		Set(key, valuer.Valuer(value))
+	} else {
+		Set(key, value)
+	}
+}
+
+// Del del cache of key
+func Del(key string) {
+	delete(cacher, key)
+}
+
+// Get cache getter
+func Get(key string) interface{} {
+	return cacher[key]
+}
+
+// Verbose returns app verbose
+func Verbose() bool {
+	if val := Get("Flag-Verbose"); val != nil {
+		if v, ok := val.(bool); ok {
+			return v
+		}
+	}
+	return false
+}
+
+// Vverbose returns app verbose verbose
+func Vverbose() bool {
+	if val := Get("Flag-Vverbose"); val != nil {
+		if v, ok := val.(bool); ok {
+			return v
+		}
+	}
+	return false
+}
+
+func RedisAddrs() []string { return Get("Key-RedisAddrs").([]string) }
+func RedisUser() string    { return Get("Key-RedisUser").(string) }
+func RedisPass() string    { return Get("Key-RedisPass").(string) }
+
+// ClusterMgrAddrs returns cluster manager addrs
+func ClusterMgrAddrs() []string { return Get("Key-ClusterMgrAddrs").([]string) }
+func ClusterMgrSecret() string  { return Get("Key-ClusterMgrSecret").(string) }
+
+func AccessConnMode() uint8          { return Get("Key-Access-ConnMode").(uint8) }
+func AccessConsulAddr() string       { return Get("Key-Access-ConsulAddr").(string) }
+func AccessServiceIntervalMs() int64 { return Get("Key-Access-ServiceIntervalMs").(int64) }
+func AccessPriorityAddrs() []string  { return Get("Key-Access-PriorityAddrs").([]string) }
+func AccessMaxSizePutOnce() int64    { return Get("Key-Access-MaxSizePutOnce").(int64) }
+func AccessMaxPartRetry() int        { return Get("Key-Access-MaxPartRetry").(int) }
+func AccessMaxHostRetry() int        { return Get("Key-Access-MaxHostRetry").(int) }

blobstore/cli/config/config.go

@@ -0,0 +1,171 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package config
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+)
+
+// Config config in file
+type Config struct {
+	Verbose  bool `json:"verbose" cache:"Flag-Verbose" help:"enable verbose mode"`
+	Vverbose bool `json:"vverbose" cache:"Flag-Vverbose" help:"enable verbose verbose mode"`
+
+	RedisAddrs []string `json:"redis_addrs" cache:"Key-RedisAddrs" help:"redis addrs"`
+	RedisUser  string   `json:"redis_user" cache:"Key-RedisUser" help:"redis username"`
+	RedisPass  string   `json:"redis_pass" cache:"Key-RedisPass" help:"redis password"`
+
+	ClusterMgrAddrs  []string `json:"cm_addrs" cache:"Key-ClusterMgrAddrs" help:"cluster manager addrs"`
+	ClusterMgrSecret string   `json:"cm_secret" cache:"Key-ClusterMgrSecret" help:"cluster manager secret"`
+
+	Access struct { // see more in api/access/client.go
+		ConnMode          uint8    `json:"conn_mode" cache:"Key-Access-ConnMode" help:"connection mode, 4 means no timeout"`
+		ConsulAddr        string   `json:"consul_addr" cache:"Key-Access-ConsulAddr" help:"consul address"`
+		ServiceIntervalMs int64    `json:"service_interval_ms" cache:"Key-Access-ServiceIntervalMs" help:"service interval ms"`
+		PriorityAddrs     []string `json:"priority_addrs" cache:"Key-Access-PriorityAddrs" help:"priority addresses to try"`
+		MaxSizePutOnce    int64    `json:"max_size_put_once" cache:"Key-Access-MaxSizePutOnce" help:"max size put once"`
+		MaxPartRetry      int      `json:"max_part_retry" cache:"Key-Access-MaxPartRetry" help:"max times to retry part"`
+		MaxHostRetry      int      `json:"max_host_retry" cache:"Key-Access-MaxHostRetry" help:"max times to retry host"`
+	} `json:"access"`
+}
+
+func load(conf *Config) {
+	cacheSetter := func(elemT reflect.Type, elemV reflect.Value) {
+		for idx := 0; idx < elemT.NumField(); idx++ {
+			field := elemT.Field(idx)
+			value := elemV.Field(idx)
+			if key := field.Tag.Get("cache"); key != "" {
+				keyValuer[key] = typeValuer{Type: field.Type.Kind().String(), Valuer: valuer(field.Type.Kind())}
+				Set(key, value.Interface())
+			}
+		}
+	}
+
+	cacheSetter(reflect.TypeOf(conf).Elem(), reflect.ValueOf(conf).Elem())
+
+	confAccess := &conf.Access
+	cacheSetter(reflect.TypeOf(confAccess).Elem(), reflect.ValueOf(confAccess).Elem())
+}
+
+// LoadConfig load config from path
+func LoadConfig(path string) {
+	var conf *Config
+	f, err := os.Open(path)
+	if err != nil {
+		panic(err)
+	}
+	if err := common.NewDecoder(f).Decode(&conf); err != nil {
+		panic(err)
+	}
+	load(conf)
+}
+
+type typeValuer struct {
+	Type   string
+	Valuer func(string) interface{}
+}
+
+var (
+	keyValuer = make(map[string]typeValuer)
+
+	valuer = func(t reflect.Kind) func(string) interface{} {
+		switch t {
+		case reflect.Bool:
+			return func(val string) interface{} { v, _ := strconv.ParseBool(val); return v }
+
+		case reflect.Int:
+			return func(val string) interface{} { v, _ := strconv.Atoi(val); return int(v) }
+		case reflect.Int8:
+			return func(val string) interface{} { v, _ := strconv.Atoi(val); return int8(v) }
+		case reflect.Int16:
+			return func(val string) interface{} { v, _ := strconv.Atoi(val); return int16(v) }
+		case reflect.Int32:
+			return func(val string) interface{} { v, _ := strconv.Atoi(val); return int32(v) }
+		case reflect.Int64:
+			return func(val string) interface{} { v, _ := strconv.Atoi(val); return int64(v) }
+
+		case reflect.Uint:
+			return func(val string) interface{} { v, _ := strconv.Atoi(val); return uint(v) }
+		case reflect.Uint8:
+			return func(val string) interface{} { v, _ := strconv.Atoi(val); return uint8(v) }
+		case reflect.Uint16:
+			return func(val string) interface{} { v, _ := strconv.Atoi(val); return uint16(v) }
+		case reflect.Uint32:
+			return func(val string) interface{} { v, _ := strconv.Atoi(val); return uint32(v) }
+		case reflect.Uint64:
+			return func(val string) interface{} { v, _ := strconv.Atoi(val); return uint64(v) }
+
+		case reflect.Float32:
+			return func(val string) interface{} { v, _ := strconv.ParseFloat(val, 32); return float32(v) }
+		case reflect.Float64:
+			return func(val string) interface{} { v, _ := strconv.ParseFloat(val, 64); return float64(v) }
+
+		case reflect.String:
+			return func(val string) interface{} { return val }
+		case reflect.Slice: // []string
+			return func(val string) interface{} {
+				if val == "" {
+					return []string{}
+				}
+				return strings.Split(val, ",")
+			}
+
+		case reflect.Map: // map[string]string
+			return func(val string) interface{} {
+				m := make(map[string]string)
+				if val == "" {
+					return m
+				}
+				kvs := strings.Split(val, ",")
+				for _, kvStr := range kvs {
+					kv := strings.SplitN(kvStr, ":", 2)
+					k, v := strings.TrimSpace(kv[0]), ""
+					if len(kv) > 1 {
+						v = strings.TrimSpace(kv[1])
+					}
+					m[k] = v
+				}
+				return m
+			}
+		default:
+			panic(fmt.Sprintf("unknown type %v", t))
+		}
+	}
+)
+
+// PrintType print type of keyValuer
+func PrintType() {
+	keys := make([]string, 0, len(keyValuer))
+	for key := range keyValuer {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	fmt.Println("Init Cache Keys:")
+	for _, key := range keys {
+		fmt.Printf("\t| %-30s | %-10s |\n", key, keyValuer[key].Type)
+	}
+}
+
+func init() {
+	conf := &Config{}
+	load(conf)
+}

blobstore/cli/history.go

@@ -0,0 +1,75 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package cli
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/desertbit/grumble"
+	"github.com/fatih/color"
+)
+
+func cmdHistory(path string) func(c *grumble.Context) error {
+	hisColor := color.New(color.FgHiMagenta)
+	return func(c *grumble.Context) error {
+		file, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		l := c.Args.Int("line")
+		lines := make([]string, 0, l)
+
+		reader := bufio.NewReader(file)
+		n := 1
+		for {
+			line, _, err := reader.ReadLine()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				return err
+			}
+
+			lines = append(lines, fmt.Sprintf("%5.d: %s", n, line))
+			n++
+		}
+		if len(lines) > l {
+			lines = lines[len(lines)-l:]
+		}
+
+		for _, line := range lines {
+			hisColor.Println(line)
+		}
+		return nil
+	}
+}
+
+func registerHistory(app *grumble.App) {
+	historyCommand := &grumble.Command{
+		Name:     "history",
+		Help:     "show history commands",
+		LongHelp: "show history commands in file " + app.Config().HistoryFile,
+		Args: func(a *grumble.Args) {
+			a.Int("line", "show lines at the tail", grumble.Default(40))
+		},
+		Run: cmdHistory(app.Config().HistoryFile),
+	}
+	app.AddCommand(historyCommand)
+}

blobstore/cli/util.go

@@ -0,0 +1,269 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package cli
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/desertbit/grumble"
+	"github.com/dustin/go-humanize"
+	"github.com/fatih/color"
+
+	"github.com/cubefs/cubefs/blobstore/access/controller"
+	"github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+	"github.com/cubefs/cubefs/blobstore/cli/common/args"
+	"github.com/cubefs/cubefs/blobstore/cli/common/cfmt"
+	"github.com/cubefs/cubefs/blobstore/cli/config"
+	"github.com/cubefs/cubefs/blobstore/common/redis"
+	"github.com/cubefs/cubefs/blobstore/common/uptoken"
+)
+
+func newRedisCli(addrs ...string) *redis.ClusterClient {
+	if len(addrs) == 0 || (len(addrs) == 1 && addrs[0] == "") {
+		addrs = config.RedisAddrs()
+	}
+	return redis.NewClusterClient(&redis.ClusterConfig{
+		Addrs:    addrs,
+		Username: config.RedisUser(),
+		Password: config.RedisPass(),
+	})
+}
+
+func cmdTime(c *grumble.Context) error {
+	unix := c.Args.String("unix")
+	format := c.Args.String("format")
+
+	if unix == "" {
+		t := time.Now()
+		fmt.Printf("timestamp = %s (seconds = %d nanosecs = %d) \n\t--> format: %s (%s)\n\n",
+			color.RedString("%d", t.UnixNano()), t.Unix(), t.Nanosecond(),
+			color.GreenString("%s", t.Format(time.RFC3339Nano)), humanize.Time(t))
+		return nil
+	}
+
+	if format == "" {
+		format = time.RFC3339Nano
+	}
+
+	unix += strings.Repeat("0", 19)
+	sec, _ := strconv.ParseInt(unix[:10], 10, 64)
+	nsec, _ := strconv.ParseInt(unix[10:19], 10, 64)
+
+	t := time.Unix(sec, nsec)
+	fmt.Printf("timestamp = %s (seconds = %d nanosecs = %d) \n\t--> format: %s (%s)\n\n",
+		color.RedString("%s", c.Args.String("unix")), sec, nsec,
+		color.GreenString("%s", t.Format(format)), humanize.Time(t))
+	return nil
+}
+
+func cmdToken(c *grumble.Context) error {
+	tokenStr := c.Args.String("token")
+	token := uptoken.DecodeToken(tokenStr)
+	err := fmt.Errorf("invalid token: %s", tokenStr)
+
+	data := token.Data[:]
+	offset := 8
+	next := func() (uint64, bool) {
+		val, n := binary.Uvarint(data[offset:])
+		if n <= 0 {
+			return 0, false
+		}
+		offset += n
+		return val, true
+	}
+
+	var (
+		minBid, count, expiredTime uint64
+		ok                         bool
+	)
+	if minBid, ok = next(); !ok {
+		return err
+	}
+	if count, ok = next(); !ok {
+		return err
+	}
+	if expiredTime, ok = next(); !ok {
+		return err
+	}
+
+	fmt.Println("Checksum: ", strings.ToUpper(hex.EncodeToString(data[0:8])))
+	fmt.Println("MinBid  : ", minBid)
+	fmt.Println("Count   : ", count)
+
+	t := time.Unix(int64(expiredTime), 9)
+	if time.Since(t) < 0 {
+		fmt.Printf("Time    : %d (%s) (%s)", expiredTime,
+			common.Normal.Sprint(t.Format(time.RFC3339)), humanize.Time(t))
+	} else {
+		fmt.Printf("Time    : %d (%s) (%s)", expiredTime,
+			common.Danger.Sprint(t.Format(time.RFC3339)), humanize.Time(t))
+	}
+	fmt.Println()
+
+	return nil
+}
+
+func registerUtil(app *grumble.App) {
+	utilCommand := &grumble.Command{
+		Name:     "util",
+		Help:     "util commands",
+		LongHelp: "util commands, parse everything",
+	}
+	app.AddCommand(utilCommand)
+
+	utilCommand.AddCommand(&grumble.Command{
+		Name:     "time",
+		Help:     "time format [unix] [format]",
+		LongHelp: "time format, show now if no argument",
+		Run:      cmdTime,
+		Args: func(a *grumble.Args) {
+			a.String("unix", "unix timestamp", grumble.Default(""))
+			a.String("format", "format for timestamp", grumble.Default(""))
+		},
+	})
+	utilCommand.AddCommand(&grumble.Command{
+		Name: "vuid",
+		Help: "parse vuid <vuid>",
+		Args: func(a *grumble.Args) {
+			args.VuidRegister(a)
+		},
+		Run: func(c *grumble.Context) error {
+			fmt.Println("Parse VUID: ", cfmt.VuidCF(args.Vuid(c.Args)))
+			return nil
+		},
+	})
+	utilCommand.AddCommand(&grumble.Command{
+		Name: "token",
+		Help: "parse token <token>",
+		Run:  cmdToken,
+		Args: func(a *grumble.Args) {
+			a.String("token", "token of putat")
+		},
+	})
+
+	utilCommand.AddCommand(&grumble.Command{
+		Name:     "location",
+		Help:     "parse location <[json|hex|base64]>",
+		LongHelp: "parse location, or decode from string",
+		Args: func(a *grumble.Args) {
+			a.String("jsonORstr", "location json or location string")
+		},
+		Run: func(c *grumble.Context) error {
+			jsonORstr := c.Args.String("jsonORstr")
+			loc, err := cfmt.ParseLocation(jsonORstr)
+			if err == nil {
+				fmt.Println(cfmt.LocationJoin(&loc, ""))
+				return nil
+			}
+
+			src, err := hex.DecodeString(jsonORstr)
+			if err != nil {
+				return fmt.Errorf("invalid (%s) %s", jsonORstr, err.Error())
+			}
+			loc, n, err := access.DecodeLocation(src)
+			if err != nil {
+				fmt.Printf("has read bytes %d / %d\n", n, len(src))
+				fmt.Println(cfmt.LocationJoin(&loc, ""))
+				return fmt.Errorf("invalid (%s) %s", jsonORstr, err.Error())
+			}
+
+			fmt.Println(cfmt.LocationJoin(&loc, ""))
+			return nil
+		},
+	})
+
+	redisCommand := &grumble.Command{
+		Name: "redis",
+		Help: "redis tools",
+		Run: func(c *grumble.Context) error {
+			fmt.Println("redis-addrs:", config.RedisAddrs())
+			fmt.Println("redis-user :", config.RedisUser())
+			fmt.Println("redis-pass :", config.RedisPass())
+			fmt.Println()
+			fmt.Println("access volume prefix with access/volume/{cid}/{vid}")
+			return nil
+		},
+	}
+	utilCommand.AddCommand(redisCommand)
+	redisCommand.AddCommand(&grumble.Command{
+		Name:     "get",
+		Help:     "redis get <key>",
+		LongHelp: "redis get, access prefix access/volume/",
+		Args: func(a *grumble.Args) {
+			a.String("key", "redis key")
+		},
+		Flags: func(f *grumble.Flags) {
+			f.StringL("addr", "", "redis addr")
+		},
+		Run: func(c *grumble.Context) error {
+			key := c.Args.String("key")
+			cli := newRedisCli(c.Flags.String("addr"))
+
+			fmt.Println("Get Key:", key)
+			if strings.HasPrefix(key, "access/volume/") {
+				var val controller.VolumePhy
+				if err := cli.Get(common.CmdContext(), key, &val); err != nil {
+					return err
+				}
+				fmt.Println(common.Readable(val))
+				return nil
+			}
+
+			b, err := cli.ClusterClient.Get(common.CmdContext(), key).Bytes()
+			if err != nil {
+				return err
+			}
+			fmt.Println(string(b))
+			return nil
+		},
+	})
+	redisCommand.AddCommand(&grumble.Command{
+		Name:     "set",
+		Help:     "redis set <key> [value] [expiration]",
+		LongHelp: "redis set key, delete key if value is empty, expiration is duration string",
+		Args: func(a *grumble.Args) {
+			a.String("key", "redis key")
+			a.String("value", "redis value", grumble.Default(""))
+			a.String("expiration", "redis key expiration, duration string",
+				grumble.Default("0"))
+		},
+		Flags: func(f *grumble.Flags) {
+			f.StringL("addr", "", "redis addr")
+		},
+		Run: func(c *grumble.Context) error {
+			key := c.Args.String("key")
+			value := c.Args.String("value")
+			cli := newRedisCli(c.Flags.String("addr"))
+
+			if value == "" {
+				fmt.Println("Del Key:", common.Loaded.Sprint(key))
+				return cli.ClusterClient.Del(common.CmdContext(), key).Err()
+			}
+			expiration, err := time.ParseDuration(c.Args.String("expiration"))
+			if err != nil {
+				return err
+			}
+
+			fmt.Println("Set Key:", common.Loaded.Sprint(key), "expiration:", expiration)
+			return cli.Set(common.CmdContext(), key, value, expiration)
+		},
+	})
+}

blobstore/cmd/access/access.conf

@@ -0,0 +1,21 @@
+{
+    "bind_addr": ":9500",
+    "auditlog": {
+        "logdir": "/tmp/access/"
+    },
+    "log": {
+        "level": "info",
+        "filename": "/tmp/logs/access.log"
+     },
+    "consul_agent_addr": "127.0.0.1:8500",
+    "service_register": {
+        "consul_addr": "127.0.0.1:8500",
+        "service_ip": "127.0.0.1"
+    },
+    "stream": {
+        "idc": "z0",
+        "cluster_config": {
+            "region": "test-region"
+        }
+    }
+}

blobstore/cmd/access/access.conf.default

@@ -0,0 +1,112 @@
+{
+    "max_procs": 0,
+    "shutdown_timeout_s": 30,
+    "log": {
+        "level": "debug",
+        "filename": "/tmp/access.log",
+        "maxsize": 1024,
+        "maxage": 7,
+        "maxbackups": 7
+    },
+    "auditlog": {
+        "chunkbits": 29,
+        "logdir": "/tmp/access/",
+        "backup": 0,
+        "rotate_new": false
+    },
+    "bind_addr": ":9500",
+    "consul_agent_addr": "127.0.0.1:8500",
+    "service_register": {
+        "consul_addr": "127.0.0.1:8500",
+        "node": "nodename",
+        "service_ip": "service_ip"
+    },
+    "limit": {
+        "name_rps": {
+            "alloc": 0,
+            "put": 0,
+            "putat": 0,
+            "get": 0,
+            "delete": 0,
+            "sign": 0
+        },
+        "reader_mbps": 0,
+        "writer_mbps": 0
+    },
+    "stream": {
+        "idc": "idc",
+        "max_blob_size": 4194304,
+        "mem_pool_size_classes": {
+            "2048": 81920,
+            "24576": 40960,
+            "65536": 40960,
+            "524288": 20480,
+            "2097152": 10240,
+            "8389632": 4096,
+            "16777216": 1024,
+            "33554432": 512,
+            "67108864": 64
+        },
+        "code_mode_put_quorums": {
+            "1": 24,
+            "2": 11,
+            "3": 34,
+            "4": 14
+        },
+        "alloc_retry_interval_ms": 100,
+        "alloc_retry_times": 3,
+        "encoder_concurrency": 1000,
+        "encoder_enableverify": true,
+        "min_read_shards_x": 1,
+        "shard_crc_disabled": false,
+        "cluster_config": {
+            "region": "region",
+            "region_magic": "",
+            "cluster_reload_secs": 3,
+            "clustermgr_client_config": {
+                "client_timeout_ms": 3000,
+                "hosts": [],
+                "transport_config": {
+                    "auth": {
+                        "enable_auth": true,
+                        "secret": "secret key"
+                    },
+                    "dial_timeout_ms": 2000
+                }
+            },
+            "redis_client_config": {
+                "addrs": [
+                    "127.0.0.1:xxx"
+                ],
+                "password": "pass",
+                "username": "user"
+            },
+            "service_punish_threshold": 3,
+            "service_punish_valid_interval_s": 30,
+            "service_reload_secs": 3
+        },
+        "disk_punish_interval_s": 60,
+        "disk_timeout_punish_interval_s": 6,
+        "service_punish_interval_s": 60,
+        "blobnode_config": {
+            "client_timeout_ms": 10000
+        },
+        "proxy_config": {
+            "client_timeout_ms": 5000
+        },
+        "alloc_command_config": {
+            "error_percent_threshold": 50,
+            "max_concurrent_requests": 10240,
+            "request_volume_threshold": 100,
+            "sleep_window": 2000,
+            "timeout": 30000
+        },
+        "rw_command_config": {
+            "error_percent_threshold": 80,
+            "max_concurrent_requests": 102400,
+            "request_volume_threshold": 1000,
+            "sleep_window": 5000,
+            "timeout": 600000
+        }
+    }
+}

blobstore/cmd/access/main.go

@@ -0,0 +1,27 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package main
+
+import (
+	"os"
+
+	"github.com/cubefs/cubefs/blobstore/cmd"
+
+	_ "github.com/cubefs/cubefs/blobstore/access"
+)
+
+func main() {
+	cmd.Main(os.Args)
+}

blobstore/common/codemode/codemode.go

@@ -0,0 +1,337 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package codemode
+
+import "fmt"
+
+// CodeMode EC encode and decode mode
+type CodeMode uint8
+type CodeModeName string
+
+// pre-defined mode
+const (
+	EC15P12       CodeMode = 1
+	EC6P6         CodeMode = 2
+	EC16P20L2     CodeMode = 3
+	EC6P10L2      CodeMode = 4
+	EC6P3L3       CodeMode = 5
+	EC6P6Align0   CodeMode = 6
+	EC6P6Align512 CodeMode = 7
+	EC4P4L2       CodeMode = 8
+	EC12P4        CodeMode = 9
+	EC16P4        CodeMode = 10
+	EC3P3         CodeMode = 11
+	EC10P4        CodeMode = 12
+	EC6P3         CodeMode = 13
+)
+
+// Note: Don't modify it unless you know very well how codemode works.
+const (
+	// align size per shard
+	alignSize0B   = 0    // 0B
+	alignSize512B = 512  // 512B
+	alignSize2KB  = 2048 // 2KB
+)
+
+// The tactic is fixed pairing with one codemode.
+// Add a new codemode if you want other features.
+var constCodeModeTactic = map[CodeMode]Tactic{
+	EC15P12:   {N: 15, M: 12, L: 0, AZCount: 3, PutQuorum: 24, GetQuorum: 0, MinShardSize: alignSize2KB},
+	EC6P6:     {N: 6, M: 6, L: 0, AZCount: 3, PutQuorum: 11, GetQuorum: 0, MinShardSize: alignSize2KB},
+	EC16P20L2: {N: 16, M: 20, L: 2, AZCount: 2, PutQuorum: 34, GetQuorum: 0, MinShardSize: alignSize2KB},
+	EC6P10L2:  {N: 6, M: 10, L: 2, AZCount: 2, PutQuorum: 14, GetQuorum: 0, MinShardSize: alignSize2KB},
+
+	// single az
+	EC12P4: {N: 12, M: 4, L: 0, AZCount: 1, PutQuorum: 15, GetQuorum: 0, MinShardSize: alignSize2KB},
+	EC16P4: {N: 16, M: 4, L: 0, AZCount: 1, PutQuorum: 19, GetQuorum: 0, MinShardSize: alignSize2KB},
+	EC3P3:  {N: 3, M: 3, L: 0, AZCount: 1, PutQuorum: 5, GetQuorum: 0, MinShardSize: alignSize2KB},
+	EC10P4: {N: 10, M: 4, L: 0, AZCount: 1, PutQuorum: 13, GetQuorum: 0, MinShardSize: alignSize2KB},
+	EC6P3:  {N: 6, M: 3, L: 0, AZCount: 1, PutQuorum: 8, GetQuorum: 0, MinShardSize: alignSize2KB},
+	// for env test
+	EC6P3L3:       {N: 6, M: 3, L: 3, AZCount: 3, PutQuorum: 9, GetQuorum: 0, MinShardSize: alignSize2KB},
+	EC6P6Align0:   {N: 6, M: 6, L: 0, AZCount: 3, PutQuorum: 11, GetQuorum: 0, MinShardSize: alignSize0B},
+	EC6P6Align512: {N: 6, M: 6, L: 0, AZCount: 3, PutQuorum: 11, GetQuorum: 0, MinShardSize: alignSize512B},
+	EC4P4L2:       {N: 4, M: 4, L: 2, AZCount: 2, PutQuorum: 6, GetQuorum: 0, MinShardSize: alignSize2KB},
+}
+
+var constName2CodeMode = map[CodeModeName]CodeMode{
+	"EC15P12":       EC15P12,
+	"EC6P6":         EC6P6,
+	"EC16P20L2":     EC16P20L2,
+	"EC6P10L2":      EC6P10L2,
+	"EC6P3L3":       EC6P3L3,
+	"EC6P6Align0":   EC6P6Align0,
+	"EC6P6Align512": EC6P6Align512,
+	"EC4P4L2":       EC4P4L2,
+	"EC12P4":        EC12P4,
+	"EC16P4":        EC16P4,
+	"EC3P3":         EC3P3,
+	"EC10P4":        EC10P4,
+	"EC6P3":         EC6P3,
+}
+
+var constCodeMode2Name = map[CodeMode]CodeModeName{
+	EC15P12:       "EC15P12",
+	EC6P6:         "EC6P6",
+	EC16P20L2:     "EC16P20L2",
+	EC6P10L2:      "EC6P10L2",
+	EC6P3L3:       "EC6P3L3",
+	EC6P6Align0:   "EC6P6Align0",
+	EC6P6Align512: "EC6P6Align512",
+	EC4P4L2:       "EC4P4L2",
+	EC12P4:        "EC12P4",
+	EC16P4:        "EC16P4",
+	EC3P3:         "EC3P3",
+	EC10P4:        "EC10P4",
+	EC6P3:         "EC6P3",
+}
+
+//vol layout ep:EC6P10L2
+//|----N------|--------M----------------|--L--|
+//|0,1,2,3,4,5|6,7,8,9,10,11,12,13,14,15|16,17|
+
+// global stripe:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15], n=6 m=10
+// two local stripes:
+// local stripe1:[0,1,2,  6, 7, 8, 9,10, 16] n=8 m=1
+// local stripe2:[3,4,5, 11,12,13,14,15, 17] n=8 m=1
+
+// Tactic constant strategy of one CodeMode
+type Tactic struct {
+	N int
+	M int
+	// local parity count
+	L int
+	// the count of AZ, access use this for split data shards and parity shards
+	AZCount int
+
+	// PutQuorum write quorum,
+	// MUST make sure that ec data is recoverable if one AZ was down
+	// We SHOULD ignore the local shards
+	// (N + M) / AZCount + N <= PutQuorum <= M + N
+	PutQuorum int
+
+	// get quorum config
+	GetQuorum int
+
+	// MinShardSize min size per shard, fill data into shards 0-N continuously,
+	// align with zero bytes if data size less than MinShardSize*N
+	//
+	// length of data less than MinShardSize*N, size of per shard = MinShardSize
+	//  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+	//  |  data  |                 align zero bytes                     |
+	//  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+	//  |    0    |    1    |    2    |   ....                |    N    |
+	//  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+	//
+	// length of data more than MinShardSize*N, size of per shard = ceil(len(data)/N)
+	//  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+	//  |                           data                        |padding|
+	//  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+	//  |    0    |    1    |    2    |   ....                |    N    |
+	//  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+	MinShardSize int
+}
+
+func init() {
+	// assert all codemode
+	for _, pair := range []struct {
+		Mode CodeMode
+		Size int
+	}{
+		{Mode: EC15P12, Size: alignSize2KB},
+		{Mode: EC6P6, Size: alignSize2KB},
+		{Mode: EC16P20L2, Size: alignSize2KB},
+		{Mode: EC6P10L2, Size: alignSize2KB},
+
+		{Mode: EC6P3L3, Size: alignSize2KB},
+		{Mode: EC6P6Align0, Size: alignSize0B},
+		{Mode: EC6P6Align512, Size: alignSize512B},
+	} {
+		tactic := pair.Mode.Tactic()
+		if !tactic.IsValid() {
+			panic(fmt.Sprintf("Invalid codemode:%d Tactic:%+v", pair.Mode, tactic))
+		}
+
+		min := tactic.N + (tactic.N+tactic.M)/tactic.AZCount
+		max := tactic.N + tactic.M
+		if tactic.PutQuorum < min || tactic.PutQuorum > max {
+			panic(fmt.Sprintf("Invalid codemode:%d PutQuorum:%d([%d,%d])", pair.Mode,
+				tactic.PutQuorum, min, max))
+		}
+
+		if tactic.MinShardSize != pair.Size {
+			panic(fmt.Sprintf("Invalid codemode:%d MinShardSize:%d(%d)", pair.Mode,
+				tactic.MinShardSize, pair.Size))
+		}
+	}
+}
+
+// Tactic returns its constant tactic
+func (c CodeMode) Tactic() Tactic {
+	if tactic, ok := constCodeModeTactic[c]; ok {
+		return tactic
+	}
+	panic(fmt.Sprintf("Invalid codemode:%d", c))
+}
+
+func (c CodeMode) GetShardNum() int {
+	tactic := c.Tactic()
+	return tactic.L + tactic.M + tactic.N
+}
+
+// Name turn the CodeMode to CodeModeName
+func (c CodeMode) Name() CodeModeName {
+	if name, ok := constCodeMode2Name[c]; ok {
+		return name
+	}
+	panic(fmt.Sprintf("codemode: %d is invalid", c))
+}
+
+// String turn the CodeMode to string
+func (c CodeMode) String() string {
+	if name, ok := constCodeMode2Name[c]; ok {
+		return string(name)
+	}
+	return ""
+}
+
+// IsValid check the CodeMode is valid
+func (c CodeMode) IsValid() bool {
+	if _, ok := constCodeMode2Name[c]; ok {
+		return ok
+	}
+	return false
+}
+
+// GetCodeMode get the code mode by name
+func (cn CodeModeName) GetCodeMode() CodeMode {
+	if code, ok := constName2CodeMode[cn]; ok {
+		return code
+	}
+	panic(fmt.Sprintf("codemode: %s is invalid", cn))
+}
+
+// IsValid check the CodeMode is valid by Name
+func (cn CodeModeName) IsValid() bool {
+	if _, ok := constName2CodeMode[cn]; ok {
+		return ok
+	}
+	return false
+}
+
+// Tactic get tactic by code mode name
+func (cn CodeModeName) Tactic() Tactic {
+	return cn.GetCodeMode().Tactic()
+}
+
+// IsValid ec tactic valid or not
+func (c *Tactic) IsValid() bool {
+	return c.N > 0 && c.M > 0 && c.L >= 0 && c.AZCount > 0 &&
+		c.PutQuorum > 0 && c.GetQuorum >= 0 && c.MinShardSize >= 0 &&
+		c.N%c.AZCount == 0 && c.M%c.AZCount == 0 && c.L%c.AZCount == 0
+}
+
+// GetECLayoutByAZ ec layout by AZ
+func (c *Tactic) GetECLayoutByAZ() (azStripes [][]int) {
+	azStripes = make([][]int, c.AZCount)
+	n, m, l := c.N/c.AZCount, c.M/c.AZCount, c.L/c.AZCount
+	for idx := range azStripes {
+		stripe := make([]int, 0, n+m+l)
+		for i := 0; i < n; i++ {
+			stripe = append(stripe, idx*n+i)
+		}
+		for i := 0; i < m; i++ {
+			stripe = append(stripe, c.N+idx*m+i)
+		}
+		for i := 0; i < l; i++ {
+			stripe = append(stripe, c.N+c.M+idx*l+i)
+		}
+		azStripes[idx] = stripe
+	}
+	return azStripes
+}
+
+// GlobalStripe returns initial stripe	return name.GetCodeMode().Tactic()
+func (c *Tactic) GlobalStripe() (indexes []int, n, m int) {
+	indexes = make([]int, c.N+c.M)
+	for i := 0; i < c.N+c.M; i++ {
+		indexes[i] = i
+	}
+	return indexes, c.N, c.M
+}
+
+// AllLocalStripe returns all local stripes
+func (c *Tactic) AllLocalStripe() (stripes [][]int, n, m int) {
+	if c.L == 0 {
+		return
+	}
+
+	n, m, l := c.N/c.AZCount, c.M/c.AZCount, c.L/c.AZCount
+	return c.GetECLayoutByAZ(), n + m, l
+}
+
+// LocalStripe get local stripe by index
+func (c *Tactic) LocalStripe(index int) (localStripe []int, n, m int) {
+	if c.L == 0 {
+		return nil, 0, 0
+	}
+
+	n, m, l := c.N/c.AZCount, c.M/c.AZCount, c.L/c.AZCount
+	var azIdx int
+	if index < c.N {
+		azIdx = index / n
+	} else if index < c.N+c.M {
+		azIdx = (index - c.N) / m
+	} else if index < c.N+c.M+c.L {
+		azIdx = (index - c.N - c.M) / l
+	} else {
+		return nil, 0, 0
+	}
+
+	return c.LocalStripeInAZ(azIdx)
+}
+
+// LocalStripeInAZ get local stripe in az index
+func (c *Tactic) LocalStripeInAZ(azIndex int) (localStripe []int, n, m int) {
+	if c.L == 0 {
+		return nil, 0, 0
+	}
+
+	n, m, l := c.N/c.AZCount, c.M/c.AZCount, c.L/c.AZCount
+	azStripes := c.GetECLayoutByAZ()
+	if azIndex < 0 || azIndex >= len(azStripes) {
+		return nil, 0, 0
+	}
+	return azStripes[azIndex][:], n + m, l
+}
+
+// GetAllCodeModes get all the available CodeModes
+func GetAllCodeModes() []CodeMode {
+	return []CodeMode{
+		EC15P12,
+		EC6P6,
+		EC16P20L2,
+		EC6P10L2,
+		EC6P3L3,
+		EC6P6Align0,
+		EC6P6Align512,
+		EC4P4L2,
+		EC12P4,
+		EC16P4,
+		EC3P3,
+		EC10P4,
+		EC6P3,
+	}
+}

blobstore/common/codemode/codemode_test.go

@@ -0,0 +1,240 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package codemode
+
+import (
+	"math"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	ec6P10L2Stripes = [][]int{
+		{0, 1, 2, 6, 7, 8, 9, 10, 16},
+		{3, 4, 5, 11, 12, 13, 14, 15, 17},
+	}
+	ec16P20L2Stripes = [][]int{
+		{0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 36},
+		{8, 9, 10, 11, 12, 13, 14, 15, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37},
+	}
+)
+
+func TestCodeModeBase(t *testing.T) {
+	codeMode1 := EC15P12.Tactic()
+	assert.Equal(t, codeMode1.MinShardSize, 2048)
+	indexes, n, m := (&codeMode1).GlobalStripe()
+	assert.Equal(t, codeMode1.N, n)
+	assert.Equal(t, codeMode1.M, m)
+	assert.True(t, codeMode1.IsValid())
+	expectedIndex := make([]int, 0)
+	for i := 0; i < codeMode1.N+codeMode1.M; i++ {
+		expectedIndex = append(expectedIndex, i)
+	}
+	assert.Equal(t, expectedIndex, indexes)
+
+	codeMode2 := EC6P10L2.Tactic()
+	assert.Equal(t, codeMode2.MinShardSize, 2048)
+	indexes, n, m = (&codeMode2).GlobalStripe()
+	assert.Equal(t, codeMode2.N, n)
+	assert.Equal(t, codeMode2.M, m)
+	assert.True(t, codeMode2.IsValid())
+	expectedIndex = make([]int, 0)
+	for i := 0; i < codeMode2.N+codeMode2.M; i++ {
+		expectedIndex = append(expectedIndex, i)
+	}
+	assert.Equal(t, expectedIndex, indexes)
+}
+
+func TestCodeModeGetTactic(t *testing.T) {
+	cases := []struct {
+		mode    CodeMode
+		isPanic bool
+	}{
+		{0, true},
+		{1, false},
+		{4, false},
+		{(1 << 8) - 1, true},
+		{math.MaxInt8, true},
+	}
+
+	for _, cs := range cases {
+		if cs.isPanic {
+			assert.Panics(t, func() { cs.mode.Tactic() })
+		} else {
+			assert.NotPanics(t, func() { cs.mode.Tactic() })
+		}
+	}
+}
+
+func TestGetLayoutByAZ(t *testing.T) {
+	codeMode1 := EC15P12.Tactic()
+	indexes := (&codeMode1).GetECLayoutByAZ()
+	assert.Equal(t, 3, len(indexes))
+
+	for i := range indexes {
+		assert.Equal(t, 9, len(indexes[i]))
+	}
+
+	codeMode2 := EC6P10L2.Tactic()
+	indexes = (&codeMode2).GetECLayoutByAZ()
+	assert.Equal(t, 2, len(indexes))
+
+	assert.Equal(t, ec6P10L2Stripes[0], indexes[0])
+	assert.Equal(t, ec6P10L2Stripes[1], indexes[1])
+
+	{
+		codeMode := EC12P4.Tactic()
+		indexes := codeMode.GetECLayoutByAZ()
+		assert.Equal(t, 1, len(indexes))
+
+		for i := range indexes {
+			assert.Equal(t, codeMode.N+codeMode.M+codeMode.L, len(indexes[i]))
+		}
+	}
+}
+
+func TestGlobalStripe(t *testing.T) {
+	cases := []struct {
+		mode CodeMode
+		n    int
+	}{
+		{EC15P12, 27},
+		{EC6P6, 12},
+		{EC16P20L2, 36},
+		{EC6P10L2, 16},
+		{EC12P4, 16},
+		{EC16P4, 20},
+	}
+	for _, cs := range cases {
+		tactic := cs.mode.Tactic()
+		stripe, n, m := tactic.GlobalStripe()
+		assert.Equal(t, cs.n, len(stripe))
+		assert.Equal(t, tactic.N, n)
+		assert.Equal(t, tactic.M, m)
+	}
+}
+
+func TestAllLocalStripe(t *testing.T) {
+	cases := []struct {
+		mode    CodeMode
+		stripes [][]int
+		n       int
+		m       int
+	}{
+		{EC6P6, nil, 0, 0},
+		{EC6P10L2, ec6P10L2Stripes, 8, 1},
+		{EC16P20L2, ec16P20L2Stripes, 18, 1},
+	}
+	for _, cs := range cases {
+		tactic := cs.mode.Tactic()
+		stripes, n, m := tactic.AllLocalStripe()
+		assert.Equal(t, cs.stripes, stripes)
+		assert.Equal(t, cs.n, n)
+		assert.Equal(t, cs.m, m)
+	}
+}
+
+func TestLocalStripe(t *testing.T) {
+	cases := []struct {
+		mode   CodeMode
+		index  int
+		stripe []int
+		n      int
+		m      int
+	}{
+		{EC6P6, 0, nil, 0, 0},
+		{EC6P6, 1, nil, 0, 0},
+		{EC6P6, 4, nil, 0, 0},
+		{EC6P6, 100, nil, 0, 0},
+
+		{EC6P10L2, 0, ec6P10L2Stripes[0], 8, 1},
+		{EC6P10L2, 1, ec6P10L2Stripes[0], 8, 1},
+		{EC6P10L2, 16, ec6P10L2Stripes[0], 8, 1},
+		{EC6P10L2, 3, ec6P10L2Stripes[1], 8, 1},
+		{EC6P10L2, 11, ec6P10L2Stripes[1], 8, 1},
+		{EC6P10L2, 17, ec6P10L2Stripes[1], 8, 1},
+		{EC6P10L2, 18, nil, 0, 0},
+
+		{EC16P20L2, 0, ec16P20L2Stripes[0], 18, 1},
+		{EC16P20L2, 18, ec16P20L2Stripes[0], 18, 1},
+		{EC16P20L2, 36, ec16P20L2Stripes[0], 18, 1},
+		{EC16P20L2, 8, ec16P20L2Stripes[1], 18, 1},
+		{EC16P20L2, 35, ec16P20L2Stripes[1], 18, 1},
+		{EC16P20L2, 37, ec16P20L2Stripes[1], 18, 1},
+		{EC16P20L2, 38, nil, 0, 0},
+	}
+	for _, cs := range cases {
+		tactic := cs.mode.Tactic()
+		stripe, n, m := tactic.LocalStripe(cs.index)
+		assert.Equal(t, cs.stripe, stripe)
+		assert.Equal(t, cs.n, n)
+		assert.Equal(t, cs.m, m)
+	}
+}
+
+func TestLocalStripeInAZ(t *testing.T) {
+	cases := []struct {
+		mode    CodeMode
+		azIndex int
+		stripe  []int
+		n       int
+		m       int
+	}{
+		{EC6P6, 0, nil, 0, 0},
+		{EC6P6, 1, nil, 0, 0},
+		{EC6P6, 4, nil, 0, 0},
+		{EC6P6, 100, nil, 0, 0},
+
+		{EC6P10L2, 0, ec6P10L2Stripes[0], 8, 1},
+		{EC6P10L2, 1, ec6P10L2Stripes[1], 8, 1},
+		{EC6P10L2, 2, nil, 0, 0},
+
+		{EC16P20L2, 0, ec16P20L2Stripes[0], 18, 1},
+		{EC16P20L2, 1, ec16P20L2Stripes[1], 18, 1},
+		{EC16P20L2, 2, nil, 0, 0},
+	}
+	for _, cs := range cases {
+		tactic := cs.mode.Tactic()
+		stripe, n, m := tactic.LocalStripeInAZ(cs.azIndex)
+		assert.Equal(t, cs.stripe, stripe)
+		assert.Equal(t, cs.n, n)
+		assert.Equal(t, cs.m, m)
+	}
+}
+
+func BenchmarkGlobalStripe(b *testing.B) {
+	t := EC16P20L2.Tactic()
+	tactic := &t
+	for ii := 0; ii < b.N; ii++ {
+		tactic.GlobalStripe()
+	}
+}
+
+func BenchmarkGetECLayoutByAZ(b *testing.B) {
+	t := EC16P20L2.Tactic()
+	tactic := &t
+	for ii := 0; ii < b.N; ii++ {
+		tactic.GetECLayoutByAZ()
+	}
+}
+
+func BenchmarkLocalStripe(b *testing.B) {
+	t := EC16P20L2.Tactic()
+	tactic := &t
+	for ii := 0; ii < b.N; ii++ {
+		tactic.LocalStripe(37)
+	}
+}

blobstore/common/codemode/policy.go

@@ -0,0 +1,30 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package codemode
+
+// Policy will be used to adjust code mode's upload range or code mode's volume ratio and so on
+type Policy struct {
+	ModeName CodeModeName `json:"mode_name"`
+	// min size of object, access use this to put object
+	MinSize int64 `json:"min_size"`
+	// max size of object. access use this to put object
+	MaxSize int64 `json:"max_size"`
+	// code mode's cluster space size ratio, clusterMgr should use this to create specified num of volume
+	SizeRatio float64 `json:"size_ratio"`
+	// enable means this kind of code mode enable or not
+	// access/allocator will ignore this kind of code mode's allocation when enable is false
+	// clustermgr will ignore this kind of code mode's creation when enable is false
+	Enable bool `json:"enable"`
+}

blobstore/common/counter/counter.go

@@ -0,0 +1,72 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package counter
+
+import (
+	"sync/atomic"
+
+	"github.com/benbjohnson/clock"
+)
+
+const (
+	// SLOT counter resulting stored.
+	SLOT int64 = 20
+
+	// interval default is one minute.
+	interval int64 = 60
+)
+
+// time clock just for testing mock.
+var time = clock.New()
+
+// Counter resulting count in default interval period.
+// It is thread safe.
+type Counter struct {
+	counts     [SLOT]int64
+	timestamps [SLOT]int64
+}
+
+// Add counts once in now.
+func (c *Counter) Add() {
+	c.AddN(1)
+}
+
+// AddN adds n in now.
+func (c *Counter) AddN(n int) {
+	now := time.Now().Unix() / interval
+	index := now % SLOT
+
+	if ts := atomic.LoadInt64(&c.timestamps[index]); ts == now {
+		atomic.AddInt64(&c.counts[index], int64(n))
+	} else {
+		atomic.StoreInt64(&c.counts[index], int64(n))
+		atomic.StoreInt64(&c.timestamps[index], now)
+	}
+}
+
+// Show returns result from oldest to newest.
+func (c *Counter) Show() (counts [SLOT]int) {
+	now := time.Now().Unix() / interval
+	index := now % SLOT
+
+	validTs := now - SLOT
+	for ii := int64(0); ii < SLOT; ii++ {
+		idx := (index + 1 + ii) % SLOT
+		if ts := atomic.LoadInt64(&c.timestamps[idx]); ts > validTs {
+			counts[ii] = int(atomic.LoadInt64(&c.counts[idx]))
+		}
+	}
+	return
+}

blobstore/common/counter/counter_test.go

@@ -0,0 +1,109 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package counter
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/benbjohnson/clock"
+	"github.com/stretchr/testify/require"
+)
+
+const second clock.Duration = 1e9
+
+var mockTime *clock.Mock
+
+func init() {
+	mockTime = clock.NewMock()
+	mockTime.Add(1483950782 * second)
+	time = mockTime
+}
+
+func TestCounterBase(t *testing.T) {
+	var c Counter
+	var exp [SLOT]int
+	require.Equal(t, exp, c.Show())
+
+	c.Add()
+	exp[SLOT-1] = 1
+	require.Equal(t, exp, c.Show())
+
+	for ii := int64(0); ii < SLOT; ii++ {
+		mockTime.Add(clock.Duration(interval) * second)
+		c.AddN(int(ii))
+		copy(exp[:], exp[1:])
+		exp[SLOT-1] = int(ii)
+		require.Equal(t, exp, c.Show())
+	}
+
+	for range [10]struct{}{} {
+		c := &Counter{}
+		c.Add()
+		mockTime.Add(clock.Duration(interval) * second)
+		exp := [SLOT]int{}
+		exp[SLOT-2] = 1
+		require.Equal(t, exp, c.Show())
+
+		c.Add()
+		c.Add()
+		exp[SLOT-1] = 2
+		require.Equal(t, exp, c.Show())
+
+		mockTime.Add(clock.Duration(SLOT-1) * clock.Duration(interval) * second)
+		exp = [SLOT]int{}
+		exp[0] = 2
+		require.Equal(t, exp, c.Show())
+
+		c.Add()
+		exp[0] = 2
+		exp[SLOT-1] = 1
+		require.Equal(t, exp, c.Show())
+	}
+}
+
+func TestCounterSafe(t *testing.T) {
+	var c Counter
+	var wg sync.WaitGroup
+	wg.Add(1000)
+	for idx := range [1000]struct{}{} {
+		go func(ii int) {
+			mockTime.Add(clock.Duration(ii) * second)
+			c.AddN(ii)
+			if ii%3 == 0 {
+				c.Show()
+			}
+			wg.Done()
+		}(idx)
+	}
+	wg.Wait()
+}
+
+func BenchmarkCounterAdd(b *testing.B) {
+	var c Counter
+	for ii := 0; ii < b.N; ii++ {
+		c.Add()
+	}
+}
+
+func BenchmarkCounterShow(b *testing.B) {
+	var c Counter
+	c.AddN(100)
+	mockTime.Add(100 * second)
+	b.ResetTimer()
+	for ii := 0; ii < b.N; ii++ {
+		c.Show()
+	}
+}

blobstore/common/crc32block/block.go

@@ -0,0 +1,49 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package crc32block
+
+import (
+	"encoding/binary"
+	"hash/crc32"
+)
+
+const (
+	defaultCrc32BlockSize = 64 * 1024
+)
+
+var gBlockSize int64 = defaultCrc32BlockSize
+
+type blockUnit []byte
+
+func (b blockUnit) length() int {
+	return len(b)
+}
+
+func (b blockUnit) payload() int {
+	return len(b) - crc32Len
+}
+
+func (b blockUnit) check() (err error) {
+	payloadCrc := crc32.ChecksumIEEE(b[crc32Len:])
+	if binary.LittleEndian.Uint32(b) != payloadCrc {
+		return ErrMismatchedCrc
+	}
+	return nil
+}
+
+func (b blockUnit) writeCrc() {
+	payloadCrc := crc32.ChecksumIEEE(b[crc32Len:])
+	binary.LittleEndian.PutUint32(b, payloadCrc)
+}

blobstore/common/crc32block/decode.go

@@ -0,0 +1,216 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package crc32block
+
+import (
+	"bufio"
+	"io"
+	"io/ioutil"
+)
+
+/*
+journal record:
+
+	|-----block-----|------block-----|---block---|
+
+block:
+
+	|--crc--|------payload -----|
+*/
+
+type Decoder struct {
+	from    io.ReaderAt // read from
+	off     int64       // offset. readonly
+	limit   int64       // size limit, readonly
+	bufSize int64       // for speed
+	block   blockUnit   // block buffer
+}
+
+type decoderReader struct {
+	reader io.Reader //
+	block  []byte    //
+	i, j   int       // block[i:j]
+	err    error     //
+}
+
+type rangeReader struct {
+	r      io.Reader
+	limit  int64
+	skip   int64
+	skiped bool
+}
+
+type blockReader struct {
+	reader io.Reader //
+	block  blockUnit //
+	i, j   int       // block[i:j] is unread portion of the current block's payload.
+	remain int64     //
+	err    error     //
+}
+
+func (br *blockReader) Read(p []byte) (n int, err error) {
+	if br.err != nil {
+		return 0, br.err
+	}
+
+	for br.i == br.j {
+		if br.remain == 0 {
+			return n, io.EOF
+		}
+		br.err = br.nextBlock()
+		if br.err != nil {
+			return 0, br.err
+		}
+	}
+
+	n = copy(p, br.block[br.i:br.j])
+
+	br.i += n
+	br.remain -= int64(n)
+	return n, nil
+}
+
+func (br *blockReader) nextBlock() (err error) {
+	blockLen := int64(len(br.block))
+	blockPayloadLen := int64(blockLen - crc32Len)
+
+	want := blockLen
+	if br.remain < blockPayloadLen {
+		want = br.remain + crc32Len
+	}
+
+	_, err = io.ReadFull(br.reader, br.block[:want])
+	if err != nil {
+		br.err = err
+		return br.err
+	}
+
+	if err = blockUnit(br.block[:want]).check(); err != nil {
+		return err
+	}
+
+	br.i = crc32Len
+	br.j = int(want)
+
+	return nil
+}
+
+func (r *rangeReader) Read(p []byte) (n int, err error) {
+	if !r.skiped {
+		_, err := io.CopyN(ioutil.Discard, r.r, r.skip)
+		if err != nil {
+			return 0, err
+		}
+		r.skiped = true
+		r.r = io.LimitReader(r.r, r.limit)
+	}
+	return r.r.Read(p)
+}
+
+func (dec *Decoder) Reader(from, to int64) (r io.Reader, err error) {
+	blockLen := int64(dec.block.length())
+	blockPayloadLen := int64(dec.block.payload())
+
+	blockOff := (from / blockPayloadLen) * blockLen
+	encodedSize := EncodeSize(dec.limit, blockLen) - blockOff
+
+	// raw reader
+	r = io.NewSectionReader(dec.from, dec.off+blockOff, encodedSize)
+
+	// buffer
+	r = bufio.NewReaderSize(r, int(dec.bufSize))
+
+	// decode reader
+	r = NewBlockReader(r, DecodeSize(encodedSize, blockLen), dec.block)
+
+	// range reader
+	r = &rangeReader{
+		r:     r,
+		limit: to - from,
+		skip:  from % blockPayloadLen,
+	}
+
+	return r, nil
+}
+
+func (r *decoderReader) Read(b []byte) (n int, err error) {
+	if r.err != nil {
+		return 0, r.err
+	}
+
+	for len(b) > 0 {
+		if r.i == r.j {
+			if r.err = r.nextBlock(); r.err != nil {
+				if n > 0 {
+					return n, nil
+				}
+				return n, r.err
+			}
+		}
+
+		readn := copy(b, r.block[r.i:r.j])
+		r.i += readn
+
+		b = b[readn:]
+		n += readn
+	}
+	return
+}
+
+func (r *decoderReader) nextBlock() (err error) {
+	n, err := readFullOrToEnd(r.reader, r.block)
+	if err != nil {
+		return
+	}
+
+	if n <= crc32Len {
+		return ErrMismatchedCrc
+	}
+
+	if err = blockUnit(r.block[:n]).check(); err != nil {
+		return ErrMismatchedCrc
+	}
+
+	r.i, r.j = crc32Len, n
+
+	return nil
+}
+
+func NewBlockReader(r io.Reader, limit int64, block []byte) *blockReader {
+	if block == nil || !isValidBlockLen(int64(len(block))) {
+		panic(ErrInvalidBlock)
+	}
+	return &blockReader{reader: r, remain: limit, block: block}
+}
+
+// NewDecoderReader returns io.Reader
+//
+// Deprecated: no reused buffer, use NewBodyDecoder to instead.
+func NewDecoderReader(in io.Reader) io.Reader {
+	chunk := make([]byte, defaultCrc32BlockSize)
+	return &decoderReader{block: chunk, err: nil, reader: in}
+}
+
+func NewDecoderWithBlock(r io.ReaderAt, off int64, size int64, block []byte, bufferSize int64) (dec *Decoder, err error) {
+	if block == nil || !isValidBlockLen(int64(len(block))) {
+		return nil, ErrInvalidBlock
+	}
+	return &Decoder{from: r, off: off, block: block, limit: size, bufSize: bufferSize}, nil
+}
+
+func NewDecoder(r io.ReaderAt, off int64, size int64) (dec *Decoder, err error) {
+	block := make([]byte, defaultCrc32BlockSize)
+	return NewDecoderWithBlock(r, off, size, block, defaultCrc32BlockSize)
+}

blobstore/common/crc32block/decode_test.go

@@ -0,0 +1,53 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package crc32block
+
+import (
+	"bytes"
+	"hash/crc32"
+	"io"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSimpleEncoderDecoder(t *testing.T) {
+	size1 := int64(64*1024 - 4)
+	size2 := int64(64 * 1024)
+	size3 := int64(64*1024 + 4)
+	size4 := int64(1024 * 1024)
+	size5 := int64(0)
+	data := []struct {
+		bodysize int64
+	}{
+		{bodysize: size1},
+		{bodysize: size2},
+		{bodysize: size3},
+		{bodysize: size4},
+		{bodysize: size5},
+	}
+
+	for index, d := range data {
+		b1 := genRandData(d.bodysize)
+		body := bytes.NewReader(b1)
+		enc := NewEncoderReader(body)
+
+		dec := NewDecoderReader(enc)
+		b2 := make([]byte, d.bodysize)
+		_, err := io.ReadFull(dec, b2)
+		require.NoError(t, err, "index is %v", index)
+		require.Equal(t, crc32.ChecksumIEEE(b1), crc32.ChecksumIEEE(b2), "index is %v", index)
+	}
+}

blobstore/common/crc32block/encode.go

@@ -0,0 +1,172 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package crc32block
+
+import (
+	"io"
+)
+
+type ReaderError struct {
+	error
+}
+
+type WriterError struct {
+	error
+}
+
+type Encoder struct {
+	block blockUnit // block buffer
+}
+
+type limitEncoderReader struct {
+	reader io.Reader
+	block  blockUnit
+	remain int64
+	i, j   int
+	err    error
+}
+
+type encoderReader struct {
+	reader io.Reader //
+	block  blockUnit //
+	i, j   int       // block[i:j]
+	err    error     //
+}
+
+func (enc *Encoder) Encode(from io.Reader, limitSize int64, to io.Writer) (n int64, err error) {
+	if !isValidBlockLen(int64(enc.block.length())) {
+		panic(ErrInvalidBlock)
+	}
+
+	encSize := EncodeSize(limitSize, int64(enc.block.length()))
+
+	reader := &limitEncoderReader{reader: from, block: enc.block, remain: limitSize}
+
+	return io.CopyN(to, reader, encSize)
+}
+
+func (r *limitEncoderReader) Read(b []byte) (n int, err error) {
+	if r.err != nil {
+		return 0, r.err
+	}
+
+	for len(b) > 0 {
+		if r.i == r.j {
+			if r.remain == 0 {
+				return n, io.EOF
+			}
+			if r.err = r.nextBlock(); r.err != nil {
+				if n > 0 {
+					return n, nil
+				}
+				return n, r.err
+			}
+		}
+
+		readn := copy(b, r.block[r.i:r.j])
+		r.i += readn
+
+		b = b[readn:]
+		n += readn
+	}
+	return
+}
+
+func (r *limitEncoderReader) nextBlock() (err error) {
+	blockPayloadLen := r.block.payload()
+
+	needn := blockPayloadLen
+	if r.remain < int64(blockPayloadLen) {
+		needn = int(r.remain)
+	}
+
+	block := blockUnit(r.block[:crc32Len+needn])
+
+	n, err := io.ReadFull(r.reader, block[crc32Len:])
+	if err != nil {
+		return ReaderError{err}
+	}
+
+	r.i = 0
+	r.j = crc32Len + n
+
+	blockUnit(r.block[r.i:r.j]).writeCrc()
+	r.remain -= int64(block.payload())
+
+	return nil
+}
+
+func (r *encoderReader) Read(b []byte) (n int, err error) {
+	if r.err != nil {
+		return 0, r.err
+	}
+
+	for len(b) > 0 {
+		if r.i == r.j {
+			if r.err = r.nextBlock(); r.err != nil {
+				if n > 0 {
+					return n, nil
+				}
+				return n, r.err
+			}
+		}
+
+		readn := copy(b, r.block[r.i:r.j])
+		r.i += readn
+
+		b = b[readn:]
+		n += readn
+	}
+	return
+}
+
+func (r *encoderReader) nextBlock() (err error) {
+	n, err := readFullOrToEnd(r.reader, r.block[crc32Len:])
+	if err != nil {
+		return err
+	}
+
+	r.i = 0
+	r.j = crc32Len + n
+
+	blockUnit(r.block[r.i:r.j]).writeCrc()
+
+	return nil
+}
+
+func NewEncoder(block []byte) (enc *Encoder, err error) {
+	if block != nil && !isValidBlockLen(int64(len(block))) {
+		return nil, ErrInvalidBlock
+	}
+	if block == nil {
+		block = make([]byte, defaultCrc32BlockSize)
+	}
+
+	return &Encoder{block: block}, nil
+}
+
+// NewEncoderReader returns io.Reader
+//
+// Deprecated: no reused buffer, use NewBodyEncoder to instead.
+func NewEncoderReader(r io.Reader) io.Reader {
+	block := make([]byte, defaultCrc32BlockSize)
+	return &encoderReader{block: block, reader: r}
+}
+
+func NewLimitEncoderReader(r io.Reader, limitSize int64) (enc *limitEncoderReader) {
+	block := make([]byte, defaultCrc32BlockSize)
+	enc = &limitEncoderReader{reader: r, block: block, remain: limitSize}
+	return
+}

blobstore/common/crc32block/encode_test.go

@@ -0,0 +1,260 @@
+// Copyright 2022 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package crc32block
+
+import (
+	"bytes"
+	"crypto/rand"
+	"io"
+	"io/ioutil"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	_KiB = int64(1024)
+)
+
+func genRandData(size int64) []byte {
+	b := make([]byte, size)
+	rand.Read(b)
+	return b
+}
+
+func TestEncodeData(t *testing.T) {
+	data := genRandData(128*1024 + 80)
+	fsize := int64(len(data)) // 128k+80
+	r := bytes.NewReader(data)
+	w := bytes.NewBuffer(nil)
+
+	encoder, err := NewEncoder(nil)
+	require.NoError(t, err)
+	require.NotNil(t, encoder)
+
+	// encode
+	n, err := encoder.Encode(r, fsize, w)
+	require.NoError(t, err)
+	require.Equal(t, n, EncodeSize(int64(fsize), defaultCrc32BlockSize))
+}
+
+func TestDecodeData(t *testing.T) {
+	data := genRandData(128*1024 + 80)
+	fsize := int64(len(data)) // 128k+80
+	r := bytes.NewReader(data)
+	w := bytes.NewBuffer(nil)
+
+	encoder, err := NewEncoder(nil)
+	require.NoError(t, err)
+	require.NotNil(t, encoder)
+
+	// encode
+	n, err := encoder.Encode(r, fsize, w)
+	require.NoError(t, err)
+	require.Equal(t, n, EncodeSize(fsize, defaultCrc32BlockSize))
+
+	// decode
+	testDatas := []struct {
+		off     int64
+		from    int64
+		to      int64
+		fsize   int64
+		padding int64
+	}{
+		{from: 0, to: 0, fsize: 0},
+		{from: fsize, to: fsize, fsize: fsize},
+		{from: 0, to: fsize, fsize: fsize},
+
+		{from: 64*_KiB - 4, to: fsize, fsize: fsize},
+		{from: 64*_KiB + 0, to: fsize, fsize: fsize},
+		{from: 64*_KiB + 4, to: fsize, fsize: fsize},
+		{from: 64*_KiB + 5, to: fsize, fsize: fsize},
+
+		{from: 64*_KiB - 4, to: fsize - 1, fsize: fsize},
+		{from: 64*_KiB + 0, to: fsize - 1, fsize: fsize},
+		{from: 64*_KiB + 4, to: fsize - 1, fsize: fsize},
+		{from: 64*_KiB + 5, to: fsize - 1, fsize: fsize},
+
+		{from: 64*_KiB + 4, to: fsize - 64*_KiB, fsize: fsize},
+		{from: 64*_KiB + 4, to: fsize - 64*_KiB - 4, fsize: fsize},
+		{from: 64*_KiB + 4, to: fsize - 64*_KiB - 5, fsize: fsize},
+		{from: 0, to: fsize - 64*_KiB - 4 - 64*_KiB - 4, fsize: fsize},
+	}
+
+	for _, tb := range testDatas {
+		disk := append(make([]byte, tb.off), w.Bytes()...)
+		disk = append(disk, make([]byte, tb.padding)...)
+
+		decoder, err := NewDecoder(bytes.NewReader(disk), int64(tb.off), tb.fsize)
+		require.NoError(t, err)
+		require.NotNil(t, decoder)
+
+		reader, err := decoder.Reader(tb.from, tb.to)
+		require.NoError(t, err)
+		require.NotNil(t, reader)
+
+		cont, err := ioutil.ReadAll(reader)
+		require.NoError(t, err)
+		require.Equal(t, data[tb.from:tb.to], cont)
+	}
+}
+
+func TestLimitEncoderReader(t *testing.T) {
+	data := genRandData(128*1024 + 80)
+	fsize := int64(len(data)) // 128k+80
+	r := bytes.NewReader(data)
+	w := bytes.NewBuffer(nil)
+
+	encReader := NewLimitEncoderReader(r, fsize)
+	io.Copy(w, encReader)
+
+	// decode
+	testDatas := []struct {
+		off     int64
+		from    int64
+		to      int64
+		fsize   int64
+		padding int64
+	}{
+		{from: 0, to: 0, fsize: 0},
+		{from: fsize, to: fsize, fsize: fsize},
+		{from: 0, to: fsize, fsize: fsize},
+
+		{from: 0, to: 64, fsize: fsize},
+		{from: 64*_KiB - 4, to: fsize, fsize: fsize},
+		{from: 64*_KiB + 0, to: fsize, fsize: fsize},
+		{from: 64*_KiB + 4, to: fsize, fsize: fsize},
+		{from: 64*_KiB + 5, to: fsize, fsize: fsize},
+
+		{from: 64*_KiB - 4, to: fsize - 1, fsize: fsize},
+		{from: 64*_KiB + 0, to: fsize - 1, fsize: fsize},
+		{from: 64*_KiB + 4, to: fsize - 1, fsize: fsize},
+		{from: 64*_KiB + 5, to: fsize - 1, fsize: fsize},
+
+		{from: 64*_KiB + 4, to: fsize - 64*_KiB, fsize: fsize},
+		{from: 64*_KiB + 4, to: fsize - 64*_KiB - 4, fsize: fsize},
+		{from: 64*_KiB + 4, to: fsize - 64*_KiB - 5, fsize: fsize},
+		{from: 0, to: fsize - 64*_KiB - 4 - 64*_KiB - 4, fsize: fsize},
+	}
+
+	for _, tb := range testDatas {
+		disk := append(make([]byte, tb.off), w.Bytes()...)
+		disk = append(disk, make([]byte, tb.padding)...)
+
+		decoder, err := NewDecoder(bytes.NewReader(disk), int64(tb.off), tb.fsize)
+		require.NoError(t, err)
+		require.NotNil(t, decoder)
+
+		reader, err := decoder.Reader(tb.from, tb.to)
+		require.NoError(t, err)
+		require.NotNil(t, reader)
+
+		cont, err := ioutil.ReadAll(reader)
+		require.NoError(t, err)
+		require.Equal(t, data[tb.from:tb.to], cont)
+	}
+}
+
+func TestLimitEncoderReader2(t *testing.T) {
+	testDatas := []struct {
+		offset   int64
+		fsize    int64
+		from, to int64
+		padding  int64
+		err      error
+	}{
+		{offset: 0, fsize: 1, from: 0, to: 1},
+		{offset: 0, fsize: 64*_KiB - 4, from: 0, to: 64},
+		{offset: 0, fsize: 64*_KiB - 4, from: 0, to: 64*_KiB - 4},
+		{offset: 10, fsize: 64 * _KiB, from: 0, to: 64*_KiB - 4},
+		{offset: 0, fsize: 64 * _KiB, from: 0, to: 64 * _KiB},
+		{offset: 10, fsize: 64 * _KiB, from: 0, to: 64 * _KiB},
+		{offset: 10, fsize: 64*_KiB + 5, from: 0, to: 64 * _KiB},
+
+		{offset: 10, fsize: 128 * _KiB, from: 0, to: 128 * _KiB},
+		{offset: 10, fsize: 128 * _KiB, from: 128*_KiB - 4, to: 128 * _KiB},
+		{offset: 10, fsize: 128 * _KiB, from: 64*_KiB - 4, to: 64 * _KiB},
+		{offset: 10, fsize: 128 * _KiB, from: 64*_KiB + 4, to: 64*_KiB + 4},
+		{offset: 10, fsize: 128 * _KiB, from: 64*_KiB + 4, to: 64*_KiB + 8},
+		{offset: 10, fsize: 128 * _KiB, from: 64*_KiB + 7, to: 64*_KiB + 10},
+		{offset: 10, fsize: 128 * _KiB, from: 64*_KiB - 4, to: 64*_KiB - 1},
+		{offset: 10, fsize: 128 * _KiB, from: 4, to: 64*_KiB - 1},
+		{offset: 10, fsize: 128 * _KiB, from: 4, to: 64*_KiB - 4},
+		{offset: 10, fsize: 128 * _KiB, from: 4, to: 64 * _KiB},
+		{offset: 10, fsize: 128 * _KiB, from: 64 * _KiB, to: 128 * _KiB},
+		{offset: 10, fsize: 128*_KiB + 7, from: 64 * _KiB, to: 64 * _KiB},
+		{offset: 10, fsize: 128*_KiB + 7, from: 64*_KiB + 4, to: 128 * _KiB},
+		{offset: 10, fsize: 128*_KiB + 7, from: 64*_KiB - 4, to: 128*_KiB + 7},
+	}
+
+	for _, tb := range testDatas {
+		// encoder
+		data := genRandData(tb.fsize)
+		fsize := int64(len(data))
+		r := bytes.NewReader(data)
+		w := bytes.NewBuffer(nil)
+
+		encReader := NewLimitEncoderReader(r, fsize)
+		io.Copy(w, encReader)
+
+		// decoder
+		content := append(make([]byte, tb.offset), w.Bytes()...)
+		content = append(content, make([]byte, tb.padding)...)
+
+		decoder, err := NewDecoder(bytes.NewReader(content), int64(tb.offset), tb.fsize)
+		require.NoError(t, err)
+		require.NotNil(t, decoder)
+
+		reader, err := decoder.Reader(tb.from, tb.to)
+		require.NoError(t, err)
+		require.NotNil(t, reader)
+
+		cont, err := ioutil.ReadAll(reader)
+		require.NoError(t, err)
+		require.Equal(t, data[tb.from:tb.to], cont)
+	}
+}
+
+func TestLimitEncoderReaderFailed(t *testing.T) {
+	// encoder
+	data := genRandData(64 * _KiB)
+	fsize := int64(len(data))
+	r := bytes.NewReader(data)
+	w := bytes.NewBuffer(nil)
+
+	enc, err := NewEncoder(make([]byte, 1))
+	require.Error(t, err)
+	require.Nil(t, enc)
+
+	enc, err = NewEncoder(nil)
+	require.NoError(t, err)
+	require.NotNil(t, enc)
+
+	n, err := enc.Encode(r, fsize, w)
+	require.NoError(t, err)
+	require.Equal(t, n, EncodeSize(fsize, defaultCrc32BlockSize))
+
+	r = bytes.NewReader(data)
+	_, err = enc.Encode(r, fsize+1, ioutil.Discard)
+	require.Error(t, err)
+	_, ok := err.(ReaderError)
+	require.Equal(t, ok, true)
+}
+
+func BenchmarkEncode(b *testing.B) {
+	benchmarkCode(b, func(r io.Reader) io.ReadCloser {
+		return io.NopCloser(NewEncoderReader(r))
+	})
+}