fix(access): limit writer in pipe

@formatter:off

Signed-off-by: slasher <shenjie1@oppo.com>

.gitattributes

@@ -2,3 +2,6 @@
 * text=auto eol=lf
 # Do not modify line endings for binary files
 *.png binary
+*.pdf binary
+*.tar.gz binary
+*.jpg binary

.github/pull_request_template.md

@@ -1,17 +1,110 @@
-<!-- Thanks for sending a pull request! -->
+<!-- Thanks for sending the pull request! -->
 
-**What this PR does / why we need it**:
+<!--
+### Contribution Checklist
 
-**Which issue this PR fixes**:
-<!-- *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: -->
-fixes #
+  - PR title format should be *type(scope): subject*. For details, see *[Pull Request Title](https://github.com/cubefs/cubefs/blob/master/.github/workflows/check_pull_request.yml)*.
 
-**Special notes for your reviewer**:
+  - Each pull request should address only one issue, not mix up code from multiple issues.
 
-**Release note**:
-<!--  Steps to write your release note:
-1. Use the release-note-* labels to set the release note state (if you have access)
-2. Enter your extended release note in the below block; leaving it blank means using the PR title as the release note. If no release note is required, just write `NONE`.
+  - Each commit in the pull request has a meaningful commit message. For details, see *[Commit Message](https://github.com/cubefs/cubefs/blob/master/.github/workflows/check_pull_request.yml)*.
+
+  - Fill out the template below to describe the changes contributed by the pull request. That will give reviewers the context they need to do the review.
+
+  - Once all items of the checklist are addressed, remove the above text and this checklist, leaving only the filled out template below.
 -->
-```release-note
+
+<!-- Either this PR fixes an issue, -->
+
+Fixes: #xyz
+
+<!-- or this PR is one task of an issue. -->
+
+Main Issue: #xyz
+
+
+### Motivation
+<!-- Explain here the context, and why you're making that change. What is the problem you're trying to solve. -->
+
+blaaaaa
+
+### Modifications
+<!-- Describe the modifications you've done. -->
+
+``` text
+blaaaaa
 ```
+
+### Types of changes
+<!-- Show in a checkbox-style, the expected types of changes your project is supposed to have: -->
+<!-- _Put an `x` in the boxes that apply_ -->
+
+- [ ] New feature (non-breaking change which adds functionality)
+- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ] Bugfix (non-breaking change which fixes an issue)
+- [ ] Documentation Update (if none of the other choices apply)
+- [ ] So on...
+
+### Verifying this change
+<!-- Please pick either of the following options. -->
+
+- [ ] Make sure that the change passes the testing checks.
+
+This change is a trivial rework / code cleanup without any test coverage.
+
+*(or)*
+
+This change is already covered by existing tests, such as *(please describe tests)*.
+
+*(or)*
+
+This change added tests and can be verified as follows:
+
+*(example:)*
+  - *This can be verified in development debugging*
+  - *This can be realized in a mocked environment, like a test cluster consisting in docker*
+
+*(or)*
+
+This change `MUST` reappear in online clusters, or occur in that specific scenarios.
+
+### Does this pull request potentially affect one of the following parts:
+<!-- Which of the following parts are affected by this change? -->
+
+- [ ] Master
+- [ ] MetaNode
+- [ ] DataNode
+- [ ] ObjectNode
+- [ ] AuthNode
+- [ ] LcNode
+- [ ] Blobstore
+- [ ] Client
+- [ ] Cli
+- [ ] SDK
+- [ ] Other Tools
+- [ ] Common Packages
+- [ ] Dependencies
+- [ ] Anything that affects deployment
+
+### Documentation
+<!-- Is there a chinese and english document modification? -->
+
+- [ ] `doc` <!-- Your PR contains doc changes. -->
+- [ ] `doc-required` <!-- Your PR changes impact docs and you will update later -->
+- [ ] `doc-not-needed` <!-- Your PR changes do not impact docs -->
+- [ ] `doc-complete` <!-- Docs have been already added -->
+
+### Review Expection
+<!-- How long would you like the team to be completed in your contributing? -->
+
+- [ ] `in-two-days`
+- [ ] `weekly`
+- [ ] `free-time`
+- [ ] `whenever`
+
+### Matching PR in forked repository
+<!-- enter the url if has PR in forked repository. -->
+
+PR in forked repository: <!-- ENTER URL HERE -->
+
+<!-- Thanks for contributing, best days! -->

.github/slsa/slsa-preload-linux-amd64.yml

@@ -1,23 +0,0 @@
-version: 1
-
-env:
-  - CGO_ENABLED=0
-
-flags:
-  - -trimpath
-
-goos: linux
-goarch: amd64
-
-# (Optional) Entrypoint to compile.
-main: ./preload/preload.go
-
-binary: cfs-preload-{{ .Os }}-{{ .Arch }}
-
-ldflags:
-  - "-X github.com/cubefs/cubefs/proto.Version={{ .Env.VERSION }}"
-  - "-X github.com/cubefs/cubefs/proto.CommitID={{ .Env.COMMIT_ID }}"
-  - "-X github.com/cubefs/cubefs/proto.BranchName={{ .Env.BRANCH_NAME }}"
-  - "-X github.com/cubefs/cubefs/proto.BuildTime={{ .Env.BUILD_TIME }}"
-  - "-X github.com/cubefs/cubefs/blobstore/util/version.version={{ .Env.BRANCH_NAME }}/{{ .Env.COMMIT_ID }}"
-  - "-w -s"

.github/workflows/blobstore_checks.yml

@@ -1,36 +0,0 @@
-name: BlobStore-Checks
-
-on:
-  push:
-    paths:
-        - 'blobstore/**.go'
-  pull_request:
-    types: [opened, synchronize, reopened]
-    paths:
-        - 'blobstore/**'
-
-permissions:
-  contents: read
-
-jobs:
-  GolangFormat:
-    name: format
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
-
-      - name: Go code format with gofumpt
-        run: |
-          docker/run_docker.sh --bsgofumpt
-
-  GolangCI-Lint:
-    name: lint
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
-
-      - name: run golangci-lint
-        run: |
-          docker/run_docker.sh --bsgolint

.github/workflows/check_pull_request.yml

@@ -8,6 +8,9 @@ on:
       - reopened
       - synchronize
 
+permissions:
+  contents: read
+
 jobs:
   check-pr-title:
     name: Check Pull Request Title
@@ -45,31 +48,41 @@ jobs:
   check-commit-message:
     name: Check Commit Message
     runs-on: ubuntu-latest
+    env:
+      JOB_COMMIT_FILE: '/tmp/commits.json'
     steps:
       - name: Get PR Commits
         id: 'get-pr-commits'
-        uses: tim-actions/get-pr-commits@3efc1387ead42029a0d488ab98f24b7452dc3cde # v1.3.0
+        uses: sejust/get-pr-commits@21ca1696fc716fa9423291cddc3d7a82668cfbc2 # v1.3.2
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
+          output-file: ${{ env.JOB_COMMIT_FILE }}
 
       - name: Check Title
-        uses: tim-actions/commit-message-checker-with-regex@094fc16ff83d04e2ec73edb5eaf6aa267db33791 # v0.3.2
+        uses: sejust/commit-message-checker-with-regex@5bedef5c21ee29bb438572bdb715ad65159264f8 # v0.3.3
         with:
-          commits: ${{ steps.get-pr-commits.outputs.commits }}
-          pattern: '^[a-z]+\([a-z]+\): .+\n(\n.*)*$'
+          commits: ${{ env.JOB_COMMIT_FILE }}
+          pattern: '^[a-z]+\([a-z0-9_\-\.]+\): .+\n(\n.*)*$'
           error: 'Title likes `<type>(<scope>): <subject>`'
 
-      - name: Check Subject Line Length
-        uses: tim-actions/commit-message-checker-with-regex@094fc16ff83d04e2ec73edb5eaf6aa267db33791 # v0.3.2
+      - name: Check Title Space
+        uses: sejust/commit-message-checker-with-regex@5bedef5c21ee29bb438572bdb715ad65159264f8 # v0.3.3
         with:
-          commits: ${{ steps.get-pr-commits.outputs.commits }}
+          commits: ${{ env.JOB_COMMIT_FILE }}
+          pattern: '^[^ ]+(?: [^ ]+)*\n(\n.*)*$'
+          error: 'Title has consecutive spaces'
+
+      - name: Check Subject Line Length
+        uses: sejust/commit-message-checker-with-regex@5bedef5c21ee29bb438572bdb715ad65159264f8 # v0.3.3
+        with:
+          commits: ${{ env.JOB_COMMIT_FILE }}
           pattern: '^.{0,100}\n(\n.*)*$'
           error: 'Subject too long (max 100)'
 
       - name: Check Body Line Length
-        uses: tim-actions/commit-message-checker-with-regex@094fc16ff83d04e2ec73edb5eaf6aa267db33791 # v0.3.2
+        uses: sejust/commit-message-checker-with-regex@5bedef5c21ee29bb438572bdb715ad65159264f8 # v0.3.3
         with:
-          commits: ${{ steps.get-pr-commits.outputs.commits }}
+          commits: ${{ env.JOB_COMMIT_FILE }}
           pattern: '^.+\n(\n.{0,100})*$'
           error: 'Body line too long (max 100)'
 

.github/workflows/ci.yml

@@ -16,12 +16,8 @@ on:
       - release-*
       - develop-*
       - blobstore-*
-    paths-ignore:
+    # paths-ignore:
       # - 'blobstore/**'
-      # - '.github/**'
-      # - 'docs/**'
-      # - 'docs-zh/**'
-      # - '**.md'
 
 permissions:
   contents: read
@@ -33,7 +29,7 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
 
-      - name: Find changed files
+      - name: Find changed files of document
         id: changed-files
         uses: tj-actions/changed-files@87697c0dca7dd44e37a2b79a79489332556ff1f3 # v37.6.0
         with:
@@ -43,13 +39,42 @@ jobs:
             docs-zh/**
             **.md
 
+      - name: Find changed files of blobstore
+        id: changed-blobs
+        uses: tj-actions/changed-files@87697c0dca7dd44e37a2b79a79489332556ff1f3 # v37.6.0
+        with:
+          files: |
+            blobstore/**
+
+      - name: All changed documents
+        if: steps.changed-files.outputs.only_changed == 'true'
+        env:
+          CI_ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
+        run: |
+          for file in ${CI_ALL_CHANGED_FILES}; do
+            echo "<$file> was changed"
+          done
+
       - name: Check gofmt
         if: steps.changed-files.outputs.only_changed != 'true'
         run: |
           docker/run_docker.sh --format
 
-      - name: Unit tests
-        if: steps.changed-files.outputs.only_changed != 'true'
+      - name: Unit test for blobstore
+        if: steps.changed-blobs.outputs.only_changed == 'true'
+        run: |
+          docker/run_docker.sh --testblobstore
+
+      - name: Unit test for cubefs
+        if: ${{ (steps.changed-files.outputs.only_changed != 'true') &&
+                (steps.changed-blobs.outputs.any_changed != 'true') }}
+        run: |
+          docker/run_docker.sh --testcubefs
+
+      - name: Unit test for all
+        if: ${{ (steps.changed-files.outputs.only_changed != 'true') &&
+                (steps.changed-blobs.outputs.only_changed != 'true') &&
+                (steps.changed-blobs.outputs.any_changed == 'true') }}
         run: |
           docker/run_docker.sh --test
 

.github/workflows/codeql.yml

@@ -41,11 +41,21 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        language: [ 'java', 'python' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby', 'swift' ]
-        # Use only 'java' to analyze code written in Java, Kotlin or both
-        # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both
-        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+        include:
+        - language: go
+          build-mode: autobuild
+        - language: java-kotlin
+          build-mode: none # This mode only analyzes Java. Set this to 'autobuild' or 'manual' to analyze Kotlin too.
+        - language: python
+          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
 
     steps:
     - name: Checkout repository
@@ -53,9 +63,10 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@a09933a12a80f87b87005513f0abb1494c27a716 # v2.21.4
+      uses: github/codeql-action/init@9e8d0789d4a0fa9ceb6b1738f7e269594bdd67f0 # v3.28.9
       with:
         languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
         # If you wish to specify custom queries, you can do so here or in a config file.
         # By default, queries listed here will override any specified in a config file.
         # Prefix the list here with "+" to use these queries and those in the config file.
@@ -63,12 +74,6 @@ jobs:
         # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
         # queries: security-extended,security-and-quality
 
-
-    # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@a09933a12a80f87b87005513f0abb1494c27a716 # v2.21.4
-
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
 
@@ -80,6 +85,6 @@ jobs:
     #     ./location_of_script_within_repo/buildscript.sh
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@a09933a12a80f87b87005513f0abb1494c27a716 # v2.21.4
+      uses: github/codeql-action/analyze@9e8d0789d4a0fa9ceb6b1738f7e269594bdd67f0 # v3.28.9
       with:
         category: "/language:${{matrix.language}}"

.github/workflows/issue_similarity.yml

@@ -14,7 +14,7 @@ jobs:
       - name: analysis
         uses: actions-cool/issues-similarity-analysis@8f46978e3e8b79d736997a225c95d27d9029f294 # v1.3.1
         with:
-          filter-threshold: 0.6
+          filter-threshold: 0.8
           comment-title: '### See'
           comment-body: '${index}. ${similarity} #${number}'
           show-footer: false

.github/workflows/scorecard.yml

@@ -32,12 +32,12 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
         with:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@08b4669551908b1024bb425080c797723083c031 # v2.2.0
+        uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1
         with:
           results_file: results.sarif
           results_format: sarif
@@ -67,6 +67,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@0ba4244466797eb048eb91a6cd43d5c03ca8bd05 # v2.21.2
+        uses: github/codeql-action/upload-sarif@4355270be187e1b672a7a1c7c7bae5afdc1ab94a # v3.24.10
         with:
           sarif_file: results.sarif

.gitignore

@@ -20,3 +20,8 @@ java/src/main/resources/*.so
 /cover.output
 /cubefs_unittest.output
 .version
+.cache
+.clangd
+blobstore/cpp/build
+blobstore/cpp/**/*.pb.cc
+blobstore/cpp/**/*.pb.h

.goreleaser.yaml

@@ -110,24 +110,6 @@ builds:
       - -X {{.Env.PROTO}}.BuildTime={{.Date}}
       - -X {{.Env.VV}}={{.Branch}}/{{.Date}}
       - -w -s
-  - id: "preload"
-    main: ./preload
-    binary: cfs-preload
-    env:
-      - CGO_ENABLED=0
-    goos:
-      - linux
-    goarch:
-      - amd64
-    flags:
-      - -trimpath
-    ldflags:
-      - -X {{.Env.PROTO}}.Version={{.Version}}
-      - -X {{.Env.PROTO}}.CommitID={{.FullCommit}}
-      - -X {{.Env.PROTO}}.BranchName={{.Branch}}
-      - -X {{.Env.PROTO}}.BuildTime={{.Date}}
-      - -X {{.Env.VV}}={{.Branch}}/{{.Date}}
-      - -w -s
 
   - id: "server"
     main: ./cmd

.semgrep.yml

@@ -8,11 +8,14 @@ rules:
         - '*.go'
     patterns:
       - pattern-regex: '(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)'
+      - pattern-not-regex: '127\.0\.0\.\d+'
       - pattern-not-regex: '10\.\d+\.\d+.\d+'
       - pattern-not-regex: '192\.168\.\d+.\d+'
       - pattern-not-regex: '192\.0\.2\.\d+'    # 192.0.2.0/24 (TEST-NET-1, rfc5737)
       - pattern-not-regex: '198\.51\.100\.\d+' # 198.51.100.0/24 (TEST-NET-2, rfc5737)
       - pattern-not-regex: '203\.0\.113\.\d+'  # 203.0.113.0/24 (TEST-NET-3, rfc5737)
+      - pattern-not-regex: '172\.16\.\d+\.\d+' # 172.16.0.0/12
+      - pattern-not-regex: '169\.254\.\d+\.\d+'# 169.254.0.0/16
     severity: WARNING
   - id: rfc-3849-ip-address
     languages:
@@ -22,5 +25,5 @@ rules:
       include:
         - '*.go'
     patterns:
-      - pattern-regex: '(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))'
-    severity: WARNING
+      - pattern-regex: '(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7})|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))'
+    severity: WARNING

.semgrepignore

@@ -0,0 +1,2 @@
+depends/
+vendor/

ADOPTERS.md

@@ -14,6 +14,7 @@ This page contains a list of organizations who are using CubeFS in production or
 - **[LinkSure Network](https://cn.wifi.com)**: LinkSure uses CubeFS to store application logs running inside container environments as well as nginx logs. Also, they use CubeFS as the backend storage for Elasticsearch.
 - **[Reconova](http://www.reconova.com):** Reconova uses CubeFS to store massive small files in the production environment. It currently uses one CubeFS volume to store more than 80 million small files and each file around 40 kilobytes in size.
 - **[BIGO](https://www.bigo.sg/):** BIGO uses CubeFS to store logs for AI platform applications that running in the container environment, because CubeFS have excellent concurrent processing capability.
+- **[Vipshop](https://www.vip.com/):** Cubefs is used in the offline hybrid deployment scenario of YARN on Kubernetes.After some adjustments, it has become quite stable.
 
 ## Adopters
 
@@ -29,11 +30,19 @@ This page contains a list of organizations who are using CubeFS in production or
 | [LinkSure Network](https://cn.wifi.com) | ![production](https://img.shields.io/badge/-production-blue?style=flat) |
 | [BIGO LIVE](https://www.bigo.tv/cn/) | ![production](https://img.shields.io/badge/-production-blue?style=flat) |
 | [Xiaomi](https://www.mi.com/global/) | ![production](https://img.shields.io/badge/-production-blue?style=flat) |
+| [shengwang.cn](https://www.shengwang.cn/) | ![production](https://img.shields.io/badge/-production-blue?style=flat) |
+| [Vipshop](https://www.vip.com/) | ![production](https://img.shields.io/badge/-production-blue?style=flat) |
+| [CreditEase](https://www.creditease.com/) | ![production](https://img.shields.io/badge/-production-blue?style=flat) |
+| [TD Tech](https://www.td-tech.com/) | ![production](https://img.shields.io/badge/-production-blue?style=flat) |
+| [Digtal guangdong](https://www.digitalgd.com.cn/) | ![production](https://img.shields.io/badge/-production-blue?style=flat) |
 | [PITS Global Data Recovery Services](https://www.pitsdatarecovery.net/) | ![testing](https://img.shields.io/badge/-testing-green?style=flat) |
 | [Da-Jiang Innovations](https://www.dji.com/cn) | ![testing](https://img.shields.io/badge/-testing-green?style=flat) |
-| [Yanhuang Data](https://yanhuangdata.com/) | ![testing](https://img.shields.io/badge/-testing-green?style=flat) |
 | [Sinosoft](http://www.sinosoft.com.cn) | ![testing](https://img.shields.io/badge/-testing-green?style=flat) |
 | [DADA](https://about.imdada.cn) | ![testing](https://img.shields.io/badge/-testing-green?style=flat) |
 | [Club Factory](https://www.wholeeprime.com/) | ![testing](https://img.shields.io/badge/-testing-green?style=flat) |
 | [Nanjing University](https://www.nju.edu.cn/en/) | ![testing](https://img.shields.io/badge/-testing-green?style=flat) |
 | [PSBC](https://www.psbc.com/) | ![testing](https://img.shields.io/badge/-testing-green?style=flat) |
+| [DeepMirror](https://deepmirror.vercel.app/) | ![testing](https://img.shields.io/badge/-testing-green?style=flat) |
+| [shoppe](https://shopee.com/) | ![testing](https://img.shields.io/badge/-testing-green?style=flat) |
+| [houdutech](https://www.houdutech.cn) | ![testing](https://img.shields.io/badge/-testing-green?style=flat) |
+| [cecloud](https://www.cecloud.com/) | ![testing](https://img.shields.io/badge/-testing-green?style=flat) |

CHANGELOG.md

@@ -1,3 +1,159 @@
+## Release v3.5.3 - 2025/12/31
+
+### **UPGRAGDE NOTICE**
+
+If you are using a CubeFS version earlier than v3.5.0, please refer to the UPGRADE NOTICE in version v3.5.0 for detailed upgrade steps and upgrade to v3.5.0 first.
+
+Upgrade nodes in this order: flshnode → master → datanode → metanode → objectnode → lcnode → cli → client.
+
+Upgrade lcnode and  flashnode when needed.
+
+Deploy flashgroupmanager when needed
+
+Clients should use versions later than 3.2.0. Older versions need to be upgraded promptly; otherwise, there will be a risk of compromising stability.
+
+### **Main Feature**
+
+#### High-throughput LLM/MLLM training with 8ms+ compute–storage latency tolerance
++ `client`: Support asynchronous flush for extent handler to improve write performance. Write speed exceeds 1.2 GB/s; on a high-spec H20 training node, a single client can achieve 10+ GB/s aggregate throughput with 10 concurrent large-file writes.(#3973,@bboyCH4)
++ `client`: Optimize the client read-ahead mechanism and memory footprint; single-file read speeds exceed 2 GB/s. (#3982,@bboyCH4)
++ `client`: Metadata cache acceleration for small-file prewarm (#3995,@Victor1319)
+
+Note: Refer to the latest community documentation for enabling and tuning.
+
+#### Distributed cache can run as an independent service
++ `flashgroupmanager`: Introduce flashgroupmanager node and topology to support flashnode cluster management. (@bboyCH4)
++ `flashnode`: Support block-level data read and write operations.  (#3977, @clinx)
++ `tools`: Add `rctest` (benchmark) and `rcconfig` (config) tools for remote cache system. (#3981,@bboyCH4,@clinx)
++ `client`: Provide SDK for FlashNode object storage data block upload/download service （#3985,@bboyCH4,@clinx）
++ `client`: Implement NearRead strategy to prioritize reading from the nearest replica to reduce latency. (#3976,@zhumingze1108)
+
+### **Enhance**
++ `client`: Fuse library supports parallel processing of FUSE requests to improve concurrency. (#3974,@Victor1319)
++ `client`: Optimize metadata cache performance.  (#3974,@Victor1319)
++ `client`: Add `tcpAliveTime` parameter for better TCP connection management. (#3974,@Victor1319)
++ `master`: Support DP decommission status evolution history query. (#3987,@shuqiang-zheng)
++ `master`: Add `TryDecommissionRunningDiskIgnoreDps` to support differentiated strategies for disk decommission based on different reasons. (#3975,@shuqiang-zheng)
++ `master`: Add audit logs for `migrateMetaPartition` and record reasons for DP migration/rollback. (#3975,@shuqiang-zheng)
++ `datanode`: Support reason passthrough for DP migration. (#3975,@shuqiang-zheng)
++ `flashnode`: Optimize cache operation opcodes and processing logic. (#3988,@clinx,@bboyCH4)
+
+
+
+### **Bugfix**
+* `master`: Decommission token consumed twice on restart during two-replica DP decommissioning(#3978,@shuqiang-zheng)
+* `client`: Fix `ltp iogen01` test failure when pre-reading (ahead read) is enabled. (#3980,@clinx)
+* `client`: Offset calculation error during client readahead with partial hits(#3979,@bboyCH4)
+* `master`: Some DPs remain in decommission queue when disk offline marking fails, affecting subsequent decommissions (#3983,@shuqiang-zheng)
+* `master`: Incorrect disk/node decommission progress display for 2-replica DPs due to leader change(#3984,@Victor1319)
+
+
+## Release v3.5.2 - 2025/07/31
+
+### **UPGRAGDE NOTICE**
+
+If you are using a CubeFS version earlier than v3.5.0, please refer to the UPGRADE NOTICE in version v3.5.0 for detailed upgrade steps and upgrade to v3.5.0 first.
+
+Upgrade nodes in this order: master → metanode → datanode → objectnode → cli → client.
+
+Upgrade lcnode and deploy flashnode when needed.
+
+Clients should use versions later than 3.2.0. Older versions need to be upgraded promptly; otherwise, there will be a risk of compromising stability.
+
+### **Main Feature**
+
+####  For large language models (LLMs) and multimodal LLM (MLLM) training, delivers high throughput (LLM checkpoints) and tolerates high-latency compute–storage separation (8 ms+), achieving training durations comparable to public cloud deployments in the same region
+
+
+
+
+
+
+
+
++ `master/lcnode`: Lifecycle adds filtering rule based on file size. (#3893, @Victor1319)
++ `master`: dp decommission support priority&concurrency control. (#3891, @shuqiang-zheng)
++ `master`: Add master replica abnormality alarm. (#3882, @zhumingze1108)
++ `master`: Add disk decommission success alarm. (#3886, @zhumingze1108)
++ `meta`: Support mp reload capability. (#3894, @leonrayang)
++ `meta`: Volume file size distribution statistics. (#3884, @M1eyu2018,  @zhumingze1108)
++ `client`: Implement client pre-reading function. (#3889, @yanbin027, @Victor1319)
++ `client`: Client delay monitoring statistics. (#3885, @M1eyu2018,  @aaronwu2010)
++ `data`: Bad disk detection and lost disk discovery. (#3878, @zhumingze1108)
++ `data`: Support asynchronous limitio restrictions. (#3881, @zhumingze1108)
++ `master/data/meta/cli`: dp and mp read-only reasons display. (#3880, @zhumingze1108)
+
+### **Enhance**
++ `all`: Remove the part of the code that uses datanode as cache. (#3888, @Victor1319)
++ `master`: DataNode&Disk&dp Decommission logic optimization. (#3891, @shuqiang-zheng, @zhumingze1108)
++ `meta`: Optimize metanode memory usage. (#3892, @Victor1319)
++ `data`: Optimize datanode memory usage. (#3887, @aaronwu2010)
++ `data`: Optimize and repair the process of tinyDeleteRecord synchronization logic. (#3890, @Victor1319)
++ `cli`: cli datapartition check display optimization. (#3879, @zhumingze1108)
+
+### **Bugfix**
+* `master/data`: Repair process blocked by host0 replica. (@shuqiang-zheng)
+* `master/raft`: Fixed the issue of dp no leader caused by failure to add raft members during decommission process. (@shuqiang-zheng)
+
+## Release v3.5.1 - 2025/05/28
+
+### **UPGRAGDE NOTICE**
+
+
+If you are using a CubeFS version earlier than v3.5.0, please refer to the UPGRADE NOTICE in version v3.5.0 for detailed upgrade steps and upgrade to v3.5.0 first.
+
+
+### **Main Feature**
++ `all`: flash cache in cluster. （#2943 @bboyCH4, @longerfly, @slasher, @shuqiang-zheng, @clinx）
++ `flash`: scale out the cache layer by adding more cache nodes to handle increased read traffic.
++ `master`: save the FlashNode topology state and push FlashNode topology data to the client.
++ `client`: data reads are routed to the appropriate cache node based on consistent hashing.
++ `cli`: use CLI commands to query the current cache status and control its behavior.
+
+### **Enhance**
++ `data/meta`: support for dynamic adjustment of gogc. (#3816, @shuqiang-zheng)
++ `client`: actively release part of the client's memory to reduce memory footprint. (@bboyCH4)
++ `client`: support reading data with quorum consistency. (@zhumingze1108)
+
+### **Bugfix**
+* `meta`: tune the retry mechanism for failed volume creation to minimize the impact on volume deletion performance. (@bboyCH4)
+* `data`: no longer allow single replica dp raftForce deletion. (@zhumingze1108)
+* `data`: add CRC check for extent ID allocation. (@leonrayang)
+* `client`: monitor already contains grouping label commit. (@zhumingze1108)
+* `client`: failure to update the local extent cache generation resulted in an LTP failure.(@bboyCH4)
+* `flash`: Removing an fn immediately after a single 200ms timeout on origin fetch is too sensitive. A better approach would be to remove it only after multiple consecutive timeouts. (@longerfly)
+* `object`: copy data between different buckets. (@clinx)
+
+
+
+## Release v3.5.0 - 2025/03/13
+
+### **UPGRAGDE NOTICE**
+
+If you are using a CubeFS version earlier than v3.4.0, please refer to the UPGRADE NOTICE in version v3.4.0 for detailed upgrade steps and upgrade to v3.4.0 first.
+
+### **Attention:**
+If you deployed the cluster based on an older version, please refer to the documentation [upgrade 3.5.0](https://github.com/cubefs/cubefs/wiki/CubeFS-v3.5.0-upgrade-manual) for upgrade steps to version 3.5.+
+
+### **Main Feature**
++ `all`: Supports the capability to manage different storage media. （#3603，@bboyCH4, @true1064, @Victor1319 ）
++ `master/lcnode`: Supports automatic migration of cold data via lifecycle management . (#3604  @bboyCH4, @honeyvinnie, @Victor1319)
++ `master/client`: Support querying all client versions and IP information. (#3606 ,  @Victor1319)
++ `datanode`: Support for direct I/O read operations with volume. (#3630, @Victor1319)
+
+### **Enhance**
++ `bcache`: bcache supports configuration switches that work for non-SSD types. (#3607, @longerfly)
++ `sdk`: Support the ability of the tool to count the size of the directory by access time. (#3608, @longerfly)
++ `sdk`: Use distributed locks to prevent concurrent deletions from multiple clients in the recycle bin. (#3610, @Victor1319)
++ `sdk/datanode`: Optimize data read path performance for storage and compute separation scenarios. (#3631, @Victor1319)
++ `sdk/metanode`: Support metadata reads in a leaderless environment using quorum mode and meta follower mode. (#3632, @Victor1319)
+
+### **Bugfix**
++ `raft`: Conflicts between Raft metadata and WAL logs (#3605, @Victor1319）
++ `sdk`: Deleting files can cause client panic when the recycle bin is enabled (#3609, @bboyCH4)
++ `all`: Fix bugs related to historical version faults and anomalies. ( @Victor1319）
+
+
 ## Release v3.4.0 - 2024/10/28
 
 ### **UPGRAGDE NOTICE**
@@ -187,7 +343,7 @@ If your Blobstore version is v1.1.0 or before which built with cubefs-blobstore
 
 ### **Bugfix**
 * `master`: master snapshot recover not reset local rocksdb info (#1522, @wuchunhuan )
-* `master`:Memory cost too fast during restart in case of data partition‘s count is magnity (#1774 , @leonrayang)
+* `master`:Memory cost too fast during restart in case of data partition's count is magnity (#1774 , @leonrayang)
 * `client`: Readonly dp can still accept write request from client (#1779, @bboyCH4)
 * `metanode`: Metanode should not establish connection to blobstore for cold volume (#1781, @bboyCH4)
 * `client`: blockcache service may be oom when the client caches many large files concurrently (#1783, @zhangtianjiong)
@@ -472,7 +628,7 @@ https://zhuanlan.zhihu.com/p/28417779
   faultDomainGrpBatchCnt，default count:3，can also set 2 or 1
     
   If zone is unavaliable caused by network partition interruption，create nodeset group according to usable zone
-Set “faultDomainBuildAsPossible”  true, default is false
+Set "faultDomainBuildAsPossible"  true, default is false
 
   The distribution of nodesets under the number of different faultDomainGrpBatchCnt
   3 zone（1 nodeset per zone）
@@ -509,7 +665,7 @@ Set “faultDomainBuildAsPossible”  true, default is false
 **3.  Note**
 **1) After the fault domain is enabled, all devices in the new zone will join the fault domain**
 **2) The created volume will preferentially select the resources of the original zone**
-**3) Need add configuration items to use domain resources when creating a new volume according to the table below. By default, the original  zone resources are used first if it’s avaliable**
+**3) Need add configuration items to use domain resources when creating a new volume according to the table below. By default, the original  zone resources are used first if it's avaliable**
 
 | Cluster:faultDomain | Vol:crossZone | Vol:defaultPriority | Rules for volume to use domain |
 h| ------ | ------ | ------ |------ |
@@ -525,16 +681,14 @@ example :` curl "http://10.177.200.119:17010/admin/createVol?name=vol_cross5&cap
 ### **Content Summary**
 
 **1. Purpose**
-In order to query the content summary information of a directory efficiently, e.g. total file size, total files and total directories, v2.5 stores such information as the parent directory’s xattr. 
+In order to query the content summary information of a directory efficiently, e.g. total file size, total files and total directories, v2.5 stores such information as the parent directory's xattr. 
 
 The parent directory stores the files, directories and total file size of the current directory. Then only need to make recursive of the sub directories, and accumulate the information stored by the directories to query the content summary information of a directory.
 
 **2. Configuration**
   Client config file: fuse.json
   **1) Enable XAttr**
-    ”enableXattr”:”true”
-  **2) Enable Summary**
-    ”enableSummary”:”true”
+    "enableXattr":"true"
   Both of xattr and summay have to be set if you want to mount a volume to the local disk. 
   Set summary is enough if you want to access the volume via libsdk.so.
 
@@ -547,7 +701,7 @@ The parent directory stores the files, directories and total file size of the cu
     cfs_getsummary (libsdk/libsdk.go)
 
   **4. Note**
-  1)The incremental files’ summary information will be held by their parent directories. But the old files will not. Use cfs_refreshsummary 
+  1)The incremental files' summary information will be held by their parent directories. But the old files will not. Use cfs_refreshsummary 
     (libsdk/libsdk.go) interface to rebuild the content summary information.
   2)The files, directories and total file size are updated asynchronously in the background. Users are not aware of these operations, but it does 
     increase the requests to meta servers (usually doubled). You are recommended to evaluate the impact to your cluster before using this 
@@ -791,7 +945,7 @@ Release2.1.0 did a lot of work to optimize memory usage.
 * `object`: Change from hard link to soft link in **CopyObject** action. [#563](https://github.com/cubefs/cubefs/pull/563)
 * `object`: Solved **parallel-safety** issue; Clean up useless data on failure in **upload** part. [#553](https://github.com/cubefs/cubefs/pull/553)
 * `object`: Fixed a problem in listing multipart uploads. [#595](https://github.com/cubefs/cubefs/pull/595) 
-* `object`: Solve the problem that back-end report “NotExistErr” error when uploading files with the same key in parallel. [#685](https://github.com/cubefs/cubefs/pull/685)
+* `object`: Solve the problem that back-end report "NotExistErr" error when uploading files with the same key in parallel. [#685](https://github.com/cubefs/cubefs/pull/685)
 * `fuse`: Evict inode cache when dealing with forget. [#523](https://github.com/cubefs/cubefs/pull/523)
 
 ### Document

MAINTAINERS.md

@@ -1,63 +1,63 @@
 # Technical Steering Committee(TSC)
 
-| Name                                                                   | Email                                                           | Organization |
-|------------------------------------------------------------------------|-----------------------------------------------------------------|--------------|
-| Haifeng Liu ([@bladehliu](https://github.com/bladehliu))               | [bladehliu@qq.com](mailto:bladehliu@qq.com)                     | [Individual]            |
-| Weilong Guo ([@awzhgw](https://github.com/awzhgw))                     | [guowl18702995996@gmail.com](mailto:guowl18702995996@gmail.com) | [JD.com]     |
-| Xiaochun He ([@xiaochunhe](https://github.com/xiaochunhe))             | [626148589@qq.com](mailto:626148589@qq.com)                     | [OPPO]       |
-| Mofei Zhang ([@mervinkid](https://github.com/mervinkid))               | [mofei2816@gmail.com](mailto:mofei2816@gmail.com)               | [JD.com]     |
-| Liang Chang ([@leonrayang](https://github.com/leonrayang))             | [chl696@sina.com](mailto:chl696@sina.com)                       | [OPPO]       |
+| Name                                                       | Email                                                           | Organization |
+| ---------------------------------------------------------- | --------------------------------------------------------------- | ------------ |
+| Haifeng Liu ([@bladehliu](https://github.com/bladehliu))   | [bladehliu@gmail.com](mailto:bladehliu@gmail.com)                  | [Individual] |
+| Weilong Guo ([@awzhgw](https://github.com/awzhgw))         | [guowl18702995996@gmail.com](mailto:guowl18702995996@gmail.com) | [JD.com]     |
+| Xiaochun He ([@xiaochunhe](https://github.com/xiaochunhe)) | [626148589@qq.com](mailto:626148589@qq.com)                     | [OPPO]       |
+| Mofei Zhang ([@mervinkid](https://github.com/mervinkid))   | [mofei2816@gmail.com](mailto:mofei2816@gmail.com)               | [JD.com]     |
+| Liang Chang ([@leonrayang](https://github.com/leonrayang)) | [chl696@sina.com](mailto:chl696@sina.com)                       | [OPPO]       |
 
 # Maintainers
 
-| Name                                                                   | Email                                                           | Organization |
-|------------------------------------------------------------------------|-----------------------------------------------------------------|--------------|
-| Haifeng Liu ([@bladehliu](https://github.com/bladehliu))               | [bladehliu@qq.com](mailto:bladehliu@qq.com)                     | -            |
-| Weilong Guo ([@awzhgw](https://github.com/awzhgw))                     | [guowl18702995996@gmail.com](mailto:guowl18702995996@gmail.com) | [JD.com]     |
-| Shuoran Liu ([@shuoranliu](https://github.com/shuoranliu))             | [shuoranliu@gmail.com](mailto:shuoranliu@gmail.com)             | [BEIKE]      |
-| Xiaochun He ([@xiaochunhe](https://github.com/xiaochunhe))             | [626148589@qq.com](mailto:626148589@qq.com)                     | [OPPO]       |
-| Mofei Zhang ([@mervinkid](https://github.com/mervinkid))               | [mofei2816@gmail.com](mailto:mofei2816@gmail.com)               | [JD.com]     |
-| Liang Chang ([@leonrayang](https://github.com/leonrayang))             | [chl696@sina.com](mailto:chl696@sina.com)                       | [OPPO]       |
-| Xuewei Zeng ([@Victor1319](https://github.com/Victor1319))             | [834863182@qq.com](mailto:834863182@qq.com)                     | [OPPO]       |
-| Dr. Wei Ding ([@wding109](https://github.com/wding109))                 | [wding109@gmail.com](mailto:wding109@gmail.com)                 | [ByteDance]  |
-| Dr. Junyuan Zeng ([@jzeng4](https://github.com/jzeng4))                 | [jzeng04@gmail.com](mailto:jzeng04@gmail.com)                  | [LinkedIn]   |
-| hooklee2000 ([@hooklee2000](https://github.com/hooklee2000))           | [hooklee2000@gmail.com](mailto:hooklee2000@gmail.com)           | [XFusion]    | 
-| Zhendong Li ([@lizhendong666](https://github.com/lizhendong666))       | [lizhendong666@gmail.com](mailto:lizhendong666@gmail.com)       | [JD.com]     |
-| Xiaobo Yu ([@cessory](https://github.com/cessory))                     | [yxbstorm@gmail.com](mailto:yxbstorm@gmail.com)                 | [OPPO]       |
-| Cloudstriff ([@Cloudstriff](https://github.com/Cloudstriff))           | [chenjiongwendao@qq.com](mailto:chenjiongwendao@qq.com)         | [OPPO]       |
-| slasher ([@sejust](https://github.com/sejust))                         | [mcq.sejust@gmail.com](mailto:mcq.sejust@gmail.com)             | [OPPO]       |
+| Name                                                             | Email                                                           | Organization |
+| ---------------------------------------------------------------- | --------------------------------------------------------------- | ------------ |
+| Haifeng Liu ([@bladehliu](https://github.com/bladehliu))         | [bladehliu@gmail.com](mailto:bladehliu@gmail.com)                  | [Individual] |
+| Weilong Guo ([@awzhgw](https://github.com/awzhgw))               | [guowl18702995996@gmail.com](mailto:guowl18702995996@gmail.com) | [JD.com]     |
+| Shuoran Liu ([@shuoranliu](https://github.com/shuoranliu))       | [shuoranliu@gmail.com](mailto:shuoranliu@gmail.com)             | [BEIKE]      |
+| Xiaochun He ([@xiaochunhe](https://github.com/xiaochunhe))       | [626148589@qq.com](mailto:626148589@qq.com)                     | [OPPO]       |
+| Mofei Zhang ([@mervinkid](https://github.com/mervinkid))         | [mofei2816@gmail.com](mailto:mofei2816@gmail.com)               | [JD.com]     |
+| Liang Chang ([@leonrayang](https://github.com/leonrayang))       | [chl696@sina.com](mailto:chl696@sina.com)                       | [OPPO]       |
+| Xuewei Zeng ([@Victor1319](https://github.com/Victor1319))       | [834863182@qq.com](mailto:834863182@qq.com)                     | [OPPO]       |
+| Dr. Wei Ding ([@wding109](https://github.com/wding109))          | [wding109@gmail.com](mailto:wding109@gmail.com)                 | [ByteDance]  |
+| Dr. Junyuan Zeng ([@jzeng4](https://github.com/jzeng4))          | [jzeng04@gmail.com](mailto:jzeng04@gmail.com)                   | [LinkedIn]   |
+| hooklee2000 ([@hooklee2000](https://github.com/hooklee2000))     | [hooklee2000@gmail.com](mailto:hooklee2000@gmail.com)           | [XFusion]    |
+| Zhendong Li ([@lizhendong666](https://github.com/lizhendong666)) | [lizhendong666@gmail.com](mailto:lizhendong666@gmail.com)       | [JD.com]     |
+| Xiaobo Yu ([@cessory](https://github.com/cessory))               | [yxbstorm@gmail.com](mailto:yxbstorm@gmail.com)                 | [OPPO]       |
+| Cloudstriff ([@Cloudstriff](https://github.com/Cloudstriff))     | [chenjiongwendao@qq.com](mailto:chenjiongwendao@qq.com)         | [OPPO]       |
+| slasher ([@sejust](https://github.com/sejust))                   | [mcq.sejust@gmail.com](mailto:mcq.sejust@gmail.com)             | [OPPO]       |
 
 # Committers
 
-| Name                                                                            | Email                                                            | Organization |
-|---------------------------------------------------------------------------------|------------------------------------------------------------------|--------------|
-| Hongyan Wang ([@jadewang198510](https://github.com/jadewang198510))             | [741773046@qq.com](mailto:741773046@qq.com)                      | [OPPO]       |
-| Chi He ([bboyCH4](https://github.com/bboyCH4))                                  | [hechi1014@126.com](mailto:hechi1014@126.com)                    | [OPPO]       |
-| Jianxing Zhao ([@znlstar](https://github.com/znlstar))                          | [znlstar@163.com](mailto:znlstar@163.com)                        | [JD.com]     |
-| Yong Sheng ([@shyodx](https://github.com/shyodx))                               | [shengyong2021@gmail.com](mailto:shengyong2021@gmail.com)        | [BEIKE]      |
-| Zhengyi Zhu ([@zhuzhengyi](https://github.com/wding109))                        | [zhengyi.zhu.hust@gmail.com](mailto:zhengyi.zhu.hust@gmail.com)  | [BEIKE]      |
-| Lei Yin ([@yinlei-jinan](https://github.com/yinlei-jinan))                     | [297155992@qq.com](mailto:297155992@qq.com)                      | [JD.com]     |
-| Liying Zhang ([@Vivian7755](https://github.com/Vivian7755))                     | [zly7755@163.com](mailto:zly7755@163.com)                        | [JD.com]     |
-| Xihao Xu ([@xxscott](https://github.com/xxscott))                               | [xxscott@163.com](mailto:xxscott@163.com)                        | [JD.com]     |
-| Wenjia Wu ([@wenjia322](https://github.com/wenjia322))                          | [buaa1214wwj@126.com](mailto:buaa1214wwj@126.com)                | [JD.com]     |
-| pengtianyue ([@pengtianyue025](https://github.com/pengtianyue025))              | [pengtianyue025@gmail.com](mailto:pengtianyue025@gmail.com)      | [ByteDance]  |
-| baijiaruo ([@baijiaruo](https://github.com/baijiaruo))                          | [505892459@qq.com](mailto:505892459@qq.com)                      | [China United Telecommunications Co] |
-| Tianjiong Zhang ([@tianjiongzhang](https://github.com/tianjiongzhang))          | [236556116@qq.com](mailto:236556116@qq.com)                      | [Sangfor]    |
-| Zongchao Hu ([@JasonHu520](https://github.com/JasonHu520))                      | [hastyjason500@gmail.com](mailto:hastyjason500@gmail.com)        | [OPPO]       |
-| Tianpeng Li ([@Skypigltp](https://github.com/skypigltp))                        | [skypigltp@gmail.com](mailto:skypigltp@gmail.com)                | [VIVO]       |
-| Hongyin Zhu ([@zhuhyc](https://github.com/zhuhyc))                              | [zzhniy.163.niy@163.com](mailto:zzhniy.163.niy@163.com)          | [JD.com]     |
-| Zhixiang Tang ([@xiangcai1215](https://github.com/xiangcai1215))                | [505892459@qq.com](mailto:505892459@qq.com)                      | [Xiaohongshu]|
-| Yubo Li ([@yuboLee](https://github.com/yuboLee))                                | [pangbolee@gmail.com](mailto:pangbolee@gmail.com)                | [JD.com]     |
-| Tao Li ([@tomscut](https://github.com/tomscut))                                 | [tomleescut@gmail.com](mailto:tomleescut@gmail.com)              | [BIGO]       |
-| Junhao Guo ([@M1eyu2018](https://github.com/M1eyu2018))                         | [857037797@qq.com](mailto:857037797@qq.com)                      | [BIGO]       |
-| Bingxing Liu ([@liubingxing](https://github.com/liubingxing))                   | [liubbingxing@gmail.com](mailto:liubbingxing@gmail.com)          | [BIGO]       |
-| Xiang Li ([@lixiang](https://github.com/lixiang))                               | [960754123@qq.com](mailto:960754123@qq.com)                      | [Sangfor]    |
-| Qing Li ([@qingli](https://github.com/liqingqiya))                              | [liqing.qiya@gmail.com](mailto:liqing.qiya@gmail.com)            | [ByteDance]  |
-| Zhihao Wang ([@Cresc](https://github.com/zhihao-wang))                          | [wzh07@hotmail.com](mailto:liqing.qiya@gmail.com)                | [ByteDance]  |
-| NaturalSelect ([@NaturalSelect](https://github.com/NaturalSelect))               | [2145973003@qq.com](mailto:2145973003@qq.com)                    | [Chengdu University of Information Technology] |
-| setcy ([@setcy](https://github.com/setcy))                                      | [asetcy@gmail.com](mailto:asetcy@gmail.com)                      | [Hangzhou Dianzi University] |
-| Shuqiang Zheng ([@shuqiang-zheng](https://github.com/shuqiang-zheng))           | [782879301@qq.com](mailto:782879301@qq.com)                      | [OPPO]       |
-| Chuanqing Zhang ([@zhangchuanqing5658](https://github.com/zhangchuanqing5658))  | [zhang691753540@gmail.com](mailto:zhang691753540@gmail.com)      | [JD.com]     |
+| Name                                                                           | Email                                                           | Organization                                   |
+| ------------------------------------------------------------------------------ | --------------------------------------------------------------- | ---------------------------------------------- |
+| Hongyan Wang ([@jadewang198510](https://github.com/jadewang198510))            | [741773046@qq.com](mailto:741773046@qq.com)                     | [OPPO]                                         |
+| Chi He ([bboyCH4](https://github.com/bboyCH4))                                 | [hechi1014@126.com](mailto:hechi1014@126.com)                   | [OPPO]                                         |
+| Jianxing Zhao ([@znlstar](https://github.com/znlstar))                         | [znlstar@163.com](mailto:znlstar@163.com)                       | [JD.com]                                       |
+| Yong Sheng ([@shyodx](https://github.com/shyodx))                              | [shengyong2021@gmail.com](mailto:shengyong2021@gmail.com)       | [BEIKE]                                        |
+| Zhengyi Zhu ([@zhuzhengyi](https://github.com/wding109))                       | [zhengyi.zhu.hust@gmail.com](mailto:zhengyi.zhu.hust@gmail.com) | [BEIKE]                                        |
+| Lei Yin ([@yinlei-jinan](https://github.com/yinlei-jinan))                     | [297155992@qq.com](mailto:297155992@qq.com)                     | [JD.com]                                       |
+| Liying Zhang ([@Vivian7755](https://github.com/Vivian7755))                    | [zly7755@163.com](mailto:zly7755@163.com)                       | [JD.com]                                       |
+| Xihao Xu ([@xxscott](https://github.com/xxscott))                              | [xxscott@163.com](mailto:xxscott@163.com)                       | [JD.com]                                       |
+| Wenjia Wu ([@wenjia322](https://github.com/wenjia322))                         | [buaa1214wwj@126.com](mailto:buaa1214wwj@126.com)               | [JD.com]                                       |
+| pengtianyue ([@pengtianyue025](https://github.com/pengtianyue025))             | [pengtianyue025@gmail.com](mailto:pengtianyue025@gmail.com)     | [ByteDance]                                    |
+| baijiaruo ([@baijiaruo](https://github.com/baijiaruo))                         | [505892459@qq.com](mailto:505892459@qq.com)                     | [China United Telecommunications Co]           |
+| Tianjiong Zhang ([@tianjiongzhang](https://github.com/tianjiongzhang))         | [236556116@qq.com](mailto:236556116@qq.com)                     | [Sangfor]                                      |
+| Zongchao Hu ([@JasonHu520](https://github.com/JasonHu520))                     | [hastyjason500@gmail.com](mailto:hastyjason500@gmail.com)       | [OPPO]                                         |
+| Tianpeng Li ([@Skypigltp](https://github.com/skypigltp))                       | [skypigltp@gmail.com](mailto:skypigltp@gmail.com)               | [VIVO]                                         |
+| Hongyin Zhu ([@zhuhyc](https://github.com/zhuhyc))                             | [zzhniy.163.niy@163.com](mailto:zzhniy.163.niy@163.com)         | [JD.com]                                       |
+| Zhixiang Tang ([@xiangcai1215](https://github.com/xiangcai1215))               | [505892459@qq.com](mailto:505892459@qq.com)                     | [Xiaohongshu]                                  |
+| Yubo Li ([@yuboLee](https://github.com/yuboLee))                               | [pangbolee@gmail.com](mailto:pangbolee@gmail.com)               | [JD.com]                                       |
+| Tao Li ([@tomscut](https://github.com/tomscut))                                | [tomleescut@gmail.com](mailto:tomleescut@gmail.com)             | [BIGO]                                         |
+| Junhao Guo ([@M1eyu2018](https://github.com/M1eyu2018))                        | [857037797@qq.com](mailto:857037797@qq.com)                     | [BIGO]                                         |
+| Bingxing Liu ([@liubingxing](https://github.com/liubingxing))                  | [liubbingxing@gmail.com](mailto:liubbingxing@gmail.com)         | [BIGO]                                         |
+| Xiang Li ([@lixiang](https://github.com/lixiang))                              | [960754123@qq.com](mailto:960754123@qq.com)                     | [Sangfor]                                      |
+| Qing Li ([@qingli](https://github.com/liqingqiya))                             | [liqing.qiya@gmail.com](mailto:liqing.qiya@gmail.com)           | [ByteDance]                                    |
+| Zhihao Wang ([@Cresc](https://github.com/zhihao-wang))                         | [wzh07@hotmail.com](mailto:liqing.qiya@gmail.com)               | [ByteDance]                                    |
+| NaturalSelect ([@NaturalSelect](https://github.com/NaturalSelect))             | [2145973003@qq.com](mailto:2145973003@qq.com)                   | [Chengdu University of Information Technology] |
+| setcy ([@setcy](https://github.com/setcy))                                     | [asetcy@gmail.com](mailto:asetcy@gmail.com)                     | [Hangzhou Dianzi University]                   |
+| Shuqiang Zheng ([@shuqiang-zheng](https://github.com/shuqiang-zheng))          | [782879301@qq.com](mailto:782879301@qq.com)                     | [OPPO]                                         |
+| Chuanqing Zhang ([@zhangchuanqing5658](https://github.com/zhangchuanqing5658)) | [zhang691753540@gmail.com](mailto:zhang691753540@gmail.com)     | [JD.com]                                       |
 
 
 [OPPO]: https://www.oppo.com/en/

Makefile

@@ -8,8 +8,8 @@ default: all
 phony := all
 all: build
 
-phony += build server authtool client cli libsdkpre libsdk fsck fdstore preload bcache blobstore deploy
-build: server authtool client cli libsdk fsck fdstore preload bcache blobstore deploy
+phony += build server authtool client cli libsdkpre libsdk fsck fdstore bcache blobstore deploy
+build: server authtool client cli libsdk fsck fdstore bcache blobstore deploy
 
 server:
 	@build/build.sh server $(GOMOD) --threads=$(threads)
@@ -22,6 +22,9 @@ deploy:
 blobstore:
 	@build/build.sh blobstore $(GOMOD) --threads=$(threads)
 
+blobstoredialtest:
+	@build/build.sh blobstoredialtest $(GOMOD) --threads=$(threads)
+
 client:
 	@build/build.sh client $(GOMOD) --threads=$(threads)
 
@@ -43,12 +46,15 @@ libsdk:
 fdstore:
 	@build/build.sh fdstore $(GOMOD) --threads=$(threads)
 
-preload:
-	@build/build.sh preload $(GOMOD) --threads=$(threads)
-
 bcache:
 	@build/build.sh bcache $(GOMOD) --threads=$(threads)
 
+rctest:
+	@build/build.sh rctest $(GOMOD) --threads=$(threads)
+
+rcconfig:
+	@build/build.sh rcconfig $(GOMOD) --threads=$(threads)
+
 phony += clean
 clean:
 	@$(RM) -rf build/bin
@@ -61,9 +67,13 @@ phony += test
 test:
 	@build/build.sh test $(GOMOD) --threads=$(threads)
 
-phony += testcover
+phony += testcover testcovercubefs testcoverblobstore
 testcover:
 	@build/build.sh testcover $(GOMOD) --threads=$(threads)
+testcovercubefs:
+	@build/build.sh testcovercubefs $(GOMOD) --threads=$(threads)
+testcoverblobstore:
+	@build/build.sh testcoverblobstore $(GOMOD) --threads=$(threads)
 
 phony += mock
 mock:

OWNERS.md

@@ -0,0 +1,6 @@
+code approvers:
+  - maintainers
+code reviewers:
+  - contributors && maintainers
+docs:
+  - sig-docs

README.md

@@ -1,6 +1,6 @@
 # CubeFS
 
-[![CNCF Status](https://img.shields.io/badge/cncf%20status-incubating-blue.svg)](https://www.cncf.io/projects)
+[![CNCF Status](https://img.shields.io/badge/cncf%20status-graduated-blue.svg)](https://www.cncf.io/projects)
 [![Build Status](https://github.com/cubefs/cubefs/actions/workflows/ci.yml/badge.svg)](https://github.com/cubefs/cubefs/actions/workflows/ci.yml)
 [![LICENSE](https://img.shields.io/github/license/cubefs/cubefs.svg)](https://github.com/cubefs/cubefs/blob/master/LICENSE)
 [![Language](https://img.shields.io/badge/Language-Go-blue.svg)](https://golang.org/)
@@ -14,6 +14,7 @@
 [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fcubefs%2Fcubefs.svg?type=shield&issueType=security)](https://app.fossa.com/projects/git%2Bgithub.com%2Fcubefs%2Fcubefs?ref=badge_shield)
 [![Release](https://img.shields.io/github/v/release/cubefs/cubefs.svg?color=161823&style=flat-square&logo=smartthings)](https://github.com/cubefs/cubefs/releases)
 [![Tag](https://img.shields.io/github/v/tag/cubefs/cubefs.svg?color=ee8936&logo=fitbit&style=flat-square)](https://github.com/cubefs/cubefs/tags)
+[![Gurubase](https://img.shields.io/badge/Gurubase-Ask%20CubeFS%20Guru-006BFF)](https://gurubase.io/g/cubefs)
 
 |<img src="https://user-images.githubusercontent.com/5708406/91202310-31eaab80-e734-11ea-84fc-c1b1882ae71c.png" height="24"/>&nbsp;Community Meeting|
 |------------------|
@@ -25,13 +26,14 @@
 
 ## Overview
 
-CubeFS ("储宝" in Chinese) is an open-source cloud-native file storage system, hosted by the [Cloud Native Computing Foundation](https://cncf.io) (CNCF) as an [incubating](https://www.cncf.io/projects/) project.
-
+CubeFS ("储宝" in Chinese) is an open-source cloud-native distributed file & object storage system, hosted by the [Cloud Native Computing Foundation](https://cncf.io) (CNCF) as a [graduated](https://www.cncf.io/projects/) project.
 
 ## What can you build with CubeFS
 
-As an open-source distributed storage, CubeFS can serve as your datacenter filesystem, data lake storage infra, and private or hybrid cloud storage. 
-In particular, CubeFS enables the separation of storage/compute architecture for databases and AI/ML applications.
+* As an open-source distributed storage, CubeFS can serve as your datacenter filesystem, data lake storage infra, and private or hybrid cloud storage. 
+* Moreover, it can be run in public cloud services, providing cache acceleration and file system semantics on top of public cloud storage such as S3.
+
+* In particular, CubeFS enables the separation of storage/compute architecture for databases, search systems, and AI/ML applications.
 
 Some key features of CubeFS include:
 
@@ -43,7 +45,7 @@ Some key features of CubeFS include:
 - Flexible storage policies, high-performance replication or low-cost erasure coding
 
 
-<div width="100%" style="text-align:center;"><img alt="CubeFS Architecture" src="https://raw.githubusercontent.com/cubefs/cubefs/master/docs/source/pic/cfs-arch-ec.png"/></div>
+<div width="100%" style="text-align:center;"><img alt="CubeFS Architecture" src="https://raw.githubusercontent.com/cubefs/cubefs/master/docs/source/overview/pic/cfs-arch-ec.png"/></div>
 
 ## Documents
 

ROADMAP.md

@@ -1,4 +1,12 @@
-# Roadmap of 2024
+# Roadmap of 2026
 
-https://github.com/cubefs/cubefs/issues/3064
+### Release Scheduled
 
+| Feature | Type | Version | Status | Development Branch | Scheduled Release Date | Details |
+|:--|:--|:--|:--|:--|:--|:--|
+| Hybrid Cloud Support & Metadata Cost Reduction | Feature | Release-3.6.0 | System Testing | develop-v3.6.0 | July | 1) Public Cloud Data Writeback<br>2) RocksDB metadata supports RocksDB persistence (Learner priority)<br>3) Learner capability: Raft group supports Learner capability<br>4) MP&&DP multi-region distribution and directed automatic migration |
+| System Operations Automation and Stability Enhancement | Feature | Release-3.6.1 | In Development | develop-v3.6.1 | November | 1) DP supports capacity and quantity balancing<br>2) Master & Datanode IO tiered flow control + adaptive load<br>3) Rack balancing, NodeSet balancing<br>4) Cache node massive small files support and performance optimization<br>5) Cache prefetch optimization |
+
+
+
+CubeFS will prioritize performance and feature requirements for AI and similar scenarios, and may adjust release content accordingly.

authnode/api_service.go

@@ -29,10 +29,6 @@ import (
 	"github.com/cubefs/cubefs/util/log"
 )
 
-const (
-	nodeType = "auth"
-)
-
 func (m *Server) getTicket(w http.ResponseWriter, r *http.Request) {
 	var (
 		plaintext []byte
@@ -43,7 +39,7 @@ func (m *Server) getTicket(w http.ResponseWriter, r *http.Request) {
 		message   string
 	)
 
-	if m.metaReady == false {
+	if !m.metaReady {
 		log.LogWarnf("action[handlerWithInterceptor] leader meta has not ready")
 		http.Error(w, m.leaderInfo.addr, http.StatusBadRequest)
 	}
@@ -79,7 +75,6 @@ func (m *Server) getTicket(w http.ResponseWriter, r *http.Request) {
 	}
 
 	sendOkReply(w, r, newSuccessHTTPAuthReply(message))
-	return
 }
 
 func (m *Server) raftNodeOp(w http.ResponseWriter, r *http.Request) {
@@ -145,21 +140,14 @@ func (m *Server) raftNodeOp(w http.ResponseWriter, r *http.Request) {
 	}
 
 	sendOkReply(w, r, newSuccessHTTPAuthReply(message))
-	return
 }
 
 func (m *Server) handleAddRaftNode(raftNodeInfo *proto.AuthRaftNodeInfo) (err error) {
-	if err = m.cluster.addRaftNode(raftNodeInfo.ID, raftNodeInfo.Addr); err != nil {
-		return
-	}
-	return
+	return m.cluster.addRaftNode(raftNodeInfo.ID, raftNodeInfo.Addr)
 }
 
 func (m *Server) handleRemoveRaftNode(raftNodeInfo *proto.AuthRaftNodeInfo) (err error) {
-	if err = m.cluster.removeRaftNode(raftNodeInfo.ID, raftNodeInfo.Addr); err != nil {
-		return
-	}
-	return
+	return m.cluster.removeRaftNode(raftNodeInfo.ID, raftNodeInfo.Addr)
 }
 
 func genAuthRaftNodeOpResp(req *proto.APIAccessReq, ts int64, key []byte, msg string) (message string, err error) {
@@ -289,28 +277,18 @@ func (m *Server) apiAccessEntry(w http.ResponseWriter, r *http.Request) {
 	}
 
 	sendOkReply(w, r, newSuccessHTTPAuthReply(message))
-	return
 }
 
 func (m *Server) handleCreateKey(keyInfo *keystore.KeyInfo) (res *keystore.KeyInfo, err error) {
-	if res, err = m.cluster.CreateNewKey(keyInfo.ID, keyInfo); err != nil {
-		return
-	}
-	return
+	return m.cluster.CreateNewKey(keyInfo.ID, keyInfo)
 }
 
 func (m *Server) handleDeleteKey(keyInfo *keystore.KeyInfo) (res *keystore.KeyInfo, err error) {
-	if res, err = m.cluster.DeleteKey(keyInfo.ID); err != nil {
-		return
-	}
-	return
+	return m.cluster.DeleteKey(keyInfo.ID)
 }
 
 func (m *Server) handleGetKey(keyInfo *keystore.KeyInfo) (res *keystore.KeyInfo, err error) {
-	if res, err = m.getSecretKeyInfo(keyInfo.ID); err != nil {
-		return
-	}
-	return
+	return m.getSecretKeyInfo(keyInfo.ID)
 }
 
 func (m *Server) handleAddCaps(keyInfo *keystore.KeyInfo) (res *keystore.KeyInfo, err error) {
@@ -437,7 +415,6 @@ func (m *Server) osCapsOp(w http.ResponseWriter, r *http.Request) {
 	}
 
 	sendOkReply(w, r, newSuccessHTTPAuthReply(message))
-	return
 }
 
 func (m *Server) genTicket(key []byte, serviceID string, IP string, caps []byte) (ticket cryptoutil.Ticket) {
@@ -680,7 +657,6 @@ func send(w http.ResponseWriter, r *http.Request, reply []byte) {
 		return
 	}
 	log.LogInfof("URL[%v],remoteAddr[%v],response ok", r.URL, r.RemoteAddr)
-	return
 }
 
 func keyNotFound(name string) (err error) {
@@ -700,5 +676,4 @@ func sendErrReply(w http.ResponseWriter, r *http.Request, HTTPAuthReply *proto.H
 	if _, err = w.Write(reply); err != nil {
 		log.LogErrorf("fail to write http reply[%s] len[%d].URL[%v],remoteAddr[%v] err:[%v]", string(reply), len(reply), r.URL, r.RemoteAddr, err)
 	}
-	return
 }

authnode/authnode_manager.go

@@ -36,7 +36,7 @@ func (m *Server) handleLeaderChange(leader uint64) {
 	log.LogWarnf("action[handleLeaderChange] change leader to [%v] ", m.leaderInfo.addr)
 	m.authProxy = m.newAuthProxy() // TODO no lock?
 
-	if m.metaReady == false {
+	if !m.metaReady {
 		if err := m.cluster.loadKeystore(); err != nil {
 			panic(err)
 		}
@@ -57,7 +57,7 @@ func (m *Server) handlePeerChange(confChange *proto.ConfChange) (err error) {
 			msg = fmt.Sprintf("action[handlePeerChange] clusterID[%v] nodeAddr[%v] is invalid", m.clusterName, addr)
 			break
 		}
-		m.raftStore.AddNodeWithPort(confChange.Peer.ID, arr[0], int(m.config.heartbeatPort), int(m.config.replicaPort))
+		m.raftStore.AddNodeWithPort(confChange.Peer.ID, arr[0], m.config.heartbeatPort, m.config.replicaPort)
 		AddrDatabase[confChange.Peer.ID] = string(confChange.Context)
 		msg = fmt.Sprintf("clusterID[%v] peerID:%v,nodeAddr[%v] has been add", m.clusterName, confChange.Peer.ID, addr)
 	case proto.ConfRemoveNode:
@@ -73,5 +73,4 @@ func (m *Server) handlePeerChange(confChange *proto.ConfChange) (err error) {
 func (m *Server) handleApplySnapshot() {
 	log.LogInfof("clusterID[%v] peerID:%v action[handleApplySnapshot]", m.clusterName, m.id)
 	m.fsm.restore()
-	return
 }

authnode/authtool/authtool.go

@@ -57,10 +57,7 @@ var action2PathMap = map[string]string{
 	OSGetCaps:      proto.OSGetCaps,
 }
 
-var (
-	cflag    string
-	flaginfo flagInfo
-)
+var flaginfo flagInfo
 
 type ticketFlag struct {
 	key     string
@@ -364,7 +361,7 @@ func accessAuthServer() {
 				panic(err)
 			}
 		} else {
-			if res, err = resp.KeyInfo.DumpJSONStr(resp.AuthIDKey); err != nil {
+			if _, err = resp.KeyInfo.DumpJSONStr(resp.AuthIDKey); err != nil {
 				panic(err)
 			}
 		}

authnode/cluster.go

@@ -57,8 +57,8 @@ func newCluster(name string, leaderInfo *LeaderInfo, fsm *KeystoreFsm, partition
 	c.cfg = cfg
 	c.fsm = fsm
 	c.partition = partition
-	c.fsm.keystore = make(map[string]*keystore.KeyInfo, 0)
-	c.fsm.accessKeystore = make(map[string]*keystore.AccessKeyInfo, 0)
+	c.fsm.keystore = make(map[string]*keystore.KeyInfo)
+	c.fsm.accessKeystore = make(map[string]*keystore.AccessKeyInfo)
 	return
 }
 
@@ -66,10 +66,6 @@ func (c *Cluster) scheduleTask() {
 	c.scheduleToCheckHeartbeat()
 }
 
-func (c *Cluster) authAddr() (addr string) {
-	return c.leaderInfo.addr
-}
-
 func (c *Cluster) scheduleToCheckHeartbeat() {
 	go func() {
 		for {

authnode/config.go

@@ -42,8 +42,8 @@ const (
 type clusterConfig struct {
 	peers         []raftstore.PeerAddress
 	peerAddrs     []string
-	heartbeatPort int64
-	replicaPort   int64
+	heartbeatPort int
+	replicaPort   int
 }
 
 // AddrDatabase is a map that stores the address of a given host (e.g., the leader)
@@ -76,7 +76,7 @@ func (cfg *clusterConfig) parsePeers(peerStr string) error {
 		if err != nil {
 			return err
 		}
-		cfg.peers = append(cfg.peers, raftstore.PeerAddress{Peer: proto.Peer{ID: id}, Address: ip, HeartbeatPort: int(cfg.heartbeatPort), ReplicaPort: int(cfg.replicaPort)})
+		cfg.peers = append(cfg.peers, raftstore.PeerAddress{Peer: proto.Peer{ID: id}, Address: ip, HeartbeatPort: cfg.heartbeatPort, ReplicaPort: cfg.replicaPort})
 		address := fmt.Sprintf("%v:%v", ip, port)
 		syslog.Println(address)
 		AddrDatabase[id] = address

authnode/const.go

@@ -37,3 +37,9 @@ const (
 	akAcronym = "ak"
 	akPrefix  = keySeparator + akAcronym + keySeparator
 )
+
+// TODO: unused
+var (
+	_ = opSyncGetKey
+	_ = opSyncGetCaps
+)

authnode/http_server.go

@@ -54,7 +54,6 @@ func (m *Server) startHTTPService() {
 			}
 		}
 	}()
-	return
 }
 
 func (m *Server) newAuthProxy() *AuthProxy {
@@ -122,7 +121,6 @@ func (m *Server) handleFunctions() {
 	http.Handle(proto.OSAddCaps, m.handlerWithInterceptor())
 	http.Handle(proto.OSDeleteCaps, m.handlerWithInterceptor())
 	http.Handle(proto.OSGetCaps, m.handlerWithInterceptor())
-	return
 }
 
 func (m *Server) handlerWithInterceptor() http.Handler {

authnode/keystore_cache_op.go

@@ -30,7 +30,6 @@ func (mf *KeystoreFsm) DeleteKey(id string) {
 	mf.ksMutex.Lock()
 	defer mf.ksMutex.Unlock()
 	delete(mf.keystore, id)
-	return
 }
 
 func (mf *KeystoreFsm) PutAKInfo(akInfo *keystore.AccessKeyInfo) {
@@ -55,5 +54,4 @@ func (mf *KeystoreFsm) DeleteAKInfo(accessKey string) {
 	mf.aksMutex.Lock()
 	defer mf.aksMutex.Unlock()
 	delete(mf.accessKeystore, accessKey)
-	return
 }

authnode/keystore_fsm.go

@@ -37,8 +37,6 @@ type raftLeaderChangeHandler func(leader uint64)
 
 type raftPeerChangeHandler func(confChange *proto.ConfChange) (err error)
 
-type raftCmdApplyHandler func(cmd *RaftCmd) (err error)
-
 type raftApplySnapshotHandler func()
 
 // KeystoreFsm represents the finite state machine of a keystore

authnode/keystore_fsm_op.go

@@ -118,7 +118,7 @@ func (c *Cluster) syncPutAccessKeyInfo(opType uint32, accessKeyInfo *keystore.Ac
 }
 
 func (c *Cluster) loadKeystore() (err error) {
-	ks := make(map[string]*keystore.KeyInfo, 0)
+	ks := make(map[string]*keystore.KeyInfo)
 	log.LogInfof("action[loadKeystore]")
 	result, err := c.fsm.store.SeekForPrefix([]byte(ksPrefix))
 	if err != nil {
@@ -143,14 +143,8 @@ func (c *Cluster) loadKeystore() (err error) {
 	return
 }
 
-func (c *Cluster) clearKeystore() {
-	c.fsm.ksMutex.Lock()
-	defer c.fsm.ksMutex.Unlock()
-	c.fsm.keystore = nil
-}
-
 func (c *Cluster) loadAKstore() (err error) {
-	aks := make(map[string]*keystore.AccessKeyInfo, 0)
+	aks := make(map[string]*keystore.AccessKeyInfo)
 	log.LogInfof("action[loadAccessKeystore]")
 	result, err := c.fsm.store.SeekForPrefix([]byte(akPrefix))
 	if err != nil {
@@ -175,12 +169,6 @@ func (c *Cluster) loadAKstore() (err error) {
 	return
 }
 
-func (c *Cluster) clearAKstore() {
-	c.fsm.aksMutex.Lock()
-	defer c.fsm.aksMutex.Unlock()
-	c.fsm.accessKeystore = nil
-}
-
 func (c *Cluster) addRaftNode(nodeID uint64, addr string) (err error) {
 	peer := proto.Peer{ID: nodeID}
 	_, err = c.partition.ChangeMember(proto.ConfAddNode, peer, []byte(addr))

authnode/server.go

@@ -109,8 +109,8 @@ func (m *Server) checkConfig(cfg *config.Config) (err error) {
 	if m.id, err = strconv.ParseUint(cfg.GetString(ID), 10, 64); err != nil {
 		return fmt.Errorf("%v,err:%v", proto.ErrInvalidCfg, err.Error())
 	}
-	m.config.heartbeatPort = cfg.GetInt64(heartbeatPortKey)
-	m.config.replicaPort = cfg.GetInt64(replicaPortKey)
+	m.config.heartbeatPort = cfg.GetInt(heartbeatPortKey)
+	m.config.replicaPort = cfg.GetInt(replicaPortKey)
 	if m.config.heartbeatPort <= 1024 {
 		m.config.heartbeatPort = raftstore.DefaultHeartbeatPort
 	}
@@ -162,8 +162,8 @@ func (m *Server) createRaftServer(cfg *config.Config) (err error) {
 		NodeID:            m.id,
 		RaftPath:          m.walDir,
 		NumOfLogsToRetain: m.retainLogs,
-		HeartbeatPort:     int(m.config.heartbeatPort),
-		ReplicaPort:       int(m.config.replicaPort),
+		HeartbeatPort:     m.config.heartbeatPort,
+		ReplicaPort:       m.config.replicaPort,
 		TickInterval:      m.tickInterval,
 		ElectionTick:      m.electionTick,
 	}
@@ -213,7 +213,7 @@ func (m *Server) Start(cfg *config.Config) (err error) {
 		return fmt.Errorf("action[Start] failed %v,err: auth root Key invalid=%s", proto.ErrInvalidCfg, AuthRootKey)
 	}
 
-	if cfg.GetBool(EnableHTTPS) == true {
+	if cfg.GetBool(EnableHTTPS) {
 		m.cluster.PKIKey.EnableHTTPS = true
 		if m.cluster.PKIKey.AuthRootPublicKey, err = os.ReadFile("/app/server.crt"); err != nil {
 			return fmt.Errorf("action[Start] failed,err[%v]", err)

blobstore/Dockerfile

@@ -1,5 +1,7 @@
-FROM golang:1.17.13@sha256:87262e4a4c7db56158a80a18fefdc4fee5accc41b59cde821e691d05541bbb18
+FROM golang:1.18.10@sha256:50c889275d26f816b5314fc99f55425fa76b18fcaf16af255f5d57f09e1f48da
 
+RUN sed -i 's/deb.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list && \
+    sed -i 's/security.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list
 
 ENV JAVA_HOME=bin/jdk1.8.0_321
 ENV CLASSPATH=$CLASSPATH:$JAVA_HOME/lib

blobstore/Makefile

@@ -34,7 +34,7 @@ INSTALL=CGO_ENABLED=0 $(BUILD)
 CGOINSTALL=CGO_ENABLED=1 $(BUILD)
 PROJECTMOD=github.com/cubefs/cubefs/blobstore
 CMDDIR=$(PROJECTMOD)/cmd
-TARGETS=clustermgr blobnode access scheduler proxy cli
+TARGETS=clustermgr blobnode access scheduler proxy cli shardnode
 
 .PHONY: clean all $(TARGETS)
 all:$(TARGETS)
@@ -63,6 +63,10 @@ cli:
 	@echo "building blobstore-cli"
 	@$(CGOINSTALL) -o $(BINDIR)/blobstore-cli $(PROJECTMOD)/cli/cli
 
+shardnode:
+	@echo "building shardnode"
+	@$(CGOINSTALL) $(CMDDIR)/shardnode
+
 clean:
 	@go clean -i ./...
 	@rm -f $(BINDIR)/*

blobstore/access/controller/cluster.go

@@ -38,6 +38,9 @@ import (
 	"github.com/cubefs/cubefs/blobstore/util/log"
 )
 
+// just for cli write to readonly initialised cluster.
+var ForceWriteReadonlyCluster = false
+
 // AlgChoose algorithm of choose cluster
 type AlgChoose uint32
 
@@ -52,14 +55,17 @@ const (
 	maxAlg
 )
 
-var cachedClient = struct {
-	mu    sync.Mutex
-	cache map[string]*cmapi.Client
-}{
-	cache: make(map[string]*cmapi.Client),
+type keyClient struct {
+	key string
+	cli *cmapi.Client
 }
 
-func getClusterClient(conf cmapi.Config) *cmapi.Client {
+var cachedClient = struct {
+	mu    sync.Mutex
+	cache map[proto.ClusterID]*keyClient
+}{cache: make(map[proto.ClusterID]*keyClient)}
+
+func getClusterClient(ctx context.Context, clusterID proto.ClusterID, conf cmapi.Config) *cmapi.Client {
 	hosts := make([]string, len(conf.Hosts))
 	copy(hosts, conf.Hosts[:])
 	sort.Strings(hosts)
@@ -67,13 +73,20 @@ func getClusterClient(conf cmapi.Config) *cmapi.Client {
 
 	cachedClient.mu.Lock()
 	defer cachedClient.mu.Unlock()
-	cli, ok := cachedClient.cache[key]
-	if ok {
+	keyCli, ok := cachedClient.cache[clusterID]
+	if !ok {
+		cli := cmapi.New(&conf)
+		cachedClient.cache[clusterID] = &keyClient{key: key, cli: cli}
 		return cli
 	}
-	cli = cmapi.New(&conf)
-	cachedClient.cache[key] = cli
-	return cli
+	if keyCli.key != key {
+		span := trace.SpanFromContextSafe(ctx)
+		span.Warnf("change cluster(%d) clustermgr hosts %s -> %s", clusterID, keyCli.key, key)
+		cli := cmapi.New(&conf)
+		keyCli.key = key
+		*keyCli.cli = *cli
+	}
+	return keyCli.cli
 }
 
 // IsValid returns valid algorithm or not.
@@ -115,6 +128,8 @@ type ClusterController interface {
 	GetConfig(ctx context.Context, key string) (string, error)
 	// ChangeChooseAlg change alloc algorithm
 	ChangeChooseAlg(alg AlgChoose) error
+	// GetShardController return IShardController in specified cluster
+	GetShardController(clusterID proto.ClusterID) (IShardController, error)
 }
 
 // ClusterConfig cluster config
@@ -126,13 +141,15 @@ type ClusterConfig struct {
 	Region            string       `json:"region"`
 	RegionMagic       string       `json:"region_magic"`
 	ClusterReloadSecs int          `json:"cluster_reload_secs"`
-	ServiceReloadSecs int          `json:"service_reload_secs"`
+	ShardReloadSecs   int          `json:"shard_reload_secs"`
 	CMClientConfig    cmapi.Config `json:"clustermgr_client_config"`
 
-	ServicePunishThreshold      uint32 `json:"service_punish_threshold"`
-	ServicePunishValidIntervalS int    `json:"service_punish_valid_interval_s"`
+	ServiceConfig
+	VolumeConfig
 
 	ConsulAgentAddr string    `json:"consul_agent_addr"`
+	ConsulToken     string    `json:"consul_token"`
+	ConsulTokenFile string    `json:"consul_token_file"`
 	Clusters        []Cluster `json:"clusters"`
 }
 
@@ -140,6 +157,7 @@ type ClusterConfig struct {
 type Cluster struct {
 	ClusterID proto.ClusterID `json:"cluster_id"`
 	Hosts     []string        `json:"hosts"`
+	Space     SpaceConf       `json:"space"` // one space - one cluster
 }
 
 type cluster struct {
@@ -160,7 +178,8 @@ type clusterControllerImpl struct {
 	available       atomic.Value // available clusters
 	serviceMgrs     sync.Map
 	volumeGetters   sync.Map
-	roundRobinCount uint64 // a count for round robin
+	shardMgrs       sync.Map // cid -> shardController
+	roundRobinCount uint64   // a count for round robin
 	proxy           proxy.Cacher
 	stopCh          <-chan struct{}
 
@@ -173,9 +192,15 @@ func NewClusterController(cfg *ClusterConfig, proxy proxy.Cacher, stopCh <-chan
 
 	consulConf := api.DefaultConfig()
 	consulConf.Address = cfg.ConsulAgentAddr
+	if cfg.ConsulTokenFile != "" {
+		consulConf.TokenFile = cfg.ConsulTokenFile
+	}
+	if cfg.ConsulToken != "" {
+		consulConf.Token = cfg.ConsulToken
+	}
 	var client *api.Client
 	var err error
-	if consulConf.Address != "" {
+	if cfg.ConsulAgentAddr != "" {
 		client, err = api.NewClient(consulConf)
 		if err != nil {
 			return nil, fmt.Errorf("new consul client failed, err: %v", err)
@@ -230,7 +255,7 @@ func (c *clusterControllerImpl) loadWithConfig() error {
 	for _, cs := range c.config.Clusters {
 		conf := c.config.CMClientConfig
 		conf.Hosts = cs.Hosts
-		cmCli := getClusterClient(conf)
+		cmCli := getClusterClient(ctx, cs.ClusterID, conf)
 
 		stat, err := cmCli.Stat(ctx)
 		if err != nil {
@@ -239,10 +264,13 @@ func (c *clusterControllerImpl) loadWithConfig() error {
 		}
 		clusterInfo := &cmapi.ClusterInfo{}
 		clusterInfo.ClusterID = cs.ClusterID
-		clusterInfo.Capacity = stat.SpaceStat.TotalSpace
-		clusterInfo.Available = stat.SpaceStat.WritableSpace
+		clusterInfo.Capacity = stat.BlobNodeSpaceStat.TotalSpace
+		clusterInfo.Available = stat.BlobNodeSpaceStat.WritableSpace
 		clusterInfo.Nodes = cs.Hosts
 		clusterInfo.Readonly = stat.ReadOnly
+		if clusterInfo.Readonly && ForceWriteReadonlyCluster {
+			clusterInfo.Readonly = false
+		}
 
 		allClusters[cs.ClusterID] = &cluster{client: cmCli, clusterInfo: clusterInfo}
 
@@ -254,7 +282,6 @@ func (c *clusterControllerImpl) loadWithConfig() error {
 			span.Debug("readonly or no available cluster", clusterInfo.ClusterID)
 		}
 	}
-
 	return c.deal(ctx, available, allClusters, totalAvailable)
 }
 
@@ -305,7 +332,12 @@ func (c *clusterControllerImpl) loadWithConsul() error {
 	return c.deal(ctx, available, allClusters, totalAvailable)
 }
 
-func (c *clusterControllerImpl) deal(ctx context.Context, available []*cmapi.ClusterInfo, allClusters clusterMap, totalAvailable int64) error {
+func (c *clusterControllerImpl) deal(ctx context.Context,
+	available []*cmapi.ClusterInfo, allClusters clusterMap, totalAvailable int64,
+) error {
+	if len(allClusters) == 0 {
+		return nil
+	}
 	span := trace.SpanFromContextSafe(ctx)
 
 	sort.Slice(available, func(i, j int) bool {
@@ -313,9 +345,14 @@ func (c *clusterControllerImpl) deal(ctx context.Context, available []*cmapi.Clu
 	})
 
 	newClusters := make([]*cmapi.ClusterInfo, 0, len(allClusters))
-	for clusterID := range allClusters {
+	for clusterID, cluster := range allClusters {
 		if _, ok := c.serviceMgrs.Load(clusterID); !ok {
-			newClusters = append(newClusters, allClusters[clusterID].clusterInfo)
+			newClusters = append(newClusters, cluster.clusterInfo)
+		} else {
+			// update cluster client if hosts changed
+			conf := c.config.CMClientConfig
+			conf.Hosts = cluster.clusterInfo.Nodes[:]
+			getClusterClient(ctx, clusterID, conf)
 		}
 	}
 
@@ -325,7 +362,7 @@ func (c *clusterControllerImpl) deal(ctx context.Context, available []*cmapi.Clu
 		if allClusters[clusterID].client == nil {
 			conf := c.config.CMClientConfig
 			conf.Hosts = newCluster.Nodes
-			allClusters[clusterID].client = getClusterClient(conf)
+			allClusters[clusterID].client = getClusterClient(ctx, clusterID, conf)
 		}
 
 		cmCli := allClusters[clusterID].client
@@ -343,26 +380,39 @@ func (c *clusterControllerImpl) deal(ctx context.Context, available []*cmapi.Clu
 			}
 		}
 
-		serviceController, err := NewServiceController(ServiceConfig{
-			ClusterID:                   clusterID,
-			IDC:                         c.config.IDC,
-			ReloadSec:                   c.config.ServiceReloadSecs,
-			ServicePunishThreshold:      c.config.ServicePunishThreshold,
-			ServicePunishValidIntervalS: c.config.ServicePunishValidIntervalS,
-		}, cmCli, c.proxy, c.stopCh)
+		serviceConfig := c.config.ServiceConfig
+		serviceConfig.ClusterID = clusterID
+		serviceConfig.IDC = c.config.IDC
+		serviceController, err := NewServiceController(serviceConfig, cmCli, c.proxy, c.stopCh)
 		if err != nil {
 			removeThisCluster()
 			span.Warn("new service manager failed", clusterID, err)
 			continue
 		}
 
-		volumeGetter, err := NewVolumeGetter(clusterID, serviceController, c.proxy, -1)
+		volumeConfig := c.config.VolumeConfig
+		volumeConfig.ClusterID = clusterID
+		volumeGetter, err := NewVolumeGetter(volumeConfig, serviceController, c.proxy, c.stopCh)
 		if err != nil {
 			removeThisCluster()
 			span.Warn("new volume getter failed", clusterID, err)
 			continue
 		}
 
+		space := c.getSpaceConf(clusterID)
+		span.Debugf("get space config valid:%t, cluster:%d", space.IsValid(), clusterID)
+		if space.IsValid() { // need shard node, meta system
+			shardMgr, err := NewShardController(shardCtrlConf{
+				clusterID:  clusterID,
+				reloadSecs: c.config.ShardReloadSecs,
+				space:      space,
+			}, cmCli, serviceController, c.stopCh)
+			if err != nil {
+				span.Fatalf("new shard controller failed, clusterID=%d, err=%+v", clusterID, err)
+			}
+			c.shardMgrs.Store(clusterID, shardMgr)
+		}
+
 		c.serviceMgrs.Store(clusterID, serviceController)
 		c.volumeGetters.Store(clusterID, volumeGetter)
 		span.Debug("loaded new cluster", clusterID)
@@ -475,3 +525,22 @@ func (c *clusterControllerImpl) GetConfig(ctx context.Context, key string) (ret
 	}
 	return
 }
+
+func (c *clusterControllerImpl) GetShardController(clusterID proto.ClusterID) (IShardController, error) {
+	if shardMgr, exist := c.shardMgrs.Load(clusterID); exist {
+		if controller, ok := shardMgr.(IShardController); ok {
+			return controller, nil
+		}
+		return nil, fmt.Errorf("not shard controller for %d", clusterID)
+	}
+	return nil, fmt.Errorf("no shard controller of %d", clusterID)
+}
+
+func (c *clusterControllerImpl) getSpaceConf(clusterID proto.ClusterID) SpaceConf {
+	for _, cs := range c.config.Clusters {
+		if clusterID == cs.ClusterID {
+			return cs.Space
+		}
+	}
+	return SpaceConf{}
+}

blobstore/access/controller/cluster_test.go

@@ -146,7 +146,7 @@ func stat(w http.ResponseWriter, req *http.Request) {
 	info := &clustermgr.StatInfo{
 		LeaderHost: hostAddr,
 		ReadOnly:   false,
-		SpaceStat: clustermgr.SpaceStatInfo{
+		BlobNodeSpaceStat: clustermgr.SpaceStatInfo{
 			TotalSpace:    1 << 40,
 			WritableSpace: 1 << 20,
 		},

blobstore/access/controller/mock_test.go

@@ -17,12 +17,12 @@ package controller_test
 import (
 	"context"
 	"math/rand"
+	"sync"
 	"testing"
 	"time"
 
 	"github.com/golang/mock/gomock"
 
-	bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
 	cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr"
 	"github.com/cubefs/cubefs/blobstore/api/proxy"
 	"github.com/cubefs/cubefs/blobstore/common/codemode"
@@ -48,10 +48,11 @@ var (
 	cmcli    cmapi.APIAccess
 	proxycli proxy.Cacher
 
+	dataMu      sync.Mutex
 	dataCalled  map[proto.Vid]int
 	dataNodes   map[string]cmapi.ServiceInfo
 	dataVolumes map[proto.Vid]cmapi.VolumeInfo
-	dataDisks   map[proto.DiskID]bnapi.DiskInfo
+	dataDisks   map[proto.DiskID]cmapi.BlobNodeDiskInfo
 )
 
 func init() {
@@ -63,15 +64,15 @@ func init() {
 	dataVolumes[1] = cmapi.VolumeInfo{
 		VolumeInfoBase: cmapi.VolumeInfoBase{Vid: 1, CodeMode: codemode.EC6P10L2},
 		Units: []cmapi.Unit{
-			{Vuid: 1011, DiskID: 1021, Host: "1031"},
-			{Vuid: 1012, DiskID: 1022, Host: "1032"},
+			{Vuid: 1011, DiskID: 1021},
+			{Vuid: 1012, DiskID: 1022},
 		},
 	}
 	dataVolumes[9] = cmapi.VolumeInfo{
 		VolumeInfoBase: cmapi.VolumeInfoBase{Vid: 9, CodeMode: codemode.EC16P20L2},
 		Units: []cmapi.Unit{
-			{Vuid: 9011, DiskID: 9021, Host: "9031"},
-			{Vuid: 9012, DiskID: 9022, Host: "9032"},
+			{Vuid: 9011, DiskID: 9021},
+			{Vuid: 9012, DiskID: 9022},
 		},
 	}
 	dataVolumes[vid404] = cmapi.VolumeInfo{VolumeInfoBase: cmapi.VolumeInfoBase{Vid: vid404}}
@@ -84,20 +85,24 @@ func init() {
 		},
 	}
 
-	dataDisks = make(map[proto.DiskID]bnapi.DiskInfo)
-	dataDisks[10001] = bnapi.DiskInfo{
-		ClusterID: 1,
-		Idc:       idc,
-		Host:      "blobnode-1",
-		DiskHeartBeatInfo: bnapi.DiskHeartBeatInfo{
+	dataDisks = make(map[proto.DiskID]cmapi.BlobNodeDiskInfo)
+	dataDisks[10001] = cmapi.BlobNodeDiskInfo{
+		DiskInfo: cmapi.DiskInfo{
+			ClusterID: 1,
+			Idc:       idc,
+			Host:      "blobnode-1",
+		},
+		DiskHeartBeatInfo: cmapi.DiskHeartBeatInfo{
 			DiskID: 10001,
 		},
 	}
-	dataDisks[10002] = bnapi.DiskInfo{
-		ClusterID: 1,
-		Idc:       idc,
-		Host:      "blobnode-2",
-		DiskHeartBeatInfo: bnapi.DiskHeartBeatInfo{
+	dataDisks[10002] = cmapi.BlobNodeDiskInfo{
+		DiskInfo: cmapi.DiskInfo{
+			ClusterID: 1,
+			Idc:       idc,
+			Host:      "blobnode-2",
+		},
+		DiskHeartBeatInfo: cmapi.DiskHeartBeatInfo{
 			DiskID: 10002,
 		},
 	}
@@ -112,26 +117,31 @@ func init() {
 			return cmapi.ServiceInfo{}, errNotFound
 		})
 	cli.EXPECT().ListDisk(A, A).AnyTimes().Return(cmapi.ListDiskRet{}, nil)
+	cli.EXPECT().ListShardNodeDisk(A, A).AnyTimes().Return(cmapi.ListShardNodeDiskRet{}, nil)
 	cmcli = cli
 
 	pcli := mocks.NewMockProxyClient(C(&testing.T{}))
 	pcli.EXPECT().GetCacheVolume(A, A, A).AnyTimes().DoAndReturn(
-		func(_ context.Context, _ string, args *proxy.CacheVolumeArgs) (*proxy.VersionVolume, error) {
-			volume := new(proxy.VersionVolume)
+		func(ctx context.Context, _ string, args *proxy.CacheVolumeArgs) (*cmapi.VolumeInfo, error) {
+			select {
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			default:
+			}
 			vid := args.Vid
+			dataMu.Lock()
 			dataCalled[vid]++
+			dataMu.Unlock()
 			if val, ok := dataVolumes[vid]; ok {
 				if vid == vid404 {
 					return nil, errcode.ErrVolumeNotExist
 				}
-				volume.VolumeInfo = val
-				volume.Version = volume.GetVersion()
-				return volume, nil
+				return &val, nil
 			}
 			return nil, errNotFound
 		})
 	pcli.EXPECT().GetCacheDisk(A, A, A).AnyTimes().DoAndReturn(
-		func(_ context.Context, _ string, args *proxy.CacheDiskArgs) (*bnapi.DiskInfo, error) {
+		func(_ context.Context, _ string, args *proxy.CacheDiskArgs) (*cmapi.BlobNodeDiskInfo, error) {
 			if val, ok := dataDisks[args.DiskID]; ok {
 				return &val, nil
 			}

blobstore/access/controller/service.go

@@ -24,7 +24,6 @@ import (
 
 	"golang.org/x/sync/singleflight"
 
-	"github.com/cubefs/cubefs/blobstore/api/blobnode"
 	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
 	"github.com/cubefs/cubefs/blobstore/api/proxy"
 	"github.com/cubefs/cubefs/blobstore/common/proto"
@@ -35,12 +34,8 @@ import (
 )
 
 const (
-	_diskHostServicePrefix = "diskhost"
-
-	// default service punish check valid interval
-	defaultServicePinishValidIntervalS int = 30
-	// default service punish check threshold
-	defaultServicePinishThreshold uint32 = 3
+	_primaryDisk          = "_disk_"
+	_primaryShardnodeDisk = "_sddisk_"
 )
 
 // HostIDC item of host with idc
@@ -68,6 +63,15 @@ type ServiceController interface {
 	// PunishDiskWithThreshold will punish a disk host for
 	// an punishTimeSec interval if disk host failed times satisfied with threshold
 	PunishDiskWithThreshold(ctx context.Context, diskID proto.DiskID, punishTimeSec int)
+	// GetShardnodeHost return shardnode host
+	GetShardnodeHost(ctx context.Context, diskID proto.DiskID) (hostIDC *HostIDC, err error)
+	// PunishShardnode will punish a shardnode disk host for an punishTimeSec interval
+	PunishShardnode(ctx context.Context, diskID proto.DiskID, punishTimeSec int)
+	// PunishShardnodeDiskWithThreshold will punish a disk host for
+	// an punishTimeSec interval if disk host failed times satisfied with threshold
+	PunishShardnodeDiskWithThreshold(ctx context.Context, diskID proto.DiskID, punishTimeSec int)
+	// IsPunishShardnode return shardnode disk is punish
+	IsPunishShardnode(diskID proto.DiskID) bool
 }
 
 type (
@@ -86,6 +90,8 @@ type hostItem struct {
 	lastModifyTime int64
 	// failedTimes record the service host failed times during some interval
 	failedTimes uint32
+
+	createAt time.Time
 }
 
 func (h *hostItem) isPunish() bool {
@@ -94,19 +100,24 @@ func (h *hostItem) isPunish() bool {
 
 // ServiceConfig service config
 type ServiceConfig struct {
-	ClusterID                   proto.ClusterID
-	IDC                         string
-	ReloadSec                   int
-	LoadDiskInterval            int
-	ServicePunishThreshold      uint32
-	ServicePunishValidIntervalS int
+	ClusterID proto.ClusterID `json:"-"`
+	IDC       string          `json:"-"`
+
+	ServiceReloadSecs           int    `json:"service_reload_secs"`
+	LoadDiskIntervalS           int    `json:"load_disk_interval_s"`
+	DiskPunishThreshold         uint32 `json:"disk_punish_threshold"`
+	DiskPunishValidIntervalS    int    `json:"disk_punish_valid_interval_s"`
+	DiskMemoryExpirationS       int    `json:"disk_memory_expiration_s"` // <= 0 means no expiration
+	ServicePunishThreshold      uint32 `json:"service_punish_threshold"`
+	ServicePunishValidIntervalS int    `json:"service_punish_valid_interval_s"`
 }
 
 type serviceControllerImpl struct {
 	// allServices hold all disk/service host map, use for quickly find out
-	allServices  sync.Map
-	serviceHosts serviceMap
-	brokenDisks  sync.Map
+	allServices   sync.Map
+	serviceHosts  serviceMap
+	brokenDisks   sync.Map
+	sdBrokenDisks sync.Map // shard node broken disks
 
 	group        singleflight.Group
 	serviceLocks map[string]*sync.RWMutex
@@ -117,12 +128,15 @@ type serviceControllerImpl struct {
 }
 
 // NewServiceController returns a service controller
-func NewServiceController(cfg ServiceConfig, cmCli clustermgr.APIAccess, proxy proxy.Cacher,
-	stopCh <-chan struct{}) (ServiceController, error) {
-	defaulter.Equal(&cfg.ServicePunishThreshold, defaultServicePinishThreshold)
-	defaulter.LessOrEqual(&cfg.ServicePunishValidIntervalS, defaultServicePinishValidIntervalS)
-	defaulter.LessOrEqual(&cfg.LoadDiskInterval, int(300))
-	defaulter.LessOrEqual(&cfg.ReloadSec, int(10))
+func NewServiceController(cfg ServiceConfig,
+	cmCli clustermgr.APIAccess, proxy proxy.Cacher, stopCh <-chan struct{},
+) (ServiceController, error) {
+	defaulter.IntegerLessOrEqual(&cfg.ServiceReloadSecs, 10)
+	defaulter.IntegerLessOrEqual(&cfg.LoadDiskIntervalS, 300)
+	defaulter.IntegerEqual(&cfg.DiskPunishThreshold, 3)
+	defaulter.IntegerLessOrEqual(&cfg.DiskPunishValidIntervalS, 30)
+	defaulter.IntegerEqual(&cfg.ServicePunishThreshold, 3)
+	defaulter.IntegerLessOrEqual(&cfg.ServicePunishValidIntervalS, 30)
 
 	controller := &serviceControllerImpl{
 		serviceHosts: serviceMap{
@@ -145,7 +159,7 @@ func NewServiceController(cfg ServiceConfig, cmCli clustermgr.APIAccess, proxy p
 		return controller, nil
 	}
 	go func() {
-		tick := time.NewTicker(time.Duration(cfg.ReloadSec) * time.Second)
+		tick := time.NewTicker(time.Duration(cfg.ServiceReloadSecs) * time.Second)
 		defer tick.Stop()
 		for {
 			select {
@@ -159,13 +173,12 @@ func NewServiceController(cfg ServiceConfig, cmCli clustermgr.APIAccess, proxy p
 		}
 	}()
 	go func() {
-		controller.loadBrokenDisks()
-		tick := time.NewTicker(time.Duration(cfg.LoadDiskInterval) * time.Second)
+		tick := time.NewTicker(time.Duration(cfg.LoadDiskIntervalS) * time.Second)
 		defer tick.Stop()
 		for {
+			controller.loadBrokenDisks()
 			select {
 			case <-tick.C:
-				controller.loadBrokenDisks()
 			case <-stopCh:
 				return
 			}
@@ -196,7 +209,7 @@ func (s *serviceControllerImpl) load(cid proto.ClusterID, idc string) error {
 	}
 	if len(hostItems) > 0 {
 		for _, item := range hostItems {
-			s.allServices.Store(serviceName+item.host, item)
+			s.allServices.Store(s.getServiceKey(serviceName, item.host), item)
 			span.Debugf("store node %+v", item)
 		}
 		s.serviceHosts[serviceName].Store(hostItems)
@@ -205,29 +218,65 @@ func (s *serviceControllerImpl) load(cid proto.ClusterID, idc string) error {
 }
 
 func (s *serviceControllerImpl) loadBrokenDisks() {
-	span, ctx := trace.StartSpanFromContext(context.Background(), "access_cluster_load_disks")
+	_, ctx := trace.StartSpanFromContext(context.Background(), "access_cluster_load_disks")
+
+	fnBlobnode := func(ctx context.Context, args *clustermgr.ListOptionArgs, diskMap map[proto.DiskID]struct{}) error {
+		list, err := s.cmClient.ListDisk(ctx, args)
+		if err != nil {
+			return err
+		}
+
+		for _, disk := range list.Disks {
+			diskMap[disk.DiskID] = struct{}{}
+		}
+		args.Marker = list.Marker
+		return nil
+	}
+
+	fnShardnode := func(ctx context.Context, args *clustermgr.ListOptionArgs, diskMap map[proto.DiskID]struct{}) error {
+		list, err := s.cmClient.ListShardNodeDisk(ctx, args)
+		if err != nil {
+			return err
+		}
+
+		for _, disk := range list.Disks {
+			diskMap[disk.DiskID] = struct{}{}
+		}
+		args.Marker = list.Marker
+		return nil
+	}
+
+	s.processBrokenDisks(ctx, fnBlobnode, &s.brokenDisks)
+	s.processBrokenDisks(ctx, fnShardnode, &s.sdBrokenDisks)
+}
+
+func (s *serviceControllerImpl) processBrokenDisks(
+	ctx context.Context,
+	fn func(context.Context, *clustermgr.ListOptionArgs, map[proto.DiskID]struct{}) error,
+	disks *sync.Map,
+) {
+	span := trace.SpanFromContextSafe(ctx)
 
 	brokenDiskIDs := make(map[proto.DiskID]struct{})
-	for _, st := range []proto.DiskStatus{proto.DiskStatusBroken, proto.DiskStatusRepairing} {
+	for _, st := range []proto.DiskStatus{proto.DiskStatusBroken} {
 		span.Debugf("to load disks of cluster %d %s", s.config.ClusterID, st.String())
 
-		args := &clustermgr.ListOptionArgs{Status: st, Marker: 1, Count: 1 << 10}
-		for args.Marker > proto.InvalidDiskID {
-			list, err := s.cmClient.ListDisk(ctx, args)
+		args := &clustermgr.ListOptionArgs{Status: st, Count: 1 << 10}
+		for {
+			err := fn(ctx, args, brokenDiskIDs)
 			if err != nil {
-				span.Errorf("load disks of cluster %d %s", s.config.ClusterID, err.Error())
+				span.Errorf("load disks of cluster %d, err:%+v", s.config.ClusterID, err)
 				return
 			}
-			for _, disk := range list.Disks {
-				brokenDiskIDs[disk.DiskID] = struct{}{}
+			if args.Marker <= proto.InvalidDiskID {
+				break
 			}
-			args.Marker = list.Marker
 		}
 	}
 
 	// clean cached disks, ignore cases when concurrency getting disk.
-	s.brokenDisks.Range(func(key, value interface{}) bool {
-		s.brokenDisks.Delete(key)
+	disks.Range(func(key, value any) bool {
+		disks.Delete(key)
 		return true
 	})
 	if len(brokenDiskIDs) == 0 {
@@ -235,7 +284,7 @@ func (s *serviceControllerImpl) loadBrokenDisks() {
 	}
 	span.Warnf("load disks of cluster %d broken %v", s.config.ClusterID, brokenDiskIDs)
 	for diskID := range brokenDiskIDs {
-		s.brokenDisks.Store(diskID, struct{}{})
+		disks.Store(diskID, struct{}{})
 	}
 }
 
@@ -330,16 +379,19 @@ func (s *serviceControllerImpl) GetDiskHost(ctx context.Context, diskID proto.Di
 
 	_, broken := s.brokenDisks.Load(diskID)
 
-	v, ok := s.allServices.Load(_diskHostServicePrefix + (diskID.ToString()))
+	v, ok := s.allServices.Load(s.getServiceKey(_primaryDisk, diskID))
 	if ok {
 		item := v.(*hostItem)
-		return &HostIDC{
-			Host:     item.host,
-			IDC:      item.idc,
-			Punished: broken || item.isPunish(),
-		}, nil
+		expiration := time.Second * time.Duration(s.config.DiskMemoryExpirationS)
+		if expiration <= 0 || time.Since(item.createAt) < expiration {
+			return &HostIDC{
+				Host:     item.host,
+				IDC:      item.idc,
+				Punished: broken || item.isPunish(),
+			}, nil
+		}
 	}
-	ret, err, _ := s.group.Do("get-diskinfo-"+diskID.ToString(), func() (interface{}, error) {
+	ret, err, _ := s.group.Do("get-diskinfo-"+diskID.ToString(), func() (any, error) {
 		hosts, err := s.GetServiceHosts(ctx, proto.ServiceNameProxy)
 		if err != nil {
 			return nil, err
@@ -358,10 +410,48 @@ func (s *serviceControllerImpl) GetDiskHost(ctx context.Context, diskID proto.Di
 		span.Error("can't get disk host from proxy", err)
 		return nil, errors.Base(err, "get disk info", diskID)
 	}
-	diskInfo := ret.(*blobnode.DiskInfo)
+	diskInfo := ret.(*clustermgr.BlobNodeDiskInfo)
+
+	item := &hostItem{host: diskInfo.Host, idc: diskInfo.Idc, createAt: time.Now()}
+	s.allServices.Store(s.getServiceKey(_primaryDisk, diskInfo.DiskID), item)
+	return &HostIDC{
+		Host:     item.host,
+		IDC:      item.idc,
+		Punished: broken || item.isPunish(),
+	}, nil
+}
+
+func (s *serviceControllerImpl) GetShardnodeHost(ctx context.Context, diskID proto.DiskID) (hostIDC *HostIDC, err error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	_, broken := s.sdBrokenDisks.Load(diskID)
+	v, ok := s.allServices.Load(s.getServiceKey(_primaryShardnodeDisk, diskID))
+	if ok {
+		item := v.(*hostItem)
+		return &HostIDC{
+			Host:     item.host,
+			IDC:      item.idc,
+			Punished: broken || item.isPunish(),
+		}, nil
+	}
+
+	ret, err, _ := s.group.Do("get-shardnode-diskinfo-"+diskID.ToString(), func() (any, error) {
+		// todo: support proxy get disk host, next version
+		info, err := s.cmClient.ShardNodeDiskInfo(ctx, diskID)
+		if err != nil {
+			return nil, err
+		}
+
+		return info, nil
+	})
+	if err != nil {
+		span.Error("can't get shardnode disk host from cm", err)
+		return nil, errors.Base(err, "get shardnode disk info", diskID)
+	}
+	diskInfo := ret.(*clustermgr.ShardNodeDiskInfo)
 
 	item := &hostItem{host: diskInfo.Host, idc: diskInfo.Idc}
-	s.allServices.Store(_diskHostServicePrefix+(diskInfo.DiskID.ToString()), item)
+	s.allServices.Store(s.getServiceKey(_primaryShardnodeDisk, diskInfo.DiskID), item)
 	return &HostIDC{
 		Host:     item.host,
 		IDC:      item.idc,
@@ -371,9 +461,9 @@ func (s *serviceControllerImpl) GetDiskHost(ctx context.Context, diskID proto.Di
 
 // PunishService will punish an service host for an punishTimeSec interval
 func (s *serviceControllerImpl) PunishService(ctx context.Context, service, host string, punishTimeSec int) {
-	v, ok := s.allServices.Load(service + host)
+	v, ok := s.allServices.Load(s.getServiceKey(service, host))
 	if !ok {
-		panic(fmt.Sprintf("can't find host in all services map, %s-%s", service, host))
+		panic(fmt.Sprintf("can't find host in all services map, %s", s.getServiceKey(service, host)))
 	}
 	item := v.(*hostItem)
 
@@ -383,28 +473,63 @@ func (s *serviceControllerImpl) PunishService(ctx context.Context, service, host
 
 // PunishDisk will punish a disk host for an punishTimeSec interval
 func (s *serviceControllerImpl) PunishDisk(ctx context.Context, diskID proto.DiskID, punishTimeSec int) {
-	s.PunishService(ctx, _diskHostServicePrefix, diskID.ToString(), punishTimeSec)
+	s.PunishService(ctx, _primaryDisk, diskID.ToString(), punishTimeSec)
+}
+
+// PunishShardnode will punish a shardnode disk host for an punishTimeSec interval
+func (s *serviceControllerImpl) PunishShardnode(ctx context.Context, diskID proto.DiskID, punishTimeSec int) {
+	s.PunishService(ctx, _primaryShardnodeDisk, diskID.ToString(), punishTimeSec)
+}
+
+// PunishShardnodeDiskWithThreshold will punish a disk host for
+// an punishTimeSec interval if disk host failed times satisfied with threshold
+func (s *serviceControllerImpl) PunishShardnodeDiskWithThreshold(ctx context.Context, diskID proto.DiskID, punishTimeSec int) {
+	s.punishWith(ctx, _primaryShardnodeDisk, diskID.ToString(), punishTimeSec,
+		s.config.DiskPunishThreshold, s.config.DiskPunishValidIntervalS)
+}
+
+func (s *serviceControllerImpl) IsPunishShardnode(diskID proto.DiskID) bool {
+	return s.isPunishService(_primaryShardnodeDisk, diskID.ToString())
+}
+
+func (s *serviceControllerImpl) isPunishService(primary, secondary string) bool {
+	v, ok := s.allServices.Load(s.getServiceKey(primary, secondary))
+	if !ok {
+		panic(fmt.Sprintf("can't find host in all services map, %s", s.getServiceKey(primary, secondary)))
+	}
+
+	item := v.(*hostItem)
+	return item.isPunish()
 }
 
 // PunishDiskWithThreshold will punish a disk host for
 // an punishTimeSec interval if disk host failed times satisfied with threshold
 func (s *serviceControllerImpl) PunishDiskWithThreshold(ctx context.Context, diskID proto.DiskID, punishTimeSec int) {
-	s.PunishServiceWithThreshold(ctx, _diskHostServicePrefix, diskID.ToString(), punishTimeSec)
+	s.punishWith(ctx, _primaryDisk, diskID.ToString(), punishTimeSec,
+		s.config.DiskPunishThreshold, s.config.DiskPunishValidIntervalS)
 }
 
 // PunishServiceWithThreshold will punish an service host for
 // an punishTimeSec interval if service failed times satisfied with threshold
 func (s *serviceControllerImpl) PunishServiceWithThreshold(ctx context.Context, service, host string, punishTimeSec int) {
-	v, ok := s.allServices.Load(service + host)
+	s.punishWith(ctx, service, host, punishTimeSec, s.config.ServicePunishThreshold, s.config.ServicePunishValidIntervalS)
+}
+
+func (s *serviceControllerImpl) punishWith(ctx context.Context,
+	primary, secondary string, punishTimeSec int,
+	threshold uint32, interval int,
+) {
+	serviceKey := s.getServiceKey(primary, secondary)
+	v, ok := s.allServices.Load(serviceKey)
 	if !ok {
-		panic(fmt.Sprintf("can't can host in all services map, %s-%s", service, host))
+		panic(fmt.Sprintf("can't load host in all services map, %s", serviceKey))
 	}
 	item := v.(*hostItem)
 	new := atomic.AddUint32(&item.failedTimes, 1)
 	// failedTimes larger than threshold, then check the lastModifyTime
-	if new >= s.config.ServicePunishThreshold {
-		if time.Since(time.Unix(atomic.LoadInt64(&item.lastModifyTime), 0)) < time.Duration(s.config.ServicePunishValidIntervalS)*time.Second {
-			s.PunishService(ctx, service, host, punishTimeSec)
+	if new >= threshold {
+		if time.Since(time.Unix(atomic.LoadInt64(&item.lastModifyTime), 0)) < time.Duration(interval)*time.Second {
+			s.PunishService(ctx, primary, secondary, punishTimeSec)
 			return
 		}
 		atomic.AddUint32(&item.failedTimes, -(new - 1))
@@ -415,3 +540,7 @@ func (s *serviceControllerImpl) PunishServiceWithThreshold(ctx context.Context,
 func (s *serviceControllerImpl) getServiceLock(name string) *sync.RWMutex {
 	return s.serviceLocks[name]
 }
+
+func (s *serviceControllerImpl) getServiceKey(primary string, secondary any) string {
+	return fmt.Sprintf("%s/%v", primary, secondary)
+}

blobstore/access/controller/service_test.go

@@ -23,7 +23,6 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/cubefs/cubefs/blobstore/access/controller"
-	bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode"
 	cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr"
 	"github.com/cubefs/cubefs/blobstore/api/proxy"
 	"github.com/cubefs/cubefs/blobstore/common/proto"
@@ -57,7 +56,7 @@ func TestAccessServiceNew(t *testing.T) {
 	}
 	{
 		sc, err := controller.NewServiceController(
-			controller.ServiceConfig{IDC: idc + "x", ReloadSec: 1}, cmcli, proxycli, nil)
+			controller.ServiceConfig{IDC: idc + "x", ServiceReloadSecs: 1}, cmcli, proxycli, nil)
 		require.NoError(t, err)
 
 		_, err = sc.GetServiceHost(serviceCtx, serviceName)
@@ -67,7 +66,7 @@ func TestAccessServiceNew(t *testing.T) {
 
 func TestAccessServiceGetServiceHost(t *testing.T) {
 	sc, err := controller.NewServiceController(
-		controller.ServiceConfig{IDC: idc, ReloadSec: 1}, cmcli, proxycli, nil)
+		controller.ServiceConfig{IDC: idc, ServiceReloadSecs: 1}, cmcli, proxycli, nil)
 	require.NoError(t, err)
 
 	keys := make(hostSet)
@@ -87,7 +86,7 @@ func TestAccessServicePunishService(t *testing.T) {
 	stop := closer.New()
 	defer stop.Close()
 	sc, err := controller.NewServiceController(
-		controller.ServiceConfig{IDC: idc, ReloadSec: 1}, cmcli, proxycli, stop.Done())
+		controller.ServiceConfig{IDC: idc, ServiceReloadSecs: 1}, cmcli, proxycli, stop.Done())
 	require.NoError(t, err)
 
 	{
@@ -138,7 +137,7 @@ func TestAccessServicePunishServiceWithThreshold(t *testing.T) {
 	sc, err := controller.NewServiceController(
 		controller.ServiceConfig{
 			IDC:                         idc,
-			ReloadSec:                   1,
+			ServiceReloadSecs:           1,
 			ServicePunishThreshold:      threshold,
 			ServicePunishValidIntervalS: 2,
 		}, cmcli, proxycli, stop.Done())
@@ -190,7 +189,7 @@ func TestAccessServicePunishServiceWithThreshold(t *testing.T) {
 
 func TestAccessServiceGetDiskHost(t *testing.T) {
 	sc, err := controller.NewServiceController(
-		controller.ServiceConfig{IDC: idc, ReloadSec: 1}, cmcli, proxycli, nil)
+		controller.ServiceConfig{IDC: idc, ServiceReloadSecs: 1}, cmcli, proxycli, nil)
 	require.NoError(t, err)
 
 	{
@@ -204,11 +203,24 @@ func TestAccessServiceGetDiskHost(t *testing.T) {
 	}
 }
 
+func TestAccessServiceGetDiskHostExpired(t *testing.T) {
+	sc, err := controller.NewServiceController(
+		controller.ServiceConfig{IDC: idc, ServiceReloadSecs: 1, DiskMemoryExpirationS: 1}, cmcli, proxycli, nil)
+	require.NoError(t, err)
+
+	host, err := sc.GetDiskHost(serviceCtx, proto.DiskID(10001))
+	require.NoError(t, err)
+	require.True(t, host.Host == "blobnode-1")
+	time.Sleep(time.Second)
+	_, err = sc.GetDiskHost(serviceCtx, proto.DiskID(10001))
+	require.NoError(t, err)
+}
+
 func TestAccessServiceGetBrokenDiskHost(t *testing.T) {
 	brokenRet := cmapi.ListDiskRet{}
-	brokenRet.Disks = make([]*bnapi.DiskInfo, 2)
-	brokenRet.Disks[0] = &bnapi.DiskInfo{}
-	brokenRet.Disks[1] = &bnapi.DiskInfo{}
+	brokenRet.Disks = make([]*cmapi.BlobNodeDiskInfo, 2)
+	brokenRet.Disks[0] = &cmapi.BlobNodeDiskInfo{}
+	brokenRet.Disks[1] = &cmapi.BlobNodeDiskInfo{}
 
 	cli := mocks.NewMockClientAPI(C(t))
 	cli.EXPECT().GetService(A, A).Times(5).DoAndReturn(
@@ -218,11 +230,12 @@ func TestAccessServiceGetBrokenDiskHost(t *testing.T) {
 			}
 			return cmapi.ServiceInfo{}, errNotFound
 		})
-	cli.EXPECT().ListDisk(A, A).Times(6).Return(brokenRet, nil)
+	cli.EXPECT().ListDisk(A, A).Times(3).Return(brokenRet, nil)
+	cli.EXPECT().ListShardNodeDisk(A, A).Times(3).Return(cmapi.ListShardNodeDiskRet{}, nil)
 
 	pcli := mocks.NewMockProxyClient(C(t))
 	pcli.EXPECT().GetCacheDisk(A, A, A).AnyTimes().DoAndReturn(
-		func(_ context.Context, _ string, args *proxy.CacheDiskArgs) (*bnapi.DiskInfo, error) {
+		func(_ context.Context, _ string, args *proxy.CacheDiskArgs) (*cmapi.BlobNodeDiskInfo, error) {
 			if val, ok := dataDisks[args.DiskID]; ok {
 				return &val, nil
 			}
@@ -232,7 +245,7 @@ func TestAccessServiceGetBrokenDiskHost(t *testing.T) {
 	stop := closer.New()
 	defer stop.Close()
 	sc, err := controller.NewServiceController(
-		controller.ServiceConfig{IDC: idc, ReloadSec: 1, LoadDiskInterval: 1}, cli, pcli, stop.Done())
+		controller.ServiceConfig{IDC: idc, ServiceReloadSecs: 1, LoadDiskIntervalS: 1}, cli, pcli, stop.Done())
 	require.NoError(t, err)
 
 	{
@@ -270,6 +283,7 @@ func TestAccessServiceGetBrokenDiskHost(t *testing.T) {
 	brokenRet.Disks[0].DiskID = 10000
 	brokenRet.Disks[1].DiskID = 10000
 	cli.EXPECT().ListDisk(A, A).Times(1).Return(brokenRet, errors.New("list error"))
+	cli.EXPECT().ListShardNodeDisk(A, A).Times(1).Return(cmapi.ListShardNodeDiskRet{}, errors.New("list error"))
 	time.Sleep(time.Second)
 	{
 		host, err := sc.GetDiskHost(serviceCtx, 10001)
@@ -281,7 +295,8 @@ func TestAccessServiceGetBrokenDiskHost(t *testing.T) {
 	}
 
 	brokenRet.Disks = brokenRet.Disks[:0]
-	cli.EXPECT().ListDisk(A, A).Times(2).Return(brokenRet, nil)
+	cli.EXPECT().ListDisk(A, A).Times(1).Return(brokenRet, nil)
+	cli.EXPECT().ListShardNodeDisk(A, A).Times(1).Return(cmapi.ListShardNodeDiskRet{}, nil)
 	time.Sleep(time.Second)
 	{
 		host, err := sc.GetDiskHost(serviceCtx, 10001)
@@ -297,7 +312,7 @@ func TestAccessServicePunishDisk(t *testing.T) {
 	stop := closer.New()
 	defer stop.Close()
 	sc, err := controller.NewServiceController(
-		controller.ServiceConfig{IDC: idc, ReloadSec: 1}, cmcli, proxycli, stop.Done())
+		controller.ServiceConfig{IDC: idc, ServiceReloadSecs: 1}, cmcli, proxycli, stop.Done())
 	require.NoError(t, err)
 
 	{

blobstore/access/controller/shard.go

@@ -0,0 +1,728 @@
+// Copyright 2024 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package controller
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"sync"
+	"time"
+
+	"github.com/google/btree"
+	"golang.org/x/sync/singleflight"
+
+	acapi "github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/api/shardnode"
+	"github.com/cubefs/cubefs/blobstore/cli/common"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/sharding"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/util/defaulter"
+	"github.com/cubefs/cubefs/blobstore/util/errors"
+)
+
+const (
+	defaultBTreeDegree     = 16
+	defaultShardReloadSecs = 120
+)
+
+var (
+	errCatalogInvalid  = errors.New("invalid catalog")
+	errCatalogNoLeader = errors.New("catalog item no leader")
+	errShardInvalid    = errors.New("invalid shard info")
+)
+
+type IShardController interface {
+	GetShard(ctx context.Context, shardKeys []string) (Shard, error)
+	GetShardByID(ctx context.Context, shardID proto.ShardID) (Shard, error)
+	GetShardByRange(ctx context.Context, shardRange sharding.Range) (Shard, error)
+	GetFisrtShard(ctx context.Context) (Shard, error)
+	GetNextShard(ctx context.Context, shardRange sharding.Range) (Shard, error)
+	GetSpaceID() proto.SpaceID
+	UpdateRoute(ctx context.Context) error
+	UpdateShard(ctx context.Context, ss shardnode.ShardStats) error
+	GetShardSubRangeCount(ctx context.Context) int
+}
+
+type shardCtrlConf struct {
+	clusterID  proto.ClusterID
+	reloadSecs int
+	space      SpaceConf
+}
+
+type SpaceConf struct {
+	Name string `json:"name"`
+	AK   string `json:"ak"`
+	SK   string `json:"sk"`
+}
+
+func (c *SpaceConf) IsValid() bool {
+	return c.Name != "" && c.AK != "" && c.SK != ""
+}
+
+func NewShardController(conf shardCtrlConf, cmCli clustermgr.ClientAPI, punishCtrl ServiceController, stopCh <-chan struct{}) (IShardController, error) {
+	defaulter.Equal(&conf.reloadSecs, defaultShardReloadSecs)
+
+	s := &shardControllerImpl{
+		shards: make(map[proto.ShardID]*shard),
+		ranges: btree.New(defaultBTreeDegree),
+		conf:   conf,
+		cmCli:  cmCli,
+		stopCh: stopCh,
+
+		punishCtrl: punishCtrl,
+	}
+
+	span, ctx := trace.StartSpanFromContext(context.Background(), "")
+	span.Debugf("start new shard controller, conf:%+v", conf)
+
+	err := s.initSpace(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	err = s.initRoute(ctx)
+	// if err is nil, errCatalogNoLeader: OK. we can get leader from sn when write leader node ; else : FATAL
+	if err != nil && !errors.Is(err, errCatalogNoLeader) {
+		return nil, err
+	}
+
+	sd, err := s.GetFisrtShard(ctx)
+	if err != nil {
+		return nil, err
+	}
+	s.subRangeCnt = len(sd.(*shard).rangeExt.Subs)
+
+	go s.incrementalRoute()
+
+	span.Debugf("success to new shard controller, clusterID:%d, space:%s", conf.clusterID, conf.space.Name)
+	return s, nil
+}
+
+type shardControllerImpl struct {
+	shards       map[proto.ShardID]*shard
+	ranges       *btree.BTree
+	version      proto.RouteVersion
+	spaceID      proto.SpaceID
+	groupRun     singleflight.Group
+	subRangeCnt  int
+	sync.RWMutex // todo: I will optimize locker in the next version
+
+	conf       shardCtrlConf
+	cmCli      clustermgr.ClientAPI
+	punishCtrl ServiceController
+	stopCh     <-chan struct{}
+}
+
+func (s *shardControllerImpl) GetShard(ctx context.Context, shardKeys []string) (Shard, error) {
+	// shard_1 ranges [1, 100) , shard 2: [100, 200), shard 3: [200, 300) ...
+	// if compare shard keys=20, it belong to shard 1 ; if keys=100, it belong to shard 2 ; keys=220, belong to shard 3
+	// if keys=120, will walk [shard 2, shard end]
+	s.RLock()
+	defer s.RUnlock()
+
+	span := trace.SpanFromContextSafe(ctx)
+	ci := sharding.NewCompareItem(sharding.RangeType_RangeTypeHash, shardKeys)
+	var si *shard
+	pivot := &compareItem{ci: *ci}
+
+	s.ranges.AscendGreaterOrEqual(pivot, func(i btree.Item) bool {
+		si = i.(*shard)
+		// span.Debugf("shardID=%d, max boundary=%d, compare=%d, shard=%+v", si.shardID, si.rangeExt.MaxBoundary(), ci.GetBoundary(), *si)
+		if si.belong(ci) {
+			return false
+		}
+		si = nil
+		return true
+	})
+
+	if si == nil { // not found expect shard
+		span.Errorf("not find shard. name:%s, shard len:%d, key boundary:%s", shardKeys, s.ranges.Len(), ci.GetBoundary())
+		return nil, errcode.ErrAccessNotFoundShard
+	}
+	return si, nil
+}
+
+func (s *shardControllerImpl) GetShardByID(ctx context.Context, shardID proto.ShardID) (Shard, error) {
+	sd, ok := s.getShardByID(shardID)
+	if ok {
+		return sd, nil
+	}
+
+	// not found expect shard
+	return nil, errcode.ErrAccessNotFoundShard
+}
+
+func (s *shardControllerImpl) GetFisrtShard(ctx context.Context) (Shard, error) {
+	s.RLock()
+	defer s.RUnlock()
+
+	span := trace.SpanFromContextSafe(ctx)
+	min := s.ranges.Min()
+	if min == nil { // not found expect shard
+		span.Errorf("not find shard. name:%s, shard len:%d", s.ranges.Len())
+		return nil, errcode.ErrAccessNotFoundShard
+	}
+
+	return min.(*shard), nil
+}
+
+func (s *shardControllerImpl) GetShardByRange(ctx context.Context, shardRange sharding.Range) (Shard, error) {
+	s.RLock()
+	defer s.RUnlock()
+
+	span := trace.SpanFromContextSafe(ctx)
+	var si *shard
+	pivot := &shard{rangeExt: shardRange}
+
+	// todo: shard ranges split, old shard range cant find
+	s.ranges.DescendLessOrEqual(pivot, func(i btree.Item) bool {
+		si = i.(*shard)
+		if si.contain(&shardRange) {
+			return false
+		}
+		si = nil
+		return true
+	})
+
+	if si == nil { // not found expect shard
+		span.Errorf("not find shard. range:%s, shard len:%d", common.RawString(shardRange), s.ranges.Len())
+		return nil, errcode.ErrAccessNotFoundShard
+	}
+	return si, nil
+}
+
+func (s *shardControllerImpl) GetNextShard(ctx context.Context, shardRange sharding.Range) (Shard, error) {
+	s.RLock()
+	defer s.RUnlock()
+
+	span := trace.SpanFromContextSafe(ctx)
+	// range is the end, the last one
+	if s.ranges.Max().(*shard).contain(&shardRange) {
+		return nil, nil
+	}
+
+	var si *shard
+	pivot := &shard{rangeExt: shardRange}
+	// todo: If two shard merge, it is possible that the shard queried contains the shard range
+	s.ranges.AscendGreaterOrEqual(pivot, func(i btree.Item) bool {
+		si = i.(*shard)
+		if !si.contain(&shardRange) {
+			return false
+		}
+		si = nil
+		return true
+	})
+
+	if si == nil { // not found expect shard
+		span.Errorf("not find shard. range:%s, shard len:%d", common.RawString(shardRange), s.ranges.Len())
+		return nil, errcode.ErrAccessNotFoundShard
+	}
+	return si, nil
+}
+
+func (s *shardControllerImpl) GetSpaceID() proto.SpaceID {
+	return s.spaceID
+}
+
+func (s *shardControllerImpl) UpdateRoute(ctx context.Context) error {
+	// Aggregation blob operations which comes from upper-layer
+	_, err, _ := s.groupRun.Do("updateRoute", func() (interface{}, error) {
+		// there is only one updateRoute, the same time. no concurrence
+		err1 := s.updateRoute(ctx)
+		return nil, err1
+	})
+	return err
+}
+
+// UpdateShard  update leader disk id and units info
+func (s *shardControllerImpl) UpdateShard(ctx context.Context, sd shardnode.ShardStats) error {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Debugf("will update shard, leaderDiskID=%d, LeaderSuid=%d, version=%d, suid=%d", sd.LeaderDiskID, sd.LeaderSuid, sd.RouteVersion, sd.Suid)
+
+	_, err, _ := s.groupRun.Do("shardID-"+sd.Suid.ShardID().ToString(), func() (interface{}, error) {
+		if isInvalidShardStat(sd) {
+			span.Errorf("invalid shard get from shard node. shard info:%+v", sd)
+			return nil, errShardInvalid
+		}
+		// todo: optimize lock at next version; will split smaller Lock or do a copy update
+		s.Lock()
+		defer s.Unlock()
+
+		// shard exist
+		oldShard, exist := s.getShardNoLock(sd.Suid.ShardID())
+		if !exist {
+			span.Warnf("dont need update shard, exist:%t, current shard:%+v, replace shard:%+v", exist, oldShard, sd)
+			return nil, errcode.ErrAccessNotFoundShard
+		}
+
+		// only update leader diskID/suid ; sd.LeaderDiskID must is in units
+		// don't need to judge or change RouteVersion, when switch the primary shardNode. only update version in cm GetCatalogChanges
+		if sd.LeaderSuid.Epoch() >= oldShard.units[sd.LeaderSuid.Index()].Suid.Epoch() {
+			oldShard.leaderDiskID = sd.LeaderDiskID
+			oldShard.leaderSuid = sd.LeaderSuid
+			return nil, nil
+		} else {
+			span.Warnf("skip update shard, leader suid epoch is less than old. old:%d, new:%d", oldShard.leaderSuid.Epoch(), sd.LeaderSuid.Epoch())
+			return nil, errCatalogNoLeader
+		}
+	})
+	return err
+}
+
+func (s *shardControllerImpl) GetShardSubRangeCount(ctx context.Context) int {
+	return s.subRangeCnt
+}
+
+func (s *shardControllerImpl) initSpace(ctx context.Context) error {
+	token, err := clustermgr.EncodeAuthInfo(&clustermgr.AuthInfo{
+		AccessKey: s.conf.space.AK,
+		SecretKey: s.conf.space.SK,
+	})
+	if err != nil {
+		return err
+	}
+
+	err = s.cmCli.AuthSpace(ctx, &clustermgr.AuthSpaceArgs{
+		Name:  s.conf.space.Name,
+		Token: token,
+	})
+	if err != nil {
+		return err
+	}
+
+	ret, err := s.cmCli.GetSpaceByName(ctx, &clustermgr.GetSpaceByNameArgs{
+		Name: s.conf.space.Name,
+	})
+	if err != nil {
+		return err
+	}
+
+	s.spaceID = ret.SpaceID
+	return nil
+}
+
+func (s *shardControllerImpl) initRoute(ctx context.Context) error {
+	return s.updateRoute(ctx)
+}
+
+func (s *shardControllerImpl) incrementalRoute() {
+	tk := time.NewTicker(time.Second * time.Duration(s.conf.reloadSecs))
+	defer tk.Stop()
+
+	for {
+		span, ctx := trace.StartSpanFromContext(context.Background(), "")
+		select {
+		case <-tk.C:
+			// there is only one updateRoute, the same time. no concurrence
+			err := s.UpdateRoute(ctx)
+			span.Debugf("loop update catalog route, err:%+v", err)
+
+		case <-s.stopCh:
+			span.Info("exit shard controller")
+			return
+		}
+	}
+}
+
+// called by period task, or read/write fail, init route
+// if err is nil, errCatalogNoLeader: OK. we can get leader from sn when write leader node ; else : FATAL
+func (s *shardControllerImpl) updateRoute(ctx context.Context) error {
+	span := trace.SpanFromContextSafe(ctx)
+
+	s.RLock()
+	version := s.version
+	s.RUnlock()
+
+	// $RouteVersion is 0: means full catalog route ; RouteVersion is greater than 0: means fetch incremental route
+	// full catalog route: all type is CatalogChangeItemAddShard (may contain init item and modify update item)
+	// incremental route: all type is ItemUpdateShard at 1.5.0 branch; If splitting features are supported, there may be multiple types
+	// todo: shard ranges split, incremental route will handles the add and update types
+	ret, err := s.cmCli.GetCatalogChanges(ctx, &clustermgr.GetCatalogChangesArgs{
+		RouteVersion: version,
+	})
+	if err != nil {
+		span.Errorf("fail to get catalog from clusterMgr. err:%+v", err)
+		return err
+	}
+
+	// skip
+	if version >= ret.RouteVersion || len(ret.Items) == 0 {
+		span.Debugf("skip get catalog changes, version=%d, items=%+v", version, *ret)
+		return nil
+	}
+
+	// todo: optimize lock at next version; will split smaller Lock or do a copy update
+	s.Lock()
+	defer s.Unlock()
+
+	// Try to process the correct item in this batch, skip error item and wait for the next fetch catalog, or fetch from sn
+	// 1. catalog normal: leader disk is not zero, and it is in units
+	// 2. leader disk is 0: in the election: shardNode is restart, or leader disk is broken re_election
+	// 3. leader disk not zero, and not in units
+	succ, wrong, itemErr := 0, 0, error(nil)
+	for _, item := range ret.Items {
+		switch item.Type {
+		case proto.CatalogChangeItemAddShard:
+			itemErr = s.handleShardAdd(ctx, item)
+		case proto.CatalogChangeItemUpdateShard:
+			itemErr = s.handleShardUpdate(ctx, item)
+		default:
+			itemErr = fmt.Errorf("not expected catalog")
+		}
+		// Skip the item that failed. and then fetch from sn, or wait for the next cm catalog
+		if errors.Is(itemErr, errCatalogNoLeader) {
+			wrong++
+			err = errCatalogNoLeader
+			continue
+		}
+		if itemErr != nil {
+			span.Errorf("update shard catalog error:%+v, item:%+v", err, item)
+			return itemErr
+		}
+		succ++
+	}
+
+	s.version = ret.RouteVersion
+	span.Debugf("success to update catalog, version from %d to %d, correct:%d, wrong:%d, local range min:%s, max:%s",
+		version, ret.RouteVersion, succ, wrong, s.ranges.Min().(*shard).String(), s.ranges.Max().(*shard).String())
+
+	return err
+}
+
+func (s *shardControllerImpl) handleShardAdd(ctx context.Context, item clustermgr.CatalogChangeItem) error {
+	span := trace.SpanFromContextSafe(ctx)
+
+	val := clustermgr.CatalogChangeShardAdd{}
+	err := val.Unmarshal(item.Item.Value)
+	if err != nil {
+		span.Warnf("catalog json unmarshal failed. type=%d, version=%d, err=%+v", item.Type, item.RouteVersion, err)
+		return err
+	}
+
+	// check invalid item
+	leaderIdx, err := findAndCheckCatalogShardAdd(val)
+	if err != nil && !errors.Is(err, errCatalogNoLeader) {
+		return err
+	}
+
+	sh := &shard{
+		shardID:      val.ShardID,
+		version:      val.RouteVersion,
+		leaderDiskID: val.Units[leaderIdx].DiskID,
+		leaderSuid:   val.Units[leaderIdx].Suid,
+		rangeExt:     val.Units[leaderIdx].Range,
+		units:        convertShardUnitInfo(val.Units),
+		punishCtrl:   s.punishCtrl,
+	}
+	s.addShardNoLock(sh)
+	span.Debugf("handle one catalog item add :%+v", val)
+
+	// insert a no leader item. because we need shardID. and will fetch correct item from sn
+	if errors.Is(err, errCatalogNoLeader) {
+		span.Warnf("catalog handle item add, no leader disk, item:%+v", val)
+		return errCatalogNoLeader
+	}
+	return nil
+}
+
+func (s *shardControllerImpl) handleShardUpdate(ctx context.Context, item clustermgr.CatalogChangeItem) error {
+	span := trace.SpanFromContextSafe(ctx)
+
+	val := clustermgr.CatalogChangeShardUpdate{}
+	if err := val.Unmarshal(item.Item.Value); err != nil {
+		span.Warnf("catalog json unmarshal failed. type=%d, version=%d, err=%+v", item.Type, item.RouteVersion, err)
+		return err
+	}
+
+	// shard id not exist
+	info, exist := s.shards[val.ShardID]
+	if !exist {
+		return errCatalogInvalid
+	}
+
+	// RouteVersion must be monotonically increasing. The version is too small, less than expected, should discard invalid item
+	// In general, cm make sure that the version is correct, and will not happen here
+	if val.RouteVersion < info.version {
+		span.Warnf("catalog skip invalid version item update, item:%+v", val)
+		return errCatalogInvalid
+	}
+
+	// skip invalid item
+	err := checkCatalogShardUpdate(val)
+	if errors.Is(err, errCatalogInvalid) {
+		span.Warnf("catalog skip invalid item update, item:%+v", val)
+		return errCatalogInvalid
+	}
+	if val.Unit.Suid.Epoch() <= info.units[val.Unit.Suid.Index()].Suid.Epoch() {
+		span.Warnf("catalog skip invalid item update, old:%+v, new:%+v", info.units[val.Unit.Suid.Index()], val)
+		return errCatalogInvalid
+	}
+
+	// fix leader disk is 0: use new disk unit as leaderDisk, and we will fetch the correct leader later from sn
+	if errors.Is(err, errCatalogNoLeader) {
+		span.Warnf("catalog handle item update, no leader disk, item:%+v", val)
+		val.Unit.LeaderDiskID = val.Unit.DiskID
+	}
+
+	// we will update shard, after fix catalog val
+	s.setShardByID(ctx, info, &val)
+	span.Debugf("handle one catalog item update:%+v, shard:%+v", val, *info)
+	return err
+}
+
+func (s *shardControllerImpl) delShardNoLock(si *shard) {
+	sd, ok := s.shards[si.shardID]
+	if ok {
+		s.ranges.Delete(sd)
+		delete(s.shards, si.shardID)
+	}
+}
+
+func (s *shardControllerImpl) addShardNoLock(si *shard) {
+	s.shards[si.shardID] = si
+	s.ranges.ReplaceOrInsert(si)
+}
+
+func (s *shardControllerImpl) getShardNoLock(id proto.ShardID) (*shard, bool) {
+	sd, ok := s.shards[id]
+	return sd, ok
+}
+
+func (s *shardControllerImpl) getShardByID(shardID proto.ShardID) (*shard, bool) {
+	s.RLock()
+	defer s.RUnlock()
+
+	info, ok := s.shards[shardID]
+	return info, ok
+}
+
+func (s *shardControllerImpl) setShardByID(ctx context.Context, info *shard, val *clustermgr.CatalogChangeShardUpdate) {
+	info.version = val.RouteVersion
+
+	// info.rangeExt = val.Unit.Range  // todo: will update range next version
+	idx := val.Unit.Suid.Index()
+	info.units[idx] = clustermgr.ShardUnit{
+		Suid:    val.Unit.Suid,
+		DiskID:  val.Unit.DiskID,
+		Learner: val.Unit.Learner, // most time, $learner is false
+	}
+
+	// update leader disk id and suid ; leader disk may not in units ; leader disk may not val.disk
+	for _, unit := range info.units {
+		if unit.DiskID == val.Unit.LeaderDiskID {
+			info.leaderDiskID = unit.DiskID
+			info.leaderSuid = unit.Suid
+			return
+		}
+	}
+
+	// leader disk not in units, fix it
+	span := trace.SpanFromContextSafe(ctx)
+	span.Infof("leader disk not in units. old leader:(%d, %d), new leader:%d, units:%+v",
+		info.leaderDiskID, info.leaderSuid, val.Unit.LeaderDiskID, info.units)
+	info.leaderDiskID = info.units[0].DiskID
+	info.leaderSuid = info.units[0].Suid
+}
+
+// ShardOpInfo for upper level(stream) use, get ShardOpHeader information
+type ShardOpInfo struct {
+	DiskID       proto.DiskID
+	Suid         proto.Suid
+	RouteVersion proto.RouteVersion
+}
+
+type Shard interface {
+	GetShardID() proto.ShardID
+	GetRange() sharding.Range
+	GetMember(context.Context, acapi.GetShardMode, map[proto.DiskID]struct{}) (ShardOpInfo, error)
+}
+
+// shard implement btree.Item interface, shard route information
+type shard struct {
+	shardID      proto.ShardID
+	leaderDiskID proto.DiskID
+	leaderSuid   proto.Suid
+	version      proto.RouteVersion
+	rangeExt     sharding.Range
+	units        []clustermgr.ShardUnit
+	punishCtrl   ServiceController
+}
+
+func (i *shard) Less(item btree.Item) bool {
+	switch than := item.(type) {
+	case *shard:
+		return i.rangeExt.MaxBoundary().Less(than.rangeExt.MaxBoundary())
+	case *compareItem:
+		return i.rangeExt.MaxBoundary().Less(than.ci.GetBoundary())
+	default:
+		return false
+	}
+}
+
+func (i *shard) String() string {
+	return i.rangeExt.String()
+}
+
+func (i *shard) GetShardID() proto.ShardID {
+	return i.shardID
+}
+
+func (i *shard) GetRange() sharding.Range {
+	return i.rangeExt
+}
+
+func (i *shard) GetMember(ctx context.Context, mode acapi.GetShardMode, exclude map[proto.DiskID]struct{}) (ShardOpInfo, error) {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Debugf("get shard member, mode:%d, exclude:%d, shard:%+v", mode, exclude, *i)
+
+	// 1. get member exclude disk id
+	if len(exclude) != 0 {
+		return i.getMemberExcluded(ctx, exclude)
+	}
+
+	// 2. get member by mode
+	if mode == acapi.GetShardModeLeader {
+		return i.getMemberLeader(ctx)
+	}
+	return i.getMemberRandom(ctx, nil)
+}
+
+func (i *shard) getMemberExcluded(ctx context.Context, exclude map[proto.DiskID]struct{}) (ShardOpInfo, error) {
+	return i.getMemberRandom(ctx, exclude)
+}
+
+func (i *shard) getMemberLeader(ctx context.Context) (ShardOpInfo, error) {
+	return ShardOpInfo{
+		DiskID:       i.leaderDiskID,
+		Suid:         i.leaderSuid,
+		RouteVersion: i.version,
+	}, nil
+}
+
+func (i *shard) getMemberRandom(ctx context.Context, exclude map[proto.DiskID]struct{}) (ShardOpInfo, error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	n := len(i.units)
+	initIdx := rand.Intn(n)
+	idx := initIdx
+
+	for {
+		disk, err := i.punishCtrl.GetShardnodeHost(ctx, i.units[idx].DiskID)
+		if err != nil {
+			return ShardOpInfo{}, err
+		}
+		if _, exist := exclude[i.units[idx].DiskID]; !exist && !disk.Punished && !i.units[idx].Learner {
+			return i.getShardOpInfo(idx), nil
+		}
+
+		span.Warnf("skip invalid unit, punished or exclude: %+v, diskID:%d, suid:%d, learner:%t", disk, i.units[idx].DiskID, i.units[idx].Suid, i.units[idx].Learner)
+		idx = (idx + 1) % n
+		if idx == initIdx {
+			break
+		}
+	}
+
+	span.Warnf("can not find expect disk, exclude disk=%d, shard:%+v", exclude, *i)
+	return ShardOpInfo{}, fmt.Errorf("can not find expect shardnode disk")
+}
+
+func (i *shard) getShardOpInfo(idx int) ShardOpInfo {
+	return ShardOpInfo{
+		DiskID:       i.units[idx].DiskID,
+		Suid:         i.units[idx].Suid,
+		RouteVersion: i.version,
+	}
+}
+
+func (i *shard) belong(ci *sharding.CompareItem) bool {
+	return i.rangeExt.Belong(ci)
+}
+
+func (i *shard) contain(rg *sharding.Range) bool {
+	return i.rangeExt.Contain(rg)
+}
+
+type compareItem struct {
+	ci sharding.CompareItem
+}
+
+func (i *compareItem) Less(item btree.Item) bool {
+	than := item.(*shard)
+	return i.ci.GetBoundary().Less(than.rangeExt.MaxBoundary())
+}
+
+func (i *compareItem) String() string {
+	return i.ci.String()
+}
+
+func convertShardUnitInfo(units []clustermgr.ShardUnitInfo) []clustermgr.ShardUnit {
+	ret := make([]clustermgr.ShardUnit, len(units))
+
+	for i, unit := range units {
+		ret[i] = clustermgr.ShardUnit{
+			Suid:    unit.Suid,
+			DiskID:  unit.DiskID,
+			Learner: unit.Learner, // most time, $learner is false
+		}
+	}
+
+	return ret
+}
+
+func isInvalidShardStat(sd shardnode.ShardStats) bool {
+	if sd.Suid == 0 || sd.RouteVersion == 0 || sd.LeaderDiskID == 0 || sd.LeaderSuid == 0 {
+		return true
+	}
+
+	return false
+}
+
+func checkCatalogShardUpdate(val clustermgr.CatalogChangeShardUpdate) error {
+	if val.RouteVersion == 0 || val.ShardID == 0 || val.Unit.Suid == 0 || val.Unit.DiskID == 0 {
+		return errCatalogInvalid
+	}
+	if val.Unit.LeaderDiskID == 0 {
+		return errCatalogNoLeader
+	}
+	return nil
+}
+
+func findAndCheckCatalogShardAdd(val clustermgr.CatalogChangeShardAdd) (int, error) {
+	leaderIdx := -1
+	if val.ShardID == 0 || val.RouteVersion == 0 {
+		return leaderIdx, errCatalogInvalid
+	}
+
+	for i, unit := range val.Units {
+		if unit.Suid == 0 || unit.DiskID == 0 {
+			return leaderIdx, errCatalogInvalid
+		}
+
+		// leader disk may be not in the units
+		if unit.LeaderDiskID == unit.DiskID {
+			leaderIdx = i
+			break
+		}
+	}
+
+	if leaderIdx == -1 {
+		return 0, errCatalogNoLeader
+	}
+	return leaderIdx, nil
+}

blobstore/access/controller/shard_test.go

@@ -0,0 +1,856 @@
+package controller
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/gogo/protobuf/types"
+	"github.com/golang/mock/gomock"
+	"github.com/google/btree"
+	"github.com/stretchr/testify/require"
+
+	acapi "github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/api/shardnode"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/sharding"
+	"github.com/cubefs/cubefs/blobstore/testing/mocks"
+)
+
+var (
+	gAny    = gomock.Any()
+	errMock = errors.New("fake error")
+)
+
+func TestShardController(t *testing.T) {
+	ctx := context.Background()
+
+	stopCh := make(chan struct{})
+	ctr := gomock.NewController(t)
+	cmCli := mocks.NewMockClientAPI(ctr)
+	cmCli.EXPECT().AuthSpace(gAny, gAny).Return(nil)
+	cmCli.EXPECT().GetSpaceByName(gAny, gAny).Return(&clustermgr.Space{
+		SpaceID: 1,
+		Name:    "spaceTest",
+	}, nil)
+	retCatlog := &clustermgr.GetCatalogChangesRet{}
+	retCatlog.RouteVersion = 1
+	cmCli.EXPECT().GetCatalogChanges(gAny, gAny).Return(retCatlog, nil)
+
+	cmCli.EXPECT().GetService(gAny, gAny).Return(clustermgr.ServiceInfo{
+		Nodes: []clustermgr.ServiceNode{
+			{ClusterID: 1, Name: proto.ServiceNameProxy, Host: "proxy-1", Idc: "test-idc"},
+			{ClusterID: 1, Name: proto.ServiceNameProxy, Host: "proxy-2", Idc: "test-idc"},
+		},
+	}, nil)
+	svrCtrl, err := NewServiceController(ServiceConfig{IDC: "test-idc"}, cmCli, nil, nil)
+	require.NoError(t, err)
+
+	require.NoError(t, err)
+	s, err := NewShardController(shardCtrlConf{}, cmCli, svrCtrl, stopCh)
+	require.NotNil(t, err)
+	require.NotEqual(t, errMock, err)
+	require.ErrorIs(t, err, errcode.ErrAccessNotFoundShard)
+	require.Nil(t, s)
+
+	blobName := "blob1"
+	shardKeys := []string{blobName}
+	// empty tree
+	// _, err = s.GetShard(ctx, shardKeys)
+	// require.NotNil(t, err)
+
+	sh := &shard{
+		shardID: 1,
+		version: 1,
+	}
+	// rangePtr := sharding.New(sharding.RangeType_RangeTypeHash, 2)  // keys len=1; subs len=2. will panic
+	rangePtr := sharding.New(sharding.RangeType_RangeTypeHash, 1)
+	for i := range rangePtr.Subs {
+		rangePtr.Subs[i].Min = uint64(i * 1)
+		rangePtr.Subs[i].Max = uint64(i+1) * 1
+	}
+	sh.rangeExt = *rangePtr
+	svr := &shardControllerImpl{
+		shards:      make(map[proto.ShardID]*shard),
+		ranges:      btree.New(defaultBTreeDegree),
+		subRangeCnt: 1,
+	}
+
+	// add one, not found blob
+	svr.addShardNoLock(sh)
+	require.Equal(t, 1, len(svr.shards))
+	_, err = svr.GetShard(ctx, shardKeys)
+	require.NotNil(t, err)
+	svr.delShardNoLock(sh)
+	require.Equal(t, 0, len(svr.shards))
+
+	svr.subRangeCnt = 2
+	ranges := sharding.InitShardingRange(sharding.RangeType_RangeTypeHash, 1, 8)
+	{
+		// add 8 shard
+		shards := make([]*shard, 8)
+		for i := 0; i < 8; i++ {
+			sd := &shard{
+				shardID:      proto.ShardID(i + 1),
+				leaderDiskID: 1,
+				version:      proto.RouteVersion(i + 1),
+				units: []clustermgr.ShardUnit{
+					{
+						Suid:   proto.EncodeSuid(proto.ShardID(i+1), 0, 0),
+						DiskID: 1,
+						// Host:   "testHost1",
+					},
+					{
+						Suid:   proto.EncodeSuid(proto.ShardID(i+1), 1, 0),
+						DiskID: 2,
+					},
+					{
+						Suid:   proto.EncodeSuid(proto.ShardID(i+1), 2, 0),
+						DiskID: 3,
+					},
+				},
+				punishCtrl: svrCtrl,
+			}
+			sd.rangeExt = *ranges[i]
+			shards[i] = sd
+		}
+
+		for i := 0; i < 8; i++ {
+			svr.addShardNoLock(shards[i])
+		}
+		require.Equal(t, 8, len(svr.shards))
+		svr.punishCtrl = svrCtrl
+		svr.version = 8
+
+		// ret, err := svr.GetShard(ctx, []byte("blob1__xxx")) // expect 2 keys
+		ret, err := svr.GetShard(ctx, []string{"blob1__xxx"}) // expect 2 keys
+		sk := []string{"blob1__xxx"}
+		bd := sharding.NewCompareItem(sharding.RangeType_RangeTypeHash, sk).GetBoundary()
+		t.Logf("shard key 1, key boundary=%d, shardBounary=%d, range=%s, treeLen=%d", bd, ret.(*shard).rangeExt.MaxBoundary(), ret.(*shard).String(), svr.ranges.Len())
+		// for i := range shards {
+		//	sBd := shards[i].rangeExt.MaxBoundary()
+		//	t.Logf("shard=%d, boundary=%d, isLess=%v", shards[i].shardID, sBd, bd.Less(sBd))
+		// }
+		require.Nil(t, err)
+		require.Equal(t, proto.ShardID(2), ret.(*shard).shardID)
+
+		// ret, err = svr.GetShard(ctx, []byte("{blob2__yy}{11}"))
+		ret, err = svr.GetShard(ctx, []string{"blob2__yy", "11"})
+		bd = sharding.NewCompareItem(sharding.RangeType_RangeTypeHash, []string{"blob2__yy", "11"}).GetBoundary()
+		t.Logf("shard key 2,  get boundary=%d", bd)
+		require.Nil(t, err)
+		require.Equal(t, proto.ShardID(7), ret.(*shard).shardID)
+	}
+
+	{
+		// get
+		si, ok := svr.getShardByID(1)
+		require.True(t, ok)
+		require.Equal(t, proto.ShardID(1), si.shardID)
+
+		svr.spaceID = 1
+		spID := svr.GetSpaceID()
+		require.Equal(t, proto.SpaceID(1), spID)
+
+		// update shard, switch leader 3 -> 2 ; (1, 2 leader, 4)
+		newShard := *si
+		// newShard.version++ // shard node switch leader don't increment version
+		newShard.leaderDiskID = 2
+		newShard.units[2].Learner = true
+		newShard.units = append(newShard.units, clustermgr.ShardUnit{
+			Suid:   proto.EncodeSuid(newShard.shardID, 2, 1),
+			DiskID: 4,
+		})
+		err = svr.UpdateShard(ctx, shardnode.ShardStats{
+			Suid:         proto.EncodeSuid(newShard.shardID, 1, 1),
+			LeaderDiskID: newShard.leaderDiskID,
+			LeaderSuid:   proto.EncodeSuid(newShard.shardID, 1, 1),
+			RouteVersion: newShard.version,
+			Range:        newShard.rangeExt,
+			Units:        newShard.units,
+		})
+		require.NoError(t, err)
+		require.Equal(t, newShard.leaderDiskID, si.leaderDiskID)
+
+		si, ok = svr.getShardByID(1)
+		require.True(t, ok)
+		require.Equal(t, clustermgr.ShardUnit{Suid: proto.EncodeSuid(1, 0, 0), DiskID: 1}, si.units[0])
+		require.Equal(t, clustermgr.ShardUnit{Suid: proto.EncodeSuid(1, 1, 0), DiskID: 2}, si.units[1])
+		// require.Equal(t, clustermgr.ShardUnit{Suid: proto.EncodeSuid(1, 2, 1), DiskID: 4}, si.units[2])
+
+		opInfo, err := si.GetMember(ctx, acapi.GetShardModeLeader, nil)
+		require.NoError(t, err)
+		require.Equal(t, ShardOpInfo{
+			DiskID:       2,
+			Suid:         proto.EncodeSuid(1, 1, 1),
+			RouteVersion: 1,
+		}, opInfo)
+
+		// update route, disk:3 -> disk:4
+		si.units[2] = clustermgr.ShardUnit{Suid: proto.EncodeSuid(1, 2, 1), DiskID: 4}
+
+		// switch leader 2 -> 4;  (1, 2, 4 leader)
+		newShard.units[2] = newShard.units[3]
+		newShard.units = newShard.units[:3]
+		newShard.leaderDiskID = 4
+		newShard.leaderSuid = proto.EncodeSuid(newShard.shardID, 2, 2)
+		newShard.version = si.version
+		err = svr.UpdateShard(ctx, shardnode.ShardStats{
+			Suid:         newShard.units[2].Suid,
+			LeaderDiskID: newShard.leaderDiskID,
+			LeaderSuid:   newShard.leaderSuid,
+			RouteVersion: newShard.version,
+			Range:        newShard.rangeExt,
+			Units:        newShard.units,
+		})
+		require.NoError(t, err)
+		require.Equal(t, newShard.leaderDiskID, si.leaderDiskID)
+
+		si, ok = svr.getShardByID(1)
+		require.True(t, ok)
+		require.Equal(t, newShard, *si)
+	}
+
+	{
+		// switch leader: oldLeader 4 -> leader 2, units[1,2,4]
+		si, ok := svr.getShardByID(1)
+		require.True(t, ok)
+		require.Equal(t, proto.ShardID(1), si.shardID)
+
+		newShard := *si
+		newShard.leaderDiskID = 2
+		newShard.leaderSuid = si.units[1].Suid // disk 2 old suid proto.EncodeSuid(newShard.shardID, 1, 0)
+
+		err = svr.UpdateShard(ctx, shardnode.ShardStats{
+			Suid:         newShard.units[2].Suid,
+			LeaderDiskID: newShard.leaderDiskID,
+			LeaderSuid:   newShard.leaderSuid,
+			RouteVersion: newShard.version,
+		})
+		require.NoError(t, err)
+		require.Equal(t, newShard.leaderDiskID, si.leaderDiskID)
+		require.Equal(t, newShard.leaderSuid, si.leaderSuid)
+
+		si, ok = svr.getShardByID(1)
+		require.True(t, ok)
+		require.Equal(t, newShard, *si)
+	}
+
+	{
+		err = svr.UpdateShard(ctx, shardnode.ShardStats{
+			Suid:         1,
+			LeaderDiskID: 2,
+			LeaderSuid:   0,
+			RouteVersion: 1,
+		})
+		require.ErrorIs(t, err, errShardInvalid)
+	}
+}
+
+func TestShardUpdate(t *testing.T) {
+	ctx := context.Background()
+	ctr := gomock.NewController(t)
+	cmCli := mocks.NewMockClientAPI(ctr)
+	svr := &shardControllerImpl{
+		shards: make(map[proto.ShardID]*shard),
+		ranges: btree.New(defaultBTreeDegree),
+	}
+
+	// concurrence
+	{
+		// update
+		cmCli.EXPECT().GetCatalogChanges(gAny, gAny).DoAndReturn(
+			func(ctx context.Context, args *clustermgr.GetCatalogChangesArgs) (ret *clustermgr.GetCatalogChangesRet, err error) {
+				time.Sleep(time.Millisecond * 10)
+				return &clustermgr.GetCatalogChangesRet{}, errMock
+			}).Times(1)
+		svr.cmCli = cmCli
+
+		resultCh := make(chan error, 3)
+		for i := 0; i < 3; i++ {
+			go func() {
+				err := svr.UpdateRoute(ctx)
+				resultCh <- err
+			}()
+		}
+		for i := 0; i < 3; i++ {
+			err := <-resultCh
+			require.ErrorIs(t, err, errMock)
+		}
+	}
+
+	ranges := sharding.InitShardingRange(sharding.RangeType_RangeTypeHash, 1, 10)
+	{
+		// error
+		val := clustermgr.CatalogChangeShardAdd{
+			ShardID: 0,
+		}
+		data, err := val.Marshal()
+		require.NoError(t, err)
+
+		retCatlog := &clustermgr.GetCatalogChangesRet{
+			RouteVersion: 1,
+			Items: []clustermgr.CatalogChangeItem{
+				{
+					Type:         proto.CatalogChangeItemAddShard,
+					RouteVersion: 1,
+					Item: &types.Any{
+						TypeUrl: "",
+						Value:   data,
+					},
+				},
+			},
+		}
+		cmCli.EXPECT().GetCatalogChanges(gAny, gAny).Return(retCatlog, nil)
+
+		err = svr.UpdateRoute(ctx)
+		require.ErrorIs(t, err, errCatalogInvalid)
+	}
+
+	{
+		// update, leader disk is 0, 2 success and 2 wrong
+		val := clustermgr.CatalogChangeShardAdd{
+			ShardID:      1,
+			RouteVersion: 1,
+			Units: []clustermgr.ShardUnitInfo{
+				{
+					Suid:         proto.EncodeSuid(1, 0, 0),
+					DiskID:       1,
+					AppliedIndex: 0,
+					LeaderDiskID: 1,
+					Range:        *ranges[0],
+					RouteVersion: 1,
+					Host:         "testHost1",
+					Learner:      false,
+				},
+			},
+		}
+		data1, err := val.Marshal()
+		require.NoError(t, err)
+
+		val = clustermgr.CatalogChangeShardAdd{
+			ShardID:      2,
+			RouteVersion: 2,
+			Units: []clustermgr.ShardUnitInfo{
+				{
+					Suid:         proto.EncodeSuid(2, 0, 0),
+					DiskID:       2,
+					LeaderDiskID: 0, // wrong
+					Range:        *ranges[1],
+					RouteVersion: 2,
+				},
+			},
+		}
+		data2, err := val.Marshal()
+		require.NoError(t, err)
+
+		val3 := clustermgr.CatalogChangeShardUpdate{
+			ShardID:      1,
+			RouteVersion: 3,
+			Unit: clustermgr.ShardUnitInfo{
+				Suid:         proto.EncodeSuid(1, 0, 1),
+				DiskID:       3,
+				LeaderDiskID: 0, // wrong
+				RouteVersion: 3,
+				Range:        *ranges[0],
+			},
+		}
+		data3, err := val3.Marshal()
+		require.NoError(t, err)
+
+		val4 := clustermgr.CatalogChangeShardUpdate{
+			ShardID:      1,
+			RouteVersion: 4,
+			Unit: clustermgr.ShardUnitInfo{
+				Suid:         proto.EncodeSuid(1, 0, 2),
+				DiskID:       3,
+				LeaderDiskID: 3,
+				RouteVersion: 3,
+				Range:        *ranges[0],
+			},
+		}
+		data4, err := val4.Marshal()
+		require.NoError(t, err)
+
+		retCatlog := &clustermgr.GetCatalogChangesRet{
+			RouteVersion: val4.RouteVersion,
+			Items: []clustermgr.CatalogChangeItem{
+				{
+					Type:         proto.CatalogChangeItemAddShard,
+					RouteVersion: 1,
+					Item:         &types.Any{TypeUrl: "", Value: data1}, // success
+				},
+				{
+					Type:         proto.CatalogChangeItemAddShard,
+					RouteVersion: 2,
+					Item:         &types.Any{Value: data2}, // wrong
+				},
+				{
+					Type:         proto.CatalogChangeItemUpdateShard,
+					RouteVersion: 3,
+					Item:         &types.Any{Value: data3}, // wrong
+				},
+				{
+					Type:         proto.CatalogChangeItemUpdateShard,
+					RouteVersion: 4,
+					Item:         &types.Any{Value: data4}, // success
+				},
+			},
+		}
+		cmCli.EXPECT().GetCatalogChanges(gAny, gAny).Return(retCatlog, nil)
+
+		err = svr.UpdateRoute(ctx)
+		require.Equal(t, errCatalogNoLeader, err)
+		require.Equal(t, proto.RouteVersion(4), svr.version)
+		require.Equal(t, 2, len(svr.shards))
+	}
+
+	{
+		// add shard id=9
+		const shardID = 9
+		const version = 5
+		val := clustermgr.CatalogChangeShardAdd{
+			ShardID:      shardID,
+			RouteVersion: version,
+			Units: []clustermgr.ShardUnitInfo{
+				{
+					Suid:         proto.EncodeSuid(shardID, 0, 0),
+					DiskID:       1,
+					AppliedIndex: 0,
+					LeaderDiskID: 2,
+					Range:        *ranges[8],
+					RouteVersion: version,
+					Host:         "testHost1",
+					Learner:      false,
+				},
+				{
+					Suid:         proto.EncodeSuid(shardID, 1, 0),
+					DiskID:       2,
+					AppliedIndex: 0,
+					LeaderDiskID: 2,
+					Range:        *ranges[8],
+					RouteVersion: version,
+					Host:         "testHost2",
+					Learner:      false,
+				},
+				{
+					Suid:         proto.EncodeSuid(shardID, 2, 0),
+					DiskID:       3,
+					AppliedIndex: 0,
+					LeaderDiskID: 2,
+					Range:        *ranges[8],
+					RouteVersion: version,
+					Host:         "testHost3",
+					Learner:      false,
+				},
+			},
+		}
+		data, err := val.Marshal()
+		require.NoError(t, err)
+		retCatlog := &clustermgr.GetCatalogChangesRet{
+			RouteVersion: version,
+			Items: []clustermgr.CatalogChangeItem{
+				{
+					RouteVersion: version,
+					Type:         proto.CatalogChangeItemAddShard,
+					Item: &types.Any{
+						TypeUrl: "",
+						Value:   data,
+					},
+				},
+			},
+		}
+		cmCli.EXPECT().GetCatalogChanges(gAny, gAny).Return(retCatlog, nil)
+		svr.cmCli = cmCli
+		err = svr.UpdateRoute(ctx)
+		require.NoError(t, err)
+		require.Equal(t, proto.RouteVersion(version), svr.version)
+
+		si, ok := svr.getShardByID(shardID)
+		require.True(t, ok)
+		require.Equal(t, proto.ShardID(shardID), si.shardID)
+
+		require.Equal(t, 3, len(svr.shards))
+
+		expect := shard{
+			shardID:      shardID,
+			leaderDiskID: val.Units[0].LeaderDiskID,
+			leaderSuid:   val.Units[1].Suid,
+			version:      version,
+			rangeExt:     val.Units[1].Range,
+			units:        convertShardUnitInfo(val.Units),
+			punishCtrl:   svr.punishCtrl,
+		}
+		require.Equal(t, expect, *si)
+	}
+
+	{
+		// add shard id=10
+		const shardID = 10
+		const oldVersion = 4
+		const version = 5
+		sd := &shard{
+			shardID: proto.ShardID(shardID),
+			version: oldVersion,
+			units:   make([]clustermgr.ShardUnit, 1),
+		}
+		sd.rangeExt = *ranges[9]
+		svr.addShardNoLock(sd)
+		require.Equal(t, 4, len(svr.shards))
+
+		val := clustermgr.CatalogChangeShardUpdate{
+			ShardID:      shardID,
+			RouteVersion: version,
+			Unit: clustermgr.ShardUnitInfo{
+				Suid:         proto.EncodeSuid(shardID, 0, 0),
+				DiskID:       2,
+				AppliedIndex: 0,
+				LeaderDiskID: 2,
+				Range:        *ranges[9],
+				RouteVersion: version,
+				Host:         "testHost2",
+				Learner:      false,
+			},
+		}
+		data, err := val.Marshal()
+		require.NoError(t, err)
+		retCatlog := &clustermgr.GetCatalogChangesRet{
+			RouteVersion: version,
+			Items: []clustermgr.CatalogChangeItem{
+				{
+					Type:         proto.CatalogChangeItemUpdateShard,
+					RouteVersion: version,
+					Item: &types.Any{
+						TypeUrl: "",
+						Value:   data,
+					},
+				},
+			},
+		}
+		cmCli.EXPECT().GetCatalogChanges(gAny, gAny).Return(retCatlog, nil)
+		svr.cmCli = cmCli
+		err = svr.UpdateRoute(ctx)
+		require.NoError(t, err)
+		require.Equal(t, proto.RouteVersion(version), svr.version)
+
+		si, ok := svr.getShardByID(shardID)
+		require.True(t, ok)
+		require.Equal(t, proto.ShardID(shardID), si.shardID)
+
+		require.Equal(t, 4, len(svr.shards))
+	}
+
+	{
+		// update version, leader disk, suid
+		rv := svr.version + 1
+		shardID := proto.ShardID(9)
+		oldShard, exist := svr.getShardNoLock(shardID)
+		require.True(t, exist)
+
+		val := clustermgr.CatalogChangeShardUpdate{
+			ShardID:      shardID,
+			RouteVersion: rv,
+			Unit: clustermgr.ShardUnitInfo{
+				Suid:         proto.EncodeSuid(shardID, 2, 1),
+				DiskID:       3,
+				LeaderDiskID: 2,
+				Range:        *ranges[8],
+				RouteVersion: rv,
+				Host:         "testHost3",
+				Learner:      false,
+			},
+		}
+		data, err := val.Marshal()
+		require.NoError(t, err)
+
+		item := clustermgr.CatalogChangeItem{
+			RouteVersion: svr.version + 1,
+			Type:         proto.CatalogChangeItemUpdateShard,
+			Item: &types.Any{
+				TypeUrl: "",
+				Value:   data,
+			},
+		}
+		err = svr.handleShardUpdate(ctx, item)
+		require.NoError(t, err)
+
+		sd, exist := svr.getShardNoLock(shardID)
+		require.True(t, exist)
+
+		expect := shard{
+			shardID:      oldShard.shardID,
+			leaderDiskID: 2,
+			leaderSuid:   proto.EncodeSuid(shardID, 1, 0),
+			version:      rv,
+			rangeExt:     oldShard.rangeExt,
+			units:        oldShard.units,
+			punishCtrl:   oldShard.punishCtrl,
+		}
+		idx := val.Unit.Suid.Index()
+		expect.units[idx] = clustermgr.ShardUnit{
+			Suid:    val.Unit.Suid,
+			DiskID:  val.Unit.DiskID,
+			Learner: val.Unit.Learner,
+		}
+
+		require.Equal(t, expect, *sd)
+	}
+
+	{
+		// update version, leader disk not in units(old leader)
+		rv := svr.version + 1
+		shardID := proto.ShardID(9)
+		oldShard, exist := svr.getShardNoLock(shardID)
+		require.True(t, exist)
+
+		val := clustermgr.CatalogChangeShardUpdate{
+			ShardID:      shardID,
+			RouteVersion: rv,
+			Unit: clustermgr.ShardUnitInfo{
+				Suid:         proto.EncodeSuid(shardID, 2, 2),
+				DiskID:       5,
+				LeaderDiskID: 3,
+				Range:        *ranges[8],
+				RouteVersion: rv,
+				Host:         "testHost3",
+				Learner:      false,
+			},
+		}
+		data, err := val.Marshal()
+		require.NoError(t, err)
+
+		item := clustermgr.CatalogChangeItem{
+			RouteVersion: svr.version + 1,
+			Type:         proto.CatalogChangeItemUpdateShard,
+			Item: &types.Any{
+				TypeUrl: "",
+				Value:   data,
+			},
+		}
+		err = svr.handleShardUpdate(ctx, item)
+		require.NoError(t, err)
+
+		sd, exist := svr.getShardNoLock(shardID)
+		require.True(t, exist)
+
+		opHeader, err := sd.GetMember(ctx, acapi.GetShardModeLeader, nil)
+		require.NoError(t, err)
+		expect := ShardOpInfo{
+			DiskID:       1,
+			Suid:         proto.EncodeSuid(shardID, 0, 0),
+			RouteVersion: rv,
+		}
+
+		require.Equal(t, expect, opHeader)
+		require.Equal(t, oldShard.leaderDiskID, expect.DiskID)
+	}
+
+	{
+		// leader disk is 0
+		rv := svr.version + 1
+		shardID := proto.ShardID(9)
+		oldShard, exist := svr.getShardNoLock(shardID)
+		require.True(t, exist)
+
+		val := clustermgr.CatalogChangeShardUpdate{
+			ShardID:      shardID,
+			RouteVersion: rv,
+			Unit: clustermgr.ShardUnitInfo{
+				Suid:         proto.EncodeSuid(shardID, 1, 1),
+				DiskID:       4,
+				LeaderDiskID: 0,
+				Range:        *ranges[8],
+				RouteVersion: rv,
+			},
+		}
+		data, err := val.Marshal()
+		require.NoError(t, err)
+
+		item := clustermgr.CatalogChangeItem{
+			RouteVersion: rv,
+			Type:         proto.CatalogChangeItemUpdateShard,
+			Item: &types.Any{
+				TypeUrl: "",
+				Value:   data,
+			},
+		}
+		err = svr.handleShardUpdate(ctx, item)
+		require.ErrorIs(t, err, errCatalogNoLeader)
+
+		sd, exist := svr.getShardNoLock(shardID)
+		require.True(t, exist)
+
+		expect := shard{
+			shardID:      oldShard.shardID,
+			leaderDiskID: 4,
+			leaderSuid:   proto.EncodeSuid(shardID, 1, 1),
+			version:      rv,
+			rangeExt:     oldShard.rangeExt,
+			units:        oldShard.units,
+			punishCtrl:   oldShard.punishCtrl,
+		}
+		idx := val.Unit.Suid.Index()
+		expect.units[idx] = clustermgr.ShardUnit{
+			Suid:    val.Unit.Suid,
+			DiskID:  val.Unit.DiskID,
+			Learner: val.Unit.Learner,
+		}
+
+		require.Equal(t, expect, *sd)
+
+		// error
+		val.RouteVersion = 1
+		data, err = val.Marshal()
+		require.NoError(t, err)
+		item.RouteVersion = val.RouteVersion
+		item.Item.Value = data
+		err = svr.handleShardUpdate(ctx, item)
+		require.ErrorIs(t, err, errCatalogInvalid)
+	}
+}
+
+func TestShardGetShard(t *testing.T) {
+	ctx := context.Background()
+	svr := &shardControllerImpl{
+		shards:      make(map[proto.ShardID]*shard),
+		ranges:      btree.New(defaultBTreeDegree),
+		subRangeCnt: 2,
+	}
+
+	ctr := gomock.NewController(t)
+	cmCli := mocks.NewMockClientAPI(ctr)
+	cmCli.EXPECT().GetService(gAny, gAny).Return(clustermgr.ServiceInfo{
+		Nodes: []clustermgr.ServiceNode{
+			{ClusterID: 1, Name: proto.ServiceNameProxy, Host: "proxy-1", Idc: "test-idc"},
+		},
+	}, nil)
+	cmCli.EXPECT().ShardNodeDiskInfo(gAny, proto.DiskID(1)).Return(&clustermgr.ShardNodeDiskInfo{
+		DiskInfo:                   clustermgr.DiskInfo{Host: "testHost1", Idc: "test-idc"},
+		ShardNodeDiskHeartbeatInfo: clustermgr.ShardNodeDiskHeartbeatInfo{DiskID: 1},
+	}, nil)
+	cmCli.EXPECT().ShardNodeDiskInfo(gAny, proto.DiskID(2)).Return(&clustermgr.ShardNodeDiskInfo{
+		DiskInfo:                   clustermgr.DiskInfo{Host: "testHost2", Idc: "test-idc"},
+		ShardNodeDiskHeartbeatInfo: clustermgr.ShardNodeDiskHeartbeatInfo{DiskID: 2},
+	}, nil)
+	cmCli.EXPECT().ShardNodeDiskInfo(gAny, proto.DiskID(3)).Return(&clustermgr.ShardNodeDiskInfo{
+		DiskInfo:                   clustermgr.DiskInfo{Host: "testHost3", Idc: "test-idc"},
+		ShardNodeDiskHeartbeatInfo: clustermgr.ShardNodeDiskHeartbeatInfo{DiskID: 3},
+	}, nil)
+	svrCtrl, err := NewServiceController(ServiceConfig{IDC: "test-idc"}, cmCli, nil, nil)
+	require.NoError(t, err)
+	svrCtrl.GetShardnodeHost(ctx, 1)
+	svrCtrl.GetShardnodeHost(ctx, 2)
+	svrCtrl.GetShardnodeHost(ctx, 3)
+
+	// add 8 shard
+	shards := make([]*shard, 8)
+	ranges := sharding.InitShardingRange(sharding.RangeType_RangeTypeHash, 2, 8)
+
+	for i := 0; i < 8; i++ {
+		sd := &shard{
+			shardID:      proto.ShardID(i + 1),
+			version:      1,
+			leaderDiskID: 1,
+			leaderSuid:   proto.EncodeSuid(proto.ShardID(i+1), 0, 0),
+			units: []clustermgr.ShardUnit{
+				{
+					Suid:   proto.EncodeSuid(proto.ShardID(i+1), 0, 0),
+					DiskID: 1,
+					Host:   "testHost1",
+				},
+				{
+					Suid:   proto.EncodeSuid(proto.ShardID(i+1), 1, 0),
+					DiskID: 2,
+					Host:   "testHost2",
+				},
+				{
+					Suid:   proto.EncodeSuid(proto.ShardID(i+1), 2, 0),
+					DiskID: 3,
+					Host:   "testHost3",
+				},
+			},
+			punishCtrl: svrCtrl,
+		}
+		sd.rangeExt = *ranges[i]
+		shards[i] = sd
+	}
+	// shards[7].rangeExt.Subs[0].Max = math.MaxUint64
+
+	for i := 0; i < 8; i++ {
+		svr.addShardNoLock(shards[i])
+	}
+	require.Equal(t, 8, len(svr.shards))
+
+	{
+		// sd, err := svr.GetShard(ctx, []byte("{blob1}{1}"))
+		sd, err := svr.GetShard(ctx, []string{"blob1", "1"})
+		require.NoError(t, err)
+		require.Equal(t, proto.ShardID(2), sd.GetShardID())
+
+		// sd, err = svr.GetShard(ctx, []byte("{blob1}{}"))
+		sd, err = svr.GetShard(ctx, []string{"blob1", ""})
+		require.NoError(t, err)
+		require.Equal(t, proto.ShardID(2), sd.GetShardID())
+
+		// shard others
+		sd, err = svr.GetShardByID(ctx, proto.ShardID(1))
+		require.NoError(t, err)
+
+		shardInfo, err := sd.GetMember(ctx, acapi.GetShardModeLeader, nil)
+		require.NoError(t, err)
+		require.Equal(t, shards[0].shardID, shardInfo.Suid.ShardID())
+
+		shardInfo, err = sd.GetMember(ctx, acapi.GetShardModeRandom, nil)
+		require.NoError(t, err)
+		require.Equal(t, shards[0].shardID, shardInfo.Suid.ShardID())
+
+		shardID := sd.GetShardID()
+		require.Equal(t, shards[0].shardID, shardID)
+	}
+
+	// get first shard
+	{
+		sd, err := svr.GetFisrtShard(ctx)
+		require.NoError(t, err)
+		require.Equal(t, proto.ShardID(1), sd.GetShardID())
+
+		newDisk, err := sd.GetMember(ctx, acapi.GetShardModeRandom, map[proto.DiskID]struct{}{2: {}, 1: {}})
+		require.NoError(t, err)
+		require.NotEqual(t, proto.DiskID(2), newDisk.DiskID)
+		require.NotEqual(t, proto.DiskID(1), newDisk.DiskID)
+		require.Contains(t, []proto.DiskID{3}, newDisk.DiskID)
+	}
+
+	// get shard by range
+	{
+		sd, err := svr.GetShardByID(ctx, proto.ShardID(2))
+		require.NoError(t, err)
+		shardRange := sd.GetRange()
+
+		sd, err = svr.GetShardByRange(ctx, shardRange)
+		require.NoError(t, err)
+		require.Equal(t, proto.ShardID(2), sd.GetShardID())
+		require.Equal(t, shardRange, sd.GetRange())
+
+		sd, err = svr.GetShardByRange(ctx, shards[0].rangeExt)
+		require.NoError(t, err)
+		require.Equal(t, proto.ShardID(1), sd.GetShardID())
+
+		sd, err = svr.GetShardByRange(ctx, shards[7].rangeExt)
+		require.NoError(t, err)
+		require.Equal(t, proto.ShardID(8), sd.GetShardID())
+
+		sd, err = svr.GetNextShard(ctx, shardRange)
+		require.NoError(t, err)
+		require.Equal(t, proto.ShardID(3), sd.GetShardID())
+		require.NotEqual(t, shardRange, sd.GetRange())
+
+		// err == nil && shard == nil, means last shard, reach end
+		sd, err = svr.GetNextShard(ctx, shards[7].rangeExt)
+		require.NoError(t, err)
+		require.Nil(t, sd)
+	}
+}

blobstore/access/controller/volume.go

@@ -17,6 +17,7 @@ package controller
 import (
 	"context"
 	"fmt"
+	"sync"
 	"time"
 
 	"golang.org/x/sync/singleflight"
@@ -29,15 +30,11 @@ import (
 	"github.com/cubefs/cubefs/blobstore/common/proto"
 	"github.com/cubefs/cubefs/blobstore/common/rpc"
 	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/util/defaulter"
 	"github.com/cubefs/cubefs/blobstore/util/errors"
 	"github.com/cubefs/cubefs/blobstore/util/retry"
 )
 
-const (
-	_defaultCacheSize       = 1 << 20
-	_defaultCacheExpiration = int64(2 * time.Minute)
-)
-
 // Unit alias of clustermgr.Unit
 type Unit = clustermgr.Unit
 
@@ -51,23 +48,23 @@ type VolumePhy struct {
 	Vid       proto.Vid
 	CodeMode  codemode.CodeMode
 	IsPunish  bool
-	Version   uint32
+	Version   uint64
 	Timestamp int64
 	Units     []Unit
 }
 
 // VolumeGetter getter of volume physical location
 //
-// ctx: context with trace or something
-//
-// isCache: is false means reading from proxy cluster then updating memcache
-//
-//	otherwise reading from memcache -> proxy -> cluster
+//	ctx: context with trace or something
+//	isCache: is false means reading from proxy cluster then updating memcache
+//		otherwise reading from memcache -> proxy -> cluster
 type VolumeGetter interface {
 	// Get returns volume physical location of vid
 	Get(ctx context.Context, vid proto.Vid, isCache bool) *VolumePhy
 	// Punish punish vid with interval seconds
 	Punish(ctx context.Context, vid proto.Vid, punishIntervalS int)
+	// Update try to flush volume of proxy
+	Update(ctx context.Context, vid proto.Vid)
 }
 type cvid uint64
 
@@ -106,9 +103,6 @@ func (vc *volumeMemCache) Set(key cvid, value *VolumePhy) {
 }
 
 type volumeGetterImpl struct {
-	ctx context.Context
-	cid proto.ClusterID
-
 	volumeMemCache volumePhyCacher
 	memExpiration  int64
 	punishCache    *memcache.MemCache
@@ -116,40 +110,71 @@ type volumeGetterImpl struct {
 	service   ServiceController
 	proxy     proxy.Cacher
 	singleRun *singleflight.Group
+
+	unusualLock   sync.Mutex
+	unusualVolume map[proto.Vid]int
+
+	config VolumeConfig
+}
+
+// VolumeConfig controller of volume's config
+type VolumeConfig struct {
+	ClusterID proto.ClusterID `json:"-"`
+
+	VolumeMemcacheSize         int   `json:"volume_memcache_size"`
+	VolumeMemcachePunishSize   int   `json:"volume_memcache_punish_size"`
+	VolumeMemcacheExpirationMs int64 `json:"volume_memcache_expiration_ms"` // -1 means no expiration
+	VolumePunishThreshold      int   `json:"volume_punish_threshold"`
+	VolumePunishIntervalS      int   `json:"volume_punish_interval_s"`
 }
 
 // NewVolumeGetter new a volume getter
-//
-//	memExpiration expiration of memcache, 0 means no expiration
-func NewVolumeGetter(clusterID proto.ClusterID, service ServiceController,
-	proxy proxy.Cacher, memExpiration time.Duration) (VolumeGetter, error) {
-	_, ctx := trace.StartSpanFromContext(context.Background(), "")
+func NewVolumeGetter(cfg VolumeConfig, service ServiceController,
+	proxy proxy.Cacher, stop <-chan struct{},
+) (VolumeGetter, error) {
+	expiration := cfg.VolumeMemcacheExpirationMs * int64(time.Millisecond)
+	defaulter.IntegerEqual(&expiration, int64(2*time.Minute))
+	defaulter.IntegerLess(&expiration, 0)
 
-	expiration := int64(memExpiration)
-	if expiration < 0 {
-		expiration = _defaultCacheExpiration
-	}
+	defaulter.IntegerLessOrEqual(&cfg.VolumeMemcacheSize, 1<<20)
+	defaulter.IntegerLessOrEqual(&cfg.VolumeMemcachePunishSize, 1<<10)
+	defaulter.IntegerLessOrEqual(&cfg.VolumePunishThreshold, 10)
+	defaulter.IntegerLessOrEqual(&cfg.VolumePunishIntervalS, 600)
 
-	mc, err := memcache.NewMemCache(_defaultCacheSize)
+	mc, err := memcache.NewMemCache(cfg.VolumeMemcacheSize)
 	if err != nil {
 		return nil, err
 	}
-	punishCache, err := memcache.NewMemCache(1024)
+	punishCache, err := memcache.NewMemCache(cfg.VolumeMemcachePunishSize)
 	if err != nil {
 		return nil, err
 	}
 
 	getter := &volumeGetterImpl{
-		ctx:            ctx,
-		cid:            clusterID,
 		volumeMemCache: &volumeMemCache{cache: mc},
 		memExpiration:  expiration,
 		punishCache:    punishCache,
 		service:        service,
 		proxy:          proxy,
 		singleRun:      new(singleflight.Group),
+		unusualVolume:  make(map[proto.Vid]int),
+		config:         cfg,
 	}
 
+	go func() {
+		ticker := time.NewTicker(time.Duration(cfg.VolumePunishIntervalS) * time.Second)
+		defer ticker.Stop()
+		for {
+			getter.tickerUpdate()
+			select {
+			case <-ticker.C:
+			case <-stop:
+				getter.tickerUpdate()
+				return
+			}
+		}
+	}()
+
 	return getter, nil
 }
 
@@ -159,8 +184,8 @@ func NewVolumeGetter(clusterID proto.ClusterID, service ServiceController,
 //	2.second level cache from proxy cluster
 func (v *volumeGetterImpl) Get(ctx context.Context, vid proto.Vid, isCache bool) (phy *VolumePhy) {
 	span := trace.SpanFromContextSafe(ctx)
-	cid := v.cid.ToString()
-	id := addCVid(v.cid, vid)
+	cid := v.config.ClusterID.ToString()
+	id := addCVid(v.config.ClusterID, vid)
 
 	// check if volume punish
 	defer func() {
@@ -215,7 +240,7 @@ func (v *volumeGetterImpl) Get(ctx context.Context, vid proto.Vid, isCache bool)
 	}
 
 	singleID := fmt.Sprintf("get-volume-%d", id)
-	ver := uint32(0)
+	ver := uint64(0)
 	if phy != nil {
 		ver = phy.Version
 	}
@@ -225,6 +250,7 @@ func (v *volumeGetterImpl) Get(ctx context.Context, vid proto.Vid, isCache bool)
 	if err != nil {
 		cacheMetric.WithLabelValues(cid, "proxy", "miss").Inc()
 		span.Error("get volume location from proxy failed", errors.Detail(err))
+		phy = nil
 		return
 	}
 	cacheMetric.WithLabelValues(cid, "proxy", "hit").Inc()
@@ -238,34 +264,41 @@ func (v *volumeGetterImpl) Get(ctx context.Context, vid proto.Vid, isCache bool)
 	return
 }
 
-func (v *volumeGetterImpl) Punish(ctx context.Context, vid proto.Vid, punishIntervalS int) {
-	v.punishCache.Set(addCVid(v.cid, vid), time.Now().Add(time.Duration(punishIntervalS)*time.Second).Unix())
+func (v *volumeGetterImpl) Punish(_ context.Context, vid proto.Vid, punishIntervalS int) {
+	v.punishCache.Set(addCVid(v.config.ClusterID, vid), time.Now().Add(time.Duration(punishIntervalS)*time.Second).Unix())
 }
 
-func (v *volumeGetterImpl) setToLocalCache(ctx context.Context, id cvid, phy *VolumePhy) {
+func (v *volumeGetterImpl) Update(_ context.Context, vid proto.Vid) {
+	v.unusualLock.Lock()
+	v.unusualVolume[vid] += 1
+	v.unusualLock.Unlock()
+}
+
+func (v *volumeGetterImpl) setToLocalCache(_ context.Context, id cvid, phy *VolumePhy) {
 	v.volumeMemCache.Set(id, phy)
 }
 
-func (v *volumeGetterImpl) getFromLocalCache(ctx context.Context, id cvid) *VolumePhy {
+func (v *volumeGetterImpl) getFromLocalCache(_ context.Context, id cvid) *VolumePhy {
 	return v.volumeMemCache.Get(id)
 }
 
-func (v *volumeGetterImpl) getFromProxy(ctx context.Context, vid proto.Vid, flush bool, ver uint32) (*VolumePhy, error) {
+func (v *volumeGetterImpl) getFromProxy(ctx context.Context, vid proto.Vid, flush bool, ver uint64) (*VolumePhy, error) {
 	span := trace.SpanFromContextSafe(ctx)
 	hosts, err := v.service.GetServiceHosts(ctx, proto.ServiceNameProxy)
 	if err != nil {
 		return nil, err
 	}
 
-	var volume *proxy.VersionVolume
-	id := addCVid(v.cid, vid)
+	var volume *clustermgr.VolumeInfo
+	cid := v.config.ClusterID
+	id := addCVid(cid, vid)
 	triedHosts := make(map[string]struct{})
 	if err = retry.ExponentialBackoff(3, 30).RuptOn(func() (bool, error) {
 		for _, host := range hosts {
 			triedHosts[host] = struct{}{}
 			if volume, err = v.proxy.GetCacheVolume(ctx, host,
 				&proxy.CacheVolumeArgs{Vid: vid, Flush: flush, Version: ver}); err != nil {
-				if rpc.DetectStatusCode(err) == errcode.CodeVolumeNotExist {
+				if err == context.Canceled || rpc.DetectStatusCode(err) == errcode.CodeVolumeNotExist {
 					return true, err
 				}
 				span.Warnf("get from proxy(%s) volume(%d) error(%s)", host, vid, err.Error())
@@ -281,22 +314,26 @@ func (v *volumeGetterImpl) getFromProxy(ctx context.Context, vid proto.Vid, flus
 				Vid:       vid,
 				Timestamp: -time.Now().UnixNano(),
 			}
-			span.Infof("to update memcache on not exist volume(%d-%d) %+v", v.cid, vid, phy)
+			span.Infof("to update memcache on not exist volume(%d-%d) %+v", cid, vid, phy)
 			v.setToLocalCache(ctx, id, phy)
+		} else if flush {
+			span.Warnf("to flush force on all proxy of volume(%d-%d)", cid, vid)
+			v.setToLocalCache(ctx, id, nil)
+			v.flush(ctx, vid, 0, hosts, map[string]struct{}{})
 		}
-		return nil, errors.Base(err, "get volume from proxy", v.cid, vid)
+		return nil, errors.Base(err, "get volume from proxy", cid, vid)
 	}
 
 	phy := &VolumePhy{
 		Vid:       volume.Vid,
 		CodeMode:  volume.CodeMode,
-		Version:   volume.Version,
+		Version:   uint64(volume.RouteVersion),
 		Timestamp: time.Now().UnixNano(),
 		Units:     make([]Unit, len(volume.Units)),
 	}
 	copy(phy.Units, volume.Units[:])
 
-	span.Debugf("to update memcache on volume(%d-%d) %+v", v.cid, vid, phy)
+	span.Debugf("to update memcache on volume(%d-%d) %+v", cid, vid, phy)
 	v.setToLocalCache(ctx, id, phy)
 
 	if flush {
@@ -307,7 +344,7 @@ func (v *volumeGetterImpl) getFromProxy(ctx context.Context, vid proto.Vid, flus
 }
 
 // flush update all proxy cache of this idc
-func (v *volumeGetterImpl) flush(ctx context.Context, vid proto.Vid, ver uint32, hosts []string, except map[string]struct{}) {
+func (v *volumeGetterImpl) flush(ctx context.Context, vid proto.Vid, ver uint64, hosts []string, except map[string]struct{}) {
 	span := trace.SpanFromContextSafe(ctx)
 	span.Infof("to flush volume cache %d on proxy:%v version:%d except:%v", vid, hosts, ver, except)
 
@@ -316,15 +353,16 @@ func (v *volumeGetterImpl) flush(ctx context.Context, vid proto.Vid, ver uint32,
 			continue
 		}
 
+		bgSpan, bgCtx := trace.StartSpanFromContextWithTraceID(context.Background(), "flush_proxy_volume", span.TraceID())
 		go func(host string) {
 			retry.ExponentialBackoff(2, 10).RuptOn(func() (bool, error) {
-				if _, err := v.proxy.GetCacheVolume(ctx, host,
+				if _, err := v.proxy.GetCacheVolume(bgCtx, host,
 					&proxy.CacheVolumeArgs{Vid: vid, Flush: true, Version: ver}); err != nil {
 					if rpc.DetectStatusCode(err) == errcode.CodeVolumeNotExist {
-						span.Info("not found volume", vid)
+						bgSpan.Info("not found volume", vid)
 						return true, err
 					}
-					span.Warnf("flush volume:%d error:%s", vid, err.Error())
+					bgSpan.Warnf("flush volume:%d error:%s", vid, err.Error())
 					return false, err
 				}
 				return true, nil
@@ -332,3 +370,23 @@ func (v *volumeGetterImpl) flush(ctx context.Context, vid proto.Vid, ver uint32,
 		}(host)
 	}
 }
+
+func (v *volumeGetterImpl) tickerUpdate() {
+	var vids []proto.Vid
+	v.unusualLock.Lock()
+	for vid, n := range v.unusualVolume {
+		if n > v.config.VolumePunishThreshold {
+			vids = append(vids, vid)
+			if len(vids) >= 10 {
+				break
+			}
+			delete(v.unusualVolume, vid)
+		}
+	}
+	v.unusualLock.Unlock()
+
+	for _, vid := range vids {
+		_, ctx := trace.StartSpanFromContext(context.Background(), "update-unusual")
+		v.getFromProxy(ctx, vid, true, 0)
+	}
+}

blobstore/access/controller/volume_test.go

@@ -26,6 +26,12 @@ import (
 	"github.com/cubefs/cubefs/blobstore/common/trace"
 )
 
+func closedCh() <-chan struct{} {
+	c := make(chan struct{})
+	close(c)
+	return c
+}
+
 func proxyService() controller.ServiceController {
 	service, _ := controller.NewServiceController(controller.ServiceConfig{IDC: idc}, cmcli, proxycli, nil)
 	return service
@@ -34,7 +40,8 @@ func proxyService() controller.ServiceController {
 func TestAccessVolumeGetterNew(t *testing.T) {
 	_, ctx := trace.StartSpanFromContext(context.Background(), "TestAccessVolumeGetterNew")
 
-	getter, err := controller.NewVolumeGetter(1, proxyService(), proxycli, time.Millisecond*200)
+	cfg := controller.VolumeConfig{ClusterID: 1, VolumeMemcacheExpirationMs: 200}
+	getter, err := controller.NewVolumeGetter(cfg, proxyService(), proxycli, closedCh())
 	require.Nil(t, err)
 	require.Nil(t, getter.Get(ctx, proto.Vid(0), true))
 
@@ -62,7 +69,8 @@ func TestAccessVolumeGetterNew(t *testing.T) {
 func TestAccessVolumeGetterNotExistVolume(t *testing.T) {
 	_, ctx := trace.StartSpanFromContext(context.Background(), "TestAccessVolumeGetterNotExistVolume")
 
-	getter, err := controller.NewVolumeGetter(0xfe, proxyService(), proxycli, time.Millisecond*200)
+	cfg := controller.VolumeConfig{ClusterID: 0xfe, VolumeMemcacheExpirationMs: 200}
+	getter, err := controller.NewVolumeGetter(cfg, proxyService(), proxycli, closedCh())
 	require.NoError(t, err)
 
 	id := vid404
@@ -78,7 +86,8 @@ func TestAccessVolumeGetterNotExistVolume(t *testing.T) {
 	getter.Get(ctx, id, true)
 	require.Equal(t, 2, dataCalled[id])
 
-	getter, err = controller.NewVolumeGetter(0xee, proxyService(), proxycli, 0)
+	cfg = controller.VolumeConfig{ClusterID: 0xee, VolumeMemcacheExpirationMs: -1}
+	getter, err = controller.NewVolumeGetter(cfg, proxyService(), proxycli, closedCh())
 	require.NoError(t, err)
 	id = vid404
 	dataCalled[id] = 0
@@ -94,10 +103,34 @@ func TestAccessVolumeGetterNotExistVolume(t *testing.T) {
 	require.Equal(t, 6, dataCalled[id])
 }
 
+func TestAccessVolumeGetterNotExistFlush(t *testing.T) {
+	_, ctx := trace.StartSpanFromContext(context.Background(), "TestAccessVolumeGetterNotExistVolumeFlush")
+
+	cfg := controller.VolumeConfig{ClusterID: 0xfe, VolumeMemcacheExpirationMs: 200}
+	getter, err := controller.NewVolumeGetter(cfg, proxyService(), proxycli, closedCh())
+	require.NoError(t, err)
+
+	id := vid404
+	require.Nil(t, getter.Get(ctx, id, true))
+	for range [10]struct{}{} {
+		require.Nil(t, getter.Get(ctx, id, false))
+	}
+
+	time.Sleep(time.Millisecond * 210)
+
+	cfg = controller.VolumeConfig{ClusterID: 0xee, VolumeMemcacheExpirationMs: -1}
+	getter, err = controller.NewVolumeGetter(cfg, proxyService(), proxycli, closedCh())
+	require.NoError(t, err)
+	for range [10]struct{}{} {
+		require.Nil(t, getter.Get(ctx, id, false))
+	}
+}
+
 func TestAccessVolumeGetterExpiration(t *testing.T) {
 	_, ctx := trace.StartSpanFromContext(context.Background(), "TestAccessVolumeGetterExpiration")
 
-	getter, err := controller.NewVolumeGetter(1, proxyService(), proxycli, time.Millisecond*200)
+	cfg := controller.VolumeConfig{ClusterID: 1, VolumeMemcacheExpirationMs: 200}
+	getter, err := controller.NewVolumeGetter(cfg, proxyService(), proxycli, closedCh())
 	require.Nil(t, err)
 
 	id := proto.Vid(1)
@@ -118,7 +151,8 @@ func TestAccessVolumeGetterExpiration(t *testing.T) {
 func TestAccessVolumePunish(t *testing.T) {
 	_, ctx := trace.StartSpanFromContext(context.Background(), "TestAccessVolumePunish")
 
-	getter, err := controller.NewVolumeGetter(1, proxyService(), proxycli, 0)
+	cfg := controller.VolumeConfig{ClusterID: 1, VolumeMemcacheExpirationMs: -1}
+	getter, err := controller.NewVolumeGetter(cfg, proxyService(), proxycli, closedCh())
 	require.Nil(t, err)
 	require.Nil(t, getter.Get(ctx, proto.Vid(0), true))
 
@@ -141,3 +175,50 @@ func TestAccessVolumePunish(t *testing.T) {
 	require.NotNil(t, info)
 	require.False(t, info.IsPunish)
 }
+
+func TestAccessVolumeUpdate(t *testing.T) {
+	_, ctx := trace.StartSpanFromContext(context.Background(), "TestAccessVolumeUpdate")
+
+	ch := make(chan struct{})
+	go func() {
+		time.Sleep(time.Second)
+		close(ch)
+	}()
+	cfg := controller.VolumeConfig{ClusterID: 1, VolumeMemcacheExpirationMs: -1}
+	getter, err := controller.NewVolumeGetter(cfg, proxyService(), proxycli, ch)
+	require.NoError(t, err)
+	getter.Update(ctx, 1)
+	for range [11]struct{}{} {
+		getter.Update(ctx, vid404)
+	}
+	for idx := range [11]struct{}{} {
+		for range [11]struct{}{} {
+			getter.Update(ctx, proto.Vid(idx+1))
+		}
+	}
+	getter.Update(ctx, 123)
+	getter.Update(ctx, 11)
+
+	<-ch
+}
+
+func TestAccessVolumeUpdateForce(t *testing.T) {
+	_, ctx := trace.StartSpanFromContext(context.Background(), "TestAccessVolumeUpdateForce")
+	ctxDone, cancel := context.WithCancel(ctx)
+	cancel()
+
+	ch := make(chan struct{})
+	go func() {
+		time.Sleep(time.Second)
+		close(ch)
+	}()
+	cfg := controller.VolumeConfig{ClusterID: 1, VolumeMemcacheExpirationMs: -1}
+	getter, err := controller.NewVolumeGetter(cfg, proxyService(), proxycli, ch)
+	require.NoError(t, err)
+
+	id := proto.Vid(1)
+	require.Nil(t, getter.Get(ctxDone, id, false))
+	require.NotNil(t, getter.Get(ctx, id, true))
+
+	<-ch
+}

blobstore/access/controller_mock_test.go

@@ -1,305 +0,0 @@
-// Code generated by MockGen. DO NOT EDIT.
-// Source: github.com/cubefs/cubefs/blobstore/access/controller (interfaces: ClusterController,ServiceController,VolumeGetter)
-
-// Package access is a generated GoMock package.
-package access
-
-import (
-	context "context"
-	reflect "reflect"
-
-	controller "github.com/cubefs/cubefs/blobstore/access/controller"
-	clustermgr "github.com/cubefs/cubefs/blobstore/api/clustermgr"
-	proto "github.com/cubefs/cubefs/blobstore/common/proto"
-	gomock "github.com/golang/mock/gomock"
-)
-
-// MockClusterController is a mock of ClusterController interface.
-type MockClusterController struct {
-	ctrl     *gomock.Controller
-	recorder *MockClusterControllerMockRecorder
-}
-
-// MockClusterControllerMockRecorder is the mock recorder for MockClusterController.
-type MockClusterControllerMockRecorder struct {
-	mock *MockClusterController
-}
-
-// NewMockClusterController creates a new mock instance.
-func NewMockClusterController(ctrl *gomock.Controller) *MockClusterController {
-	mock := &MockClusterController{ctrl: ctrl}
-	mock.recorder = &MockClusterControllerMockRecorder{mock}
-	return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use.
-func (m *MockClusterController) EXPECT() *MockClusterControllerMockRecorder {
-	return m.recorder
-}
-
-// All mocks base method.
-func (m *MockClusterController) All() []*clustermgr.ClusterInfo {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "All")
-	ret0, _ := ret[0].([]*clustermgr.ClusterInfo)
-	return ret0
-}
-
-// All indicates an expected call of All.
-func (mr *MockClusterControllerMockRecorder) All() *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "All", reflect.TypeOf((*MockClusterController)(nil).All))
-}
-
-// ChangeChooseAlg mocks base method.
-func (m *MockClusterController) ChangeChooseAlg(arg0 controller.AlgChoose) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "ChangeChooseAlg", arg0)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// ChangeChooseAlg indicates an expected call of ChangeChooseAlg.
-func (mr *MockClusterControllerMockRecorder) ChangeChooseAlg(arg0 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChangeChooseAlg", reflect.TypeOf((*MockClusterController)(nil).ChangeChooseAlg), arg0)
-}
-
-// ChooseOne mocks base method.
-func (m *MockClusterController) ChooseOne() (*clustermgr.ClusterInfo, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "ChooseOne")
-	ret0, _ := ret[0].(*clustermgr.ClusterInfo)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// ChooseOne indicates an expected call of ChooseOne.
-func (mr *MockClusterControllerMockRecorder) ChooseOne() *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChooseOne", reflect.TypeOf((*MockClusterController)(nil).ChooseOne))
-}
-
-// GetConfig mocks base method.
-func (m *MockClusterController) GetConfig(arg0 context.Context, arg1 string) (string, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "GetConfig", arg0, arg1)
-	ret0, _ := ret[0].(string)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// GetConfig indicates an expected call of GetConfig.
-func (mr *MockClusterControllerMockRecorder) GetConfig(arg0, arg1 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfig", reflect.TypeOf((*MockClusterController)(nil).GetConfig), arg0, arg1)
-}
-
-// GetServiceController mocks base method.
-func (m *MockClusterController) GetServiceController(arg0 proto.ClusterID) (controller.ServiceController, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "GetServiceController", arg0)
-	ret0, _ := ret[0].(controller.ServiceController)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// GetServiceController indicates an expected call of GetServiceController.
-func (mr *MockClusterControllerMockRecorder) GetServiceController(arg0 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetServiceController", reflect.TypeOf((*MockClusterController)(nil).GetServiceController), arg0)
-}
-
-// GetVolumeGetter mocks base method.
-func (m *MockClusterController) GetVolumeGetter(arg0 proto.ClusterID) (controller.VolumeGetter, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "GetVolumeGetter", arg0)
-	ret0, _ := ret[0].(controller.VolumeGetter)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// GetVolumeGetter indicates an expected call of GetVolumeGetter.
-func (mr *MockClusterControllerMockRecorder) GetVolumeGetter(arg0 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVolumeGetter", reflect.TypeOf((*MockClusterController)(nil).GetVolumeGetter), arg0)
-}
-
-// Region mocks base method.
-func (m *MockClusterController) Region() string {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Region")
-	ret0, _ := ret[0].(string)
-	return ret0
-}
-
-// Region indicates an expected call of Region.
-func (mr *MockClusterControllerMockRecorder) Region() *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Region", reflect.TypeOf((*MockClusterController)(nil).Region))
-}
-
-// MockServiceController is a mock of ServiceController interface.
-type MockServiceController struct {
-	ctrl     *gomock.Controller
-	recorder *MockServiceControllerMockRecorder
-}
-
-// MockServiceControllerMockRecorder is the mock recorder for MockServiceController.
-type MockServiceControllerMockRecorder struct {
-	mock *MockServiceController
-}
-
-// NewMockServiceController creates a new mock instance.
-func NewMockServiceController(ctrl *gomock.Controller) *MockServiceController {
-	mock := &MockServiceController{ctrl: ctrl}
-	mock.recorder = &MockServiceControllerMockRecorder{mock}
-	return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use.
-func (m *MockServiceController) EXPECT() *MockServiceControllerMockRecorder {
-	return m.recorder
-}
-
-// GetDiskHost mocks base method.
-func (m *MockServiceController) GetDiskHost(arg0 context.Context, arg1 proto.DiskID) (*controller.HostIDC, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "GetDiskHost", arg0, arg1)
-	ret0, _ := ret[0].(*controller.HostIDC)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// GetDiskHost indicates an expected call of GetDiskHost.
-func (mr *MockServiceControllerMockRecorder) GetDiskHost(arg0, arg1 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDiskHost", reflect.TypeOf((*MockServiceController)(nil).GetDiskHost), arg0, arg1)
-}
-
-// GetServiceHost mocks base method.
-func (m *MockServiceController) GetServiceHost(arg0 context.Context, arg1 string) (string, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "GetServiceHost", arg0, arg1)
-	ret0, _ := ret[0].(string)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// GetServiceHost indicates an expected call of GetServiceHost.
-func (mr *MockServiceControllerMockRecorder) GetServiceHost(arg0, arg1 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetServiceHost", reflect.TypeOf((*MockServiceController)(nil).GetServiceHost), arg0, arg1)
-}
-
-// GetServiceHosts mocks base method.
-func (m *MockServiceController) GetServiceHosts(arg0 context.Context, arg1 string) ([]string, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "GetServiceHosts", arg0, arg1)
-	ret0, _ := ret[0].([]string)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// GetServiceHosts indicates an expected call of GetServiceHosts.
-func (mr *MockServiceControllerMockRecorder) GetServiceHosts(arg0, arg1 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetServiceHosts", reflect.TypeOf((*MockServiceController)(nil).GetServiceHosts), arg0, arg1)
-}
-
-// PunishDisk mocks base method.
-func (m *MockServiceController) PunishDisk(arg0 context.Context, arg1 proto.DiskID, arg2 int) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "PunishDisk", arg0, arg1, arg2)
-}
-
-// PunishDisk indicates an expected call of PunishDisk.
-func (mr *MockServiceControllerMockRecorder) PunishDisk(arg0, arg1, arg2 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PunishDisk", reflect.TypeOf((*MockServiceController)(nil).PunishDisk), arg0, arg1, arg2)
-}
-
-// PunishDiskWithThreshold mocks base method.
-func (m *MockServiceController) PunishDiskWithThreshold(arg0 context.Context, arg1 proto.DiskID, arg2 int) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "PunishDiskWithThreshold", arg0, arg1, arg2)
-}
-
-// PunishDiskWithThreshold indicates an expected call of PunishDiskWithThreshold.
-func (mr *MockServiceControllerMockRecorder) PunishDiskWithThreshold(arg0, arg1, arg2 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PunishDiskWithThreshold", reflect.TypeOf((*MockServiceController)(nil).PunishDiskWithThreshold), arg0, arg1, arg2)
-}
-
-// PunishService mocks base method.
-func (m *MockServiceController) PunishService(arg0 context.Context, arg1, arg2 string, arg3 int) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "PunishService", arg0, arg1, arg2, arg3)
-}
-
-// PunishService indicates an expected call of PunishService.
-func (mr *MockServiceControllerMockRecorder) PunishService(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PunishService", reflect.TypeOf((*MockServiceController)(nil).PunishService), arg0, arg1, arg2, arg3)
-}
-
-// PunishServiceWithThreshold mocks base method.
-func (m *MockServiceController) PunishServiceWithThreshold(arg0 context.Context, arg1, arg2 string, arg3 int) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "PunishServiceWithThreshold", arg0, arg1, arg2, arg3)
-}
-
-// PunishServiceWithThreshold indicates an expected call of PunishServiceWithThreshold.
-func (mr *MockServiceControllerMockRecorder) PunishServiceWithThreshold(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PunishServiceWithThreshold", reflect.TypeOf((*MockServiceController)(nil).PunishServiceWithThreshold), arg0, arg1, arg2, arg3)
-}
-
-// MockVolumeGetter is a mock of VolumeGetter interface.
-type MockVolumeGetter struct {
-	ctrl     *gomock.Controller
-	recorder *MockVolumeGetterMockRecorder
-}
-
-// MockVolumeGetterMockRecorder is the mock recorder for MockVolumeGetter.
-type MockVolumeGetterMockRecorder struct {
-	mock *MockVolumeGetter
-}
-
-// NewMockVolumeGetter creates a new mock instance.
-func NewMockVolumeGetter(ctrl *gomock.Controller) *MockVolumeGetter {
-	mock := &MockVolumeGetter{ctrl: ctrl}
-	mock.recorder = &MockVolumeGetterMockRecorder{mock}
-	return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use.
-func (m *MockVolumeGetter) EXPECT() *MockVolumeGetterMockRecorder {
-	return m.recorder
-}
-
-// Get mocks base method.
-func (m *MockVolumeGetter) Get(arg0 context.Context, arg1 proto.Vid, arg2 bool) *controller.VolumePhy {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2)
-	ret0, _ := ret[0].(*controller.VolumePhy)
-	return ret0
-}
-
-// Get indicates an expected call of Get.
-func (mr *MockVolumeGetterMockRecorder) Get(arg0, arg1, arg2 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockVolumeGetter)(nil).Get), arg0, arg1, arg2)
-}
-
-// Punish mocks base method.
-func (m *MockVolumeGetter) Punish(arg0 context.Context, arg1 proto.Vid, arg2 int) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "Punish", arg0, arg1, arg2)
-}
-
-// Punish indicates an expected call of Punish.
-func (mr *MockVolumeGetterMockRecorder) Punish(arg0, arg1, arg2 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Punish", reflect.TypeOf((*MockVolumeGetter)(nil).Punish), arg0, arg1, arg2)
-}

blobstore/access/server.go

@@ -15,14 +15,13 @@
 package access
 
 import (
-	"crypto/sha1"
 	"fmt"
 	"net/http"
 	"strconv"
 	"sync"
-	"time"
 
 	"github.com/cubefs/cubefs/blobstore/access/controller"
+	"github.com/cubefs/cubefs/blobstore/access/stream"
 	"github.com/cubefs/cubefs/blobstore/api/access"
 	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
 	"github.com/cubefs/cubefs/blobstore/cmd"
@@ -32,8 +31,8 @@ import (
 	"github.com/cubefs/cubefs/blobstore/common/proto"
 	"github.com/cubefs/cubefs/blobstore/common/resourcepool"
 	"github.com/cubefs/cubefs/blobstore/common/rpc"
+	"github.com/cubefs/cubefs/blobstore/common/security"
 	"github.com/cubefs/cubefs/blobstore/common/trace"
-	"github.com/cubefs/cubefs/blobstore/common/uptoken"
 	"github.com/cubefs/cubefs/blobstore/util/closer"
 	"github.com/cubefs/cubefs/blobstore/util/errors"
 	"github.com/cubefs/cubefs/blobstore/util/log"
@@ -48,46 +47,11 @@ const (
 	limitNameSign   = "sign"
 )
 
-const (
-	_tokenExpiration = time.Hour * 12
-)
-
-var (
-	// tokenSecretKeys alloc token with the first secret key always,
-	// so that you can change the secret key.
-	//
-	// parse-1: insert a new key at the first index,
-	// parse-2: delete the old key at the last index after _tokenExpiration duration.
-	tokenSecretKeys = [...][20]byte{
-		{0x5f, 0x00, 0x88, 0x96, 0x00, 0xa1, 0xfe, 0x1b},
-		{0xff, 0x1f, 0x2f, 0x4f, 0x7f, 0xaf, 0xef, 0xff},
-	}
-	_initTokenSecret sync.Once
-)
-
-func initTokenSecret(b []byte) {
-	_initTokenSecret.Do(func() {
-		for idx := range tokenSecretKeys {
-			copy(tokenSecretKeys[idx][7:], b)
-		}
-	})
-}
-
-func initWithRegionMagic(regionMagic string) {
-	if regionMagic == "" {
-		log.Warn("no region magic setting, using default secret keys for checksum")
-		return
-	}
-	b := sha1.Sum([]byte(regionMagic))
-	initTokenSecret(b[:8])
-	initLocationSecret(b[:8])
-}
-
 type accessStatus struct {
-	Limit Status              `json:"limit"`
+	Limit stream.Status       `json:"limit"`
 	Pool  resourcepool.Status `json:"pool"`
 
-	Config   StreamConfig                            `json:"config"`
+	Config   stream.StreamConfig                     `json:"config"`
 	Clusters []*clustermgr.ClusterInfo               `json:"clusters"`
 	Services map[proto.ClusterID]map[string][]string `json:"services"`
 }
@@ -96,29 +60,34 @@ type accessStatus struct {
 type Config struct {
 	cmd.Config
 
-	ServiceRegister consul.Config `json:"service_register"`
-	Stream          StreamConfig  `json:"stream"`
-	Limit           LimitConfig   `json:"limit"`
+	ServiceRegister consul.Config       `json:"service_register"`
+	Stream          stream.StreamConfig `json:"stream"`
+	Limit           stream.LimitConfig  `json:"limit"`
 }
 
 // Service rpc service
 type Service struct {
 	config        Config
-	streamHandler StreamHandler
-	limiter       Limiter
+	streamHandler stream.StreamHandler
+	limiter       stream.Limiter
 	closer        closer.Closer
 }
 
 // New returns an access service
 func New(cfg Config) *Service {
 	// add region magic checksum to the secret keys
-	initWithRegionMagic(cfg.Stream.ClusterConfig.RegionMagic)
+	security.InitWithRegionMagic(cfg.Stream.ClusterConfig.RegionMagic)
 
 	cl := closer.New()
+	h, err := stream.NewStreamHandler(&cfg.Stream, cl.Done())
+	if err != nil {
+		log.Fatalf("new stream handler failed, err: %+v", err)
+	}
+
 	return &Service{
 		config:        cfg,
-		streamHandler: NewStreamHandler(&cfg.Stream, cl.Done()),
-		limiter:       NewLimiter(cfg.Limit),
+		streamHandler: h,
+		limiter:       stream.NewLimiter(cfg.Limit),
 		closer:        cl,
 	}
 }
@@ -142,9 +111,9 @@ func (s *Service) RegisterService() {
 // RegisterAdminHandler register admin handler to profile
 func (s *Service) RegisterAdminHandler() {
 	profile.HandleFunc(http.MethodGet, "/access/status", func(c *rpc.Context) {
-		var admin *streamAdmin
+		var admin *stream.StreamAdmin
 		if sa := s.streamHandler.Admin(); sa != nil {
-			if ad, ok := sa.(*streamAdmin); ok {
+			if ad, ok := sa.(*stream.StreamAdmin); ok {
 				admin = ad
 			}
 		}
@@ -158,13 +127,13 @@ func (s *Service) RegisterAdminHandler() {
 
 		status := new(accessStatus)
 		status.Limit = s.limiter.Status()
-		status.Pool = admin.memPool.Status()
-		status.Config = admin.config
-		status.Clusters = admin.controller.All()
+		status.Pool = admin.MemPool.Status()
+		status.Config = admin.Config
+		status.Clusters = admin.Controller.All()
 		status.Services = make(map[proto.ClusterID]map[string][]string, len(status.Clusters))
 
 		for _, cluster := range status.Clusters {
-			service, err := admin.controller.GetServiceController(cluster.ClusterID)
+			service, err := admin.Controller.GetServiceController(cluster.ClusterID)
 			if err != nil {
 				span.Warn(err.Error())
 				continue
@@ -191,8 +160,8 @@ func (s *Service) RegisterAdminHandler() {
 
 		alg := controller.AlgChoose(algInt)
 		if sa := s.streamHandler.Admin(); sa != nil {
-			if admin, ok := sa.(*streamAdmin); ok {
-				if err := admin.controller.ChangeChooseAlg(alg); err != nil {
+			if admin, ok := sa.(*stream.StreamAdmin); ok {
+				if err := admin.Controller.ChangeChooseAlg(alg); err != nil {
 					c.RespondWith(http.StatusForbidden, "", []byte(err.Error()))
 					return
 				}
@@ -265,7 +234,7 @@ func (s *Service) Put(c *rpc.Context) {
 	}
 
 	rc := s.limiter.Reader(ctx, c.Request.Body)
-	loc, err := s.streamHandler.Put(ctx, rc, args.Size, hasherMap)
+	loc, err := s.streamHandler.Put(ctx, rc, args.Size, hasherMap, args.AssignClusterID, args.CodeMode)
 	if err != nil {
 		span.Error("stream put failed", errors.Detail(err))
 		c.RespondError(httpError(err))
@@ -277,7 +246,7 @@ func (s *Service) Put(c *rpc.Context) {
 		hashSumMap[alg] = hasher.Sum(nil)
 	}
 
-	if err := fillCrc(loc); err != nil {
+	if err := security.LocationCrcFill(loc); err != nil {
 		span.Error("stream put fill location crc", err)
 		c.RespondError(httpError(err))
 		return
@@ -308,8 +277,8 @@ func (s *Service) PutAt(c *rpc.Context) {
 	}
 
 	valid := false
-	for _, secretKey := range tokenSecretKeys {
-		token := uptoken.DecodeToken(args.Token)
+	for _, secretKey := range security.TokenSecretKeys() {
+		token := security.DecodeToken(args.Token)
 		if token.IsValid(args.ClusterID, args.Vid, args.BlobID, uint32(args.Size), secretKey[:]) {
 			valid = true
 			break
@@ -369,7 +338,7 @@ func (s *Service) Alloc(c *rpc.Context) {
 		return
 	}
 
-	if err := fillCrc(location); err != nil {
+	if err := security.LocationCrcFill(location); err != nil {
 		span.Error("stream alloc fill location crc", err)
 		c.RespondError(httpError(err))
 		return
@@ -377,7 +346,7 @@ func (s *Service) Alloc(c *rpc.Context) {
 
 	resp := access.AllocResp{
 		Location: *location,
-		Tokens:   genTokens(location),
+		Tokens:   security.StreamGenTokens(location),
 	}
 	c.RespondJSON(resp)
 	span.Infof("done /alloc request resp:%+v", resp)
@@ -395,7 +364,7 @@ func (s *Service) Get(c *rpc.Context) {
 	span := trace.SpanFromContextSafe(ctx)
 
 	span.Debugf("accept /get request args:%+v", args)
-	if !args.IsValid() || !verifyCrc(&args.Location) {
+	if !args.IsValid() || !security.LocationCrcVerify(&args.Location) {
 		c.RespondError(errcode.ErrIllegalArguments)
 		return
 	}
@@ -411,9 +380,9 @@ func (s *Service) Get(c *rpc.Context) {
 
 	w.Header().Set(rpc.HeaderContentType, rpc.MIMEStream)
 	w.Header().Set(rpc.HeaderContentLength, strconv.FormatInt(int64(args.ReadSize), 10))
-	if args.ReadSize > 0 && args.ReadSize != args.Location.Size {
+	if args.ReadSize > 0 && args.ReadSize != args.Location.Size_ {
 		w.Header().Set(rpc.HeaderContentRange, fmt.Sprintf("bytes %d-%d/%d",
-			args.Offset, args.Offset+args.ReadSize-1, args.Location.Size))
+			args.Offset, args.Offset+args.ReadSize-1, args.Location.Size_))
 		c.RespondStatus(http.StatusPartialContent)
 	} else {
 		c.RespondStatus(http.StatusOK)
@@ -424,7 +393,7 @@ func (s *Service) Get(c *rpc.Context) {
 
 	err = transfer()
 	if err != nil {
-		reportDownload(args.Location.ClusterID, "StatusOKError", "-")
+		stream.SteamReportDownload(args.Location.ClusterID, "StatusOKError", "-")
 		span.Error("stream get transfer failed", errors.Detail(err))
 		return
 	}
@@ -471,19 +440,19 @@ func (s *Service) Delete(c *rpc.Context) {
 
 	clusterBlobsN := make(map[proto.ClusterID]int, 4)
 	for _, loc := range args.Locations {
-		if !verifyCrc(&loc) {
+		if !security.LocationCrcVerify(&loc) {
 			span.Infof("invalid crc %+v", loc)
 			err = errcode.ErrIllegalArguments
 			return
 		}
-		clusterBlobsN[loc.ClusterID] += len(loc.Blobs)
+		clusterBlobsN[loc.ClusterID] += len(loc.Slices)
 	}
 
 	if len(args.Locations) == 1 {
 		loc := args.Locations[0]
 		if err := s.streamHandler.Delete(ctx, &loc); err != nil {
 			span.Error("stream delete failed", errors.Detail(err))
-			resp.FailedLocations = []access.Location{loc}
+			resp.FailedLocations = []proto.Location{loc}
 		}
 		return
 	}
@@ -495,12 +464,12 @@ func (s *Service) Delete(c *rpc.Context) {
 	// max delete locations is 1024, one location is max to 5G,
 	// merged message max size about 40MB.
 
-	merged := make(map[proto.ClusterID][]access.SliceInfo, len(clusterBlobsN))
+	merged := make(map[proto.ClusterID][]proto.Slice, len(clusterBlobsN))
 	for id, n := range clusterBlobsN {
-		merged[id] = make([]access.SliceInfo, 0, n)
+		merged[id] = make([]proto.Slice, 0, n)
 	}
 	for _, loc := range args.Locations {
-		merged[loc.ClusterID] = append(merged[loc.ClusterID], loc.Blobs...)
+		merged[loc.ClusterID] = append(merged[loc.ClusterID], loc.Slices...)
 	}
 
 	var wg sync.WaitGroup
@@ -509,7 +478,7 @@ func (s *Service) Delete(c *rpc.Context) {
 	go func() {
 		for id := range failedCh {
 			if resp.FailedLocations == nil {
-				resp.FailedLocations = make([]access.Location, 0, len(args.Locations))
+				resp.FailedLocations = make([]proto.Location, 0, len(args.Locations))
 			}
 			for _, loc := range args.Locations {
 				if loc.ClusterID == id {
@@ -523,10 +492,10 @@ func (s *Service) Delete(c *rpc.Context) {
 	wg.Add(len(merged))
 	for id := range merged {
 		go func(id proto.ClusterID) {
-			if err := s.streamHandler.Delete(ctx, &access.Location{
+			if err := s.streamHandler.Delete(ctx, &proto.Location{
 				ClusterID: id,
-				BlobSize:  1,
-				Blobs:     merged[id],
+				SliceSize: 1,
+				Slices:    merged[id],
 			}); err != nil {
 				span.Error("stream delete failed", id, errors.Detail(err))
 				failedCh <- id
@@ -558,8 +527,8 @@ func (s *Service) DeleteBlob(c *rpc.Context) {
 	}
 
 	valid := false
-	for _, secretKey := range tokenSecretKeys {
-		token := uptoken.DecodeToken(args.Token)
+	for _, secretKey := range security.TokenSecretKeys() {
+		token := security.DecodeToken(args.Token)
 		if token.IsValid(args.ClusterID, args.Vid, args.BlobID, uint32(args.Size), secretKey[:]) {
 			valid = true
 			break
@@ -571,13 +540,13 @@ func (s *Service) DeleteBlob(c *rpc.Context) {
 		return
 	}
 
-	if err := s.streamHandler.Delete(ctx, &access.Location{
+	if err := s.streamHandler.Delete(ctx, &proto.Location{
 		ClusterID: args.ClusterID,
-		BlobSize:  1,
-		Blobs: []access.SliceInfo{{
-			MinBid: args.BlobID,
-			Vid:    args.Vid,
-			Count:  1,
+		SliceSize: 1,
+		Slices: []proto.Slice{{
+			MinSliceID: args.BlobID,
+			Vid:        args.Vid,
+			Count:      1,
 		}},
 	}); err != nil {
 		span.Error("stream delete blob failed", errors.Detail(err))
@@ -608,7 +577,7 @@ func (s *Service) Sign(c *rpc.Context) {
 
 	loc := args.Location
 	crcOld := loc.Crc
-	if err := signCrc(&loc, args.Locations); err != nil {
+	if err := security.LocationCrcSign(&loc, args.Locations); err != nil {
 		span.Error("stream sign failed", errors.Detail(err))
 		c.RespondError(errcode.ErrIllegalArguments)
 		return
@@ -627,39 +596,3 @@ func httpError(err error) error {
 	}
 	return errcode.ErrUnexpected
 }
-
-// genTokens generate tokens
-//  1. Returns 0 token if has no blobs.
-//  2. Returns 1 token if file size less than blobsize.
-//  3. Returns len(blobs) tokens if size divided by blobsize.
-//  4. Otherwise returns len(blobs)+1 tokens, the last token
-//     will be used by the last blob, even if the last slice blobs' size
-//     less than blobsize.
-//  5. Each segment blob has its specified token include the last blob.
-func genTokens(location *access.Location) []string {
-	tokens := make([]string, 0, len(location.Blobs)+1)
-
-	hasMultiBlobs := location.Size >= uint64(location.BlobSize)
-	lastSize := uint32(location.Size % uint64(location.BlobSize))
-	for idx, blob := range location.Blobs {
-		// returns one token if size < blobsize
-		if hasMultiBlobs {
-			count := blob.Count
-			if idx == len(location.Blobs)-1 && lastSize > 0 {
-				count--
-			}
-			tokens = append(tokens, uptoken.EncodeToken(uptoken.NewUploadToken(location.ClusterID,
-				blob.Vid, blob.MinBid, count,
-				location.BlobSize, _tokenExpiration, tokenSecretKeys[0][:])))
-		}
-
-		// token of the last blob
-		if idx == len(location.Blobs)-1 && lastSize > 0 {
-			tokens = append(tokens, uptoken.EncodeToken(uptoken.NewUploadToken(location.ClusterID,
-				blob.Vid, blob.MinBid+proto.BlobID(blob.Count)-1, 1,
-				lastSize, _tokenExpiration, tokenSecretKeys[0][:])))
-		}
-	}
-
-	return tokens
-}

blobstore/access/server_location.go

@@ -1,127 +0,0 @@
-// Copyright 2022 The CubeFS Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-// implied. See the License for the specific language governing
-// permissions and limitations under the License.
-
-package access
-
-import (
-	"fmt"
-	"hash/crc32"
-	"sync"
-
-	"github.com/cubefs/cubefs/blobstore/api/access"
-	"github.com/cubefs/cubefs/blobstore/common/proto"
-	"github.com/cubefs/cubefs/blobstore/util/bytespool"
-)
-
-const (
-	// DO NOT CHANGE IT.
-	_crcPoly = uint32(0x59c8943c)
-)
-
-var (
-	// DO NOT CHANGE IT.
-	_crcTable    = crc32.MakeTable(_crcPoly)
-	_crcMagicKey = [20]byte{
-		0x52, 0xe, 0x53, 0x53, 0x81,
-		0x1f, 0x51, 0xb7, 0xa4, 0x72,
-		0x10, 0x33, 0x64, 0xa7, 0x3a,
-		0x10, 0x19, 0xbc, 0x60, 0x7,
-	}
-	_initLocationSecret sync.Once
-)
-
-func initLocationSecret(b []byte) {
-	_initLocationSecret.Do(func() {
-		copy(_crcMagicKey[7:], b)
-	})
-}
-
-func calcCrc(loc *access.Location) (uint32, error) {
-	crcWriter := crc32.New(_crcTable)
-
-	buf := bytespool.Alloc(1024)
-	defer bytespool.Free(buf)
-
-	n := loc.Encode2(buf)
-	if n < 4 {
-		return 0, fmt.Errorf("no enough bytes(%d) fill into buf", n)
-	}
-
-	if _, err := crcWriter.Write(_crcMagicKey[:]); err != nil {
-		return 0, fmt.Errorf("fill crc %s", err.Error())
-	}
-	if _, err := crcWriter.Write(buf[4:n]); err != nil {
-		return 0, fmt.Errorf("fill crc %s", err.Error())
-	}
-
-	return crcWriter.Sum32(), nil
-}
-
-func fillCrc(loc *access.Location) error {
-	crc, err := calcCrc(loc)
-	if err != nil {
-		return err
-	}
-	loc.Crc = crc
-	return nil
-}
-
-func verifyCrc(loc *access.Location) bool {
-	crc, err := calcCrc(loc)
-	if err != nil {
-		return false
-	}
-	return loc.Crc == crc
-}
-
-func signCrc(loc *access.Location, locs []access.Location) error {
-	first := locs[0]
-	bids := make(map[proto.BlobID]struct{}, 64)
-
-	if loc.ClusterID != first.ClusterID ||
-		loc.CodeMode != first.CodeMode ||
-		loc.BlobSize != first.BlobSize {
-		return fmt.Errorf("not equal in constant field")
-	}
-
-	for _, l := range locs {
-		if !verifyCrc(&l) {
-			return fmt.Errorf("not equal in crc %d", l.Crc)
-		}
-
-		// assert
-		if l.ClusterID != first.ClusterID ||
-			l.CodeMode != first.CodeMode ||
-			l.BlobSize != first.BlobSize {
-			return fmt.Errorf("not equal in constant field")
-		}
-
-		for _, blob := range l.Blobs {
-			for c := 0; c < int(blob.Count); c++ {
-				bids[blob.MinBid+proto.BlobID(c)] = struct{}{}
-			}
-		}
-	}
-
-	for _, blob := range loc.Blobs {
-		for c := 0; c < int(blob.Count); c++ {
-			bid := blob.MinBid + proto.BlobID(c)
-			if _, ok := bids[bid]; !ok {
-				return fmt.Errorf("not equal in blob_id(%d)", bid)
-			}
-		}
-	}
-
-	return fillCrc(loc)
-}

blobstore/access/server_test.go

@@ -30,26 +30,65 @@ import (
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/require"
 
+	"github.com/cubefs/cubefs/blobstore/access/stream"
 	"github.com/cubefs/cubefs/blobstore/api/access"
 	"github.com/cubefs/cubefs/blobstore/common/codemode"
 	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
 	"github.com/cubefs/cubefs/blobstore/common/proto"
 	"github.com/cubefs/cubefs/blobstore/common/rpc"
-	"github.com/cubefs/cubefs/blobstore/common/uptoken"
+	"github.com/cubefs/cubefs/blobstore/common/security"
+	mocks "github.com/cubefs/cubefs/blobstore/testing/mocks"
+	_ "github.com/cubefs/cubefs/blobstore/testing/nolog"
 )
 
 var (
 	ctx              = context.Background()
 	_blobSize uint32 = 1 << 20
-	location         = &access.Location{
+	location         = &proto.Location{
 		ClusterID: 1,
-		CodeMode:  1,
-		BlobSize:  _blobSize,
+		CodeMode:  codemode.EC15P12,
+		SliceSize: _blobSize,
 		Crc:       0,
-		Blobs: []access.SliceInfo{{
-			MinBid: 111,
-			Vid:    1111,
-			Count:  11,
+		Slices: []proto.Slice{{
+			MinSliceID: 111,
+			Vid:        1111,
+			Count:      11,
+		}},
+	}
+
+	locationForClusterID = &proto.Location{
+		ClusterID: 11,
+		CodeMode:  codemode.EC3P3,
+		SliceSize: _blobSize,
+		Crc:       0,
+		Slices: []proto.Slice{{
+			MinSliceID: 111,
+			Vid:        1111,
+			Count:      1,
+		}},
+	}
+
+	locationForCodeMode = &proto.Location{
+		ClusterID: 22,
+		CodeMode:  codemode.EC6P6,
+		SliceSize: _blobSize,
+		Crc:       0,
+		Slices: []proto.Slice{{
+			MinSliceID: 111,
+			Vid:        1111,
+			Count:      1,
+		}},
+	}
+
+	locationForClusterIDAndCodeMode = &proto.Location{
+		ClusterID: 99,
+		CodeMode:  codemode.EC12P4,
+		SliceSize: _blobSize,
+		Crc:       0,
+		Slices: []proto.Slice{{
+			MinSliceID: 111,
+			Vid:        1111,
+			Count:      1,
 		}},
 	}
 
@@ -66,54 +105,66 @@ func runMockService(s *Service) string {
 
 func newService() *Service {
 	ctr := gomock.NewController(&testing.T{})
-	s := NewMockStreamHandler(ctr)
+	s := mocks.NewMockStreamHandler(ctr)
 
 	s.EXPECT().Alloc(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
 		func(ctx context.Context, size uint64, blobSize uint32,
-			assignClusterID proto.ClusterID, codeMode codemode.CodeMode) (*access.Location, error) {
+			assignClusterID proto.ClusterID, codeMode codemode.CodeMode,
+		) (*proto.Location, error) {
 			if size < 1024 {
 				return nil, errors.New("fake alloc location")
 			}
 			loc := location.Copy()
-			loc.Size = uint64(size)
-			fillCrc(&loc)
+			loc.Size_ = uint64(size)
+			security.LocationCrcFill(&loc)
 			return &loc, nil
 		})
 
 	s.EXPECT().PutAt(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(),
 		gomock.Any()).AnyTimes().DoAndReturn(
 		func(ctx context.Context, rc io.Reader,
-			clusterID proto.ClusterID, vid proto.Vid, bid proto.BlobID, size int64,
-			hasherMap access.HasherMap) error {
+			clusterID proto.ClusterID, vid proto.Vid, bid proto.BlobID, size int64, hasherMap access.HasherMap,
+		) error {
 			if size < 1024 {
 				return errcode.ErrAccessLimited
 			}
 			return nil
 		})
 
-	s.EXPECT().Put(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
-		func(ctx context.Context, rc io.Reader, size int64, hasherMap access.HasherMap) (*access.Location, error) {
+	s.EXPECT().Put(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
+		func(ctx context.Context, rc io.Reader, size int64, hasherMap access.HasherMap,
+			assignClusterID proto.ClusterID, codeMode codemode.CodeMode,
+		) (*proto.Location, error) {
 			if size < 1024 {
 				return nil, errors.New("fake put nil body")
 			}
-			loc := location.Copy()
-			loc.Size = uint64(size)
-			fillCrc(&loc)
+			var loc proto.Location
+			if assignClusterID == 0 && codeMode == codemode.CodeModeNone {
+				loc = location.Copy()
+			} else if assignClusterID == 11 && codeMode == codemode.CodeModeNone {
+				loc = locationForClusterID.Copy()
+			} else if assignClusterID == 0 && codeMode == codemode.EC6P6 {
+				loc = locationForCodeMode.Copy()
+			} else if assignClusterID == 99 && codeMode == codemode.EC12P4 {
+				loc = locationForClusterIDAndCodeMode.Copy()
+			}
+			loc.Size_ = uint64(size)
+			security.LocationCrcFill(&loc)
 			return &loc, nil
 		})
 
 	s.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
-		func(ctx context.Context, w io.Writer, location access.Location, readSize, offset uint64) (func() error, error) {
+		func(ctx context.Context, w io.Writer, location proto.Location, readSize, offset uint64) (func() error, error) {
 			if readSize < 1024 {
 				return nil, errors.New("fake get nil body")
 			}
 			return func() error { return nil }, nil
 		})
 	s.EXPECT().Delete(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
-		func(ctx context.Context, location *access.Location) error {
+		func(ctx context.Context, location *proto.Location) error {
 			if location.ClusterID >= 10 {
 				return errors.New("fake delete error with cluster")
-			} else if location.ClusterID == 1 && location.Crc > 0 && location.Size < 1024 {
+			} else if location.ClusterID == 1 && location.Crc > 0 && location.Size_ < 1024 {
 				return errors.New("fake delete error")
 			}
 			return nil
@@ -121,7 +172,7 @@ func newService() *Service {
 
 	return &Service{
 		streamHandler: s,
-		limiter: NewLimiter(LimitConfig{
+		limiter: stream.NewLimiter(stream.LimitConfig{
 			NameRps: map[string]int{
 				limitNameAlloc: 2,
 			},
@@ -168,21 +219,21 @@ func TestAccessServiceAlloc(t *testing.T) {
 		resp := &access.AllocResp{}
 		err := cli.PostWith(ctx, url(), resp, args)
 		require.NoError(t, err)
-		require.Equal(t, uint64(1024), resp.Location.Size)
+		require.Equal(t, uint64(1024), resp.Location.Size_)
 	}
 	{
 		args.Size = uint64(_blobSize)
 		resp := &access.AllocResp{}
 		err := cli.PostWith(ctx, url(), resp, args)
 		require.NoError(t, err)
-		require.Equal(t, uint64(_blobSize), resp.Location.Size)
+		require.Equal(t, uint64(_blobSize), resp.Location.Size_)
 	}
 	{
 		args.Size = uint64(_blobSize) + 1
 		resp := &access.AllocResp{}
 		err := cli.PostWith(ctx, url(), resp, args)
 		require.NoError(t, err)
-		require.Equal(t, uint64(_blobSize)+1, resp.Location.Size)
+		require.Equal(t, uint64(_blobSize)+1, resp.Location.Size_)
 	}
 }
 
@@ -220,7 +271,7 @@ func TestAccessServicePutAt(t *testing.T) {
 			resp := &access.PutAtResp{}
 			req, _ := http.NewRequest(method, url(args.Size, "c1fdcecaacbfafd86f0b00"), bytes.NewReader(buf))
 			err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode())
-			assertErrorCode(t, 552, err)
+			assertErrorCode(t, errcode.CodeAccessLimited, err)
 		}
 		{
 			args.Size = 1024
@@ -245,6 +296,21 @@ func TestAccessServicePut(t *testing.T) {
 		return fmt.Sprintf("%s/put?size=%d&hashes=%d", host, size, hashes)
 	}
 
+	urlForClusterID := func(size int64, hashes access.HashAlgorithm, assignClusterID proto.ClusterID) string {
+		return fmt.Sprintf("%s/put?size=%d&hashes=%d&assign_cluster_id=%d", host, size, hashes, assignClusterID)
+	}
+
+	urlForCodeMode := func(size int64, hashes access.HashAlgorithm, codeMode codemode.CodeMode) string {
+		return fmt.Sprintf("%s/put?size=%d&hashes=%d&code_mode=%d", host, size, hashes, codeMode)
+	}
+
+	urlForClusterIDAndCodeMode := func(size int64, hashes access.HashAlgorithm,
+		assignClusterID proto.ClusterID, codeMode codemode.CodeMode,
+	) string {
+		return fmt.Sprintf("%s/put?size=%d&hashes=%d&assign_cluster_id=%d&code_mode=%d",
+			host, size, hashes, assignClusterID, codeMode)
+	}
+
 	for _, method := range []string{http.MethodPut, http.MethodPost} {
 		args := access.PutArgs{
 			Size:   0,
@@ -276,7 +342,38 @@ func TestAccessServicePut(t *testing.T) {
 			resp := &access.PutResp{}
 			err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode())
 			require.NoError(t, err)
-			require.Equal(t, uint64(1024), resp.Location.Size)
+			require.Equal(t, uint64(1024), resp.Location.Size_)
+			require.Equal(t, proto.ClusterID(1), resp.Location.ClusterID)
+			require.Equal(t, codemode.EC15P12, resp.Location.CodeMode)
+		}
+		{
+			args.Body = bytes.NewReader(make([]byte, 1024))
+			req, _ := http.NewRequest(method,
+				urlForClusterIDAndCodeMode(1024, args.Hashes, proto.ClusterID(99), codemode.EC12P4), args.Body)
+			resp := &access.PutResp{}
+			err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode())
+			require.NoError(t, err)
+			require.Equal(t, uint64(1024), resp.Location.Size_)
+			require.Equal(t, proto.ClusterID(99), resp.Location.ClusterID)
+			require.Equal(t, codemode.EC12P4, resp.Location.CodeMode)
+		}
+		{
+			args.Body = bytes.NewReader(make([]byte, 1024))
+			req, _ := http.NewRequest(method, urlForClusterID(1024, args.Hashes, proto.ClusterID(11)), args.Body)
+			resp := &access.PutResp{}
+			err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode())
+			require.NoError(t, err)
+			require.Equal(t, uint64(1024), resp.Location.Size_)
+			require.Equal(t, proto.ClusterID(11), resp.Location.ClusterID)
+		}
+		{
+			args.Body = bytes.NewReader(make([]byte, 1024))
+			req, _ := http.NewRequest(method, urlForCodeMode(1024, args.Hashes, codemode.EC6P6), args.Body)
+			resp := &access.PutResp{}
+			err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode())
+			require.NoError(t, err)
+			require.Equal(t, uint64(1024), resp.Location.Size_)
+			require.Equal(t, codemode.EC6P6, resp.Location.CodeMode)
 		}
 	}
 }
@@ -301,28 +398,28 @@ func TestAccessServiceGet(t *testing.T) {
 		require.Equal(t, 400, resp.StatusCode, resp.Status)
 	}
 	{
-		args.Location.Size = 1023
+		args.Location.Size_ = 1023
 		args.ReadSize = 1023
-		fillCrc(&args.Location)
+		security.LocationCrcFill(&args.Location)
 		resp, err := cli.Post(ctx, url(), args)
 		require.NoError(t, err)
 		resp.Body.Close()
 		require.Equal(t, 500, resp.StatusCode, resp.Status)
 	}
 	{
-		args.Location.Size = 1024
+		args.Location.Size_ = 1024
 		args.ReadSize = 1024
-		fillCrc(&args.Location)
+		security.LocationCrcFill(&args.Location)
 		resp, err := cli.Post(ctx, url(), args)
 		require.NoError(t, err)
 		resp.Body.Close()
 		require.Equal(t, 200, resp.StatusCode, resp.Status)
 	}
 	{
-		args.Location.Size = 10240
+		args.Location.Size_ = 10240
 		args.Offset = 1000
 		args.ReadSize = 1024
-		fillCrc(&args.Location)
+		security.LocationCrcFill(&args.Location)
 		resp, err := cli.Post(ctx, url(), args)
 		require.NoError(t, err)
 		resp.Body.Close()
@@ -362,7 +459,7 @@ func TestAccessServiceDelete(t *testing.T) {
 	}
 
 	args := access.DeleteArgs{
-		Locations: []access.Location{location.Copy()},
+		Locations: []proto.Location{location.Copy()},
 	}
 	{
 		code, _, err := deleteRequest(access.DeleteArgs{})
@@ -375,7 +472,7 @@ func TestAccessServiceDelete(t *testing.T) {
 		require.Equal(t, 400, code)
 	}
 	{
-		fillCrc(&args.Locations[0])
+		security.LocationCrcFill(&args.Locations[0])
 		code, resp, err := deleteRequest(args)
 		require.NoError(t, err)
 		require.Equal(t, 226, code)
@@ -383,17 +480,17 @@ func TestAccessServiceDelete(t *testing.T) {
 	}
 	{
 		loc := &args.Locations[0]
-		loc.Size = 1024
-		fillCrc(loc)
+		loc.Size_ = 1024
+		security.LocationCrcFill(loc)
 		code, _, err := deleteRequest(args)
 		require.NoError(t, err)
 		require.Equal(t, 200, code)
 	}
 	{
 		loc := location.Copy()
-		loc.Size = 1024
-		fillCrc(&loc)
-		locs := make([]access.Location, access.MaxDeleteLocations)
+		loc.Size_ = 1024
+		security.LocationCrcFill(&loc)
+		locs := make([]proto.Location, access.MaxDeleteLocations)
 		for idx := range locs {
 			locs[idx] = loc
 		}
@@ -404,9 +501,9 @@ func TestAccessServiceDelete(t *testing.T) {
 	}
 	{
 		loc := location.Copy()
-		loc.Size = 1024
-		fillCrc(&loc)
-		locs := make([]access.Location, access.MaxDeleteLocations+1)
+		loc.Size_ = 1024
+		security.LocationCrcFill(&loc)
+		locs := make([]proto.Location, access.MaxDeleteLocations+1)
 		for idx := range locs {
 			locs[idx] = loc
 		}
@@ -416,22 +513,22 @@ func TestAccessServiceDelete(t *testing.T) {
 	}
 	{
 		loc := location.Copy()
-		loc.Size = 1024
+		loc.Size_ = 1024
 		loc.ClusterID = proto.ClusterID(11)
-		fillCrc(&loc)
-		code, resp, err := deleteRequest(access.DeleteArgs{Locations: []access.Location{loc}})
+		security.LocationCrcFill(&loc)
+		code, resp, err := deleteRequest(access.DeleteArgs{Locations: []proto.Location{loc}})
 		require.NoError(t, err)
 		require.Equal(t, 226, code)
 		require.Equal(t, 1, len(resp.FailedLocations))
 		require.Equal(t, proto.ClusterID(11), resp.FailedLocations[0].ClusterID)
 	}
 	{
-		locs := make([]access.Location, access.MaxDeleteLocations)
+		locs := make([]proto.Location, access.MaxDeleteLocations)
 		for idx := range locs {
 			loc := location.Copy()
-			loc.Size = 1024
+			loc.Size_ = 1024
 			loc.ClusterID = proto.ClusterID(idx % 11)
-			fillCrc(&loc)
+			security.LocationCrcFill(&loc)
 			locs[idx] = loc
 		}
 		code, resp, err := deleteRequest(access.DeleteArgs{Locations: locs})
@@ -502,7 +599,7 @@ func TestAccessServiceSign(t *testing.T) {
 		return fmt.Sprintf("%s/sign", host)
 	}
 	args := access.SignArgs{
-		Locations: []access.Location{location.Copy()},
+		Locations: []proto.Location{location.Copy()},
 		Location:  location.Copy(),
 	}
 	{
@@ -516,7 +613,7 @@ func TestAccessServiceSign(t *testing.T) {
 		assertErrorCode(t, 400, err)
 	}
 	{
-		fillCrc(&args.Locations[0])
+		security.LocationCrcFill(&args.Locations[0])
 		resp := &access.SignResp{}
 		err := cli.PostWith(ctx, url(), resp, args)
 		require.NoError(t, err)
@@ -530,153 +627,162 @@ func assertErrorCode(t *testing.T, code int, err error) {
 }
 
 func TestAccessServiceTokens(t *testing.T) {
-	skey := tokenSecretKeys[0][:]
-	checker := func(loc *access.Location, tokens []string) {
-		if loc.Size == 0 {
+	skey := security.TokenSecretKeys()[0][:]
+	checker := func(loc *proto.Location, tokens []string) {
+		if loc.Size_ == 0 {
 			require.Equal(t, 0, len(tokens))
 			return
 		}
 
-		hasMultiBlobs := loc.Size >= uint64(loc.BlobSize)
-		lastSize := uint32(loc.Size % uint64(loc.BlobSize))
+		hasMultiBlobs := loc.Size_ >= uint64(loc.SliceSize)
+		lastSize := uint32(loc.Size_ % uint64(loc.SliceSize))
 		if !hasMultiBlobs {
 			require.Equal(t, 1, len(tokens))
 
-			token := uptoken.DecodeToken(tokens[0])
-			blob := loc.Blobs[0]
-			for bid := blob.MinBid - 100; bid < blob.MinBid+100; bid++ {
-				require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
+			token := security.DecodeToken(tokens[0])
+			blob := loc.Slices[0]
+			for bid := blob.MinSliceID - 100; bid < blob.MinSliceID+100; bid++ {
+				require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey))
 			}
-			require.True(t, token.IsValid(loc.ClusterID, blob.Vid, blob.MinBid, lastSize, skey))
+			require.True(t, token.IsValid(loc.ClusterID, blob.Vid, blob.MinSliceID, lastSize, skey))
 			return
 		}
 
 		if lastSize == 0 {
-			require.Equal(t, len(loc.Blobs), len(tokens))
-			for idx, blob := range loc.Blobs {
-				token := uptoken.DecodeToken(tokens[idx])
+			require.Equal(t, len(loc.Slices), len(tokens))
+			for idx, blob := range loc.Slices {
+				token := security.DecodeToken(tokens[idx])
 				for ii := uint32(0); ii < 100; ii++ {
-					bid := blob.MinBid - proto.BlobID(ii) - 1
-					require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
-					bid = blob.MinBid + proto.BlobID(blob.Count+ii)
-					require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
+					bid := blob.MinSliceID - proto.BlobID(ii) - 1
+					require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey))
+					bid = blob.MinSliceID + proto.BlobID(blob.Count+ii)
+					require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey))
 				}
 				for ii := uint32(0); ii < blob.Count; ii++ {
-					bid := blob.MinBid + proto.BlobID(ii)
-					require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
+					bid := blob.MinSliceID + proto.BlobID(ii)
+					require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey))
 				}
 			}
 			return
 		}
 
-		require.Equal(t, len(loc.Blobs)+1, len(tokens))
-		for ii := 0; ii < len(loc.Blobs)-1; ii++ {
-			token := uptoken.DecodeToken(tokens[ii])
-			blob := loc.Blobs[ii]
+		if loc.Slices[len(loc.Slices)-1].Count == 1 {
+			require.Equal(t, len(loc.Slices), len(tokens))
+			idx := len(loc.Slices) - 1
+			blob := loc.Slices[idx]
+			token := security.DecodeToken(tokens[idx])
+			require.True(t, token.IsValid(loc.ClusterID, blob.Vid, blob.MinSliceID, lastSize, skey))
+			return
+		}
+
+		require.Equal(t, len(loc.Slices)+1, len(tokens))
+		for ii := 0; ii < len(loc.Slices)-1; ii++ {
+			token := security.DecodeToken(tokens[ii])
+			blob := loc.Slices[ii]
 			for ii := uint32(0); ii < blob.Count; ii++ {
-				bid := blob.MinBid + proto.BlobID(ii)
-				require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
+				bid := blob.MinSliceID + proto.BlobID(ii)
+				require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey))
 			}
 		}
 
-		token := uptoken.DecodeToken(tokens[len(loc.Blobs)-1])
-		blob := loc.Blobs[len(loc.Blobs)-1]
+		token := security.DecodeToken(tokens[len(loc.Slices)-1])
+		blob := loc.Slices[len(loc.Slices)-1]
 		for ii := uint32(0); ii < 100; ii++ {
-			bid := blob.MinBid - proto.BlobID(ii) - 1
-			require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
-			bid = blob.MinBid + proto.BlobID(blob.Count+ii) - 1
-			require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
+			bid := blob.MinSliceID - proto.BlobID(ii) - 1
+			require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey))
+			bid = blob.MinSliceID + proto.BlobID(blob.Count+ii) - 1
+			require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey))
 		}
 		for ii := uint32(0); ii < blob.Count-1; ii++ {
-			bid := blob.MinBid + proto.BlobID(ii)
-			require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey))
+			bid := blob.MinSliceID + proto.BlobID(ii)
+			require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.SliceSize, skey))
 		}
 
-		token = uptoken.DecodeToken(tokens[len(loc.Blobs)])
-		lastbid := blob.MinBid + proto.BlobID(blob.Count) - 1
+		token = security.DecodeToken(tokens[len(loc.Slices)])
+		lastbid := blob.MinSliceID + proto.BlobID(blob.Count) - 1
 		require.True(t, token.IsValid(loc.ClusterID, blob.Vid, lastbid, lastSize, skey))
 	}
 
 	{
-		loc := &access.Location{
-			Size:     0,
-			BlobSize: 333,
-			Blobs:    []access.SliceInfo{},
+		loc := &proto.Location{
+			Size_:     0,
+			SliceSize: 333,
+			Slices:    []proto.Slice{},
 		}
-		checker(loc, genTokens(loc))
+		checker(loc, security.StreamGenTokens(loc))
 	}
 	{
-		loc := &access.Location{
-			Size:     1,
-			BlobSize: 1024,
-			Blobs: []access.SliceInfo{
-				{MinBid: 100, Vid: 1000, Count: 1},
+		loc := &proto.Location{
+			Size_:     1,
+			SliceSize: 1024,
+			Slices: []proto.Slice{
+				{MinSliceID: 100, Vid: 1000, Count: 1},
 			},
 		}
-		checker(loc, genTokens(loc))
+		checker(loc, security.StreamGenTokens(loc))
 	}
 	{
-		loc := &access.Location{
-			Size:     1024,
-			BlobSize: 1024,
-			Blobs: []access.SliceInfo{
-				{MinBid: 100, Vid: 1000, Count: 1},
+		loc := &proto.Location{
+			Size_:     1024,
+			SliceSize: 1024,
+			Slices: []proto.Slice{
+				{MinSliceID: 100, Vid: 1000, Count: 1},
 			},
 		}
-		checker(loc, genTokens(loc))
+		checker(loc, security.StreamGenTokens(loc))
 	}
 	{
-		loc := &access.Location{
-			Size:     1025,
-			BlobSize: 1024,
-			Blobs: []access.SliceInfo{
-				{MinBid: 100, Vid: 1000, Count: 2},
+		loc := &proto.Location{
+			Size_:     1025,
+			SliceSize: 1024,
+			Slices: []proto.Slice{
+				{MinSliceID: 100, Vid: 1000, Count: 2},
 			},
 		}
-		checker(loc, genTokens(loc))
+		checker(loc, security.StreamGenTokens(loc))
 	}
 	{
-		loc := &access.Location{
-			Size:     2048,
-			BlobSize: 1024,
-			Blobs: []access.SliceInfo{
-				{MinBid: 100, Vid: 1000, Count: 2},
+		loc := &proto.Location{
+			Size_:     2048,
+			SliceSize: 1024,
+			Slices: []proto.Slice{
+				{MinSliceID: 100, Vid: 1000, Count: 2},
 			},
 		}
-		checker(loc, genTokens(loc))
+		checker(loc, security.StreamGenTokens(loc))
 	}
 	{
-		loc := &access.Location{
-			Size:     10240,
-			BlobSize: 1024,
-			Blobs: []access.SliceInfo{
-				{MinBid: 100, Vid: 1000, Count: 4},
-				{MinBid: 200, Vid: 1000, Count: 6},
+		loc := &proto.Location{
+			Size_:     10240,
+			SliceSize: 1024,
+			Slices: []proto.Slice{
+				{MinSliceID: 100, Vid: 1000, Count: 4},
+				{MinSliceID: 200, Vid: 1000, Count: 6},
 			},
 		}
-		checker(loc, genTokens(loc))
+		checker(loc, security.StreamGenTokens(loc))
 	}
 	{
-		loc := &access.Location{
-			Size:     1025,
-			BlobSize: 1024,
-			Blobs: []access.SliceInfo{
-				{MinBid: 100, Vid: 1000, Count: 1},
-				{MinBid: 200, Vid: 1000, Count: 1},
+		loc := &proto.Location{
+			Size_:     1025,
+			SliceSize: 1024,
+			Slices: []proto.Slice{
+				{MinSliceID: 100, Vid: 1000, Count: 1},
+				{MinSliceID: 200, Vid: 1000, Count: 1},
 			},
 		}
-		checker(loc, genTokens(loc))
+		checker(loc, security.StreamGenTokens(loc))
 	}
 	{
-		loc := &access.Location{
-			Size:     10242,
-			BlobSize: 1024,
-			Blobs: []access.SliceInfo{
-				{MinBid: 100, Vid: 1000, Count: 5},
-				{MinBid: 200, Vid: 1000, Count: 6},
+		loc := &proto.Location{
+			Size_:     10242,
+			SliceSize: 1024,
+			Slices: []proto.Slice{
+				{MinSliceID: 100, Vid: 1000, Count: 5},
+				{MinSliceID: 200, Vid: 1000, Count: 6},
 			},
 		}
-		checker(loc, genTokens(loc))
+		checker(loc, security.StreamGenTokens(loc))
 	}
 }
 
@@ -698,7 +804,7 @@ func TestAccessServiceLimited(t *testing.T) {
 			if err != nil {
 				assertErrorCode(t, errcode.CodeAccessLimited, err)
 			} else {
-				require.Equal(t, uint64(1024), resp.Location.Size)
+				require.Equal(t, uint64(1024), resp.Location.Size_)
 			}
 		}()
 	}

blobstore/access/stream/codemode.go

@@ -12,7 +12,7 @@
 // implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package access
+package stream
 
 import (
 	"fmt"
@@ -43,3 +43,22 @@ func (c CodeModePairs) SelectCodeMode(size int64) codemode.CodeMode {
 
 	panic(fmt.Sprintf("no codemode policy to be selected by size %d, %+v", size, c))
 }
+
+// Verify select codemode
+func (c CodeModePairs) VerifySelectCodeMode(selectCodeMode codemode.CodeMode) bool {
+	if !selectCodeMode.IsValid() {
+		return false
+	}
+
+	for codeMode, pair := range c {
+		policy := pair.Policy
+		if !policy.Enable {
+			continue
+		}
+		if selectCodeMode == codeMode {
+			return true
+		}
+	}
+
+	return false
+}

blobstore/access/stream/codemode_test.go

@@ -12,20 +12,20 @@
 // implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package access_test
+package stream_test
 
 import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
 
-	"github.com/cubefs/cubefs/blobstore/access"
+	"github.com/cubefs/cubefs/blobstore/access/stream"
 	"github.com/cubefs/cubefs/blobstore/common/codemode"
 )
 
 func TestAccessStreamCodeModePairs(t *testing.T) {
-	m := access.CodeModePairs{
-		codemode.EC6P6: access.CodeModePair{
+	m := stream.CodeModePairs{
+		codemode.EC6P6: stream.CodeModePair{
 			Policy: codemode.Policy{
 				ModeName: codemode.EC6P6.Name(),
 				MinSize:  1 << 10,
@@ -34,7 +34,7 @@ func TestAccessStreamCodeModePairs(t *testing.T) {
 			},
 			Tactic: codemode.EC6P6.Tactic(),
 		},
-		codemode.EC6P10L2: access.CodeModePair{
+		codemode.EC6P10L2: stream.CodeModePair{
 			Policy: codemode.Policy{
 				ModeName: codemode.EC6P10L2.Name(),
 				MinSize:  1 << 30,

blobstore/access/stream/config_defaulter.go

@@ -12,17 +12,19 @@
 // implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package access
+package stream
 
 const (
 	defaultMaxBlobSize uint32 = 1 << 22 // 4MB
 
-	defaultDiskPunishIntervalS    int = 60
-	defaultServicePunishIntervalS int = 60
-	defaultAllocRetryTimes        int = 3
-	defaultAllocRetryIntervalMS   int = 100
-	defaultEncoderConcurrency     int = 1000
-	defaultMinReadShardsX         int = 1
+	defaultDiskPunishIntervalS      int = 60
+	defaultServicePunishIntervalS   int = 60
+	defaultAllocRetryTimes          int = 3
+	defaultAllocRetryIntervalMS     int = 100
+	defaultEncoderConcurrency       int = 1000
+	defaultMinReadShardsX           int = 1
+	defaultShardnodeRetryTimes      int = 3
+	defaultShardnodeRetryIntervalMS int = 200
 
 	// client timeout ms
 	defaultTimeoutClusterMgr int64 = 1000 * 3

blobstore/access/stream/controller_mock_test.go

@@ -0,0 +1,608 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/cubefs/cubefs/blobstore/access/controller (interfaces: ClusterController,ServiceController,VolumeGetter,IShardController,Shard)
+
+// Package stream is a generated GoMock package.
+package stream
+
+import (
+	context "context"
+	reflect "reflect"
+
+	controller "github.com/cubefs/cubefs/blobstore/access/controller"
+	access "github.com/cubefs/cubefs/blobstore/api/access"
+	clustermgr "github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	shardnode "github.com/cubefs/cubefs/blobstore/api/shardnode"
+	proto "github.com/cubefs/cubefs/blobstore/common/proto"
+	sharding "github.com/cubefs/cubefs/blobstore/common/sharding"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockClusterController is a mock of ClusterController interface.
+type MockClusterController struct {
+	ctrl     *gomock.Controller
+	recorder *MockClusterControllerMockRecorder
+}
+
+// MockClusterControllerMockRecorder is the mock recorder for MockClusterController.
+type MockClusterControllerMockRecorder struct {
+	mock *MockClusterController
+}
+
+// NewMockClusterController creates a new mock instance.
+func NewMockClusterController(ctrl *gomock.Controller) *MockClusterController {
+	mock := &MockClusterController{ctrl: ctrl}
+	mock.recorder = &MockClusterControllerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockClusterController) EXPECT() *MockClusterControllerMockRecorder {
+	return m.recorder
+}
+
+// All mocks base method.
+func (m *MockClusterController) All() []*clustermgr.ClusterInfo {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "All")
+	ret0, _ := ret[0].([]*clustermgr.ClusterInfo)
+	return ret0
+}
+
+// All indicates an expected call of All.
+func (mr *MockClusterControllerMockRecorder) All() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "All", reflect.TypeOf((*MockClusterController)(nil).All))
+}
+
+// ChangeChooseAlg mocks base method.
+func (m *MockClusterController) ChangeChooseAlg(arg0 controller.AlgChoose) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ChangeChooseAlg", arg0)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// ChangeChooseAlg indicates an expected call of ChangeChooseAlg.
+func (mr *MockClusterControllerMockRecorder) ChangeChooseAlg(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChangeChooseAlg", reflect.TypeOf((*MockClusterController)(nil).ChangeChooseAlg), arg0)
+}
+
+// ChooseOne mocks base method.
+func (m *MockClusterController) ChooseOne() (*clustermgr.ClusterInfo, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ChooseOne")
+	ret0, _ := ret[0].(*clustermgr.ClusterInfo)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ChooseOne indicates an expected call of ChooseOne.
+func (mr *MockClusterControllerMockRecorder) ChooseOne() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChooseOne", reflect.TypeOf((*MockClusterController)(nil).ChooseOne))
+}
+
+// GetConfig mocks base method.
+func (m *MockClusterController) GetConfig(arg0 context.Context, arg1 string) (string, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetConfig", arg0, arg1)
+	ret0, _ := ret[0].(string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetConfig indicates an expected call of GetConfig.
+func (mr *MockClusterControllerMockRecorder) GetConfig(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfig", reflect.TypeOf((*MockClusterController)(nil).GetConfig), arg0, arg1)
+}
+
+// GetServiceController mocks base method.
+func (m *MockClusterController) GetServiceController(arg0 proto.ClusterID) (controller.ServiceController, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetServiceController", arg0)
+	ret0, _ := ret[0].(controller.ServiceController)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetServiceController indicates an expected call of GetServiceController.
+func (mr *MockClusterControllerMockRecorder) GetServiceController(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetServiceController", reflect.TypeOf((*MockClusterController)(nil).GetServiceController), arg0)
+}
+
+// GetShardController mocks base method.
+func (m *MockClusterController) GetShardController(arg0 proto.ClusterID) (controller.IShardController, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetShardController", arg0)
+	ret0, _ := ret[0].(controller.IShardController)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetShardController indicates an expected call of GetShardController.
+func (mr *MockClusterControllerMockRecorder) GetShardController(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetShardController", reflect.TypeOf((*MockClusterController)(nil).GetShardController), arg0)
+}
+
+// GetVolumeGetter mocks base method.
+func (m *MockClusterController) GetVolumeGetter(arg0 proto.ClusterID) (controller.VolumeGetter, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetVolumeGetter", arg0)
+	ret0, _ := ret[0].(controller.VolumeGetter)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetVolumeGetter indicates an expected call of GetVolumeGetter.
+func (mr *MockClusterControllerMockRecorder) GetVolumeGetter(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVolumeGetter", reflect.TypeOf((*MockClusterController)(nil).GetVolumeGetter), arg0)
+}
+
+// Region mocks base method.
+func (m *MockClusterController) Region() string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Region")
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// Region indicates an expected call of Region.
+func (mr *MockClusterControllerMockRecorder) Region() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Region", reflect.TypeOf((*MockClusterController)(nil).Region))
+}
+
+// MockServiceController is a mock of ServiceController interface.
+type MockServiceController struct {
+	ctrl     *gomock.Controller
+	recorder *MockServiceControllerMockRecorder
+}
+
+// MockServiceControllerMockRecorder is the mock recorder for MockServiceController.
+type MockServiceControllerMockRecorder struct {
+	mock *MockServiceController
+}
+
+// NewMockServiceController creates a new mock instance.
+func NewMockServiceController(ctrl *gomock.Controller) *MockServiceController {
+	mock := &MockServiceController{ctrl: ctrl}
+	mock.recorder = &MockServiceControllerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockServiceController) EXPECT() *MockServiceControllerMockRecorder {
+	return m.recorder
+}
+
+// GetDiskHost mocks base method.
+func (m *MockServiceController) GetDiskHost(arg0 context.Context, arg1 proto.DiskID) (*controller.HostIDC, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetDiskHost", arg0, arg1)
+	ret0, _ := ret[0].(*controller.HostIDC)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetDiskHost indicates an expected call of GetDiskHost.
+func (mr *MockServiceControllerMockRecorder) GetDiskHost(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDiskHost", reflect.TypeOf((*MockServiceController)(nil).GetDiskHost), arg0, arg1)
+}
+
+// GetServiceHost mocks base method.
+func (m *MockServiceController) GetServiceHost(arg0 context.Context, arg1 string) (string, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetServiceHost", arg0, arg1)
+	ret0, _ := ret[0].(string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetServiceHost indicates an expected call of GetServiceHost.
+func (mr *MockServiceControllerMockRecorder) GetServiceHost(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetServiceHost", reflect.TypeOf((*MockServiceController)(nil).GetServiceHost), arg0, arg1)
+}
+
+// GetServiceHosts mocks base method.
+func (m *MockServiceController) GetServiceHosts(arg0 context.Context, arg1 string) ([]string, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetServiceHosts", arg0, arg1)
+	ret0, _ := ret[0].([]string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetServiceHosts indicates an expected call of GetServiceHosts.
+func (mr *MockServiceControllerMockRecorder) GetServiceHosts(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetServiceHosts", reflect.TypeOf((*MockServiceController)(nil).GetServiceHosts), arg0, arg1)
+}
+
+// GetShardnodeHost mocks base method.
+func (m *MockServiceController) GetShardnodeHost(arg0 context.Context, arg1 proto.DiskID) (*controller.HostIDC, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetShardnodeHost", arg0, arg1)
+	ret0, _ := ret[0].(*controller.HostIDC)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetShardnodeHost indicates an expected call of GetShardnodeHost.
+func (mr *MockServiceControllerMockRecorder) GetShardnodeHost(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetShardnodeHost", reflect.TypeOf((*MockServiceController)(nil).GetShardnodeHost), arg0, arg1)
+}
+
+// IsPunishShardnode mocks base method.
+func (m *MockServiceController) IsPunishShardnode(arg0 proto.DiskID) bool {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "IsPunishShardnode", arg0)
+	ret0, _ := ret[0].(bool)
+	return ret0
+}
+
+// IsPunishShardnode indicates an expected call of IsPunishShardnode.
+func (mr *MockServiceControllerMockRecorder) IsPunishShardnode(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsPunishShardnode", reflect.TypeOf((*MockServiceController)(nil).IsPunishShardnode), arg0)
+}
+
+// PunishDisk mocks base method.
+func (m *MockServiceController) PunishDisk(arg0 context.Context, arg1 proto.DiskID, arg2 int) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "PunishDisk", arg0, arg1, arg2)
+}
+
+// PunishDisk indicates an expected call of PunishDisk.
+func (mr *MockServiceControllerMockRecorder) PunishDisk(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PunishDisk", reflect.TypeOf((*MockServiceController)(nil).PunishDisk), arg0, arg1, arg2)
+}
+
+// PunishDiskWithThreshold mocks base method.
+func (m *MockServiceController) PunishDiskWithThreshold(arg0 context.Context, arg1 proto.DiskID, arg2 int) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "PunishDiskWithThreshold", arg0, arg1, arg2)
+}
+
+// PunishDiskWithThreshold indicates an expected call of PunishDiskWithThreshold.
+func (mr *MockServiceControllerMockRecorder) PunishDiskWithThreshold(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PunishDiskWithThreshold", reflect.TypeOf((*MockServiceController)(nil).PunishDiskWithThreshold), arg0, arg1, arg2)
+}
+
+// PunishService mocks base method.
+func (m *MockServiceController) PunishService(arg0 context.Context, arg1, arg2 string, arg3 int) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "PunishService", arg0, arg1, arg2, arg3)
+}
+
+// PunishService indicates an expected call of PunishService.
+func (mr *MockServiceControllerMockRecorder) PunishService(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PunishService", reflect.TypeOf((*MockServiceController)(nil).PunishService), arg0, arg1, arg2, arg3)
+}
+
+// PunishServiceWithThreshold mocks base method.
+func (m *MockServiceController) PunishServiceWithThreshold(arg0 context.Context, arg1, arg2 string, arg3 int) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "PunishServiceWithThreshold", arg0, arg1, arg2, arg3)
+}
+
+// PunishServiceWithThreshold indicates an expected call of PunishServiceWithThreshold.
+func (mr *MockServiceControllerMockRecorder) PunishServiceWithThreshold(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PunishServiceWithThreshold", reflect.TypeOf((*MockServiceController)(nil).PunishServiceWithThreshold), arg0, arg1, arg2, arg3)
+}
+
+// PunishShardnode mocks base method.
+func (m *MockServiceController) PunishShardnode(arg0 context.Context, arg1 proto.DiskID, arg2 int) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "PunishShardnode", arg0, arg1, arg2)
+}
+
+// PunishShardnode indicates an expected call of PunishShardnode.
+func (mr *MockServiceControllerMockRecorder) PunishShardnode(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PunishShardnode", reflect.TypeOf((*MockServiceController)(nil).PunishShardnode), arg0, arg1, arg2)
+}
+
+// PunishShardnodeDiskWithThreshold mocks base method.
+func (m *MockServiceController) PunishShardnodeDiskWithThreshold(arg0 context.Context, arg1 proto.DiskID, arg2 int) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "PunishShardnodeDiskWithThreshold", arg0, arg1, arg2)
+}
+
+// PunishShardnodeDiskWithThreshold indicates an expected call of PunishShardnodeDiskWithThreshold.
+func (mr *MockServiceControllerMockRecorder) PunishShardnodeDiskWithThreshold(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PunishShardnodeDiskWithThreshold", reflect.TypeOf((*MockServiceController)(nil).PunishShardnodeDiskWithThreshold), arg0, arg1, arg2)
+}
+
+// MockVolumeGetter is a mock of VolumeGetter interface.
+type MockVolumeGetter struct {
+	ctrl     *gomock.Controller
+	recorder *MockVolumeGetterMockRecorder
+}
+
+// MockVolumeGetterMockRecorder is the mock recorder for MockVolumeGetter.
+type MockVolumeGetterMockRecorder struct {
+	mock *MockVolumeGetter
+}
+
+// NewMockVolumeGetter creates a new mock instance.
+func NewMockVolumeGetter(ctrl *gomock.Controller) *MockVolumeGetter {
+	mock := &MockVolumeGetter{ctrl: ctrl}
+	mock.recorder = &MockVolumeGetterMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockVolumeGetter) EXPECT() *MockVolumeGetterMockRecorder {
+	return m.recorder
+}
+
+// Get mocks base method.
+func (m *MockVolumeGetter) Get(arg0 context.Context, arg1 proto.Vid, arg2 bool) *controller.VolumePhy {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2)
+	ret0, _ := ret[0].(*controller.VolumePhy)
+	return ret0
+}
+
+// Get indicates an expected call of Get.
+func (mr *MockVolumeGetterMockRecorder) Get(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockVolumeGetter)(nil).Get), arg0, arg1, arg2)
+}
+
+// Punish mocks base method.
+func (m *MockVolumeGetter) Punish(arg0 context.Context, arg1 proto.Vid, arg2 int) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "Punish", arg0, arg1, arg2)
+}
+
+// Punish indicates an expected call of Punish.
+func (mr *MockVolumeGetterMockRecorder) Punish(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Punish", reflect.TypeOf((*MockVolumeGetter)(nil).Punish), arg0, arg1, arg2)
+}
+
+// Update mocks base method.
+func (m *MockVolumeGetter) Update(arg0 context.Context, arg1 proto.Vid) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "Update", arg0, arg1)
+}
+
+// Update indicates an expected call of Update.
+func (mr *MockVolumeGetterMockRecorder) Update(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockVolumeGetter)(nil).Update), arg0, arg1)
+}
+
+// MockShardController is a mock of IShardController interface.
+type MockShardController struct {
+	ctrl     *gomock.Controller
+	recorder *MockShardControllerMockRecorder
+}
+
+// MockShardControllerMockRecorder is the mock recorder for MockShardController.
+type MockShardControllerMockRecorder struct {
+	mock *MockShardController
+}
+
+// NewMockShardController creates a new mock instance.
+func NewMockShardController(ctrl *gomock.Controller) *MockShardController {
+	mock := &MockShardController{ctrl: ctrl}
+	mock.recorder = &MockShardControllerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockShardController) EXPECT() *MockShardControllerMockRecorder {
+	return m.recorder
+}
+
+// GetFisrtShard mocks base method.
+func (m *MockShardController) GetFisrtShard(arg0 context.Context) (controller.Shard, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetFisrtShard", arg0)
+	ret0, _ := ret[0].(controller.Shard)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetFisrtShard indicates an expected call of GetFisrtShard.
+func (mr *MockShardControllerMockRecorder) GetFisrtShard(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFisrtShard", reflect.TypeOf((*MockShardController)(nil).GetFisrtShard), arg0)
+}
+
+// GetNextShard mocks base method.
+func (m *MockShardController) GetNextShard(arg0 context.Context, arg1 sharding.Range) (controller.Shard, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetNextShard", arg0, arg1)
+	ret0, _ := ret[0].(controller.Shard)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetNextShard indicates an expected call of GetNextShard.
+func (mr *MockShardControllerMockRecorder) GetNextShard(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNextShard", reflect.TypeOf((*MockShardController)(nil).GetNextShard), arg0, arg1)
+}
+
+// GetShard mocks base method.
+func (m *MockShardController) GetShard(arg0 context.Context, arg1 []string) (controller.Shard, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetShard", arg0, arg1)
+	ret0, _ := ret[0].(controller.Shard)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetShard indicates an expected call of GetShard.
+func (mr *MockShardControllerMockRecorder) GetShard(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetShard", reflect.TypeOf((*MockShardController)(nil).GetShard), arg0, arg1)
+}
+
+// GetShardByID mocks base method.
+func (m *MockShardController) GetShardByID(arg0 context.Context, arg1 proto.ShardID) (controller.Shard, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetShardByID", arg0, arg1)
+	ret0, _ := ret[0].(controller.Shard)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetShardByID indicates an expected call of GetShardByID.
+func (mr *MockShardControllerMockRecorder) GetShardByID(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetShardByID", reflect.TypeOf((*MockShardController)(nil).GetShardByID), arg0, arg1)
+}
+
+// GetShardByRange mocks base method.
+func (m *MockShardController) GetShardByRange(arg0 context.Context, arg1 sharding.Range) (controller.Shard, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetShardByRange", arg0, arg1)
+	ret0, _ := ret[0].(controller.Shard)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetShardByRange indicates an expected call of GetShardByRange.
+func (mr *MockShardControllerMockRecorder) GetShardByRange(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetShardByRange", reflect.TypeOf((*MockShardController)(nil).GetShardByRange), arg0, arg1)
+}
+
+// GetShardSubRangeCount mocks base method.
+func (m *MockShardController) GetShardSubRangeCount(arg0 context.Context) int {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetShardSubRangeCount", arg0)
+	ret0, _ := ret[0].(int)
+	return ret0
+}
+
+// GetShardSubRangeCount indicates an expected call of GetShardSubRangeCount.
+func (mr *MockShardControllerMockRecorder) GetShardSubRangeCount(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetShardSubRangeCount", reflect.TypeOf((*MockShardController)(nil).GetShardSubRangeCount), arg0)
+}
+
+// GetSpaceID mocks base method.
+func (m *MockShardController) GetSpaceID() proto.SpaceID {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetSpaceID")
+	ret0, _ := ret[0].(proto.SpaceID)
+	return ret0
+}
+
+// GetSpaceID indicates an expected call of GetSpaceID.
+func (mr *MockShardControllerMockRecorder) GetSpaceID() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSpaceID", reflect.TypeOf((*MockShardController)(nil).GetSpaceID))
+}
+
+// UpdateRoute mocks base method.
+func (m *MockShardController) UpdateRoute(arg0 context.Context) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateRoute", arg0)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// UpdateRoute indicates an expected call of UpdateRoute.
+func (mr *MockShardControllerMockRecorder) UpdateRoute(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateRoute", reflect.TypeOf((*MockShardController)(nil).UpdateRoute), arg0)
+}
+
+// UpdateShard mocks base method.
+func (m *MockShardController) UpdateShard(arg0 context.Context, arg1 shardnode.ShardStats) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateShard", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// UpdateShard indicates an expected call of UpdateShard.
+func (mr *MockShardControllerMockRecorder) UpdateShard(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateShard", reflect.TypeOf((*MockShardController)(nil).UpdateShard), arg0, arg1)
+}
+
+// MockShard is a mock of Shard interface.
+type MockShard struct {
+	ctrl     *gomock.Controller
+	recorder *MockShardMockRecorder
+}
+
+// MockShardMockRecorder is the mock recorder for MockShard.
+type MockShardMockRecorder struct {
+	mock *MockShard
+}
+
+// NewMockShard creates a new mock instance.
+func NewMockShard(ctrl *gomock.Controller) *MockShard {
+	mock := &MockShard{ctrl: ctrl}
+	mock.recorder = &MockShardMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockShard) EXPECT() *MockShardMockRecorder {
+	return m.recorder
+}
+
+// GetMember mocks base method.
+func (m *MockShard) GetMember(arg0 context.Context, arg1 access.GetShardMode, arg2 map[proto.DiskID]struct{}) (controller.ShardOpInfo, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetMember", arg0, arg1, arg2)
+	ret0, _ := ret[0].(controller.ShardOpInfo)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetMember indicates an expected call of GetMember.
+func (mr *MockShardMockRecorder) GetMember(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMember", reflect.TypeOf((*MockShard)(nil).GetMember), arg0, arg1, arg2)
+}
+
+// GetRange mocks base method.
+func (m *MockShard) GetRange() sharding.Range {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetRange")
+	ret0, _ := ret[0].(sharding.Range)
+	return ret0
+}
+
+// GetRange indicates an expected call of GetRange.
+func (mr *MockShardMockRecorder) GetRange() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRange", reflect.TypeOf((*MockShard)(nil).GetRange))
+}
+
+// GetShardID mocks base method.
+func (m *MockShard) GetShardID() proto.ShardID {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetShardID")
+	ret0, _ := ret[0].(proto.ShardID)
+	return ret0
+}
+
+// GetShardID indicates an expected call of GetShardID.
+func (mr *MockShardMockRecorder) GetShardID() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetShardID", reflect.TypeOf((*MockShard)(nil).GetShardID))
+}

blobstore/access/stream/limiter.go

@@ -12,7 +12,7 @@
 // implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package access
+package stream
 
 import (
 	"context"
@@ -118,20 +118,20 @@ type Writer struct {
 var _ io.Writer = &Writer{}
 
 func (w *Writer) Write(p []byte) (n int, err error) {
-	n, err = w.underlying.Write(p)
-
 	now := time.Now()
-	reserve := w.rate.ReserveN(now, n)
+	reserve := w.rate.ReserveN(now, len(p))
 
 	// Wait if necessary
 	delay := reserve.DelayFrom(now)
 	if delay == 0 {
+		n, err = w.underlying.Write(p)
 		return
 	}
 
 	span := trace.SpanFromContextSafe(w.ctx)
 	if !reserve.OK() {
-		span.Warnf("writer exceeds limiter n:%d, burst:%d", n, w.rate.Burst())
+		span.Warnf("writer exceeds limiter n:%d, burst:%d", len(p), w.rate.Burst())
+		n, err = w.underlying.Write(p)
 		return
 	}
 	t := time.NewTimer(delay)
@@ -143,6 +143,7 @@ func (w *Writer) Write(p []byte) (n int, err error) {
 	select {
 	case <-t.C:
 		// We can proceed.
+		n, err = w.underlying.Write(p)
 		return
 	case <-w.ctx.Done():
 		// Context was canceled before we could proceed.  Cancel the

blobstore/access/stream/limiter_test.go

@@ -12,7 +12,7 @@
 // implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package access
+package stream
 
 import (
 	"context"

blobstore/access/stream/metric.go

@@ -12,12 +12,15 @@
 // implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package access
+package stream
 
 import (
+	"os"
+
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/rpc/auditlog"
 )
 
 var unhealthMetric = prometheus.NewCounterVec(
@@ -40,9 +43,27 @@ var downloadMetric = prometheus.NewCounterVec(
 	[]string{"cluster", "way", "reason"},
 )
 
+var readwriteMetric *prometheus.HistogramVec
+
+var SteamReportDownload = reportDownload
+
 func init() {
 	prometheus.MustRegister(unhealthMetric)
 	prometheus.MustRegister(downloadMetric)
+
+	hostname, _ := os.Hostname()
+	readwriteMetric = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace:   "blobstore",
+			Subsystem:   "access",
+			Name:        "read_write_duration_ms",
+			Help:        "read write duration ms",
+			Buckets:     auditlog.Buckets,
+			ConstLabels: map[string]string{"host": hostname},
+		},
+		[]string{"cluster", "idc", "api"},
+	)
+	prometheus.MustRegister(readwriteMetric)
 }
 
 func reportUnhealth(cid proto.ClusterID, action, module, host, reason string) {
@@ -52,3 +73,8 @@ func reportUnhealth(cid proto.ClusterID, action, module, host, reason string) {
 func reportDownload(cid proto.ClusterID, way, reason string) {
 	downloadMetric.WithLabelValues(cid.ToString(), way, reason).Inc()
 }
+
+// upload_read, upload_write, download_read, download_write
+func reportReadwrite(cid, idc, api string, ms int64) {
+	readwriteMetric.WithLabelValues(cid, idc, api).Observe(float64(ms))
+}

blobstore/access/stream/stream.go

@@ -12,7 +12,7 @@
 // implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package access
+package stream
 
 import (
 	"context"
@@ -20,12 +20,17 @@ import (
 	"fmt"
 	"io"
 	"strings"
+	"sync/atomic"
 
 	"github.com/afex/hystrix-go/hystrix"
+	"golang.org/x/sync/singleflight"
+
 	"github.com/cubefs/cubefs/blobstore/access/controller"
 	"github.com/cubefs/cubefs/blobstore/api/access"
 	"github.com/cubefs/cubefs/blobstore/api/blobnode"
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
 	"github.com/cubefs/cubefs/blobstore/api/proxy"
+	"github.com/cubefs/cubefs/blobstore/api/shardnode"
 	"github.com/cubefs/cubefs/blobstore/common/codemode"
 	"github.com/cubefs/cubefs/blobstore/common/ec"
 	"github.com/cubefs/cubefs/blobstore/common/proto"
@@ -33,7 +38,6 @@ import (
 	"github.com/cubefs/cubefs/blobstore/common/trace"
 	"github.com/cubefs/cubefs/blobstore/util/defaulter"
 	"github.com/cubefs/cubefs/blobstore/util/errors"
-	"github.com/cubefs/cubefs/blobstore/util/log"
 	"github.com/cubefs/cubefs/blobstore/util/retry"
 )
 
@@ -45,6 +49,7 @@ const (
 	rwCommand    = "rw"
 
 	serviceProxy = proto.ServiceNameProxy
+	serviceShard = proto.ServiceNameShardNode
 )
 
 // StreamHandler stream http handler
@@ -56,7 +61,7 @@ type StreamHandler interface {
 	//               codeMode > 0, alloc in this codemode
 	//     return: a location of file
 	Alloc(ctx context.Context, size uint64, blobSize uint32,
-		assignClusterID proto.ClusterID, codeMode codemode.CodeMode) (*access.Location, error)
+		assignClusterID proto.ClusterID, codeMode codemode.CodeMode) (*proto.Location, error)
 
 	// PutAt access interface /putat, put one blob
 	//     required: rc file reader
@@ -69,7 +74,9 @@ type StreamHandler interface {
 	// Put put one object
 	//     required: size, file size
 	//     optional: hasher map to calculate hash.Hash
-	Put(ctx context.Context, rc io.Reader, size int64, hasherMap access.HasherMap) (*access.Location, error)
+	//     optional: code to specify codemode and not choose codemode by size
+	Put(ctx context.Context, rc io.Reader, size int64, hasherMap access.HasherMap,
+		assignClusterID proto.ClusterID, codeMode codemode.CodeMode) (*proto.Location, error)
 
 	// Get read file
 	//     required: location, readSize
@@ -93,19 +100,35 @@ type StreamHandler interface {
 	//...
 	//read-9 [d4                                       p5]
 	//failed
-	Get(ctx context.Context, w io.Writer, location access.Location, readSize, offset uint64) (func() error, error)
+	Get(ctx context.Context, w io.Writer, location proto.Location, readSize, offset uint64) (func() error, error)
 
 	// Delete delete all blobs in this location
-	Delete(ctx context.Context, location *access.Location) error
+	Delete(ctx context.Context, location *proto.Location) error
 
 	// Admin returns internal admin interface.
-	Admin() interface{}
+	Admin() any
+	// GetBlob returns location
+	GetBlob(ctx context.Context, args *access.GetBlobArgs) (*proto.Location, error)
+	// DeleteBlob returns error
+	DeleteBlob(ctx context.Context, args *access.DelBlobArgs) error
+	// SealBlob returns error
+	SealBlob(ctx context.Context, args *access.SealBlobArgs) error
+	// CreateBlob returns location
+	CreateBlob(ctx context.Context, args *access.CreateBlobArgs) (*proto.Location, error)
+	// ListBlob returns blobs
+	ListBlob(ctx context.Context, args *access.ListBlobArgs) (shardnode.ListBlobRet, error)
+	// AllocSlice returns alloc blob
+	AllocSlice(ctx context.Context, args *access.AllocSliceArgs) (shardnode.AllocSliceRet, error)
 }
 
-type streamAdmin struct {
-	config     StreamConfig
-	memPool    *resourcepool.MemPool
-	controller controller.ClusterController
+type StreamAdmin struct {
+	Config     StreamConfig
+	MemPool    *resourcepool.MemPool
+	Controller controller.ClusterController
+}
+
+type ShardnodeConfig struct {
+	shardnode.Config
 }
 
 // StreamConfig access stream handler config
@@ -113,25 +136,52 @@ type StreamConfig struct {
 	IDC string `json:"idc"`
 
 	MaxBlobSize                uint32 `json:"max_blob_size"`
+	VolumePunishIntervalS      int    `json:"volume_punish_interval_s"`
 	DiskPunishIntervalS        int    `json:"disk_punish_interval_s"`
 	DiskTimeoutPunishIntervalS int    `json:"disk_timeout_punish_interval_s"`
-	ServicePunishIntervalS     int    `json:"service_punish_interval_s"`
+	ServicePunishIntervalS     int    `json:"service_punish_interval_s"` // just service of proxy
+	ShardnodePunishIntervalS   int    `json:"shardnode_punish_interval_s"`
 	AllocRetryTimes            int    `json:"alloc_retry_times"`
 	AllocRetryIntervalMS       int    `json:"alloc_retry_interval_ms"`
 	EncoderEnableVerify        bool   `json:"encoder_enableverify"`
 	EncoderConcurrency         int    `json:"encoder_concurrency"`
 	MinReadShardsX             int    `json:"min_read_shards_x"`
-	ShardCrcDisabled           bool   `json:"shard_crc_disabled"`
+	ReadDataOnlyTimeoutMS      int    `json:"read_data_only_timeout_ms"`
+	ShardCrcWriteDisable       bool   `json:"shard_crc_write_disable"`
+	ShardCrcReadEnable         bool   `json:"shard_crc_read_enable"`
+	ShardnodeRetryTimes        int    `json:"shardnode_retry_times"`
+	ShardnodeRetryIntervalMS   int    `json:"shardnode_retry_interval_ms"`
+
+	LogSlowBaseTimeMS  int     `json:"log_slow_base_time_ms"`
+	LogSlowBaseSpeedKB int     `json:"log_slow_base_speed_kb"`
+	LogSlowTimeFator   float32 `json:"log_slow_time_fator"`
+
+	// DeleteIntoShardnodePercentage roundrobin percentage [1-100]
+	DeleteIntoShardnodePercentage int64 `json:"delete_into_shardnode_percentage"`
+	deleteRoundrobin              int64 `json:"-"`
+
+	// RepairIntoShardnodePercentage roundrobin percentage [1-100]
+	RepairIntoShardnodePercentage int64 `json:"repair_into_shardnode_percentage"`
+	repairRoundrobin              int64 `json:"-"`
 
 	MemPoolSizeClasses map[int]int `json:"mem_pool_size_classes"`
 
 	// CodeModesPutQuorums
 	// just for one AZ is down, cant write quorum in all AZs
 	CodeModesPutQuorums map[codemode.CodeMode]int `json:"code_mode_put_quorums"`
+	// CodeModesGetOrdered shards with volume unit order to
+	// reduce reedsolom's inverted matrix cache
+	//
+	// EC24P8 in 1AZ, C(32, 8) = 10518300 matrix
+	// Inverted matrix memory: (24 + 24*24 + 24*24*8) * 10518300 ~= 51 GB
+	CodeModesGetOrdered map[codemode.CodeMode]bool `json:"code_mode_get_ordered"`
+	// CodeModesGetIgnoreIDC no distance when getting cross idc
+	CodeModesGetIgnoreIDC map[codemode.CodeMode]bool `json:"code_mode_get_ignore_idc"`
 
-	ClusterConfig  controller.ClusterConfig `json:"cluster_config"`
-	BlobnodeConfig blobnode.Config          `json:"blobnode_config"`
-	ProxyConfig    proxy.Config             `json:"proxy_config"`
+	ClusterConfig   controller.ClusterConfig `json:"cluster_config"`
+	BlobnodeConfig  blobnode.Config          `json:"blobnode_config"`
+	ProxyConfig     proxy.Config             `json:"proxy_config"`
+	ShardnodeConfig *ShardnodeConfig         `json:"shardnode_config"`
 
 	// hystrix command config
 	AllocCommandConfig hystrix.CommandConfig `json:"alloc_command_config"`
@@ -160,9 +210,11 @@ type Handler struct {
 	memPool           *resourcepool.MemPool
 	encoder           map[codemode.CodeMode]ec.Encoder
 	clusterController controller.ClusterController
+	groupRun          singleflight.Group
 
-	blobnodeClient blobnode.StorageAPI
-	proxyClient    proxy.Client
+	blobnodeClient  blobnode.StorageAPI
+	proxyClient     proxy.Client
+	shardnodeClient shardnode.AccessAPI
 
 	allCodeModes  CodeModePairs
 	maxObjectSize int64
@@ -173,12 +225,12 @@ type Handler struct {
 	StreamConfig
 }
 
-func confCheck(cfg *StreamConfig) {
+func confCheck(cfg *StreamConfig) error {
 	if cfg.IDC == "" {
-		log.Fatal("idc config can not be null")
+		return errors.New("idc config can not be null")
 	}
 	if cfg.ClusterConfig.ConsulAgentAddr == "" && len(cfg.ClusterConfig.Clusters) == 0 {
-		log.Panic("consul or clusters can not all be empty")
+		return errors.New("consul or clusters can not all be empty")
 	}
 	cfg.ClusterConfig.IDC = cfg.IDC
 
@@ -189,20 +241,37 @@ func confCheck(cfg *StreamConfig) {
 	for mode, quorum := range cfg.CodeModesPutQuorums {
 		tactic := mode.Tactic()
 		if quorum < tactic.N+tactic.L+1 || quorum > mode.GetShardNum() {
-			log.Fatalf("invalid put quorum(%d) in codemode(%d): %+v", quorum, mode, tactic)
+			return errors.Newf("invalid put quorum(%d) in codemode(%d): %+v", quorum, mode, tactic)
 		}
 	}
 
 	defaulter.Equal(&cfg.MaxBlobSize, defaultMaxBlobSize)
+	defaulter.IntegerLessOrEqual(&cfg.VolumePunishIntervalS, 60)
 	defaulter.LessOrEqual(&cfg.DiskPunishIntervalS, defaultDiskPunishIntervalS)
 	defaulter.LessOrEqual(&cfg.DiskTimeoutPunishIntervalS, defaultDiskPunishIntervalS/10)
 	defaulter.LessOrEqual(&cfg.ServicePunishIntervalS, defaultServicePunishIntervalS)
+	defaulter.IntegerLessOrEqual(&cfg.ShardnodePunishIntervalS, 60)
 	defaulter.LessOrEqual(&cfg.AllocRetryTimes, defaultAllocRetryTimes)
-	if cfg.AllocRetryIntervalMS <= 100 {
-		cfg.AllocRetryIntervalMS = defaultAllocRetryIntervalMS
+	defaulter.LessOrEqual(&cfg.AllocRetryIntervalMS, defaultAllocRetryIntervalMS)
+	defaulter.LessOrEqual(&cfg.ShardnodeRetryTimes, defaultShardnodeRetryTimes)
+	if cfg.ShardnodeRetryIntervalMS <= defaultShardnodeRetryIntervalMS {
+		cfg.ShardnodeRetryIntervalMS = defaultShardnodeRetryIntervalMS
 	}
 	defaulter.LessOrEqual(&cfg.EncoderConcurrency, defaultEncoderConcurrency)
 	defaulter.LessOrEqual(&cfg.MinReadShardsX, defaultMinReadShardsX)
+	defaulter.LessOrEqual(&cfg.ReadDataOnlyTimeoutMS, 3*1000)
+
+	defaulter.LessOrEqual(&cfg.LogSlowBaseTimeMS, 500)
+	defaulter.Equal(&cfg.LogSlowBaseSpeedKB, 1<<10)
+	defaulter.LessOrEqual(&cfg.LogSlowTimeFator, float32(2.0))
+	defaulter.IntegerLess(&cfg.DeleteIntoShardnodePercentage, 0)
+	if cfg.DeleteIntoShardnodePercentage > 100 {
+		cfg.DeleteIntoShardnodePercentage = 100
+	}
+	defaulter.IntegerLess(&cfg.RepairIntoShardnodePercentage, 0)
+	if cfg.RepairIntoShardnodePercentage > 100 {
+		cfg.RepairIntoShardnodePercentage = 100
+	}
 
 	defaulter.LessOrEqual(&cfg.ClusterConfig.CMClientConfig.Config.ClientTimeoutMs, defaultTimeoutClusterMgr)
 	defaulter.LessOrEqual(&cfg.BlobnodeConfig.ClientTimeoutMs, defaultTimeoutBlobnode)
@@ -223,39 +292,60 @@ func confCheck(cfg *StreamConfig) {
 	defaulter.LessOrEqual(&hc.SleepWindow, defaultBlobnodeSleepWindow)
 	defaulter.LessOrEqual(&hc.ErrorPercentThreshold, defaultBlobnodeErrorPercentThreshold)
 	cfg.RWCommandConfig = hc
+
+	return nil
 }
 
 // NewStreamHandler returns a stream handler
-func NewStreamHandler(cfg *StreamConfig, stopCh <-chan struct{}) StreamHandler {
-	confCheck(cfg)
+func NewStreamHandler(cfg *StreamConfig, stopCh <-chan struct{}) (h StreamHandler, e error) {
+	if e = confCheck(cfg); e != nil {
+		return nil, e
+	}
 	proxyClient := proxy.New(&cfg.ProxyConfig)
 	clusterController, err := controller.NewClusterController(&cfg.ClusterConfig, proxyClient, stopCh)
 	if err != nil {
-		log.Fatalf("new cluster controller failed, err: %v", err)
+		e = errors.Newf("new cluster controller failed, err: %v", err)
+		return
 	}
 
 	handler := &Handler{
 		memPool:           resourcepool.NewMemPool(cfg.MemPoolSizeClasses),
 		clusterController: clusterController,
 
-		blobnodeClient: blobnode.New(&cfg.BlobnodeConfig),
-		proxyClient:    proxyClient,
+		blobnodeClient:  blobnode.New(&cfg.BlobnodeConfig),
+		proxyClient:     proxyClient,
+		shardnodeClient: shardnode.NewNonsupportShardnode(),
 
 		maxObjectSize: defaultMaxObjectSize,
 		StreamConfig:  *cfg,
 	}
+	if cfg.ShardnodeConfig != nil { // enable shard node
+		// Do not use rpc retry, because the stream blob handles retries itself
+		defaulter.LessOrEqual(&cfg.ShardnodeConfig.Config.Retry, int(1))
+		handler.shardnodeClient = shardnode.New(cfg.ShardnodeConfig.Config)
+	} else { // disable write delete/repair msg to shardnode
+		handler.StreamConfig.DeleteIntoShardnodePercentage = 0
+		handler.StreamConfig.RepairIntoShardnodePercentage = 0
+	}
 
+	if err = clustermgr.LoadExtendCodemode(context.Background(), handler.clusterController); err != nil {
+		e = errors.Newf("load extend codemode failed, err: %+v", err)
+		return
+	}
 	rawCodeModePolicies, err := handler.clusterController.GetConfig(context.Background(), proto.CodeModeConfigKey)
 	if err != nil {
-		log.Fatal("get codemode policy from cluster manager failed, err: ", err)
+		e = errors.Newf("get codemode policy from cluster manager failed, err: %+v", err)
+		return
 	}
 	codeModePolicies := make([]codemode.Policy, 0)
 	err = json.Unmarshal([]byte(rawCodeModePolicies), &codeModePolicies)
 	if err != nil {
-		log.Fatal("json decode codemode policy failed, err: ", err)
+		e = errors.Newf("json decode codemode policy failed, err: %+v", err)
+		return
 	}
 	if len(codeModePolicies) <= 0 {
-		log.Fatal("invalid codemode policy raw: ", rawCodeModePolicies)
+		e = errors.Newf("invalid codemode policy raw: %s", rawCodeModePolicies)
+		return
 	}
 
 	allCodeModes := make(CodeModePairs)
@@ -277,7 +367,8 @@ func NewStreamHandler(cfg *StreamConfig, stopCh <-chan struct{}) StreamHandler {
 			Concurrency:  cfg.EncoderConcurrency,
 		})
 		if err != nil {
-			log.Fatalf("new encoder failed, err: %v", err)
+			e = errors.Newf("new encoder failed, err: %v", err)
+			return
 		}
 		encoders[codeMode] = encoder
 	}
@@ -293,22 +384,29 @@ func NewStreamHandler(cfg *StreamConfig, stopCh <-chan struct{}) StreamHandler {
 	handler.discardVidChan = make(chan discardVid, 8)
 	handler.stopCh = stopCh
 	handler.loopDiscardVids()
-	return handler
+	return handler, nil
 }
 
 // Delete delete all blobs in this location
-func (h *Handler) Delete(ctx context.Context, location *access.Location) error {
+func (h *Handler) Delete(ctx context.Context, location *proto.Location) error {
 	span := trace.SpanFromContextSafe(ctx)
-	span.Debugf("to delete %+v", location)
+	if h.DeleteIntoShardnodePercentage > 0 {
+		percentage := (atomic.AddInt64(&h.deleteRoundrobin, 1) % 100) + 1
+		if percentage <= h.DeleteIntoShardnodePercentage {
+			span.Debugf("to delete into shardnode %+v", location)
+			return h.clearGarbageIntoShardnode(ctx, location)
+		}
+	}
+	span.Debugf("to delete into proxy %+v", location)
 	return h.clearGarbage(ctx, location)
 }
 
 // Admin returns internal admin interface.
-func (h *Handler) Admin() interface{} {
-	return &streamAdmin{
-		config:     h.StreamConfig,
-		memPool:    h.memPool,
-		controller: h.clusterController,
+func (h *Handler) Admin() any {
+	return &StreamAdmin{
+		Config:     h.StreamConfig,
+		MemPool:    h.memPool,
+		Controller: h.clusterController,
 	}
 }
 
@@ -322,6 +420,15 @@ func (h *Handler) sendRepairMsg(ctx context.Context, blob blobIdent, badIdxes []
 	span := trace.SpanFromContextSafe(ctx)
 	span.Infof("to repair %s indexes(%+v)", blob.String(), badIdxes)
 
+	if h.RepairIntoShardnodePercentage > 0 {
+		percentage := (atomic.AddInt64(&h.repairRoundrobin, 1) % 100) + 1
+		if percentage <= h.RepairIntoShardnodePercentage {
+			span.Debugf("to repair into shardnode %s", blob.String())
+			h.sendRepairMsgIntoShardnode(ctx, blob, badIdxes)
+			return
+		}
+	}
+
 	clusterID := blob.cid
 	serviceController, err := h.clusterController.GetServiceController(clusterID)
 	if err != nil {
@@ -338,24 +445,30 @@ func (h *Handler) sendRepairMsg(ctx context.Context, blob blobIdent, badIdxes []
 		Reason:    "access-repair",
 	}
 
+	hosts, err := serviceController.GetServiceHosts(ctx, serviceProxy)
+	if err != nil {
+		span.Error(errors.Detail(err))
+		return
+	}
+	for len(hosts) < 3 {
+		hosts = append(hosts, hosts...)
+	}
 	if err := retry.Timed(3, 200).On(func() error {
-		host, err := serviceController.GetServiceHost(ctx, serviceProxy)
-		if err != nil {
-			span.Warn(err)
-			return err
-		}
+		host := hosts[0]
+		hosts = hosts[1:]
 		err = h.proxyClient.SendShardRepairMsg(ctx, host, repairArgs)
 		if err != nil {
 			if errorTimeout(err) || errorConnectionRefused(err) {
 				serviceController.PunishServiceWithThreshold(ctx, serviceProxy, host, h.ServicePunishIntervalS)
+				reportUnhealth(clusterID, "punish", serviceProxy, host, "failed")
+			} else {
+				reportUnhealth(clusterID, "repair.msg", serviceProxy, host, "failed")
 			}
 			span.Warnf("send to %s repair message(%+v) %s", host, repairArgs, err.Error())
-			reportUnhealth(clusterID, "punish", serviceProxy, host, "failed")
 			err = errors.Base(err, host)
 		}
 		return err
 	}); err != nil {
-		reportUnhealth(clusterID, "repair.msg", serviceProxy, "-", "failed")
 		span.Errorf("send repair message(%+v) failed %s", repairArgs, errors.Detail(err))
 		return
 	}
@@ -363,7 +476,7 @@ func (h *Handler) sendRepairMsg(ctx context.Context, blob blobIdent, badIdxes []
 	span.Infof("send repair message(%+v)", repairArgs)
 }
 
-func (h *Handler) clearGarbage(ctx context.Context, location *access.Location) error {
+func (h *Handler) clearGarbage(ctx context.Context, location *proto.Location) error {
 	span := trace.SpanFromContextSafe(ctx)
 	serviceController, err := h.clusterController.GetServiceController(location.ClusterID)
 	if err != nil {
@@ -384,28 +497,34 @@ func (h *Handler) clearGarbage(ctx context.Context, location *access.Location) e
 		})
 	}
 
-	var logMsg interface{} = location
+	var logMsg any = location
 	if len(deleteArgs.Blobs) <= 20 {
 		logMsg = deleteArgs
 	}
+	hosts, err := serviceController.GetServiceHosts(ctx, serviceProxy)
+	if err != nil {
+		span.Error(err)
+		return err
+	}
+	for len(hosts) < 3 {
+		hosts = append(hosts, hosts...)
+	}
 	if err := retry.Timed(3, 200).On(func() error {
-		host, err := serviceController.GetServiceHost(ctx, serviceProxy)
-		if err != nil {
-			span.Warn(err)
-			return err
-		}
+		host := hosts[0]
+		hosts = hosts[1:]
 		err = h.proxyClient.SendDeleteMsg(ctx, host, deleteArgs)
 		if err != nil {
 			if errorTimeout(err) || errorConnectionRefused(err) {
 				serviceController.PunishServiceWithThreshold(ctx, serviceProxy, host, h.ServicePunishIntervalS)
+				reportUnhealth(location.ClusterID, "punish", serviceProxy, host, "failed")
+			} else {
+				reportUnhealth(location.ClusterID, "delete.msg", serviceProxy, host, "failed")
 			}
 			span.Warnf("send to %s delete message(%+v) %s", host, logMsg, err.Error())
-			reportUnhealth(location.ClusterID, "punish", serviceProxy, host, "failed")
 			err = errors.Base(err, host)
 		}
 		return err
 	}); err != nil {
-		reportUnhealth(location.ClusterID, "delete.msg", serviceProxy, "-", "failed")
 		span.Errorf("send delete message(%+v) failed %s", logMsg, errors.Detail(err))
 		return errors.Base(err, "send delete message:", logMsg)
 	}
@@ -420,19 +539,25 @@ func (h *Handler) getVolume(ctx context.Context, clusterID proto.ClusterID, vid
 	if err != nil {
 		return nil, err
 	}
-
 	volume := volumeGetter.Get(ctx, vid, isCache)
 	if volume == nil {
 		return nil, errors.Newf("not found volume of (%d %d)", clusterID, vid)
 	}
-
 	return volume, nil
 }
 
+func (h *Handler) updateVolume(ctx context.Context, clusterID proto.ClusterID, vid proto.Vid) {
+	volumeGetter, err := h.clusterController.GetVolumeGetter(clusterID)
+	if err != nil {
+		return
+	}
+	volumeGetter.Update(ctx, vid)
+}
+
 func (h *Handler) punishVolume(ctx context.Context, clusterID proto.ClusterID, vid proto.Vid, host, reason string) {
 	reportUnhealth(clusterID, "punish", "volume", host, reason)
 	if volumeGetter, err := h.clusterController.GetVolumeGetter(clusterID); err == nil {
-		volumeGetter.Punish(ctx, vid, h.DiskPunishIntervalS)
+		volumeGetter.Punish(ctx, vid, h.VolumePunishIntervalS)
 	}
 }
 
@@ -450,16 +575,18 @@ func (h *Handler) punishDiskWith(ctx context.Context, clusterID proto.ClusterID,
 	}
 }
 
-// blobCount blobSize > 0 is certain
-func blobCount(size uint64, blobSize uint32) uint64 {
-	return (size + uint64(blobSize) - 1) / uint64(blobSize)
+func (h *Handler) punishShardnodeDisk(ctx context.Context, clusterID proto.ClusterID, diskID proto.DiskID, host, reason string) {
+	reportUnhealth(clusterID, "punish", "shardnode", host, reason)
+	if serviceController, err := h.clusterController.GetServiceController(clusterID); err == nil {
+		serviceController.PunishShardnode(ctx, diskID, h.ShardnodePunishIntervalS)
+	}
 }
 
-func minU64(a, b uint64) uint64 {
-	if a <= b {
-		return a
+func (h *Handler) punishShardnodeDiskWith(ctx context.Context, clusterID proto.ClusterID, diskID proto.DiskID, host, reason string) {
+	reportUnhealth(clusterID, "punish", "shardnode", host, reason)
+	if serviceController, err := h.clusterController.GetServiceController(clusterID); err == nil {
+		serviceController.PunishShardnodeDiskWithThreshold(ctx, diskID, h.ShardnodePunishIntervalS)
 	}
-	return b
 }
 
 func errorTimeout(err error) bool {

blobstore/access/stream/stream_alloc.go

@@ -12,19 +12,21 @@
 // implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package access
+package stream
 
 import (
 	"context"
 	"sync/atomic"
 
 	"github.com/afex/hystrix-go/hystrix"
+
 	"github.com/cubefs/cubefs/blobstore/api/access"
 	"github.com/cubefs/cubefs/blobstore/api/proxy"
 	"github.com/cubefs/cubefs/blobstore/common/codemode"
 	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
 	"github.com/cubefs/cubefs/blobstore/common/proto"
 	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/util"
 	"github.com/cubefs/cubefs/blobstore/util/errors"
 	"github.com/cubefs/cubefs/blobstore/util/retry"
 )
@@ -39,7 +41,8 @@ var errAllocatePunishedVolume = errors.New("allocate punished volume")
 //	          codeMode > 0, alloc in this codemode
 //	return: a location of file
 func (h *Handler) Alloc(ctx context.Context, size uint64, blobSize uint32,
-	assignClusterID proto.ClusterID, codeMode codemode.CodeMode) (*access.Location, error) {
+	assignClusterID proto.ClusterID, codeMode codemode.CodeMode,
+) (*proto.Location, error) {
 	span := trace.SpanFromContextSafe(ctx)
 	span.Debugf("alloc request with size:%d blobsize:%d cluster:%d codemode:%d",
 		size, blobSize, assignClusterID, codeMode)
@@ -54,7 +57,7 @@ func (h *Handler) Alloc(ctx context.Context, size uint64, blobSize uint32,
 		span.Debugf("fill blobsize:%d", blobSize)
 	}
 
-	if codeMode == 0 {
+	if codeMode == codemode.CodeModeNone {
 		codeMode = h.allCodeModes.SelectCodeMode(int64(size))
 		span.Debugf("select codemode:%d", codeMode)
 	}
@@ -70,19 +73,20 @@ func (h *Handler) Alloc(ctx context.Context, size uint64, blobSize uint32,
 	}
 	span.Debugf("allocated from %d %+v", clusterID, blobs)
 
-	location := &access.Location{
+	location := &proto.Location{
 		ClusterID: clusterID,
 		CodeMode:  codeMode,
-		Size:      size,
-		BlobSize:  blobSize,
-		Blobs:     blobs,
+		Size_:     size,
+		SliceSize: blobSize,
+		Slices:    blobs,
 	}
 	span.Debugf("alloc ok %+v", location)
 	return location, nil
 }
 
-func (h *Handler) allocFromAllocatorWithHystrix(ctx context.Context, codeMode codemode.CodeMode, size uint64, blobSize uint32,
-	clusterID proto.ClusterID) (cid proto.ClusterID, bidRets []access.SliceInfo, err error) {
+func (h *Handler) allocFromAllocatorWithHystrix(ctx context.Context,
+	codeMode codemode.CodeMode, size uint64, blobSize uint32, clusterID proto.ClusterID,
+) (cid proto.ClusterID, bidRets []proto.Slice, err error) {
 	err = hystrix.Do(allocCommand, func() error {
 		cid, bidRets, err = h.allocFromAllocator(ctx, codeMode, size, blobSize, clusterID)
 		return err
@@ -90,8 +94,9 @@ func (h *Handler) allocFromAllocatorWithHystrix(ctx context.Context, codeMode co
 	return
 }
 
-func (h *Handler) allocFromAllocator(ctx context.Context, codeMode codemode.CodeMode, size uint64, blobSize uint32,
-	clusterID proto.ClusterID) (proto.ClusterID, []access.SliceInfo, error) {
+func (h *Handler) allocFromAllocator(ctx context.Context,
+	codeMode codemode.CodeMode, size uint64, blobSize uint32, clusterID proto.ClusterID,
+) (proto.ClusterID, []proto.Slice, error) {
 	span := trace.SpanFromContextSafe(ctx)
 
 	if blobSize == 0 {
@@ -108,32 +113,28 @@ func (h *Handler) allocFromAllocator(ctx context.Context, codeMode codemode.Code
 	args := proxy.AllocVolsArgs{
 		Fsize:    size,
 		CodeMode: codeMode,
-		BidCount: blobCount(size, blobSize),
+		BidCount: util.AlignedBlocks(size, uint64(blobSize)),
+	}
+
+	serviceController, err := h.clusterController.GetServiceController(clusterID)
+	if err != nil {
+		span.Error(err)
+		return 0, nil, err
+	}
+	hosts, err := serviceController.GetServiceHosts(ctx, serviceProxy)
+	if err != nil {
+		span.Error(err)
+		return 0, nil, err
+	}
+	for len(hosts) < h.AllocRetryTimes {
+		hosts = append(hosts, hosts...)
 	}
 
 	var allocRets []proxy.AllocRet
 	var allocHost string
-	hostsSet := make(map[string]struct{}, 1)
-	if err := retry.ExponentialBackoff(h.AllocRetryTimes, uint32(h.AllocRetryIntervalMS)).On(func() error {
-		serviceController, err := h.clusterController.GetServiceController(clusterID)
-		if err != nil {
-			span.Warn(err)
-			return errors.Info(err, "get service controller", clusterID)
-		}
-
-		var host string
-		for range [10]struct{}{} {
-			host, err = serviceController.GetServiceHost(ctx, serviceProxy)
-			if err != nil {
-				span.Warn(err)
-				return errors.Info(err, "get proxy host", clusterID)
-			}
-			if _, ok := hostsSet[host]; ok {
-				continue
-			}
-			hostsSet[host] = struct{}{}
-			break
-		}
+	if err := retry.ExponentialBackoff(h.AllocRetryTimes, uint32(h.AllocRetryIntervalMS)).RuptOn(func() (bool, error) {
+		host := hosts[0]
+		hosts = hosts[1:]
 		allocHost = host
 
 		allocRets, err = h.proxyClient.VolumeAlloc(ctx, host, &args)
@@ -144,7 +145,10 @@ func (h *Handler) allocFromAllocator(ctx context.Context, codeMode codemode.Code
 				serviceController.PunishServiceWithThreshold(ctx, serviceProxy, host, h.ServicePunishIntervalS)
 			}
 			span.Warn(host, err)
-			return errors.Base(err, "alloc from proxy", host)
+			if err == context.Canceled {
+				return true, err
+			}
+			return false, errors.Base(err, "alloc from proxy", host)
 		}
 
 		// filter punished volume in allocating progress
@@ -152,18 +156,18 @@ func (h *Handler) allocFromAllocator(ctx context.Context, codeMode codemode.Code
 			vInfo, err := h.getVolume(ctx, clusterID, ret.Vid, true)
 			if err != nil {
 				span.Warn(err)
-				return err
+				return false, err
 			}
 			if vInfo.IsPunish {
 				// return err and retry allocate
 				err = errAllocatePunishedVolume
 				args.Excludes = append(args.Excludes, vInfo.Vid)
 				span.Warn("next retry exclude vid:", vInfo.Vid, err)
-				return err
+				return false, err
 			}
 		}
 
-		return nil
+		return true, nil
 	}); err != nil {
 		if err != errAllocatePunishedVolume {
 			reportUnhealth(clusterID, "allocate", "-", "-", "failed")
@@ -178,20 +182,20 @@ func (h *Handler) allocFromAllocator(ctx context.Context, codeMode codemode.Code
 		setCacheVidHost(clusterID, ret.Vid, allocHost)
 	}
 
-	blobN := blobCount(size, blobSize)
-	blobs := make([]access.SliceInfo, 0, blobN)
+	blobN := util.AlignedBlocks(size, uint64(blobSize))
+	blobs := make([]proto.Slice, 0, blobN)
 	for _, bidRet := range allocRets {
 		if blobN <= 0 {
 			break
 		}
 
-		count := minU64(blobN, uint64(bidRet.BidEnd)-uint64(bidRet.BidStart)+1)
+		count := util.Min(blobN, uint64(bidRet.BidEnd)-uint64(bidRet.BidStart)+1)
 		blobN -= count
 
-		blobs = append(blobs, access.SliceInfo{
-			MinBid: bidRet.BidStart,
-			Vid:    bidRet.Vid,
-			Count:  uint32(count),
+		blobs = append(blobs, proto.Slice{
+			MinSliceID: bidRet.BidStart,
+			Vid:        bidRet.Vid,
+			Count:      uint32(count),
 		})
 	}
 	if blobN > 0 {

blobstore/access/stream/stream_alloc_test.go

@@ -12,9 +12,10 @@
 // implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package access
+package stream
 
 import (
+	"context"
 	"testing"
 	"time"
 
@@ -32,26 +33,26 @@ func TestAccessStreamAllocBase(t *testing.T) {
 		require.NoError(t, err)
 		require.Equal(t, clusterID, loc.ClusterID)
 		require.Equal(t, codemode.EC6P6, loc.CodeMode)
-		require.Equal(t, uint64(1<<30), loc.Size)
-		require.Equal(t, uint32(1<<22), loc.BlobSize)
-		require.Equal(t, 2, len(loc.Blobs))
-		require.Equal(t, uint32(1), loc.Blobs[0].Count)
-		require.Equal(t, uint32((1<<8)-1), loc.Blobs[1].Count)
+		require.Equal(t, uint64(1<<30), loc.Size_)
+		require.Equal(t, uint32(1<<22), loc.SliceSize)
+		require.Equal(t, 2, len(loc.Slices))
+		require.Equal(t, uint32(1), loc.Slices[0].Count)
+		require.Equal(t, uint32((1<<8)-1), loc.Slices[1].Count)
 	}
 	{
 		loc, err := streamer.Alloc(ctx(), (1<<30)+1, 0, 0, 0)
 		require.NoError(t, err)
-		require.Equal(t, 2, len(loc.Blobs))
-		require.Equal(t, uint32(1), loc.Blobs[0].Count)
-		require.Equal(t, uint32(1<<8), loc.Blobs[1].Count)
+		require.Equal(t, 2, len(loc.Slices))
+		require.Equal(t, uint32(1), loc.Slices[0].Count)
+		require.Equal(t, uint32(1<<8), loc.Slices[1].Count)
 	}
 	// 1M blobsize
 	{
 		loc, err := streamer.Alloc(ctx(), 1<<30, 1<<20, 0, 0)
 		require.NoError(t, err)
-		require.Equal(t, 2, len(loc.Blobs))
-		require.Equal(t, uint32(1), loc.Blobs[0].Count)
-		require.Equal(t, uint32((1<<10)-1), loc.Blobs[1].Count)
+		require.Equal(t, 2, len(loc.Slices))
+		require.Equal(t, uint32(1), loc.Slices[0].Count)
+		require.Equal(t, uint32((1<<10)-1), loc.Slices[1].Count)
 	}
 	// max size + 1
 	{
@@ -68,3 +69,11 @@ func TestAccessStreamAllocBase(t *testing.T) {
 		require.Error(t, err)
 	}
 }
+
+func TestAccessStreamAllocCanceled(t *testing.T) {
+	ctxfunc := ctxWithName("TestAccessStreamAllocCanceled")
+	ctx, cancel := context.WithCancel(ctxfunc())
+	cancel()
+	_, err := streamer.Alloc(ctx, 1, 0, 0, 0)
+	require.ErrorIs(t, err, context.Canceled)
+}

blobstore/access/stream/stream_blob.go

@@ -0,0 +1,784 @@
+package stream
+
+import (
+	"context"
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	"github.com/cubefs/cubefs/blobstore/access/controller"
+	acapi "github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/api/shardnode"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/rpc"
+	"github.com/cubefs/cubefs/blobstore/common/sharding"
+	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/util/errors"
+	"github.com/cubefs/cubefs/blobstore/util/retry"
+)
+
+func (h *Handler) GetBlob(ctx context.Context, args *acapi.GetBlobArgs) (*proto.Location, error) {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Debugf("get blob args:%+v", *args)
+
+	var blob shardnode.GetBlobRet
+	rerr := retry.ExponentialBackoff(3, 200).RuptOn(func() (bool, error) {
+		header, err := h.getShardOpHeader(ctx, acapi.GetShardCommonArgs{
+			ClusterID: args.ClusterID,
+			BlobName:  args.BlobName,
+			Mode:      args.Mode,
+		})
+		if err != nil {
+			return true, err // not retry
+		}
+
+		host, err := h.getShardHost(ctx, args.ClusterID, header.DiskID)
+		if err != nil {
+			return true, err
+		}
+
+		blob, err = h.shardnodeClient.GetBlob(ctx, host, shardnode.GetBlobArgs{
+			Header: header,
+			Name:   args.BlobName,
+		})
+		if err != nil {
+			return h.punishAndUpdate(ctx, &punishArgs{
+				ShardOpHeader: header,
+				clusterID:     args.ClusterID,
+				host:          host,
+				mode:          args.Mode,
+				err:           err,
+			})
+		}
+
+		return true, nil
+	})
+
+	if rerr != nil {
+		span.Errorf("get blob failed, args:%+v, err:%+v", *args, rerr)
+	}
+	return &blob.Blob.Location, rerr
+}
+
+func (h *Handler) CreateBlob(ctx context.Context, args *acapi.CreateBlobArgs) (*proto.Location, error) {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Debugf("create blob args:%+v", *args)
+
+	err := h.fixCreateBlobArgs(ctx, args)
+	if err != nil {
+		return nil, err
+	}
+
+	var blob shardnode.CreateBlobRet
+	rerr := retry.ExponentialBackoff(3, 200).RuptOn(func() (bool, error) {
+		header, err := h.getShardOpHeader(ctx, acapi.GetShardCommonArgs{
+			ClusterID: args.ClusterID,
+			BlobName:  args.BlobName,
+			Mode:      acapi.GetShardModeLeader,
+		})
+		if err != nil {
+			return true, err
+		}
+
+		host, err := h.getShardHost(ctx, args.ClusterID, header.DiskID)
+		if err != nil {
+			return true, err
+		}
+
+		blob, err = h.shardnodeClient.CreateBlob(ctx, host, shardnode.CreateBlobArgs{
+			Header:    header,
+			Name:      args.BlobName,
+			CodeMode:  args.CodeMode,
+			Size_:     args.Size,
+			SliceSize: args.SliceSize,
+		})
+		if err != nil {
+			return h.punishAndUpdate(ctx, &punishArgs{
+				ShardOpHeader: header,
+				clusterID:     args.ClusterID,
+				host:          host,
+				mode:          acapi.GetShardModeLeader,
+				err:           err,
+			})
+		}
+		return true, nil
+	})
+
+	if rerr != nil {
+		span.Errorf("create blob failed, args:%+v, err:%+v", *args, rerr)
+	}
+	return &blob.Blob.Location, rerr
+}
+
+func (h *Handler) DeleteBlob(ctx context.Context, args *acapi.DelBlobArgs) error {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Debugf("delete blob args:%+v", *args)
+
+	rerr := retry.ExponentialBackoff(3, 200).RuptOn(func() (bool, error) {
+		header, err := h.getShardOpHeader(ctx, acapi.GetShardCommonArgs{
+			ClusterID: args.ClusterID,
+			BlobName:  args.BlobName,
+			Mode:      acapi.GetShardModeLeader,
+		})
+		if err != nil {
+			return true, err
+		}
+
+		host, err := h.getShardHost(ctx, args.ClusterID, header.DiskID)
+		if err != nil {
+			return true, err
+		}
+
+		if err = h.shardnodeClient.DeleteBlob(ctx, host, shardnode.DeleteBlobArgs{
+			Header: header,
+			Name:   args.BlobName,
+		}); err != nil {
+			return h.punishAndUpdate(ctx, &punishArgs{
+				ShardOpHeader: header,
+				clusterID:     args.ClusterID,
+				host:          host,
+				mode:          acapi.GetShardModeLeader,
+				err:           err,
+			})
+		}
+
+		return true, nil
+	})
+
+	if rerr != nil {
+		span.Errorf("delete blob failed, args:%+v, err:%+v", *args, rerr)
+	}
+	return rerr
+}
+
+func (h *Handler) SealBlob(ctx context.Context, args *acapi.SealBlobArgs) error {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Debugf("seal blob args:%+v", *args)
+
+	rerr := retry.ExponentialBackoff(3, 200).RuptOn(func() (bool, error) {
+		header, err := h.getShardOpHeader(ctx, acapi.GetShardCommonArgs{
+			ClusterID: args.ClusterID,
+			BlobName:  args.BlobName,
+			Mode:      acapi.GetShardModeLeader,
+		})
+		if err != nil {
+			return true, err
+		}
+
+		host, err := h.getShardHost(ctx, args.ClusterID, header.DiskID)
+		if err != nil {
+			return true, err
+		}
+
+		err = h.shardnodeClient.SealBlob(ctx, host, shardnode.SealBlobArgs{
+			Header: header,
+			Name:   args.BlobName,
+			Size_:  args.Size,
+			Slices: args.Slices,
+		})
+		if err != nil {
+			return h.punishAndUpdate(ctx, &punishArgs{
+				ShardOpHeader: header,
+				clusterID:     args.ClusterID,
+				host:          host,
+				mode:          acapi.GetShardModeLeader,
+				err:           err,
+			})
+		}
+		return true, nil
+	})
+
+	if rerr != nil {
+		span.Errorf("seal blob failed, args:%+v, err:%+v", *args, rerr)
+	}
+	return rerr
+}
+
+func (h *Handler) ListBlob(ctx context.Context, args *acapi.ListBlobArgs) (ret shardnode.ListBlobRet, err error) {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Debugf("list blob args:%+v", *args)
+	defer func() {
+		if err != nil {
+			span.Errorf("list blob failed, args:%+v, err:%+v", *args, err)
+		}
+	}()
+
+	if args.ShardID != 0 {
+		return h.listSpecificShard(ctx, args)
+	}
+
+	return h.listManyShards(ctx, args)
+}
+
+func (h *Handler) AllocSlice(ctx context.Context, args *acapi.AllocSliceArgs) (shardnode.AllocSliceRet, error) {
+	span := trace.SpanFromContextSafe(ctx)
+	span.Debugf("alloc blob args:%+v", *args)
+
+	var slices shardnode.AllocSliceRet
+	rerr := retry.ExponentialBackoff(3, 200).RuptOn(func() (bool, error) {
+		header, err := h.getShardOpHeader(ctx, acapi.GetShardCommonArgs{
+			ClusterID: args.ClusterID,
+			BlobName:  args.BlobName,
+			Mode:      acapi.GetShardModeLeader,
+		})
+		if err != nil {
+			return true, err
+		}
+
+		host, err := h.getShardHost(ctx, args.ClusterID, header.DiskID)
+		if err != nil {
+			return true, err
+		}
+
+		slices, err = h.shardnodeClient.AllocSlice(ctx, host, shardnode.AllocSliceArgs{
+			Header:      header,
+			Name:        args.BlobName,
+			CodeMode:    args.CodeMode,
+			Size_:       args.Size,
+			FailedSlice: args.FailSlice,
+		})
+		if err != nil {
+			return h.punishAndUpdate(ctx, &punishArgs{
+				ShardOpHeader: header,
+				clusterID:     args.ClusterID,
+				host:          host,
+				mode:          acapi.GetShardModeLeader,
+				err:           err,
+			})
+		}
+		return true, nil
+	})
+
+	if rerr != nil {
+		span.Errorf("alloc slice failed, args:%+v, err:%+v", *args, rerr)
+	}
+	return slices, rerr
+}
+
+func (h *Handler) listSpecificShard(ctx context.Context, args *acapi.ListBlobArgs) (shardnode.ListBlobRet, error) {
+	span := trace.SpanFromContextSafe(ctx)
+	var ret shardnode.ListBlobRet
+	rerr := retry.ExponentialBackoff(3, 200).RuptOn(func() (bool, error) {
+		header, err := h.getOpHeaderByID(ctx, args.ClusterID, args.ShardID, args.Mode)
+		if err != nil {
+			return true, err
+		}
+
+		interrupt := false
+		ret, interrupt, err = h.listSingleShardEnough(ctx, args, header)
+		span.Debugf("list blob, shardID=%d, interrupt:%t, length:%d, err:%+v", args.ShardID, interrupt, len(ret.Blobs), err)
+
+		if err != nil {
+			return interrupt, err
+		}
+		return true, nil
+	})
+	return ret, rerr
+}
+
+func (h *Handler) listManyShards(ctx context.Context, args *acapi.ListBlobArgs) (shardnode.ListBlobRet, error) {
+	span := trace.SpanFromContextSafe(ctx)
+	shardMgr, err := h.clusterController.GetShardController(args.ClusterID)
+	if err != nil {
+		return shardnode.ListBlobRet{}, err
+	}
+
+	var (
+		shard   controller.Shard
+		allBlob shardnode.ListBlobRet
+	)
+	if len(args.Marker) == 0 {
+		shard, err = shardMgr.GetFisrtShard(ctx)
+		if err != nil {
+			return shardnode.ListBlobRet{}, err
+		}
+	} else {
+		unionMarker := acapi.ListBlobEncodeMarker{}
+		if err = unionMarker.UnmarshalFromString(args.Marker); err != nil {
+			return shardnode.ListBlobRet{}, fmt.Errorf("fail to unmarshal marker, err: %+v", err)
+		}
+		allBlob.NextMarker = unionMarker.Marker
+		shard, err = shardMgr.GetShardByRange(ctx, unionMarker.Range)
+		if err != nil {
+			return shardnode.ListBlobRet{}, err
+		}
+		span.Debugf("list blob at multi shards, prefix=%s, range=%s, marker=%s", args.Prefix, unionMarker.Range.String(), unionMarker.Marker)
+	}
+
+	lastRange := shard.GetRange()
+	count := int64(args.Count)
+	for count > 0 {
+		var ret shardnode.ListBlobRet
+		interrupt := false
+		rerr := retry.ExponentialBackoff(3, 200).RuptOn(func() (bool, error) {
+			header, err := h.getOpHeaderByShard(ctx, shardMgr, shard, args.Mode)
+			if err != nil {
+				return interrupt, err
+			}
+			args.Marker = allBlob.NextMarker
+			args.Count = uint64(count)
+			ret, interrupt, err = h.listSingleShardEnough(ctx, args, header)
+			span.Debugf("list blob, shardID=%d, interrupt:%t, length:%d, err:%+v", args.ShardID, interrupt, len(ret.Blobs), err)
+
+			if err != nil {
+				return interrupt, err
+			}
+			return true, nil
+		})
+		if rerr != nil {
+			return shardnode.ListBlobRet{}, rerr
+		}
+
+		allBlob.Blobs = append(allBlob.Blobs, ret.Blobs...)
+		allBlob.NextMarker = ret.NextMarker
+		count -= int64(len(ret.Blobs))
+		if ret.NextMarker == "" {
+			shard, err = shardMgr.GetNextShard(ctx, lastRange)
+			if err != nil {
+				return shardnode.ListBlobRet{}, err
+			}
+			// err == nil && shard == nil, means last shard, reach end
+			if shard == nil {
+				lastRange = sharding.Range{}
+				break // reach end
+			}
+			lastRange = shard.GetRange()
+		}
+	}
+
+	// reach end, don't need marshal
+	if len(allBlob.NextMarker) == 0 && lastRange.Type == 0 {
+		return allBlob, nil
+	}
+
+	markers := acapi.ListBlobEncodeMarker{
+		Range:  lastRange,          // empty, means reach the end; else, means next expect shard
+		Marker: allBlob.NextMarker, // empty, means current shard list end; else, means expect begin blob name
+	}
+	unionMarker, err := markers.MarshalToString()
+	allBlob.NextMarker = unionMarker
+	return allBlob, err
+}
+
+func (h *Handler) listSingleShardEnough(ctx context.Context, args *acapi.ListBlobArgs, header shardnode.ShardOpHeader) (shardnode.ListBlobRet, bool, error) {
+	host, err := h.getShardHost(ctx, args.ClusterID, header.DiskID)
+	if err != nil {
+		return shardnode.ListBlobRet{}, true, err
+	}
+
+	ret, err := h.shardnodeClient.ListBlob(ctx, host, shardnode.ListBlobArgs{
+		Header: header,
+		Prefix: args.Prefix,
+		Marker: args.Marker,
+		Count:  args.Count,
+	})
+	if err != nil {
+		interrupt, err1 := h.punishAndUpdate(ctx, &punishArgs{
+			ShardOpHeader: header,
+			clusterID:     args.ClusterID,
+			host:          host,
+			mode:          args.Mode,
+			err:           err,
+		})
+		return shardnode.ListBlobRet{}, interrupt, err1
+	}
+
+	return ret, true, nil
+}
+
+func (h *Handler) getShardOpHeader(ctx context.Context, args acapi.GetShardCommonArgs) (shardnode.ShardOpHeader, error) {
+	shardMgr, err := h.clusterController.GetShardController(args.ClusterID)
+	if err != nil {
+		return shardnode.ShardOpHeader{}, err
+	}
+
+	shardKeys := shardnode.DecodeShardKeys(args.BlobName, shardMgr.GetShardSubRangeCount(ctx))
+	shard, err := shardMgr.GetShard(ctx, shardKeys)
+	if err != nil {
+		return shardnode.ShardOpHeader{}, err
+	}
+
+	oh, err := h.getOpHeaderByShard(ctx, shardMgr, shard, args.Mode)
+	return oh, err
+}
+
+func (h *Handler) getOpHeaderByID(ctx context.Context, clusterID proto.ClusterID, shardID proto.ShardID, mode acapi.GetShardMode) (shardnode.ShardOpHeader, error) {
+	shardMgr, err := h.clusterController.GetShardController(clusterID)
+	if err != nil {
+		return shardnode.ShardOpHeader{}, err
+	}
+
+	shard, err := shardMgr.GetShardByID(ctx, shardID)
+	if err != nil {
+		return shardnode.ShardOpHeader{}, err
+	}
+
+	return h.getOpHeaderByShard(ctx, shardMgr, shard, mode)
+}
+
+func (h *Handler) getOpHeaderByShard(ctx context.Context, shardMgr controller.IShardController, shard controller.Shard,
+	mode acapi.GetShardMode,
+) (shardnode.ShardOpHeader, error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	spaceID := shardMgr.GetSpaceID()
+	info, err := shard.GetMember(ctx, mode, nil)
+	if err != nil {
+		return shardnode.ShardOpHeader{}, err
+	}
+
+	oh := shardnode.ShardOpHeader{
+		SpaceID:      spaceID,
+		DiskID:       info.DiskID,
+		Suid:         info.Suid,
+		RouteVersion: info.RouteVersion,
+	}
+
+	span.Debugf("shard op header: %+v", oh)
+	return oh, nil
+}
+
+func (h *Handler) getShardHost(ctx context.Context, clusterID proto.ClusterID, diskID proto.DiskID) (string, error) {
+	span := trace.SpanFromContextSafe(ctx)
+	s, err := h.clusterController.GetServiceController(clusterID)
+	if err != nil {
+		return "", err
+	}
+
+	hostInfo, err := s.GetShardnodeHost(ctx, diskID)
+	if err != nil {
+		return "", err
+	}
+
+	span.Debugf("get shard host:%+v", *hostInfo)
+	return hostInfo.Host, nil
+}
+
+type punishArgs struct {
+	shardnode.ShardOpHeader
+	clusterID proto.ClusterID
+	host      string
+	mode      acapi.GetShardMode
+	exclude   map[proto.DiskID]struct{}
+	err       error
+}
+
+func (h *Handler) punishAndUpdate(ctx context.Context, args *punishArgs) (bool, error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	// This error is coming from the shardnode interface, and we want to make sure that the error can be parsed into an error code
+	code := rpc.DetectStatusCode(args.err)
+
+	// leader disk status: normal->EIO->broken->repairing->repaired. it greater than broken will mark punished
+	// if bad disk(punished), we select another disk as leader; old leader is not in disk units, after update route(replace suid index unit)
+	// and then call sn return NoLeader, and we fetch and update new leader shard
+	//    1. old leader: repaired(eio, or status >= broken), sn will remove disk and return DiskNotFound, update route
+	//    2. old leader: broken(eio, broken, repairing), sn return DiskBroken
+	// cm catalog units is always correct, but its leaderDiskID may be wrong
+	switch code {
+	case errcode.CodeDiskBroken: // read shard at bad disk, but shard/disk is reparing
+		// if follow node broken disk, it will not election, just try again, change other shard;
+		// if leader node broken disk, it cant get shard stats, wait new leader
+		h.punishShardnodeDisk(ctx, args.clusterID, args.DiskID, args.host, "Broken")
+		if args.mode == acapi.GetShardModeLeader {
+			err1 := h.updateLeaderFromNewHost(ctx, args)
+			if err1 != nil {
+				span.Warnf("fail to change other shard node, cluster:%d, err:%+v", args.clusterID, err1)
+			}
+		}
+		return false, args.err
+
+		// update route and punish
+	case errcode.CodeShardNodeDiskNotFound: // read shard at bad disk, but old broken disk is repaired, all shard repaired
+		h.punishShardnodeDisk(ctx, args.clusterID, args.DiskID, args.host, "NotFound")
+		if err1 := h.updateShardRoute(ctx, args.clusterID); err1 != nil {
+			span.Warnf("fail to update shard route, cluster:%d, err:%+v", args.clusterID, err1)
+		}
+		return false, args.err
+
+		// update route
+	case errcode.CodeShardDoesNotExist, // intermediate state disk, not a final state; shard is removed, disk is repairing/repaired ; suid not match disk id
+		errcode.CodeShardRouteVersionNeedUpdate: // header op version less than shardnode version
+		if err1 := h.updateShardRoute(ctx, args.clusterID); err1 != nil {
+			span.Warnf("fail to update shard route, cluster:%d, err:%+v", args.clusterID, err1)
+		}
+		return false, args.err
+
+		// select master
+	case errcode.CodeShardNodeNotLeader: // leader disk id error when create/delete/seal
+		if err1 := h.updateLeaderFromNewHost(ctx, args); err1 != nil {
+			span.Warnf("fail to update leader and shard info, cluster:%d, err:%+v", args.clusterID, err1)
+		}
+		return false, args.err
+
+	default:
+	}
+
+	// err:dial tcp 127.0.0.1:9100: connect: connection refused  ； code:500
+	if errorConnectionRefused(args.err) {
+		span.Warnf("shardnode connection refused/timeout, args:%+v, err:%+v", *args, args.err)
+		h.groupRun.Do("shardnode-leader-"+args.DiskID.ToString(), func() (interface{}, error) {
+			// must wait have master leader, block wait
+			h.punishShardnodeDisk(ctx, args.clusterID, args.DiskID, args.host, "Refused")
+			err1 := h.updateLeaderFromNewHost(ctx, args)
+			if err1 != nil {
+				span.Warnf("fail to change other shard node, cluster:%d, err:%+v", args.clusterID, err1)
+			}
+			return nil, err1
+		})
+		return false, errcode.ErrConnectionRefused
+	}
+
+	if errorTimeout(args.err) {
+		h.punishShardnodeDiskWith(ctx, args.clusterID, args.DiskID, args.host, "Timeout")
+		return false, args.err
+	}
+
+	// eio or other error; if shardNode restarts quickly so wait for it to start, and try again
+	return false, args.err
+}
+
+func (h *Handler) updateShardRoute(ctx context.Context, clusterID proto.ClusterID) error {
+	shardMgr, err := h.clusterController.GetShardController(clusterID)
+	if err != nil {
+		return err
+	}
+
+	return shardMgr.UpdateRoute(ctx)
+}
+
+// updateLeaderFromCurrentHost from old current shard host/disk, get leader and update shard
+func (h *Handler) updateLeaderFromCurrentHost(ctx context.Context, args *punishArgs) error {
+	shardMgr, err := h.clusterController.GetShardController(args.clusterID)
+	if err != nil {
+		return err
+	}
+
+	shardStat, err := h.getLeaderShardInfo(ctx, args.clusterID, args.host, args.DiskID, args.Suid, 0)
+	if err != nil {
+		return err
+	}
+
+	return shardMgr.UpdateShard(ctx, shardStat)
+}
+
+// updateLeaderFromNewHost from other shard host/disk, get leader and update shard
+func (h *Handler) updateLeaderFromNewHost(ctx context.Context, args *punishArgs) error {
+	shardMgr, err := h.clusterController.GetShardController(args.clusterID)
+	if err != nil {
+		return err
+	}
+	shard, err := shardMgr.GetShardByID(ctx, args.Suid.ShardID())
+	if err != nil {
+		return err
+	}
+
+	if args.exclude == nil {
+		args.exclude = make(map[proto.DiskID]struct{})
+		args.exclude[args.DiskID] = struct{}{}
+	}
+
+	// we get new disk, exclude bad diskID
+	newDisk, err := shard.GetMember(ctx, acapi.GetShardModeRandom, args.exclude)
+	if err != nil {
+		return err
+	}
+	newHost, err := h.getShardHost(ctx, args.clusterID, newDisk.DiskID)
+	if err != nil {
+		return err
+	}
+	// span := trace.SpanFromContextSafe(ctx)
+	// span.Debugf("get newDisk:%+v, old host:%s, old disk:%d", newDisk, args.host, args.DiskID)
+
+	shardStat, err := h.getLeaderShardInfo(ctx, args.clusterID, newHost, newDisk.DiskID, newDisk.Suid, args.DiskID)
+	if err != nil {
+		args.exclude[newDisk.DiskID] = struct{}{}
+		return err
+	}
+
+	return shardMgr.UpdateShard(ctx, shardStat)
+}
+
+func (h *Handler) getLeaderShardInfo(ctx context.Context, clusterID proto.ClusterID, host string, diskID proto.DiskID, suid proto.Suid, badDisk proto.DiskID) (shardnode.ShardStats, error) {
+	span := trace.SpanFromContextSafe(ctx)
+
+	for i := 0; i < h.ShardnodeRetryTimes; i++ {
+		// 1. get leader info
+		leader, err := h.shardnodeClient.GetShardStats(ctx, host, shardnode.GetShardArgs{
+			DiskID: diskID,
+			Suid:   suid,
+		})
+		if err != nil {
+			if code := rpc.DetectStatusCode(err); code == errcode.CodeShardNoLeader {
+				span.Warnf("shard node is in the election, host:%s, disk:%d, suid:%d, badDisk:%d", host, diskID, suid, badDisk)
+				time.Sleep(time.Millisecond * time.Duration(h.ShardnodeRetryIntervalMS))
+				continue
+			}
+			return shardnode.ShardStats{}, err
+		}
+
+		// skip bad host. LeaderDiskID is 0 means in the election. bad disk is last leader, not start election yet
+		if leader.LeaderDiskID == 0 || leader.LeaderDiskID == badDisk {
+			span.Warnf("shard node is in the election, host:%s, disk:%d, suid:%d, badDisk:%d", host, diskID, suid, badDisk)
+			time.Sleep(time.Millisecond * time.Duration(h.ShardnodeRetryIntervalMS))
+			continue
+		}
+
+		return leader, nil
+	}
+
+	return shardnode.ShardStats{}, errcode.ErrShardNoLeader
+}
+
+func (h *Handler) fixCreateBlobArgs(ctx context.Context, args *acapi.CreateBlobArgs) error {
+	span := trace.SpanFromContextSafe(ctx)
+	if int64(args.Size) > h.maxObjectSize {
+		span.Info("exceed max object size", h.maxObjectSize)
+		return errcode.ErrAccessExceedSize
+	}
+
+	if args.SliceSize == 0 {
+		args.SliceSize = atomic.LoadUint32(&h.MaxBlobSize)
+		span.Debugf("fill slice size:%d", args.SliceSize)
+	}
+
+	if args.CodeMode == 0 {
+		args.CodeMode = h.allCodeModes.SelectCodeMode(int64(args.Size))
+		span.Debugf("select codemode:%d", args.CodeMode)
+	}
+
+	if !args.CodeMode.IsValid() {
+		span.Infof("invalid codemode:%d", args.CodeMode)
+		return errcode.ErrIllegalArguments
+	}
+
+	if args.ClusterID == 0 {
+		cluster, err := h.clusterController.ChooseOne()
+		if err != nil {
+			return err
+		}
+		args.ClusterID = cluster.ClusterID
+		span.Debugf("choose cluster[%+v]", cluster)
+	}
+
+	return nil
+}
+
+func (h *Handler) getRepairMessageShardnode(ctx context.Context,
+	shardController controller.IShardController, clusterID proto.ClusterID, blob blobIdent, badIdxes []uint8,
+) (args shardnode.RepairSliceArgs, host string, err error) {
+	args.Vid = blob.vid
+	args.Bid = blob.bid
+	args.Reason = "access-repair"
+	for _, idx := range badIdxes {
+		args.BadIdx = append(args.BadIdx, uint32(idx))
+	}
+	tagNum := shardController.GetShardSubRangeCount(ctx)
+	var shard controller.Shard
+	shard, err = shardController.GetShard(ctx, args.GetShardKeys(tagNum))
+	if err != nil {
+		return
+	}
+	args.Header, err = h.getOpHeaderByShard(ctx, shardController, shard, acapi.GetShardModeLeader)
+	if err != nil {
+		return
+	}
+	host, err = h.getShardHost(ctx, clusterID, args.Header.DiskID)
+	return
+}
+
+func (h *Handler) sendRepairMsgIntoShardnode(ctx context.Context, blob blobIdent, badIdxes []uint8) {
+	span := trace.SpanFromContextSafe(ctx)
+	clusterID := blob.cid
+	shardController, err := h.clusterController.GetShardController(clusterID)
+	if err != nil {
+		span.Error(errors.Detail(err))
+		return
+	}
+
+	if err := retry.Timed(3, 100).On(func() error {
+		args, host, err := h.getRepairMessageShardnode(ctx, shardController, clusterID, blob, badIdxes)
+		if err != nil {
+			reportUnhealth(clusterID, "repair.msg", serviceShard, "-", "failed")
+			span.Warn(err)
+			return err
+		}
+		if err = h.shardnodeClient.RepairSlice(ctx, host, args); err != nil {
+			span.Warnf("send to shardnode %s repair message(%+v) %s", host, args, err.Error())
+			reportUnhealth(clusterID, "repair.msg", serviceShard, host, "failed")
+			_, err = h.punishAndUpdate(ctx, &punishArgs{
+				ShardOpHeader: args.Header,
+				clusterID:     clusterID,
+				host:          host,
+				mode:          acapi.GetShardModeLeader,
+				err:           err,
+			})
+			err = errors.Base(err, host)
+		}
+		return err
+	}); err != nil {
+		span.Errorf("send shardnode repair message(%+v) failed %s", blob, errors.Detail(err))
+		return
+	}
+
+	span.Infof("send shardnode repair message(%+v)", blob)
+}
+
+func (h *Handler) getDeleteMessageShardnode(ctx context.Context,
+	shardController controller.IShardController, clusterID proto.ClusterID, slice proto.Slice,
+) (args shardnode.DeleteBlobRawArgs, host string, err error) {
+	args.Slice = slice
+	tagNum := shardController.GetShardSubRangeCount(ctx)
+	var shard controller.Shard
+	shard, err = shardController.GetShard(ctx, args.GetShardKeys(tagNum))
+	if err != nil {
+		return
+	}
+	args.Header, err = h.getOpHeaderByShard(ctx, shardController, shard, acapi.GetShardModeLeader)
+	if err != nil {
+		return
+	}
+	host, err = h.getShardHost(ctx, clusterID, args.Header.DiskID)
+	return
+}
+
+func (h *Handler) clearGarbageIntoShardnode(ctx context.Context, location *proto.Location) error {
+	span := trace.SpanFromContextSafe(ctx)
+	shardController, err := h.clusterController.GetShardController(location.ClusterID)
+	if err != nil {
+		span.Error(errors.Detail(err))
+		return errors.Base(err, "clear location:", *location)
+	}
+
+	clusterID := location.ClusterID
+	for _, slice := range location.Slices {
+		if err := retry.Timed(3, 100).On(func() error {
+			args, host, err := h.getDeleteMessageShardnode(ctx, shardController, clusterID, slice)
+			if err != nil {
+				reportUnhealth(clusterID, "delete.msg", serviceShard, "-", "failed")
+				span.Warn(err)
+				return err
+			}
+			if err = h.shardnodeClient.DeleteBlobRaw(ctx, host, args); err != nil {
+				span.Warnf("send to shardnode %s delete message(%+v) %s", host, slice, err.Error())
+				reportUnhealth(clusterID, "delete.msg", serviceShard, host, "failed")
+				_, err = h.punishAndUpdate(ctx, &punishArgs{
+					ShardOpHeader: args.Header,
+					clusterID:     clusterID,
+					host:          host,
+					mode:          acapi.GetShardModeLeader,
+					err:           err,
+				})
+				err = errors.Base(err, host)
+			}
+			return err
+		}); err != nil {
+			span.Errorf("send shardnode delete message(%+v) failed %s", slice, errors.Detail(err))
+			return errors.Base(err, "send shardnode delete message:", slice)
+		}
+	}
+
+	span.Infof("send shardnode delete message(%+v)", location)
+	return nil
+}

blobstore/access/stream/stream_blob_test.go

@@ -0,0 +1,568 @@
+package stream
+
+import (
+	"context"
+	"errors"
+	"io"
+	"math"
+	"strings"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/require"
+
+	"github.com/cubefs/cubefs/blobstore/access/controller"
+	acapi "github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
+	"github.com/cubefs/cubefs/blobstore/api/shardnode"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+	"github.com/cubefs/cubefs/blobstore/common/sharding"
+	"github.com/cubefs/cubefs/blobstore/testing/mocks"
+)
+
+func newStreamHandlerSuccess(t *testing.T) *Handler {
+	ctr := gomock.NewController(t)
+	gAny := gomock.Any()
+
+	info := controller.ShardOpInfo{
+		DiskID:       101,
+		Suid:         proto.EncodeSuid(1, 0, 1),
+		RouteVersion: 1,
+	}
+
+	shardInfo := NewMockShard(ctr)
+	shardInfo.EXPECT().GetMember(gAny, gAny, gAny).Return(info, nil).AnyTimes()
+
+	shardMgr := NewMockShardController(ctr)
+	shardMgr.EXPECT().GetShard(gAny, gAny).Return(shardInfo, nil).AnyTimes()
+	shardMgr.EXPECT().GetSpaceID().Return(proto.SpaceID(1)).AnyTimes()
+	shardMgr.EXPECT().UpdateRoute(gAny).Return(nil).AnyTimes()
+	shardMgr.EXPECT().GetShardSubRangeCount(gAny).Return(2).AnyTimes()
+
+	svrCtrl := NewMockServiceController(ctr)
+	svrCtrl.EXPECT().GetShardnodeHost(gAny, gAny).Return(&controller.HostIDC{Host: "host"}, nil).AnyTimes()
+
+	clu := NewMockClusterController(ctr)
+	clu.EXPECT().GetShardController(gAny).Return(shardMgr, nil).AnyTimes()
+	clu.EXPECT().GetServiceController(gAny).Return(svrCtrl, nil).AnyTimes()
+
+	shardCli := mocks.NewMockShardnodeAccess(ctr)
+	proxyClient := mocks.NewMockProxyClient(ctr)
+
+	allCodeModes := CodeModePairs{
+		codemode.EC3P3: CodeModePair{
+			Policy: codemode.Policy{
+				ModeName: codemode.EC3P3.Name(),
+				MaxSize:  math.MaxInt64,
+				Enable:   true,
+			},
+			Tactic: codemode.EC3P3.Tactic(),
+		},
+	}
+	handler := &Handler{
+		clusterController: clu,
+		shardnodeClient:   shardCli,
+		proxyClient:       proxyClient,
+
+		maxObjectSize: 100,
+		allCodeModes:  allCodeModes,
+	}
+
+	return handler
+}
+
+func TestStreamBlobGet(t *testing.T) {
+	ctx := context.Background()
+	ctr := gomock.NewController(t)
+	gAny := gomock.Any()
+
+	info := controller.ShardOpInfo{}
+
+	shardInfo := NewMockShard(ctr)
+	shardInfo.EXPECT().GetMember(gAny, gAny, gAny).Return(info, nil).Times(2)
+
+	shardMgr := NewMockShardController(ctr)
+	shardMgr.EXPECT().GetShard(gAny, gAny).Return(shardInfo, nil).Times(2)
+	shardMgr.EXPECT().GetSpaceID().Return(proto.SpaceID(1)).Times(2)
+	shardMgr.EXPECT().UpdateRoute(gAny).Return(nil)
+	shardMgr.EXPECT().GetShardSubRangeCount(gAny).Return(2).AnyTimes()
+
+	svrCtrl := NewMockServiceController(ctr)
+	svrCtrl.EXPECT().GetShardnodeHost(gAny, gAny).Return(&controller.HostIDC{Host: "host"}, nil).Times(2)
+
+	clu := NewMockClusterController(ctr)
+	clu.EXPECT().GetShardController(gAny).Return(shardMgr, nil).Times(3)
+	clu.EXPECT().GetServiceController(gAny).Return(svrCtrl, nil).Times(2)
+
+	blobName := "blob1"
+	blob := &proto.Blob{
+		Name: blobName,
+		Location: proto.Location{
+			ClusterID: 1,
+			CodeMode:  codemode.EC3P3,
+			Size_:     1,
+			SliceSize: 1,
+			Crc:       1,
+			Slices:    nil,
+		},
+	}
+	ret := shardnode.GetBlobRet{
+		Blob: *blob,
+	}
+	shardCli := mocks.NewMockShardnodeAccess(ctr)
+	shardCli.EXPECT().GetBlob(gAny, gAny, gAny).Return(ret, errcode.ErrShardRouteVersionNeedUpdate)
+	shardCli.EXPECT().GetBlob(gAny, gAny, gAny).Return(ret, nil)
+
+	handler := &Handler{
+		clusterController: clu,
+		shardnodeClient:   shardCli,
+	}
+	args := acapi.GetBlobArgs{
+		BlobName: string(blobName),
+		Mode:     acapi.GetShardModeRandom,
+	}
+
+	loc, err := handler.GetBlob(ctx, &args)
+	require.NoError(t, err)
+	require.Equal(t, ret.Blob.Location, *loc)
+}
+
+func TestStreamBlobCreate(t *testing.T) {
+	ctx := context.Background()
+	gAny := gomock.Any()
+	h := newStreamHandlerSuccess(t)
+
+	args := acapi.CreateBlobArgs{
+		BlobName:  ("blob-create"),
+		CodeMode:  0,
+		ClusterID: 0,
+		Size:      10,
+		SliceSize: 4,
+	}
+
+	ret := shardnode.CreateBlobRet{
+		Blob: proto.Blob{
+			Name: "blob-create",
+			Location: proto.Location{
+				ClusterID: 1,
+				CodeMode:  codemode.EC3P3,
+				Size_:     10,
+				SliceSize: 4,
+				Crc:       1,
+				Slices:    nil,
+			},
+		},
+	}
+	h.shardnodeClient.(*mocks.MockShardnodeAccess).EXPECT().CreateBlob(gAny, gAny, gAny).Return(ret, errcode.ErrShardRouteVersionNeedUpdate)
+	h.shardnodeClient.(*mocks.MockShardnodeAccess).EXPECT().CreateBlob(gAny, gAny, gAny).Return(ret, nil)
+	h.clusterController.(*MockClusterController).EXPECT().ChooseOne().Return(&clustermgr.ClusterInfo{
+		ClusterID: 1,
+	}, nil)
+
+	loc, err := h.CreateBlob(ctx, &args)
+	require.NoError(t, err)
+	require.Equal(t, ret.Blob.Location, *loc)
+}
+
+func TestStreamBlobDelete(t *testing.T) {
+	ctx := context.Background()
+	gAny := gomock.Any()
+	h := newStreamHandlerSuccess(t)
+
+	args := acapi.DelBlobArgs{ClusterID: 1, BlobName: "blob-del"}
+	h.shardnodeClient.(*mocks.MockShardnodeAccess).EXPECT().DeleteBlob(gAny, gAny, gAny).Return(errcode.ErrShardRouteVersionNeedUpdate)
+	h.shardnodeClient.(*mocks.MockShardnodeAccess).EXPECT().DeleteBlob(gAny, gAny, gAny).Return(nil)
+	require.NoError(t, h.DeleteBlob(ctx, &args))
+}
+
+func TestStreamBlobDeleteRaw(t *testing.T) {
+	ctx, gAny := context.Background(), gomock.Any()
+	h := newStreamHandlerSuccess(t)
+	h.StreamConfig.DeleteIntoShardnodePercentage = 100
+
+	h.shardnodeClient.(*mocks.MockShardnodeAccess).EXPECT().DeleteBlobRaw(gAny, gAny, gAny).Return(errcode.ErrUnexpected).Times(3)
+	require.Error(t, h.Delete(ctx, &proto.Location{Slices: []proto.Slice{{}, {}}}))
+
+	h.shardnodeClient.(*mocks.MockShardnodeAccess).EXPECT().DeleteBlobRaw(gAny, gAny, gAny).Return(nil).Times(2)
+	require.NoError(t, h.Delete(ctx, &proto.Location{Slices: []proto.Slice{{}, {}}}))
+}
+
+func TestStreamBlobSeal(t *testing.T) {
+	ctx := context.Background()
+	gAny := gomock.Any()
+	h := newStreamHandlerSuccess(t)
+
+	args := acapi.SealBlobArgs{
+		BlobName:  ("blob-seal"),
+		ClusterID: 1,
+		Slices:    make([]proto.Slice, 1),
+	}
+
+	h.shardnodeClient.(*mocks.MockShardnodeAccess).EXPECT().SealBlob(gAny, gAny, gAny).Return(errcode.ErrShardRouteVersionNeedUpdate)
+	h.shardnodeClient.(*mocks.MockShardnodeAccess).EXPECT().SealBlob(gAny, gAny, gAny).Return(nil)
+
+	err := h.SealBlob(ctx, &args)
+	require.NoError(t, err)
+}
+
+func TestStreamBlobList(t *testing.T) {
+	ctx := context.Background()
+	gAny := gomock.Any()
+	errMock := errors.New("fake error")
+	ctr := gomock.NewController(t)
+
+	info := controller.ShardOpInfo{
+		DiskID:       101,
+		Suid:         proto.EncodeSuid(1, 0, 1),
+		RouteVersion: 1,
+	}
+
+	shardInfo := NewMockShard(ctr)
+	shardInfo.EXPECT().GetMember(gAny, gAny, gAny).Return(info, nil).Times(3)
+
+	shardMgr := NewMockShardController(ctr)
+	shardMgr.EXPECT().GetShardByID(gAny, gAny).Return(shardInfo, nil).Times(3)
+	shardMgr.EXPECT().GetSpaceID().Return(proto.SpaceID(1)).Times(3)
+	shardMgr.EXPECT().UpdateRoute(gAny).Return(nil).Times(3)
+	shardMgr.EXPECT().GetShardSubRangeCount(gAny).Return(2).AnyTimes()
+
+	svrCtrl := NewMockServiceController(ctr)
+	svrCtrl.EXPECT().GetShardnodeHost(gAny, gAny).Return(&controller.HostIDC{Host: "host"}, nil).Times(3)
+
+	clu := NewMockClusterController(ctr)
+	clu.EXPECT().GetShardController(gAny).Return(shardMgr, nil).Times(3 * 2)
+	clu.EXPECT().GetServiceController(gAny).Return(svrCtrl, nil).Times(3)
+
+	h := &Handler{
+		clusterController: clu,
+		shardnodeClient:   mocks.NewMockShardnodeAccess(ctr),
+	}
+
+	args := acapi.ListBlobArgs{
+		ClusterID: 1,
+		ShardID:   1,
+		Prefix:    ("test-"),
+		Marker:    ("test-blob-1"),
+		Count:     4,
+	}
+	// list one shard
+	h.shardnodeClient.(*mocks.MockShardnodeAccess).EXPECT().ListBlob(gAny, gAny, gAny).Return(shardnode.ListBlobRet{}, errcode.ErrShardRouteVersionNeedUpdate).Times(3)
+	ret, err := h.ListBlob(ctx, &args)
+	require.NotNil(t, err)
+	require.ErrorIs(t, err, errcode.ErrShardRouteVersionNeedUpdate)
+	require.Equal(t, 0, len(ret.Blobs))
+
+	// list one shard, 3 blob
+	shardInfo.EXPECT().GetMember(gAny, gAny, gAny).Return(info, nil)
+	shardMgr.EXPECT().GetShardByID(gAny, gAny).Return(shardInfo, nil)
+	shardMgr.EXPECT().GetSpaceID().Return(proto.SpaceID(1))
+	svrCtrl.EXPECT().GetShardnodeHost(gAny, gAny).Return(&controller.HostIDC{Host: "host"}, nil)
+	clu.EXPECT().GetShardController(gAny).Return(shardMgr, nil)
+	clu.EXPECT().GetServiceController(gAny).Return(svrCtrl, nil)
+	listRet := shardnode.ListBlobRet{
+		Blobs: []proto.Blob{
+			{Name: "test-blob-1"},
+			{Name: "test-blob-2"},
+			{Name: "test-blob-3"},
+		},
+		NextMarker: "",
+	}
+	h.shardnodeClient.(*mocks.MockShardnodeAccess).EXPECT().ListBlob(gAny, gAny, gAny).Return(listRet, nil)
+
+	ret, err = h.ListBlob(ctx, &args)
+	require.NoError(t, err)
+	require.Equal(t, "", ret.NextMarker) // require.Equal(t, []byte(nil), ret.NextMarker)
+	require.Equal(t, 3, len(ret.Blobs))
+
+	// list all
+	shards := make([]controller.Shard, 4)
+	ranges := sharding.InitShardingRange(sharding.RangeType_RangeTypeHash, 1, 3)
+	for i := range shards {
+		shards[i] = NewMockShard(ctr)
+		shards[i].(*MockShard).EXPECT().GetShardID().Return(proto.ShardID(i + 1)).AnyTimes()
+		shards[i].(*MockShard).EXPECT().GetRange().Return(*ranges[i]).AnyTimes()
+		shards[i].(*MockShard).EXPECT().GetMember(gAny, gAny, gAny).Return(info, nil).AnyTimes()
+	}
+
+	shardMgr.EXPECT().GetFisrtShard(gAny).Return(shards[0], nil).Times(1)
+	shardMgr.EXPECT().GetNextShard(gAny, gAny).Return(shards[1], nil)
+	shardMgr.EXPECT().GetNextShard(gAny, gAny).Return(shards[2], nil)
+	shardMgr.EXPECT().GetSpaceID().Return(proto.SpaceID(1)).Times(2)
+
+	svrCtrl.EXPECT().GetShardnodeHost(gAny, gAny).Return(&controller.HostIDC{Host: "host"}, nil).Times(2)
+	h.clusterController.(*MockClusterController).EXPECT().GetShardController(gAny).Return(shardMgr, nil).Times(1)
+	h.clusterController.(*MockClusterController).EXPECT().GetServiceController(gAny).Return(svrCtrl, nil).Times(2)
+	listRet = shardnode.ListBlobRet{
+		Blobs: []proto.Blob{
+			{Name: "test-blob-1"},
+			{Name: "test-blob-2"},
+		},
+		NextMarker: "",
+	}
+	h.shardnodeClient.(*mocks.MockShardnodeAccess).EXPECT().ListBlob(gAny, gAny, gAny).Return(listRet, nil).Times(2)
+
+	args.ShardID = 0
+	args.Marker = ""
+	args.Count = 4
+	ret, err = h.ListBlob(ctx, &args)
+	expectMarker := acapi.ListBlobEncodeMarker{
+		Range:  *ranges[2],
+		Marker: "",
+	}
+	require.NoError(t, err)
+	require.Equal(t, 4, len(ret.Blobs))
+
+	actual := acapi.ListBlobEncodeMarker{}
+	err = actual.Unmarshal([]byte(ret.NextMarker))
+	require.NoError(t, err)
+	require.Equal(t, expectMarker, actual) // string(ret.NextMarker))
+
+	// list all, from next shard 3, list twice
+	args.ShardID = 0
+	args.Count = 4
+	args.Marker = ret.NextMarker
+
+	shardMgr.EXPECT().GetSpaceID().Return(proto.SpaceID(1)).Times(2)
+	shardMgr.EXPECT().GetShardByRange(gAny, expectMarker.Range).Return(shards[2], nil)
+	shardMgr.EXPECT().GetNextShard(gAny, gAny).Return(nil, errMock)
+	svrCtrl.EXPECT().GetShardnodeHost(gAny, gAny).Return(&controller.HostIDC{Host: "host"}, nil).Times(2)
+	h.clusterController.(*MockClusterController).EXPECT().GetShardController(gAny).Return(shardMgr, nil).Times(1)
+	h.clusterController.(*MockClusterController).EXPECT().GetServiceController(gAny).Return(svrCtrl, nil).Times(2)
+	listRet.NextMarker = ret.NextMarker
+	h.shardnodeClient.(*mocks.MockShardnodeAccess).EXPECT().ListBlob(gAny, gAny, gAny).Return(listRet, nil)
+	listRet.NextMarker = ""
+	h.shardnodeClient.(*mocks.MockShardnodeAccess).EXPECT().ListBlob(gAny, gAny, gAny).Return(listRet, nil)
+
+	ret, err = h.ListBlob(ctx, &args)
+	require.NotNil(t, err)
+	require.ErrorIs(t, err, errMock)
+	require.Equal(t, 0, len(ret.Blobs))
+
+	// list all, access until last shard not enough count
+	lastEndMarker := acapi.ListBlobEncodeMarker{
+		Range:  *ranges[2], // total count 4
+		Marker: args.Marker,
+	}
+	lastEnd, err := lastEndMarker.Marshal()
+	require.NoError(t, err)
+	args.ShardID = 0
+	args.Count = 100
+	args.Marker = string(lastEnd)
+
+	h.clusterController.(*MockClusterController).EXPECT().GetShardController(gAny).Return(shardMgr, nil)
+	shardMgr.EXPECT().GetShardByRange(gAny, lastEndMarker.Range).Return(shards[2], nil)
+	shardMgr.EXPECT().GetSpaceID().Return(proto.SpaceID(1)).Times(2)
+	shardMgr.EXPECT().GetNextShard(gAny, gAny).Return(shards[3], nil)
+	shardMgr.EXPECT().GetNextShard(gAny, gAny).Return(nil, nil)
+
+	svrCtrl.EXPECT().GetShardnodeHost(gAny, gAny).Return(&controller.HostIDC{Host: "host"}, nil).Times(2)
+	h.clusterController.(*MockClusterController).EXPECT().GetServiceController(gAny).Return(svrCtrl, nil).Times(2)
+	listRet.NextMarker = ""
+	h.shardnodeClient.(*mocks.MockShardnodeAccess).EXPECT().ListBlob(gAny, gAny, gAny).Return(listRet, nil).Times(2)
+
+	ret, err = h.ListBlob(ctx, &args)
+	require.NoError(t, err)
+	require.Equal(t, 4, len(ret.Blobs))
+	// endOneMarker := acapi.ListBlobEncodeMarker{
+	//	Range:  sharding.Range{}, // *ranges[3],
+	//	Marker: args.Marker,
+	// }
+	// endOne, err := endOneMarker.Marshal()
+	// require.NoError(t, err)
+	require.Equal(t, "", ret.NextMarker) // require.Equal(t, []byte(nil), ret.NextMarker)
+
+	// list all, error marker
+	args = acapi.ListBlobArgs{
+		ClusterID: 1,
+		Mode:      1,
+		ShardID:   0,
+		Prefix:    "", //[]byte("test-"),
+		Marker:    ("abcd"),
+		Count:     100,
+	}
+	h.clusterController.(*MockClusterController).EXPECT().GetShardController(gAny).Return(shardMgr, nil)
+	ret, err = h.ListBlob(ctx, &args)
+	require.NotNil(t, err)
+	require.True(t, strings.Contains(err.Error(), "fail to unmarshal marker"))
+}
+
+func TestStreamBlobAlloc(t *testing.T) {
+	ctx := context.Background()
+	gAny := gomock.Any()
+	h := newStreamHandlerSuccess(t)
+
+	args := acapi.AllocSliceArgs{
+		BlobName:  ("blob-seal"),
+		ClusterID: 1,
+		CodeMode:  1,
+		Size:      1,
+		FailSlice: proto.Slice{},
+	}
+
+	ret := shardnode.AllocSliceRet{
+		Slices: make([]proto.Slice, 1),
+	}
+	ret.Slices[0].Vid = 1
+	h.shardnodeClient.(*mocks.MockShardnodeAccess).EXPECT().AllocSlice(gAny, gAny, gAny).Return(ret, errcode.ErrShardRouteVersionNeedUpdate)
+	h.shardnodeClient.(*mocks.MockShardnodeAccess).EXPECT().AllocSlice(gAny, gAny, gAny).Return(ret, nil)
+
+	loc, err := h.AllocSlice(ctx, &args)
+	require.NoError(t, err)
+	require.NotNil(t, loc)
+	require.Equal(t, 1, len(loc.Slices))
+	require.Equal(t, proto.Vid(1), loc.Slices[0].Vid)
+}
+
+func TestStreamBlobOther(t *testing.T) {
+	ctx := context.Background()
+	gAny := gomock.Any()
+	ctr := gomock.NewController(t)
+
+	svrCtrl := NewMockServiceController(ctr)
+	svrCtrl.EXPECT().PunishShardnode(gAny, gAny, gAny).Times(2)
+
+	shardMgr := NewMockShardController(ctr)
+	shardMgr.EXPECT().UpdateRoute(gAny).Return(nil).Times(2)
+	shardMgr.EXPECT().UpdateShard(gAny, gAny).Return(nil)
+
+	clu := NewMockClusterController(ctr)
+	clu.EXPECT().GetServiceController(gAny).Return(svrCtrl, nil).Times(2)
+	clu.EXPECT().GetShardController(gAny).Return(shardMgr, nil).Times(3)
+
+	h := &Handler{
+		clusterController: clu,
+	}
+
+	interrupt, err1 := h.punishAndUpdate(ctx, &punishArgs{
+		err: errcode.ErrShardNodeDiskNotFound,
+	})
+	require.Equal(t, false, interrupt)
+	require.ErrorIs(t, err1, errcode.ErrShardNodeDiskNotFound)
+
+	interrupt, err1 = h.punishAndUpdate(ctx, &punishArgs{
+		err: errcode.ErrShardDoesNotExist,
+	})
+	require.Equal(t, false, interrupt)
+	require.ErrorIs(t, err1, errcode.ErrShardDoesNotExist)
+
+	interrupt, err1 = h.punishAndUpdate(ctx, &punishArgs{
+		err: io.EOF,
+	})
+	require.Equal(t, false, interrupt)
+	require.ErrorIs(t, err1, io.EOF)
+
+	// broken disk
+	interrupt, err1 = h.punishAndUpdate(ctx, &punishArgs{
+		ShardOpHeader: shardnode.ShardOpHeader{},
+		clusterID:     0,
+		host:          "",
+		err:           errcode.ErrDiskBroken,
+	})
+	require.Equal(t, false, interrupt)
+	require.ErrorIs(t, err1, errcode.ErrDiskBroken)
+
+	shardnodeClient := mocks.NewMockShardnodeAccess(ctr)
+	shardnodeClient.EXPECT().GetShardStats(gAny, gAny, gAny).Return(shardnode.ShardStats{LeaderDiskID: 11}, nil).Times(1)
+	h.shardnodeClient = shardnodeClient
+	h.ShardnodeRetryTimes = defaultShardnodeRetryTimes
+	err1 = h.updateLeaderFromCurrentHost(ctx, &punishArgs{
+		err: errcode.ErrShardNodeNotLeader,
+	})
+	require.NoError(t, err1)
+
+	// wait connect refused
+	h.ShardnodeRetryTimes = defaultShardnodeRetryTimes
+	info := controller.ShardOpInfo{
+		DiskID:       101,
+		Suid:         proto.EncodeSuid(1, 0, 1),
+		RouteVersion: 1,
+	}
+	clu.EXPECT().GetShardController(gAny).Return(shardMgr, nil)
+	clu.EXPECT().GetServiceController(gAny).Return(svrCtrl, nil).Times(2)
+	shardInfo := NewMockShard(ctr)
+	shardInfo.EXPECT().GetMember(gAny, gAny, map[proto.DiskID]struct{}{1: {}}).Return(info, nil)
+	shardMgr.EXPECT().GetShardByID(gAny, gAny).Return(shardInfo, nil)
+	shardMgr.EXPECT().UpdateShard(gAny, gAny).Return(nil)
+	svrCtrl.EXPECT().GetShardnodeHost(gAny, proto.DiskID(101)).Return(&controller.HostIDC{Host: "host101"}, nil)
+	svrCtrl.EXPECT().PunishShardnode(gAny, gAny, gAny)
+	shardnodeClient.EXPECT().GetShardStats(gAny, gAny, gAny).Return(shardnode.ShardStats{LeaderDiskID: 1}, nil)
+	shardnodeClient.EXPECT().GetShardStats(gAny, "host101", shardnode.GetShardArgs{
+		DiskID: proto.DiskID(101),
+		Suid:   info.Suid,
+	}).Return(shardnode.ShardStats{LeaderDiskID: 102}, nil)
+	interrupt, err1 = h.punishAndUpdate(ctx, &punishArgs{
+		ShardOpHeader: shardnode.ShardOpHeader{
+			DiskID: 1,
+			Suid:   2,
+		},
+		err: errors.New("dial tcp localhost:9100: connect: connection refused"),
+	})
+	require.Equal(t, false, interrupt)
+	// require.True(t, strings.Contains(err1.Error(), "connection refused"))
+	require.ErrorIs(t, err1, errcode.ErrConnectionRefused)
+
+	// interrupt, err1 = convertError(kvstore.ErrNotFound)
+	// require.Equal(t, true, interrupt)
+	// require.ErrorIs(t, err1, errcode.ErrCallShardNodeFail)
+}
+
+func TestStreamBlob_NotLeader_RetrySuccess(t *testing.T) {
+	ctx := context.Background()
+	ctr := gomock.NewController(t)
+	gAny := gomock.Any()
+
+	// old leader, Not Leader
+	oldInfo := controller.ShardOpInfo{
+		DiskID:       101,
+		Suid:         proto.EncodeSuid(1, 0, 1),
+		RouteVersion: 1,
+	}
+	// new leader
+	newInfo := controller.ShardOpInfo{
+		DiskID:       102,
+		Suid:         proto.EncodeSuid(1, 0, 2),
+		RouteVersion: 1,
+	}
+
+	// shard mock
+	shard := NewMockShard(ctr)
+	shard.EXPECT().GetMember(gAny, gAny, gAny).Return(oldInfo, nil).AnyTimes()                                                              // getShardOpHeader
+	shard.EXPECT().GetMember(gAny, acapi.GetShardModeRandom, map[proto.DiskID]struct{}{oldInfo.DiskID: {}}).Return(newInfo, nil).AnyTimes() // waitShardnodeNextLeader
+	shard.EXPECT().GetShardID().Return(proto.ShardID(1)).AnyTimes()
+
+	// shard controller mock
+	shardMgr := NewMockShardController(ctr)
+	shardMgr.EXPECT().GetShard(gAny, gAny).Return(shard, nil).AnyTimes()
+	shardMgr.EXPECT().GetShardByID(gAny, proto.ShardID(1)).Return(shard, nil).AnyTimes()
+	shardMgr.EXPECT().GetSpaceID().Return(proto.SpaceID(1)).AnyTimes()
+	shardMgr.EXPECT().UpdateRoute(gAny).Return(nil).AnyTimes()
+	shardMgr.EXPECT().GetShardSubRangeCount(gAny).Return(2).AnyTimes()
+	shardMgr.EXPECT().UpdateShard(gAny, gAny).Return(nil).Times(1) // waitShardnodeNextLeader finally
+
+	// service controller mock
+	svrCtrl := NewMockServiceController(ctr)
+	svrCtrl.EXPECT().GetShardnodeHost(gAny, proto.DiskID(oldInfo.DiskID)).Return(&controller.HostIDC{Host: "host-old"}, nil).AnyTimes()
+	svrCtrl.EXPECT().GetShardnodeHost(gAny, proto.DiskID(newInfo.DiskID)).Return(&controller.HostIDC{Host: "host-new"}, nil).AnyTimes()
+
+	// cluster controller mock
+	clu := NewMockClusterController(ctr)
+	clu.EXPECT().GetShardController(gAny).Return(shardMgr, nil).AnyTimes()
+	clu.EXPECT().GetServiceController(gAny).Return(svrCtrl, nil).AnyTimes()
+
+	// shardnode client mock：first NotLeader，and then success
+	shardCli := mocks.NewMockShardnodeAccess(ctr)
+	shardCli.EXPECT().DeleteBlob(gAny, gAny, gAny).Return(errcode.ErrShardNodeNotLeader)
+	shardCli.EXPECT().GetShardStats(gAny, gAny, gAny).Return(shardnode.ShardStats{LeaderDiskID: newInfo.DiskID}, nil)
+	shardCli.EXPECT().DeleteBlob(gAny, gAny, gAny).Return(nil)
+
+	h := &Handler{
+		clusterController: clu,
+		shardnodeClient:   shardCli,
+	}
+	h.ShardnodeRetryTimes = defaultShardnodeRetryTimes
+
+	args := acapi.DelBlobArgs{ClusterID: 1, BlobName: "blob-notleader"}
+	require.NoError(t, h.DeleteBlob(ctx, &args))
+}

blobstore/access/stream/stream_get.go

@@ -12,11 +12,12 @@
 // implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package access
+package stream
 
 import (
 	"context"
 	"fmt"
+	"hash/crc32"
 	"io"
 	"math/rand"
 	"sort"
@@ -26,7 +27,6 @@ import (
 	"github.com/afex/hystrix-go/hystrix"
 
 	"github.com/cubefs/cubefs/blobstore/access/controller"
-	"github.com/cubefs/cubefs/blobstore/api/access"
 	"github.com/cubefs/cubefs/blobstore/api/blobnode"
 	"github.com/cubefs/cubefs/blobstore/common/codemode"
 	"github.com/cubefs/cubefs/blobstore/common/ec"
@@ -34,6 +34,7 @@ import (
 	"github.com/cubefs/cubefs/blobstore/common/proto"
 	"github.com/cubefs/cubefs/blobstore/common/rpc"
 	"github.com/cubefs/cubefs/blobstore/common/trace"
+	"github.com/cubefs/cubefs/blobstore/util"
 	"github.com/cubefs/cubefs/blobstore/util/errors"
 	"github.com/cubefs/cubefs/blobstore/util/retry"
 )
@@ -66,6 +67,7 @@ type shardData struct {
 	index  int
 	status bool
 	buffer []byte
+	time   int
 }
 
 type sortedVuid struct {
@@ -110,7 +112,7 @@ type pipeBuffer struct {
 // ...
 // read-9 [d4                                       p5]
 // failed
-func (h *Handler) Get(ctx context.Context, w io.Writer, location access.Location, readSize, offset uint64) (func() error, error) {
+func (h *Handler) Get(ctx context.Context, w io.Writer, location proto.Location, readSize, offset uint64) (func() error, error) {
 	span := trace.SpanFromContextSafe(ctx)
 	span.Debugf("get request cluster:%d size:%d offset:%d", location.ClusterID, readSize, offset)
 
@@ -141,6 +143,7 @@ func (h *Handler) Get(ctx context.Context, w io.Writer, location access.Location
 		getTime := new(timeReadWrite)
 		defer func() {
 			span.AppendRPCTrackLog([]string{getTime.String()})
+			getTime.Report(clusterID.ToString(), h.IDC, false)
 		}()
 
 		// try to read data shard only,
@@ -166,6 +169,10 @@ func (h *Handler) Get(ctx context.Context, w io.Writer, location access.Location
 			}
 		}
 
+		var spanpipe trace.Span
+		spanpipe, ctx = trace.StartSpanFromContextWithTraceID(context.Background(), "", span.TraceID())
+		defer spanpipe.Finish()
+
 		// data stream flow:
 		// client <--copy-- pipeline <--swap-- readBlob <--copy-- blobnode
 		//
@@ -185,18 +192,21 @@ func (h *Handler) Get(ctx context.Context, w io.Writer, location access.Location
 					if blobVolume == nil || blobVolume.Vid != blob.Vid {
 						blobVolume, err = h.getVolume(ctx, clusterID, blob.Vid, true)
 						if err != nil {
-							span.Error("get volume", err)
+							spanpipe.Error("get volume", err)
 							ch <- pipeBuffer{err: err}
 							return
 						}
 
 						// do not use local shards
-						sortedVuids = genSortedVuidByIDC(ctx, serviceController, h.IDC, blobVolume.Units[:tactic.N+tactic.M])
-						span.Debugf("to read %s with read-shard-x:%d active-shard-n:%d of data-n:%d party-n:%d",
+						ordered := h.CodeModesGetOrdered[blobVolume.CodeMode]
+						ignoreIDC := h.CodeModesGetIgnoreIDC[blobVolume.CodeMode]
+						sortedVuids = genSortedVuidByIDC(ctx,
+							serviceController, h.IDC, blobVolume.Units[:tactic.N+tactic.M], ordered, ignoreIDC)
+						spanpipe.Debugf("to read %s with read-shard-x:%d active-shard-n:%d of data-n:%d party-n:%d",
 							blob.ID(), h.MinReadShardsX, len(sortedVuids), tactic.N, tactic.M)
 						if len(sortedVuids) < tactic.N {
 							err = fmt.Errorf("broken %s", blob.ID())
-							span.Error(err)
+							spanpipe.Error(err)
 							ch <- pipeBuffer{err: err}
 							return
 						}
@@ -212,7 +222,7 @@ func (h *Handler) Get(ctx context.Context, w io.Writer, location access.Location
 
 					err = h.readOneBlob(ctx, getTime, serviceController, blob, sortedVuids, shards)
 					if err != nil {
-						span.Error("read one blob", blob.ID(), err)
+						spanpipe.Error("read one blob", blob.ID(), err)
 						for _, buf := range shards {
 							h.memPool.Put(buf)
 						}
@@ -255,7 +265,7 @@ func (h *Handler) Get(ctx context.Context, w io.Writer, location access.Location
 					continue
 				}
 
-				toRead := minU64(toReadSize, l-off)
+				toRead := util.Min(toReadSize, l-off)
 				if _, e := w.Write(buf[off : off+toRead]); e != nil {
 					err = errors.Info(e, "write to response")
 					break
@@ -301,7 +311,8 @@ func (h *Handler) Get(ctx context.Context, w io.Writer, location access.Location
 // 4. Just read essential bytes if the data is a segment of one shard.
 func (h *Handler) readOneBlob(ctx context.Context, getTime *timeReadWrite,
 	serviceController controller.ServiceController,
-	blob blobGetArgs, sortedVuids []sortedVuid, shards [][]byte) error {
+	blob blobGetArgs, sortedVuids []sortedVuid, shards [][]byte,
+) error {
 	span := trace.SpanFromContextSafe(ctx)
 
 	tactic := blob.CodeMode.Tactic()
@@ -369,7 +380,31 @@ func (h *Handler) readOneBlob(ctx context.Context, getTime *timeReadWrite,
 
 	startRead := time.Now()
 	reconstructed := false
+	got, mostTime := 0, 0
 	for shard := range shardPipe {
+		if got++; got == dataN-1 {
+			mostTime = shard.time
+		}
+
+		var shardSpeed float32
+		if shard.time > 0 {
+			shardSpeed = float32(blob.ShardReadSize) / (float32(shard.time) / 1e9) / (1 << 10)
+		}
+		// find slow data shard, if it speed greater than index data-1 shard time.
+		if mostTime > 0 && shard.time/1e6 > h.LogSlowBaseTimeMS &&
+			shardSpeed < float32(h.LogSlowBaseSpeedKB) &&
+			float32(shard.time) > h.LogSlowTimeFator*float32(mostTime) {
+			var logvuid sortedVuid
+			for idx := range sortedVuids {
+				if sortedVuids[idx].index == shard.index {
+					logvuid = sortedVuids[idx]
+					break
+				}
+			}
+			span.Warnf("slow disk(host:%s diskid:%d) time(most:%dms shard:%dms) speed:%.2fKB/s",
+				logvuid.host, logvuid.diskID, mostTime/1e6, shard.time/1e6, shardSpeed)
+		}
+
 		// swap shard buffer
 		if shard.status {
 			buf := shards[shard.index]
@@ -462,7 +497,8 @@ func (h *Handler) readOneBlob(ctx context.Context, getTime *timeReadWrite,
 }
 
 func (h *Handler) readOneShard(ctx context.Context, serviceController controller.ServiceController,
-	blob blobGetArgs, vuid sortedVuid, stopChan <-chan struct{}) shardData {
+	blob blobGetArgs, vuid sortedVuid, stopChan <-chan struct{},
+) shardData {
 	clusterID, vid := blob.Cid, blob.Vid
 	shardOffset, shardReadSize := blob.ShardOffset, blob.ShardReadSize
 	span := trace.SpanFromContextSafe(ctx)
@@ -470,12 +506,14 @@ func (h *Handler) readOneShard(ctx context.Context, serviceController controller
 		index:  vuid.index,
 		status: false,
 	}
+	shardStart := time.Now()
 
 	args := blobnode.RangeGetShardArgs{
 		GetShardArgs: blobnode.GetShardArgs{
 			DiskID: vuid.diskID,
 			Vuid:   vuid.vuid,
 			Bid:    blob.Bid,
+			Type:   blobnode.ReadIO,
 		},
 		Offset: int64(shardOffset),
 		Size:   int64(shardReadSize),
@@ -484,9 +522,10 @@ func (h *Handler) readOneShard(ctx context.Context, serviceController controller
 	var (
 		err  error
 		body io.ReadCloser
+		crc  uint32
 	)
 	if hErr := hystrix.Do(rwCommand, func() error {
-		body, err = h.getOneShardFromHost(ctx, serviceController, vuid.host, vuid.diskID, args,
+		body, crc, err = h.getOneShardFromHost(ctx, serviceController, vuid.host, vuid.diskID, args,
 			vuid.index, clusterID, vid, 3, stopChan)
 		if err != nil && (errorTimeout(err) || rpc.DetectStatusCode(err) == errcode.CodeOverload) {
 			return err
@@ -498,10 +537,13 @@ func (h *Handler) readOneShard(ctx context.Context, serviceController controller
 	}
 
 	if err != nil {
-		if err == errPunishedDisk || err == errCanceledReadShard {
+		if err == errPunishedDisk {
 			span.Warnf("read %s on %s: %s", blob.ID(), vuid.ID(), err.Error())
 			return shardResult
 		}
+		if err == errCanceledReadShard {
+			return shardResult
+		}
 		span.Warnf("rpc read %s on %s: %s", blob.ID(), vuid.ID(), errors.Detail(err))
 		return shardResult
 	}
@@ -519,14 +561,26 @@ func (h *Handler) readOneShard(ctx context.Context, serviceController controller
 		span.Warnf("io read %s on %s: %s", blob.ID(), vuid.ID(), err.Error())
 		return shardResult
 	}
+	if h.ShardCrcReadEnable && crc > 0 {
+		newCrc := crc32.ChecksumIEEE(buf[shardOffset : shardOffset+shardReadSize])
+		if newCrc != crc {
+			h.memPool.Put(buf)
+			reportDownload(clusterID, "Download", "CrcMismatch")
+			span.Errorf("blob:%+v vuid:%s crc mismatch 0x%x(%d) != 0x%x(%d)",
+				blob, vuid.ID(), crc, crc, newCrc, newCrc)
+			return shardResult
+		}
+	}
 
 	shardResult.status = true
 	shardResult.buffer = buf
+	shardResult.time = int(time.Since(shardStart))
 	return shardResult
 }
 
 func (h *Handler) getDataShardOnly(ctx context.Context, getTime *timeReadWrite,
-	w io.Writer, serviceController controller.ServiceController, blob blobGetArgs) error {
+	w io.Writer, serviceController controller.ServiceController, blob blobGetArgs,
+) error {
 	span := trace.SpanFromContextSafe(ctx)
 	if blob.ReadSize == 0 {
 		return nil
@@ -549,6 +603,10 @@ func (h *Handler) getDataShardOnly(ctx context.Context, getTime *timeReadWrite,
 	firstShardIdx := int(blob.Offset) / shardSize
 	shardOffset := int(blob.Offset) % shardSize
 
+	ctx, cancel := context.WithDeadline(ctx,
+		time.Now().Add(time.Millisecond*time.Duration(h.ReadDataOnlyTimeoutMS)))
+	defer cancel()
+
 	startRead := time.Now()
 	remainSize := blob.ReadSize
 	bufOffset := 0
@@ -557,22 +615,30 @@ func (h *Handler) getDataShardOnly(ctx context.Context, getTime *timeReadWrite,
 			break
 		}
 
-		toReadSize := minU64(remainSize, uint64(shardSize-shardOffset))
+		diskInfo, err := serviceController.GetDiskHost(ctx, shard.DiskID)
+		if err != nil {
+			span.Warnf("get disk host failed: %s", err)
+			return errNeedReconstructRead
+		}
+		host := diskInfo.Host
+
+		toReadSize := util.Min(remainSize, uint64(shardSize-shardOffset))
 		args := blobnode.RangeGetShardArgs{
 			GetShardArgs: blobnode.GetShardArgs{
 				DiskID: shard.DiskID,
 				Vuid:   shard.Vuid,
 				Bid:    blob.Bid,
+				Type:   blobnode.ReadIO,
 			},
 			Offset: int64(shardOffset),
 			Size:   int64(toReadSize),
 		}
 
-		body, err := h.getOneShardFromHost(ctx, serviceController, shard.Host, shard.DiskID, args,
+		body, crc, err := h.getOneShardFromHost(ctx, serviceController, host, shard.DiskID, args,
 			firstShardIdx+i, blob.Cid, blob.Vid, 1, nil)
 		if err != nil {
 			span.Warnf("read %s on blobnode(vuid:%d disk:%d host:%s) ecidx(%02d): %s", blob.ID(),
-				shard.Vuid, shard.DiskID, shard.Host, firstShardIdx+i, errors.Detail(err))
+				shard.Vuid, shard.DiskID, host, firstShardIdx+i, errors.Detail(err))
 			return errNeedReconstructRead
 		}
 		defer body.Close()
@@ -583,6 +649,14 @@ func (h *Handler) getDataShardOnly(ctx context.Context, getTime *timeReadWrite,
 			span.Warn(err)
 			return errNeedReconstructRead
 		}
+		if h.ShardCrcReadEnable && crc > 0 {
+			if newCrc := crc32.ChecksumIEEE(buf); newCrc != crc {
+				reportDownload(blob.Cid, "Download", "CrcMismatch")
+				span.Errorf("blob:%+v (host:%s diskid:%d vuid:%d) crc mismatch 0x%x(%d) != 0x%x(%d)",
+					blob, host, shard.DiskID, shard.Vuid, crc, crc, newCrc, newCrc)
+				return errNeedReconstructRead
+			}
+		}
 
 		// reset next shard offset
 		shardOffset = 0
@@ -610,20 +684,16 @@ func (h *Handler) getOneShardFromHost(ctx context.Context, serviceController con
 	host string, diskID proto.DiskID, args blobnode.RangeGetShardArgs, // get shard param with host diskid
 	index int, clusterID proto.ClusterID, vid proto.Vid, // param to update volume cache
 	attempts int, cancelChan <-chan struct{}, // do not retry again if cancelChan was closed
-) (io.ReadCloser, error) {
+) (rbody io.ReadCloser, rcrc uint32, rerr error) {
 	span := trace.SpanFromContextSafe(ctx)
 
 	// skip punished disk
 	if diskHost, err := serviceController.GetDiskHost(ctx, diskID); err != nil {
-		return nil, err
+		return nil, 0, err
 	} else if diskHost.Punished {
-		return nil, errPunishedDisk
+		return nil, 0, errPunishedDisk
 	}
 
-	var (
-		rbody io.ReadCloser
-		rerr  error
-	)
 	rerr = retry.ExponentialBackoff(attempts, 200).RuptOn(func() (bool, error) {
 		if cancelChan != nil {
 			select {
@@ -633,14 +703,10 @@ func (h *Handler) getOneShardFromHost(ctx context.Context, serviceController con
 			}
 		}
 
-		// new child span to get from blobnode, we should finish it here.
-		spanChild, ctxChild := trace.StartSpanFromContextWithTraceID(
-			context.Background(), "GetFromBlobnode", span.TraceID())
-		defer spanChild.Finish()
-
-		body, _, err := h.blobnodeClient.RangeGetShard(ctxChild, host, &args)
+		body, crc, err := h.blobnodeClient.RangeGetShard(ctx, host, &args)
 		if err == nil {
 			rbody = body
+			rcrc = crc
 			return true, nil
 		}
 
@@ -688,10 +754,12 @@ func (h *Handler) getOneShardFromHost(ctx context.Context, serviceController con
 
 		// do not retry on timeout then punish threshold this disk
 		if errorTimeout(err) {
+			h.updateVolume(ctx, clusterID, vid)
 			h.punishDiskWith(ctx, clusterID, diskID, host, "Timeout")
 			return true, err
 		}
 		if errorConnectionRefused(err) {
+			h.updateVolume(ctx, clusterID, vid)
 			return true, err
 		}
 		span.Debugf("read from disk:%d blobnode/%s", diskID, err.Error())
@@ -700,17 +768,17 @@ func (h *Handler) getOneShardFromHost(ctx context.Context, serviceController con
 		return false, err
 	})
 
-	return rbody, rerr
+	return
 }
 
-func genLocationBlobs(location *access.Location, readSize uint64, offset uint64) ([]blobGetArgs, error) {
-	if readSize > location.Size || offset > location.Size || offset+readSize > location.Size {
-		return nil, fmt.Errorf("FileSize:%d ReadSize:%d Offset:%d", location.Size, readSize, offset)
+func genLocationBlobs(location *proto.Location, readSize uint64, offset uint64) ([]blobGetArgs, error) {
+	if readSize > location.Size_ || offset > location.Size_ || offset+readSize > location.Size_ {
+		return nil, fmt.Errorf("FileSize:%d ReadSize:%d Offset:%d", location.Size_, readSize, offset)
 	}
 
-	blobSize := uint64(location.BlobSize)
+	blobSize := uint64(location.SliceSize)
 	if blobSize <= 0 {
-		return nil, fmt.Errorf("BlobSize:%d", blobSize)
+		return nil, fmt.Errorf("SliceSize:%d", blobSize)
 	}
 
 	remainSize := readSize
@@ -721,8 +789,8 @@ func genLocationBlobs(location *access.Location, readSize uint64, offset uint64)
 
 	idx := uint64(0)
 	blobs := make([]blobGetArgs, 0, 1+(readSize+blobOffset)/blobSize)
-	for _, blob := range location.Blobs {
-		currBlobID := blob.MinBid
+	for _, blob := range location.Slices {
+		currBlobID := blob.MinSliceID
 
 		for ii := uint32(0); ii < blob.Count; ii++ {
 			if remainSize <= 0 {
@@ -730,10 +798,10 @@ func genLocationBlobs(location *access.Location, readSize uint64, offset uint64)
 			}
 
 			if idx >= firstBlobIdx {
-				toReadSize := minU64(remainSize, blobSize-blobOffset)
+				toReadSize := util.Min(remainSize, blobSize-blobOffset)
 				if toReadSize > 0 {
 					// update the last blob size
-					fixedBlobSize := minU64(location.Size-idx*blobSize, blobSize)
+					fixedBlobSize := util.Min(location.Size_-idx*blobSize, blobSize)
 
 					sizes, _ := ec.GetBufferSizes(int(fixedBlobSize), tactic)
 					shardSize := sizes.ShardSize
@@ -771,8 +839,10 @@ func genLocationBlobs(location *access.Location, readSize uint64, offset uint64)
 	return blobs, nil
 }
 
-func genSortedVuidByIDC(ctx context.Context, serviceController controller.ServiceController, idc string,
-	vuidPhys []controller.Unit) []sortedVuid {
+func genSortedVuidByIDC(ctx context.Context,
+	serviceController controller.ServiceController, idc string, vuidPhys []controller.Unit,
+	ordered bool, ignoreIDC bool,
+) []sortedVuid {
 	span := trace.SpanFromContextSafe(ctx)
 
 	vuids := make([]sortedVuid, 0, len(vuidPhys))
@@ -792,7 +862,11 @@ func genSortedVuidByIDC(ctx context.Context, serviceController controller.Servic
 			continue
 		}
 
-		dis := distance(idc, hostIDC.IDC, hostIDC.Punished)
+		destIDC := hostIDC.IDC
+		if ignoreIDC {
+			destIDC = idc
+		}
+		dis := distance(idc, destIDC, hostIDC.Punished)
 		if _, ok := sortMap[dis]; !ok {
 			sortMap[dis] = make([]sortedVuid, 0, 8)
 		}
@@ -800,7 +874,7 @@ func genSortedVuidByIDC(ctx context.Context, serviceController controller.Servic
 			index:  idx,
 			vuid:   phy.Vuid,
 			diskID: phy.DiskID,
-			host:   phy.Host,
+			host:   hostIDC.Host,
 		})
 	}
 
@@ -812,9 +886,11 @@ func genSortedVuidByIDC(ctx context.Context, serviceController controller.Servic
 
 	for _, dis := range keys {
 		ids := sortMap[dis]
-		rand.Shuffle(len(ids), func(i, j int) {
-			ids[i], ids[j] = ids[j], ids[i]
-		})
+		if !ordered {
+			rand.Shuffle(len(ids), func(i, j int) {
+				ids[i], ids[j] = ids[j], ids[i]
+			})
+		}
 		vuids = append(vuids, ids...)
 		if dis > 1 {
 			span.Debugf("distance: %d punished vuids: %+v", dis, ids)
@@ -841,7 +917,7 @@ func emptyDataShardIndexes(sizes ec.BufferSizes) map[int]struct{} {
 	firstEmptyIdx := (sizes.DataSize + sizes.ShardSize - 1) / sizes.ShardSize
 	n := sizes.ECDataSize / sizes.ShardSize
 	if firstEmptyIdx >= n {
-		return make(map[int]struct{})
+		return nil
 	}
 
 	set := make(map[int]struct{}, n-firstEmptyIdx)

blobstore/access/stream/stream_get_test.go

@@ -12,7 +12,7 @@
 // implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package access
+package stream
 
 import (
 	"bytes"
@@ -24,7 +24,6 @@ import (
 
 	"github.com/stretchr/testify/require"
 
-	"github.com/cubefs/cubefs/blobstore/api/access"
 	"github.com/cubefs/cubefs/blobstore/common/codemode"
 	"github.com/cubefs/cubefs/blobstore/common/proto"
 )
@@ -35,7 +34,7 @@ func TestAccessStreamGetBase(t *testing.T) {
 	{
 		dataShards.clean()
 		data := []byte("x")
-		loc, err := streamer.Put(ctx(), bytes.NewReader(data), int64(len(data)), nil)
+		loc, err := streamer.Put(ctx(), bytes.NewReader(data), int64(len(data)), nil, proto.ClusterID(0), codemode.CodeModeNone)
 		require.NoError(t, err)
 
 		buff := bytes.NewBuffer(nil)
@@ -48,7 +47,7 @@ func TestAccessStreamGetBase(t *testing.T) {
 	{
 		dataShards.clean()
 		data := []byte("x")
-		loc, err := streamer.Put(ctx(), bytes.NewReader(data), int64(len(data)), nil)
+		loc, err := streamer.Put(ctx(), bytes.NewReader(data), int64(len(data)), nil, proto.ClusterID(0), codemode.CodeModeNone)
 		require.NoError(t, err)
 
 		buff := bytes.NewBuffer(nil)
@@ -84,7 +83,7 @@ func TestAccessStreamGetBase(t *testing.T) {
 		size := cs.size
 		data := make([]byte, size)
 		rand.Read(data)
-		loc, err := streamer.Put(ctx(), bytes.NewReader(data), int64(size), nil)
+		loc, err := streamer.Put(ctx(), bytes.NewReader(data), int64(size), nil, proto.ClusterID(0), codemode.CodeModeNone)
 		require.NoError(t, err)
 
 		buff := bytes.NewBuffer(nil)
@@ -118,7 +117,7 @@ func TestAccessStreamGetBroken(t *testing.T) {
 	rand.Read(data)
 	// time wait the punished services
 	time.Sleep(time.Second * time.Duration(punishServiceS))
-	loc, err := streamer.Put(ctx(), bytes.NewReader(data), int64(size), nil)
+	loc, err := streamer.Put(ctx(), bytes.NewReader(data), int64(size), nil, proto.ClusterID(0), codemode.CodeModeNone)
 	require.NoError(t, err)
 
 	cases := []struct {
@@ -184,7 +183,7 @@ func TestAccessStreamGetOffset(t *testing.T) {
 		size := cs.size
 		data := make([]byte, size)
 		rand.Read(data)
-		loc, err := streamer.Put(ctx(), bytes.NewReader(data), size, nil)
+		loc, err := streamer.Put(ctx(), bytes.NewReader(data), size, nil, proto.ClusterID(0), codemode.CodeModeNone)
 		require.NoError(t, err)
 
 		buff := bytes.NewBuffer(nil)
@@ -211,7 +210,7 @@ func TestAccessStreamGetShardTimeout(t *testing.T) {
 	size := 1 << 22
 	buff := make([]byte, size)
 	rand.Read(buff)
-	loc, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil)
+	loc, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil, proto.ClusterID(0), codemode.CodeModeNone)
 	require.NoError(t, err)
 
 	// no delay when blocking one shard, cos MinReadShardsX = 1
@@ -220,13 +219,9 @@ func TestAccessStreamGetShardTimeout(t *testing.T) {
 		vuidController.Unblock(1001)
 	}()
 	{
-		startTime := time.Now()
 		transfer, _ := streamer.Get(ctx(), bytes.NewBuffer(nil), *loc, uint64(size), 0)
 		err := transfer()
 		require.NoError(t, err)
-
-		duration := time.Since(startTime)
-		require.GreaterOrEqual(t, vuidController.duration, duration, "greater duration: ", duration)
 	}
 
 	// delay one duration when blocking two shard, cos MinReadShardsX = 1
@@ -249,6 +244,59 @@ func TestAccessStreamGetShardTimeout(t *testing.T) {
 	}
 }
 
+func TestAccessStreamGetShardSlow(t *testing.T) {
+	ctx := ctxWithName("TestAccessStreamGetShardSlow")
+	dataShards.clean()
+	vuidController.Unbreak(1005)
+	streamer.MinReadShardsX = 0
+	defer func() {
+		vuidController.SetSlowdown(1001, -1)
+		vuidController.Break(1005)
+		streamer.MinReadShardsX = minReadShardsX
+		dataShards.clean()
+	}()
+
+	size := 1 << 20
+	buff := make([]byte, size)
+	rand.Read(buff)
+	loc, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil, proto.ClusterID(0), codemode.CodeModeNone)
+	require.NoError(t, err)
+
+	vuidController.SetSlowdown(1001, 500*time.Millisecond)
+	transfer, err := streamer.Get(ctx(), bytes.NewBuffer(nil), *loc, uint64(size), 0)
+	require.NoError(t, err)
+	err = transfer()
+	require.NoError(t, err)
+}
+
+func TestAccessStreamGetShardCrcMismatch(t *testing.T) {
+	ctx := ctxWithName("TestAccessStreamGetShardCrcMismatch")
+	vuidController.Unbreak(1005)
+	streamer.MinReadShardsX = 0
+	defer func() {
+		vuidController.SetCrcMismatch(1001, false)
+		vuidController.SetCrcMismatch(1002, false)
+		vuidController.Break(1005)
+		streamer.MinReadShardsX = minReadShardsX
+		dataShards.clean()
+	}()
+
+	vuidController.SetCrcMismatch(1001, true)
+	vuidController.SetCrcMismatch(1002, true)
+	for _, size := range []int{1, 1023, 2048, 1 << 20} {
+		dataShards.clean()
+		buff := make([]byte, size)
+		rand.Read(buff)
+		loc, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil, proto.ClusterID(0), codemode.CodeModeNone)
+		require.NoError(t, err)
+
+		transfer, err := streamer.Get(ctx(), bytes.NewBuffer(nil), *loc, uint64(size), 0)
+		require.NoError(t, err)
+		err = transfer()
+		require.NoError(t, err)
+	}
+}
+
 func TestAccessStreamGetShardBroken(t *testing.T) {
 	ctx := ctxWithName("TestAccessStreamGetShardBroken")
 	dataShards.clean()
@@ -263,7 +311,7 @@ func TestAccessStreamGetShardBroken(t *testing.T) {
 	size := 1 << 22
 	buff := make([]byte, size)
 	rand.Read(buff)
-	loc, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil)
+	loc, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil, proto.ClusterID(0), codemode.CodeModeNone)
 	require.NoError(t, err)
 
 	// no delay when blocking one shard, cos MinReadShardsX = 1
@@ -291,6 +339,39 @@ func TestAccessStreamGetShardBroken(t *testing.T) {
 	}
 }
 
+func TestAccessStreamGetShardOnlyTimeout(t *testing.T) {
+	ctx := ctxWithName("TestAccessStreamGetShardOnlyTimeout")
+	dataShards.clean()
+	oldMs := streamer.ReadDataOnlyTimeoutMS
+	streamer.ReadDataOnlyTimeoutMS = 100
+	defer func() {
+		streamer.ReadDataOnlyTimeoutMS = oldMs
+		dataShards.clean()
+	}()
+
+	size := 1
+	buff := make([]byte, size)
+	rand.Read(buff)
+	loc, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil, proto.ClusterID(0), codemode.CodeModeNone)
+	require.NoError(t, err)
+
+	// blocking the data shard, force to waiting ReadDataOnlyTimeoutMS
+	vuidController.Block(1001)
+	defer func() {
+		vuidController.Unblock(1001)
+	}()
+	{
+		startTime := time.Now()
+		transfer, err := streamer.Get(ctx(), bytes.NewBuffer(nil), *loc, uint64(size), 0)
+		require.NoError(t, err)
+		err = transfer()
+		require.NoError(t, err)
+
+		duration := time.Since(startTime)
+		require.GreaterOrEqual(t, duration, 100*time.Millisecond, "greater duration:", duration)
+	}
+}
+
 func TestAccessStreamGetLocalIDC(t *testing.T) {
 	ctx := ctxWithName("TestAccessStreamGetLocalIDC")
 	dataShards.clean()
@@ -303,7 +384,7 @@ func TestAccessStreamGetLocalIDC(t *testing.T) {
 	size := 1 << 22
 	buff := make([]byte, size)
 	rand.Read(buff)
-	loc, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil)
+	loc, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil, proto.ClusterID(0), codemode.CodeModeNone)
 	require.NoError(t, err)
 
 	// no delay when blocking other idc all shards
@@ -414,7 +495,7 @@ func TestAccessStreamGetAligned(t *testing.T) {
 
 		data := make([]byte, cs.size)
 		rand.Read(data)
-		loc, err := streamer.Put(ctx(), bytes.NewReader(data), int64(cs.size), nil)
+		loc, err := streamer.Put(ctx(), bytes.NewReader(data), int64(cs.size), nil, proto.ClusterID(0), codemode.CodeModeNone)
 		require.NoError(t, err)
 
 		// cos put shards asynchronously, should wait all shard written
@@ -444,21 +525,21 @@ func TestAccessStreamGenLocationBlobs(t *testing.T) {
 	firstSliceStart := proto.BlobID(100)
 	secondSliceStart := proto.BlobID(200)
 
-	loc := access.Location{
+	loc := proto.Location{
 		ClusterID: 0,
 		CodeMode:  codemode.EC6P6,
-		Size:      1024*4 + 37 + 1024*2, // 5 fine blobs and 2 missing blobs
-		BlobSize:  1024,
-		Blobs: []access.SliceInfo{
+		Size_:     1024*4 + 37 + 1024*2, // 5 fine blobs and 2 missing blobs
+		SliceSize: 1024,
+		Slices: []proto.Slice{
 			{
-				MinBid: firstSliceStart,
-				Vid:    proto.Vid(1001),
-				Count:  3,
+				MinSliceID: firstSliceStart,
+				Vid:        proto.Vid(1001),
+				Count:      3,
 			},
 			{
-				MinBid: secondSliceStart,
-				Vid:    proto.Vid(2001),
-				Count:  2,
+				MinSliceID: secondSliceStart,
+				Vid:        proto.Vid(2001),
+				Count:      2,
 			},
 		},
 	}
@@ -594,7 +675,7 @@ func BenchmarkAccessStreamGet(b *testing.B) {
 	for _, cs := range cases {
 		b.ResetTimer()
 		b.Run(cs.name, func(b *testing.B) {
-			loc, err := streamer.Put(ctx, newReader(cs.size), int64(cs.size), nil)
+			loc, err := streamer.Put(ctx, newReader(cs.size), int64(cs.size), nil, proto.ClusterID(0), codemode.CodeModeNone)
 			require.NoError(b, err)
 
 			b.ResetTimer()

blobstore/access/stream/stream_loop.go

@@ -12,7 +12,7 @@
 // implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package access
+package stream
 
 import (
 	"context"

blobstore/access/stream/stream_mock_test.go

@@ -12,11 +12,7 @@
 // implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package access
-
-// github.com/cubefs/cubefs/blobstore/access/... module access interfaces
-//go:generate mockgen -destination=./controller_mock_test.go -package=access -mock_names ClusterController=MockClusterController,ServiceController=MockServiceController,VolumeGetter=MockVolumeGetter github.com/cubefs/cubefs/blobstore/access/controller ClusterController,ServiceController,VolumeGetter
-//go:generate mockgen -destination=./access_mock_test.go -package=access -mock_names StreamHandler=MockStreamHandler,Limiter=MockLimiter github.com/cubefs/cubefs/blobstore/access StreamHandler,Limiter
+package stream
 
 import (
 	"bytes"
@@ -25,7 +21,6 @@ import (
 	"fmt"
 	"hash/crc32"
 	"io"
-	"io/ioutil"
 	"math"
 	"math/rand"
 	"strconv"
@@ -48,6 +43,7 @@ import (
 	"github.com/cubefs/cubefs/blobstore/common/trace"
 	"github.com/cubefs/cubefs/blobstore/testing/mocks"
 	_ "github.com/cubefs/cubefs/blobstore/testing/nolog"
+	"github.com/cubefs/cubefs/blobstore/util/bytespool"
 )
 
 var (
@@ -82,10 +78,10 @@ var (
 	cc                controller.ClusterController
 
 	clusterInfo *clustermgr.ClusterInfo
-	dataVolume  *proxy.VersionVolume
+	dataVolume  *clustermgr.VolumeInfo
 	dataAllocs  []proxy.AllocRet
 	dataNodes   map[string]clustermgr.ServiceInfo
-	dataDisks   map[proto.DiskID]blobnode.DiskInfo
+	dataDisks   map[proto.DiskID]clustermgr.BlobNodeDiskInfo
 	dataShards  *shardsData
 
 	vuidController *vuidControl
@@ -147,6 +143,8 @@ type vuidControl struct {
 	mutex    sync.Mutex
 	broken   map[proto.Vuid]bool
 	blocked  map[proto.Vuid]bool
+	slowdown map[proto.Vuid]time.Duration
+	crc      map[proto.Vuid]bool
 	block    func()
 	duration time.Duration
 
@@ -191,6 +189,36 @@ func (c *vuidControl) Isblocked(id proto.Vuid) bool {
 	return ok && v
 }
 
+func (c *vuidControl) SetSlowdown(id proto.Vuid, t time.Duration) {
+	c.mutex.Lock()
+	if t < 0 {
+		delete(c.slowdown, id)
+	} else {
+		c.slowdown[id] = t
+	}
+	c.mutex.Unlock()
+}
+
+func (c *vuidControl) GetSlowdown(id proto.Vuid) time.Duration {
+	c.mutex.Lock()
+	v := c.slowdown[id]
+	c.mutex.Unlock()
+	return v
+}
+
+func (c *vuidControl) SetCrcMismatch(id proto.Vuid, crc bool) {
+	c.mutex.Lock()
+	c.crc[id] = crc
+	c.mutex.Unlock()
+}
+
+func (c *vuidControl) GetCrcMismatch(id proto.Vuid) bool {
+	c.mutex.Lock()
+	v := c.crc[id]
+	c.mutex.Unlock()
+	return v
+}
+
 func (c *vuidControl) SetBNRealError(b bool) {
 	c.mutex.Lock()
 	c.isBNRealError = b
@@ -209,7 +237,8 @@ func randBlobnodeRealError(errors []errcode.Error) error {
 }
 
 var storageAPIRangeGetShard = func(ctx context.Context, host string, args *blobnode.RangeGetShardArgs) (
-	body io.ReadCloser, shardCrc uint32, err error) {
+	body io.ReadCloser, shardCrc uint32, err error,
+) {
 	if vuidController.Isbroken(args.Vuid) {
 		err = errors.New("get shard fake error")
 		if vuidController.IsBNRealError() {
@@ -226,6 +255,9 @@ var storageAPIRangeGetShard = func(ctx context.Context, host string, args *blobn
 		}
 		return
 	}
+	if slow := vuidController.GetSlowdown(args.Vuid); slow > 0 {
+		time.Sleep(slow)
+	}
 
 	buff := dataShards.get(args.Vuid, args.Bid)
 	if len(buff) == 0 {
@@ -235,15 +267,21 @@ var storageAPIRangeGetShard = func(ctx context.Context, host string, args *blobn
 		err = errors.New("get shard concurrently")
 		return
 	}
+	if len(buff) == int(args.Size) {
+		shardCrc = crc32.ChecksumIEEE(buff)
+		if vuidController.GetCrcMismatch(args.Vuid) {
+			shardCrc++
+		}
+	}
 
 	buff = buff[int(args.Offset):int(args.Offset+args.Size)]
-	shardCrc = crc32.ChecksumIEEE(buff)
-	body = ioutil.NopCloser(bytes.NewReader(buff))
+	body = io.NopCloser(bytes.NewReader(buff))
 	return
 }
 
 var storageAPIPutShard = func(ctx context.Context, host string, args *blobnode.PutShardArgs) (
-	crc uint32, err error) {
+	crc uint32, err error,
+) {
 	if vuidController.Isbroken(args.Vuid) {
 		err = errors.New("put shard fake error")
 		if vuidController.IsBNRealError() {
@@ -256,14 +294,21 @@ var storageAPIPutShard = func(ctx context.Context, host string, args *blobnode.P
 		err = errors.New("put shard timeout")
 		return
 	}
+	if slow := vuidController.GetSlowdown(args.Vuid); slow > 0 {
+		time.Sleep(slow)
+	}
 
 	buffer, _ := memPool.Alloc(int(args.Size))
 	defer memPool.Put(buffer)
 
 	buffer = buffer[:int(args.Size)]
-	_, err = io.ReadFull(args.Body, buffer)
-	if err != nil {
-		return
+	if args.NopData {
+		bytespool.Zero(buffer)
+	} else {
+		_, err = io.ReadFull(args.Body, buffer)
+		if err != nil {
+			return
+		}
 	}
 
 	crc = crc32.ChecksumIEEE(buffer)
@@ -284,7 +329,7 @@ func initMockData() {
 		Vid:      volumeID,
 	}
 
-	dataVolume = &proxy.VersionVolume{VolumeInfo: clustermgr.VolumeInfo{
+	dataVolume = &clustermgr.VolumeInfo{
 		VolumeInfoBase: clustermgr.VolumeInfoBase{
 			Vid:      volumeID,
 			CodeMode: codemode.EC6P6,
@@ -294,12 +339,11 @@ func initMockData() {
 				units = append(units, clustermgr.Unit{
 					Vuid:   proto.Vuid(id),
 					DiskID: proto.DiskID(id),
-					Host:   strconv.Itoa(id),
 				})
 			}
 			return
 		}(),
-	}}
+	}
 
 	proxyNodes := make([]clustermgr.ServiceNode, 32)
 	for idx := range proxyNodes {
@@ -316,17 +360,17 @@ func initMockData() {
 		Nodes: proxyNodes,
 	}
 
-	dataDisks = make(map[proto.DiskID]blobnode.DiskInfo)
+	dataDisks = make(map[proto.DiskID]clustermgr.BlobNodeDiskInfo)
 	for _, id := range idcID {
-		dataDisks[proto.DiskID(id)] = blobnode.DiskInfo{
-			ClusterID: clusterID, Idc: idc, Host: strconv.Itoa(id),
-			DiskHeartBeatInfo: blobnode.DiskHeartBeatInfo{DiskID: proto.DiskID(id)},
+		dataDisks[proto.DiskID(id)] = clustermgr.BlobNodeDiskInfo{
+			DiskInfo:          clustermgr.DiskInfo{ClusterID: clusterID, Idc: idc, Host: strconv.Itoa(id)},
+			DiskHeartBeatInfo: clustermgr.DiskHeartBeatInfo{DiskID: proto.DiskID(id)},
 		}
 	}
 	for _, id := range idcOtherID {
-		dataDisks[proto.DiskID(id)] = blobnode.DiskInfo{
-			ClusterID: clusterID, Idc: idcOther, Host: strconv.Itoa(id),
-			DiskHeartBeatInfo: blobnode.DiskHeartBeatInfo{DiskID: proto.DiskID(id)},
+		dataDisks[proto.DiskID(id)] = clustermgr.BlobNodeDiskInfo{
+			DiskInfo:          clustermgr.DiskInfo{ClusterID: clusterID, Idc: idcOther, Host: strconv.Itoa(id)},
+			DiskHeartBeatInfo: clustermgr.DiskHeartBeatInfo{DiskID: proto.DiskID(id)},
 		}
 	}
 
@@ -357,7 +401,7 @@ func initMockData() {
 	proxycli.EXPECT().GetCacheVolume(gomock.Any(), gomock.Any(), gomock.Any()).
 		AnyTimes().Return(dataVolume, nil)
 	proxycli.EXPECT().GetCacheDisk(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
-		func(_ context.Context, _ string, args *proxy.CacheDiskArgs) (*blobnode.DiskInfo, error) {
+		func(_ context.Context, _ string, args *proxy.CacheDiskArgs) (*clustermgr.BlobNodeDiskInfo, error) {
 			if val, ok := dataDisks[args.DiskID]; ok {
 				return &val, nil
 			}
@@ -366,17 +410,26 @@ func initMockData() {
 
 	serviceController, _ = controller.NewServiceController(
 		controller.ServiceConfig{
-			ClusterID: clusterID,
-			IDC:       idc,
-			ReloadSec: 1000,
+			ClusterID:         clusterID,
+			IDC:               idc,
+			ServiceReloadSecs: 1000,
 		}, cmcli, proxycli, nil)
-	volumeGetter, _ = controller.NewVolumeGetter(clusterID, serviceController, proxycli, 0)
+	volumeGetter, _ = controller.NewVolumeGetter(controller.VolumeConfig{
+		ClusterID:                  clusterID,
+		VolumeMemcacheExpirationMs: -1,
+	}, serviceController, proxycli, nil)
 
 	ctr = gomock.NewController(&testing.T{})
 	c := NewMockClusterController(ctr)
 	c.EXPECT().Region().AnyTimes().Return("test-region")
 	c.EXPECT().ChooseOne().AnyTimes().Return(clusterInfo, nil)
-	c.EXPECT().GetServiceController(gomock.Any()).AnyTimes().Return(serviceController, nil)
+	c.EXPECT().GetServiceController(gomock.Any()).AnyTimes().DoAndReturn(
+		func(needClusterID proto.ClusterID) (controller.ServiceController, error) {
+			if needClusterID != clusterID {
+				return nil, fmt.Errorf("no service controller of %d", needClusterID)
+			}
+			return serviceController, nil
+		})
 	c.EXPECT().GetVolumeGetter(gomock.Any()).AnyTimes().Return(volumeGetter, nil)
 	c.EXPECT().ChangeChooseAlg(gomock.Any()).AnyTimes().DoAndReturn(
 		func(alg controller.AlgChoose) error {
@@ -393,6 +446,11 @@ func initMockData() {
 	allocCli.EXPECT().SendShardRepairMsg(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
 	allocCli.EXPECT().VolumeAlloc(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(
 		func(ctx context.Context, host string, args *proxy.AllocVolsArgs) ([]proxy.AllocRet, error) {
+			select {
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			default:
+			}
 			if args.Fsize > allocTimeoutSize {
 				return nil, errAllocTimeout
 			}
@@ -466,8 +524,10 @@ func initEC() {
 
 func initController() {
 	vuidController = &vuidControl{
-		broken:  make(map[proto.Vuid]bool),
-		blocked: make(map[proto.Vuid]bool),
+		broken:   make(map[proto.Vuid]bool),
+		blocked:  make(map[proto.Vuid]bool),
+		slowdown: make(map[proto.Vuid]time.Duration),
+		crc:      make(map[proto.Vuid]bool),
 		block: func() {
 			time.Sleep(200 * time.Millisecond)
 		},
@@ -503,6 +563,8 @@ func init() {
 	initMockData()
 	initController()
 
+	hystrix.ConfigureCommand(allocCommand, hystrix.CommandConfig{Timeout: defaultAllocatorTimeout})
+
 	streamer = &Handler{
 		memPool:           memPool,
 		encoder:           encoder,
@@ -521,6 +583,11 @@ func init() {
 			AllocRetryTimes:        3,
 			AllocRetryIntervalMS:   3000,
 			MinReadShardsX:         minReadShardsX,
+			ReadDataOnlyTimeoutMS:  10000,
+			ShardCrcReadEnable:     true,
+			LogSlowBaseTimeMS:      10,
+			LogSlowBaseSpeedKB:     1 << 10,
+			LogSlowTimeFator:       1.3,
 		},
 		discardVidChan: make(chan discardVid, 8),
 		stopCh:         make(chan struct{}),

blobstore/access/stream/stream_put.go

@@ -12,7 +12,7 @@
 // implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package access
+package stream
 
 import (
 	"bytes"
@@ -28,23 +28,24 @@ import (
 
 	"github.com/cubefs/cubefs/blobstore/api/access"
 	"github.com/cubefs/cubefs/blobstore/api/blobnode"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
 	"github.com/cubefs/cubefs/blobstore/common/ec"
 	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
 	"github.com/cubefs/cubefs/blobstore/common/rpc"
 	"github.com/cubefs/cubefs/blobstore/common/trace"
 	"github.com/cubefs/cubefs/blobstore/util/errors"
 	"github.com/cubefs/cubefs/blobstore/util/retry"
 )
 
-// TODO: To Be Continue
-//  put empty shard to blobnode if file has been aligned.
-
 // Put put one object
 //
 //	required: size, file size
 //	optional: hasher map to calculate hash.Hash
-func (h *Handler) Put(ctx context.Context, rc io.Reader, size int64,
-	hasherMap access.HasherMap) (*access.Location, error) {
+func (h *Handler) Put(ctx context.Context,
+	rc io.Reader, size int64, hasherMap access.HasherMap,
+	assignClusterID proto.ClusterID, codeMode codemode.CodeMode,
+) (*proto.Location, error) {
 	span := trace.SpanFromContextSafe(ctx)
 	span.Debugf("put request size:%d hashes:b(%b)", size, hasherMap.ToHashAlgorithm())
 
@@ -62,11 +63,20 @@ func (h *Handler) Put(ctx context.Context, rc io.Reader, size int64,
 	}
 
 	// 2.choose cluster and alloc volume from allocator
-	selectedCodeMode := h.allCodeModes.SelectCodeMode(size)
-	span.Debugf("select codemode %d", selectedCodeMode)
+	selectedCodeMode := codeMode
+	if selectedCodeMode == codemode.CodeModeNone {
+		selectedCodeMode = h.allCodeModes.SelectCodeMode(size)
+	} else {
+		valid := h.allCodeModes.VerifySelectCodeMode(selectedCodeMode)
+		if !valid {
+			span.Errorf("specify codemode %d not found in codemode policy", selectedCodeMode)
+			return nil, errcode.ErrIllegalArguments
+		}
+	}
+	span.Debugf("select codemode %d, specify codemode %d", selectedCodeMode, codeMode)
 
 	blobSize := atomic.LoadUint32(&h.MaxBlobSize)
-	clusterID, blobs, err := h.allocFromAllocatorWithHystrix(ctx, selectedCodeMode, uint64(size), blobSize, 0)
+	clusterID, blobs, err := h.allocFromAllocatorWithHystrix(ctx, selectedCodeMode, uint64(size), blobSize, assignClusterID)
 	if err != nil {
 		span.Error("alloc failed", errors.Detail(err))
 		return nil, err
@@ -75,19 +85,20 @@ func (h *Handler) Put(ctx context.Context, rc io.Reader, size int64,
 
 	// 3.read body and split, alloc from mem pool;ec encode and put into data node
 	limitReader := io.LimitReader(rc, int64(size))
-	location := &access.Location{
+	location := &proto.Location{
 		ClusterID: clusterID,
 		CodeMode:  selectedCodeMode,
-		Size:      uint64(size),
-		BlobSize:  blobSize,
-		Blobs:     blobs,
+		Size_:     uint64(size),
+		SliceSize: blobSize,
+		Slices:    blobs,
 	}
 
 	uploadSucc := false
 	defer func() {
 		if !uploadSucc {
 			span.Infof("put failed clean location %+v", location)
-			if err := h.clearGarbage(ctx, location); err != nil {
+			_, newCtx := trace.StartSpanFromContextWithTraceID(context.Background(), "", span.TraceID())
+			if err := h.clearGarbage(newCtx, location); err != nil {
 				span.Warn(errors.Detail(err))
 			}
 		}
@@ -99,6 +110,7 @@ func (h *Handler) Put(ctx context.Context, rc io.Reader, size int64,
 		// release ec buffer which have not takeover
 		buffer.Release()
 		span.AppendRPCTrackLog([]string{putTime.String()})
+		putTime.Report(clusterID.ToString(), h.IDC, true)
 	}()
 
 	// concurrent buffer in per request
@@ -121,6 +133,7 @@ func (h *Handler) Put(ctx context.Context, rc io.Reader, size int64,
 		if err != nil {
 			return nil, err
 		}
+		empties := emptyDataShardIndexes(buffer.BufferSizes)
 
 		readBuff := buffer.DataBuf[:bsize]
 		shards, err := encoder.Split(buffer.ECDataBuf)
@@ -153,7 +166,7 @@ func (h *Handler) Put(ctx context.Context, rc io.Reader, size int64,
 		buffer = nil
 		<-ready
 		startWrite := time.Now()
-		err = h.writeToBlobnodesWithHystrix(ctx, blobident, shards, func() {
+		err = h.writeToBlobnodesWithHystrix(ctx, blobident, shards, empties, func() {
 			takeoverBuffer.Release()
 			ready <- struct{}{}
 		})
@@ -168,11 +181,12 @@ func (h *Handler) Put(ctx context.Context, rc io.Reader, size int64,
 }
 
 func (h *Handler) writeToBlobnodesWithHystrix(ctx context.Context,
-	blob blobIdent, shards [][]byte, callback func()) error {
+	blob blobIdent, shards [][]byte, empties map[int]struct{}, callback func(),
+) error {
 	safe := make(chan struct{}, 1)
 	err := hystrix.Do(rwCommand, func() error {
 		safe <- struct{}{}
-		return h.writeToBlobnodes(ctx, blob, shards, callback)
+		return h.writeToBlobnodes(ctx, blob, shards, empties, callback)
 	}, nil)
 
 	select {
@@ -192,8 +206,8 @@ type shardPutStatus struct {
 // takeover ec buffer release by callback.
 // return if had quorum successful shards, then wait all shards in background.
 func (h *Handler) writeToBlobnodes(ctx context.Context,
-	blob blobIdent, shards [][]byte, callback func()) (err error) {
-	span := trace.SpanFromContextSafe(ctx)
+	blob blobIdent, shards [][]byte, empties map[int]struct{}, callback func(),
+) (err error) {
 	clusterID, vid, bid := blob.cid, blob.vid, blob.bid
 
 	wg := &sync.WaitGroup{}
@@ -221,6 +235,14 @@ func (h *Handler) writeToBlobnodes(ctx context.Context,
 		putQuorum = uint32(num)
 	}
 
+	span := trace.SpanFromContextSafe(ctx)
+	// new context span to write blobnode in background
+	span, ctx = trace.StartSpanFromContextWithTraceID(context.Background(), "", span.TraceID())
+	defer span.Finish()
+
+	writeStart := time.Now()
+	writeTime := int32(0)
+
 	// writtenNum ONLY apply on data and partiy shards
 	// TODO: count N and M in each AZ,
 	//    decision ec data is recoverable or not.
@@ -238,28 +260,27 @@ func (h *Handler) writeToBlobnodes(ctx context.Context,
 				wg.Done()
 			}()
 
+			_, empty := empties[index]
+
 			diskID := unit.DiskID
 			args := &blobnode.PutShardArgs{
 				DiskID: diskID,
 				Vuid:   unit.Vuid,
 				Bid:    bid,
 				Size:   int64(len(shards[index])),
-				Type:   blobnode.NormalIO,
+				Type:   blobnode.WriteIO,
+
+				NopData: empty,
 			}
 
-			crcDisabled := h.ShardCrcDisabled
+			crcDisable := h.ShardCrcWriteDisable
 			var crcOrigin uint32
-			if !crcDisabled {
+			if !crcDisable {
 				crcOrigin = crc32.ChecksumIEEE(shards[index])
 			}
 
-			// new child span to write to blobnode, we should finish it here.
-			spanChild, ctxChild := trace.StartSpanFromContextWithTraceID(
-				context.Background(), "WriteToBlobnode", span.TraceID())
-			defer spanChild.Finish()
-
 		RETRY:
-			hostInfo, err := serviceController.GetDiskHost(ctxChild, diskID)
+			hostInfo, err := serviceController.GetDiskHost(ctx, diskID)
 			if err != nil {
 				span.Error("get disk host failed", errors.Detail(err))
 				return
@@ -278,14 +299,28 @@ func (h *Handler) writeToBlobnodes(ctx context.Context,
 				crc       uint32
 			)
 			writeErr = retry.ExponentialBackoff(3, 200).RuptOn(func() (bool, error) {
-				args.Body = bytes.NewReader(shards[index])
+				if !args.NopData {
+					args.Body = bytes.NewReader(shards[index])
+				}
 
-				crc, err = h.blobnodeClient.PutShard(ctxChild, host, args)
+				crc, err = h.blobnodeClient.PutShard(ctx, host, args)
 				if err == nil {
-					if !crcDisabled && crc != crcOrigin {
+					if !crcDisable && crc != crcOrigin {
 						return false, fmt.Errorf("crc mismatch 0x%x != 0x%x", crc, crcOrigin)
 					}
 
+					// slow disk if speed lower and time greater than most shards, also retry.
+					if mostTime := atomic.LoadInt32(&writeTime); mostTime > 0 {
+						shardTime := time.Since(writeStart)
+						shardSpeed := float32(args.Size) / (float32(shardTime) / 1e9) / (1 << 10)
+						if int(shardTime/1e6) > h.LogSlowBaseTimeMS &&
+							shardSpeed < float32(h.LogSlowBaseSpeedKB) &&
+							float32(shardTime) > h.LogSlowTimeFator*float32(mostTime) {
+							span.Warnf("slow disk(host:%s diskid:%d) time(most:%dms shard:%dms) speed:%.2fKB/s",
+								host, diskID, mostTime/1e6, shardTime/1e6, shardSpeed)
+						}
+					}
+
 					needRetry = false
 					return true, nil
 				}
@@ -304,7 +339,7 @@ func (h *Handler) writeToBlobnodes(ctx context.Context,
 
 				switch code {
 				// EIO and Readonly error, then we need to punish disk in local and no necessary to retry
-				case errcode.CodeDiskBroken, errcode.CodeVUIDReadonly:
+				case errcode.CodeVUIDReadonly:
 					h.punishVolume(ctx, clusterID, vid, host, "BrokenOrRO")
 					h.punishDisk(ctx, clusterID, diskID, host, "BrokenOrRO")
 					span.Warnf("punish disk:%d volume:%d cos:blobnode/%d", diskID, vid, code)
@@ -316,9 +351,10 @@ func (h *Handler) writeToBlobnodes(ctx context.Context,
 					span.Warnf("punish volume:%d cos:blobnode/%d", vid, code)
 					return true, err
 
+				// disk broken may be some chunks have repaired
 				// vuid not found means the reflection between vuid and diskID has change, we should refresh the volume
 				// disk not found means disk has been repaired or offline
-				case errcode.CodeDiskNotFound, errcode.CodeVuidNotFound:
+				case errcode.CodeDiskBroken, errcode.CodeDiskNotFound, errcode.CodeVuidNotFound:
 					latestVolume, e := h.getVolume(ctx, clusterID, vid, false)
 					if e != nil {
 						return true, errors.Base(err, "get volume with no cache failed").Detail(e)
@@ -335,8 +371,12 @@ func (h *Handler) writeToBlobnodes(ctx context.Context,
 						return true, err
 					}
 
-					h.punishVolume(ctx, clusterID, vid, host, "NotFound")
-					h.punishDisk(ctx, clusterID, diskID, host, "NotFound")
+					reason := "NotFound"
+					if code == errcode.CodeDiskBroken {
+						reason = "Broken"
+					}
+					h.punishVolume(ctx, clusterID, vid, host, reason)
+					h.punishDisk(ctx, clusterID, diskID, host, reason)
 					span.Warnf("punish disk:%d volume:%d cos:blobnode/%d", diskID, vid, code)
 					return true, err
 				default:
@@ -344,8 +384,9 @@ func (h *Handler) writeToBlobnodes(ctx context.Context,
 
 				// in timeout case and writtenNum is not satisfied with putQuorum, then should retry
 				if errorTimeout(err) && atomic.LoadUint32(&writtenNum) < putQuorum {
+					h.updateVolume(ctx, clusterID, vid)
 					h.punishDiskWith(ctx, clusterID, diskID, host, "Timeout")
-					span.Warn("connect timeout, need to punish threshold disk", diskID, host)
+					span.Warn("timeout need to punish threshold disk", diskID, host)
 					return false, err
 				}
 
@@ -373,6 +414,11 @@ func (h *Handler) writeToBlobnodes(ctx context.Context,
 	for len(received) < len(volume.Units) && atomic.LoadUint32(&writtenNum) < putQuorum {
 		st := <-statusCh
 		received[st.index] = st
+
+		// trace slow disk after written 3/4 shards
+		if atomic.LoadInt32(&writeTime) == 0 && len(received) > len(volume.Units)*3/4 {
+			atomic.StoreInt32(&writeTime, int32(time.Since(writeStart)))
+		}
 	}
 
 	writeDone := make(chan struct{}, 1)

blobstore/access/stream/stream_put_test.go

@@ -12,7 +12,7 @@
 // implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package access
+package stream
 
 import (
 	"bytes"
@@ -28,6 +28,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/cubefs/cubefs/blobstore/api/access"
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
 	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
 	"github.com/cubefs/cubefs/blobstore/common/proto"
 )
@@ -42,41 +43,41 @@ func TestAccessStreamPutBase(t *testing.T) {
 	// 0
 	{
 		size := 0
-		_, err := streamer.Put(ctx(), newReader(size), int64(size), nil)
+		_, err := streamer.Put(ctx(), newReader(size), int64(size), nil, proto.ClusterID(0), codemode.CodeModeNone)
 		require.Error(t, err)
 	}
 	// 1 byte
 	{
 		size := 1
-		loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil)
+		loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil, proto.ClusterID(0), codemode.CodeModeNone)
 		require.NoError(t, err)
-		require.Equal(t, 1, len(loc.Blobs))
-		require.Equal(t, uint32(1), loc.Blobs[0].Count)
+		require.Equal(t, 1, len(loc.Slices))
+		require.Equal(t, uint32(1), loc.Slices[0].Count)
 		// time wait the punished services
 		time.Sleep(time.Second * time.Duration(punishServiceS))
 	}
 	// <4M
 	{
 		size := 1 << 18
-		loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil)
+		loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil, proto.ClusterID(0), codemode.CodeModeNone)
 		require.NoError(t, err)
-		require.Equal(t, 1, len(loc.Blobs))
-		require.Equal(t, uint32(1), loc.Blobs[0].Count)
+		require.Equal(t, 1, len(loc.Slices))
+		require.Equal(t, uint32(1), loc.Slices[0].Count)
 		time.Sleep(time.Second * time.Duration(punishServiceS))
 	}
 	// 8M + 1k
 	{
 		size := (1 << 23) + 1024
-		loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil)
+		loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil, proto.ClusterID(0), codemode.CodeModeNone)
 		require.NoError(t, err)
-		require.Equal(t, 2, len(loc.Blobs))
-		require.Equal(t, uint32(2), loc.Blobs[1].Count)
+		require.Equal(t, 2, len(loc.Slices))
+		require.Equal(t, uint32(2), loc.Slices[1].Count)
 		time.Sleep(time.Second * time.Duration(punishServiceS))
 	}
 	// max size + 1
 	{
 		size := defaultMaxObjectSize + 1
-		_, err := streamer.Put(ctx(), nil, int64(size), nil)
+		_, err := streamer.Put(ctx(), nil, int64(size), nil, proto.ClusterID(0), codemode.CodeModeNone)
 		require.EqualError(t, errcode.ErrAccessExceedSize, err.Error())
 	}
 
@@ -97,7 +98,7 @@ func TestAccessStreamPutSum(t *testing.T) {
 		}
 		hashSumMap := make(access.HashSumMap, len(hasherMap))
 
-		_, err := streamer.Put(ctx(), bytes.NewReader(data), int64(len(data)), hasherMap)
+		_, err := streamer.Put(ctx(), bytes.NewReader(data), int64(len(data)), hasherMap, proto.ClusterID(0), codemode.CodeModeNone)
 		require.NoError(t, err)
 		for alg, hasher := range hasherMap {
 			hashSumMap[alg] = hasher.Sum(nil)
@@ -231,7 +232,7 @@ func TestAccessStreamPutShardTimeout(t *testing.T) {
 	buff := make([]byte, size)
 	rand.Read(buff)
 	startTime := time.Now()
-	loc, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil)
+	loc, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil, proto.ClusterID(0), codemode.CodeModeNone)
 	require.NoError(t, err)
 
 	// response immediately if had quorum shards
@@ -247,7 +248,7 @@ func TestAccessStreamPutShardTimeout(t *testing.T) {
 	vuidController.Block(1002)
 	{
 		startTime := time.Now()
-		_, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil)
+		_, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil, proto.ClusterID(0), codemode.CodeModeNone)
 		require.Error(t, err)
 
 		duration := time.Since(startTime)
@@ -259,6 +260,25 @@ func TestAccessStreamPutShardTimeout(t *testing.T) {
 	}
 }
 
+func TestAccessStreamPutShardSlow(t *testing.T) {
+	ctx := ctxWithName("TestAccessStreamPutShardSlow")
+	dataShards.clean()
+	vuidController.SetSlowdown(1001, time.Second)
+	vuidController.Unbreak(1005)
+	defer func() {
+		vuidController.SetSlowdown(1001, -1)
+		vuidController.Break(1005)
+		dataShards.clean()
+	}()
+
+	size := 3
+	buff := make([]byte, size)
+	rand.Read(buff)
+	_, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil, proto.ClusterID(0), codemode.CodeModeNone)
+	require.NoError(t, err)
+	time.Sleep(time.Second)
+}
+
 func TestAccessStreamPutQuorum(t *testing.T) {
 	ctx := ctxWithName("TestAccessStreamPutQuorum")
 	defer func() {
@@ -299,7 +319,7 @@ func TestAccessStreamPutQuorum(t *testing.T) {
 			vuidController.Break(id)
 		}
 
-		_, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil)
+		_, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil, proto.ClusterID(0), codemode.CodeModeNone)
 		if cs.hasError {
 			require.NotNil(t, err)
 		} else {
@@ -313,6 +333,49 @@ func TestAccessStreamPutQuorum(t *testing.T) {
 	}
 }
 
+func TestAccessStreamPutWithClusterIDAndCodeMode(t *testing.T) {
+	ctx := ctxWithName("TestAccessStreamPutWithClusterIDAndCodeMode")
+	{
+		size := 1 << 22
+		_, err := streamer.Put(ctx(), newReader(size), int64(size), nil, proto.ClusterID(0), codemode.EC16P20L2)
+		require.Error(t, err)
+	}
+	{
+		size := 1 << 22
+		_, err := streamer.Put(ctx(), newReader(size), int64(size), nil, proto.ClusterID(2), codemode.CodeModeNone)
+		require.Error(t, err)
+	}
+	{
+		size := 1 << 22
+		_, err := streamer.Put(ctx(), newReader(size), int64(size), nil, proto.ClusterID(2), codemode.EC15P12)
+		require.Error(t, err)
+	}
+	{
+		size := 1 << 20
+		loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil, proto.ClusterID(0), codemode.EC6P6)
+		require.NoError(t, err)
+		require.Equal(t, codemode.EC6P6, loc.CodeMode)
+		require.Equal(t, 1, len(loc.Slices))
+		require.Equal(t, uint32(1), loc.Slices[0].Count)
+	}
+	{
+		size := 1 << 22
+		loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil, proto.ClusterID(1), codemode.EC6P6)
+		require.NoError(t, err)
+		require.Equal(t, codemode.EC6P6, loc.CodeMode)
+		require.Equal(t, 1, len(loc.Slices))
+		require.Equal(t, uint32(1), loc.Slices[0].Count)
+	}
+	{
+		size := 1 << 22
+		loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil, proto.ClusterID(1), codemode.CodeModeNone)
+		require.NoError(t, err)
+		require.Equal(t, codemode.EC6P6, loc.CodeMode)
+		require.Equal(t, 1, len(loc.Slices))
+		require.Equal(t, uint32(1), loc.Slices[0].Count)
+	}
+}
+
 func BenchmarkAccessStreamPut(b *testing.B) {
 	ctx := ctxWithName("BenchmarkAccessStreamPut")()
 	vuidController.Unbreak(1005)
@@ -338,7 +401,7 @@ func BenchmarkAccessStreamPut(b *testing.B) {
 		b.ResetTimer()
 		b.Run(cs.name, func(b *testing.B) {
 			for ii := 0; ii <= b.N; ii++ {
-				streamer.Put(ctx, bytes.NewReader(buff[:cs.size]), int64(cs.size), nil)
+				streamer.Put(ctx, bytes.NewReader(buff[:cs.size]), int64(cs.size), nil, proto.ClusterID(0), codemode.CodeModeNone)
 			}
 		})
 	}

blobstore/access/stream/stream_putat.go

@@ -12,7 +12,7 @@
 // implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package access
+package stream
 
 import (
 	"context"
@@ -33,8 +33,9 @@ import (
 //	required: size, one blob size
 //	optional: hasherMap, computing hash
 func (h *Handler) PutAt(ctx context.Context, rc io.Reader,
-	clusterID proto.ClusterID, vid proto.Vid, bid proto.BlobID, size int64,
-	hasherMap access.HasherMap) error {
+	clusterID proto.ClusterID, vid proto.Vid, bid proto.BlobID,
+	size int64, hasherMap access.HasherMap,
+) error {
 	span := trace.SpanFromContextSafe(ctx)
 	span.Debugf("putat request cluster:%d vid:%d bid:%d size:%d hashes:b(%b)",
 		clusterID, vid, bid, size, hasherMap.ToHashAlgorithm())
@@ -55,12 +56,14 @@ func (h *Handler) PutAt(ctx context.Context, rc io.Reader,
 	if err != nil {
 		return err
 	}
+	empties := emptyDataShardIndexes(buffer.BufferSizes)
 
 	putTime := new(timeReadWrite)
 	putTime.IncA(time.Since(st))
 	defer func() {
 		buffer.Release()
 		span.AppendRPCTrackLog([]string{putTime.String()})
+		putTime.Report(clusterID.ToString(), h.IDC, true)
 	}()
 
 	shards, err := encoder.Split(buffer.ECDataBuf)
@@ -90,7 +93,7 @@ func (h *Handler) PutAt(ctx context.Context, rc io.Reader,
 	takeoverBuffer := buffer
 	buffer = nil
 	startWrite := time.Now()
-	err = h.writeToBlobnodesWithHystrix(ctx, blobident, shards, func() {
+	err = h.writeToBlobnodesWithHystrix(ctx, blobident, shards, empties, func() {
 		takeoverBuffer.Release()
 	})
 	putTime.IncW(time.Since(startWrite))

blobstore/access/stream/stream_putat_test.go

@@ -12,7 +12,7 @@
 // implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package access
+package stream
 
 import (
 	"bytes"

blobstore/access/stream/stream_test.go

@@ -12,7 +12,7 @@
 // implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package access
+package stream
 
 import (
 	"bytes"
@@ -24,6 +24,7 @@ import (
 
 	"github.com/cubefs/cubefs/blobstore/access/controller"
 	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
 )
 
 func newReader(size int) io.Reader {
@@ -44,25 +45,42 @@ func TestAccessStreamConfig(t *testing.T) {
 			ConsulAgentAddr: "http://127.0.0.1:8500",
 		},
 	}
-	confCheck(&cfg)
+	err := confCheck(&cfg)
 
+	require.NoError(t, err)
 	require.Equal(t, idcOther, cfg.IDC)
 	require.Equal(t, map[int]int{1024: 1}, cfg.MemPoolSizeClasses)
 	require.Equal(t, defaultDiskPunishIntervalS, cfg.DiskPunishIntervalS)
+
+	cfg = StreamConfig{
+		IDC:                idcOther,
+		MemPoolSizeClasses: map[int]int{1024: 1},
+		CodeModesPutQuorums: map[codemode.CodeMode]int{
+			codemode.EC15P12:  16,
+			codemode.EC6P10L2: 18,
+		},
+		ClusterConfig: controller.ClusterConfig{
+			Clusters: []controller.Cluster{
+				{ClusterID: 1, Hosts: []string{"host1"}},
+				{ClusterID: 2, Hosts: []string{"host2"}},
+			},
+		},
+	}
+	err = confCheck(&cfg)
+	require.NoError(t, err)
 }
 
 func TestAccessStreamNew(t *testing.T) {
 	require.Equal(t, idc, streamer.IDC)
 
-	require.Panics(t, func() {
-		NewStreamHandler(&StreamConfig{IDC: "idc"}, nil)
-	})
+	_, err := NewStreamHandler(&StreamConfig{IDC: "idc"}, nil)
+	require.NotNil(t, err)
 }
 
 func TestAccessStreamDelete(t *testing.T) {
 	ctx := ctxWithName("TestAccessStreamDelete")
 	size := 1 << 18
-	loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil)
+	loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil, proto.ClusterID(0), codemode.CodeModeNone)
 	require.NoError(t, err)
 
 	err = streamer.Delete(ctx(), loc)
@@ -77,19 +95,19 @@ func TestAccessStreamAdmin(t *testing.T) {
 		sa := handler.Admin()
 		require.NotNil(t, sa)
 
-		admin := sa.(*streamAdmin)
-		require.Nil(t, admin.memPool)
-		require.Nil(t, admin.controller)
+		admin := sa.(*StreamAdmin)
+		require.Nil(t, admin.MemPool)
+		require.Nil(t, admin.Controller)
 	}
 	{
 		sa := streamer.Admin()
 		require.NotNil(t, sa)
 
-		admin := sa.(*streamAdmin)
-		require.NotNil(t, admin.memPool)
-		t.Log("mempool status:", admin.memPool.Status())
+		admin := sa.(*StreamAdmin)
+		require.NotNil(t, admin.MemPool)
+		t.Log("mempool status:", admin.MemPool.Status())
 
-		ctr := admin.controller
+		ctr := admin.Controller
 		require.NotNil(t, ctr)
 		t.Log("region:", ctr.Region())
 		require.Error(t, ctr.ChangeChooseAlg(controller.AlgChoose(100)))

blobstore/access/stream/stream_time.go

@@ -12,7 +12,7 @@
 // implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package access
+package stream
 
 import (
 	"fmt"
@@ -38,6 +38,16 @@ func (t *timeReadWrite) IncW(dur time.Duration) {
 	atomic.AddInt64(&t.w, int64(dur))
 }
 
+func (t *timeReadWrite) Report(cid, idc string, upload bool) {
+	if upload {
+		reportReadwrite(cid, idc, "upload_read", atomic.LoadInt64(&t.r)/1e6)
+		reportReadwrite(cid, idc, "upload_write", atomic.LoadInt64(&t.w)/1e6)
+	} else {
+		reportReadwrite(cid, idc, "download_read", atomic.LoadInt64(&t.r)/1e6)
+		reportReadwrite(cid, idc, "download_write", atomic.LoadInt64(&t.w)/1e6)
+	}
+}
+
 // String within milliseconds
 func (t *timeReadWrite) String() string {
 	a := atomic.LoadInt64(&t.a) / 1e6

blobstore/api/access/client.go

@@ -22,12 +22,13 @@ import (
 	"net/http"
 	"runtime"
 	"sort"
+	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/hashicorp/consul/api"
-	"gopkg.in/natefinch/lumberjack.v2"
 
+	"github.com/cubefs/cubefs/blobstore/api/shardnode"
 	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
 	"github.com/cubefs/cubefs/blobstore/common/proto"
 	"github.com/cubefs/cubefs/blobstore/common/resourcepool"
@@ -42,9 +43,7 @@ import (
 const (
 	defaultMaxSizePutOnce  int64 = 1 << 28 // 256MB
 	defaultMaxPartRetry    int   = 3
-	defaultMaxHostRetry    int   = 3
 	defaultPartConcurrence int   = 4
-	defaultServiceInterval int   = 3600 // one hour.
 	defaultServiceName           = "access"
 )
 
@@ -52,9 +51,7 @@ const (
 type RPCConnectMode uint8
 
 // timeout: [short - - - - - - - - -> long]
-//
-//	quick --> general --> default --> slow --> nolimit
-//
+// ----- quick --> general --> default --> slow --> nolimit
 // speed: 40MB -->  20MB   -->  10MB   --> 4MB  --> nolimit
 const (
 	DefaultConnMode RPCConnectMode = iota
@@ -91,9 +88,7 @@ func (mode RPCConnectMode) getConfig(speed float64, timeout, baseTimeout int64)
 		BodyBaseTimeoutMs: getBaseTimeout(30 * 1000),
 		Tc: rpc.TransportConfig{
 			// dial timeout
-			DialTimeoutMs: 5 * 1000,
-			// response header timeout after send the request
-			ResponseHeaderTimeoutMs: 5 * 1000,
+			DialTimeoutMs: 200,
 			// IdleConnTimeout is the maximum amount of time an idle
 			// (keep-alive) connection will remain idle before closing
 			// itself.Zero means no limit.
@@ -111,29 +106,21 @@ func (mode RPCConnectMode) getConfig(speed float64, timeout, baseTimeout int64)
 		config.ClientTimeoutMs = getTimeout(getSpeed(40))
 		config.BodyBandwidthMBPs = getSpeed(40)
 		config.BodyBaseTimeoutMs = getBaseTimeout(3 * 1000)
-		config.Tc.DialTimeoutMs = 2 * 1000
-		config.Tc.ResponseHeaderTimeoutMs = 2 * 1000
 		config.Tc.IdleConnTimeoutMs = 10 * 1000
 	case GeneralConnMode:
 		config.ClientTimeoutMs = getTimeout(getSpeed(20))
 		config.BodyBandwidthMBPs = getSpeed(20)
 		config.BodyBaseTimeoutMs = getBaseTimeout(10 * 1000)
-		config.Tc.DialTimeoutMs = 3 * 1000
-		config.Tc.ResponseHeaderTimeoutMs = 3 * 1000
 		config.Tc.IdleConnTimeoutMs = 30 * 1000
 	case SlowConnMode:
 		config.ClientTimeoutMs = getTimeout(getSpeed(4))
 		config.BodyBandwidthMBPs = getSpeed(4)
 		config.BodyBaseTimeoutMs = getBaseTimeout(120 * 1000)
-		config.Tc.DialTimeoutMs = 10 * 1000
-		config.Tc.ResponseHeaderTimeoutMs = 10 * 1000
 		config.Tc.IdleConnTimeoutMs = 60 * 1000
 	case NoLimitConnMode:
 		config.ClientTimeoutMs = 0
 		config.BodyBandwidthMBPs = getSpeed(0)
 		config.BodyBaseTimeoutMs = getBaseTimeout(0)
-		config.Tc.DialTimeoutMs = 0
-		config.Tc.ResponseHeaderTimeoutMs = 0
 		config.Tc.IdleConnTimeoutMs = 600 * 1000
 	default:
 	}
@@ -144,61 +131,62 @@ func (mode RPCConnectMode) getConfig(speed float64, timeout, baseTimeout int64)
 // Config access client config
 type Config struct {
 	// ConnMode rpc connection timeout setting
-	ConnMode RPCConnectMode
+	ConnMode RPCConnectMode `json:"connection_mode"`
 	// ClientTimeoutMs the whole request and response timeout
-	ClientTimeoutMs int64
+	ClientTimeoutMs int64 `json:"client_timeout_ms"`
 	// BodyBandwidthMBPs reading body timeout, request or response
 	//   timeout = ContentLength/BodyBandwidthMBPs + BodyBaseTimeoutMs
-	BodyBandwidthMBPs float64
+	BodyBandwidthMBPs float64 `json:"body_bandwidth_mbps"`
 	// BodyBaseTimeoutMs base timeout for read body
-	BodyBaseTimeoutMs int64
+	BodyBaseTimeoutMs int64 `json:"body_base_timeout_ms"`
 
 	// Consul is consul config for discovering service
-	Consul ConsulConfig
-	// ServiceIntervalS is interval seconds for discovering service
-	ServiceIntervalS int
+	Consul ConsulConfig `json:"consul"`
+	// ServiceIntervalS is interval seconds for discovering service hosts,
+	//   at least 5 seconds and default is 5 minutes.
+	ServiceIntervalS int `json:"service_interval_s"`
 	// PriorityAddrs priority addrs of access service when retry
-	PriorityAddrs []string
-	// MaxSizePutOnce max size using once-put object interface
-	MaxSizePutOnce int64
+	PriorityAddrs []string `json:"priority_addrs"`
+	// MaxSizePutOnce max size using once-put object interface, default is 256MB.
+	MaxSizePutOnce int64 `json:"max_size_put_once"`
 	// MaxPartRetry max retry times when putting one part, 0 means forever
-	MaxPartRetry int
-	// MaxHostRetry max retry hosts of access service
-	MaxHostRetry int
+	MaxPartRetry int `json:"max_part_retry"`
+	// MaxHostRetry max retry hosts of access, default all hosts.
+	MaxHostRetry int `json:"max_host_retry"`
 	// PartConcurrence concurrence of put parts
-	PartConcurrence int
+	PartConcurrence int `json:"part_concurrence"`
 
 	// rpc selector config
-	// Failure retry interval, default value is -1, if FailRetryIntervalS < 0,
-	// remove failed hosts will not work.
-	FailRetryIntervalS int
+	// Failure retry interval, default value is 300s,
+	// if FailRetryIntervalS < 0, remove failed hosts will not work.
+	FailRetryIntervalS int `json:"fail_retry_interval_s"`
 	// Within MaxFailsPeriodS, if the number of failures is greater than or equal to MaxFails,
 	// the host is considered disconnected.
-	MaxFailsPeriodS int
+	MaxFailsPeriodS int `json:"max_fails_period_s"`
 	// HostTryTimes Number of host failure retries
-	HostTryTimes int
+	HostTryTimes int `json:"host_try_times"`
 
 	// RPCConfig user-defined rpc config
 	// All connections will use the config if it's not nil
 	// ConnMode will be ignored if rpc config is setting
-	RPCConfig *rpc.Config
+	RPCConfig *rpc.Config `json:"rpc_config"`
 
 	// LogLevel client output logging level.
-	LogLevel log.Level
+	LogLevel log.Level `json:"log_level"`
 
 	// Logger trace all logging to the logger if setting.
 	// It is an io.WriteCloser that writes to the specified filename.
 	// YOU should CLOSE it after you do not use the client anymore.
-	Logger *Logger
+	Logger *Logger `json:"logger"`
 }
 
 // ConsulConfig alias of consul api.Config
 // Fixup: client and sdk using the same config type
 type ConsulConfig = api.Config
 
-// Logger alias of lumberjack Logger
+// Logger alias of AsyncLogger
 // See more at: https://github.com/natefinch/lumberjack
-type Logger = lumberjack.Logger
+type Logger = log.AsyncLogger
 
 // client access rpc client
 type client struct {
@@ -215,12 +203,24 @@ type API interface {
 	//
 	// If PutArgs' body is of type *bytes.Buffer, *bytes.Reader, or *strings.Reader,
 	// GetBody is populated, then the Put once request has retry ability.
-	Put(ctx context.Context, args *PutArgs) (location Location, hashSumMap HashSumMap, err error)
+	Put(ctx context.Context, args *PutArgs) (location proto.Location, hashSumMap HashSumMap, err error)
 	// Get object, range is supported.
 	Get(ctx context.Context, args *GetArgs) (body io.ReadCloser, err error)
 	// Delete all blobs in these locations.
-	// return failed locations which have yet been deleted if error is not nil.
-	Delete(ctx context.Context, args *DeleteArgs) (failedLocations []Location, err error)
+	//
+	// Returns:
+	// - (nil, nil): all blobs deleted successfully.
+	// - (nil, ErrIllegalArguments): when args is invalid.
+	// - (failedLocations, err): returns the list of locations that have not yet been deleted.
+	Delete(ctx context.Context, args *DeleteArgs) (failedLocations []proto.Location, err error)
+}
+
+type Client interface {
+	API
+	ListBlob(ctx context.Context, args *ListBlobArgs) (shardnode.ListBlobRet, error)
+	GetBlob(ctx context.Context, args *GetBlobArgs) (io.ReadCloser, error)
+	DeleteBlob(ctx context.Context, args *DelBlobArgs) error
+	PutBlob(ctx context.Context, args *PutBlobArgs) (proto.ClusterID, HashSumMap, error)
 }
 
 var _ API = (*client)(nil)
@@ -232,28 +232,42 @@ var _ io.ReadCloser = (*noopBody)(nil)
 func (rc noopBody) Read(p []byte) (n int, err error) { return 0, io.EOF }
 func (rc noopBody) Close() error                     { return nil }
 
-var memPool *resourcepool.MemPool
+var (
+	memPool  *resourcepool.MemPool
+	poolOnce sync.Once
+)
 
-func init() {
-	memPool = resourcepool.NewMemPool(map[int]int{
-		1 << 12: -1,
-		1 << 14: -1,
-		1 << 18: -1,
-		1 << 20: -1,
-		1 << 22: -1,
-		1 << 23: -1,
-		1 << 24: -1,
+func lazyInitSingletonMemPool() {
+	poolOnce.Do(func() {
+		if memPool == nil {
+			memPool = resourcepool.NewMemPool(map[int]int{
+				1 << 12: -1,
+				1 << 14: -1,
+				1 << 18: -1,
+				1 << 20: -1,
+				1 << 22: -1,
+				1 << 23: -1,
+				1 << 24: -1,
+			})
+		}
 	})
 }
 
+// ResetMemoryPool is thread unsafe, call it on init.
+func ResetMemoryPool(sizeClasses map[int]int) {
+	memPool = resourcepool.NewMemPool(sizeClasses)
+}
+
 // New returns an access API
 func New(cfg Config) (API, error) {
 	defaulter.LessOrEqual(&cfg.MaxSizePutOnce, defaultMaxSizePutOnce)
 	defaulter.Less(&cfg.MaxPartRetry, defaultMaxPartRetry)
-	defaulter.LessOrEqual(&cfg.MaxHostRetry, defaultMaxHostRetry)
 	defaulter.LessOrEqual(&cfg.PartConcurrence, defaultPartConcurrence)
-	if cfg.ServiceIntervalS < 300 { // at least 5 minutes
-		cfg.ServiceIntervalS = defaultServiceInterval
+	defaulter.Equal(&cfg.FailRetryIntervalS, 300)
+	defaulter.LessOrEqual(&cfg.MaxFailsPeriodS, 10)
+	defaulter.Equal(&cfg.ServiceIntervalS, 300) // 5 minutes
+	if cfg.ServiceIntervalS < 5 {
+		cfg.ServiceIntervalS = 5
 	}
 
 	log.SetOutputLevel(cfg.LogLevel)
@@ -261,6 +275,8 @@ func New(cfg Config) (API, error) {
 		log.SetOutput(cfg.Logger)
 	}
 
+	lazyInitSingletonMemPool()
+
 	c := &client{
 		config: cfg,
 		stop:   make(chan struct{}),
@@ -322,10 +338,11 @@ func New(cfg Config) (API, error) {
 	}
 	c.rpcClient.Store(getClient(&cfg, hosts))
 
-	ticker := time.NewTicker(time.Duration(cfg.ServiceIntervalS) * time.Second)
 	go func() {
+		ticker := time.NewTicker(time.Duration(cfg.ServiceIntervalS) * time.Second)
+		defer ticker.Stop()
 		for {
-			old := hosts
+			old := hosts[:]
 			select {
 			case <-ticker.C:
 				hosts, err = hostGetter()
@@ -338,10 +355,10 @@ func New(cfg Config) (API, error) {
 					if ok && oldClient != nil {
 						oldClient.Close()
 					}
+					log.Warnf("update hosts of client (%v) -> (%v)", old, hosts)
 					c.rpcClient.Store(getClient(&cfg, hosts))
 				}
 			case <-c.stop:
-				ticker.Stop()
 				return
 			}
 		}
@@ -387,13 +404,13 @@ func getClient(cfg *Config, hosts []string) rpc.Client {
 	return rpc.NewLbClient(lbConfig, nil)
 }
 
-func (c *client) Put(ctx context.Context, args *PutArgs) (location Location, hashSumMap HashSumMap, err error) {
+func (c *client) Put(ctx context.Context, args *PutArgs) (location proto.Location, hashSumMap HashSumMap, err error) {
 	if args.Size == 0 {
 		hashSumMap := args.Hashes.ToHashSumMap()
 		for alg := range hashSumMap {
 			hashSumMap[alg] = alg.ToHasher().Sum(nil)
 		}
-		return Location{Blobs: make([]SliceInfo, 0)}, hashSumMap, nil
+		return proto.Location{Slices: make([]proto.Slice, 0)}, hashSumMap, nil
 	}
 
 	ctx = withReqidContext(ctx)
@@ -403,14 +420,19 @@ func (c *client) Put(ctx context.Context, args *PutArgs) (location Location, has
 	return c.putParts(ctx, args)
 }
 
-func (c *client) putObject(ctx context.Context, args *PutArgs) (location Location, hashSumMap HashSumMap, err error) {
+func (c *client) putObject(ctx context.Context, args *PutArgs) (location proto.Location, hashSumMap HashSumMap, err error) {
 	rpcClient := c.rpcClient.Load().(rpc.Client)
 
-	urlStr := fmt.Sprintf("/put?size=%d&hashes=%d", args.Size, args.Hashes)
+	urlStr := fmt.Sprintf("/put?size=%d&hashes=%d&assign_cluster_id=%d&code_mode=%d",
+		args.Size, args.Hashes, args.AssignClusterID, args.CodeMode)
+
 	req, err := http.NewRequest(http.MethodPut, urlStr, args.Body)
 	if err != nil {
 		return
 	}
+	if args.GetBody != nil {
+		req.GetBody = args.GetBody
+	}
 
 	resp := &PutResp{}
 	if err = rpcClient.DoWith(ctx, req, resp, rpc.WithCrcEncode()); err == nil {
@@ -447,7 +469,8 @@ func (c *client) putPartsBatch(ctx context.Context, parts []blobPart) error {
 		})
 	}
 
-	if err := task.Run(context.Background(), tasks...); err != nil {
+	newCtx := trace.NewContextFromContext(ctx)
+	if err := task.Run(ctx, tasks...); err != nil {
 		for _, part := range parts {
 			part := part
 			// asynchronously delete blob
@@ -458,7 +481,7 @@ func (c *client) putPartsBatch(ctx context.Context, parts []blobPart) error {
 				if err != nil {
 					return
 				}
-				rpcClient.DoWith(ctx, req, nil)
+				rpcClient.DoWith(newCtx, req, nil)
 			}()
 		}
 		return err
@@ -467,7 +490,8 @@ func (c *client) putPartsBatch(ctx context.Context, parts []blobPart) error {
 }
 
 func (c *client) readerPipeline(span trace.Span, reqBody io.Reader,
-	closeCh <-chan struct{}, size, blobSize int) <-chan []byte {
+	closeCh <-chan struct{}, size, blobSize int,
+) <-chan []byte {
 	ch := make(chan []byte, c.config.PartConcurrence-1)
 	go func() {
 		for size > 0 {
@@ -501,7 +525,7 @@ func (c *client) readerPipeline(span trace.Span, reqBody io.Reader,
 	return ch
 }
 
-func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSumMap, error) {
+func (c *client) putParts(ctx context.Context, args *PutArgs) (proto.Location, HashSumMap, error) {
 	span := trace.SpanFromContextSafe(ctx)
 	rpcClient := c.rpcClient.Load().(rpc.Client)
 
@@ -517,7 +541,7 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
 	}
 
 	var (
-		loc    Location
+		loc    proto.Location
 		tokens []string
 	)
 
@@ -528,16 +552,18 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
 			return
 		}
 
+		// force to clean up, even canceled context
+		newCtx := trace.NewContextFromSpan(span)
 		locations := signArgs.Locations[:]
 		if len(locations) > 1 {
 			signArgs.Location = loc.Copy()
 			signResp := &SignResp{}
-			if err := rpcClient.PostWith(ctx, "/sign", signResp, signArgs); err == nil {
-				locations = []Location{signResp.Location.Copy()}
+			if err := rpcClient.PostWith(newCtx, "/sign", signResp, signArgs); err == nil {
+				locations = []proto.Location{signResp.Location.Copy()}
 			}
 		}
 		if len(locations) > 0 {
-			if _, err := c.Delete(ctx, &DeleteArgs{Locations: locations}); err != nil {
+			if _, err := c.Delete(newCtx, &DeleteArgs{Locations: locations}); err != nil {
 				span.Warnf("clean location '%+v' failed %s", locations, err.Error())
 			}
 		}
@@ -545,7 +571,12 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
 
 	// alloc
 	allocResp := &AllocResp{}
-	if err := rpcClient.PostWith(ctx, "/alloc", allocResp, AllocArgs{Size: uint64(args.Size)}); err != nil {
+	allocArgs := &AllocArgs{
+		Size:            uint64(args.Size),
+		AssignClusterID: args.AssignClusterID,
+		CodeMode:        args.CodeMode,
+	}
+	if err := rpcClient.PostWith(ctx, "/alloc", allocResp, allocArgs); err != nil {
 		return allocResp.Location, nil, err
 	}
 	loc = allocResp.Location
@@ -554,7 +585,7 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
 
 	// buffer pipeline
 	closeCh := make(chan struct{})
-	bufferPipe := c.readerPipeline(span, reqBody, closeCh, int(loc.Size), int(loc.BlobSize))
+	bufferPipe := c.readerPipeline(span, reqBody, closeCh, int(loc.Size_), int(loc.SliceSize))
 	defer func() {
 		close(closeCh)
 		// waiting pipeline close if has error
@@ -573,17 +604,17 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
 
 	currBlobIdx := 0
 	currBlobCount := uint32(0)
-	remainSize := loc.Size
+	remainSize := loc.Size_
 	restPartsLoc := loc
 
 	readSize := 0
-	for readSize < int(loc.Size) {
+	for readSize < int(loc.Size_) {
 		parts := make([]blobPart, 0, c.config.PartConcurrence)
 
 		// waiting at least one blob
 		buf, ok := <-bufferPipe
-		if !ok && readSize < int(loc.Size) {
-			return Location{}, nil, errcode.ErrAccessReadRequestBody
+		if !ok && readSize < int(loc.Size_) {
+			return proto.Location{}, nil, errcode.ErrAccessReadRequestBody
 		}
 		readSize += len(buf)
 		parts = append(parts, blobPart{size: len(buf), buf: buf})
@@ -593,9 +624,9 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
 			select {
 			case buf, ok := <-bufferPipe:
 				if !ok {
-					if readSize < int(loc.Size) {
+					if readSize < int(loc.Size_) {
 						releaseBuffer(parts)
-						return Location{}, nil, errcode.ErrAccessReadRequestBody
+						return proto.Location{}, nil, errcode.ErrAccessReadRequestBody
 					}
 					more = false
 				} else {
@@ -609,9 +640,9 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
 
 		tryTimes := c.config.MaxPartRetry
 		for {
-			if len(loc.Blobs) > MaxLocationBlobs {
+			if len(loc.Slices) > MaxLocationBlobs {
 				releaseBuffer(parts)
-				return Location{}, nil, errcode.ErrUnexpected
+				return proto.Location{}, nil, errcode.ErrUnexpected
 			}
 
 			// feed new params
@@ -619,16 +650,16 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
 			currCount := currBlobCount
 			for i := range parts {
 				token := tokens[currIdx]
-				if restPartsLoc.Size > uint64(loc.BlobSize) && parts[i].size < int(loc.BlobSize) {
-					token = tokens[currIdx+1]
+				if restPartsLoc.Size_ > uint64(loc.SliceSize) && parts[i].size < int(loc.SliceSize) {
+					token = tokens[len(tokens)-1]
 				}
 				parts[i].token = token
 				parts[i].cid = loc.ClusterID
-				parts[i].vid = loc.Blobs[currIdx].Vid
-				parts[i].bid = loc.Blobs[currIdx].MinBid + proto.BlobID(currCount)
+				parts[i].vid = loc.Slices[currIdx].Vid
+				parts[i].bid = loc.Slices[currIdx].MinSliceID + proto.BlobID(currCount)
 
 				currCount++
-				if loc.Blobs[currIdx].Count == currCount {
+				if loc.Slices[currIdx].Count == currCount {
 					currIdx++
 					currCount = 0
 				}
@@ -640,7 +671,7 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
 					remainSize -= uint64(part.size)
 					currBlobCount++
 					// next blobs
-					if loc.Blobs[currBlobIdx].Count == currBlobCount {
+					if loc.Slices[currBlobIdx].Count == currBlobCount {
 						currBlobIdx++
 						currBlobCount = 0
 					}
@@ -654,7 +685,7 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
 				if tryTimes == 1 {
 					releaseBuffer(parts)
 					span.Error("exceed the max retry limit", c.config.MaxPartRetry)
-					return Location{}, nil, errcode.ErrUnexpected
+					return proto.Location{}, nil, errcode.ErrUnexpected
 				}
 				tryTimes--
 			}
@@ -665,14 +696,14 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
 				resp := &AllocResp{}
 				if err := rpcClient.PostWith(ctx, "/alloc", resp, AllocArgs{
 					Size:            remainSize,
-					BlobSize:        loc.BlobSize,
+					BlobSize:        loc.SliceSize,
 					CodeMode:        loc.CodeMode,
 					AssignClusterID: loc.ClusterID,
 				}); err != nil {
 					return true, err
 				}
-				if len(resp.Location.Blobs) > 0 {
-					if newVid := resp.Location.Blobs[0].Vid; newVid == loc.Blobs[currBlobIdx].Vid {
+				if len(resp.Location.Slices) > 0 {
+					if newVid := resp.Location.Slices[0].Vid; newVid == loc.Slices[currBlobIdx].Vid {
 						return false, fmt.Errorf("alloc the same vid %d", newVid)
 					}
 				}
@@ -682,17 +713,17 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
 			if err != nil {
 				releaseBuffer(parts)
 				span.Error("alloc another parts to put", err)
-				return Location{}, nil, errcode.ErrUnexpected
+				return proto.Location{}, nil, err
 			}
 
 			restPartsLoc = restPartsResp.Location
 			signArgs.Locations = append(signArgs.Locations, restPartsLoc.Copy())
 
 			if currBlobCount > 0 {
-				loc.Blobs[currBlobIdx].Count = currBlobCount
+				loc.Slices[currBlobIdx].Count = currBlobCount
 				currBlobIdx++
 			}
-			loc.Blobs = append(loc.Blobs[:currBlobIdx], restPartsLoc.Blobs...)
+			loc.Slices = append(loc.Slices[:currBlobIdx], restPartsLoc.Slices...)
 			tokens = append(tokens[:currBlobIdx], restPartsResp.Tokens...)
 
 			currBlobCount = 0
@@ -707,7 +738,7 @@ func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSum
 		signResp := &SignResp{}
 		if err := rpcClient.PostWith(ctx, "/sign", signResp, signArgs); err != nil {
 			span.Error("sign location with crc", err)
-			return Location{}, nil, errcode.ErrUnexpected
+			return proto.Location{}, nil, err
 		}
 		loc = signResp.Location
 	}
@@ -726,7 +757,7 @@ func (c *client) Get(ctx context.Context, args *GetArgs) (body io.ReadCloser, er
 	rpcClient := c.rpcClient.Load().(rpc.Client)
 
 	ctx = withReqidContext(ctx)
-	if args.Location.Size == 0 || args.ReadSize == 0 {
+	if args.Location.Size_ == 0 || args.ReadSize == 0 {
 		return noopBody{}, nil
 	}
 
@@ -741,7 +772,7 @@ func (c *client) Get(ctx context.Context, args *GetArgs) (body io.ReadCloser, er
 	return resp.Body, nil
 }
 
-func (c *client) Delete(ctx context.Context, args *DeleteArgs) ([]Location, error) {
+func (c *client) Delete(ctx context.Context, args *DeleteArgs) ([]proto.Location, error) {
 	if !args.IsValid() {
 		if args == nil {
 			return nil, errcode.ErrIllegalArguments
@@ -751,9 +782,9 @@ func (c *client) Delete(ctx context.Context, args *DeleteArgs) ([]Location, erro
 	rpcClient := c.rpcClient.Load().(rpc.Client)
 
 	ctx = withReqidContext(ctx)
-	locations := make([]Location, 0, len(args.Locations))
+	locations := make([]proto.Location, 0, len(args.Locations))
 	for _, loc := range args.Locations {
-		if loc.Size > 0 {
+		if loc.Size_ > 0 {
 			locations = append(locations, loc.Copy())
 		}
 	}

blobstore/api/access/client_reqid.go

@@ -31,6 +31,8 @@ const (
 	reqidKey
 )
 
+var ClientWithReqidContext = withReqidContext
+
 // WithRequestID trace request id in full life of the request
 // The second parameter rid could be the one of type below:
 //

blobstore/api/access/client_test.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"hash/crc32"
 	"io"
-	"io/ioutil"
 	mrand "math/rand"
 	"net/http"
 	"net/http/httptest"
@@ -39,8 +38,8 @@ import (
 	errcode "github.com/cubefs/cubefs/blobstore/common/errors"
 	"github.com/cubefs/cubefs/blobstore/common/proto"
 	"github.com/cubefs/cubefs/blobstore/common/rpc"
+	"github.com/cubefs/cubefs/blobstore/common/security"
 	"github.com/cubefs/cubefs/blobstore/common/trace"
-	"github.com/cubefs/cubefs/blobstore/common/uptoken"
 	_ "github.com/cubefs/cubefs/blobstore/testing/nolog"
 	"github.com/cubefs/cubefs/blobstore/util/bytespool"
 	"github.com/cubefs/cubefs/blobstore/util/log"
@@ -158,60 +157,60 @@ func handleAlloc(c *rpc.Context) {
 		return
 	}
 
-	loc := access.Location{
+	loc := proto.Location{
 		ClusterID: 1,
-		Size:      args.Size,
-		BlobSize:  blobSize,
-		Blobs: []access.SliceInfo{
+		Size_:     args.Size,
+		SliceSize: blobSize,
+		Slices: []proto.Slice{
 			{
-				MinBid: proto.BlobID(mrand.Int()),
-				Vid:    proto.Vid(mrand.Int()),
-				Count:  uint32((args.Size + blobSize - 1) / blobSize),
+				MinSliceID: proto.BlobID(mrand.Int()),
+				Vid:        proto.Vid(mrand.Int()),
+				Count:      uint32((args.Size + blobSize - 1) / blobSize),
 			},
 		},
 	}
 	// split to two blobs if large enough
-	if loc.Blobs[0].Count > 2 {
-		loc.Blobs[0].Count = 2
-		loc.Blobs = append(loc.Blobs, []access.SliceInfo{
+	if loc.Slices[0].Count > 2 {
+		loc.Slices[0].Count = 2
+		loc.Slices = append(loc.Slices, []proto.Slice{
 			{
-				MinBid: proto.BlobID(mrand.Int()),
-				Vid:    proto.Vid(mrand.Int()),
-				Count:  uint32((args.Size - 2*blobSize + blobSize - 1) / blobSize),
+				MinSliceID: proto.BlobID(mrand.Int()),
+				Vid:        proto.Vid(mrand.Int()),
+				Count:      uint32((args.Size - 2*blobSize + blobSize - 1) / blobSize),
 			},
 		}...)
 	}
 	// alloc the rest parts
 	if args.AssignClusterID > 0 {
-		loc.Blobs = []access.SliceInfo{
+		loc.Slices = []proto.Slice{
 			{
-				MinBid: proto.BlobID(mrand.Int()),
-				Vid:    proto.Vid(mrand.Int()),
-				Count:  uint32((args.Size + blobSize - 1) / blobSize),
+				MinSliceID: proto.BlobID(mrand.Int()),
+				Vid:        proto.Vid(mrand.Int()),
+				Count:      uint32((args.Size + blobSize - 1) / blobSize),
 			},
 		}
 	}
 
-	tokens := make([]string, 0, len(loc.Blobs)+1)
+	tokens := make([]string, 0, len(loc.Slices)+1)
 
-	hasMultiBlobs := loc.Size >= uint64(loc.BlobSize)
-	lastSize := uint32(loc.Size % uint64(loc.BlobSize))
-	for idx, blob := range loc.Blobs {
+	hasMultiBlobs := loc.Size_ >= uint64(loc.SliceSize)
+	lastSize := uint32(loc.Size_ % uint64(loc.SliceSize))
+	for idx, blob := range loc.Slices {
 		// returns one token if size < blobsize
 		if hasMultiBlobs {
 			count := blob.Count
-			if idx == len(loc.Blobs)-1 && lastSize > 0 {
+			if idx == len(loc.Slices)-1 && lastSize > 0 {
 				count--
 			}
-			tokens = append(tokens, uptoken.EncodeToken(uptoken.NewUploadToken(loc.ClusterID,
-				blob.Vid, blob.MinBid, count,
-				loc.BlobSize, 0, tokenAlloc[:])))
+			tokens = append(tokens, security.EncodeToken(security.NewUploadToken(loc.ClusterID,
+				blob.Vid, blob.MinSliceID, count,
+				loc.SliceSize, 0, tokenAlloc[:])))
 		}
 
 		// token of the last blob
-		if idx == len(loc.Blobs)-1 && lastSize > 0 {
-			tokens = append(tokens, uptoken.EncodeToken(uptoken.NewUploadToken(loc.ClusterID,
-				blob.Vid, blob.MinBid+proto.BlobID(blob.Count)-1, 1,
+		if idx == len(loc.Slices)-1 && lastSize > 0 {
+			tokens = append(tokens, security.EncodeToken(security.NewUploadToken(loc.ClusterID,
+				blob.Vid, blob.MinSliceID+proto.BlobID(blob.Count)-1, 1,
 				lastSize, 0, tokenAlloc[:])))
 		}
 	}
@@ -252,7 +251,7 @@ func handlePut(c *rpc.Context) {
 		hashSumMap[alg] = hasher.Sum(nil)
 	}
 
-	loc := access.Location{Size: uint64(args.Size)}
+	loc := proto.Location{Size_: uint64(args.Size)}
 	fillCrc(&loc)
 	c.RespondJSON(access.PutResp{
 		Location:   loc,
@@ -272,7 +271,7 @@ func handlePutAt(c *rpc.Context) {
 		return
 	}
 
-	token := uptoken.DecodeToken(args.Token)
+	token := security.DecodeToken(args.Token)
 	if !token.IsValid(args.ClusterID, args.Vid, args.BlobID, uint32(args.Size), tokenPutat[:]) {
 		c.RespondStatus(http.StatusForbidden)
 		return
@@ -299,7 +298,7 @@ func handleGet(c *rpc.Context) {
 		return
 	}
 
-	if args.Location.Size == 100 {
+	if args.Location.Size_ == 100 {
 		c.RespondStatus(http.StatusBadRequest)
 		return
 	}
@@ -360,7 +359,7 @@ func handleSign(c *rpc.Context) {
 	c.RespondJSON(access.SignResp{Location: args.Location})
 }
 
-func calcCrc(loc *access.Location) (uint32, error) {
+func calcCrc(loc *proto.Location) (uint32, error) {
 	crcWriter := crc32.New(crc32.IEEETable)
 
 	buf := bytespool.Alloc(1024)
@@ -378,7 +377,7 @@ func calcCrc(loc *access.Location) (uint32, error) {
 	return crcWriter.Sum32(), nil
 }
 
-func fillCrc(loc *access.Location) error {
+func fillCrc(loc *proto.Location) error {
 	crc, err := calcCrc(loc)
 	if err != nil {
 		return err
@@ -387,7 +386,7 @@ func fillCrc(loc *access.Location) error {
 	return nil
 }
 
-func verifyCrc(loc *access.Location) bool {
+func verifyCrc(loc *proto.Location) bool {
 	crc, err := calcCrc(loc)
 	if err != nil {
 		return false
@@ -395,13 +394,13 @@ func verifyCrc(loc *access.Location) bool {
 	return loc.Crc == crc
 }
 
-func signCrc(loc *access.Location, locs []access.Location) error {
+func signCrc(loc *proto.Location, locs []proto.Location) error {
 	first := locs[0]
 	bids := make(map[proto.BlobID]struct{}, 64)
 
 	if loc.ClusterID != first.ClusterID ||
 		loc.CodeMode != first.CodeMode ||
-		loc.BlobSize != first.BlobSize {
+		loc.SliceSize != first.SliceSize {
 		return fmt.Errorf("not equal in constant field")
 	}
 
@@ -413,20 +412,20 @@ func signCrc(loc *access.Location, locs []access.Location) error {
 		// assert
 		if l.ClusterID != first.ClusterID ||
 			l.CodeMode != first.CodeMode ||
-			l.BlobSize != first.BlobSize {
+			l.SliceSize != first.SliceSize {
 			return fmt.Errorf("not equal in constant field")
 		}
 
-		for _, blob := range l.Blobs {
+		for _, blob := range l.Slices {
 			for c := 0; c < int(blob.Count); c++ {
-				bids[blob.MinBid+proto.BlobID(c)] = struct{}{}
+				bids[blob.MinSliceID+proto.BlobID(c)] = struct{}{}
 			}
 		}
 	}
 
-	for _, blob := range loc.Blobs {
+	for _, blob := range loc.Slices {
 		for c := 0; c < int(blob.Count); c++ {
-			bid := blob.MinBid + proto.BlobID(c)
+			bid := blob.MinSliceID + proto.BlobID(c)
 			if _, ok := bids[bid]; !ok {
 				return fmt.Errorf("not equal in blob_id(%d)", bid)
 			}
@@ -501,10 +500,10 @@ func TestAccessClientConnectionMode(t *testing.T) {
 		if cs.size <= 0 {
 			continue
 		}
-		loc := access.Location{Size: uint64(mrand.Int63n(cs.size))}
+		loc := proto.Location{Size_: uint64(mrand.Int63n(cs.size))}
 		fillCrc(&loc)
 		_, err = cli.Delete(randCtx(), &access.DeleteArgs{
-			Locations: []access.Location{loc},
+			Locations: []proto.Location{loc},
 		})
 		require.NoError(t, err)
 	}
@@ -546,7 +545,7 @@ func TestAccessClientPutGet(t *testing.T) {
 	}
 
 	// test code 400
-	_, err := client.Get(randCtx(), &access.GetArgs{Location: access.Location{Size: 100}, ReadSize: uint64(100)})
+	_, err := client.Get(randCtx(), &access.GetArgs{Location: proto.Location{Size_: 100}, ReadSize: uint64(100)})
 	require.Error(t, err)
 }
 
@@ -691,7 +690,7 @@ func TestAccessClientPutMaxBlobsLength(t *testing.T) {
 		}
 
 		_, _, err := client.Put(randCtx(), &args)
-		require.ErrorIs(t, cs.err, err)
+		require.ErrorIs(t, err, cs.err)
 	}
 }
 
@@ -724,17 +723,16 @@ func TestAccessClientPutTimeout(t *testing.T) {
 		maxMs time.Duration
 	}
 	cases := []caseT{
-		// QuickConnMode 3s + size / 40, dial and response 2s
+		// QuickConnMode 3s + size / 40
 		{access.QuickConnMode, mb * -119, ms * 0, ms * 600},
 		{access.QuickConnMode, mb * -100, ms * 500, ms * 1000},
 		{access.QuickConnMode, mb * -80, ms * 1000, ms * 1500},
-		{access.QuickConnMode, mb * -1, ms * 2000, ms * 2500},
+		{access.QuickConnMode, mb * -1, ms * 2500, ms * 3500},
 
-		// DefaultConnMode 30s + size / 10, dial and response 5s
+		// DefaultConnMode 30s + size / 10
 		{access.DefaultConnMode, mb * -299, ms * 0, ms * 600},
 		{access.DefaultConnMode, mb * -280, ms * 2000, ms * 2500},
 		{access.DefaultConnMode, mb * -270, ms * 3000, ms * 3500},
-		{access.DefaultConnMode, mb * -1, ms * 5000, ms * 5500},
 	}
 
 	var wg sync.WaitGroup
@@ -778,40 +776,40 @@ func TestAccessClientDelete(t *testing.T) {
 	{
 		locs, err := client.Delete(randCtx(), nil)
 		require.Nil(t, locs)
-		require.ErrorIs(t, errcode.ErrIllegalArguments, err)
+		require.ErrorIs(t, err, errcode.ErrIllegalArguments)
 	}
 	{
 		locs, err := client.Delete(randCtx(), &access.DeleteArgs{})
 		require.Nil(t, locs)
-		require.ErrorIs(t, errcode.ErrIllegalArguments, err)
+		require.ErrorIs(t, err, errcode.ErrIllegalArguments)
 	}
 	{
 		locs, err := client.Delete(randCtx(), &access.DeleteArgs{
-			Locations: make([]access.Location, 1),
+			Locations: make([]proto.Location, 1),
 		})
 		require.Nil(t, locs)
 		require.NoError(t, err)
 	}
 	{
 		args := &access.DeleteArgs{
-			Locations: make([]access.Location, 1000),
+			Locations: make([]proto.Location, 1000),
 		}
 		_, err := client.Delete(randCtx(), args)
 		require.NoError(t, err)
 	}
 	{
 		args := &access.DeleteArgs{
-			Locations: make([]access.Location, access.MaxDeleteLocations+1),
+			Locations: make([]proto.Location, access.MaxDeleteLocations+1),
 		}
 		locs, err := client.Delete(randCtx(), args)
 		require.Equal(t, args.Locations, locs)
-		require.ErrorIs(t, errcode.ErrIllegalArguments, err)
+		require.ErrorIs(t, err, errcode.ErrIllegalArguments)
 	}
 	{
-		loc := access.Location{Size: 100, Blobs: make([]access.SliceInfo, 0)}
+		loc := proto.Location{Size_: 100, Slices: make([]proto.Slice, 0)}
 		fillCrc(&loc)
 		args := &access.DeleteArgs{
-			Locations: make([]access.Location, 0, access.MaxDeleteLocations),
+			Locations: make([]proto.Location, 0, access.MaxDeleteLocations),
 		}
 		for i := 1; i < access.MaxDeleteLocations/10; i++ {
 			args.Locations = append(args.Locations, loc)
@@ -894,7 +892,7 @@ func TestAccessClientPutAtToken(t *testing.T) {
 			Body: bytes.NewBuffer(buff),
 		}
 		_, _, err := client.Put(randCtx(), &args)
-		require.ErrorIs(t, errcode.ErrUnexpected, err)
+		require.ErrorIs(t, err, errcode.ErrUnexpected)
 	}
 }
 
@@ -928,7 +926,7 @@ func TestAccessClientRPCConfig(t *testing.T) {
 }
 
 func TestAccessClientLogger(t *testing.T) {
-	file, err := ioutil.TempFile(os.TempDir(), "TestAccessClientLogger")
+	file, err := os.CreateTemp(os.TempDir(), "TestAccessClientLogger")
 	require.NoError(t, err)
 	require.NoError(t, file.Close())
 	defer func() {
@@ -940,7 +938,7 @@ func TestAccessClientLogger(t *testing.T) {
 		Logger: &access.Logger{
 			Filename: file.Name(),
 		},
-		PriorityAddrs: []string{"127.0.0.1:9500"},
+		PriorityAddrs: []string{"127.0.0.1:37173"},
 	}
 	client, err := access.New(cfg)
 	require.NoError(t, err)
@@ -956,3 +954,35 @@ func TestAccessClientLogger(t *testing.T) {
 	_, _, err = client.Put(randCtx(), &args)
 	require.Error(t, err)
 }
+
+func TestAccessClientPutRetryGetBody(t *testing.T) {
+	refusedAddr := "http://127.0.0.1:37173"
+	cfg := access.Config{}
+	cfg.RPCConfig = &rpc.Config{}
+	cfg.LogLevel = log.Lfatal
+	cfg.PriorityAddrs = []string{refusedAddr, refusedAddr, mockServer.URL, refusedAddr}
+	client, err := access.New(cfg)
+	require.NoError(t, err)
+	for range [1000]struct{}{} {
+		retried := false
+		args := access.PutArgs{
+			Size: int64(1),
+			Body: bytes.NewBuffer([]byte{'a'}),
+			GetBody: func() (io.ReadCloser, error) {
+				retried = true
+				return io.NopCloser(bytes.NewBuffer([]byte{'z'})), nil
+			},
+		}
+		loc, _, err := client.Put(randCtx(), &args)
+		require.NoError(t, err)
+		rc, err := client.Get(randCtx(), &access.GetArgs{Location: loc, ReadSize: uint64(1)})
+		require.NoError(t, err)
+		buff, err := io.ReadAll(rc)
+		require.NoError(t, err)
+		if len(buff) > 0 && buff[0] == 'z' {
+			require.True(t, retried)
+			return
+		}
+	}
+	require.Fail(t, "retry with get body failed")
+}

blobstore/api/access/proto.go

@@ -18,10 +18,7 @@ import (
 	"crypto/md5"
 	"crypto/sha1"
 	"crypto/sha256"
-	"encoding/base64"
-	"encoding/binary"
 	"encoding/hex"
-	"fmt"
 	"hash"
 	"hash/crc32"
 	"io"
@@ -196,298 +193,27 @@ func (h HashSumMap) All() map[string]interface{} {
 	return m
 }
 
-// Location file location, 4 + 1 + 8 + 4 + 4 + len*16 bytes
-// |                                        |
-// |   ClusterID(4)    |    CodeMode(1)     |
-// |                Size(8)                 |
-// |   BlobSize(4)     |      Crc(4)        |
-// |           len*SliceInfo(16)            |
-//
-// ClusterID which cluster file is in
-// CodeMode is ec encode mode, see defined in "common/lib/codemode"
-// Size is file size
-// BlobSize is every blob's size but the last one which's size=(Size mod BlobSize)
-// Crc is the checksum, change anything of the location, crc will mismatch
-// Blobs all blob information
-type Location struct {
-	_         [0]byte
-	ClusterID proto.ClusterID   `json:"cluster_id"`
-	CodeMode  codemode.CodeMode `json:"code_mode"`
-	Size      uint64            `json:"size"`
-	BlobSize  uint32            `json:"blob_size"`
-	Crc       uint32            `json:"crc"`
-	Blobs     []SliceInfo       `json:"blobs"`
-}
-
-// SliceInfo blobs info, 8 + 4 + 4 bytes
-//
-// MinBid is the first blob id
-// Vid is which volume all blobs in
-// Count is num of consecutive blob ids, count=1 just has one blob
-//
-// blob ids = [MinBid, MinBid+count)
-type SliceInfo struct {
-	_      [0]byte
-	MinBid proto.BlobID `json:"min_bid"`
-	Vid    proto.Vid    `json:"vid"`
-	Count  uint32       `json:"count"`
-}
-
-// Blob is one piece of data in a location
-//
-// Bid is the blob id
-// Vid is which volume the blob in
-// Size is real size of the blob
-type Blob struct {
-	Bid  proto.BlobID
-	Vid  proto.Vid
-	Size uint32
-}
-
-// Copy returns a new same Location
-func (loc *Location) Copy() Location {
-	dst := Location{
-		ClusterID: loc.ClusterID,
-		CodeMode:  loc.CodeMode,
-		Size:      loc.Size,
-		BlobSize:  loc.BlobSize,
-		Crc:       loc.Crc,
-		Blobs:     make([]SliceInfo, len(loc.Blobs)),
-	}
-	copy(dst.Blobs, loc.Blobs)
-	return dst
-}
-
-// Encode transfer Location to slice byte
-// Returns the buf created by me
-//
-//	(n) means max-n bytes
-//	- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-//	|  field  | crc | clusterid  | codemode |    size     |  blobsize  |
-//	- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-//	| n-bytes |  4  | uvarint(5) |    1     | uvarint(10) | uvarint(5) |
-//	- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-//	        25   +  (5){len(blobs)}   +   len(Blobs) * 20
-//	- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-//	|  blobs  | minbid | vid | count |           ...                   |
-//	- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-//	| n-bytes |  (10)  | (5) |  (5)  | (20) | (20) |       ...         |
-//	- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-func (loc *Location) Encode() []byte {
-	if loc == nil {
-		return nil
-	}
-	n := 25 + 5 + len(loc.Blobs)*20
-	buf := make([]byte, n)
-	n = loc.Encode2(buf)
-	return buf[:n]
-}
-
-// Encode2 transfer Location to the buf, the buf reuse by yourself
-// Returns the number of bytes read
-// If the buffer is too small, Encode2 will panic
-func (loc *Location) Encode2(buf []byte) int {
-	if loc == nil {
-		return 0
-	}
-
-	n := 0
-	binary.BigEndian.PutUint32(buf[n:], loc.Crc)
-	n += 4
-	n += binary.PutUvarint(buf[n:], uint64(loc.ClusterID))
-	buf[n] = byte(loc.CodeMode)
-	n++
-	n += binary.PutUvarint(buf[n:], uint64(loc.Size))
-	n += binary.PutUvarint(buf[n:], uint64(loc.BlobSize))
-
-	n += binary.PutUvarint(buf[n:], uint64(len(loc.Blobs)))
-	for _, blob := range loc.Blobs {
-		n += binary.PutUvarint(buf[n:], uint64(blob.MinBid))
-		n += binary.PutUvarint(buf[n:], uint64(blob.Vid))
-		n += binary.PutUvarint(buf[n:], uint64(blob.Count))
-	}
-
-	return n
-}
-
-// Decode parse location from buf
-// Returns the number of bytes read
-// Error is not nil when parsing failed
-func (loc *Location) Decode(buf []byte) (int, error) {
-	if loc == nil {
-		return 0, fmt.Errorf("location receiver is nil")
-	}
-
-	location, n, err := DecodeLocation(buf)
-	if err != nil {
-		return n, err
-	}
-
-	*loc = location
-	return n, nil
-}
-
-// ToString transfer location to hex string
-func (loc *Location) ToString() string {
-	return loc.HexString()
-}
-
-// HexString transfer location to hex string
-func (loc *Location) HexString() string {
-	return hex.EncodeToString(loc.Encode())
-}
-
-// Base64String transfer location to base64 string
-func (loc *Location) Base64String() string {
-	return base64.StdEncoding.EncodeToString(loc.Encode())
-}
-
-// Spread location blobs to slice
-func (loc *Location) Spread() []Blob {
-	count := 0
-	for _, blob := range loc.Blobs {
-		count += int(blob.Count)
-	}
-
-	blobs := make([]Blob, 0, count)
-	for _, blob := range loc.Blobs {
-		for offset := uint32(0); offset < blob.Count; offset++ {
-			blobs = append(blobs, Blob{
-				Bid:  blob.MinBid + proto.BlobID(offset),
-				Vid:  blob.Vid,
-				Size: loc.BlobSize,
-			})
-		}
-	}
-	if len(blobs) > 0 && loc.BlobSize > 0 {
-		if lastSize := loc.Size % uint64(loc.BlobSize); lastSize > 0 {
-			blobs[len(blobs)-1].Size = uint32(lastSize)
-		}
-	}
-	return blobs
-}
-
-// DecodeLocation parse location from buf
-// Returns Location and the number of bytes read
-// Error is not nil when parsing failed
-func DecodeLocation(buf []byte) (Location, int, error) {
-	var (
-		loc Location
-		n   int
-
-		val uint64
-		nn  int
-	)
-	next := func() (uint64, int) {
-		val, nn := binary.Uvarint(buf)
-		if nn <= 0 {
-			return 0, nn
-		}
-		n += nn
-		buf = buf[nn:]
-		return val, nn
-	}
-
-	if len(buf) < 4 {
-		return loc, n, fmt.Errorf("bytes crc %d", len(buf))
-	}
-	loc.Crc = binary.BigEndian.Uint32(buf)
-	n += 4
-	buf = buf[4:]
-
-	if val, nn = next(); nn <= 0 {
-		return loc, n, fmt.Errorf("bytes cluster_id %d", nn)
-	}
-	loc.ClusterID = proto.ClusterID(val)
-
-	if len(buf) < 1 {
-		return loc, n, fmt.Errorf("bytes codemode %d", len(buf))
-	}
-	loc.CodeMode = codemode.CodeMode(buf[0])
-	n++
-	buf = buf[1:]
-
-	if val, nn = next(); nn <= 0 {
-		return loc, n, fmt.Errorf("bytes size %d", nn)
-	}
-	loc.Size = val
-
-	if val, nn = next(); nn <= 0 {
-		return loc, n, fmt.Errorf("bytes blob_size %d", nn)
-	}
-	loc.BlobSize = uint32(val)
-
-	if val, nn = next(); nn <= 0 {
-		return loc, n, fmt.Errorf("bytes length blobs %d", nn)
-	}
-	length := int(val)
-
-	if length > 0 {
-		loc.Blobs = make([]SliceInfo, 0, length)
-	}
-	for index := 0; index < length; index++ {
-		var blob SliceInfo
-		if val, nn = next(); nn <= 0 {
-			return loc, n, fmt.Errorf("bytes %dth-blob min_bid %d", index, nn)
-		}
-		blob.MinBid = proto.BlobID(val)
-
-		if val, nn = next(); nn <= 0 {
-			return loc, n, fmt.Errorf("bytes %dth-blob vid %d", index, nn)
-		}
-		blob.Vid = proto.Vid(val)
-
-		if val, nn = next(); nn <= 0 {
-			return loc, n, fmt.Errorf("bytes %dth-blob count %d", index, nn)
-		}
-		blob.Count = uint32(val)
-
-		loc.Blobs = append(loc.Blobs, blob)
-	}
-
-	return loc, n, nil
-}
-
-// DecodeLocationFrom decode location from hex string
-func DecodeLocationFrom(s string) (Location, error) {
-	return DecodeLocationFromHex(s)
-}
-
-// DecodeLocationFromHex decode location from hex string
-func DecodeLocationFromHex(s string) (Location, error) {
-	var loc Location
-	src, err := hex.DecodeString(s)
-	if err != nil {
-		return loc, err
-	}
-	_, err = loc.Decode(src)
-	if err != nil {
-		return loc, err
-	}
-	return loc, nil
-}
-
-// DecodeLocationFromBase64 decode location from base64 string
-func DecodeLocationFromBase64(s string) (Location, error) {
-	var loc Location
-	src, err := base64.StdEncoding.DecodeString(s)
-	if err != nil {
-		return loc, err
-	}
-	_, err = loc.Decode(src)
-	if err != nil {
-		return loc, err
-	}
-	return loc, nil
-}
-
 // PutArgs for service /put
 // Hashes means how to calculate check sum,
 // HashAlgCRC32 | HashAlgMD5 equal 2 + 4 = 6
+// AssignClusterID > 0 means that cluster_id is assigned by the API caller
+// AssignClusterID = 0 means that cluster_id is assigned by access cluster controller
+// CodeMode > 0 means that codemode is assigned by the API caller
+// CodeMode = 0 means that codemode is assigned by code_mode_policies
 type PutArgs struct {
 	Size   int64         `json:"size"`
 	Hashes HashAlgorithm `json:"hashes,omitempty"`
 	Body   io.Reader     `json:"-"`
+
+	AssignClusterID proto.ClusterID   `json:"assign_cluster_id,omitempty"`
+	CodeMode        codemode.CodeMode `json:"code_mode,omitempty"`
+
+	// GetBody defines an optional func to return a new copy of Body.
+	// It is used for client requests when a redirect requires reading
+	// the body more than once. Use of GetBody still requires setting Body.
+	//
+	// There force reset request.GetBody if it is setting.
+	GetBody func() (io.ReadCloser, error) `json:"-"`
 }
 
 // IsValid is valid put args
@@ -500,8 +226,8 @@ func (args *PutArgs) IsValid() bool {
 
 // PutResp put response result
 type PutResp struct {
-	Location   Location   `json:"location"`
-	HashSumMap HashSumMap `json:"hashsum"`
+	Location   proto.Location `json:"location"`
+	HashSumMap HashSumMap     `json:"hashsum,omitempty"`
 }
 
 // PutAtArgs for service /putat
@@ -555,15 +281,16 @@ func (args *AllocArgs) IsValid() bool {
 // if size mod blobsize == 0, length of tokens equal length of location blobs
 // otherwise additional token for the last blob uploading
 type AllocResp struct {
-	Location Location `json:"location"`
-	Tokens   []string `json:"tokens"`
+	Location proto.Location `json:"location"`
+	Tokens   []string       `json:"tokens"`
 }
 
 // GetArgs for service /get
 type GetArgs struct {
-	Location Location `json:"location"`
-	Offset   uint64   `json:"offset"`
-	ReadSize uint64   `json:"read_size"`
+	Location proto.Location `json:"location"`
+	Offset   uint64         `json:"offset"`
+	ReadSize uint64         `json:"read_size"`
+	Writer   io.Writer      `json:"-"`
 }
 
 // IsValid is valid get args
@@ -571,14 +298,14 @@ func (args *GetArgs) IsValid() bool {
 	if args == nil {
 		return false
 	}
-	return args.Offset <= args.Location.Size &&
-		args.ReadSize <= args.Location.Size &&
-		args.Offset+args.ReadSize <= args.Location.Size
+	return args.Offset <= args.Location.Size_ &&
+		args.ReadSize <= args.Location.Size_ &&
+		args.Offset+args.ReadSize <= args.Location.Size_
 }
 
 // DeleteArgs for service /delete
 type DeleteArgs struct {
-	Locations []Location `json:"locations"`
+	Locations []proto.Location `json:"locations"`
 }
 
 // IsValid is valid delete args
@@ -591,7 +318,7 @@ func (args *DeleteArgs) IsValid() bool {
 
 // DeleteResp delete response with failed locations
 type DeleteResp struct {
-	FailedLocations []Location `json:"failed_locations,omitempty"`
+	FailedLocations []proto.Location `json:"failed_locations,omitempty"`
 }
 
 // DeleteBlobArgs for service /deleteblob
@@ -618,8 +345,8 @@ func (args *DeleteBlobArgs) IsValid() bool {
 // Locations are signed location getting from /alloc
 // Location is to be signed location which merged by yourself
 type SignArgs struct {
-	Locations []Location `json:"locations"`
-	Location  Location   `json:"location"`
+	Locations []proto.Location `json:"locations"`
+	Location  proto.Location   `json:"location"`
 }
 
 // IsValid is valid sign args
@@ -632,5 +359,141 @@ func (args *SignArgs) IsValid() bool {
 
 // SignResp sign response location with crc
 type SignResp struct {
-	Location Location `json:"location"`
+	Location proto.Location `json:"location"`
+}
+
+// Shardnode Blob
+
+type GetShardMode int
+
+const (
+	GetShardModeRandom = GetShardMode(iota)
+	GetShardModeLeader
+)
+
+type CreateBlobArgs struct {
+	ClusterID proto.ClusterID
+	CodeMode  codemode.CodeMode
+	BlobName  string
+	Size      uint64
+	SliceSize uint32
+}
+
+func (args *CreateBlobArgs) IsValid() bool {
+	if args == nil {
+		return false
+	}
+	return args.Size != 0 && len(args.BlobName) != 0
+}
+
+type CreateBlobRet struct {
+	Location proto.Location
+}
+
+type ListBlobArgs struct {
+	ClusterID proto.ClusterID
+	ShardID   proto.ShardID
+	Mode      GetShardMode
+	Prefix    string
+	Marker    string
+	Count     uint64
+}
+
+func (args *ListBlobArgs) IsValid() bool {
+	if args == nil {
+		return false
+	}
+	return args.ClusterID != 0
+}
+
+type SealBlobArgs struct {
+	ClusterID proto.ClusterID
+	BlobName  string
+	Size      uint64
+	Slices    []proto.Slice
+}
+
+func (args *SealBlobArgs) IsValid() bool {
+	if args == nil {
+		return false
+	}
+	return args.ClusterID != 0 && len(args.BlobName) != 0
+}
+
+type GetBlobArgs struct {
+	ClusterID proto.ClusterID
+	Mode      GetShardMode
+	BlobName  string
+
+	Offset   uint64
+	ReadSize uint64
+	Writer   io.Writer
+}
+
+// IsValid is valid get args
+func (args *GetBlobArgs) IsValid() bool {
+	if args == nil {
+		return false
+	}
+	return args.ClusterID != 0 && len(args.BlobName) != 0
+}
+
+type DelBlobArgs struct {
+	ClusterID proto.ClusterID
+	BlobName  string
+}
+
+func (args *DelBlobArgs) IsValid() bool {
+	if args == nil {
+		return false
+	}
+	return args.ClusterID != 0 && len(args.BlobName) != 0
+}
+
+type AllocSliceArgs struct {
+	ClusterID proto.ClusterID
+	CodeMode  codemode.CodeMode
+	BlobName  string
+	Size      uint64
+	FailSlice proto.Slice
+}
+
+func (args *AllocSliceArgs) IsValid() bool {
+	if args == nil {
+		return false
+	}
+	return args.ClusterID != 0 && args.CodeMode.IsValid() && args.Size != 0 && len(args.BlobName) != 0
+}
+
+type PutBlobArgs struct {
+	CodeMode codemode.CodeMode
+	BlobName string
+	NeedSeal bool
+
+	Size   uint64
+	Hashes HashAlgorithm
+	Body   io.Reader
+}
+
+func (args *PutBlobArgs) IsValid() bool {
+	if args == nil {
+		return false
+	}
+	return args.CodeMode != 0 && args.Size != 0 && len(args.BlobName) != 0
+}
+
+type GetShardCommonArgs struct {
+	ClusterID proto.ClusterID
+	ShardID   proto.ShardID
+	Mode      GetShardMode
+	BlobName  string
+}
+
+func (args *ListBlobEncodeMarker) MarshalToString() (string, error) {
+	raw, err := args.Marshal()
+	return string(raw), err
+}
+
+func (args *ListBlobEncodeMarker) UnmarshalFromString(marker string) error {
+	return args.Unmarshal([]byte(marker))
 }

blobstore/api/access/proto_test.go

@@ -259,192 +259,6 @@ func TestHashAlgorithm2HashSumMap(t *testing.T) {
 	}
 }
 
-func TestLocationEncodeDecodeNil(t *testing.T) {
-	var loc *access.Location
-	require.Nil(t, loc.Encode())
-	require.Equal(t, 0, loc.Encode2(nil))
-	require.Equal(t, "", loc.ToString())
-	require.Equal(t, "", loc.HexString())
-	require.Equal(t, "", loc.Base64String())
-
-	n, err := loc.Decode(nil)
-	require.Error(t, err)
-	require.Equal(t, 0, n)
-
-	locx, n, err := access.DecodeLocation(nil)
-	require.Error(t, err)
-	require.Equal(t, 0, n)
-	require.Equal(t, access.Location{}, locx)
-
-	locx, err = access.DecodeLocationFrom("")
-	require.Error(t, err)
-	require.Equal(t, access.Location{}, locx)
-
-	locx, err = access.DecodeLocationFrom("xxx")
-	require.Error(t, err)
-	require.Equal(t, access.Location{}, locx)
-
-	locx, err = access.DecodeLocationFromHex("xxx")
-	require.Error(t, err)
-	require.Equal(t, access.Location{}, locx)
-
-	locx, err = access.DecodeLocationFromBase64("xxx")
-	require.Error(t, err)
-	require.Equal(t, access.Location{}, locx)
-}
-
-func TestLocationEncodeDecode(t *testing.T) {
-	for ii := 0; ii < 100; ii++ {
-		loc := &access.Location{
-			ClusterID: proto.ClusterID(mrand.Uint32()),
-			CodeMode:  codemode.CodeMode(mrand.Intn(0xff)),
-			Size:      mrand.Uint64(),
-			BlobSize:  mrand.Uint32(),
-			Crc:       mrand.Uint32(),
-		}
-
-		num := mrand.Intn(5)
-		for i := 0; i < num; i++ {
-			loc.Blobs = append(loc.Blobs, access.SliceInfo{
-				MinBid: proto.BlobID(mrand.Uint64()),
-				Vid:    proto.Vid(mrand.Uint32()),
-				Count:  mrand.Uint32(),
-			})
-		}
-
-		buf := loc.Encode()
-		bufx := make([]byte, len(buf))
-		n := loc.Encode2(bufx)
-		require.Equal(t, len(buf), n)
-		require.Equal(t, buf, bufx)
-
-		require.Panics(t, func() { loc.Encode2(nil) })
-		require.Panics(t, func() { loc.Encode2(bufx[:3]) })
-		require.Panics(t, func() { loc.Encode2(bufx[:n/2]) })
-		require.Panics(t, func() { loc.Encode2(bufx[:n-1]) })
-
-		locx := access.Location{}
-		locx.Decode(bufx)
-		require.Equal(t, loc.ToString(), locx.ToString())
-		require.Equal(t, loc.HexString(), locx.HexString())
-		require.Equal(t, loc.Base64String(), locx.Base64String())
-
-		str := loc.ToString()
-		locx, err := access.DecodeLocationFrom(str)
-		require.NoError(t, err)
-		require.Equal(t, *loc, locx)
-
-		str = loc.HexString()
-		locx, err = access.DecodeLocationFromHex(str)
-		require.NoError(t, err)
-		require.Equal(t, *loc, locx)
-
-		str = loc.Base64String()
-		locx, err = access.DecodeLocationFromBase64(str)
-		require.NoError(t, err)
-		require.Equal(t, *loc, locx)
-	}
-}
-
-func TestLocationDecodeError(t *testing.T) {
-	loc := &access.Location{
-		ClusterID: proto.ClusterID(math.MaxUint32),
-		CodeMode:  codemode.CodeMode(math.MaxInt8),
-		Size:      math.MaxUint64,
-		BlobSize:  math.MaxUint32,
-		Crc:       math.MaxUint32,
-	}
-
-	buf := loc.Encode()
-	require.Equal(t, 25+1, len(buf))
-	for _, n := range []int{3, 8, 9, 19, 24} {
-		_, _, err := access.DecodeLocation(buf[:n])
-		require.Error(t, err)
-		t.Log(err)
-	}
-
-	loc.Blobs = append(loc.Blobs, access.SliceInfo{
-		MinBid: proto.BlobID(math.MaxUint64),
-		Vid:    proto.Vid(math.MaxUint32),
-		Count:  math.MaxUint32,
-	})
-
-	buf = loc.Encode()
-	require.Equal(t, 25+1+20, len(buf))
-	for _, n := range []int{25, 35, 40, 45} {
-		_, _, err := access.DecodeLocation(buf[:n])
-		require.Error(t, err)
-		t.Log(err)
-	}
-}
-
-func TestLocationSpread(t *testing.T) {
-	{
-		var loc access.Location
-		blobs := loc.Spread()
-		require.NotNil(t, blobs)
-		require.Equal(t, 0, len(blobs))
-	}
-	{
-		loc := &access.Location{
-			Size:     10,
-			BlobSize: 1 << 22,
-			Blobs: []access.SliceInfo{{
-				MinBid: 100,
-				Vid:    4,
-				Count:  1,
-			}},
-		}
-		blobs := loc.Spread()
-		require.Equal(t, 1, len(blobs))
-		require.Equal(t, proto.BlobID(100), blobs[0].Bid)
-		require.Equal(t, proto.Vid(4), blobs[0].Vid)
-		require.Equal(t, uint32(10), blobs[0].Size)
-	}
-	{
-		loc := &access.Location{
-			Size:     (1 << 22) + 10,
-			BlobSize: 1 << 22,
-			Blobs: []access.SliceInfo{{
-				MinBid: 100,
-				Vid:    4,
-				Count:  2,
-			}},
-		}
-		blobs := loc.Spread()
-		require.Equal(t, 2, len(blobs))
-		require.Equal(t, proto.BlobID(100), blobs[0].Bid)
-		require.Equal(t, proto.Vid(4), blobs[0].Vid)
-		require.Equal(t, uint32(1<<22), blobs[0].Size)
-		require.Equal(t, proto.BlobID(101), blobs[1].Bid)
-		require.Equal(t, proto.Vid(4), blobs[1].Vid)
-		require.Equal(t, uint32(10), blobs[1].Size)
-	}
-	{
-		loc := &access.Location{
-			Size:     1 << 23,
-			BlobSize: 1 << 22,
-			Blobs: []access.SliceInfo{{
-				MinBid: 100,
-				Vid:    4,
-				Count:  1,
-			}, {
-				MinBid: 200,
-				Vid:    4,
-				Count:  1,
-			}},
-		}
-		blobs := loc.Spread()
-		require.Equal(t, 2, len(blobs))
-		require.Equal(t, proto.BlobID(100), blobs[0].Bid)
-		require.Equal(t, proto.Vid(4), blobs[0].Vid)
-		require.Equal(t, uint32(1<<22), blobs[0].Size)
-		require.Equal(t, proto.BlobID(200), blobs[1].Bid)
-		require.Equal(t, proto.Vid(4), blobs[1].Vid)
-		require.Equal(t, uint32(1<<22), blobs[1].Size)
-	}
-}
-
 func TestPutArgs(t *testing.T) {
 	cases := []struct {
 		size  int64
@@ -538,7 +352,7 @@ func TestGetArgs(t *testing.T) {
 			Offset:   cs.offset,
 			ReadSize: cs.readSize,
 		}
-		args.Location.Size = cs.size
+		args.Location.Size_ = cs.size
 		require.Equal(t, cs.valid, args.IsValid())
 	}
 }
@@ -547,11 +361,11 @@ func TestDeleteArgs(t *testing.T) {
 	args := access.DeleteArgs{}
 	require.False(t, args.IsValid())
 	require.False(t, (*access.DeleteArgs)(nil).IsValid())
-	args.Locations = []access.Location{{}}
+	args.Locations = []proto.Location{{}}
 	require.True(t, args.IsValid())
-	args.Locations = make([]access.Location, access.MaxDeleteLocations)
+	args.Locations = make([]proto.Location, access.MaxDeleteLocations)
 	require.True(t, args.IsValid())
-	args.Locations = make([]access.Location, access.MaxDeleteLocations+1)
+	args.Locations = make([]proto.Location, access.MaxDeleteLocations+1)
 	require.False(t, args.IsValid())
 }
 
@@ -573,7 +387,7 @@ func TestSignArgs(t *testing.T) {
 	args := access.SignArgs{}
 	require.False(t, args.IsValid())
 	require.False(t, (*access.DeleteArgs)(nil).IsValid())
-	args.Locations = []access.Location{{}}
+	args.Locations = []proto.Location{{}}
 	require.True(t, args.IsValid())
 }
 

blobstore/api/access/stream_blob.pb.go

@@ -0,0 +1,384 @@
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: stream_blob.proto
+
+package access
+
+import (
+	fmt "fmt"
+	sharding "github.com/cubefs/cubefs/blobstore/common/sharding"
+	_ "github.com/gogo/protobuf/gogoproto"
+	proto "github.com/gogo/protobuf/proto"
+	io "io"
+	math "math"
+	math_bits "math/bits"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
+
+type ListBlobEncodeMarker struct {
+	Range                sharding.Range `protobuf:"bytes,1,opt,name=range,proto3" json:"range"`
+	Marker               string         `protobuf:"bytes,2,opt,name=marker,proto3" json:"marker,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}       `json:"-"`
+	XXX_unrecognized     []byte         `json:"-"`
+	XXX_sizecache        int32          `json:"-"`
+}
+
+func (m *ListBlobEncodeMarker) Reset()         { *m = ListBlobEncodeMarker{} }
+func (m *ListBlobEncodeMarker) String() string { return proto.CompactTextString(m) }
+func (*ListBlobEncodeMarker) ProtoMessage()    {}
+func (*ListBlobEncodeMarker) Descriptor() ([]byte, []int) {
+	return fileDescriptor_3f58ae6b23694640, []int{0}
+}
+func (m *ListBlobEncodeMarker) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *ListBlobEncodeMarker) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_ListBlobEncodeMarker.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *ListBlobEncodeMarker) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ListBlobEncodeMarker.Merge(m, src)
+}
+func (m *ListBlobEncodeMarker) XXX_Size() int {
+	return m.Size()
+}
+func (m *ListBlobEncodeMarker) XXX_DiscardUnknown() {
+	xxx_messageInfo_ListBlobEncodeMarker.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ListBlobEncodeMarker proto.InternalMessageInfo
+
+func (m *ListBlobEncodeMarker) GetRange() sharding.Range {
+	if m != nil {
+		return m.Range
+	}
+	return sharding.Range{}
+}
+
+func (m *ListBlobEncodeMarker) GetMarker() string {
+	if m != nil {
+		return m.Marker
+	}
+	return ""
+}
+
+func init() {
+	proto.RegisterType((*ListBlobEncodeMarker)(nil), "cubefs.blobstore.api.access.ListBlobEncodeMarker")
+}
+
+func init() { proto.RegisterFile("stream_blob.proto", fileDescriptor_3f58ae6b23694640) }
+
+var fileDescriptor_3f58ae6b23694640 = []byte{
+	// 217 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x2c, 0x2e, 0x29, 0x4a,
+	0x4d, 0xcc, 0x8d, 0x4f, 0xca, 0xc9, 0x4f, 0xd2, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x92, 0x4e,
+	0x2e, 0x4d, 0x4a, 0x4d, 0x2b, 0xd6, 0x03, 0x09, 0x15, 0x97, 0xe4, 0x17, 0xa5, 0xea, 0x25, 0x16,
+	0x64, 0xea, 0x25, 0x26, 0x27, 0xa7, 0x16, 0x17, 0x4b, 0x89, 0xa4, 0xe7, 0xa7, 0xe7, 0x83, 0xd5,
+	0xe9, 0x83, 0x58, 0x10, 0x2d, 0x52, 0x3a, 0x10, 0x2d, 0xfa, 0x70, 0x2d, 0xfa, 0xc9, 0xf9, 0xb9,
+	0xb9, 0xf9, 0x79, 0xfa, 0xc5, 0x19, 0x89, 0x45, 0x29, 0x99, 0x79, 0xe9, 0xfa, 0x45, 0x89, 0x79,
+	0xe9, 0xa9, 0x10, 0xd5, 0x4a, 0xc5, 0x5c, 0x22, 0x3e, 0x99, 0xc5, 0x25, 0x4e, 0x39, 0xf9, 0x49,
+	0xae, 0x79, 0xc9, 0xf9, 0x29, 0xa9, 0xbe, 0x89, 0x45, 0xd9, 0xa9, 0x45, 0x42, 0xce, 0x5c, 0xac,
+	0x60, 0x65, 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0xdc, 0x46, 0xea, 0x7a, 0x18, 0x0e, 0x81, 0x98, 0xaa,
+	0x07, 0x33, 0x55, 0x2f, 0x08, 0xa4, 0xdc, 0x89, 0xe5, 0xc4, 0x3d, 0x79, 0x86, 0x20, 0x88, 0x5e,
+	0x21, 0x31, 0x2e, 0xb6, 0x5c, 0xb0, 0x71, 0x12, 0x4c, 0x0a, 0x8c, 0x1a, 0x9c, 0x41, 0x50, 0x9e,
+	0x93, 0xf8, 0x89, 0x47, 0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x18, 0xc5,
+	0xa9, 0xa7, 0x6f, 0x0d, 0xf1, 0x51, 0x12, 0x1b, 0xd8, 0x51, 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff,
+	0xff, 0x17, 0x5f, 0xcb, 0x7b, 0x0a, 0x01, 0x00, 0x00,
+}
+
+func (m *ListBlobEncodeMarker) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *ListBlobEncodeMarker) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *ListBlobEncodeMarker) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.XXX_unrecognized != nil {
+		i -= len(m.XXX_unrecognized)
+		copy(dAtA[i:], m.XXX_unrecognized)
+	}
+	if len(m.Marker) > 0 {
+		i -= len(m.Marker)
+		copy(dAtA[i:], m.Marker)
+		i = encodeVarintStreamBlob(dAtA, i, uint64(len(m.Marker)))
+		i--
+		dAtA[i] = 0x12
+	}
+	{
+		size, err := m.Range.MarshalToSizedBuffer(dAtA[:i])
+		if err != nil {
+			return 0, err
+		}
+		i -= size
+		i = encodeVarintStreamBlob(dAtA, i, uint64(size))
+	}
+	i--
+	dAtA[i] = 0xa
+	return len(dAtA) - i, nil
+}
+
+func encodeVarintStreamBlob(dAtA []byte, offset int, v uint64) int {
+	offset -= sovStreamBlob(v)
+	base := offset
+	for v >= 1<<7 {
+		dAtA[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	dAtA[offset] = uint8(v)
+	return base
+}
+func (m *ListBlobEncodeMarker) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = m.Range.Size()
+	n += 1 + l + sovStreamBlob(uint64(l))
+	l = len(m.Marker)
+	if l > 0 {
+		n += 1 + l + sovStreamBlob(uint64(l))
+	}
+	if m.XXX_unrecognized != nil {
+		n += len(m.XXX_unrecognized)
+	}
+	return n
+}
+
+func sovStreamBlob(x uint64) (n int) {
+	return (math_bits.Len64(x|1) + 6) / 7
+}
+func sozStreamBlob(x uint64) (n int) {
+	return sovStreamBlob(uint64((x << 1) ^ uint64((int64(x) >> 63))))
+}
+func (m *ListBlobEncodeMarker) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowStreamBlob
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: ListBlobEncodeMarker: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: ListBlobEncodeMarker: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Range", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStreamBlob
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthStreamBlob
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthStreamBlob
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if err := m.Range.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Marker", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStreamBlob
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthStreamBlob
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthStreamBlob
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Marker = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipStreamBlob(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthStreamBlob
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func skipStreamBlob(dAtA []byte) (n int, err error) {
+	l := len(dAtA)
+	iNdEx := 0
+	depth := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return 0, ErrIntOverflowStreamBlob
+			}
+			if iNdEx >= l {
+				return 0, io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		wireType := int(wire & 0x7)
+		switch wireType {
+		case 0:
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowStreamBlob
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				iNdEx++
+				if dAtA[iNdEx-1] < 0x80 {
+					break
+				}
+			}
+		case 1:
+			iNdEx += 8
+		case 2:
+			var length int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowStreamBlob
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				length |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if length < 0 {
+				return 0, ErrInvalidLengthStreamBlob
+			}
+			iNdEx += length
+		case 3:
+			depth++
+		case 4:
+			if depth == 0 {
+				return 0, ErrUnexpectedEndOfGroupStreamBlob
+			}
+			depth--
+		case 5:
+			iNdEx += 4
+		default:
+			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
+		}
+		if iNdEx < 0 {
+			return 0, ErrInvalidLengthStreamBlob
+		}
+		if depth == 0 {
+			return iNdEx, nil
+		}
+	}
+	return 0, io.ErrUnexpectedEOF
+}
+
+var (
+	ErrInvalidLengthStreamBlob        = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowStreamBlob          = fmt.Errorf("proto: integer overflow")
+	ErrUnexpectedEndOfGroupStreamBlob = fmt.Errorf("proto: unexpected end of group")
+)

blobstore/api/access/stream_blob.proto

@@ -0,0 +1,31 @@
+// Copyright 2024 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+
+syntax = "proto3";
+
+package cubefs.blobstore.api.access;
+option go_package = "./;access";
+
+option (gogoproto.sizer_all) = true;
+option (gogoproto.marshaler_all) = true;
+option (gogoproto.unmarshaler_all) = true;
+
+import "gogoproto/gogo.proto";
+import "cubefs/blobstore/common/sharding/range.proto";
+
+message ListBlobEncodeMarker {
+    cubefs.blobstore.common.sharding.Range range = 1 [(gogoproto.nullable) = false];
+    string marker = 2;
+}

blobstore/api/blobnode/chunk.go

@@ -16,157 +16,24 @@ package blobnode
 
 import (
 	"context"
-	"encoding/binary"
-	"encoding/hex"
-	"errors"
 	"fmt"
-	"time"
 
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
 	bloberr "github.com/cubefs/cubefs/blobstore/common/errors"
 	"github.com/cubefs/cubefs/blobstore/common/proto"
 	"github.com/cubefs/cubefs/blobstore/common/rpc"
 )
 
-const (
-	ChunkStatusDefault  ChunkStatus = iota // 0
-	ChunkStatusNormal                      // 1
-	ChunkStatusReadOnly                    // 2
-	ChunkStatusRelease                     // 3
-	ChunkNumStatus                         // 4
-)
-
-const (
-	ReleaseForUser    = "release for user"
-	ReleaseForCompact = "release for compact"
-)
-
-// Chunk ID
-// vuid + timestamp
-const (
-	chunkVuidLen      = 8
-	chunkTimestmapLen = 8
-	ChunkIdLength     = chunkVuidLen + chunkTimestmapLen
-)
-
-var InvalidChunkId ChunkId = [ChunkIdLength]byte{}
-
-var (
-	_vuidHexLen      = hex.EncodedLen(chunkVuidLen)
-	_timestampHexLen = hex.EncodedLen(chunkTimestmapLen)
-
-	// ${vuid_hex}-${tiemstamp_hex}
-	// |-- 8 Bytes --|-- 1 Bytes --|-- 8 Bytes --|
-	delimiter        = []byte("-")
-	ChunkIdEncodeLen = _vuidHexLen + _timestampHexLen + len(delimiter)
-)
-
-type (
-	ChunkId     [ChunkIdLength]byte
-	ChunkStatus uint8
-)
-
-func (c ChunkId) UnixTime() uint64 {
-	return binary.BigEndian.Uint64(c[chunkVuidLen:ChunkIdLength])
-}
-
-func (c ChunkId) VolumeUnitId() proto.Vuid {
-	return proto.Vuid(binary.BigEndian.Uint64(c[:chunkVuidLen]))
-}
-
-func (c *ChunkId) Marshal() ([]byte, error) {
-	buf := make([]byte, ChunkIdEncodeLen)
-	var i int
-
-	hex.Encode(buf[i:_vuidHexLen], c[:chunkVuidLen])
-	i += _vuidHexLen
-
-	copy(buf[i:i+len(delimiter)], delimiter)
-	i += len(delimiter)
-
-	hex.Encode(buf[i:], c[chunkVuidLen:ChunkIdLength])
-
-	return buf, nil
-}
-
-func (c *ChunkId) Unmarshal(data []byte) error {
-	if len(data) != ChunkIdEncodeLen {
-		panic(errors.New("chunk buf size not match"))
-	}
-
-	var i int
-
-	_, err := hex.Decode(c[:chunkVuidLen], data[i:_vuidHexLen])
-	if err != nil {
-		return err
-	}
-
-	i += _vuidHexLen
-	i += len(delimiter)
-
-	_, err = hex.Decode(c[chunkVuidLen:], data[i:])
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (c ChunkId) String() string {
-	buf, _ := c.Marshal()
-	return string(buf[:])
-}
-
-func (c ChunkId) MarshalJSON() ([]byte, error) {
-	b := make([]byte, ChunkIdEncodeLen+2)
-	b[0], b[ChunkIdEncodeLen+1] = '"', '"'
-
-	buf, _ := c.Marshal()
-	copy(b[1:], buf)
-
-	return b, nil
-}
-
-func (c *ChunkId) UnmarshalJSON(data []byte) (err error) {
-	if len(data) != ChunkIdEncodeLen+2 {
-		return errors.New("failed unmarshal json")
-	}
-
-	return c.Unmarshal(data[1 : ChunkIdEncodeLen+1])
-}
-
-func EncodeChunk(id ChunkId) string {
-	return id.String()
-}
-
-func NewChunkId(vuid proto.Vuid) (chunkId ChunkId) {
-	binary.BigEndian.PutUint64(chunkId[:chunkVuidLen], uint64(vuid))
-	binary.BigEndian.PutUint64(chunkId[chunkVuidLen:ChunkIdLength], uint64(time.Now().UnixNano()))
-	return
-}
-
 func IsValidDiskID(id proto.DiskID) bool {
 	return id != proto.InvalidDiskID
 }
 
-func IsValidChunkId(id ChunkId) bool {
-	return id != InvalidChunkId
+func IsValidChunkID(id clustermgr.ChunkID) bool {
+	return id != clustermgr.InvalidChunkID
 }
 
-func IsValidChunkStatus(status ChunkStatus) bool {
-	return status < ChunkNumStatus
-}
-
-func DecodeChunk(name string) (id ChunkId, err error) {
-	buf := []byte(name)
-	if len(buf) != ChunkIdEncodeLen {
-		return InvalidChunkId, errors.New("invalid chunk name")
-	}
-
-	if err = id.Unmarshal(buf); err != nil {
-		return InvalidChunkId, errors.New("chunk unmarshal failed")
-	}
-
-	return
+func IsValidChunkStatus(status clustermgr.ChunkStatus) bool {
+	return status < clustermgr.ChunkNumStatus
 }
 
 type CreateChunkArgs struct {
@@ -193,14 +60,14 @@ type StatChunkArgs struct {
 	Vuid   proto.Vuid   `json:"vuid"`
 }
 
-func (c *client) StatChunk(ctx context.Context, host string, args *StatChunkArgs) (ci *ChunkInfo, err error) {
+func (c *client) StatChunk(ctx context.Context, host string, args *StatChunkArgs) (ci *clustermgr.ChunkInfo, err error) {
 	if !IsValidDiskID(args.DiskID) {
 		err = bloberr.ErrInvalidDiskId
 		return
 	}
 
 	urlStr := fmt.Sprintf("%v/chunk/stat/diskid/%v/vuid/%v", host, args.DiskID, args.Vuid)
-	ci = new(ChunkInfo)
+	ci = new(clustermgr.ChunkInfo)
 	err = c.GetWith(ctx, urlStr, ci)
 	return
 }
@@ -255,10 +122,10 @@ type ListChunkArgs struct {
 }
 
 type ListChunkRet struct {
-	ChunkInfos []*ChunkInfo `json:"chunk_infos"`
+	ChunkInfos []*clustermgr.ChunkInfo `json:"chunk_infos"`
 }
 
-func (c *client) ListChunks(ctx context.Context, host string, args *ListChunkArgs) (ret []*ChunkInfo, err error) {
+func (c *client) ListChunks(ctx context.Context, host string, args *ListChunkArgs) (ret []*clustermgr.ChunkInfo, err error) {
 	if !IsValidDiskID(args.DiskID) {
 		err = bloberr.ErrInvalidDiskId
 		return
@@ -288,4 +155,5 @@ type BadShard struct {
 	DiskID proto.DiskID
 	Vuid   proto.Vuid
 	Bid    proto.BlobID
+	Err    error
 }

blobstore/api/blobnode/chunk_test.go

@@ -21,58 +21,59 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
 	"github.com/cubefs/cubefs/blobstore/common/proto"
 	"github.com/cubefs/cubefs/blobstore/util/log"
 )
 
 func TestIsValidChunkId(t *testing.T) {
-	id := InvalidChunkId
-	require.Equal(t, false, IsValidChunkId(id))
+	id := clustermgr.InvalidChunkID
+	require.Equal(t, false, IsValidChunkID(id))
 
-	id = ChunkId{0x1}
-	require.Equal(t, true, IsValidChunkId(id))
+	id = clustermgr.ChunkID{0x1}
+	require.Equal(t, true, IsValidChunkID(id))
 }
 
 func TestChunkIdNew(t *testing.T) {
-	chunkid := NewChunkId(101)
-	require.Equal(t, ChunkIdLength, len(chunkid))
-	require.NotEqual(t, InvalidChunkId, chunkid)
+	chunkid := clustermgr.NewChunkID(101)
+	require.Equal(t, clustermgr.ChunkIDLength, len(chunkid))
+	require.NotEqual(t, clustermgr.InvalidChunkID, chunkid)
 
 	expectedVuid := chunkid.VolumeUnitId()
 	require.Equal(t, expectedVuid, proto.Vuid(101))
 
 	chunkname := chunkid.String()
-	require.Equal(t, ChunkIdEncodeLen, len(chunkname))
+	require.Equal(t, clustermgr.ChunkIDEncodeLen, len(chunkname))
 
-	arrs := strings.Split(chunkname, string(delimiter))
+	arrs := strings.Split(chunkname, "-")
 	require.Equal(t, 2, len(arrs))
 	require.Equal(t, "0000000000000065", arrs[0])
 }
 
 func TestChunkId_Marshal(t *testing.T) {
-	chunkid := NewChunkId(101)
+	chunkid := clustermgr.NewChunkID(101)
 
 	data, err := chunkid.Marshal()
 	require.NoError(t, err)
-	require.Equal(t, ChunkIdEncodeLen, len(data))
+	require.Equal(t, clustermgr.ChunkIDEncodeLen, len(data))
 	log.Infof("data:%s", data)
 
-	var newchunk ChunkId
+	var newchunk clustermgr.ChunkID
 	err = newchunk.Unmarshal(data)
 	require.NoError(t, err)
 	require.Equal(t, chunkid, newchunk)
 }
 
 func TestChunkId_MarshalJSON(t *testing.T) {
-	chunkid := NewChunkId(101)
+	chunkid := clustermgr.NewChunkID(101)
 
 	data, err := json.Marshal(chunkid)
 	require.NoError(t, err)
-	require.Equal(t, ChunkIdEncodeLen+2, len(data))
+	require.Equal(t, clustermgr.ChunkIDEncodeLen+2, len(data))
 
 	log.Infof("data:%s", data)
 
-	var newchunk ChunkId
+	var newchunk clustermgr.ChunkID
 	err = json.Unmarshal(data, &newchunk)
 	require.NoError(t, err)
 	require.Equal(t, chunkid, newchunk)

blobstore/api/blobnode/client.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/cubefs/cubefs/blobstore/api/clustermgr"
 	"github.com/cubefs/cubefs/blobstore/common/errors"
 	"github.com/cubefs/cubefs/blobstore/common/proto"
 	"github.com/cubefs/cubefs/blobstore/common/rpc"
@@ -48,9 +49,9 @@ func (c *client) Close(ctx context.Context, host string) (err error) {
 	return nil
 }
 
-func (c *client) Stat(ctx context.Context, host string) (dis []*DiskInfo, err error) {
+func (c *client) Stat(ctx context.Context, host string) (dis []*clustermgr.BlobNodeDiskInfo, err error) {
 	urlStr := fmt.Sprintf("%v/stat", host)
-	dis = make([]*DiskInfo, 0)
+	dis = make([]*clustermgr.BlobNodeDiskInfo, 0)
 	err = c.GetWith(ctx, urlStr, &dis)
 	return
 }
@@ -64,17 +65,46 @@ type InspectRateArgs struct {
 	Rate int `json:"rate"`
 }
 
+type InspectCleanMetricArgs struct {
+	DiskID proto.DiskID `json:"diskid"`
+}
+
+// LimiterStat single limiter using counter
+type LimiterStat struct {
+	Running   int `json:"running"`
+	Capacity  int `json:"capacity"`
+	Remaining int `json:"remaining"`
+}
+
+// IoLimiterStats single level qos type limiter stats
+type IoLimiterStats struct {
+	Concurrency    LimiterStat `json:"concurrency"`
+	BidConcurrency LimiterStat `json:"bid_concurrency"`
+	Bandwidth      LimiterStat `json:"bandwidth"`
+}
+
+type QosStatArgs struct {
+	DiskID proto.DiskID `json:"diskid"`
+}
+
+func (c *client) QosStat(ctx context.Context, host string, args *QosStatArgs) (stat map[proto.DiskID]map[string]IoLimiterStats, err error) {
+	urlStr := fmt.Sprintf("%s/disk/stat/diskid/%d", host, args.DiskID)
+	stat = make(map[proto.DiskID]map[string]IoLimiterStats)
+	err = c.GetWith(ctx, urlStr, stat)
+	return
+}
+
 type DiskStatArgs struct {
 	DiskID proto.DiskID `json:"diskid"`
 }
 
-func (c *client) DiskInfo(ctx context.Context, host string, args *DiskStatArgs) (di *DiskInfo, err error) {
+func (c *client) DiskInfo(ctx context.Context, host string, args *DiskStatArgs) (di *clustermgr.BlobNodeDiskInfo, err error) {
 	if !IsValidDiskID(args.DiskID) {
 		return nil, errors.ErrInvalidDiskId
 	}
 
 	urlStr := fmt.Sprintf("%v/disk/stat/diskid/%v", host, args.DiskID)
-	di = new(DiskInfo)
+	di = new(clustermgr.BlobNodeDiskInfo)
 	err = c.GetWith(ctx, urlStr, di)
 	return
 }
@@ -83,20 +113,22 @@ type StorageAPI interface {
 	String(ctx context.Context, host string) string
 	IsOnline(ctx context.Context, host string) bool
 	Close(ctx context.Context, host string) error
-	Stat(ctx context.Context, host string) (infos []*DiskInfo, err error)
-	DiskInfo(ctx context.Context, host string, args *DiskStatArgs) (di *DiskInfo, err error)
+	Stat(ctx context.Context, host string) (infos []*clustermgr.BlobNodeDiskInfo, err error)
+	DiskInfo(ctx context.Context, host string, args *DiskStatArgs) (di *clustermgr.BlobNodeDiskInfo, err error)
+	QosStat(ctx context.Context, host string, args *QosStatArgs) (stat map[proto.DiskID]map[string]IoLimiterStats, err error)
 
 	// chunks
 	CreateChunk(ctx context.Context, host string, args *CreateChunkArgs) (err error)
-	StatChunk(ctx context.Context, host string, args *StatChunkArgs) (ci *ChunkInfo, err error)
+	StatChunk(ctx context.Context, host string, args *StatChunkArgs) (ci *clustermgr.ChunkInfo, err error)
 	ReleaseChunk(ctx context.Context, host string, args *ChangeChunkStatusArgs) (err error)
 	SetChunkReadonly(ctx context.Context, host string, args *ChangeChunkStatusArgs) (err error)
 	SetChunkReadwrite(ctx context.Context, host string, args *ChangeChunkStatusArgs) (err error)
-	ListChunks(ctx context.Context, host string, args *ListChunkArgs) (cis []*ChunkInfo, err error)
+	ListChunks(ctx context.Context, host string, args *ListChunkArgs) (cis []*clustermgr.ChunkInfo, err error)
 
 	// shard
 	GetShard(ctx context.Context, host string, args *GetShardArgs) (body io.ReadCloser, shardCrc uint32, err error)
 	RangeGetShard(ctx context.Context, host string, args *RangeGetShardArgs) (body io.ReadCloser, shardCrc uint32, err error)
+	GetShards(ctx context.Context, host string, args *GetShardsArgs) (getter ShardGetter, err error)
 	PutShard(ctx context.Context, host string, args *PutShardArgs) (crc uint32, err error)
 	StatShard(ctx context.Context, host string, args *StatShardArgs) (si *ShardInfo, err error)
 	MarkDeleteShard(ctx context.Context, host string, args *DeleteShardArgs) (err error)

blobstore/api/blobnode/client_test.go

@@ -17,7 +17,7 @@ package blobnode
 import (
 	"bytes"
 	"context"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"testing"
@@ -108,7 +108,7 @@ func TestNewBlobNodeClient(t *testing.T) {
 	body, _, err := cli.RangeGetShard(ctx, mockServer.URL, getShardArgs)
 	require.NoError(t, err)
 	if body != nil {
-		b, _ := ioutil.ReadAll(body)
+		b, _ := io.ReadAll(body)
 		span.Infof("body: %s\n", b)
 	}
 

blobstore/api/blobnode/datainfo.go

@@ -1,65 +0,0 @@
-// Copyright 2022 The CubeFS Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-// implied. See the License for the specific language governing
-// permissions and limitations under the License.
-
-package blobnode
-
-import (
-	"time"
-
-	"github.com/cubefs/cubefs/blobstore/common/proto"
-)
-
-type DiskHeartBeatInfo struct {
-	DiskID       proto.DiskID `json:"disk_id"`
-	Used         int64        `json:"used"`           // disk used space
-	Free         int64        `json:"free"`           // remaining free space on the disk
-	Size         int64        `json:"size"`           // total physical disk space
-	MaxChunkCnt  int64        `json:"max_chunk_cnt"`  // note: maintained by clustermgr
-	FreeChunkCnt int64        `json:"free_chunk_cnt"` // note: maintained by clustermgr
-	UsedChunkCnt int64        `json:"used_chunk_cnt"` // current number of chunks on the disk
-}
-
-type DiskInfo struct {
-	ClusterID    proto.ClusterID  `json:"cluster_id"`
-	Idc          string           `json:"idc"`
-	Rack         string           `json:"rack"`
-	Host         string           `json:"host"`
-	Path         string           `json:"path"`
-	Status       proto.DiskStatus `json:"status"` // normal、broken、repairing、repaired、dropped
-	Readonly     bool             `json:"readonly"`
-	CreateAt     time.Time        `json:"create_time"`
-	LastUpdateAt time.Time        `json:"last_update_time"`
-	DiskHeartBeatInfo
-}
-
-type ChunkInfo struct {
-	Id         ChunkId      `json:"id"`
-	Vuid       proto.Vuid   `json:"vuid"`
-	DiskID     proto.DiskID `json:"diskid"`
-	Total      uint64       `json:"total"`  // ChunkSize
-	Used       uint64       `json:"used"`   // user data size
-	Free       uint64       `json:"free"`   // ChunkSize - Used
-	Size       uint64       `json:"size"`   // Chunk File Size (logic size)
-	Status     ChunkStatus  `json:"status"` // normal、readOnly
-	Compacting bool         `json:"compacting"`
-}
-
-type ShardInfo struct {
-	Vuid   proto.Vuid   `json:"vuid"`
-	Bid    proto.BlobID `json:"bid"`
-	Size   int64        `json:"size"`
-	Crc    uint32       `json:"crc"`
-	Flag   ShardStatus  `json:"flag"` // 1:normal,2:markDelete
-	Inline bool         `json:"inline"`
-}

blobstore/api/blobnode/iostat.go

@@ -28,35 +28,48 @@ const (
 type IOType uint64
 
 const (
-	NormalIO     IOType = iota // From: external: user io: read/write
-	BackgroundIO               // From: external: background io: shard repair;disk repair, delete, compact;balance, drop, manual migrate; internal, inspect
-	IOTypeMax                  // 2
-	IOTypeOldMax = 8           // For compatibility with previous versions
+	WriteIO      IOType = iota // From: external: user io: write
+	BackgroundIO               // From: external: background io: shard repair; disk repair, compact;balance, drop, manual migrate
+	ReadIO
+	DeleteIO
+
+	IOTypeMax // 4
 )
 
-var IOtypemap = [...]string{
-	"normal",
-	"background",
+var (
+	ioTypeArray = [...]string{
+		"write",
+		"background",
+		"read",
+		"delete",
+	}
+	revertIOMap = make(map[string]IOType, IOTypeMax)
+)
+
+var _ = ioTypeArray[IOTypeMax-1]
+
+func init() {
+	for id, str := range ioTypeArray {
+		revertIOMap[str] = IOType(id)
+	}
 }
 
-var _ = IOtypemap[IOTypeMax-1]
-
 func (it IOType) IsValid() bool {
-	return it >= NormalIO && it < IOTypeOldMax
+	return it >= WriteIO && it < IOTypeMax
 }
 
 func (it IOType) String() string {
-	return IOtypemap[it]
+	return ioTypeArray[it]
 }
 
 func (it IOType) IsHighLevel() bool {
-	return it == NormalIO
+	return it == WriteIO || it == ReadIO
 }
 
 func GetIoType(ctx context.Context) IOType {
 	v := ctx.Value(_ioFlowStatKey)
 	if v == nil {
-		return NormalIO
+		return IOTypeMax
 	}
 	return v.(IOType)
 }
@@ -64,3 +77,15 @@ func GetIoType(ctx context.Context) IOType {
 func SetIoType(ctx context.Context, iot IOType) context.Context {
 	return context.WithValue(ctx, _ioFlowStatKey, iot)
 }
+
+func StringToIOType(str string) IOType {
+	tp, exist := revertIOMap[str]
+	if exist {
+		return tp
+	}
+	return IOTypeMax
+}
+
+func GetAllIOType() [IOTypeMax]string {
+	return ioTypeArray
+}

blobstore/api/blobnode/iostat_test.go

@@ -24,14 +24,104 @@ import (
 func TestGetIoType(t *testing.T) {
 	ctx := context.TODO()
 
+	// nil ctx, should return IOTypeMax/invalid
 	iotype := GetIoType(ctx)
-	require.Equal(t, NormalIO, iotype)
+	require.Equal(t, IOTypeMax, iotype)
 
 	ctx0 := context.WithValue(ctx, _ioFlowStatKey, BackgroundIO)
 	iotype = GetIoType(ctx0)
 	require.Equal(t, BackgroundIO, iotype)
 
-	ctx1 := context.WithValue(ctx0, _ioFlowStatKey, BackgroundIO)
+	ctx1 := SetIoType(ctx, ReadIO)
 	iotype = GetIoType(ctx1)
-	require.Equal(t, BackgroundIO, iotype)
+	require.Equal(t, ReadIO, iotype)
+
+	ctx = SetIoType(ctx, DeleteIO)
+	iotype = GetIoType(ctx)
+	require.Equal(t, DeleteIO, iotype)
+}
+
+func TestIOType_IsValid(t *testing.T) {
+	tests := []struct {
+		name     string
+		ioType   IOType
+		expected bool
+	}{
+		{name: "WriteIO is valid", ioType: WriteIO, expected: true},
+		{name: "BackgroundIO is valid", ioType: BackgroundIO, expected: true},
+		{name: "ReadIO is valid", ioType: ReadIO, expected: true},
+		{name: "DeleteIO is valid", ioType: DeleteIO, expected: true},
+		{name: "IOTypeMax is invalid", ioType: IOTypeMax, expected: false},
+		{name: "IOTypeMax+1 is invalid", ioType: IOTypeMax + 1, expected: false},
+		{name: "IOTypeMax-1 is valid", ioType: IOTypeMax - 1, expected: true},
+		{name: "Zero value is valid (WriteIO)", ioType: 0, expected: true},
+		{name: "Large value is invalid", ioType: 100, expected: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := tt.ioType.IsValid()
+			require.Equal(t, tt.expected, result)
+		})
+	}
+}
+
+func TestIOType_String(t *testing.T) {
+	tests := []struct {
+		name     string
+		ioType   IOType
+		expected string
+	}{
+		{name: "WriteIO string", ioType: WriteIO, expected: "write"},
+		{name: "BackgroundIO string", ioType: BackgroundIO, expected: "background"},
+		{name: "ReadIO string", ioType: ReadIO, expected: "read"},
+		{name: "DeleteIO string", ioType: DeleteIO, expected: "delete"},
+		{name: "IOTypeMax string (should panic or return empty)", ioType: IOTypeMax, expected: ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.ioType >= IOTypeMax {
+				// Test that it doesn't panic
+				require.Panics(t, func() {
+					_ = tt.ioType.String()
+				})
+			} else {
+				result := tt.ioType.String()
+				require.Equal(t, tt.expected, result)
+			}
+		})
+	}
+}
+
+func TestIOtypemap(t *testing.T) {
+	t.Run("ioTypeArray contains correct values", func(t *testing.T) {
+		require.Len(t, ioTypeArray, int(IOTypeMax))
+		require.Equal(t, "write", ioTypeArray[WriteIO])
+		require.Equal(t, "background", ioTypeArray[BackgroundIO])
+		require.Equal(t, "read", ioTypeArray[ReadIO])
+		require.Equal(t, "delete", ioTypeArray[DeleteIO])
+	})
+
+	t.Run("ioTypeArray index validation", func(t *testing.T) {
+		// Test that all valid IO types have corresponding strings
+		for i := WriteIO; i < IOTypeMax; i++ {
+			require.NotEmpty(t, ioTypeArray[i])
+		}
+	})
+}
+
+func TestRevertIOtypeMap(t *testing.T) {
+	t.Run("revertIOMap contains correct values", func(t *testing.T) {
+		require.Len(t, revertIOMap, int(IOTypeMax))
+		require.Equal(t, revertIOMap["write"], WriteIO)
+		require.Equal(t, revertIOMap["background"], BackgroundIO)
+		require.Equal(t, revertIOMap["read"], ReadIO)
+		require.Equal(t, revertIOMap["delete"], DeleteIO)
+
+		require.Equal(t, revertIOMap[WriteIO.String()], WriteIO)
+		require.Equal(t, revertIOMap[BackgroundIO.String()], BackgroundIO)
+		require.Equal(t, revertIOMap[ReadIO.String()], ReadIO)
+		require.Equal(t, revertIOMap[DeleteIO.String()], DeleteIO)
+	})
 }

blobstore/api/blobnode/shard.go

@@ -16,12 +16,14 @@ package blobnode
 
 import (
 	"context"
+	"encoding/binary"
 	"fmt"
 	"io"
 	"math"
 	"net/http"
 	"strconv"
 
+	"github.com/cubefs/cubefs/blobstore/blobnode/base"
 	bloberr "github.com/cubefs/cubefs/blobstore/common/errors"
 	"github.com/cubefs/cubefs/blobstore/common/proto"
 	"github.com/cubefs/cubefs/blobstore/common/rpc"
@@ -29,9 +31,29 @@ import (
 )
 
 const (
-	MaxShardSize = math.MaxUint32
+	MaxShardSize        = math.MaxUint32
+	GetShardsHeaderSize = 4
 )
 
+type ShardInfo struct {
+	Vuid   proto.Vuid   `json:"vuid"`
+	Bid    proto.BlobID `json:"bid"`
+	Size   int64        `json:"size"`
+	Crc    uint32       `json:"crc"`
+	Offset int64        `json:"offset"`
+	Flag   ShardStatus  `json:"flag"` // 1:normal,2:markDelete
+	Inline bool         `json:"inline"`
+
+	NopData bool `json:"nopdata"` // data all zero
+}
+
+type BidInfo struct {
+	Bid    proto.BlobID `json:"bid"`
+	Size   int64        `json:"size"`
+	Offset int64        `json:"offset"`
+	Crc    uint32       `json:"crc"`
+}
+
 type ShardStatus uint8
 
 const (
@@ -42,8 +64,11 @@ const (
 
 const (
 	ShardDataInline = 0x80 // 1000 0000
+	ShardDataNop    = 0x40 // 0100 0000
 )
 
+var putWithCrcOption = []rpc.Option{rpc.WithCrcEncode()}
+
 type PutShardArgs struct {
 	DiskID proto.DiskID `json:"diskid"`
 	Vuid   proto.Vuid   `json:"vuid"`
@@ -51,6 +76,8 @@ type PutShardArgs struct {
 	Size   int64        `json:"size"`
 	Type   IOType       `json:"iotype,omitempty"`
 	Body   io.Reader    `json:"-"`
+
+	NopData bool `json:"nopdata,omitempty"`
 }
 
 type PutShardRet struct {
@@ -77,13 +104,20 @@ func (c *client) PutShard(ctx context.Context, host string, args *PutShardArgs)
 	}
 	urlStr := fmt.Sprintf("%v/shard/put/diskid/%v/vuid/%v/bid/%v/size/%v?iotype=%d",
 		host, args.DiskID, args.Vuid, args.Bid, args.Size, args.Type)
+	if args.NopData {
+		urlStr += "&nopdata=true"
+	}
 	req, err := http.NewRequest(http.MethodPost, urlStr, args.Body)
 	if err != nil {
+		err = convertEIO(err)
 		return
 	}
-	req.ContentLength = args.Size
-	err = c.DoWith(ctx, req, ret, rpc.WithCrcEncode())
-	if err == nil {
+	var opts []rpc.Option
+	if !args.NopData {
+		req.ContentLength = args.Size
+		opts = putWithCrcOption
+	}
+	if err = c.DoWith(ctx, req, ret, opts...); err == nil {
 		crc = ret.Crc
 	}
 
@@ -115,6 +149,7 @@ func (c *client) GetShard(ctx context.Context, host string, args *GetShardArgs)
 
 	resp, err := c.Get(ctx, urlStr)
 	if err != nil {
+		err = convertEIO(err)
 		return nil, 0, err
 	}
 
@@ -161,6 +196,7 @@ func (c *client) RangeGetShard(ctx context.Context, host string, args *RangeGetS
 
 	req, err := http.NewRequest(http.MethodGet, urlStr, nil)
 	if err != nil {
+		err = convertEIO(err)
 		span.Errorf("Failed new req. urlStr:%s, err:%v", urlStr, err)
 		return
 	}
@@ -207,7 +243,8 @@ func (c *client) MarkDeleteShard(ctx context.Context, host string, args *DeleteS
 
 	urlStr := fmt.Sprintf("%v/shard/markdelete/diskid/%v/vuid/%v/bid/%v", host, args.DiskID, args.Vuid, args.Bid)
 	err = c.PostWith(ctx, urlStr, nil, rpc.NoneBody)
-	return
+
+	return convertEIO(err)
 }
 
 func (c *client) DeleteShard(ctx context.Context, host string, args *DeleteShardArgs) (err error) {
@@ -218,13 +255,15 @@ func (c *client) DeleteShard(ctx context.Context, host string, args *DeleteShard
 
 	urlStr := fmt.Sprintf("%v/shard/delete/diskid/%v/vuid/%v/bid/%v", host, args.DiskID, args.Vuid, args.Bid)
 	err = c.PostWith(ctx, urlStr, nil, rpc.NoneBody)
-	return
+
+	return convertEIO(err)
 }
 
 type StatShardArgs struct {
 	DiskID proto.DiskID `json:"diskid"`
 	Vuid   proto.Vuid   `json:"vuid"`
 	Bid    proto.BlobID `json:"bid"`
+	Type   IOType       `json:"iotype,omitempty"`
 }
 
 func (c *client) StatShard(ctx context.Context, host string, args *StatShardArgs) (si *ShardInfo, err error) {
@@ -233,11 +272,12 @@ func (c *client) StatShard(ctx context.Context, host string, args *StatShardArgs
 		return
 	}
 
-	urlStr := fmt.Sprintf("%v/shard/stat/diskid/%v/vuid/%v/bid/%v",
-		host, args.DiskID, args.Vuid, args.Bid)
+	urlStr := fmt.Sprintf("%v/shard/stat/diskid/%v/vuid/%v/bid/%v?iotype=%d",
+		host, args.DiskID, args.Vuid, args.Bid, args.Type)
 	si = &ShardInfo{}
 	err = c.GetWith(ctx, urlStr, si)
-	return
+
+	return si, convertEIO(err)
 }
 
 type ListShardsArgs struct {
@@ -265,8 +305,129 @@ func (c *client) ListShards(ctx context.Context, host string, args *ListShardsAr
 	listRet := ListShardsRet{}
 	err = c.GetWith(ctx, urlStr, &listRet)
 	if err != nil {
+		err = convertEIO(err)
 		return nil, proto.InValidBlobID, err
 	}
 
 	return listRet.ShardInfos, listRet.Next, nil
 }
+
+func convertEIO(err error) error {
+	if base.IsEIO(err) {
+		return bloberr.ErrDiskBroken
+	}
+
+	return err
+}
+
+type GetShardsArgs struct {
+	DiskID proto.DiskID `json:"diskid"`
+	Vuid   proto.Vuid   `json:"vuid" `
+	Bids   []BidInfo    `json:"bids"`
+	Type   IOType       `json:"type"`
+}
+
+func (c *client) GetShards(ctx context.Context, host string, args *GetShardsArgs) (getter ShardGetter, err error) {
+	if !args.Type.IsValid() {
+		err = bloberr.ErrInvalidParam
+		return
+	}
+	if !IsValidDiskID(args.DiskID) {
+		err = bloberr.ErrInvalidDiskId
+		return
+	}
+	urlStr := fmt.Sprintf("%v/shards/get", host)
+
+	resp, err := c.Post(ctx, urlStr, args)
+	if err != nil {
+		err = convertEIO(err)
+		return
+	}
+	if resp.StatusCode/100 != 2 {
+		defer resp.Body.Close()
+		err = rpc.ParseResponseErr(resp)
+		return
+	}
+	return &shardGetter{bids: args.Bids, body: resp.Body}, nil
+}
+
+type ShardGetter interface {
+	// NextShard before read next shard must read all data of last shard, if not will get unexpect error
+	NextShard(ctx context.Context) (body io.ReadCloser, err error, ok bool)
+	Close() error
+}
+
+type shardGetter struct {
+	body io.ReadCloser
+	bids []BidInfo
+	idx  int
+}
+
+func (b *shardGetter) NextShard(ctx context.Context) (io.ReadCloser, error, bool) {
+	span := trace.SpanFromContextSafe(ctx)
+	if b.idx >= len(b.bids) {
+		return nil, nil, false
+	}
+	var header ShardsHeader
+	_, err := io.ReadFull(b.body, header[:])
+	if err != nil {
+		return nil, err, true
+	}
+	code := header.Get()
+	if code != 200 {
+		span.Errorf("download shard failed, errCode: %s", code)
+		return nil, bloberr.ErrBidNotMatch, true
+	}
+	bid := b.bids[b.idx]
+	b.idx++
+	return io.NopCloser(io.LimitReader(b.body, bid.Size)), nil, true
+}
+
+func (b *shardGetter) Close() error {
+	return b.body.Close()
+}
+
+type shardWriter struct {
+	header        int
+	headerWritten bool
+
+	shard io.WriterTo
+}
+
+func NewShardWriter(header int, shard io.WriterTo) io.WriterTo {
+	return &shardWriter{header: header, shard: shard}
+}
+
+func (s *shardWriter) WriteTo(w io.Writer) (int64, error) {
+	if s.headerWritten {
+		return s.shard.WriteTo(w)
+	}
+	// write header
+	var header ShardsHeader
+	header.Set(s.header)
+	start := int64(0)
+	for start < int64(len(header)) {
+		n, err := w.Write(header[start:])
+		if err != nil {
+			return start, err
+		}
+		start += int64(n)
+	}
+	s.headerWritten = true
+	if s.header != http.StatusOK {
+		return int64(len(header)), bloberr.ErrBidNotMatch
+	}
+	// write data
+	n, err := s.shard.WriteTo(w)
+	return n + start, err
+}
+
+type ShardsHeader [GetShardsHeaderSize]byte
+
+func (s *ShardsHeader) Set(code int) {
+	binary.BigEndian.PutUint32(s[:], uint32(code))
+}
+
+func (s *ShardsHeader) Get() int {
+	return int(binary.BigEndian.Uint32(s[:]))
+}

blobstore/api/blobnode/shard_test.go

@@ -15,13 +15,26 @@
 package blobnode
 
 import (
+	"syscall"
 	"testing"
 
+	"github.com/cubefs/cubefs/util/errors"
 	"github.com/stretchr/testify/require"
+
+	bloberr "github.com/cubefs/cubefs/blobstore/common/errors"
 )
 
 func TestShardStatus(t *testing.T) {
 	require.Equal(t, ShardStatusDefault, ShardStatus(0))
 	require.Equal(t, ShardStatusNormal, ShardStatus(1))
 	require.Equal(t, ShardStatusMarkDelete, ShardStatus(2))
+
+	var err error
+	require.Nil(t, convertEIO(err))
+
+	err = syscall.EIO
+	require.ErrorIs(t, convertEIO(err), bloberr.ErrDiskBroken)
+
+	err = errors.New("input/output error")
+	require.ErrorIs(t, convertEIO(err), bloberr.ErrDiskBroken)
 }

blobstore/api/clustermgr/catalog.go

@@ -0,0 +1,108 @@
+package clustermgr
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/base64"
+	"encoding/binary"
+	"fmt"
+	"time"
+)
+
+const (
+	hashBytesLength       = 16
+	authVersionV1   uint8 = 1
+)
+
+func (c *Client) CreateSpace(ctx context.Context, args *CreateSpaceArgs) (err error) {
+	err = c.PostWith(ctx, "/space/create", nil, args)
+	return
+}
+
+func (c *Client) GetSpaceByName(ctx context.Context, args *GetSpaceByNameArgs) (ret *Space, err error) {
+	ret = &Space{}
+	err = c.GetWith(ctx, "/space/get?name="+args.Name, ret)
+	return
+}
+
+func (c *Client) GetSpaceByID(ctx context.Context, args *GetSpaceByIDArgs) (ret *Space, err error) {
+	ret = &Space{}
+	err = c.GetWith(ctx, "/space/get?space_id="+args.SpaceID.ToString(), ret)
+	return
+}
+
+func (c *Client) AuthSpace(ctx context.Context, args *AuthSpaceArgs) (err error) {
+	err = c.GetWith(ctx, fmt.Sprintf("/space/auth?name=%s&token=%s", args.Name, args.Token), nil)
+	return
+}
+
+func (c *Client) ListSpace(ctx context.Context, args *ListSpaceArgs) (ret ListSpaceRet, err error) {
+	err = c.GetWith(ctx, fmt.Sprintf("/space/list?marker=%d&count=%d", args.Marker, args.Count), &ret)
+	return
+}
+
+type AuthInfo struct {
+	AccessKey string
+	SecretKey string
+}
+
+// EncodeAuthInfo SDK generates token based on ak/sk
+func EncodeAuthInfo(auth *AuthInfo) (token string, err error) {
+	timeStamp := time.Now().Unix()
+	hashBytes := CalculateHash(auth, timeStamp)
+
+	w := bytes.NewBuffer([]byte{})
+	if err = binary.Write(w, binary.LittleEndian, authVersionV1); err != nil {
+		return
+	}
+	if err = binary.Write(w, binary.LittleEndian, &timeStamp); err != nil {
+		return
+	}
+	if err = binary.Write(w, binary.LittleEndian, &hashBytes); err != nil {
+		return
+	}
+	return base64.URLEncoding.EncodeToString(w.Bytes()), nil
+}
+
+// DecodeAuthInfo server parses token
+func DecodeAuthInfo(token string) (timestamp int64, hashBytes []byte, err error) {
+	b, err := base64.URLEncoding.DecodeString(token)
+	if err != nil {
+		return
+	}
+
+	var authVersion uint8
+	hashBytes = make([]byte, hashBytesLength)
+	r := bytes.NewBuffer(b)
+	if err = binary.Read(r, binary.LittleEndian, &authVersion); err != nil {
+		return
+	}
+	if authVersion == authVersionV1 {
+		if err = binary.Read(r, binary.LittleEndian, &timestamp); err != nil {
+			return
+		}
+		if err = binary.Read(r, binary.LittleEndian, &hashBytes); err != nil {
+			return
+		}
+	}
+	return
+}
+
+// CalculateHash server caculates hash based on ak/sk and timeStamp
+func CalculateHash(auth *AuthInfo, timeStamp int64) (hashBytes []byte) {
+	b := make([]byte, 8)
+	binary.LittleEndian.PutUint64(b, uint64(timeStamp))
+	hash := md5.New()
+	hash.Write(b)
+	hash.Write([]byte(auth.AccessKey))
+	hash.Write([]byte(auth.SecretKey))
+	hashBytes = hash.Sum(nil)
+	return hashBytes
+}
+
+func (c *Client) GetCatalogChanges(ctx context.Context, args *GetCatalogChangesArgs) (ret *GetCatalogChangesRet, err error) {
+	ret = &GetCatalogChangesRet{}
+	err = c.GetWith(ctx, fmt.Sprintf("/catalogchanges/get?route_version=%d&node_id=%d", args.RouteVersion, args.NodeID), ret)
+	return
+}

blobstore/api/clustermgr/catalog.proto

@@ -0,0 +1,96 @@
+syntax = "proto3";
+
+package cubefs.blobstore.api.clustermgr;
+option go_package = "./;clustermgr";
+
+option (gogoproto.sizer_all) = true;
+option (gogoproto.marshaler_all) = true;
+option (gogoproto.unmarshaler_all) = true;
+option (gogoproto.goproto_unkeyed_all) = true;
+option (gogoproto.goproto_unrecognized_all) = true;
+option (gogoproto.goproto_sizecache_all) = true;
+option (gogoproto.goproto_stringer_all) = false;
+option (gogoproto.stringer_all) = true;
+option (gogoproto.gostring_all) = true;
+
+import "gogoproto/gogo.proto";
+import "google/protobuf/any.proto";
+import "cubefs/blobstore/api/clustermgr/shard.proto";
+
+
+message Space {
+  uint32 space_id = 1 [(gogoproto.customname) = "SpaceID", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.SpaceID"];
+  string name = 2;
+  uint32 status = 3[(gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.SpaceStatus"];
+  repeated FieldMeta field_metas = 4 [(gogoproto.nullable) = false];
+  string acc_key = 5;
+  string sec_key = 6;
+}
+
+message FieldMeta {
+  uint32 id = 1 [(gogoproto.customname) = "ID", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.FieldID"];
+  string name = 2;
+  uint32 field_type = 3 [(gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.FieldType"];
+  uint32 index_option = 4 [(gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.IndexOption"];
+}
+
+message CreateSpaceArgs {
+  string name = 1;
+  repeated FieldMeta field_metas = 2 [(gogoproto.nullable) = false];
+}
+
+message GetSpaceByNameArgs {
+  string name = 1;
+}
+
+message GetSpaceByIDArgs {
+  uint32 space_id = 1 [(gogoproto.customname) = "SpaceID", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.SpaceID"];
+}
+
+message GetSpaceArgs {
+  string name = 1;
+  uint32 space_id = 2 [(gogoproto.customname) = "SpaceID", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.SpaceID"];
+}
+
+message AuthSpaceArgs {
+  string name = 1;
+  string token = 2;
+}
+
+message CatalogChangeShardAdd {
+  uint32 shard_id = 1 [(gogoproto.customname) = "ShardID", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.ShardID"];
+  uint64 route_version = 2 [(gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.RouteVersion"];
+  repeated ShardUnitInfo units = 3 [(gogoproto.nullable) = false];
+}
+
+message CatalogChangeShardUpdate {
+  uint32 shard_id = 1 [(gogoproto.customname) = "ShardID", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.ShardID"];
+  uint64 route_version = 2 [(gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.RouteVersion"];
+  ShardUnitInfo unit = 3 [(gogoproto.nullable) = false];
+}
+
+message CatalogChangeItem {
+  uint64 route_version = 1 [(gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.RouteVersion"];
+  uint32 type = 2 [(gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.CatalogChangeItemType"];
+  google.protobuf.Any item =3;
+}
+
+message GetCatalogChangesArgs {
+  uint64 route_version = 1 [(gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.RouteVersion"];
+  uint32 node_id = 2 [(gogoproto.customname) = "NodeID", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.NodeID"];
+}
+
+message GetCatalogChangesRet {
+  uint64 route_version = 1 [(gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.RouteVersion"];
+  repeated CatalogChangeItem items = 2 [(gogoproto.nullable) = false];
+}
+
+message ListSpaceArgs {
+  uint32 marker = 1 [(gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.SpaceID"];
+  uint32 count = 2;
+}
+
+message ListSpaceRet {
+  repeated Space spaces = 1;
+  uint32 marker = 2 [(gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.SpaceID"];
+}

blobstore/api/clustermgr/chunk.go

@@ -0,0 +1,181 @@
+// Copyright 2024 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package clustermgr
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"time"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+type ChunkInfo struct {
+	Id         ChunkID      `json:"id"`
+	Vuid       proto.Vuid   `json:"vuid"`
+	DiskID     proto.DiskID `json:"diskid"`
+	Total      uint64       `json:"total"`  // ChunkSize
+	Used       uint64       `json:"used"`   // user data size
+	Free       uint64       `json:"free"`   // ChunkSize - Used
+	Size       uint64       `json:"size"`   // Chunk File Size (logic size)
+	Status     ChunkStatus  `json:"status"` // normal、readOnly
+	Compacting bool         `json:"compacting"`
+}
+
+const (
+	ChunkStatusDefault  ChunkStatus = iota // 0
+	ChunkStatusNormal                      // 1
+	ChunkStatusReadOnly                    // 2
+	ChunkStatusRelease                     // 3
+	ChunkNumStatus                         // 4
+)
+
+const (
+	ReleaseForUser    = "release for user"
+	ReleaseForCompact = "release for compact"
+)
+
+// Chunk ID
+// vuid + timestamp
+const (
+	chunkVuidLen      = 8
+	chunkTimestmapLen = 8
+	ChunkIDLength     = chunkVuidLen + chunkTimestmapLen
+)
+
+var InvalidChunkID ChunkID = [ChunkIDLength]byte{}
+
+var (
+	_vuidHexLen      = hex.EncodedLen(chunkVuidLen)
+	_timestampHexLen = hex.EncodedLen(chunkTimestmapLen)
+
+	// ${vuid_hex}-${tiemstamp_hex}
+	// |-- 8 Bytes --|-- 1 Bytes --|-- 8 Bytes --|
+	delimiter        = []byte("-")
+	ChunkIDEncodeLen = _vuidHexLen + _timestampHexLen + len(delimiter)
+)
+
+type (
+	ChunkID     [ChunkIDLength]byte
+	ChunkStatus uint8
+)
+
+func (c ChunkID) UnixTime() uint64 {
+	return binary.BigEndian.Uint64(c[chunkVuidLen:ChunkIDLength])
+}
+
+func (c ChunkID) VolumeUnitId() proto.Vuid {
+	return proto.Vuid(binary.BigEndian.Uint64(c[:chunkVuidLen]))
+}
+
+func (c *ChunkID) Marshal() ([]byte, error) {
+	buf := make([]byte, ChunkIDEncodeLen)
+	var i int
+
+	hex.Encode(buf[i:_vuidHexLen], c[:chunkVuidLen])
+	i += _vuidHexLen
+
+	copy(buf[i:i+len(delimiter)], delimiter)
+	i += len(delimiter)
+
+	hex.Encode(buf[i:], c[chunkVuidLen:ChunkIDLength])
+
+	return buf, nil
+}
+
+func (c *ChunkID) Unmarshal(data []byte) error {
+	if len(data) != ChunkIDEncodeLen {
+		panic(errors.New("chunk buf size not match"))
+	}
+
+	var i int
+
+	_, err := hex.Decode(c[:chunkVuidLen], data[i:_vuidHexLen])
+	if err != nil {
+		return err
+	}
+
+	i += _vuidHexLen
+	i += len(delimiter)
+
+	_, err = hex.Decode(c[chunkVuidLen:], data[i:])
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c ChunkID) String() string {
+	buf, _ := c.Marshal()
+	return string(buf[:])
+}
+
+func (c ChunkID) MarshalJSON() ([]byte, error) {
+	b := make([]byte, ChunkIDEncodeLen+2)
+	b[0], b[ChunkIDEncodeLen+1] = '"', '"'
+
+	buf, _ := c.Marshal()
+	copy(b[1:], buf)
+
+	return b, nil
+}
+
+func (c *ChunkID) UnmarshalJSON(data []byte) (err error) {
+	if len(data) != ChunkIDEncodeLen+2 {
+		return errors.New("failed unmarshal json")
+	}
+
+	return c.Unmarshal(data[1 : ChunkIDEncodeLen+1])
+}
+
+func EncodeChunk(id ChunkID) string {
+	return id.String()
+}
+
+func NewChunkID(vuid proto.Vuid) (chunkId ChunkID) {
+	binary.BigEndian.PutUint64(chunkId[:chunkVuidLen], uint64(vuid))
+	binary.BigEndian.PutUint64(chunkId[chunkVuidLen:ChunkIDLength], uint64(time.Now().UnixNano()))
+	return
+}
+
+func DecodeChunk(name string) (id ChunkID, err error) {
+	buf := []byte(name)
+	if len(buf) != ChunkIDEncodeLen {
+		return InvalidChunkID, errors.New("invalid chunk name")
+	}
+
+	if err = id.Unmarshal(buf); err != nil {
+		return InvalidChunkID, errors.New("chunk unmarshal failed")
+	}
+
+	return
+}
+
+func (s *ChunkStatus) String() string {
+	switch *s {
+	case ChunkStatusDefault:
+		return "default"
+	case ChunkStatusNormal:
+		return "normal"
+	case ChunkStatusReadOnly:
+		return "readOnly"
+	case ChunkStatusRelease:
+		return "release"
+	default:
+		return "unkown"
+	}
+}

blobstore/api/clustermgr/client.go

@@ -22,6 +22,7 @@ import (
 	"github.com/cubefs/cubefs/blobstore/common/errors"
 	"github.com/cubefs/cubefs/blobstore/common/proto"
 	"github.com/cubefs/cubefs/blobstore/common/rpc"
+	"github.com/cubefs/cubefs/blobstore/util"
 )
 
 const (
@@ -134,3 +135,12 @@ func (c *Client) Stat(ctx context.Context) (ret *StatInfo, err error) {
 func (c *Client) Snapshot(ctx context.Context) (*http.Response, error) {
 	return c.Get(ctx, "/snapshot/dump")
 }
+
+type SetClusterReadonlyArgs struct {
+	Readonly bool `json:"readonly"`
+}
+
+func (c *Client) SetClusterReadonly(ctx context.Context, args *SetClusterReadonlyArgs) (err error) {
+	err = c.PostWith(ctx, "/cluster/set?readonly="+util.Any2String(args.Readonly), nil, nil)
+	return
+}

blobstore/api/clustermgr/config.go

@@ -16,7 +16,10 @@ package clustermgr
 
 import (
 	"context"
+	"encoding/json"
 
+	"github.com/cubefs/cubefs/blobstore/common/codemode"
+	"github.com/cubefs/cubefs/blobstore/common/proto"
 	"github.com/cubefs/cubefs/blobstore/common/rpc"
 )
 
@@ -38,8 +41,8 @@ func (c *Client) GetConfig(ctx context.Context, key string) (ret string, err err
 	return
 }
 
-func (c *Client) SetConfig(ctx context.Context, args *ConfigSetArgs) (err error) {
-	err = c.PostWith(ctx, "/config/set", nil, args)
+func (c *Client) SetConfig(ctx context.Context, key, value string) (err error) {
+	err = c.PostWith(ctx, "/config/set", nil, &ConfigSetArgs{Key: key, Value: value})
 	return
 }
 
@@ -47,3 +50,19 @@ func (c *Client) DeleteConfig(ctx context.Context, key string) (err error) {
 	err = c.PostWith(ctx, "/config/delete?key="+key, nil, rpc.NoneBody)
 	return
 }
+
+func LoadExtendCodemode(ctx context.Context, configer interface {
+	GetConfig(context.Context, string) (string, error)
+},
+) error {
+	extend, err := configer.GetConfig(ctx, proto.CodeModeExtendKey)
+	if err != nil {
+		return err
+	}
+	extends := make([]codemode.ExtendCodeMode, 0)
+	if err = json.Unmarshal([]byte(extend), &extends); err != nil {
+		return err
+	}
+	codemode.Extend(extends...)
+	return nil
+}

blobstore/api/clustermgr/disk.go

@@ -16,14 +16,69 @@ package clustermgr
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"time"
 
-	"github.com/cubefs/cubefs/blobstore/api/blobnode"
 	"github.com/cubefs/cubefs/blobstore/common/proto"
 	"github.com/cubefs/cubefs/blobstore/common/rpc"
 )
 
+type ShardNodeDiskInfo struct {
+	DiskInfo
+	ShardNodeDiskHeartbeatInfo
+}
+
+func (s *ShardNodeDiskInfo) Marshal() ([]byte, error) {
+	return json.Marshal(s)
+}
+
+func (s *ShardNodeDiskInfo) Unmarshal(raw []byte) error {
+	return json.Unmarshal(raw, s)
+}
+
+type ShardNodeDiskHeartbeatInfo struct {
+	DiskID       proto.DiskID `json:"disk_id"`
+	Used         int64        `json:"used"`           // disk used space
+	Free         int64        `json:"free"`           // remaining free space on the disk
+	Size         int64        `json:"size"`           // total physical disk space
+	MaxShardCnt  int32        `json:"max_shard_cnt"`  // note: maintained by clustermgr
+	FreeShardCnt int32        `json:"free_shard_cnt"` // note: maintained by clustermgr
+	UsedShardCnt int32        `json:"used_shard_cnt"` // current number of shards on the disk
+}
+
+type BlobNodeDiskInfo struct {
+	DiskInfo
+	DiskHeartBeatInfo
+}
+
+type DiskHeartBeatInfo struct {
+	DiskID       proto.DiskID `json:"disk_id"`
+	Used         int64        `json:"used"`           // disk used space
+	Free         int64        `json:"free"`           // remaining free space on the disk
+	Size         int64        `json:"size"`           // total physical disk space
+	MaxChunkCnt  int64        `json:"max_chunk_cnt"`  // note: maintained by clustermgr
+	FreeChunkCnt int64        `json:"free_chunk_cnt"` // note: maintained by clustermgr
+	UsedChunkCnt int64        `json:"used_chunk_cnt"` // current number of chunks on the disk
+
+	OversoldFreeChunkCnt int64 `json:"oversold_free_chunk_cnt"` // note: maintained by clustermgr
+}
+
+type DiskInfo struct {
+	ClusterID    proto.ClusterID  `json:"cluster_id"`
+	Idc          string           `json:"idc"`
+	Rack         string           `json:"rack"`
+	Host         string           `json:"host"`
+	Path         string           `json:"path"`
+	Status       proto.DiskStatus `json:"status"` // normal、broken、repairing、repaired、dropped
+	Readonly     bool             `json:"readonly"`
+	CreateAt     time.Time        `json:"create_time"`
+	LastUpdateAt time.Time        `json:"last_update_time"`
+	DiskSetID    proto.DiskSetID  `json:"disk_set_id"`
+	NodeID       proto.NodeID     `json:"node_id"`
+}
+
 type DiskInfoArgs struct {
 	DiskID proto.DiskID `json:"disk_id"`
 }
@@ -49,12 +104,21 @@ type ListOptionArgs struct {
 }
 
 type ListDiskRet struct {
-	Disks  []*blobnode.DiskInfo `json:"disks"`
+	Disks  []*BlobNodeDiskInfo `json:"disks"`
+	Marker proto.DiskID        `json:"marker"`
+}
+
+type ListShardNodeDiskRet struct {
+	Disks  []*ShardNodeDiskInfo `json:"disks"`
 	Marker proto.DiskID         `json:"marker"`
 }
 
 type DisksHeartbeatArgs struct {
-	Disks []*blobnode.DiskHeartBeatInfo `json:"disks"`
+	Disks []*DiskHeartBeatInfo `json:"disks"`
+}
+
+type ShardNodeDisksHeartbeatArgs struct {
+	Disks []ShardNodeDiskHeartbeatInfo `json:"disks"`
 }
 
 type DisksHeartbeatRet struct {
@@ -68,26 +132,32 @@ type DiskHeartbeatRet struct {
 }
 
 type DiskStatInfo struct {
-	IDC            string `json:"idc"`
-	Total          int    `json:"total"`
-	TotalChunk     int64  `json:"total_chunk"`
-	TotalFreeChunk int64  `json:"total_free_chunk"`
-	Available      int    `json:"available"`
-	Readonly       int    `json:"readonly"`
-	Expired        int    `json:"expired"`
-	Broken         int    `json:"broken"`
-	Repairing      int    `json:"repairing"`
-	Repaired       int    `json:"repaired"`
-	Dropping       int    `json:"dropping"`
-	Dropped        int    `json:"dropped"`
+	IDC                    string `json:"idc"`
+	Total                  int    `json:"total"`
+	TotalChunk             int64  `json:"total_chunk,omitempty"`
+	TotalFreeChunk         int64  `json:"total_free_chunk,omitempty"`
+	TotalOversoldFreeChunk int64  `json:"total_oversold_free_chunk,omitempty"`
+	TotalShard             int64  `json:"total_shard,omitempty"`
+	TotalFreeShard         int64  `json:"total_free_shard,omitempty"`
+	Available              int    `json:"available"`
+	Readonly               int    `json:"readonly"`
+	Expired                int    `json:"expired"`
+	Broken                 int    `json:"broken"`
+	Repairing              int    `json:"repairing"`
+	Repaired               int    `json:"repaired"`
+	Dropping               int    `json:"dropping"`
+	Dropped                int    `json:"dropped"`
 }
 
 type SpaceStatInfo struct {
-	TotalSpace     int64          `json:"total_space"`
-	FreeSpace      int64          `json:"free_space"`
-	UsedSpace      int64          `json:"used_space"`
-	WritableSpace  int64          `json:"writable_space"`
-	TotalBlobNode  int64          `json:"total_blob_node"`
+	TotalSpace     int64          `json:"total_space"`    // total physical space
+	FreeSpace      int64          `json:"free_space"`     // free physical space which is writable
+	ReadOnlySpace  int64          `json:"readonly_space"` // free physical space which is readonly
+	UsedSpace      int64          `json:"used_space"`     // used physical space
+	ReservedSpace  int64          `json:"reserved_space"` // reserved logical space
+	WritableSpace  int64          `json:"writable_space"` // writable logical space
+	TotalBlobNode  int64          `json:"total_blob_node,omitempty"`
+	TotalShardNode int64          `json:"total_shard_node,omitempty"`
 	TotalDisk      int64          `json:"total_disk"`
 	DisksStatInfos []DiskStatInfo `json:"disk_stat_infos"`
 }
@@ -108,14 +178,14 @@ func (c *Client) AllocDiskID(ctx context.Context) (proto.DiskID, error) {
 }
 
 // DiskInfo get disk info from cluster manager
-func (c *Client) DiskInfo(ctx context.Context, id proto.DiskID) (ret *blobnode.DiskInfo, err error) {
-	ret = &blobnode.DiskInfo{}
+func (c *Client) DiskInfo(ctx context.Context, id proto.DiskID) (ret *BlobNodeDiskInfo, err error) {
+	ret = &BlobNodeDiskInfo{}
 	err = c.GetWith(ctx, "/disk/info?disk_id="+id.ToString(), ret)
 	return
 }
 
 // AddDisk add/register a new disk into cluster manager
-func (c *Client) AddDisk(ctx context.Context, info *blobnode.DiskInfo) (err error) {
+func (c *Client) AddDisk(ctx context.Context, info *BlobNodeDiskInfo) (err error) {
 	err = c.PostWith(ctx, "/disk/add", nil, info)
 	return
 }
@@ -129,7 +199,7 @@ func (c *Client) SetDisk(ctx context.Context, id proto.DiskID, status proto.Disk
 }
 
 // ListHostDisk list specified host disk info from cluster manager
-func (c *Client) ListHostDisk(ctx context.Context, host string) (ret []*blobnode.DiskInfo, err error) {
+func (c *Client) ListHostDisk(ctx context.Context, host string) (ret []*BlobNodeDiskInfo, err error) {
 	listRet := ListDiskRet{}
 	opt := &ListOptionArgs{Host: host, Count: 200}
 	for {
@@ -158,7 +228,7 @@ func (c *Client) ListDisk(ctx context.Context, options *ListOptionArgs) (ret Lis
 }
 
 // HeartbeatDisk report blobnode disk latest capacity info to cluster manager
-func (c *Client) HeartbeatDisk(ctx context.Context, infos []*blobnode.DiskHeartBeatInfo) (ret []*DiskHeartbeatRet, err error) {
+func (c *Client) HeartbeatDisk(ctx context.Context, infos []*DiskHeartBeatInfo) (ret []*DiskHeartbeatRet, err error) {
 	result := &DisksHeartbeatRet{}
 	args := &DisksHeartbeatArgs{Disks: infos}
 	err = c.PostWith(ctx, "/disk/heartbeat", result, args)
@@ -176,7 +246,7 @@ func (c *Client) DroppedDisk(ctx context.Context, id proto.DiskID) (err error) {
 	return
 }
 
-func (c *Client) ListDroppingDisk(ctx context.Context) (ret []*blobnode.DiskInfo, err error) {
+func (c *Client) ListDroppingDisk(ctx context.Context) (ret []*BlobNodeDiskInfo, err error) {
 	result := &ListDiskRet{}
 	err = c.GetWith(ctx, "/disk/droppinglist", result)
 	ret = result.Disks
@@ -187,3 +257,56 @@ func (c *Client) SetReadonlyDisk(ctx context.Context, id proto.DiskID, readonly
 	err = c.PostWith(ctx, "/disk/access", nil, &DiskAccessArgs{DiskID: id, Readonly: readonly})
 	return
 }
+
+// AddShardNodeDisk add/register a new disk of shardnode into cluster manager
+func (c *Client) AddShardNodeDisk(ctx context.Context, info *ShardNodeDiskInfo) (err error) {
+	err = c.PostWith(ctx, "/shardnode/disk/add", nil, info)
+	return
+}
+
+// HeartbeatShardNodeDisk report shardnode disk latest capacity info to cluster manager
+func (c *Client) HeartbeatShardNodeDisk(ctx context.Context, infos []ShardNodeDiskHeartbeatInfo) (err error) {
+	args := &ShardNodeDisksHeartbeatArgs{Disks: infos}
+	err = c.PostWith(ctx, "/shardnode/disk/heartbeat", nil, args)
+	return
+}
+
+// AllocShardNodeDiskID alloc shardnode diskID from cluster manager
+func (c *Client) AllocShardNodeDiskID(ctx context.Context) (proto.DiskID, error) {
+	ret := &DiskIDAllocRet{}
+	err := c.PostWith(ctx, "/shardnode/diskid/alloc", ret, rpc.NoneBody)
+	if err != nil {
+		return 0, err
+	}
+	return ret.DiskID, nil
+}
+
+// ListShardNodeDisk list disk info from cluster manager
+// when ListOptionArgs is default value, defalut return 10 diskInfos
+func (c *Client) ListShardNodeDisk(ctx context.Context, options *ListOptionArgs) (ret ListShardNodeDiskRet, err error) {
+	err = c.GetWith(ctx, fmt.Sprintf(
+		"/shardnode/disk/list?idc=%s&rack=%s&host=%s&status=%d&marker=%d&count=%d",
+		options.Idc,
+		options.Rack,
+		options.Host,
+		options.Status,
+		options.Marker,
+		options.Count,
+	), &ret)
+	return
+}
+
+// ShardNodeDiskInfo get shardnode disk info from cluster manager
+func (c *Client) ShardNodeDiskInfo(ctx context.Context, id proto.DiskID) (ret *ShardNodeDiskInfo, err error) {
+	ret = &ShardNodeDiskInfo{}
+	err = c.GetWith(ctx, "/shardnode/disk/info?disk_id="+id.ToString(), ret)
+	return
+}
+
+// SetShardNodeDisk set shardnode disk status
+func (c *Client) SetShardNodeDisk(ctx context.Context, id proto.DiskID, status proto.DiskStatus) (err error) {
+	if !status.IsValid() {
+		return errors.New("invalid status")
+	}
+	return c.PostWith(ctx, "/shardnode/disk/set", nil, &DiskSetArgs{DiskID: id, Status: status})
+}

blobstore/api/clustermgr/node.go

@@ -0,0 +1,121 @@
+// Copyright 2024 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package clustermgr
+
+import (
+	"context"
+
+	"github.com/cubefs/cubefs/blobstore/common/proto"
+)
+
+type BlobNodeInfo struct {
+	NodeInfo
+}
+
+type ShardNodeInfo struct {
+	NodeInfo
+	ShardNodeExtraInfo
+}
+
+type ShardNodeExtraInfo struct {
+	RaftHost string `json:"raft_host"`
+}
+
+type NodeInfo struct {
+	NodeID    proto.NodeID     `json:"node_id"`
+	NodeSetID proto.NodeSetID  `json:"node_set_id"`
+	ClusterID proto.ClusterID  `json:"cluster_id"`
+	DiskType  proto.DiskType   `json:"disk_type"` // one node only manages one diskType disk
+	Idc       string           `json:"idc"`
+	Rack      string           `json:"rack"`
+	Host      string           `json:"host"`
+	Role      proto.NodeRole   `json:"role"`
+	Status    proto.NodeStatus `json:"status"`
+}
+
+type NodeInfoArgs struct {
+	NodeID proto.NodeID `json:"node_id"`
+}
+
+type NodeIDAllocRet struct {
+	NodeID proto.NodeID `json:"node_id"`
+}
+
+type NodeSetInfo struct {
+	ID       proto.NodeSetID                    `json:"id"`
+	Number   int                                `json:"number"`
+	Nodes    []proto.NodeID                     `json:"nodes"`
+	DiskSets map[proto.DiskSetID][]proto.DiskID `json:"disk_sets"`
+}
+
+type TopoInfo struct {
+	CurNodeSetID proto.NodeSetID                             `json:"cur_node_set_id"`
+	CurDiskSetID proto.DiskSetID                             `json:"cur_disk_set_id"`
+	AllNodeSets  map[string]map[proto.NodeSetID]*NodeSetInfo `json:"all_node_sets"`
+}
+
+// AddNode add a new node into cluster manager and return allocated nodeID
+func (c *Client) AddNode(ctx context.Context, info *BlobNodeInfo) (proto.NodeID, error) {
+	ret := &NodeIDAllocRet{}
+	err := c.PostWith(ctx, "/node/add", ret, info)
+	if err != nil {
+		return 0, err
+	}
+	return ret.NodeID, nil
+}
+
+// DropNode drop a node from cluster manager
+func (c *Client) DropNode(ctx context.Context, id proto.NodeID) (err error) {
+	err = c.PostWith(ctx, "/node/drop", nil, &NodeInfoArgs{NodeID: id})
+	return
+}
+
+// NodeInfo get node info from cluster manager
+func (c *Client) NodeInfo(ctx context.Context, id proto.NodeID) (ret *BlobNodeInfo, err error) {
+	ret = &BlobNodeInfo{}
+	err = c.GetWith(ctx, "/node/info?node_id="+id.ToString(), ret)
+	return
+}
+
+// TopoInfo get nodeset and diskset topo info from cluster manager
+func (c *Client) TopoInfo(ctx context.Context) (ret *TopoInfo, err error) {
+	ret = &TopoInfo{}
+	err = c.GetWith(ctx, "/topo/info", ret)
+	return
+}
+
+// AddShardNode add a new shardnode into cluster manager and return allocated nodeID
+func (c *Client) AddShardNode(ctx context.Context, info *ShardNodeInfo) (proto.NodeID, error) {
+	ret := &NodeIDAllocRet{}
+	err := c.PostWith(ctx, "/shardnode/add", ret, info)
+	if err != nil {
+		return 0, err
+	}
+	return ret.NodeID, nil
+}
+
+// ShardNodeInfo get shardnode info from cluster manager
+func (c *Client) ShardNodeInfo(ctx context.Context, id proto.NodeID) (ret *ShardNodeInfo, err error) {
+	ret = &ShardNodeInfo{}
+	err = c.GetWith(ctx, "/shardnode/info?node_id="+id.ToString(), ret)
+	return
+}
+
+// ShardNodeTopoInfo get shardnode nodeset and diskset topo info from cluster manager
+func (c *Client) ShardNodeTopoInfo(ctx context.Context) (ret *TopoInfo, err error) {
+	ret = &TopoInfo{}
+	err = c.GetWith(ctx, "/shardnode/topo/info", ret)
+	return
+}

blobstore/api/clustermgr/proto.go

@@ -18,9 +18,7 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/cubefs/cubefs/blobstore/api/blobnode"
 	"github.com/cubefs/cubefs/blobstore/common/proto"
-	"github.com/cubefs/cubefs/blobstore/common/raftserver"
 )
 
 const (
@@ -39,11 +37,12 @@ type ClusterInfo struct {
 }
 
 type StatInfo struct {
-	LeaderHost string            `json:"leader_host"`
-	ReadOnly   bool              `json:"read_only"`
-	RaftStatus raftserver.Status `json:"raft_status"`
-	SpaceStat  SpaceStatInfo     `json:"space_stat"`
-	VolumeStat VolumeStatInfo    `json:"volume_stat"`
+	LeaderHost         string         `json:"leader_host"`
+	ReadOnly           bool           `json:"read_only"`
+	RaftStatus         interface{}    `json:"raft_status"`
+	BlobNodeSpaceStat  SpaceStatInfo  `json:"space_stat"`
+	ShardNodeSpaceStat SpaceStatInfo  `json:"shard_node_space_stat"`
+	VolumeStat         VolumeStatInfo `json:"volume_stat"`
 }
 
 func GetConsulClusterPath(region string) string {
@@ -54,6 +53,7 @@ func GetConsulClusterPath(region string) string {
 type ClientAPI interface {
 	APIAccess
 	APIProxy
+	APIBlobnode
 }
 
 // APIAccess sub of cluster manager api for access
@@ -61,13 +61,19 @@ type APIAccess interface {
 	GetConfig(ctx context.Context, key string) (string, error)
 	GetService(ctx context.Context, args GetServiceArgs) (ServiceInfo, error)
 	ListDisk(ctx context.Context, options *ListOptionArgs) (ListDiskRet, error)
+	AuthSpace(ctx context.Context, args *AuthSpaceArgs) (err error)
+	GetSpaceByName(ctx context.Context, args *GetSpaceByNameArgs) (ret *Space, err error)
+	GetCatalogChanges(ctx context.Context, args *GetCatalogChangesArgs) (ret *GetCatalogChangesRet, err error)
+	ShardNodeDiskInfo(ctx context.Context, id proto.DiskID) (ret *ShardNodeDiskInfo, err error)
+	ListShardNodeDisk(ctx context.Context, options *ListOptionArgs) (ret ListShardNodeDiskRet, err error)
 }
 
 // APIProxy sub of cluster manager api for allocator
 type APIProxy interface {
 	GetConfig(ctx context.Context, key string) (string, error)
 	GetVolumeInfo(ctx context.Context, args *GetVolumeArgs) (*VolumeInfo, error)
-	DiskInfo(ctx context.Context, id proto.DiskID) (*blobnode.DiskInfo, error)
+	GetVolumeRoutes(ctx context.Context, args *GetVolumeRoutesArgs) (*GetVolumeRoutesRet, error)
+	DiskInfo(ctx context.Context, id proto.DiskID) (*BlobNodeDiskInfo, error)
 	AllocVolume(ctx context.Context, args *AllocVolumeArgs) (AllocatedVolumeInfos, error)
 	AllocBid(ctx context.Context, args *BidScopeArgs) (*BidScopeRet, error)
 	RetainVolume(ctx context.Context, args *RetainVolumeArgs) (RetainVolumes, error)
@@ -77,4 +83,23 @@ type APIProxy interface {
 // APIService sub of cluster manager api for service
 type APIService interface {
 	GetService(ctx context.Context, args GetServiceArgs) (ServiceInfo, error)
+	RegisterService(ctx context.Context, node ServiceNode, tickInterval, heartbeatTicks, expiresTicks uint32) (err error)
+}
+
+type APIBlobnode interface {
+	APIService
+	GetConfig(ctx context.Context, key string) (value string, err error)
+	SetConfig(ctx context.Context, key, value string) error
+	AddNode(ctx context.Context, info *BlobNodeInfo) (proto.NodeID, error)
+	ListHostDisk(ctx context.Context, host string) (ret []*BlobNodeDiskInfo, err error)
+	ListDisk(ctx context.Context, options *ListOptionArgs) (ret ListDiskRet, err error)
+	AddDisk(ctx context.Context, info *BlobNodeDiskInfo) (err error)
+	DiskInfo(ctx context.Context, id proto.DiskID) (ret *BlobNodeDiskInfo, err error)
+	SetDisk(ctx context.Context, id proto.DiskID, status proto.DiskStatus) (err error)
+	AllocDiskID(ctx context.Context) (proto.DiskID, error)
+	SetCompactChunk(ctx context.Context, args *SetCompactChunkArgs) (err error)
+	ListVolumeUnit(ctx context.Context, args *ListVolumeUnitArgs) ([]*VolumeUnitInfo, error)
+	GetVolumeInfo(ctx context.Context, args *GetVolumeArgs) (ret *VolumeInfo, err error)
+	ReportChunk(ctx context.Context, args *ReportChunkArgs) (err error)
+	HeartbeatDisk(ctx context.Context, infos []*DiskHeartBeatInfo) (ret []*DiskHeartbeatRet, err error)
 }

blobstore/api/clustermgr/shard.go

@@ -0,0 +1,64 @@
+// Copyright 2024 The CubeFS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package clustermgr
+
+import (
+	"context"
+	"fmt"
+)
+
+func (c *Client) AllocShardUnit(ctx context.Context, args *AllocShardUnitArgs) (ret *AllocShardUnitRet, err error) {
+	ret = &AllocShardUnitRet{}
+	err = c.PostWith(ctx, "/shard/unit/alloc", ret, args)
+	return
+}
+
+func (c *Client) UpdateShard(ctx context.Context, args *UpdateShardArgs) (err error) {
+	err = c.PostWith(ctx, "/shard/update", nil, args)
+	return
+}
+
+func (c *Client) ReportShard(ctx context.Context, args *ShardReportArgs) (ret []ShardTask, err error) {
+	result := &ShardReportRet{}
+	err = c.PostWith(ctx, "/shard/report", result, args)
+	return result.ShardTasks, err
+}
+
+func (c *Client) GetShardInfo(ctx context.Context, args *GetShardArgs) (ret *Shard, err error) {
+	ret = &Shard{}
+	err = c.GetWith(ctx, "/shard/get?shard_id="+args.ShardID.ToString(), ret)
+	return
+}
+
+func (c *Client) ListShardUnit(ctx context.Context, args *ListShardUnitArgs) ([]ShardUnitInfo, error) {
+	ret := &ListShardUnitRet{}
+	err := c.GetWith(ctx, "/shard/unit/list?disk_id="+args.DiskID.ToString(), ret)
+	return ret.ShardUnitInfos, err
+}
+
+func (c *Client) ListShard(ctx context.Context, args *ListShardArgs) (ret ListShardRet, err error) {
+	err = c.GetWith(ctx, fmt.Sprintf("/shard/list?marker=%d&count=%d", args.Marker, args.Count), &ret)
+	return
+}
+
+func (c *Client) AdminUpdateShard(ctx context.Context, args *Shard) (err error) {
+	err = c.PostWith(ctx, "/admin/update/shard", nil, args)
+	return
+}
+
+func (c *Client) AdminUpdateShardUnit(ctx context.Context, args *AdminUpdateShardUnitArgs) (err error) {
+	err = c.PostWith(ctx, "/admin/update/shard/unit", nil, args)
+	return
+}

blobstore/api/clustermgr/shard.proto

@@ -0,0 +1,108 @@
+syntax = "proto3";
+
+package cubefs.blobstore.api.clustermgr;
+option go_package = "./;clustermgr";
+
+option (gogoproto.sizer_all) = true;
+option (gogoproto.marshaler_all) = true;
+option (gogoproto.unmarshaler_all) = true;
+option (gogoproto.goproto_unkeyed_all) = true;
+option (gogoproto.goproto_unrecognized_all) = true;
+option (gogoproto.goproto_sizecache_all) = true;
+option (gogoproto.goproto_stringer_all) = false;
+option (gogoproto.stringer_all) = true;
+option (gogoproto.gostring_all) = true;
+
+import "gogoproto/gogo.proto";
+import "cubefs/blobstore/common/sharding/range.proto";
+
+message Shard {
+  uint32 shard_id = 1 [(gogoproto.customname) = "ShardID", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.ShardID"];
+  uint64 applied_index = 2;
+  uint32 leader_disk_id = 3 [(gogoproto.customname) = "LeaderDiskID", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.DiskID"];
+  cubefs.blobstore.common.sharding.Range range = 4 [(gogoproto.nullable) = false];
+  repeated ShardUnit units = 5 [(gogoproto.nullable) = false];
+  uint64 route_version = 6 [(gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.RouteVersion"];
+}
+
+message ShardUnit {
+  uint64 suid = 1 [(gogoproto.customname) = "Suid", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.Suid"];
+  uint32 disk_id = 2 [(gogoproto.customname) = "DiskID", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.DiskID"];
+  bool learner = 3;
+  string host = 4;
+  uint32 status = 5 [(gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.ShardUnitStatus"];
+}
+
+message ShardUnitInfo {
+  uint64 suid = 1 [(gogoproto.customname) = "Suid", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.Suid"];
+  uint32 disk_id = 2 [(gogoproto.customname) = "DiskID", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.DiskID"];
+  uint64 applied_index = 3;
+  uint32 leader_disk_id = 4 [(gogoproto.customname) = "LeaderDiskID", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.DiskID"];
+  cubefs.blobstore.common.sharding.Range range = 5 [(gogoproto.nullable) = false];
+  uint64 route_version = 6 [(gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.RouteVersion"];
+  string host = 7;
+  bool learner = 8;
+}
+
+message ShardTask {
+  uint32 task_type = 1 [(gogoproto.customname) = "TaskType", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.ShardTaskType"];
+  uint32 disk_id = 2 [(gogoproto.customname) = "DiskID", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.DiskID"];
+  uint32 suid = 3 [(gogoproto.customname) = "Suid", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.Suid"];
+  uint64 old_route_version = 4 [(gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.RouteVersion"];
+  uint64 route_version = 5 [(gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.RouteVersion"];
+}
+
+message ShardReportArgs {
+  repeated ShardUnitInfo shards = 1 [(gogoproto.nullable) = false];
+}
+
+message ShardReportRet {
+  repeated ShardTask shard_tasks = 1 [(gogoproto.nullable) = false];
+}
+
+message AllocShardUnitArgs{
+  uint64 suid = 1 [(gogoproto.customname) = "Suid", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.Suid"];
+  repeated uint32 exclude_disk_ids = 2 [(gogoproto.customname) = "ExcludeDiskIDs", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.DiskID"];
+}
+
+message AllocShardUnitRet {
+  uint64 suid = 1 [(gogoproto.customname) = "Suid", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.Suid"];
+  uint32 disk_id = 2 [(gogoproto.customname) = "DiskID", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.DiskID"];
+  string host = 3;
+}
+
+message UpdateShardArgs {
+  uint64 new_suid = 1 [(gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.Suid"];
+  uint32 new_disk_id = 2 [(gogoproto.customname) = "NewDiskID", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.DiskID"];
+  bool new_is_leaner = 3;
+  uint64 old_suid = 4 [(gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.Suid"];
+  bool old_is_leaner = 5;
+}
+
+message GetShardArgs {
+  uint32 shard_id = 1 [(gogoproto.customname) = "ShardID", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.ShardID"];
+}
+
+message ListShardUnitArgs {
+  uint32 disk_id = 1 [(gogoproto.customname) = "DiskID", (gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.DiskID"];
+}
+
+message ListShardUnitRet {
+  repeated ShardUnitInfo shard_unit_infos = 1 [(gogoproto.nullable) = false];
+}
+
+message ListShardArgs {
+  uint32 marker = 1 [(gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.ShardID"];
+  uint32 count = 2;
+}
+
+message ListShardRet {
+  repeated Shard shards = 1 [(gogoproto.nullable) = false];
+  uint32 marker = 2 [(gogoproto.casttype) = "github.com/cubefs/cubefs/blobstore/common/proto.ShardID"];
+}
+
+message AdminUpdateShardUnitArgs {
+  uint32 epoch = 1;
+  uint32  next_epoch = 2;
+  ShardUnit unit = 3 [(gogoproto.embed) = true, (gogoproto.nullable) = false];
+}

blobstore/api/clustermgr/volume.go

@@ -24,7 +24,6 @@ import (
 	"net/http"
 	"unsafe"
 
-	"github.com/cubefs/cubefs/blobstore/api/blobnode"
 	"github.com/cubefs/cubefs/blobstore/common/codemode"
 	"github.com/cubefs/cubefs/blobstore/common/proto"
 	"github.com/cubefs/cubefs/blobstore/common/rpc"
@@ -62,16 +61,7 @@ func (v *VolumeInfo) Equal(expected *VolumeInfo) bool {
 	return true
 }
 
-type VolumeInfoBase struct {
-	Vid            proto.Vid          `json:"vid"`
-	CodeMode       codemode.CodeMode  `json:"code_mode"`
-	Status         proto.VolumeStatus `json:"status"`
-	HealthScore    int                `json:"health_score"`
-	Total          uint64             `json:"total"`
-	Free           uint64             `json:"free"`
-	Used           uint64             `json:"used"`
-	CreateByNodeID uint64             `json:"create_by_node_id"`
-}
+type VolumeInfoBase VolumeInfoBasePB
 
 type AllocVolumeInfo struct {
 	VolumeInfo
@@ -157,7 +147,8 @@ func (c *Client) RetainVolume(ctx context.Context, args *RetainVolumeArgs) (ret
 }
 
 type LockVolumeArgs struct {
-	Vid proto.Vid `json:"vid"`
+	Vid   proto.Vid `json:"vid"`
+	Epoch uint32    `json:"epoch"`
 }
 
 func (c *Client) LockVolume(ctx context.Context, args *LockVolumeArgs) (err error) {
@@ -166,7 +157,9 @@ func (c *Client) LockVolume(ctx context.Context, args *LockVolumeArgs) (err erro
 }
 
 type UnlockVolumeArgs struct {
-	Vid proto.Vid `json:"vid"`
+	Vid   proto.Vid `json:"vid"`
+	Epoch uint32    `json:"epoch"`
+	Force bool      `json:"force"`
 }
 
 func (c *Client) UnlockVolume(ctx context.Context, args *UnlockVolumeArgs) (err error) {
@@ -175,7 +168,8 @@ func (c *Client) UnlockVolume(ctx context.Context, args *UnlockVolumeArgs) (err
 }
 
 type AllocVolumeUnitArgs struct {
-	Vuid proto.Vuid `json:"vuid"`
+	Vuid      proto.Vuid `json:"vuid"`
+	IsBalance bool       `json:"is_balance"`
 }
 
 type AllocVolumeUnit struct {
@@ -202,15 +196,8 @@ type ListVolumeUnitArgs struct {
 	DiskID proto.DiskID `json:"disk_id"`
 }
 
-type VolumeUnitInfo struct {
-	Vuid       proto.Vuid   `json:"vuid"`
-	DiskID     proto.DiskID `json:"disk_id"`
-	Total      uint64       `json:"total"`
-	Free       uint64       `json:"free"`
-	Used       uint64       `json:"used"`
-	Compacting bool         `json:"compact"`
-	Host       string       `json:"host"`
-}
+// VolumeUnitInfoBase is used to maintain the VolumeUnitInfo marshal method after existing structs are defined in pb.
+type VolumeUnitInfo VolumeUnitInfoBase
 
 type ListVolumeUnitInfos struct {
 	VolumeUnitInfos []*VolumeUnitInfo `json:"volume_unit_infos"`
@@ -223,7 +210,7 @@ func (c *Client) ListVolumeUnit(ctx context.Context, args *ListVolumeUnitArgs) (
 }
 
 type ReportChunkArgs struct {
-	ChunkInfos []blobnode.ChunkInfo `json:"chunk_infos"`
+	ChunkInfos []ChunkInfo `json:"chunk_infos"`
 }
 
 func (r *ReportChunkArgs) Encode() ([]byte, error) {
@@ -274,16 +261,16 @@ func (r *ReportChunkArgs) Decode(reader io.Reader) error {
 	}
 
 	read(&count)
-	r.ChunkInfos = make([]blobnode.ChunkInfo, count)
+	if count > 1<<20 {
+		return fmt.Errorf("chunks is too much %d, limit: %d", count, 1<<20)
+	}
+	r.ChunkInfos = make([]ChunkInfo, count)
 	for i := range r.ChunkInfos {
-		raw := make([]byte, blobnode.ChunkIdEncodeLen)
+		raw := make([]byte, ChunkIDEncodeLen)
 		n, err := io.ReadFull(reader, raw)
-		if n != blobnode.ChunkIdEncodeLen || err != nil {
+		if n != ChunkIDEncodeLen || err != nil {
 			return fmt.Errorf("invalid source reader, err: %v", err)
 		}
-		if err != nil {
-			return err
-		}
 		r.ChunkInfos[i].Id.Unmarshal(raw)
 		read(&r.ChunkInfos[i].Vuid)
 		read(&r.ChunkInfos[i].DiskID)
@@ -344,12 +331,13 @@ func (c *Client) ListAllocatedVolumes(ctx context.Context, args *ListAllocatedVo
 }
 
 type VolumeStatInfo struct {
-	TotalVolume       int `json:"total_volume"`
-	IdleVolume        int `json:"idle_volume"`
-	AllocatableVolume int `json:"can_alloc_volume"`
-	ActiveVolume      int `json:"active_volume"`
-	LockVolume        int `json:"lock_volume"`
-	UnlockingVolume   int `json:"unlocking_volume"`
+	TotalVolume       int    `json:"total_volume"`
+	IdleVolume        int    `json:"idle_volume"`
+	AllocatableVolume int    `json:"can_alloc_volume"`
+	ActiveVolume      int    `json:"active_volume"`
+	LockVolume        int    `json:"lock_volume"`
+	UnlockingVolume   int    `json:"unlocking_volume"`
+	WritableSpace     uint64 `json:"writable_space"`
 }
 
 type AdminUpdateUnitArgs struct {
@@ -357,3 +345,17 @@ type AdminUpdateUnitArgs struct {
 	NextEpoch uint32 `json:"next_epoch"`
 	VolumeUnitInfo
 }
+
+func (c *Client) GetVolumeRoutes(ctx context.Context, args *GetVolumeRoutesArgs) (ret *GetVolumeRoutesRet, err error) {
+	ret = &GetVolumeRoutesRet{}
+	err = c.GetWith(ctx, fmt.Sprintf("/volumeroutes/get?route_version=%d", args.RouteVersion), ret)
+	return
+}
+
+func (m *VolumeUnitInfoBase) Convert2Unit() Unit {
+	return Unit{
+		Vuid:   m.GetVuid(),
+		DiskID: m.GetDiskID(),
+		Host:   m.GetHost(),
+	}
+}