diff --git a/blobstore/.github/workflows/codeql.yml b/blobstore/.github/workflows/codeql.yml new file mode 100644 index 000000000..d0665ab92 --- /dev/null +++ b/blobstore/.github/workflows/codeql.yml @@ -0,0 +1,52 @@ +name: CodeQL + +on: + push: + branches: + - master + - main + - release-* + tags: + - v* + pull_request: + types: [opened, synchronize, reopened] + branches: + - master + - main + - release-* + +jobs: + CodeQL-Build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + language: ["go"] + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/blobstore/.github/workflows/format.yml b/blobstore/.github/workflows/format.yml new file mode 100644 index 000000000..dd6f98229 --- /dev/null +++ b/blobstore/.github/workflows/format.yml @@ -0,0 +1,37 @@ +name: Format + +on: + push: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + GolangFormat: + runs-on: ubuntu-latest + steps: + - name: Set up Go Version 1.x + uses: actions/setup-go@v2 + with: + go-version: ^1.17 + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Make rocksdb includes + run: | + wget -O deps.tar.gz http://ocs-upload-cn.heytapcs.com/deps.tar.gz + tar -zxf deps.tar.gz + mv deps .deps + source env.sh + echo "CGO_CFLAGS=${CGO_CFLAGS}" >> $GITHUB_ENV + echo "CGO_LDFLAGS=${CGO_LDFLAGS}" >> $GITHUB_ENV + + - name: Check vet + run: | + go vet -trimpath ./... + + - name: Go code format with gofumpt + run: | + go install mvdan.cc/gofumpt@v0.2.1 + gofumpt -l -w . + git diff --exit-code diff --git a/blobstore/.github/workflows/lint.yml b/blobstore/.github/workflows/lint.yml new file mode 100644 index 000000000..1272a3195 --- /dev/null +++ b/blobstore/.github/workflows/lint.yml @@ -0,0 +1,56 @@ +name: Lint + +on: + push: + branches: + - master + - main + - release-* + tags: + - v* + pull_request: + types: [opened, synchronize, reopened] + +permissions: + contents: read + +jobs: + GolangCI-Lint: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Make rocksdb includes + run: | + wget -O deps.tar.gz http://ocs-upload-cn.heytapcs.com/deps.tar.gz + tar -zxf deps.tar.gz + mv deps .deps + source env.sh + echo "CGO_CFLAGS=${CGO_CFLAGS}" >> $GITHUB_ENV + echo "CGO_LDFLAGS=${CGO_LDFLAGS}" >> $GITHUB_ENV + + - name: golangci-lint + uses: golangci/golangci-lint-action@v2.5.2 + with: + # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version + version: v1.43.0 + + # Optional: working directory, useful for monorepos + # working-directory: somedir + + # Optional: golangci-lint command line arguments. + # args: --issues-exit-code=0 -D errcheck -D typecheck --exclude SA1019 + args: --issues-exit-code=1 -D errcheck -E bodyclose ./... + + # Optional: show only new issues if it's a pull request. The default value is `false`. + # only-new-issues: true + + # Optional: if set to true then the action will use pre-installed Go. + # skip-go-installation: true + + # Optional: if set to true then the action don't cache or restore ~/go/pkg. + # skip-pkg-cache: true + + # Optional: if set to true then the action don't cache or restore ~/.cache/go-build. + # skip-build-cache: true diff --git a/blobstore/.github/workflows/test.yml b/blobstore/.github/workflows/test.yml new file mode 100644 index 000000000..f2cc0887d --- /dev/null +++ b/blobstore/.github/workflows/test.yml @@ -0,0 +1,49 @@ +name: Test + +on: + push: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + GolangTest: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + # os: [windows-latest, ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Set up Go Version 1.x + uses: actions/setup-go@v2 + with: + go-version: ^1.17 + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Make rocksdb includes + run: | + wget -O deps.tar.gz http://ocs-upload-cn.heytapcs.com/deps.tar.gz + tar -zxf deps.tar.gz + mv deps .deps + source env.sh + echo "CGO_CFLAGS=${CGO_CFLAGS}" >> $GITHUB_ENV + echo "CGO_LDFLAGS=${CGO_LDFLAGS}" >> $GITHUB_ENV + + - name: Check and Verify Go modules + run: | + go mod tidy + git diff --exit-code go.mod go.sum + go mod verify + + - name: Go test with coverage + run: | + go test -trimpath -covermode=count --coverprofile coverage.txt ./... + env: + JENKINS_TEST: TRUE + + - name: Upload codecov + uses: codecov/codecov-action@v2 + with: + files: ./coverage.txt diff --git a/blobstore/.gitignore b/blobstore/.gitignore new file mode 100644 index 000000000..192c96b28 --- /dev/null +++ b/blobstore/.gitignore @@ -0,0 +1,12 @@ +.vscode +.idea/ +.DS_Store +.version +.cache +.deps/ +bin/ +*/*/*/run/ +node_modules/ +package.json +package-lock.json +!.gitkeep \ No newline at end of file diff --git a/blobstore/LICENSE b/blobstore/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/blobstore/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/blobstore/Makefile b/blobstore/Makefile new file mode 100644 index 000000000..6681ebd23 --- /dev/null +++ b/blobstore/Makefile @@ -0,0 +1,68 @@ +# Copyright 2022 The CubeFS Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + +PROJECTDIR=$(shell pwd) +OS=$(shell uname -s) +BINDIR=$(PROJECTDIR)/bin +GCFLAGS=all=-trimpath=$(PROJECTDIR) +ASMFLAGS=all=-trimpath=$(PROJECTDIR) +BRANCHNAME=${BUILD_BRANCH} +COMMITID=${GIT_COMMIT} + +ifeq ($(BRANCHNAME),) + BRANCHNAME=$(shell git symbolic-ref --short -q HEAD) + COMMITID=$(shell git rev-parse --short HEAD) +endif + +LDFLAGS=-w -s +ifneq ($(BRANCHNAME),) + LDFLAGS+= -X "github.com/cubefs/cubefs/blobstore/util/version.version=$(BRANCHNAME)/$(COMMITID)" +endif + +BUILD=go build -v -gcflags=$(GCFLAGS) -asmflags=$(ASMFLAGS) -ldflags='$(LDFLAGS)' -o $(BINDIR) +INSTALL=CGO_ENABLED=0 $(BUILD) +CGOINSTALL=CGO_ENABLED=1 $(BUILD) +CMDDIR=github.com/cubefs/cubefs/blobstore/cmd +TARGETS=blobnode cm access scheduler tinker proxy cli + +.PHONY: clean all $(TARGETS) +all:$(TARGETS) + +cm: + @echo "building clustermgr" + @$(CGOINSTALL) $(CMDDIR)/clustermgr + +blobnode: + @echo "building blobnode" + @$(CGOINSTALL) $(CMDDIR)/blobnode + +access: + @echo "building access" + @$(INSTALL) $(CMDDIR)/access + +scheduler: + @echo "building scheduler" + @$(INSTALL) $(CMDDIR)/scheduler + +proxy: + @echo "building proxy" + @$(INSTALL) $(CMDDIR)/proxy + +cli: + @echo "building cli" + @$(CGOINSTALL) $(PROJECTDIR)/cli/cli + +clean: + @go clean -i ./... + @rm -f $(BINDIR)/* diff --git a/blobstore/README.md b/blobstore/README.md new file mode 100644 index 000000000..8f60b11e3 --- /dev/null +++ b/blobstore/README.md @@ -0,0 +1,40 @@ +# BlobStore +- [Overview](#overview) +- [Documents](#documents) +- [Build BlobStore](#build-blobstore) +- [Deploy BlobStore](#deploy-blobstore) +- [Manage BlobStore](#manage-blobstore) +- [License](#license) + +## Overview +BlobStore is a highly reliable,highly available and ultra-large scale distributed storage system. The system adopts Reed-Solomon code, which provides higher data durability with less storage cost than use three copies backup technology, and supports multiple erasure code modes multiple availability zones,and optimizes for small file to meet the storage needs of different scenarios. +Some key features of BlobStore include: +- ultra-large scale +- high reliability +- flexible deployment +- low cost + + +## Documents + +English version: https://cubefs.readthedocs.io/en/latest/ + +Chinese version: https://cubefs.readthedocs.io/zh_CN/latest/ + +## Build BlobStore + +``` +$ git clone http://github.com/cubefs/cubefs.git +$ cd cubefs/blobstore +$ source env.sh +$ ./build.sh +``` + +## Deploy BlobStore +For more details please refer to [documentation](https://cubefs.readthedocs.io/en/latest/user-guide/blobstore.html). + +## Manage BlobStore +For more details please refer to [documentation](https://cubefs.readthedocs.io/en/latest/admin-api/blobstore/blobnode.html). + +## License +BlobStore is licensed under the Apache License, Version 2.0. For detail see LICENSE and NOTICE. diff --git a/blobstore/access/access_mock_test.go b/blobstore/access/access_mock_test.go new file mode 100644 index 000000000..134447e19 --- /dev/null +++ b/blobstore/access/access_mock_test.go @@ -0,0 +1,217 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cubefs/cubefs/blobstore/access (interfaces: StreamHandler,Limiter) + +// Package access is a generated GoMock package. +package access + +import ( + context "context" + io "io" + reflect "reflect" + + access0 "github.com/cubefs/cubefs/blobstore/api/access" + codemode "github.com/cubefs/cubefs/blobstore/common/codemode" + proto "github.com/cubefs/cubefs/blobstore/common/proto" + gomock "github.com/golang/mock/gomock" +) + +// MockStreamHandler is a mock of StreamHandler interface. +type MockStreamHandler struct { + ctrl *gomock.Controller + recorder *MockStreamHandlerMockRecorder +} + +// MockStreamHandlerMockRecorder is the mock recorder for MockStreamHandler. +type MockStreamHandlerMockRecorder struct { + mock *MockStreamHandler +} + +// NewMockStreamHandler creates a new mock instance. +func NewMockStreamHandler(ctrl *gomock.Controller) *MockStreamHandler { + mock := &MockStreamHandler{ctrl: ctrl} + mock.recorder = &MockStreamHandlerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockStreamHandler) EXPECT() *MockStreamHandlerMockRecorder { + return m.recorder +} + +// Admin mocks base method. +func (m *MockStreamHandler) Admin() interface{} { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Admin") + ret0, _ := ret[0].(interface{}) + return ret0 +} + +// Admin indicates an expected call of Admin. +func (mr *MockStreamHandlerMockRecorder) Admin() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Admin", reflect.TypeOf((*MockStreamHandler)(nil).Admin)) +} + +// Alloc mocks base method. +func (m *MockStreamHandler) Alloc(arg0 context.Context, arg1 uint64, arg2 uint32, arg3 proto.ClusterID, arg4 codemode.CodeMode) (*access0.Location, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Alloc", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].(*access0.Location) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Alloc indicates an expected call of Alloc. +func (mr *MockStreamHandlerMockRecorder) Alloc(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Alloc", reflect.TypeOf((*MockStreamHandler)(nil).Alloc), arg0, arg1, arg2, arg3, arg4) +} + +// Delete mocks base method. +func (m *MockStreamHandler) Delete(arg0 context.Context, arg1 *access0.Location) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockStreamHandlerMockRecorder) Delete(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockStreamHandler)(nil).Delete), arg0, arg1) +} + +// Get mocks base method. +func (m *MockStreamHandler) Get(arg0 context.Context, arg1 io.Writer, arg2 access0.Location, arg3, arg4 uint64) (func() error, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].(func() error) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockStreamHandlerMockRecorder) Get(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockStreamHandler)(nil).Get), arg0, arg1, arg2, arg3, arg4) +} + +// Put mocks base method. +func (m *MockStreamHandler) Put(arg0 context.Context, arg1 io.Reader, arg2 int64, arg3 access0.HasherMap) (*access0.Location, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Put", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(*access0.Location) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Put indicates an expected call of Put. +func (mr *MockStreamHandlerMockRecorder) Put(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockStreamHandler)(nil).Put), arg0, arg1, arg2, arg3) +} + +// PutAt mocks base method. +func (m *MockStreamHandler) PutAt(arg0 context.Context, arg1 io.Reader, arg2 proto.ClusterID, arg3 proto.Vid, arg4 proto.BlobID, arg5 int64, arg6 access0.HasherMap) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutAt", arg0, arg1, arg2, arg3, arg4, arg5, arg6) + ret0, _ := ret[0].(error) + return ret0 +} + +// PutAt indicates an expected call of PutAt. +func (mr *MockStreamHandlerMockRecorder) PutAt(arg0, arg1, arg2, arg3, arg4, arg5, arg6 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutAt", reflect.TypeOf((*MockStreamHandler)(nil).PutAt), arg0, arg1, arg2, arg3, arg4, arg5, arg6) +} + +// MockLimiter is a mock of Limiter interface. +type MockLimiter struct { + ctrl *gomock.Controller + recorder *MockLimiterMockRecorder +} + +// MockLimiterMockRecorder is the mock recorder for MockLimiter. +type MockLimiterMockRecorder struct { + mock *MockLimiter +} + +// NewMockLimiter creates a new mock instance. +func NewMockLimiter(ctrl *gomock.Controller) *MockLimiter { + mock := &MockLimiter{ctrl: ctrl} + mock.recorder = &MockLimiterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockLimiter) EXPECT() *MockLimiterMockRecorder { + return m.recorder +} + +// Acquire mocks base method. +func (m *MockLimiter) Acquire(arg0 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Acquire", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Acquire indicates an expected call of Acquire. +func (mr *MockLimiterMockRecorder) Acquire(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Acquire", reflect.TypeOf((*MockLimiter)(nil).Acquire), arg0) +} + +// Reader mocks base method. +func (m *MockLimiter) Reader(arg0 context.Context, arg1 io.Reader) io.Reader { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Reader", arg0, arg1) + ret0, _ := ret[0].(io.Reader) + return ret0 +} + +// Reader indicates an expected call of Reader. +func (mr *MockLimiterMockRecorder) Reader(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reader", reflect.TypeOf((*MockLimiter)(nil).Reader), arg0, arg1) +} + +// Release mocks base method. +func (m *MockLimiter) Release(arg0 string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Release", arg0) +} + +// Release indicates an expected call of Release. +func (mr *MockLimiterMockRecorder) Release(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Release", reflect.TypeOf((*MockLimiter)(nil).Release), arg0) +} + +// Status mocks base method. +func (m *MockLimiter) Status() Status { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Status") + ret0, _ := ret[0].(Status) + return ret0 +} + +// Status indicates an expected call of Status. +func (mr *MockLimiterMockRecorder) Status() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Status", reflect.TypeOf((*MockLimiter)(nil).Status)) +} + +// Writer mocks base method. +func (m *MockLimiter) Writer(arg0 context.Context, arg1 io.Writer) io.Writer { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Writer", arg0, arg1) + ret0, _ := ret[0].(io.Writer) + return ret0 +} + +// Writer indicates an expected call of Writer. +func (mr *MockLimiterMockRecorder) Writer(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Writer", reflect.TypeOf((*MockLimiter)(nil).Writer), arg0, arg1) +} diff --git a/blobstore/access/codemode.go b/blobstore/access/codemode.go new file mode 100644 index 000000000..c4f1a3834 --- /dev/null +++ b/blobstore/access/codemode.go @@ -0,0 +1,45 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "fmt" + + "github.com/cubefs/cubefs/blobstore/common/codemode" +) + +// CodeModePair codemode with pair of tactic and policy +type CodeModePair struct { + Policy codemode.Policy + Tactic codemode.Tactic +} + +// CodeModePairs map of CodeModePair +type CodeModePairs map[codemode.CodeMode]CodeModePair + +// SelectCodeMode select codemode by size +func (c CodeModePairs) SelectCodeMode(size int64) codemode.CodeMode { + for codeMode, pair := range c { + policy := pair.Policy + if !policy.Enable { + continue + } + if size >= policy.MinSize && size <= policy.MaxSize { + return codeMode + } + } + + panic(fmt.Sprintf("no codemode policy to be selected by size %d, %+v", size, c)) +} diff --git a/blobstore/access/codemode_test.go b/blobstore/access/codemode_test.go new file mode 100644 index 000000000..8a43bd0a3 --- /dev/null +++ b/blobstore/access/codemode_test.go @@ -0,0 +1,71 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/access" + "github.com/cubefs/cubefs/blobstore/common/codemode" +) + +func TestAccessStreamCodeModePairs(t *testing.T) { + m := access.CodeModePairs{ + codemode.EC6P6: access.CodeModePair{ + Policy: codemode.Policy{ + ModeName: codemode.EC6P6.Name(), + MinSize: 1 << 10, + MaxSize: 1 << 20, + Enable: true, + }, + Tactic: codemode.EC6P6.Tactic(), + }, + codemode.EC6P10L2: access.CodeModePair{ + Policy: codemode.Policy{ + ModeName: codemode.EC6P10L2.Name(), + MinSize: 1 << 30, + MaxSize: 1 << 40, + Enable: true, + }, + Tactic: codemode.EC6P10L2.Tactic(), + }, + } + + cases := []struct { + size int64 + isPanic bool + mode codemode.CodeMode + }{ + {-1, true, 0}, + {0, true, 0}, + {1 << 8, true, 0}, + {1 << 10, false, codemode.EC6P6}, + {1 << 14, false, codemode.EC6P6}, + {1 << 20, false, codemode.EC6P6}, + {1 << 25, true, 0}, + {1 << 30, false, codemode.EC6P10L2}, + {1 << 40, false, codemode.EC6P10L2}, + {1 << 50, true, 0}, + } + for _, cs := range cases { + if cs.isPanic { + require.Panics(t, func() { m.SelectCodeMode(cs.size) }) + } else { + require.Equal(t, cs.mode, m.SelectCodeMode(cs.size)) + } + } +} diff --git a/blobstore/access/config_defaulter.go b/blobstore/access/config_defaulter.go new file mode 100644 index 000000000..0f5c6b3f8 --- /dev/null +++ b/blobstore/access/config_defaulter.go @@ -0,0 +1,75 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +const ( + defaultMaxBlobSize uint32 = 1 << 22 // 4MB + + defaultDiskPunishIntervalS int = 60 + defaultServicePunishIntervalS int = 60 + defaultAllocRetryTimes int = 3 + defaultAllocRetryIntervalMS int = 100 + defaultEncoderConcurrency int = 1000 + defaultMinReadShardsX int = 1 + + // client timeout ms + defaultTimeoutClusterMgr int64 = 1000 * 3 + defaultTimeoutProxy int64 = 1000 * 5 + defaultTimeoutBlobnode int64 = 1000 * 5 + + defaultAllocatorErrorPercentThreshold int = 50 + defaultAllocatorMaxConcurrentRequests int = 10240 + defaultAllocatorRequestVolumeThreshold int = 100 + defaultAllocatorSleepWindow int = 2 * 1000 + defaultAllocatorTimeout int = 30 * 1000 + defaultBlobnodeErrorPercentThreshold int = 80 + defaultBlobnodeMaxConcurrentRequests int = 102400 + defaultBlobnodeRequestVolumeThreshold int = 1000 + defaultBlobnodeSleepWindow int = 5 * 1000 + defaultBlobnodeTimeout int = 600 * 1000 +) + +// ec buffer +// | blobsize | EC15P12 | EC6P6 | EC16P20L2 | EC6P10L2 | EC3P3 | +// | 1(1 B) | 55296(54 KiB) | 24576(24 KiB) | 77824(76 KiB) | 36864(36 KiB) | 12288(12 KiB) | +// | 2048(2.0 KiB) | 55296(54 KiB) | 24576(24 KiB) | 77824(76 KiB) | 36864(36 KiB) | 12288(12 KiB) | +// | 12288(12 KiB) | 55296(54 KiB) | 24576(24 KiB) | 77824(76 KiB) | 36864(36 KiB) | 24576(24 KiB) | +// | 65536(64 KiB) | 117990(115 KiB) | 131076(128 KiB) | 155648(152 KiB) | 196614(192 KiB) | 131076(128 KiB) | +// | 524288(512 KiB) | 943731(922 KiB) | 1048584(1.0 MiB) | 1245184(1.2 MiB) | 1572876(1.5 MiB) | 1048578(1.0 MiB) | +// | 1048576(1.0 MiB) | 1887462(1.8 MiB) | 2097156(2.0 MiB) | 2490368(2.4 MiB) | 3145734(3.0 MiB) | 2097156(2.0 MiB) | +// | 2097152(2.0 MiB) | 3774897(3.6 MiB) | 4194312(4.0 MiB) | 4980736(4.8 MiB) | 6291468(6.0 MiB) | 4194306(4.0 MiB) | +// | 4194304(4.0 MiB) | 7549767(7.2 MiB) | 8388612(8.0 MiB) | 9961472(9.5 MiB) | 12582918(12 MiB) | 8388612(8.0 MiB) | +// | 8388608(8.0 MiB) | 15099507(14 MiB) | 16777224(16 MiB) | 19922944(19 MiB) | 25165836(24 MiB) | 16777218(16 MiB) | +// | 16777216(16 MiB) | 30199014(29 MiB) | 33554436(32 MiB) | 39845888(38 MiB) | 50331654(48 MiB) | 33554436(32 MiB) | + +const ( + _kib = 1 << 10 + _mib = 1 << 20 +) + +func getDefaultMempoolSize() map[int]int { + return map[int]int{ + _kib * 2: -1, // 2KiB for aligned ranged object + _kib * 16: -1, + _kib * 128: -1, + _kib * 512: -1, + _mib * 1: -1, + _mib * 2: -1, + _mib * 4: -1, + _mib * 8: -1, + _mib * 16: -1, + _mib * 32: -1, + } +} diff --git a/blobstore/access/controller/cluster.go b/blobstore/access/controller/cluster.go new file mode 100644 index 000000000..e196d42e9 --- /dev/null +++ b/blobstore/access/controller/cluster.go @@ -0,0 +1,376 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package controller + +import ( + "context" + "encoding/json" + "fmt" + "math/rand" + "path/filepath" + "sort" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/hashicorp/consul/api" + + cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/redis" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/util/errors" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +// TODO: how to stop service of one cluster??? + +// AlgChoose algorithm of choose cluster +type AlgChoose uint32 + +const ( + minAlg AlgChoose = iota + // AlgAvailable available capacity and some random alloc + AlgAvailable + // AlgRoundRobin alloc cluster round robin + AlgRoundRobin + // AlgRandom completely random alloc + AlgRandom + maxAlg +) + +// IsValid returns valid algorithm or not. +func (alg AlgChoose) IsValid() bool { + return alg > minAlg && alg < maxAlg +} + +func (alg AlgChoose) String() string { + switch alg { + case AlgAvailable: + return "Available" + case AlgRandom: + return "Random" + default: + return "Unknow" + } +} + +// errors +var ( + ErrNoSuchCluster = errors.New("controller: no such cluster") + ErrInvalidChooseAlg = errors.New("controller: invalid cluster chosen algorithm") +) + +// ClusterController controller of clusters in one region +type ClusterController interface { + // Region returns region in configuration + Region() string + // All returns all cluster info in this region + All() []*cmapi.ClusterInfo + // ChooseOne returns a available cluster to upload + ChooseOne() (*cmapi.ClusterInfo, error) + // GetServiceController return ServiceController in specified cluster + GetServiceController(clusterID proto.ClusterID) (ServiceController, error) + // GetVolumeGetter return VolumeGetter in specified cluster + GetVolumeGetter(clusterID proto.ClusterID) (VolumeGetter, error) + // GetConfig get specified config of key from cluster manager + GetConfig(ctx context.Context, key string) (string, error) + // ChangeChooseAlg change alloc algorithm + ChangeChooseAlg(alg AlgChoose) error +} + +// ClusterConfig cluster config +// +// Region and RegionMagic are paired, +// magic cannot change if one region was deployed. +type ClusterConfig struct { + IDC string `json:"-"` + Region string `json:"region"` + RegionMagic string `json:"region_magic"` + ClusterReloadSecs int `json:"cluster_reload_secs"` + ServiceReloadSecs int `json:"service_reload_secs"` + CMClientConfig cmapi.Config `json:"clustermgr_client_config"` + RedisClientConfig redis.ClusterConfig `json:"redis_client_config"` + + ServicePunishThreshold uint32 `json:"service_punish_threshold"` + ServicePunishValidIntervalS int `json:"service_punish_valid_interval_s"` +} + +type cluster struct { + clusterInfo *cmapi.ClusterInfo + client *cmapi.Client +} + +type clusterMap map[proto.ClusterID]*cluster + +type clusterQueue []*cmapi.ClusterInfo + +type clusterControllerImpl struct { + region string + kvClient *api.Client + allocAlg uint32 + totalAvailable int64 // available space of all clusters + clusters atomic.Value // all clusters + available atomic.Value // available clusters + serviceMgrs sync.Map + volumeGetters sync.Map + roundRobinCount uint64 // a count for round robin + + config ClusterConfig +} + +// NewClusterController returns a cluster controller +func NewClusterController(cfg *ClusterConfig, kvClient *api.Client) (ClusterController, error) { + controller := &clusterControllerImpl{ + region: cfg.Region, + kvClient: kvClient, + config: *cfg, + } + atomic.StoreUint32(&controller.allocAlg, uint32(AlgAvailable)) + + if err := controller.load(); err != nil { + return nil, errors.Base(err, "load cluster failed") + } + + if cfg.ClusterReloadSecs <= 0 { + cfg.ClusterReloadSecs = 3 + } + tick := time.NewTicker(time.Duration(cfg.ClusterReloadSecs) * time.Second) + go func() { + defer tick.Stop() + for range tick.C { + if err := controller.load(); err != nil { + log.Warn("load timer error", err) + } + } + }() + return controller, nil +} + +func (c *clusterControllerImpl) load() error { + span := trace.SpanFromContextSafe(context.Background()) + + path := cmapi.GetConsulClusterPath(c.region) + span.Debug("to list consul path", path) + + pairs, _, err := c.kvClient.KV().List(path, nil) + if err != nil { + return err + } + span.Debugf("found %d clusters", len(pairs)) + + allClusters := make(clusterMap) + available := make([]*cmapi.ClusterInfo, 0, len(pairs)) + totalAvailable := int64(0) + for _, pair := range pairs { + clusterInfo := &cmapi.ClusterInfo{} + err := json.Unmarshal(pair.Value, clusterInfo) + if err != nil { + span.Warnf("decode failed, raw:%s, error:%s", string(pair.Value), err.Error()) + continue + } + + clusterKey := filepath.Base(pair.Key) + span.Debug("found cluster", clusterKey) + + clusterID, err := strconv.Atoi(clusterKey) + if err != nil { + span.Warn("invalid cluster id", clusterKey, err) + continue + } + if clusterInfo.ClusterID != proto.ClusterID(clusterID) { + span.Warn("mismatch cluster id", clusterInfo.ClusterID, clusterID) + continue + } + + allClusters[proto.ClusterID(clusterID)] = &cluster{clusterInfo: clusterInfo} + if !clusterInfo.Readonly && clusterInfo.Available > 0 { + available = append(available, clusterInfo) + totalAvailable += clusterInfo.Available + } else { + span.Debug("readonly or no available cluster", clusterID) + } + } + + sort.Slice(available, func(i, j int) bool { + return available[i].Capacity < available[j].Capacity + }) + + newClusters := make([]*cmapi.ClusterInfo, 0, len(allClusters)) + for clusterID := range allClusters { + if _, ok := c.serviceMgrs.Load(clusterID); !ok { + newClusters = append(newClusters, allClusters[clusterID].clusterInfo) + } + } + + for _, newCluster := range newClusters { + conf := c.config.CMClientConfig + conf.Hosts = newCluster.Nodes + cmCli := cmapi.New(&conf) + + clusterID := newCluster.ClusterID + allClusters[clusterID].client = cmCli + + removeThisCluster := func() { + delete(allClusters, clusterID) + if !newCluster.Readonly && newCluster.Available > 0 { + totalAvailable -= newCluster.Available + for j := range available { + if available[j].ClusterID == clusterID { + available = append(available[:j], available[j+1:]...) + break + } + } + } + } + + serviceController, err := NewServiceController(ServiceConfig{ + ClusterID: clusterID, + IDC: c.config.IDC, + ReloadSec: c.config.ServiceReloadSecs, + ServicePunishThreshold: c.config.ServicePunishThreshold, + ServicePunishValidIntervalS: c.config.ServicePunishValidIntervalS, + }, cmCli) + if err != nil { + removeThisCluster() + span.Warn("new service manager failed", clusterID, err) + continue + } + + var redisCli *redis.ClusterClient + if len(c.config.RedisClientConfig.Addrs) > 0 { + redisCli = redis.NewClusterClient(&c.config.RedisClientConfig) + } + volumeGetter, err := NewVolumeGetter(clusterID, cmCli, redisCli, -1) + if err != nil { + removeThisCluster() + span.Warn("new volume getter failed", clusterID, err) + continue + } + + c.serviceMgrs.Store(clusterID, serviceController) + c.volumeGetters.Store(clusterID, volumeGetter) + + span.Debug("loaded new cluster", clusterID) + } + + c.clusters.Store(allClusters) + c.available.Store(clusterQueue(available)) + atomic.StoreInt64(&c.totalAvailable, totalAvailable) + + span.Infof("loaded %d clusters, and %d available, total available space %.3fTB", + len(allClusters), len(available), float64(totalAvailable)/(1<<40)) + return nil +} + +func (c *clusterControllerImpl) Region() string { + return c.region +} + +func (c *clusterControllerImpl) All() []*cmapi.ClusterInfo { + allClusters := c.clusters.Load().(clusterMap) + + ret := make([]*cmapi.ClusterInfo, 0, len(allClusters)) + for _, clusterInfo := range allClusters { + ret = append(ret, clusterInfo.clusterInfo) + } + + return ret +} + +func (c *clusterControllerImpl) ChooseOne() (*cmapi.ClusterInfo, error) { + alg := AlgChoose(atomic.LoadUint32(&c.allocAlg)) + + switch alg { + case AlgAvailable: + totalAvailable := atomic.LoadInt64(&c.totalAvailable) + if totalAvailable <= 0 { + return nil, fmt.Errorf("no available space %d", totalAvailable) + } + randValue := rand.Int63n(totalAvailable) + available := c.available.Load().(clusterQueue) + for _, cluster := range available { + if cluster.Available >= randValue { + return cluster, nil + } + randValue -= cluster.Available + } + + case AlgRoundRobin: + available := c.available.Load().(clusterQueue) + if length := uint64(len(available)); length > 0 { + count := atomic.AddUint64(&c.roundRobinCount, 1) + return available[count%length], nil + } + + case AlgRandom: + available := c.available.Load().(clusterQueue) + if length := int64(len(available)); length > 0 { + return available[rand.Int63()%length], nil + } + + default: + return nil, fmt.Errorf("not implemented algorithm %s(%d)", alg.String(), alg) + } + + return nil, fmt.Errorf("no available cluster by %s", alg.String()) +} + +func (c *clusterControllerImpl) ChangeChooseAlg(alg AlgChoose) error { + if !alg.IsValid() { + return ErrInvalidChooseAlg + } + + atomic.StoreUint32(&c.allocAlg, uint32(alg)) + return nil +} + +func (c *clusterControllerImpl) GetServiceController(clusterID proto.ClusterID) (ServiceController, error) { + if serviceController, exist := c.serviceMgrs.Load(clusterID); exist { + if controller, ok := serviceController.(ServiceController); ok { + return controller, nil + } + return nil, fmt.Errorf("not service controller for %d", clusterID) + } + return nil, fmt.Errorf("no service controller of %d", clusterID) +} + +func (c *clusterControllerImpl) GetVolumeGetter(clusterID proto.ClusterID) (VolumeGetter, error) { + if volumeGetter, exist := c.volumeGetters.Load(clusterID); exist { + if getter, ok := volumeGetter.(VolumeGetter); ok { + return getter, nil + } + return nil, fmt.Errorf("not volume getter for %d", clusterID) + } + return nil, fmt.Errorf("no volume getter for %d", clusterID) +} + +func (c *clusterControllerImpl) GetConfig(ctx context.Context, key string) (ret string, err error) { + span := trace.SpanFromContextSafe(ctx) + + allClusters := c.clusters.Load().(clusterMap) + if len(allClusters) == 0 { + return "", ErrNoSuchCluster + } + + for _, cluster := range allClusters { + if ret, err = cluster.client.GetConfig(ctx, key); err == nil { + return + } + span.Warnf("get config[%s] from cluster[%d] failed, err: %v", key, cluster.clusterInfo.ClusterID, err) + } + return +} diff --git a/blobstore/access/controller/cluster_test.go b/blobstore/access/controller/cluster_test.go new file mode 100644 index 000000000..ea1efc183 --- /dev/null +++ b/blobstore/access/controller/cluster_test.go @@ -0,0 +1,341 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package controller_test + +import ( + "context" + "encoding/json" + "math/rand" + "net" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/hashicorp/consul/api" + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/access/controller" + "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/redis" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +var ( + hostAddr string + consulKV api.KVPairs = nil + + stableCluster = &atomic.Value{} + + cc, cc0, cc1, cc2, cc3, cc19 controller.ClusterController +) + +func initCluster() { + mux := http.NewServeMux() + mux.HandleFunc("/", consul) + mux.HandleFunc("/service/get", serviceGet) + + testServer := httptest.NewServer(mux) + hostAddr = testServer.URL + + cluster1 := clustermgr.ClusterInfo{ + Region: region, + ClusterID: 1, + Capacity: 1 << 50, + Available: 1 << 45, + Readonly: false, + Nodes: []string{hostAddr, hostAddr}, + } + data1, _ := json.Marshal(cluster1) + + cluster2 := clustermgr.ClusterInfo{ + Region: region, + ClusterID: 2, + Capacity: 1 << 50, + Available: 1 << 45, + Readonly: true, + Nodes: []string{hostAddr, hostAddr}, + } + data2, _ := json.Marshal(cluster2) + + cluster3 := clustermgr.ClusterInfo{ + Region: region, + ClusterID: 3, + Capacity: 1 << 50, + Available: -1024, + Readonly: false, + Nodes: []string{hostAddr, hostAddr}, + } + data3, _ := json.Marshal(cluster3) + + cluster9 := clustermgr.ClusterInfo{ + Region: region, + ClusterID: 9, + Capacity: 1 << 50, + Available: 1 << 40, + Readonly: false, + Nodes: []string{hostAddr, hostAddr}, + } + data9, _ := json.Marshal(cluster9) + + consulKV = api.KVPairs{ + &api.KVPair{Key: "1", Value: data1}, + &api.KVPair{Key: "2", Value: data2}, + &api.KVPair{Key: "3", Value: data3}, + &api.KVPair{Key: "9", Value: data9}, + &api.KVPair{Key: "4", Value: data1}, + &api.KVPair{Key: "5", Value: []byte("{invalid json")}, + &api.KVPair{Key: "cannot-parse-key", Value: []byte("{}")}, + } + + time.Sleep(time.Millisecond * 200) + initCC() +} + +func consul(w http.ResponseWriter, req *http.Request) { + val := stableCluster.Load() + if val != nil { + if b := val.([]byte); b != nil { + w.Write(b) + return + } + } + + data, _ := json.Marshal(consulKV) + w.Write(data) +} + +func serviceGet(w http.ResponseWriter, req *http.Request) { + val := stableCluster.Load() + if val != nil { + if b := val.([]byte); b != nil { + w.Write([]byte("{}")) + return + } + } + + if rand.Int31()%2 == 0 { + w.Write([]byte("{cannot unmarshal")) + } else { + w.Write([]byte("{}")) + } +} + +func initCC() { + defer func() { + var data []byte + stableCluster.Store(data) + }() + + count := 0 + for cc == nil || cc0 == nil || cc1 == nil { + c := newCC() + if cc == nil { + cc = c + } + + clusters := c.All() + switch len(clusters) { + case 0: + cc0 = c + case 1: + switch clusters[0].ClusterID { + case 1: + cc1 = c + case 2: + cc2 = c + case 3: + cc3 = c + } + } + + count++ + } + log.Debug("init clusters run:", count) + + // init cluster 2 + if cc2 == nil { + var clusters api.KVPairs + clusters = append(clusters, consulKV[1]) + data, _ := json.Marshal(clusters) + stableCluster.Store(data) + cc2 = newCC() + } + // init cluster 3 + if cc3 == nil { + var clusters api.KVPairs + clusters = append(clusters, consulKV[2]) + data, _ := json.Marshal(clusters) + stableCluster.Store(data) + cc3 = newCC() + } + + // init cluster 1 and 9 + if cc19 == nil { + var clusters api.KVPairs + clusters = append(clusters, consulKV[0]) + clusters = append(clusters, consulKV[3]) + data, _ := json.Marshal(clusters) + stableCluster.Store(data) + cc19 = newCC() + } +} + +func newCC() controller.ClusterController { + cfg := controller.ClusterConfig{ + Region: region, + ClusterReloadSecs: 0, + CMClientConfig: clustermgr.Config{ + LbConfig: rpc.LbConfig{ + Hosts: []string{"http://localhost"}, + }, + }, + RedisClientConfig: redis.ClusterConfig{ + Addrs: []string{redismr.Addr()}, + }, + } + if rand.Int31()%2 == 0 { + cfg.RedisClientConfig.Addrs = nil + } + conf := api.DefaultConfig() + conf.Address = hostAddr + conf.Transport = &http.Transport{ + MaxIdleConns: 1000, + IdleConnTimeout: time.Minute, + } + conf.Transport.DialContext = (&net.Dialer{KeepAlive: time.Minute}).DialContext + + client, _ := api.NewClient(conf) + cc, err := controller.NewClusterController(&cfg, client) + if err != nil { + panic(err) + } + return cc +} + +func TestAccessClusterNew(t *testing.T) { + require.Equal(t, region, cc.Region()) +} + +func TestAccessClusterAll(t *testing.T) { + for range [5]struct{}{} { + require.LessOrEqual(t, len(newCC().All()), 4) + } +} + +func TestAccessClusterChooseOne(t *testing.T) { + { + cluster, err := cc0.ChooseOne() + require.Error(t, err) + require.Equal(t, (*clustermgr.ClusterInfo)(nil), cluster) + } + { + _, err := cc3.ChooseOne() + require.Error(t, err) + + _, err = cc2.ChooseOne() + require.Error(t, err) + + cc2.ChangeChooseAlg(controller.AlgRoundRobin) + _, err = cc2.ChooseOne() + require.Error(t, err) + } + { + cluster, err := cc1.ChooseOne() + require.NoError(t, err) + require.NotNil(t, cluster) + + for _, alg := range []controller.AlgChoose{ + controller.AlgAvailable, + controller.AlgRoundRobin, + controller.AlgRandom, + } { + cc1.ChangeChooseAlg(alg) + for range [100]struct{}{} { + clusterx, err := cc1.ChooseOne() + require.NoError(t, err) + require.Equal(t, cluster, clusterx) + } + } + } +} + +func TestAccessClusterGetHandler(t *testing.T) { + { + service, err := cc2.GetServiceController(1) + require.Error(t, err) + require.Equal(t, nil, service) + + getter, err := cc2.GetVolumeGetter(1) + require.Error(t, err) + require.Equal(t, nil, getter) + + _, err = cc2.GetConfig(context.TODO(), "key") + require.Error(t, err) + } + { + service, err := cc1.GetServiceController(1) + require.NoError(t, err) + require.NotNil(t, service) + + getter, err := cc1.GetVolumeGetter(1) + require.NoError(t, err) + require.NotNil(t, getter) + + _, err = cc1.GetConfig(context.TODO(), "key") + require.Error(t, err) + } +} + +func TestAccessClusterChangeChooseAlg(t *testing.T) { + cases := []struct { + alg controller.AlgChoose + err error + }{ + {0, controller.ErrInvalidChooseAlg}, + {controller.AlgAvailable, nil}, + {controller.AlgRoundRobin, nil}, + {controller.AlgRandom, nil}, + {1024, controller.ErrInvalidChooseAlg}, + } + for _, cs := range cases { + err := cc.ChangeChooseAlg(cs.alg) + require.Equal(t, err, cs.err) + } +} + +func TestAccessClusterChooseBalance(t *testing.T) { + cc := cc19 + for _, alg := range []controller.AlgChoose{ + controller.AlgAvailable, + controller.AlgRoundRobin, + controller.AlgRandom, + } { + err := cc.ChangeChooseAlg(alg) + require.NoError(t, err) + + m := make(map[proto.ClusterID]int, 2) + for range [10000]struct{}{} { + cluster, err := cc.ChooseOne() + require.NoError(t, err) + m[cluster.ClusterID]++ + } + + t.Logf("balance with algorithm %s: %+v", alg, m) + } +} diff --git a/blobstore/access/controller/metric.go b/blobstore/access/controller/metric.go new file mode 100644 index 000000000..8b189d7ac --- /dev/null +++ b/blobstore/access/controller/metric.go @@ -0,0 +1,33 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package controller + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +var cacheMetric = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "blobstore", + Subsystem: "access", + Name: "cache_hit_rate", + Help: "cache hit rate", + }, + []string{"cluster", "service", "status"}, +) + +func init() { + prometheus.MustRegister(cacheMetric) +} diff --git a/blobstore/access/controller/mock_test.go b/blobstore/access/controller/mock_test.go new file mode 100644 index 000000000..3be90ee4c --- /dev/null +++ b/blobstore/access/controller/mock_test.go @@ -0,0 +1,157 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package controller_test + +import ( + "context" + "math/rand" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/golang/mock/gomock" + + bnapi "github.com/cubefs/cubefs/blobstore/api/blobnode" + cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/common/codemode" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/redis" + "github.com/cubefs/cubefs/blobstore/testing/mocks" + "github.com/cubefs/cubefs/blobstore/util/errors" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +const ( + idc = "test-idc" + region = "test_region" +) + +var ( + vid404 = proto.Vid(404) + errNotFound = errors.New("not found") + + redismr *miniredis.Miniredis + rediscli *redis.ClusterClient + cmcli cmapi.APIAccess + + dataCalled map[proto.Vid]int + dataNodes map[string]cmapi.ServiceInfo + dataVolumes map[proto.Vid]cmapi.VolumeInfo + dataDisks map[proto.DiskID]bnapi.DiskInfo +) + +func init() { + rand.Seed(int64(time.Now().Nanosecond())) + log.SetOutputLevel(log.Lfatal) + + dataCalled = map[proto.Vid]int{1: 0, 9: 0} + + dataVolumes = make(map[proto.Vid]cmapi.VolumeInfo, 4) + dataVolumes[proto.Vid(1)] = cmapi.VolumeInfo{ + VolumeInfoBase: cmapi.VolumeInfoBase{ + Vid: 1, + CodeMode: codemode.EC6P10L2, + }, + Units: []cmapi.Unit{ + {Vuid: 1011, DiskID: 1021, Host: "1031"}, + {Vuid: 1012, DiskID: 1022, Host: "1032"}, + }, + } + dataVolumes[proto.Vid(9)] = cmapi.VolumeInfo{ + VolumeInfoBase: cmapi.VolumeInfoBase{ + Vid: 9, + CodeMode: codemode.EC16P20L2, + }, + Units: []cmapi.Unit{ + {Vuid: 9011, DiskID: 9021, Host: "9031"}, + {Vuid: 9012, DiskID: 9022, Host: "9032"}, + }, + } + dataVolumes[vid404] = cmapi.VolumeInfo{VolumeInfoBase: cmapi.VolumeInfoBase{Vid: vid404}} + + dataNodes = make(map[string]cmapi.ServiceInfo) + dataNodes[proto.ServiceNameProxy] = cmapi.ServiceInfo{ + Nodes: []cmapi.ServiceNode{ + { + ClusterID: 1, + Name: proto.ServiceNameProxy, + Host: "proxy-1", + Idc: idc, + }, + { + ClusterID: 1, + Name: proto.ServiceNameProxy, + Host: "proxy-2", + Idc: idc, + }, + }, + } + + dataDisks = make(map[proto.DiskID]bnapi.DiskInfo) + dataDisks[10001] = bnapi.DiskInfo{ + ClusterID: 1, + Idc: idc, + Host: "blobnode-1", + DiskHeartBeatInfo: bnapi.DiskHeartBeatInfo{ + DiskID: 10001, + }, + } + dataDisks[10002] = bnapi.DiskInfo{ + ClusterID: 1, + Idc: idc, + Host: "blobnode-2", + DiskHeartBeatInfo: bnapi.DiskHeartBeatInfo{ + DiskID: 10002, + }, + } + + redismr, _ = miniredis.Run() + rediscli = redis.NewClusterClient(&redis.ClusterConfig{ + Addrs: []string{redismr.Addr()}, + }) + + ctr := gomock.NewController(&testing.T{}) + cli := mocks.NewMockClientAPI(ctr) + cli.EXPECT().GetConfig(gomock.Any(), gomock.Any()).AnyTimes().Return("abc", nil) + cli.EXPECT().GetService(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( + func(ctx context.Context, args cmapi.GetServiceArgs) (cmapi.ServiceInfo, error) { + if val, ok := dataNodes[args.Name]; ok { + return val, nil + } + return cmapi.ServiceInfo{}, errNotFound + }) + cli.EXPECT().GetVolumeInfo(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( + func(ctx context.Context, args *cmapi.GetVolumeArgs) (*cmapi.VolumeInfo, error) { + if val, ok := dataVolumes[args.Vid]; ok { + dataCalled[args.Vid]++ + if args.Vid == vid404 { + return nil, errcode.ErrVolumeNotExist + } + return &val, nil + } + return nil, errNotFound + }) + cli.EXPECT().DiskInfo(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( + func(ctx context.Context, id proto.DiskID) (*bnapi.DiskInfo, error) { + if val, ok := dataDisks[id]; ok { + return &val, nil + } + return nil, errNotFound + }) + cmcli = cli + + initCluster() +} diff --git a/blobstore/access/controller/service.go b/blobstore/access/controller/service.go new file mode 100644 index 000000000..93675f6b8 --- /dev/null +++ b/blobstore/access/controller/service.go @@ -0,0 +1,335 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package controller + +import ( + "context" + "fmt" + "math/rand" + "sync" + "sync/atomic" + "time" + + "golang.org/x/sync/singleflight" + + "github.com/cubefs/cubefs/blobstore/api/blobnode" + "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/util/defaulter" + "github.com/cubefs/cubefs/blobstore/util/errors" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +const ( + _diskHostServicePrefix = "diskhost" + + // default service punish check valid interval + defaultServicePinishValidIntervalS int = 30 + // default service punish check threshold + defaultServicePinishThreshold uint32 = 3 +) + +// HostIDC item of host with idc +type HostIDC struct { + Host string + IDC string + Punished bool +} + +// ServiceController support for both data node discovery and normal service discovery +type ServiceController interface { + // GetServiceHost return an available service host + GetServiceHost(ctx context.Context, name string) (host string, err error) + // GetServiceHosts return all available service random sorted hosts + GetServiceHosts(ctx context.Context, name string) (hosts []string, err error) + // GetDiskHost return an disk's related data node host + GetDiskHost(ctx context.Context, diskID proto.DiskID) (hostIDC *HostIDC, err error) + // PunishService will punish an service host for an punishTimeSec interval + PunishService(ctx context.Context, service, host string, punishTimeSec int) + // PunishServiceWithThreshold will punish an service host for + // an punishTimeSec interval if service failed times satisfied with threshold during some interval time + PunishServiceWithThreshold(ctx context.Context, service, host string, punishTimeSec int) + // PunishDisk will punish a disk host for an punishTimeSec interval + PunishDisk(ctx context.Context, diskID proto.DiskID, punishTimeSec int) + // PunishDiskWithThreshold will punish a disk host for + // an punishTimeSec interval if disk host failed times satisfied with threshold + PunishDiskWithThreshold(ctx context.Context, diskID proto.DiskID, punishTimeSec int) +} + +type ( + serviceList []*hostItem + serviceMap map[string]*atomic.Value +) + +// hostItem represent a service or host item info +type hostItem struct { + host string + idc string + + // punish time record the punish end time unix of host item + punishTimeUnix int64 + // modify time record the last modify time unix of host item + lastModifyTime int64 + // failedTimes record the service host failed times during some interval + failedTimes uint32 +} + +func (h *hostItem) isPunish() bool { + return time.Since(time.Unix(atomic.LoadInt64(&h.punishTimeUnix), 0)) < 0 +} + +// ServiceConfig service config +type ServiceConfig struct { + ClusterID proto.ClusterID + IDC string + ReloadSec int + ServicePunishThreshold uint32 + ServicePunishValidIntervalS int +} + +type serviceControllerImpl struct { + // allServices hold all disk/service host map, use for quickly find out + allServices sync.Map + serviceHosts serviceMap + + group singleflight.Group + serviceLocks map[string]*sync.RWMutex + cmClient clustermgr.APIAccess + + config ServiceConfig +} + +// NewServiceController returns a service controller +func NewServiceController(cfg ServiceConfig, cmCli clustermgr.APIAccess) (ServiceController, error) { + defaulter.Equal(&cfg.ServicePunishThreshold, defaultServicePinishThreshold) + defaulter.LessOrEqual(&cfg.ServicePunishValidIntervalS, defaultServicePinishValidIntervalS) + + controller := &serviceControllerImpl{ + serviceHosts: serviceMap{ + proto.ServiceNameProxy: &atomic.Value{}, + }, + cmClient: cmCli, + serviceLocks: map[string]*sync.RWMutex{ + proto.ServiceNameProxy: {}, + }, + config: cfg, + } + + err := controller.load(cfg.ClusterID, cfg.IDC) + if err != nil { + return nil, errors.Base(err, "load service failed") + } + + if cfg.ReloadSec <= 0 { + cfg.ReloadSec = 10 + } + tick := time.NewTicker(time.Duration(cfg.ReloadSec) * time.Second) + go func() { + defer tick.Stop() + for range tick.C { + if err := controller.load(cfg.ClusterID, cfg.IDC); err != nil { + log.Warn("load timer error", err) + } + } + }() + return controller, nil +} + +// load initial all service and service hosts +func (s *serviceControllerImpl) load(cid proto.ClusterID, idc string) error { + span, ctx := trace.StartSpanFromContext(context.Background(), "access_cluster_service") + span.Debug("service loader for cluster:", cid) + + serviceName := proto.ServiceNameProxy + service, err := s.cmClient.GetService(ctx, clustermgr.GetServiceArgs{Name: serviceName}) + if err != nil { + span.Warn("get service from cluster manager failed", err) + return err + } + + span.Debugf("found %d server nodes of %s in the cluster", len(service.Nodes), serviceName) + hostItems := make(serviceList, 0, len(service.Nodes)) + for _, node := range service.Nodes { + if node.Idc != idc { + continue + } + hostItems = append(hostItems, &hostItem{idc: node.Idc, host: node.Host}) + } + if len(hostItems) > 0 { + for _, item := range hostItems { + s.allServices.Store(serviceName+item.host, item) + span.Debugf("store node %+v", item) + } + s.serviceHosts[serviceName].Store(hostItems) + } + return nil +} + +// GetServiceHost return an available service host +func (s *serviceControllerImpl) GetServiceHost(ctx context.Context, name string) (host string, err error) { + serviceList, ok := s.serviceHosts[name].Load().(serviceList) + if !ok { + return "", errors.Newf("not found host of %s", name) + } + + lock := s.getServiceLock(name) + idx := 0 + +RETRY: + + lock.RLock() + length := len(serviceList) + if length == 0 { + lock.RUnlock() + return "", errors.Newf("no any host of %s", name) + } + idx = rand.Intn(length) + + item := serviceList[idx] + lock.RUnlock() + + if !item.isPunish() { + return item.host, nil + } + + lock.Lock() + // double check + v := serviceList[idx] + // if serviceList[idx] still equal to item, then remove it + if v == item { + serviceList = append(serviceList[:idx], serviceList[idx+1:]...) + } + s.serviceHosts[name].Store(serviceList) + lock.Unlock() + + goto RETRY +} + +// GetServiceHosts return all available random-sorted hosts of service +func (s *serviceControllerImpl) GetServiceHosts(ctx context.Context, name string) (hosts []string, err error) { + serviceList, ok := s.serviceHosts[name].Load().(serviceList) + if !ok { + return nil, errors.Newf("not found host of %s", name) + } + + lock := s.getServiceLock(name) + + lock.RLock() + length := len(serviceList) + if length == 0 { + lock.RUnlock() + return nil, errors.Newf("no any host of %s", name) + } + + hosts = make([]string, 0, length) + for _, item := range serviceList { + if !item.isPunish() { + hosts = append(hosts, item.host) + } + } + lock.RUnlock() + + if len(hosts) == 0 { + return nil, errors.Newf("no available host of %s", name) + } + + rand.Shuffle(len(hosts), func(i, j int) { + hosts[i], hosts[j] = hosts[j], hosts[i] + }) + return hosts, nil +} + +// GetDiskHost return an disk's related data node host +func (s *serviceControllerImpl) GetDiskHost(ctx context.Context, diskID proto.DiskID) (*HostIDC, error) { + span := trace.SpanFromContextSafe(ctx) + + v, ok := s.allServices.Load(_diskHostServicePrefix + (diskID.ToString())) + if ok { + item := v.(*hostItem) + return &HostIDC{ + Host: item.host, + IDC: item.idc, + Punished: item.isPunish(), + }, nil + } + ret, err, _ := s.group.Do("get-diskinfo-"+diskID.ToString(), func() (interface{}, error) { + diskInfo, err := s.cmClient.DiskInfo(ctx, diskID) + if err != nil { + return nil, err + } + return diskInfo, nil + }) + if err != nil { + span.Error("can't get disk host from clustermgr", err) + return nil, errors.Base(err, "get disk info", diskID) + } + diskInfo := ret.(*blobnode.DiskInfo) + + item := &hostItem{host: diskInfo.Host, idc: diskInfo.Idc} + s.allServices.Store(_diskHostServicePrefix+(diskInfo.DiskID.ToString()), item) + return &HostIDC{ + Host: item.host, + IDC: item.idc, + Punished: item.isPunish(), + }, nil +} + +// PunishService will punish an service host for an punishTimeSec interval +func (s *serviceControllerImpl) PunishService(ctx context.Context, service, host string, punishTimeSec int) { + v, ok := s.allServices.Load(service + host) + if !ok { + panic(fmt.Sprintf("can't find host in all services map, %s-%s", service, host)) + } + item := v.(*hostItem) + + // atomic set item's punish time unix + atomic.StoreInt64(&item.punishTimeUnix, time.Now().Add(time.Duration(punishTimeSec)*time.Second).Unix()) +} + +// PunishDisk will punish a disk host for an punishTimeSec interval +func (s *serviceControllerImpl) PunishDisk(ctx context.Context, diskID proto.DiskID, punishTimeSec int) { + s.PunishService(ctx, _diskHostServicePrefix, diskID.ToString(), punishTimeSec) +} + +// PunishDiskWithThreshold will punish a disk host for +// an punishTimeSec interval if disk host failed times satisfied with threshold +func (s *serviceControllerImpl) PunishDiskWithThreshold(ctx context.Context, diskID proto.DiskID, punishTimeSec int) { + s.PunishServiceWithThreshold(ctx, _diskHostServicePrefix, diskID.ToString(), punishTimeSec) +} + +// PunishServiceWithThreshold will punish an service host for +// an punishTimeSec interval if service failed times satisfied with threshold +func (s *serviceControllerImpl) PunishServiceWithThreshold(ctx context.Context, service, host string, punishTimeSec int) { + v, ok := s.allServices.Load(service + host) + if !ok { + panic(fmt.Sprintf("can't can host in all services map, %s-%s", service, host)) + } + item := v.(*hostItem) + new := atomic.AddUint32(&item.failedTimes, 1) + // failedTimes larger than threshold, then check the lastModifyTime + if new >= s.config.ServicePunishThreshold { + if time.Since(time.Unix(atomic.LoadInt64(&item.lastModifyTime), 0)) < time.Duration(s.config.ServicePunishValidIntervalS)*time.Second { + s.PunishService(ctx, service, host, punishTimeSec) + return + } + atomic.AddUint32(&item.failedTimes, -(new - 1)) + } + atomic.StoreInt64(&item.lastModifyTime, time.Now().Unix()) +} + +func (s *serviceControllerImpl) getServiceLock(name string) *sync.RWMutex { + return s.serviceLocks[name] +} diff --git a/blobstore/access/controller/service_test.go b/blobstore/access/controller/service_test.go new file mode 100644 index 000000000..b310111ba --- /dev/null +++ b/blobstore/access/controller/service_test.go @@ -0,0 +1,243 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package controller_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/access/controller" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/trace" +) + +var serviceName = proto.ServiceNameProxy + +type hostSet map[string]struct{} + +func (set hostSet) Keys() []string { + keys := make([]string, 0, len(set)) + for k := range set { + keys = append(keys, k) + } + return keys +} + +var _, serviceCtx = trace.StartSpanFromContext(context.Background(), "TestAccessService") + +func TestAccessServiceNew(t *testing.T) { + { + sc, err := controller.NewServiceController( + controller.ServiceConfig{ + ClusterID: 0, + IDC: idc, + }, cmcli) + require.NoError(t, err) + + _, err = sc.GetServiceHost(serviceCtx, serviceName) + require.NoError(t, err) + } + { + sc, err := controller.NewServiceController( + controller.ServiceConfig{ + ClusterID: 0, + IDC: idc + "x", + ReloadSec: 1, + }, cmcli) + require.NoError(t, err) + + _, err = sc.GetServiceHost(serviceCtx, serviceName) + require.Error(t, err) + + time.Sleep(time.Second * 2) + } +} + +func TestAccessServiceGetServiceHost(t *testing.T) { + sc, err := controller.NewServiceController( + controller.ServiceConfig{ + ClusterID: 0, + IDC: idc, + ReloadSec: 1, + }, cmcli) + require.NoError(t, err) + + keys := make(hostSet) + for ii := 0; ii < 100; ii++ { + host, err := sc.GetServiceHost(serviceCtx, serviceName) + require.NoError(t, err) + keys[host] = struct{}{} + } + require.ElementsMatch(t, keys.Keys(), []string{"proxy-1", "proxy-2"}) + + hosts, err := sc.GetServiceHosts(serviceCtx, serviceName) + require.NoError(t, err) + require.ElementsMatch(t, []string{"proxy-1", "proxy-2"}, hosts) +} + +func TestAccessServicePunishService(t *testing.T) { + sc, err := controller.NewServiceController( + controller.ServiceConfig{ + ClusterID: 0, + IDC: idc, + ReloadSec: 1, + }, cmcli) + require.NoError(t, err) + + { + keys := make(hostSet) + for ii := 0; ii < 100; ii++ { + host, err := sc.GetServiceHost(serviceCtx, serviceName) + require.NoError(t, err) + keys[host] = struct{}{} + } + require.ElementsMatch(t, keys.Keys(), []string{"proxy-1", "proxy-2"}) + } + + sc.PunishService(serviceCtx, serviceName, "proxy-2", 2) + for ii := 0; ii < 100; ii++ { + host, err := sc.GetServiceHost(serviceCtx, serviceName) + require.NoError(t, err) + require.True(t, host == "proxy-1") + } + + time.Sleep(time.Second * 2) + { + keys := make(hostSet) + for ii := 0; ii < 100; ii++ { + host, err := sc.GetServiceHost(serviceCtx, serviceName) + require.NoError(t, err) + keys[host] = struct{}{} + } + require.ElementsMatch(t, keys.Keys(), []string{"proxy-1", "proxy-2"}) + } +} + +func TestAccessServicePunishServiceWithThreshold(t *testing.T) { + threshold := uint32(5) + sc, err := controller.NewServiceController( + controller.ServiceConfig{ + ClusterID: 0, + IDC: idc, + ReloadSec: 1, + ServicePunishThreshold: threshold, + ServicePunishValidIntervalS: 2, + }, cmcli) + require.NoError(t, err) + + { + keys := make(hostSet) + for ii := 0; ii < 100; ii++ { + host, err := sc.GetServiceHost(serviceCtx, serviceName) + require.NoError(t, err) + keys[host] = struct{}{} + } + require.ElementsMatch(t, keys.Keys(), []string{"proxy-1", "proxy-2"}) + } + + // not punish + for ii := threshold; ii > 1; ii-- { + sc.PunishServiceWithThreshold(serviceCtx, serviceName, "proxy-1", 2) + } + { + keys := make(hostSet) + for ii := 0; ii < 100; ii++ { + host, err := sc.GetServiceHost(serviceCtx, serviceName) + require.NoError(t, err) + keys[host] = struct{}{} + } + require.ElementsMatch(t, keys.Keys(), []string{"proxy-1", "proxy-2"}) + } + + // punished + sc.PunishServiceWithThreshold(serviceCtx, serviceName, "proxy-1", 2) + for ii := 0; ii < 100; ii++ { + host, err := sc.GetServiceHost(serviceCtx, serviceName) + require.NoError(t, err) + require.True(t, host == "proxy-2") + } + + time.Sleep(time.Second * 2) + { + keys := make(hostSet) + for ii := 0; ii < 100; ii++ { + host, err := sc.GetServiceHost(serviceCtx, serviceName) + require.NoError(t, err) + keys[host] = struct{}{} + } + require.ElementsMatch(t, keys.Keys(), []string{"proxy-1", "proxy-2"}) + } +} + +func TestAccessServiceGetDiskHost(t *testing.T) { + sc, err := controller.NewServiceController( + controller.ServiceConfig{ + ClusterID: 0, + IDC: idc, + ReloadSec: 1, + }, cmcli) + require.NoError(t, err) + + { + host, err := sc.GetDiskHost(serviceCtx, proto.DiskID(10001)) + require.NoError(t, err) + require.True(t, host.Host == "blobnode-1") + } + { + _, err := sc.GetDiskHost(serviceCtx, proto.DiskID(10000)) + require.Error(t, err) + } +} + +func TestAccessServicePunishDisk(t *testing.T) { + sc, err := controller.NewServiceController( + controller.ServiceConfig{ + ClusterID: 0, + IDC: idc, + ReloadSec: 1, + }, cmcli) + require.NoError(t, err) + + { + host, err := sc.GetDiskHost(serviceCtx, proto.DiskID(10001)) + require.NoError(t, err) + require.True(t, host.Host == "blobnode-1") + require.False(t, host.Punished) + } + sc.PunishDisk(serviceCtx, 10001, 1) + { + host, err := sc.GetDiskHost(serviceCtx, proto.DiskID(10001)) + require.NoError(t, err) + require.True(t, host.Host == "blobnode-1") + require.True(t, host.Punished) + } + time.Sleep(time.Second) + { + host, err := sc.GetDiskHost(serviceCtx, proto.DiskID(10001)) + require.NoError(t, err) + require.True(t, host.Host == "blobnode-1") + require.False(t, host.Punished) + } + sc.PunishDiskWithThreshold(serviceCtx, 10001, 1) + { + host, err := sc.GetDiskHost(serviceCtx, proto.DiskID(10001)) + require.NoError(t, err) + require.True(t, host.Host == "blobnode-1") + require.False(t, host.Punished) + } +} diff --git a/blobstore/access/controller/volume.go b/blobstore/access/controller/volume.go new file mode 100644 index 000000000..dfe4ffb02 --- /dev/null +++ b/blobstore/access/controller/volume.go @@ -0,0 +1,326 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package controller + +import ( + "context" + "fmt" + "math/rand" + "time" + + "golang.org/x/sync/singleflight" + + "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/common/codemode" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/memcache" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/redis" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/util/errors" + "github.com/cubefs/cubefs/blobstore/util/retry" +) + +const ( + _defaultCacheSize = 1 << 17 + _defaultCacheExpiration = int64(2 * time.Minute) +) + +// Unit alias of clustermgr.Unit +type Unit = clustermgr.Unit + +// VolumePhy volume physical info +// Vid, CodeMode and Units are from cluster mgr +// IsPunish is cached in memory +// Timestamp is cached in redis to clear outdate info +type VolumePhy struct { + Vid proto.Vid `json:"vid"` + CodeMode codemode.CodeMode `json:"codemode"` + IsPunish bool `json:"-"` + Timestamp int64 `json:"timestamp"` + Units []Unit `json:"units"` +} + +// VolumeGetter getter of volume physical location +// +// ctx: context with trace or something +// +// isCache: is false means reading from cluster then updating redis and memcache +// otherwise reading from memcache -> redis -> cluster +type VolumeGetter interface { + // Get returns volume physical location of vid + Get(ctx context.Context, vid proto.Vid, isCache bool) *VolumePhy + // Punish punish vid with interval seconds + Punish(ctx context.Context, vid proto.Vid, punishIntervalS int) +} +type cvid uint64 + +var ( + fmtKeyGroupVolume = func(id cvid) string { return fmt.Sprintf("get-volume-%d", id) } + fmtKeyRedisVolume = func(cid proto.ClusterID, vid proto.Vid) string { + return fmt.Sprintf("access/volume/%d/%d", cid, vid) + } +) + +func addCVid(cid proto.ClusterID, vid proto.Vid) cvid { + return cvid((uint64(cid) << 32) | uint64(vid)) +} + +// volumePhyCacher local k-v cache for (Vid, VolumePhy) +type volumePhyCacher interface { + Get(key cvid) *VolumePhy + Set(key cvid, value VolumePhy) +} + +type volumeMemCache struct { + cache *memcache.MemCache +} + +var _ volumePhyCacher = (*volumeMemCache)(nil) + +// Get implements volumePhyCacher.Get +func (vc *volumeMemCache) Get(key cvid) *VolumePhy { + value := vc.cache.Get(key) + if value == nil { + return nil + } + + phy, ok := value.(VolumePhy) + if !ok { + return nil + } + + return &phy +} + +// Set implements volumePhyCacher.Set +func (vc *volumeMemCache) Set(key cvid, value VolumePhy) { + vc.cache.Set(key, value) +} + +type volumeGetterImpl struct { + ctx context.Context + cid proto.ClusterID + + volumeMemCache volumePhyCacher + memExpiration int64 + punishCache *memcache.MemCache + + redisClient *redis.ClusterClient + cmClient clustermgr.APIAccess + + singleRun *singleflight.Group +} + +// NewVolumeGetter new a volume getter +// memExpiration expiration of memcache, 0 means no expiration +func NewVolumeGetter(clusterID proto.ClusterID, cmCli clustermgr.APIAccess, redisCli *redis.ClusterClient, + memExpiration time.Duration) (VolumeGetter, error) { + _, ctx := trace.StartSpanFromContext(context.Background(), "") + + expiration := int64(memExpiration) + if expiration < 0 { + expiration = _defaultCacheExpiration + } + + mc, err := memcache.NewMemCache(ctx, _defaultCacheSize) + if err != nil { + return nil, err + } + punishCache, err := memcache.NewMemCache(ctx, 1024) + if err != nil { + return nil, err + } + + getter := &volumeGetterImpl{ + ctx: ctx, + cid: clusterID, + volumeMemCache: &volumeMemCache{cache: mc}, + memExpiration: expiration, + punishCache: punishCache, + redisClient: redisCli, + cmClient: cmCli, + singleRun: new(singleflight.Group), + } + + return getter, nil +} + +// Get implements interface VolumeGetter +// 1.top level cache from local +// 2.second level cache from redis cluster +func (v *volumeGetterImpl) Get(ctx context.Context, vid proto.Vid, isCache bool) (phy *VolumePhy) { + span := trace.SpanFromContextSafe(ctx) + cid := v.cid.ToString() + id := addCVid(v.cid, vid) + + // check if volume punish + defer func() { + if phy == nil { + return + } + + val := v.punishCache.Get(id) + if val == nil { + return + } + + if time.Since(time.Unix(val.(int64), 0)) < 0 { + phy.IsPunish = true + } else { + v.punishCache.Set(id, nil) + } + }() + + // if isCache, reading from memcache -> redis -> cluster + if isCache { + if phy = v.getFromLocalCache(ctx, id); phy != nil { + if v.memExpiration == 0 { + if phy.Timestamp < 0 { + phy = nil + } + cacheMetric.WithLabelValues(cid, "memcache", "hit").Inc() + return + } + + now := time.Now().UnixNano() + // exist volume and not expired + if phy.Timestamp > 0 && now < phy.Timestamp+v.memExpiration { + span.Debug("got from memcache volume", cid, vid) + cacheMetric.WithLabelValues(cid, "memcache", "hit").Inc() + return + } + + // not exist volume and not expired, return nil + if phy.Timestamp < 0 && now < -phy.Timestamp+v.memExpiration { + span.Debug("got from memcache not exist volume", cid, vid) + cacheMetric.WithLabelValues(cid, "memcache", "hit").Inc() + phy = nil + return + } + + cacheMetric.WithLabelValues(cid, "memcache", "expired").Inc() + } + cacheMetric.WithLabelValues(cid, "memcache", "miss").Inc() + + if phy = v.getFromRedis(ctx, fmtKeyRedisVolume(v.cid, vid)); phy != nil { + span.Debug("got from redis volume", cid, vid) + cacheMetric.WithLabelValues(cid, "redis", "hit").Inc() + + phy.Timestamp = time.Now().UnixNano() + v.setToLocalCache(ctx, id, phy) + return + } + cacheMetric.WithLabelValues(cid, "redis", "miss").Inc() + } + + val, err, _ := v.singleRun.Do(fmtKeyGroupVolume(id), func() (interface{}, error) { + return v.getFromClusterAndUpdate(ctx, vid) + }) + if err != nil { + cacheMetric.WithLabelValues(cid, "clustermgr", "miss").Inc() + span.Error("get volume location from clustermgr failed", errors.Detail(err)) + return + } + cacheMetric.WithLabelValues(cid, "clustermgr", "hit").Inc() + + phy, ok := val.(*VolumePhy) + if !ok { + span.Errorf("%s has bad value in singleflight group %+v", fmtKeyGroupVolume(id), val) + return + } + span.Debug("got from cluster volume", cid, vid) + return +} + +func (v *volumeGetterImpl) Punish(ctx context.Context, vid proto.Vid, punishIntervalS int) { + v.punishCache.Set(addCVid(v.cid, vid), + time.Now().Add(time.Duration(punishIntervalS)*time.Second).Unix()) +} + +func (v *volumeGetterImpl) setToLocalCache(ctx context.Context, id cvid, phy *VolumePhy) error { + v.volumeMemCache.Set(id, *phy) + return nil +} + +func (v *volumeGetterImpl) getFromLocalCache(ctx context.Context, id cvid) *VolumePhy { + return v.volumeMemCache.Get(id) +} + +// expiration random 30 - 60 minutes +func (v *volumeGetterImpl) setToRedis(ctx context.Context, key string, phy *VolumePhy) error { + if v.redisClient == nil { + return nil + } + expiration := rand.Intn(30) + 30 + return v.redisClient.Set(ctx, key, phy, time.Minute*time.Duration(expiration)) +} + +func (v *volumeGetterImpl) getFromRedis(ctx context.Context, key string) *VolumePhy { + if v.redisClient == nil { + return nil + } + value := &VolumePhy{} + if err := v.redisClient.Get(ctx, key, value); err != nil { + span := trace.SpanFromContextSafe(ctx) + span.Info(key, err) + return nil + } + return value +} + +func (v *volumeGetterImpl) getFromClusterAndUpdate(ctx context.Context, vid proto.Vid) (*VolumePhy, error) { + span := trace.SpanFromContextSafe(ctx) + var ( + vInfo *clustermgr.VolumeInfo + err error + ) + + id := addCVid(v.cid, vid) + if err = retry.ExponentialBackoff(3, 100).On(func() error { + if vInfo, err = v.cmClient.GetVolumeInfo(ctx, &clustermgr.GetVolumeArgs{Vid: vid}); err != nil { + return err + } + return nil + }); err != nil { + // set local cache as nil(Timestamp is negative) when volume not exist + if rpc.DetectStatusCode(err) == errcode.CodeVolumeNotExist { + phy := &VolumePhy{ + Vid: vid, + Timestamp: -time.Now().UnixNano(), + } + span.Infof("to update memcache on not exist volume(%d-%d) %+v", v.cid, vid, phy) + v.setToLocalCache(ctx, id, phy) + } + return nil, errors.Base(err, "get volume from clustermgr", v.cid, vid) + } + + phy := &VolumePhy{ + Vid: vInfo.Vid, + CodeMode: vInfo.CodeMode, + Timestamp: time.Now().UnixNano(), + Units: make([]Unit, len(vInfo.Units)), + } + copy(phy.Units, vInfo.Units[:]) + + span.Debugf("to update memcache and redis volume(%d-%d) %+v", v.cid, vid, phy) + v.setToLocalCache(ctx, id, phy) + if err := v.setToRedis(ctx, fmtKeyRedisVolume(v.cid, vid), phy); err != nil { + span.Warn("set volume location into redis failed", err) + } + + return phy, nil +} diff --git a/blobstore/access/controller/volume_test.go b/blobstore/access/controller/volume_test.go new file mode 100644 index 000000000..e40048b84 --- /dev/null +++ b/blobstore/access/controller/volume_test.go @@ -0,0 +1,239 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package controller_test + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/access/controller" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/redis" + "github.com/cubefs/cubefs/blobstore/common/trace" +) + +var fmtKeyRedisVolume = func(cid proto.ClusterID, vid proto.Vid) string { + return fmt.Sprintf("access/volume/%d/%d", cid, vid) +} + +func TestAccessVolumeGetterNew(t *testing.T) { + _, ctx := trace.StartSpanFromContext(context.Background(), "TestAccessVolumeGetterNew") + + getter, err := controller.NewVolumeGetter(1, cmcli, rediscli, time.Millisecond*200) + require.Nil(t, err) + info := getter.Get(ctx, proto.Vid(0), true) + require.Nil(t, info) + + id := proto.Vid(1) + dataCalled[id] = 0 + info = getter.Get(ctx, id, true) + require.NotNil(t, info) + require.Equal(t, 1, dataCalled[id]) + require.Equal(t, proto.Vuid(1011), info.Units[0].Vuid) + + info = getter.Get(ctx, id, true) + require.Equal(t, 1, dataCalled[id]) + require.Equal(t, proto.Vuid(1012), info.Units[1].Vuid) + + // delete redis cache + rediscli.Del(context.TODO(), fmtKeyRedisVolume(1, id)) + getter.Get(ctx, id, true) + require.Equal(t, 1, dataCalled[id]) + + time.Sleep(time.Millisecond * 250) + getter.Get(ctx, id, true) + require.Equal(t, 2, dataCalled[id]) + + getter.Get(ctx, id, false) + require.Equal(t, 3, dataCalled[id]) + + rediscli.Del(context.TODO(), fmtKeyRedisVolume(1, id)) + id = proto.Vid(9) + info = getter.Get(ctx, id, true) + require.NotNil(t, info) + require.Equal(t, 1, dataCalled[id]) + + time.Sleep(time.Millisecond * 250) + info = getter.Get(ctx, id, true) + require.NotNil(t, info) + require.Equal(t, 1, dataCalled[id]) + + info = getter.Get(ctx, proto.Vid(10), false) + require.Nil(t, info) +} + +func TestAccessVolumeGetterNotExistVolume(t *testing.T) { + _, ctx := trace.StartSpanFromContext(context.Background(), "TestAccessVolumeGetterNotExistVolume") + + getter, err := controller.NewVolumeGetter(0xfe, cmcli, rediscli, time.Millisecond*500) + require.Nil(t, err) + info := getter.Get(ctx, proto.Vid(0), true) + require.Nil(t, info) + + id := proto.Vid(1) + dataCalled[id] = 0 + info = getter.Get(ctx, id, true) + require.NotNil(t, info) + require.Equal(t, 1, dataCalled[id]) + require.Equal(t, proto.Vuid(1011), info.Units[0].Vuid) + + id = vid404 + dataCalled[id] = 0 + info = getter.Get(ctx, id, true) + require.Nil(t, info) + require.Equal(t, 3, dataCalled[id]) + require.ErrorIs(t, redis.ErrNotFound, rediscli.Get(ctx, fmtKeyRedisVolume(0xfe, id), nil)) + + var wg sync.WaitGroup + wg.Add(10) + for ii := 0; ii < 10; ii++ { + go func() { + getter.Get(ctx, id, true) + wg.Done() + }() + } + wg.Wait() + require.Equal(t, 3, dataCalled[id]) + + time.Sleep(time.Millisecond * 500) + getter.Get(ctx, id, true) + require.Equal(t, 6, dataCalled[id]) + require.ErrorIs(t, redis.ErrNotFound, rediscli.Get(ctx, fmtKeyRedisVolume(0xfe, id), nil)) + + getter, err = controller.NewVolumeGetter(0xee, cmcli, rediscli, 0) + require.NoError(t, err) + id = vid404 + dataCalled[id] = 0 + for ii := 0; ii < 10; ii++ { + getter.Get(ctx, id, true) + require.Equal(t, 3, dataCalled[id]) + time.Sleep(time.Millisecond * 100) + } +} + +func TestAccessVolumeGetterExpiration(t *testing.T) { + _, ctx := trace.StartSpanFromContext(context.Background(), "TestAccessVolumeGetterExpiration") + mockRedis := redis.NewClusterClient(&redis.ClusterConfig{ + Addrs: []string{"127.0.0.1:5678"}, + }) + + getter, err := controller.NewVolumeGetter(1, cmcli, mockRedis, time.Millisecond*200) + require.Nil(t, err) + + info := getter.Get(ctx, proto.Vid(0), true) + require.Nil(t, info) + + id := proto.Vid(1) + dataCalled[id] = 0 + info = getter.Get(ctx, id, true) + require.NotNil(t, info) + require.Equal(t, 1, dataCalled[id]) + + info = getter.Get(ctx, id, true) + require.NotNil(t, info) + require.Equal(t, 1, dataCalled[id]) + + // expiration + time.Sleep(time.Millisecond * 250) + info = getter.Get(ctx, id, true) + require.NotNil(t, info) + require.Equal(t, 2, dataCalled[id]) + + info = getter.Get(ctx, id, false) + require.NotNil(t, info) + require.Equal(t, 3, dataCalled[id]) +} + +func TestAccessVolumeGetterBrokenRedis(t *testing.T) { + _, ctx := trace.StartSpanFromContext(context.Background(), "TestAccessVolumeGetterBrokenRedis") + mockRedis := redis.NewClusterClient(&redis.ClusterConfig{ + Addrs: []string{"127.0.0.1:5678"}, + }) + + getter, err := controller.NewVolumeGetter(1, cmcli, mockRedis, -1) + require.Nil(t, err) + + info := getter.Get(ctx, proto.Vid(0), true) + require.Nil(t, info) + + id := proto.Vid(1) + dataCalled[id] = 0 + info = getter.Get(ctx, id, true) + require.NotNil(t, info) + require.Equal(t, 1, dataCalled[id]) + require.Equal(t, proto.Vuid(1011), info.Units[0].Vuid) + + info = getter.Get(ctx, id, true) + require.NotNil(t, info) + require.Equal(t, 1, dataCalled[id]) + require.Equal(t, proto.Vuid(1012), info.Units[1].Vuid) +} + +func TestAccessVolumeGetterNoRedis(t *testing.T) { + _, ctx := trace.StartSpanFromContext(context.Background(), "TestAccessVolumeGetterNoRedis") + + getter, err := controller.NewVolumeGetter(1, cmcli, nil, -1) + require.Nil(t, err) + + info := getter.Get(ctx, proto.Vid(0), true) + require.Nil(t, info) + + id := proto.Vid(1) + dataCalled[id] = 0 + info = getter.Get(ctx, id, true) + require.NotNil(t, info) + require.Equal(t, 1, dataCalled[id]) + require.Equal(t, proto.Vuid(1011), info.Units[0].Vuid) + + info = getter.Get(ctx, id, true) + require.NotNil(t, info) + require.Equal(t, 1, dataCalled[id]) + require.Equal(t, proto.Vuid(1012), info.Units[1].Vuid) +} + +func TestAccessVolumePunish(t *testing.T) { + _, ctx := trace.StartSpanFromContext(context.Background(), "TestAccessVolumePunish") + + getter, err := controller.NewVolumeGetter(1, cmcli, rediscli, 0) + require.Nil(t, err) + + info := getter.Get(ctx, proto.Vid(0), true) + require.Nil(t, info) + + getter.Punish(ctx, proto.Vid(0), 10) + info = getter.Get(ctx, proto.Vid(0), true) + require.Nil(t, info) + + id := proto.Vid(1) + info = getter.Get(ctx, id, true) + require.NotNil(t, info) + require.Equal(t, proto.Vuid(1011), info.Units[0].Vuid) + require.False(t, info.IsPunish) + + getter.Punish(ctx, id, 1) + info = getter.Get(ctx, id, true) + require.NotNil(t, info) + require.True(t, info.IsPunish) + + time.Sleep(time.Second) + info = getter.Get(ctx, id, true) + require.NotNil(t, info) + require.False(t, info.IsPunish) +} diff --git a/blobstore/access/controller_mock_test.go b/blobstore/access/controller_mock_test.go new file mode 100644 index 000000000..d758542c1 --- /dev/null +++ b/blobstore/access/controller_mock_test.go @@ -0,0 +1,305 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cubefs/cubefs/blobstore/access/controller (interfaces: ClusterController,ServiceController,VolumeGetter) + +// Package access is a generated GoMock package. +package access + +import ( + context "context" + reflect "reflect" + + controller "github.com/cubefs/cubefs/blobstore/access/controller" + clustermgr "github.com/cubefs/cubefs/blobstore/api/clustermgr" + proto "github.com/cubefs/cubefs/blobstore/common/proto" + gomock "github.com/golang/mock/gomock" +) + +// MockClusterController is a mock of ClusterController interface. +type MockClusterController struct { + ctrl *gomock.Controller + recorder *MockClusterControllerMockRecorder +} + +// MockClusterControllerMockRecorder is the mock recorder for MockClusterController. +type MockClusterControllerMockRecorder struct { + mock *MockClusterController +} + +// NewMockClusterController creates a new mock instance. +func NewMockClusterController(ctrl *gomock.Controller) *MockClusterController { + mock := &MockClusterController{ctrl: ctrl} + mock.recorder = &MockClusterControllerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockClusterController) EXPECT() *MockClusterControllerMockRecorder { + return m.recorder +} + +// All mocks base method. +func (m *MockClusterController) All() []*clustermgr.ClusterInfo { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "All") + ret0, _ := ret[0].([]*clustermgr.ClusterInfo) + return ret0 +} + +// All indicates an expected call of All. +func (mr *MockClusterControllerMockRecorder) All() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "All", reflect.TypeOf((*MockClusterController)(nil).All)) +} + +// ChangeChooseAlg mocks base method. +func (m *MockClusterController) ChangeChooseAlg(arg0 controller.AlgChoose) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ChangeChooseAlg", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// ChangeChooseAlg indicates an expected call of ChangeChooseAlg. +func (mr *MockClusterControllerMockRecorder) ChangeChooseAlg(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChangeChooseAlg", reflect.TypeOf((*MockClusterController)(nil).ChangeChooseAlg), arg0) +} + +// ChooseOne mocks base method. +func (m *MockClusterController) ChooseOne() (*clustermgr.ClusterInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ChooseOne") + ret0, _ := ret[0].(*clustermgr.ClusterInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ChooseOne indicates an expected call of ChooseOne. +func (mr *MockClusterControllerMockRecorder) ChooseOne() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChooseOne", reflect.TypeOf((*MockClusterController)(nil).ChooseOne)) +} + +// GetConfig mocks base method. +func (m *MockClusterController) GetConfig(arg0 context.Context, arg1 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetConfig", arg0, arg1) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetConfig indicates an expected call of GetConfig. +func (mr *MockClusterControllerMockRecorder) GetConfig(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfig", reflect.TypeOf((*MockClusterController)(nil).GetConfig), arg0, arg1) +} + +// GetServiceController mocks base method. +func (m *MockClusterController) GetServiceController(arg0 proto.ClusterID) (controller.ServiceController, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetServiceController", arg0) + ret0, _ := ret[0].(controller.ServiceController) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetServiceController indicates an expected call of GetServiceController. +func (mr *MockClusterControllerMockRecorder) GetServiceController(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetServiceController", reflect.TypeOf((*MockClusterController)(nil).GetServiceController), arg0) +} + +// GetVolumeGetter mocks base method. +func (m *MockClusterController) GetVolumeGetter(arg0 proto.ClusterID) (controller.VolumeGetter, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVolumeGetter", arg0) + ret0, _ := ret[0].(controller.VolumeGetter) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetVolumeGetter indicates an expected call of GetVolumeGetter. +func (mr *MockClusterControllerMockRecorder) GetVolumeGetter(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVolumeGetter", reflect.TypeOf((*MockClusterController)(nil).GetVolumeGetter), arg0) +} + +// Region mocks base method. +func (m *MockClusterController) Region() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Region") + ret0, _ := ret[0].(string) + return ret0 +} + +// Region indicates an expected call of Region. +func (mr *MockClusterControllerMockRecorder) Region() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Region", reflect.TypeOf((*MockClusterController)(nil).Region)) +} + +// MockServiceController is a mock of ServiceController interface. +type MockServiceController struct { + ctrl *gomock.Controller + recorder *MockServiceControllerMockRecorder +} + +// MockServiceControllerMockRecorder is the mock recorder for MockServiceController. +type MockServiceControllerMockRecorder struct { + mock *MockServiceController +} + +// NewMockServiceController creates a new mock instance. +func NewMockServiceController(ctrl *gomock.Controller) *MockServiceController { + mock := &MockServiceController{ctrl: ctrl} + mock.recorder = &MockServiceControllerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockServiceController) EXPECT() *MockServiceControllerMockRecorder { + return m.recorder +} + +// GetDiskHost mocks base method. +func (m *MockServiceController) GetDiskHost(arg0 context.Context, arg1 proto.DiskID) (*controller.HostIDC, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetDiskHost", arg0, arg1) + ret0, _ := ret[0].(*controller.HostIDC) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetDiskHost indicates an expected call of GetDiskHost. +func (mr *MockServiceControllerMockRecorder) GetDiskHost(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDiskHost", reflect.TypeOf((*MockServiceController)(nil).GetDiskHost), arg0, arg1) +} + +// GetServiceHost mocks base method. +func (m *MockServiceController) GetServiceHost(arg0 context.Context, arg1 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetServiceHost", arg0, arg1) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetServiceHost indicates an expected call of GetServiceHost. +func (mr *MockServiceControllerMockRecorder) GetServiceHost(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetServiceHost", reflect.TypeOf((*MockServiceController)(nil).GetServiceHost), arg0, arg1) +} + +// GetServiceHosts mocks base method. +func (m *MockServiceController) GetServiceHosts(arg0 context.Context, arg1 string) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetServiceHosts", arg0, arg1) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetServiceHosts indicates an expected call of GetServiceHosts. +func (mr *MockServiceControllerMockRecorder) GetServiceHosts(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetServiceHosts", reflect.TypeOf((*MockServiceController)(nil).GetServiceHosts), arg0, arg1) +} + +// PunishDisk mocks base method. +func (m *MockServiceController) PunishDisk(arg0 context.Context, arg1 proto.DiskID, arg2 int) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "PunishDisk", arg0, arg1, arg2) +} + +// PunishDisk indicates an expected call of PunishDisk. +func (mr *MockServiceControllerMockRecorder) PunishDisk(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PunishDisk", reflect.TypeOf((*MockServiceController)(nil).PunishDisk), arg0, arg1, arg2) +} + +// PunishDiskWithThreshold mocks base method. +func (m *MockServiceController) PunishDiskWithThreshold(arg0 context.Context, arg1 proto.DiskID, arg2 int) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "PunishDiskWithThreshold", arg0, arg1, arg2) +} + +// PunishDiskWithThreshold indicates an expected call of PunishDiskWithThreshold. +func (mr *MockServiceControllerMockRecorder) PunishDiskWithThreshold(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PunishDiskWithThreshold", reflect.TypeOf((*MockServiceController)(nil).PunishDiskWithThreshold), arg0, arg1, arg2) +} + +// PunishService mocks base method. +func (m *MockServiceController) PunishService(arg0 context.Context, arg1, arg2 string, arg3 int) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "PunishService", arg0, arg1, arg2, arg3) +} + +// PunishService indicates an expected call of PunishService. +func (mr *MockServiceControllerMockRecorder) PunishService(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PunishService", reflect.TypeOf((*MockServiceController)(nil).PunishService), arg0, arg1, arg2, arg3) +} + +// PunishServiceWithThreshold mocks base method. +func (m *MockServiceController) PunishServiceWithThreshold(arg0 context.Context, arg1, arg2 string, arg3 int) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "PunishServiceWithThreshold", arg0, arg1, arg2, arg3) +} + +// PunishServiceWithThreshold indicates an expected call of PunishServiceWithThreshold. +func (mr *MockServiceControllerMockRecorder) PunishServiceWithThreshold(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PunishServiceWithThreshold", reflect.TypeOf((*MockServiceController)(nil).PunishServiceWithThreshold), arg0, arg1, arg2, arg3) +} + +// MockVolumeGetter is a mock of VolumeGetter interface. +type MockVolumeGetter struct { + ctrl *gomock.Controller + recorder *MockVolumeGetterMockRecorder +} + +// MockVolumeGetterMockRecorder is the mock recorder for MockVolumeGetter. +type MockVolumeGetterMockRecorder struct { + mock *MockVolumeGetter +} + +// NewMockVolumeGetter creates a new mock instance. +func NewMockVolumeGetter(ctrl *gomock.Controller) *MockVolumeGetter { + mock := &MockVolumeGetter{ctrl: ctrl} + mock.recorder = &MockVolumeGetterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockVolumeGetter) EXPECT() *MockVolumeGetterMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockVolumeGetter) Get(arg0 context.Context, arg1 proto.Vid, arg2 bool) *controller.VolumePhy { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2) + ret0, _ := ret[0].(*controller.VolumePhy) + return ret0 +} + +// Get indicates an expected call of Get. +func (mr *MockVolumeGetterMockRecorder) Get(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockVolumeGetter)(nil).Get), arg0, arg1, arg2) +} + +// Punish mocks base method. +func (m *MockVolumeGetter) Punish(arg0 context.Context, arg1 proto.Vid, arg2 int) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Punish", arg0, arg1, arg2) +} + +// Punish indicates an expected call of Punish. +func (mr *MockVolumeGetterMockRecorder) Punish(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Punish", reflect.TypeOf((*MockVolumeGetter)(nil).Punish), arg0, arg1, arg2) +} diff --git a/blobstore/access/limiter.go b/blobstore/access/limiter.go new file mode 100644 index 000000000..f0ef0df86 --- /dev/null +++ b/blobstore/access/limiter.go @@ -0,0 +1,238 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "context" + "io" + "time" + + "golang.org/x/time/rate" + + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/util/limit" + "github.com/cubefs/cubefs/blobstore/util/limit/count" +) + +const ( + _tagLimitedR = "limitedr" + _tagLimitedW = "limitedw" +) + +// Limiter rps and bps limiter +type Limiter interface { + // Acquire acquire with one request per second + Acquire(name string) error + // Release release of one request per second + Release(name string) + + // Reader return io.Reader with bandwidth rate limit + Reader(ctx context.Context, r io.Reader) io.Reader + // Writer return io.Writer with bandwidth rate limit + Writer(ctx context.Context, w io.Writer) io.Writer + + // Status returns running status + // TODO: calculate rate limit wait concurrent + Status() Status +} + +// LimitConfig configuration of limiter +type LimitConfig struct { + NameRps map[string]int `json:"name_rps"` // request with name n/s + ReaderMBps int `json:"reader_mbps"` // read with MB/s + WriterMBps int `json:"writer_mbps"` // write with MB/s +} + +// Status running status +type Status struct { + Config LimitConfig `json:"config"` // configuration status + Running map[string]int `json:"running"` // running request + ReadWait int `json:"read_wait"` // wait reading duration + WriteWait int `json:"write_wait"` // wait writing duration +} + +// Reader limited reader +type Reader struct { + ctx context.Context + rate *rate.Limiter + underlying io.Reader +} + +var _ io.Reader = &Reader{} + +func (r *Reader) Read(p []byte) (n int, err error) { + n, err = r.underlying.Read(p) + + now := time.Now() + reserve := r.rate.ReserveN(now, n) + + // Wait if necessary + delay := reserve.DelayFrom(now) + if delay == 0 { + return + } + t := time.NewTimer(delay) + defer t.Stop() + + span := trace.SpanFromContextSafe(r.ctx) + // for access PUT request is Read from client + span.SetTag(_tagLimitedW, delay.Milliseconds()) + + select { + case <-t.C: + // We can proceed. + return + case <-r.ctx.Done(): + // Context was canceled before we could proceed. Cancel the + // reservation, which may permit other events to proceed sooner. + reserve.Cancel() + err = r.ctx.Err() + return + } +} + +// Writer limited writer +type Writer struct { + ctx context.Context + rate *rate.Limiter + underlying io.Writer +} + +var _ io.Writer = &Writer{} + +func (w *Writer) Write(p []byte) (n int, err error) { + n, err = w.underlying.Write(p) + + now := time.Now() + reserve := w.rate.ReserveN(now, n) + + // Wait if necessary + delay := reserve.DelayFrom(now) + if delay == 0 { + return + } + t := time.NewTimer(delay) + defer t.Stop() + + span := trace.SpanFromContextSafe(w.ctx) + // for access GET request is Write to client + span.SetTag(_tagLimitedR, delay.Milliseconds()) + + select { + case <-t.C: + // We can proceed. + return + case <-w.ctx.Done(): + // Context was canceled before we could proceed. Cancel the + // reservation, which may permit other events to proceed sooner. + reserve.Cancel() + err = w.ctx.Err() + return + } +} + +type limiter struct { + config LimitConfig + limiters map[string]limit.Limiter + rateReader *rate.Limiter + rateWriter *rate.Limiter +} + +// NewLimiter returns a Limiter +func NewLimiter(cfg LimitConfig) Limiter { + mb := 1 << 20 + lim := &limiter{ + config: cfg, + limiters: make(map[string]limit.Limiter, len(cfg.NameRps)), + } + + for name, rps := range cfg.NameRps { + if rps > 0 { + lim.limiters[name] = count.New(rps) + } + } + + if cfg.ReaderMBps > 0 { + lim.rateReader = rate.NewLimiter(rate.Limit(cfg.ReaderMBps*mb), 2*cfg.ReaderMBps*mb) + } + if cfg.WriterMBps > 0 { + lim.rateWriter = rate.NewLimiter(rate.Limit(cfg.WriterMBps*mb), 2*cfg.WriterMBps*mb) + } + + return lim +} + +func (lim *limiter) Acquire(name string) error { + if l := lim.limiters[name]; l != nil { + return l.Acquire() + } + return nil +} + +func (lim *limiter) Release(name string) { + if l := lim.limiters[name]; l != nil { + l.Release() + } +} + +func (lim *limiter) Reader(ctx context.Context, r io.Reader) io.Reader { + if lim.rateReader != nil { + return &Reader{ + ctx: ctx, + rate: lim.rateReader, + underlying: r, + } + } + return r +} + +func (lim *limiter) Writer(ctx context.Context, w io.Writer) io.Writer { + if lim.rateWriter != nil { + return &Writer{ + ctx: ctx, + rate: lim.rateWriter, + underlying: w, + } + } + return w +} + +func (lim *limiter) Status() Status { + st := Status{ + Config: lim.config, + } + + st.Running = make(map[string]int, len(lim.limiters)) + for name, nl := range lim.limiters { + st.Running[name] = nl.Running() + } + + st.ReadWait = rateWait(lim.rateReader) + st.WriteWait = rateWait(lim.rateWriter) + + return st +} + +// rateWait get duration of waiting half of limit +func rateWait(r *rate.Limiter) int { + if r == nil { + return 0 + } + now := time.Now() + reserve := r.ReserveN(now, int(r.Limit())/2) + duration := reserve.DelayFrom(now) + reserve.Cancel() + return int(duration.Milliseconds()) +} diff --git a/blobstore/access/limiter_test.go b/blobstore/access/limiter_test.go new file mode 100644 index 000000000..d5cbb0f6f --- /dev/null +++ b/blobstore/access/limiter_test.go @@ -0,0 +1,302 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "context" + "io" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/time/rate" + + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/util/limit" +) + +type ( + limitReader struct { + size int + read int + } + limitWriter struct{} +) + +func (r *limitReader) Read(p []byte) (n int, err error) { + if r.read >= r.size { + return 0, io.EOF + } + + b := make([]byte, 1<<20) + n = copy(p, b) + r.read += n + return +} + +func (w *limitWriter) Write(p []byte) (n int, err error) { + b := make([]byte, 1<<20) + n = 0 + for len(p) > 0 { + nn := copy(p, b) + n += nn + p = p[nn:] + } + return +} + +func TestAccessLimitReader(t *testing.T) { + ctx, cancel := context.WithCancel(ctxWithName("TestAccessLimitReader")()) + r := &Reader{ + ctx: ctx, + rate: rate.NewLimiter(rate.Limit(1<<20), 1<<20), + underlying: &limitReader{size: 1 << 24}, + } + errCh := make(chan error) + go func() { + buf := make([]byte, 1<<24) + _, err := io.ReadFull(r, buf) + errCh <- err + }() + + time.Sleep(time.Second) + cancel() + + err := <-errCh + require.Error(t, err) +} + +func TestAccessLimitWriter(t *testing.T) { + ctx, cancel := context.WithCancel(ctxWithName("TestAccessLimitWriter")()) + w := &Writer{ + ctx: ctx, + rate: rate.NewLimiter(rate.Limit(1<<20), 1<<20), + underlying: &limitWriter{}, + } + errCh := make(chan error) + go func() { + _, err := w.Write(make([]byte, 1<<24)) + errCh <- err + }() + + time.Sleep(time.Second) + cancel() + + err := <-errCh + require.Error(t, err) +} + +func TestAccessLimiterBase(t *testing.T) { + nameGet := "get" + namePut := "put" + cfg := LimitConfig{ + NameRps: map[string]int{ + nameGet: 1, + namePut: 1, + }, + ReaderMBps: 1, + WriterMBps: 1, + } + l := NewLimiter(cfg) + + { + for range [100]struct{}{} { + err := l.Acquire("") + assert.NoError(t, err) + } + l.Release("") + } + { + err := l.Acquire(nameGet) + assert.NoError(t, err) + err = l.Acquire(nameGet) + assert.Equal(t, limit.ErrLimited, err) + l.Release(nameGet) + err = l.Acquire(nameGet) + assert.NoError(t, err) + l.Release(nameGet) + } + { + err := l.Acquire(nameGet) + assert.NoError(t, err) + err = l.Acquire(namePut) + assert.NoError(t, err) + + err = l.Acquire(nameGet) + assert.Equal(t, limit.ErrLimited, err) + err = l.Acquire(namePut) + assert.Equal(t, limit.ErrLimited, err) + + l.Release(nameGet) + l.Release(namePut) + } + + { + ctx := ctxWithName("TestAccessLimiterBase")() + rbuff := &limitReader{size: 1 << 8} + r := l.Reader(ctx, rbuff) + + wbuff := &limitWriter{} + w := l.Writer(ctx, wbuff) + buf := make([]byte, 1<<20) + var wg sync.WaitGroup + wg.Add(1) + go func() { + io.CopyBuffer(w, r, buf) + wg.Done() + }() + wg.Wait() + + span := trace.SpanFromContextSafe(ctx) + tags := span.Tags() + require.Empty(t, tags, span.TraceID(), tags) + } + { + ctx, cancel := context.WithCancel(ctxWithName("TestAccessLimiterBase")()) + rbuff := &limitReader{size: 1 << 30} + r := l.Reader(ctx, rbuff) + + wbuff := &limitWriter{} + w := l.Writer(ctx, wbuff) + + var wg sync.WaitGroup + wg.Add(1) + closeCh := make(chan struct{}) + go func() { + defer wg.Done() + + for { + select { + case <-closeCh: + return + default: + } + + _, err := io.CopyN(w, r, 1<<20) + if err != nil { + return + } + } + }() + + time.Sleep(time.Second) + cancel() + close(closeCh) + wg.Wait() + + span := trace.SpanFromContextSafe(ctx) + tags := span.Tags() + require.NotEmpty(t, tags, span.TraceID(), tags) + } +} + +func TestAccessLimiterNoop(t *testing.T) { + l := NewLimiter(LimitConfig{ + NameRps: nil, + ReaderMBps: 0, + WriterMBps: 0, + }) + + name := "noop" + err := l.Acquire(name) + assert.NoError(t, err) + err = l.Acquire(name) + assert.NoError(t, err) + l.Release(name) + l.Release(name) + + ctx := ctxWithName("TestAccessLimiterNoop")() + rbuff := &limitReader{size: 1 << 10} + r := l.Reader(ctx, rbuff) + + wbuff := &limitWriter{} + w := l.Writer(ctx, wbuff) + buf := make([]byte, 1<<5) + ch := make(chan struct{}) + go func() { + io.CopyBuffer(w, r, buf) + close(ch) + }() + + <-ch + span := trace.SpanFromContextSafe(ctx) + tags := span.Tags() + require.Empty(t, tags, span.TraceID(), tags) +} + +func TestAccessLimiterStatus(t *testing.T) { + { + l := NewLimiter(LimitConfig{ + NameRps: nil, + ReaderMBps: 0, + WriterMBps: 0, + }) + for range [100]struct{}{} { + l.Acquire("foo") + } + t.Logf("%+v\n", l.Status()) + } + { + ctx := ctxWithName("TestAccessLimiterStatus")() + name := "foo" + l := NewLimiter(LimitConfig{ + NameRps: map[string]int{name: 10}, + ReaderMBps: 2, + WriterMBps: 10, + }) + + ch := make(chan struct{}) + for range [7]struct{}{} { + go func() { + l.Acquire(name) + <-ch + l.Release(name) + }() + } + + var wg sync.WaitGroup + wg.Add(1) + go func() { + rbuff := &limitReader{size: 1 << 24} + r := l.Reader(ctx, rbuff) + io.CopyBuffer(&limitWriter{}, r, make([]byte, 1<<20)) + wg.Done() + }() + + w := l.Writer(ctx, &limitWriter{}) + wg.Add(8) + for range [8]struct{}{} { + go func() { + defer wg.Done() + for { + select { + case <-ch: + return + default: + } + w.Write(make([]byte, 1<<20)) + } + }() + } + + time.Sleep(time.Second * 3) + t.Logf("%+v\n", l.Status()) + + close(ch) + wg.Wait() + } +} diff --git a/blobstore/access/metric.go b/blobstore/access/metric.go new file mode 100644 index 000000000..04ee1a85d --- /dev/null +++ b/blobstore/access/metric.go @@ -0,0 +1,54 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "github.com/prometheus/client_golang/prometheus" + + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +var unhealthMetric = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "blobstore", + Subsystem: "access", + Name: "unhealth", + Help: "unhealth action on access", + }, + []string{"cluster", "action", "module", "host", "reason"}, +) + +var downloadMetric = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "blobstore", + Subsystem: "access", + Name: "download", + Help: "download way on access", + }, + []string{"cluster", "way", "reason"}, +) + +func init() { + prometheus.MustRegister(unhealthMetric) + prometheus.MustRegister(downloadMetric) +} + +func reportUnhealth(cid proto.ClusterID, action, module, host, reason string) { + unhealthMetric.WithLabelValues(cid.ToString(), action, module, host, reason).Inc() +} + +func reportDownload(cid proto.ClusterID, way, reason string) { + downloadMetric.WithLabelValues(cid.ToString(), way, reason).Inc() +} diff --git a/blobstore/access/server.go b/blobstore/access/server.go new file mode 100644 index 000000000..a770d45e7 --- /dev/null +++ b/blobstore/access/server.go @@ -0,0 +1,681 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "crypto/sha1" + "fmt" + "net/http" + "strconv" + "sync" + "time" + + consulapi "github.com/hashicorp/consul/api" + + "github.com/cubefs/cubefs/blobstore/access/controller" + "github.com/cubefs/cubefs/blobstore/api/access" + "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/cmd" + "github.com/cubefs/cubefs/blobstore/common/consul" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/profile" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/resourcepool" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/common/uptoken" + "github.com/cubefs/cubefs/blobstore/util/closer" + "github.com/cubefs/cubefs/blobstore/util/errors" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +const ( + limitNameAlloc = "alloc" + limitNamePut = "put" + limitNamePutAt = "putat" + limitNameGet = "get" + limitNameDelete = "delete" + limitNameSign = "sign" +) + +const ( + _tokenExpiration = time.Hour * 12 +) + +var ( + // tokenSecretKeys alloc token with the first secret key always, + // so that you can change the secret key. + // + // parse-1: insert a new key at the first index, + // parse-2: delete the old key at the last index after _tokenExpiration duration. + tokenSecretKeys = [...][20]byte{ + {0x5f, 0x00, 0x88, 0x96, 0x00, 0xa1, 0xfe, 0x1b}, + {0xff, 0x1f, 0x2f, 0x4f, 0x7f, 0xaf, 0xef, 0xff}, + } + _initTokenSecret sync.Once +) + +func initTokenSecret(b []byte) { + _initTokenSecret.Do(func() { + for idx := range tokenSecretKeys { + copy(tokenSecretKeys[idx][7:], b) + } + }) +} + +func initWithRegionMagic(regionMagic string) { + if regionMagic == "" { + log.Warn("no region magic setting, using default secret keys for checksum") + return + } + + log.Info("using magic secret keys for checksum with:", regionMagic) + b := sha1.Sum([]byte(regionMagic)) + initTokenSecret(b[:8]) + initLocationSecret(b[:8]) +} + +type accessStatus struct { + Limit Status `json:"limit"` + Pool resourcepool.Status `json:"pool"` + + Config StreamConfig `json:"config"` + Clusters []*clustermgr.ClusterInfo `json:"clusters"` + Services map[proto.ClusterID]map[string][]string `json:"services"` +} + +// Config service configs +type Config struct { + cmd.Config + ConsulAgentAddr string `json:"consul_agent_addr"` + ServiceRegister consul.Config `json:"service_register"` + Stream StreamConfig `json:"stream"` + Limit LimitConfig `json:"limit"` +} + +// Service rpc service +type Service struct { + config Config + streamHandler StreamHandler + limiter Limiter + closer closer.Closer +} + +// New returns an access service +func New(cfg Config) *Service { + consulConf := consulapi.DefaultConfig() + consulConf.Address = cfg.ConsulAgentAddr + + client, err := consulapi.NewClient(consulConf) + if err != nil { + log.Fatalf("new consul client failed, err: %v", err) + } + + // add region magic checksum to the secret keys + initWithRegionMagic(cfg.Stream.ClusterConfig.RegionMagic) + + cl := closer.New() + return &Service{ + config: cfg, + streamHandler: NewStreamHandler(&cfg.Stream, client, cl.Done()), + limiter: NewLimiter(cfg.Limit), + closer: cl, + } +} + +// Close close server +func (s *Service) Close() { + s.closer.Close() +} + +// RegisterService register service to rpc +func (s *Service) RegisterService() { + _, err := consul.ServiceRegister(s.config.BindAddr, &s.config.ServiceRegister) + if err != nil { + log.Fatalf("service register failed, err: %v", err) + } +} + +// RegisterAdminHandler register admin handler to profile +func (s *Service) RegisterAdminHandler() { + profile.HandleFunc(http.MethodGet, "/access/status", func(c *rpc.Context) { + var admin *streamAdmin + if sa := s.streamHandler.Admin(); sa != nil { + if ad, ok := sa.(*streamAdmin); ok { + admin = ad + } + } + if admin == nil { + c.RespondStatus(http.StatusServiceUnavailable) + return + } + + ctx := c.Request.Context() + span := trace.SpanFromContextSafe(ctx) + + status := new(accessStatus) + status.Limit = s.limiter.Status() + status.Pool = admin.memPool.Status() + status.Config = admin.config + status.Clusters = admin.controller.All() + status.Services = make(map[proto.ClusterID]map[string][]string, len(status.Clusters)) + + for _, cluster := range status.Clusters { + service, err := admin.controller.GetServiceController(cluster.ClusterID) + if err != nil { + span.Warn(err.Error()) + continue + } + + svrs := make(map[string][]string, 1) + svrName := proto.ServiceNameProxy + if hosts, err := service.GetServiceHosts(ctx, svrName); err == nil { + svrs[svrName] = hosts + } else { + span.Warn(err.Error()) + } + status.Services[cluster.ClusterID] = svrs + } + c.RespondJSON(status) + }) + + profile.HandleFunc(http.MethodPost, "/access/stream/controller/alg/:alg", func(c *rpc.Context) { + algInt, err := strconv.Atoi(c.Param.ByName("alg")) + if err != nil { + c.RespondWith(http.StatusBadRequest, "", []byte(err.Error())) + return + } + if algInt < 0 { + c.RespondWith(http.StatusBadRequest, "", []byte("invalid algorithm")) + return + } + + alg := controller.AlgChoose(algInt) + if sa := s.streamHandler.Admin(); sa != nil { + if admin, ok := sa.(*streamAdmin); ok { + if err := admin.controller.ChangeChooseAlg(alg); err != nil { + c.RespondWith(http.StatusForbidden, "", []byte(err.Error())) + return + } + + span := trace.SpanFromContextSafe(c.Request.Context()) + span.Warnf("change cluster choose algorithm to (%d %s)", alg, alg.String()) + c.Respond() + return + } + } + + c.RespondStatus(http.StatusServiceUnavailable) + }, rpc.OptArgsURI()) +} + +// Limit rps controller +func (s *Service) Limit(c *rpc.Context) { + name := "" + switch c.Request.URL.Path { + case "/alloc": + name = limitNameAlloc + case "/put": + name = limitNamePut + case "/putat": + name = limitNamePutAt + case "/get": + name = limitNameGet + case "/delete": + name = limitNameDelete + case "/sign": + name = limitNameSign + } + if name == "" { + return + } + + if err := s.limiter.Acquire(name); err != nil { + span := trace.SpanFromContextSafe(c.Request.Context()) + span.Info("access concurrent limited", name, err) + c.AbortWithError(errcode.ErrAccessLimited) + return + } + defer s.limiter.Release(name) + c.Next() +} + +// Put one object +func (s *Service) Put(c *rpc.Context) { + args := new(access.PutArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + ctx := c.Request.Context() + span := trace.SpanFromContextSafe(ctx) + + span.Debugf("accept /put request args:%+v", args) + if !args.IsValid() { + span.Debugf("invalid args:%+v", args) + c.RespondError(errcode.ErrIllegalArguments) + return + } + + hashSumMap := args.Hashes.ToHashSumMap() + hasherMap := make(access.HasherMap, len(hashSumMap)) + // make hashser + for alg := range hashSumMap { + hasherMap[alg] = alg.ToHasher() + } + + rc := s.limiter.Reader(ctx, c.Request.Body) + loc, err := s.streamHandler.Put(ctx, rc, args.Size, hasherMap) + if err != nil { + span.Error("stream put failed", errors.Detail(err)) + c.RespondError(httpError(err)) + return + } + + // hasher sum + for alg, hasher := range hasherMap { + hashSumMap[alg] = hasher.Sum(nil) + } + + if err := fillCrc(loc); err != nil { + span.Error("stream put fill location crc", err) + c.RespondError(httpError(err)) + return + } + + c.RespondJSON(access.PutResp{ + Location: *loc, + HashSumMap: hashSumMap, + }) + span.Infof("done /put request location:%+v hash:%+v", loc, hashSumMap.All()) +} + +// PutAt put one blob +func (s *Service) PutAt(c *rpc.Context) { + args := new(access.PutAtArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + ctx := c.Request.Context() + span := trace.SpanFromContextSafe(ctx) + + span.Debugf("accept /putat request args:%+v", args) + if !args.IsValid() { + span.Debugf("invalid args:%+v", args) + c.RespondError(errcode.ErrIllegalArguments) + return + } + + valid := false + for _, secretKey := range tokenSecretKeys { + token := uptoken.DecodeToken(args.Token) + if token.IsValid(args.ClusterID, args.Vid, args.Blobid, uint32(args.Size), secretKey[:]) { + valid = true + break + } + } + if !valid { + span.Debugf("invalid token:%s", args.Token) + c.RespondError(errcode.ErrIllegalArguments) + return + } + + hashSumMap := args.Hashes.ToHashSumMap() + hasherMap := make(access.HasherMap, len(hashSumMap)) + // make hashser + for alg := range hashSumMap { + hasherMap[alg] = alg.ToHasher() + } + + rc := s.limiter.Reader(ctx, c.Request.Body) + err := s.streamHandler.PutAt(ctx, rc, args.ClusterID, args.Vid, args.Blobid, args.Size, hasherMap) + if err != nil { + span.Error("stream putat failed", errors.Detail(err)) + c.RespondError(httpError(err)) + return + } + + // hasher sum + for alg, hasher := range hasherMap { + hashSumMap[alg] = hasher.Sum(nil) + } + + c.RespondJSON(access.PutAtResp{HashSumMap: hashSumMap}) + span.Infof("done /putat request hash:%+v", hashSumMap.All()) +} + +// Alloc alloc one location +func (s *Service) Alloc(c *rpc.Context) { + args := new(access.AllocArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + ctx := c.Request.Context() + span := trace.SpanFromContextSafe(ctx) + + span.Debugf("accept /alloc request args:%+v", args) + if !args.IsValid() { + span.Debugf("invalid args:%+v", args) + c.RespondError(errcode.ErrIllegalArguments) + return + } + + location, err := s.streamHandler.Alloc(ctx, args.Size, args.BlobSize, args.AssignClusterID, args.CodeMode) + if err != nil { + span.Error("stream alloc failed", errors.Detail(err)) + c.RespondError(httpError(err)) + return + } + + if err := fillCrc(location); err != nil { + span.Error("stream alloc fill location crc", err) + c.RespondError(httpError(err)) + return + } + + resp := access.AllocResp{ + Location: *location, + Tokens: genTokens(location), + } + c.RespondJSON(resp) + span.Infof("done /alloc request resp:%+v", resp) +} + +// Get read file +func (s *Service) Get(c *rpc.Context) { + args := new(access.GetArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + ctx := c.Request.Context() + span := trace.SpanFromContextSafe(ctx) + + span.Debugf("accept /get request args:%+v", args) + if !args.IsValid() || !verifyCrc(&args.Location) { + span.Debugf("invalid args:%+v", args) + c.RespondError(errcode.ErrIllegalArguments) + return + } + + w := c.Writer + writer := s.limiter.Writer(ctx, w) + transfer, err := s.streamHandler.Get(ctx, writer, args.Location, args.ReadSize, args.Offset) + if err != nil { + span.Error("stream get prepare failed", errors.Detail(err)) + c.RespondError(httpError(err)) + return + } + + w.Header().Set(rpc.HeaderContentType, rpc.MIMEStream) + w.Header().Set(rpc.HeaderContentLength, strconv.FormatInt(int64(args.ReadSize), 10)) + if args.ReadSize > 0 && args.ReadSize != args.Location.Size { + w.Header().Set(rpc.HeaderContentRange, fmt.Sprintf("bytes %d-%d/%d", + args.Offset, args.Offset+args.ReadSize-1, args.Location.Size)) + c.RespondStatus(http.StatusPartialContent) + } else { + c.RespondStatus(http.StatusOK) + } + + // flush headers to client firstly + c.Flush() + + err = transfer() + if err != nil { + span.Error("stream get transfer failed", errors.Detail(err)) + return + } + span.Info("done /get request") +} + +// Delete all blobs in this location +func (s *Service) Delete(c *rpc.Context) { + args := new(access.DeleteArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + ctx := c.Request.Context() + span := trace.SpanFromContextSafe(ctx) + + var err error + var resp access.DeleteResp + defer func() { + if err != nil { + c.RespondError(httpError(err)) + return + } + + if len(resp.FailedLocations) > 0 { + span.Errorf("failed locations N %d of %d", len(resp.FailedLocations), len(args.Locations)) + // must return 2xx even if has failed locations, + // cos rpc read body only on 2xx. + // TODO: return other http status code + c.RespondStatusData(http.StatusIMUsed, resp) + return + } + + c.RespondJSON(resp) + }() + + if !args.IsValid() { + err = errcode.ErrIllegalArguments + return + } + span.Debugf("accept /delete request args: locations %d", len(args.Locations)) + defer span.Info("done /delete request") + + clusterBlobsN := make(map[proto.ClusterID]int, 4) + for _, loc := range args.Locations { + if !verifyCrc(&loc) { + span.Infof("invalid crc %+v", loc) + err = errcode.ErrIllegalArguments + return + } + clusterBlobsN[loc.ClusterID] += len(loc.Blobs) + } + + if len(args.Locations) == 1 { + loc := args.Locations[0] + if err := s.streamHandler.Delete(ctx, &loc); err != nil { + span.Error("stream delete failed", errors.Detail(err)) + resp.FailedLocations = []access.Location{loc} + } + return + } + + // merge the same cluster locations to one delete message, + // anyone of this cluster failed, all locations mark failure, + // + // a min delete message about 10-20 bytes, + // max delete locations is 1024, one location is max to 5G, + // merged message max size about 40MB. + + merged := make(map[proto.ClusterID][]access.SliceInfo, len(clusterBlobsN)) + for id, n := range clusterBlobsN { + merged[id] = make([]access.SliceInfo, 0, n) + } + for _, loc := range args.Locations { + merged[loc.ClusterID] = append(merged[loc.ClusterID], loc.Blobs...) + } + + var wg sync.WaitGroup + failedCh := make(chan proto.ClusterID, 1) + done := make(chan struct{}) + go func() { + for id := range failedCh { + if resp.FailedLocations == nil { + resp.FailedLocations = make([]access.Location, 0, len(args.Locations)) + } + for _, loc := range args.Locations { + if loc.ClusterID == id { + resp.FailedLocations = append(resp.FailedLocations, loc) + } + } + } + close(done) + }() + + wg.Add(len(merged)) + for id := range merged { + go func(id proto.ClusterID) { + if err := s.streamHandler.Delete(ctx, &access.Location{ + ClusterID: id, + BlobSize: 1, + Blobs: merged[id], + }); err != nil { + span.Error("stream delete failed", id, errors.Detail(err)) + failedCh <- id + } + wg.Done() + }(id) + } + + wg.Wait() + close(failedCh) + <-done +} + +// DeleteBlob delete one blob +func (s *Service) DeleteBlob(c *rpc.Context) { + args := new(access.DeleteBlobArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + ctx := c.Request.Context() + span := trace.SpanFromContextSafe(ctx) + + span.Debugf("accept /deleteblob request args:%+v", args) + if !args.IsValid() { + span.Debugf("invalid args:%+v", args) + c.RespondError(errcode.ErrIllegalArguments) + return + } + + valid := false + for _, secretKey := range tokenSecretKeys { + token := uptoken.DecodeToken(args.Token) + if token.IsValid(args.ClusterID, args.Vid, args.Blobid, uint32(args.Size), secretKey[:]) { + valid = true + break + } + } + if !valid { + span.Debugf("invalid token:%s", args.Token) + c.RespondError(errcode.ErrIllegalArguments) + return + } + + if err := s.streamHandler.Delete(ctx, &access.Location{ + ClusterID: args.ClusterID, + BlobSize: 1, + Blobs: []access.SliceInfo{{ + MinBid: args.Blobid, + Vid: args.Vid, + Count: 1, + }}, + }); err != nil { + span.Error("stream delete blob failed", errors.Detail(err)) + c.RespondError(httpError(err)) + return + } + + c.Respond() + span.Info("done /deleteblob request") +} + +// Sign generate crc with locations +func (s *Service) Sign(c *rpc.Context) { + args := new(access.SignArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + ctx := c.Request.Context() + span := trace.SpanFromContextSafe(ctx) + + if !args.IsValid() { + c.RespondError(errcode.ErrIllegalArguments) + return + } + span.Debugf("accept /sign request args: %+v", args) + + loc := args.Location + crcOld := loc.Crc + if err := signCrc(&loc, args.Locations); err != nil { + span.Error("stream sign failed", errors.Detail(err)) + c.RespondError(errcode.ErrIllegalArguments) + return + } + + c.RespondJSON(access.SignResp{Location: loc}) + span.Infof("done /sign request crc %d -> %d, resp:%+v", crcOld, loc.Crc, loc) +} + +func httpError(err error) error { + if e, ok := err.(rpc.HTTPError); ok { + return e + } + if e, ok := err.(*errors.Error); ok { + return rpc.NewError(http.StatusInternalServerError, "ServerError", e.Cause()) + } + return errcode.ErrUnexpected +} + +// genTokens generate tokens +// 1. Returns 0 token if has no blobs. +// 2. Returns 1 token if file size less than blobsize. +// 3. Returns len(blobs) tokens if size divided by blobsize. +// 4. Otherwise returns len(blobs)+1 tokens, the last token +// will be used by the last blob, even if the last slice blobs' size +// less than blobsize. +// 5. Each segment blob has its specified token include the last blob. +func genTokens(location *access.Location) []string { + tokens := make([]string, 0, len(location.Blobs)+1) + + hasMultiBlobs := location.Size >= uint64(location.BlobSize) + lastSize := uint32(location.Size % uint64(location.BlobSize)) + for idx, blob := range location.Blobs { + // returns one token if size < blobsize + if hasMultiBlobs { + count := blob.Count + if idx == len(location.Blobs)-1 && lastSize > 0 { + count-- + } + tokens = append(tokens, uptoken.EncodeToken(uptoken.NewUploadToken(location.ClusterID, + blob.Vid, blob.MinBid, count, + location.BlobSize, _tokenExpiration, tokenSecretKeys[0][:]))) + } + + // token of the last blob + if idx == len(location.Blobs)-1 && lastSize > 0 { + tokens = append(tokens, uptoken.EncodeToken(uptoken.NewUploadToken(location.ClusterID, + blob.Vid, blob.MinBid+proto.BlobID(blob.Count)-1, 1, + lastSize, _tokenExpiration, tokenSecretKeys[0][:]))) + } + } + + return tokens +} diff --git a/blobstore/access/server_location.go b/blobstore/access/server_location.go new file mode 100644 index 000000000..a2f683793 --- /dev/null +++ b/blobstore/access/server_location.go @@ -0,0 +1,127 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "fmt" + "hash/crc32" + "sync" + + "github.com/cubefs/cubefs/blobstore/api/access" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/util/bytespool" +) + +const ( + // DO NOT CHANGE IT. + _crcPoly = uint32(0x59c8943c) +) + +var ( + // DO NOT CHANGE IT. + _crcTable = crc32.MakeTable(_crcPoly) + _crcMagicKey = [20]byte{ + 0x52, 0xe, 0x53, 0x53, 0x81, + 0x1f, 0x51, 0xb7, 0xa4, 0x72, + 0x10, 0x33, 0x64, 0xa7, 0x3a, + 0x10, 0x19, 0xbc, 0x60, 0x7, + } + _initLocationSecret sync.Once +) + +func initLocationSecret(b []byte) { + _initLocationSecret.Do(func() { + copy(_crcMagicKey[7:], b) + }) +} + +func calcCrc(loc *access.Location) (uint32, error) { + crcWriter := crc32.New(_crcTable) + + buf := bytespool.Alloc(1024) + defer bytespool.Free(buf) + + n := loc.Encode2(buf) + if n < 4 { + return 0, fmt.Errorf("no enough bytes(%d) fill into buf", n) + } + + if _, err := crcWriter.Write(_crcMagicKey[:]); err != nil { + return 0, fmt.Errorf("fill crc %s", err.Error()) + } + if _, err := crcWriter.Write(buf[4:n]); err != nil { + return 0, fmt.Errorf("fill crc %s", err.Error()) + } + + return crcWriter.Sum32(), nil +} + +func fillCrc(loc *access.Location) error { + crc, err := calcCrc(loc) + if err != nil { + return err + } + loc.Crc = crc + return nil +} + +func verifyCrc(loc *access.Location) bool { + crc, err := calcCrc(loc) + if err != nil { + return false + } + return loc.Crc == crc +} + +func signCrc(loc *access.Location, locs []access.Location) error { + first := locs[0] + bids := make(map[proto.BlobID]struct{}, 64) + + if loc.ClusterID != first.ClusterID || + loc.CodeMode != first.CodeMode || + loc.BlobSize != first.BlobSize { + return fmt.Errorf("not equal in constant field") + } + + for _, l := range locs { + if !verifyCrc(&l) { + return fmt.Errorf("not equal in crc %d", l.Crc) + } + + // assert + if l.ClusterID != first.ClusterID || + l.CodeMode != first.CodeMode || + l.BlobSize != first.BlobSize { + return fmt.Errorf("not equal in constant field") + } + + for _, blob := range l.Blobs { + for c := 0; c < int(blob.Count); c++ { + bids[blob.MinBid+proto.BlobID(c)] = struct{}{} + } + } + } + + for _, blob := range loc.Blobs { + for c := 0; c < int(blob.Count); c++ { + bid := blob.MinBid + proto.BlobID(c) + if _, ok := bids[bid]; !ok { + return fmt.Errorf("not equal in blob_id(%d)", bid) + } + } + } + + return fillCrc(loc) +} diff --git a/blobstore/access/server_location_test.go b/blobstore/access/server_location_test.go new file mode 100644 index 000000000..c2e5e4b9f --- /dev/null +++ b/blobstore/access/server_location_test.go @@ -0,0 +1,203 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "fmt" + "hash/crc32" + "math" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/api/access" + "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/util/bytespool" +) + +var ( + testMaxBlob = access.SliceInfo{ + MinBid: proto.BlobID(math.MaxUint64), + Vid: proto.Vid(math.MaxInt32), + Count: math.MaxUint32, + } + testMaxLoc = access.Location{ + ClusterID: proto.ClusterID(math.MaxUint32), + CodeMode: codemode.CodeMode(math.MaxInt8), + Size: math.MaxUint64, + BlobSize: math.MaxUint32, + Crc: math.MaxUint32, + } + + testMinBlob = access.SliceInfo{} + testMinLoc = access.Location{} +) + +func TestAccessServiceLocationCrc(t *testing.T) { + { + _, err := calcCrc(nil) + require.Error(t, err) + } + { + crc, err := calcCrc(&testMinLoc) + require.NoError(t, err) + require.Equal(t, uint32(0x8a7370cc), crc) + } + { + crc, err := calcCrc(&testMaxLoc) + require.NoError(t, err) + require.Equal(t, uint32(0xda55150a), crc) + } + { + loc := testMinLoc.Copy() + loc.Size = 1 << 30 + + err := fillCrc(&loc) + require.NoError(t, err) + require.Equal(t, uint32(0x9e17bc9e), loc.Crc) + } + { + loc := testMinLoc.Copy() + loc.Size = 1 << 30 + require.False(t, verifyCrc(&loc)) + + loc.Crc = 0x9e17bc9e + require.True(t, verifyCrc(&loc)) + } +} + +func TestAccessServiceLocationSecret(t *testing.T) { + secret := make([]byte, len(_crcMagicKey)) + copy(secret, _crcMagicKey[:]) + defer func() { + copy(_crcMagicKey[:], secret) + }() + { + initLocationSecret([]byte{0x34, 0x45, 0x18, 0x4f}) + crc, err := calcCrc(&testMinLoc) + require.NoError(t, err) + require.NotEqual(t, uint32(0x8a7370cc), crc) + require.Equal(t, uint32(0xdbe8df90), crc) + } + { + // init once + initLocationSecret([]byte{0x1, 0x2, 0x3, 0x4}) + crc, err := calcCrc(&testMinLoc) + require.NoError(t, err) + require.Equal(t, uint32(0xdbe8df90), crc) + } +} + +func TestAccessServiceLocationSignCrc(t *testing.T) { + loc := &access.Location{ + ClusterID: 1, + CodeMode: 1, + Size: 1023, + BlobSize: 6, + Crc: 0, + Blobs: []access.SliceInfo{{ + MinBid: 11, + Vid: 199, + Count: 10, + }}, + } + fillCrc(loc) + require.True(t, verifyCrc(loc)) + + { + loc1, loc2 := loc.Copy(), loc.Copy() + require.NoError(t, signCrc(loc, []access.Location{loc1, loc2})) + } + { + loc1, loc2 := loc.Copy(), loc.Copy() + loc1.BlobSize = 100 + fillCrc(&loc1) + require.Error(t, signCrc(loc, []access.Location{loc1, loc2})) + } + { + loc1, loc2 := loc.Copy(), loc.Copy() + loc2.Crc = 0 + require.Error(t, signCrc(loc, []access.Location{loc1, loc2})) + } + { + loc1, loc2 := loc.Copy(), loc.Copy() + loc2.ClusterID = 2 + fillCrc(&loc2) + require.Error(t, signCrc(loc, []access.Location{loc1, loc2})) + } + { + loc1, loc2 := loc.Copy(), loc.Copy() + loc2.CodeMode = 100 + fillCrc(&loc2) + require.Error(t, signCrc(loc, []access.Location{loc1, loc2})) + } + { + loc1, loc2 := loc.Copy(), loc.Copy() + loc1.Blobs = nil + loc2.Blobs[0].Count = 5 + fillCrc(&loc1) + fillCrc(&loc2) + require.Error(t, signCrc(loc, []access.Location{loc1, loc2})) + } +} + +func calcCrcWithoutMagic(loc *access.Location) (uint32, error) { + crcWriter := crc32.New(_crcTable) + + buf := bytespool.Alloc(1024) + defer bytespool.Free(buf) + + n := loc.Encode2(buf) + crcWriter.Write(buf[4:n]) + + return crcWriter.Sum32(), nil +} + +func benchmarkCrc(b *testing.B, key string, + location access.Location, blob access.SliceInfo, + run func(loc *access.Location) (uint32, error)) { + cases := []int{0, 2, 4, 8, 16, 32} + for _, l := range cases { + b.ResetTimer() + b.Run(fmt.Sprintf(key+"-%d", l), func(b *testing.B) { + loc := location.Copy() + loc.Blobs = make([]access.SliceInfo, l) + for idx := range loc.Blobs { + loc.Blobs[idx] = blob + } + b.ResetTimer() + for ii := 0; ii <= b.N; ii++ { + run(&loc) + } + }) + } +} + +func BenchmarkAccessServerCrcWithMagicMin(b *testing.B) { + benchmarkCrc(b, "min-with-magic", testMinLoc, testMinBlob, calcCrc) +} + +func BenchmarkAccessServerCrcWithoutMagicMin(b *testing.B) { + benchmarkCrc(b, "min-without-magic", testMinLoc, testMinBlob, calcCrcWithoutMagic) +} + +func BenchmarkAccessServerCrcWithMagicMax(b *testing.B) { + benchmarkCrc(b, "max-with-magic", testMaxLoc, testMaxBlob, calcCrc) +} + +func BenchmarkAccessServerCrcWithoutMagicMax(b *testing.B) { + benchmarkCrc(b, "max-without-magic", testMaxLoc, testMaxBlob, calcCrcWithoutMagic) +} diff --git a/blobstore/access/server_test.go b/blobstore/access/server_test.go new file mode 100644 index 000000000..dbd1b3722 --- /dev/null +++ b/blobstore/access/server_test.go @@ -0,0 +1,706 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strconv" + "sync" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/api/access" + "github.com/cubefs/cubefs/blobstore/common/codemode" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/uptoken" +) + +var ( + ctx = context.Background() + _blobSize uint32 = 1 << 20 + location = &access.Location{ + ClusterID: 1, + CodeMode: 1, + BlobSize: _blobSize, + Crc: 0, + Blobs: []access.SliceInfo{{ + MinBid: 111, + Vid: 1111, + Count: 11, + }}, + } + + testServer *httptest.Server + once sync.Once +) + +func runMockService(s *Service) string { + once.Do(func() { + testServer = httptest.NewServer(NewHandler(s)) + }) + return testServer.URL +} + +func newService() *Service { + ctr := gomock.NewController(&testing.T{}) + s := NewMockStreamHandler(ctr) + + s.EXPECT().Alloc(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( + func(ctx context.Context, size uint64, blobSize uint32, + assignClusterID proto.ClusterID, codeMode codemode.CodeMode) (*access.Location, error) { + if size < 1024 { + return nil, errors.New("fake alloc location") + } + loc := location.Copy() + loc.Size = uint64(size) + fillCrc(&loc) + return &loc, nil + }) + + s.EXPECT().PutAt(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), + gomock.Any()).AnyTimes().DoAndReturn( + func(ctx context.Context, rc io.Reader, + clusterID proto.ClusterID, vid proto.Vid, bid proto.BlobID, size int64, + hasherMap access.HasherMap) error { + if size < 1024 { + return errcode.ErrAccessLimited + } + return nil + }) + + s.EXPECT().Put(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( + func(ctx context.Context, rc io.Reader, size int64, hasherMap access.HasherMap) (*access.Location, error) { + if size < 1024 { + return nil, errors.New("fake put nil body") + } + loc := location.Copy() + loc.Size = uint64(size) + fillCrc(&loc) + return &loc, nil + }) + + s.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( + func(ctx context.Context, w io.Writer, location access.Location, readSize, offset uint64) (func() error, error) { + if readSize < 1024 { + return nil, errors.New("fake get nil body") + } + return func() error { return nil }, nil + }) + s.EXPECT().Delete(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( + func(ctx context.Context, location *access.Location) error { + if location.ClusterID >= 10 { + return errors.New("fake delete error with cluster") + } else if location.ClusterID == 1 && location.Crc > 0 && location.Size < 1024 { + return errors.New("fake delete error") + } + return nil + }) + + return &Service{ + streamHandler: s, + limiter: NewLimiter(LimitConfig{ + NameRps: map[string]int{ + limitNameAlloc: 2, + }, + ReaderMBps: 0, + WriterMBps: 0, + }), + } +} + +func newClient() rpc.Client { + return rpc.NewClient(&rpc.Config{}) +} + +func TestAccessServiceNew(t *testing.T) { + runMockService(newService()) +} + +func TestAccessServiceAlloc(t *testing.T) { + host := runMockService(newService()) + cli := newClient() + + url := func() string { + return fmt.Sprintf("%s/alloc", host) + } + args := access.AllocArgs{ + Size: 0, + BlobSize: 0, + AssignClusterID: 0, + CodeMode: 0, + } + { + resp := &access.AllocResp{} + err := cli.PostWith(ctx, url(), resp, args) + assertErrorCode(t, 400, err) + } + { + args.Size = 1023 + resp := &access.AllocResp{} + err := cli.PostWith(ctx, url(), resp, args) + assertErrorCode(t, 500, err) + } + { + args.Size = 1024 + resp := &access.AllocResp{} + err := cli.PostWith(ctx, url(), resp, args) + require.NoError(t, err) + require.Equal(t, uint64(1024), resp.Location.Size) + } + { + args.Size = uint64(_blobSize) + resp := &access.AllocResp{} + err := cli.PostWith(ctx, url(), resp, args) + require.NoError(t, err) + require.Equal(t, uint64(_blobSize), resp.Location.Size) + } + { + args.Size = uint64(_blobSize) + 1 + resp := &access.AllocResp{} + err := cli.PostWith(ctx, url(), resp, args) + require.NoError(t, err) + require.Equal(t, uint64(_blobSize)+1, resp.Location.Size) + } +} + +func TestAccessServicePutAt(t *testing.T) { + host := runMockService(newService()) + cli := newClient() + + url := func(size int64, token string) string { + return fmt.Sprintf("%s/putat?clusterid=1&volumeid=1111&blobid=111&size=%d&hashes=14&token=%s", + host, size, token) + } + + for _, method := range []string{http.MethodPut, http.MethodPost} { + args := access.PutArgs{ + Size: 0, + } + { + buf := make([]byte, args.Size) + req, _ := http.NewRequest(method, url(args.Size, ""), bytes.NewReader(buf)) + resp := &access.PutAtResp{} + err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode()) + assertErrorCode(t, 400, err) + } + { + args.Size = 1023 + buf := make([]byte, args.Size) + req, _ := http.NewRequest(method, url(args.Size, ""), bytes.NewReader(buf)) + resp := &access.PutAtResp{} + err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode()) + assertErrorCode(t, 400, err) + } + { + args.Size = 1023 + buf := make([]byte, args.Size) + resp := &access.PutAtResp{} + req, _ := http.NewRequest(method, url(args.Size, "c1fdcecaacbfafd86f0b00"), bytes.NewReader(buf)) + err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode()) + assertErrorCode(t, 552, err) + } + { + args.Size = 1024 + buf := make([]byte, args.Size) + req, _ := http.NewRequest(method, url(args.Size, "8238436d05ecf2366f0b00"), bytes.NewReader(buf)) + resp := &access.PutAtResp{} + err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode()) + require.NoError(t, err) + + req, _ = http.NewRequest(method, url(args.Size, "1238436d05ecf2366f0b00"), bytes.NewReader(buf)) + err = cli.DoWith(ctx, req, resp, rpc.WithCrcEncode()) + assertErrorCode(t, 400, err) + } + } +} + +func TestAccessServicePut(t *testing.T) { + host := runMockService(newService()) + cli := newClient() + + url := func(size int64, hashes access.HashAlgorithm) string { + return fmt.Sprintf("%s/put?size=%d&hashes=%d", host, size, hashes) + } + + for _, method := range []string{http.MethodPut, http.MethodPost} { + args := access.PutArgs{ + Size: 0, + Hashes: 14, + Body: nil, + } + { + req, _ := http.NewRequest(method, fmt.Sprintf("%s/put?size=size", host), args.Body) + resp := &access.PutResp{} + err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode()) + assertErrorCode(t, 400, err) + } + { + req, _ := http.NewRequest(method, url(args.Size, args.Hashes), args.Body) + resp := &access.PutResp{} + err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode()) + assertErrorCode(t, 400, err) + } + { + args.Body = bytes.NewReader(make([]byte, 1023)) + req, _ := http.NewRequest(method, url(1023, args.Hashes), args.Body) + resp := &access.PutResp{} + err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode()) + assertErrorCode(t, 500, err) + } + { + args.Body = bytes.NewReader(make([]byte, 1024)) + req, _ := http.NewRequest(method, url(1024, args.Hashes), args.Body) + resp := &access.PutResp{} + err := cli.DoWith(ctx, req, resp, rpc.WithCrcEncode()) + require.NoError(t, err) + require.Equal(t, uint64(1024), resp.Location.Size) + } + } +} + +func TestAccessServiceGet(t *testing.T) { + host := runMockService(newService()) + cli := newClient() + + url := func() string { + return fmt.Sprintf("%s/get", host) + } + args := access.GetArgs{ + Location: location.Copy(), + Offset: 0, + ReadSize: 0, + } + { + args.ReadSize = 10 + resp, err := cli.Post(ctx, url(), args) + require.NoError(t, err) + resp.Body.Close() + require.Equal(t, 400, resp.StatusCode, resp.Status) + } + { + args.Location.Size = 1023 + args.ReadSize = 1023 + fillCrc(&args.Location) + resp, err := cli.Post(ctx, url(), args) + require.NoError(t, err) + resp.Body.Close() + require.Equal(t, 500, resp.StatusCode, resp.Status) + } + { + args.Location.Size = 1024 + args.ReadSize = 1024 + fillCrc(&args.Location) + resp, err := cli.Post(ctx, url(), args) + require.NoError(t, err) + resp.Body.Close() + require.Equal(t, 200, resp.StatusCode, resp.Status) + } + { + args.Location.Size = 10240 + args.Offset = 1000 + args.ReadSize = 1024 + fillCrc(&args.Location) + resp, err := cli.Post(ctx, url(), args) + require.NoError(t, err) + resp.Body.Close() + require.Equal(t, 206, resp.StatusCode, resp.Status) + } +} + +func TestAccessServiceDelete(t *testing.T) { + host := runMockService(newService()) + cli := newClient() + + url := fmt.Sprintf("%s/delete", host) + deleteRequest := func(args interface{}) (code int, ret access.DeleteResp, err error) { + resp, err := cli.Post(ctx, url, args) + if err != nil { + return + } + defer resp.Body.Close() + + code = resp.StatusCode + if code/100 == 2 { + size, _ := strconv.Atoi(resp.Header.Get("Content-Length")) + buf := make([]byte, size) + _, err = io.ReadFull(resp.Body, buf) + if err != nil { + return + } + if err = json.Unmarshal(buf, &ret); err != nil { + return + } + } + if code >= 400 { + err = rpc.NewError(code, "Code", fmt.Errorf("httpcode: %d", code)) + return + } + return + } + + args := access.DeleteArgs{ + Locations: []access.Location{location.Copy()}, + } + { + code, _, err := deleteRequest(access.DeleteArgs{}) + require.Error(t, err) + require.Equal(t, 400, code) + } + { + code, _, err := deleteRequest(args) + require.Error(t, err) + require.Equal(t, 400, code) + } + { + fillCrc(&args.Locations[0]) + code, resp, err := deleteRequest(args) + require.NoError(t, err) + require.Equal(t, 226, code) + require.Equal(t, args.Locations[0], resp.FailedLocations[0]) + } + { + loc := &args.Locations[0] + loc.Size = 1024 + fillCrc(loc) + code, _, err := deleteRequest(args) + require.NoError(t, err) + require.Equal(t, 200, code) + } + { + loc := location.Copy() + loc.Size = 1024 + fillCrc(&loc) + locs := make([]access.Location, access.MaxDeleteLocations) + for idx := range locs { + locs[idx] = loc + } + code, resp, err := deleteRequest(access.DeleteArgs{Locations: locs}) + require.NoError(t, err) + require.Equal(t, 200, code) + require.Equal(t, 0, len(resp.FailedLocations)) + } + { + loc := location.Copy() + loc.Size = 1024 + fillCrc(&loc) + locs := make([]access.Location, access.MaxDeleteLocations+1) + for idx := range locs { + locs[idx] = loc + } + code, _, err := deleteRequest(access.DeleteArgs{Locations: locs}) + require.Error(t, err) + require.Equal(t, 400, code) + } + { + loc := location.Copy() + loc.Size = 1024 + loc.ClusterID = proto.ClusterID(11) + fillCrc(&loc) + code, resp, err := deleteRequest(access.DeleteArgs{Locations: []access.Location{loc}}) + require.NoError(t, err) + require.Equal(t, 226, code) + require.Equal(t, 1, len(resp.FailedLocations)) + require.Equal(t, proto.ClusterID(11), resp.FailedLocations[0].ClusterID) + } + { + locs := make([]access.Location, access.MaxDeleteLocations) + for idx := range locs { + loc := location.Copy() + loc.Size = 1024 + loc.ClusterID = proto.ClusterID(idx % 11) + fillCrc(&loc) + locs[idx] = loc + } + code, resp, err := deleteRequest(access.DeleteArgs{Locations: locs}) + require.NoError(t, err) + require.Equal(t, 226, code) + require.Equal(t, 93, len(resp.FailedLocations)) + } +} + +func TestAccessServiceDeleteBlob(t *testing.T) { + host := runMockService(newService()) + cli := newClient() + + url := func(size int64, token string) string { + return fmt.Sprintf("%s/deleteblob?clusterid=1&volumeid=1111&blobid=111&size=%d&token=%s", + host, size, token) + } + + method := http.MethodDelete + args := access.PutArgs{ + Size: 0, + } + { + req, _ := http.NewRequest(method, url(args.Size, "xxx"), nil) + err := cli.DoWith(ctx, req, nil) + assertErrorCode(t, 400, err) + } + { + req, _ := http.NewRequest(method, url(args.Size, ""), nil) + err := cli.DoWith(ctx, req, nil) + assertErrorCode(t, 400, err) + } + { + args.Size = 1023 + req, _ := http.NewRequest(method, url(args.Size, ""), nil) + err := cli.DoWith(ctx, req, nil) + assertErrorCode(t, 400, err) + } + { + args.Size = 1023 + req, _ := http.NewRequest(method, url(args.Size, "xxx"), nil) + err := cli.DoWith(ctx, req, nil) + assertErrorCode(t, 400, err) + } + { + args.Size = 1023 + req, _ := http.NewRequest(method, url(args.Size, "c1fdcecaacbfafd86f0b00"), nil) + err := cli.DoWith(ctx, req, nil) + require.NoError(t, err) + } + + { + url := func() string { + return fmt.Sprintf("%s/deleteblob?clusterid=11&volumeid=1111&blobid=111&size=%d&token=%s", + host, 1024, "f034db4503d5dc3f6f0100") + } + req, _ := http.NewRequest(method, url(), nil) + err := cli.DoWith(ctx, req, nil) + assertErrorCode(t, 500, err) + } +} + +func TestAccessServiceSign(t *testing.T) { + host := runMockService(newService()) + cli := newClient() + + url := func() string { + return fmt.Sprintf("%s/sign", host) + } + args := access.SignArgs{ + Locations: []access.Location{location.Copy()}, + Location: location.Copy(), + } + { + resp := &access.SignResp{} + err := cli.PostWith(ctx, url(), resp, access.SignArgs{}) + assertErrorCode(t, 400, err) + } + { + resp := &access.SignResp{} + err := cli.PostWith(ctx, url(), resp, args) + assertErrorCode(t, 400, err) + } + { + fillCrc(&args.Locations[0]) + resp := &access.SignResp{} + err := cli.PostWith(ctx, url(), resp, args) + require.NoError(t, err) + } +} + +func assertErrorCode(t *testing.T, code int, err error) { + require.Error(t, err) + codeActual := rpc.DetectStatusCode(err) + require.Equal(t, code, codeActual, err.Error()) +} + +func TestAccessServiceTokens(t *testing.T) { + skey := tokenSecretKeys[0][:] + checker := func(loc *access.Location, tokens []string) { + if loc.Size == 0 { + require.Equal(t, 0, len(tokens)) + return + } + + hasMultiBlobs := loc.Size >= uint64(loc.BlobSize) + lastSize := uint32(loc.Size % uint64(loc.BlobSize)) + if !hasMultiBlobs { + require.Equal(t, 1, len(tokens)) + + token := uptoken.DecodeToken(tokens[0]) + blob := loc.Blobs[0] + for bid := blob.MinBid - 100; bid < blob.MinBid+100; bid++ { + require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey)) + } + require.True(t, token.IsValid(loc.ClusterID, blob.Vid, blob.MinBid, lastSize, skey)) + return + } + + if lastSize == 0 { + require.Equal(t, len(loc.Blobs), len(tokens)) + for idx, blob := range loc.Blobs { + token := uptoken.DecodeToken(tokens[idx]) + for ii := uint32(0); ii < 100; ii++ { + bid := blob.MinBid - proto.BlobID(ii) - 1 + require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey)) + bid = blob.MinBid + proto.BlobID(blob.Count+ii) + require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey)) + } + for ii := uint32(0); ii < blob.Count; ii++ { + bid := blob.MinBid + proto.BlobID(ii) + require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey)) + } + } + return + } + + require.Equal(t, len(loc.Blobs)+1, len(tokens)) + for ii := 0; ii < len(loc.Blobs)-1; ii++ { + token := uptoken.DecodeToken(tokens[ii]) + blob := loc.Blobs[ii] + for ii := uint32(0); ii < blob.Count; ii++ { + bid := blob.MinBid + proto.BlobID(ii) + require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey)) + } + } + + token := uptoken.DecodeToken(tokens[len(loc.Blobs)-1]) + blob := loc.Blobs[len(loc.Blobs)-1] + for ii := uint32(0); ii < 100; ii++ { + bid := blob.MinBid - proto.BlobID(ii) - 1 + require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey)) + bid = blob.MinBid + proto.BlobID(blob.Count+ii) - 1 + require.False(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey)) + } + for ii := uint32(0); ii < blob.Count-1; ii++ { + bid := blob.MinBid + proto.BlobID(ii) + require.True(t, token.IsValid(loc.ClusterID, blob.Vid, bid, loc.BlobSize, skey)) + } + + token = uptoken.DecodeToken(tokens[len(loc.Blobs)]) + lastbid := blob.MinBid + proto.BlobID(blob.Count) - 1 + require.True(t, token.IsValid(loc.ClusterID, blob.Vid, lastbid, lastSize, skey)) + } + + { + loc := &access.Location{ + Size: 0, + BlobSize: 333, + Blobs: []access.SliceInfo{}, + } + checker(loc, genTokens(loc)) + } + { + loc := &access.Location{ + Size: 1, + BlobSize: 1024, + Blobs: []access.SliceInfo{ + {MinBid: 100, Vid: 1000, Count: 1}, + }, + } + checker(loc, genTokens(loc)) + } + { + loc := &access.Location{ + Size: 1024, + BlobSize: 1024, + Blobs: []access.SliceInfo{ + {MinBid: 100, Vid: 1000, Count: 1}, + }, + } + checker(loc, genTokens(loc)) + } + { + loc := &access.Location{ + Size: 1025, + BlobSize: 1024, + Blobs: []access.SliceInfo{ + {MinBid: 100, Vid: 1000, Count: 2}, + }, + } + checker(loc, genTokens(loc)) + } + { + loc := &access.Location{ + Size: 2048, + BlobSize: 1024, + Blobs: []access.SliceInfo{ + {MinBid: 100, Vid: 1000, Count: 2}, + }, + } + checker(loc, genTokens(loc)) + } + { + loc := &access.Location{ + Size: 10240, + BlobSize: 1024, + Blobs: []access.SliceInfo{ + {MinBid: 100, Vid: 1000, Count: 4}, + {MinBid: 200, Vid: 1000, Count: 6}, + }, + } + checker(loc, genTokens(loc)) + } + { + loc := &access.Location{ + Size: 1025, + BlobSize: 1024, + Blobs: []access.SliceInfo{ + {MinBid: 100, Vid: 1000, Count: 1}, + {MinBid: 200, Vid: 1000, Count: 1}, + }, + } + checker(loc, genTokens(loc)) + } + { + loc := &access.Location{ + Size: 10242, + BlobSize: 1024, + Blobs: []access.SliceInfo{ + {MinBid: 100, Vid: 1000, Count: 5}, + {MinBid: 200, Vid: 1000, Count: 6}, + }, + } + checker(loc, genTokens(loc)) + } +} + +func TestAccessServiceLimited(t *testing.T) { + host := runMockService(newService()) + cli := newClient() + + url := func() string { + return fmt.Sprintf("%s/alloc", host) + } + args := access.AllocArgs{Size: 1024} + var wg sync.WaitGroup + wg.Add(100) + for i := 0; i < 100; i++ { + go func() { + defer wg.Done() + resp := &access.AllocResp{} + err := cli.PostWith(ctx, url(), resp, args) + if err != nil { + assertErrorCode(t, errcode.CodeAccessLimited, err) + } else { + require.Equal(t, uint64(1024), resp.Location.Size) + } + }() + } + wg.Wait() +} diff --git a/blobstore/access/service.go b/blobstore/access/service.go new file mode 100644 index 000000000..9e0718bf6 --- /dev/null +++ b/blobstore/access/service.go @@ -0,0 +1,104 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "github.com/cubefs/cubefs/blobstore/api/access" + "github.com/cubefs/cubefs/blobstore/cmd" + "github.com/cubefs/cubefs/blobstore/common/config" + "github.com/cubefs/cubefs/blobstore/common/rpc" +) + +var ( + gConfig Config + gService *Service +) + +func init() { + mod := &cmd.Module{ + Name: "ACCESS", + InitConfig: initConfig, + SetUp: setUp, + TearDown: tearDown, + } + cmd.RegisterGracefulModule(mod) +} + +func initConfig(args []string) (*cmd.Config, error) { + config.Init("f", "", "access.conf") + if err := config.Load(&gConfig); err != nil { + return nil, err + } + return &gConfig.Config, nil +} + +func setUp() (*rpc.Router, []rpc.ProgressHandler) { + gService = New(gConfig) + // register all self functions of service + gService.RegisterService() + gService.RegisterAdminHandler() + return NewHandler(gService), nil +} + +func tearDown() { + gService.Close() +} + +// NewHandler returns app server handler +func NewHandler(service *Service) *rpc.Router { + rpc.RegisterArgsParser(&access.PutArgs{}, "json") + rpc.RegisterArgsParser(&access.PutAtArgs{}, "json") + rpc.RegisterArgsParser(&access.DeleteBlobArgs{}, "json") + + rpc.Use(service.Limit) + + // POST /put?size={size}&hashes={hashes} + // request body: DataStream + // response body: json + rpc.POST("/put", service.Put, rpc.OptArgsQuery()) + // PUT /put?size={size}&hashes={hashes} + rpc.PUT("/put", service.Put, rpc.OptArgsQuery()) + + // POST /putat?clusterid={clusterid}&volumeid={volumeid}&blobid={blobid}&size={size}&hashes={hashes}&token={token} + // request body: DataStream + // response body: json + rpc.POST("/putat", service.PutAt, rpc.OptArgsQuery()) + // PUT /putat?clusterid={clusterid}&volumeid={volumeid}&blobid={blobid}&size={size}&hashes={hashes}&token={token} + rpc.PUT("/putat", service.PutAt, rpc.OptArgsQuery()) + + // POST /alloc + // request body: json + // response body: json + rpc.POST("/alloc", service.Alloc, rpc.OptArgsBody()) + + // POST /get + // request body: json + // response body: DataStream + rpc.POST("/get", service.Get, rpc.OptArgsBody()) + + // POST /delete + // request body: json + // response body: json + rpc.POST("/delete", service.Delete, rpc.OptArgsBody()) + // DELETE /deleteblob + rpc.DELETE("/deleteblob", service.DeleteBlob, rpc.OptArgsQuery()) + + // POST /sign + // request body: json + // response body: json + rpc.POST("/sign", service.Sign, rpc.OptArgsBody()) + + return rpc.DefaultRouter +} diff --git a/blobstore/access/stream.go b/blobstore/access/stream.go new file mode 100644 index 000000000..9eed92a15 --- /dev/null +++ b/blobstore/access/stream.go @@ -0,0 +1,467 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "context" + "encoding/json" + "fmt" + "io" + "strings" + + "github.com/afex/hystrix-go/hystrix" + "github.com/hashicorp/consul/api" + + "github.com/cubefs/cubefs/blobstore/access/controller" + "github.com/cubefs/cubefs/blobstore/api/access" + "github.com/cubefs/cubefs/blobstore/api/blobnode" + "github.com/cubefs/cubefs/blobstore/api/proxy" + "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/cubefs/cubefs/blobstore/common/ec" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/resourcepool" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/util/defaulter" + "github.com/cubefs/cubefs/blobstore/util/errors" + "github.com/cubefs/cubefs/blobstore/util/log" + "github.com/cubefs/cubefs/blobstore/util/retry" +) + +const ( + defaultMaxObjectSize int64 = 5 * (1 << 30) // 5GB + + // hystrix command define + allocCommand = "alloc" + rwCommand = "rw" + + serviceProxy = proto.ServiceNameProxy +) + +// StreamHandler stream http handler +type StreamHandler interface { + // Alloc access interface /alloc + // required: size, file size + // optional: blobSize > 0, alloc with blobSize + // assignClusterID > 0, assign to alloc in this cluster certainly + // codeMode > 0, alloc in this codemode + // return: a location of file + Alloc(ctx context.Context, size uint64, blobSize uint32, + assignClusterID proto.ClusterID, codeMode codemode.CodeMode) (*access.Location, error) + + // PutAt access interface /putat, put one blob + // required: rc file reader + // required: clusterID VolumeID BlobID + // required: size, one blob size + // optional: hasherMap, computing hash + PutAt(ctx context.Context, rc io.Reader, + clusterID proto.ClusterID, vid proto.Vid, bid proto.BlobID, size int64, hasherMap access.HasherMap) error + + // Put put one object + // required: size, file size + // optional: hasher map to calculate hash.Hash + Put(ctx context.Context, rc io.Reader, size int64, hasherMap access.HasherMap) (*access.Location, error) + + // Get read file + // required: location, readSize + // optional: offset(default is 0) + // + // first return value is data transfer to copy data after argument checking + // + // Read data shards firstly, if blob size is small or read few bytes + // then ec reconstruct-read, try to reconstruct from N+X to N+M + // + // sorted N+X is, such as we use mode EC6P10L2, X=2 and Read from idc=2 + // shards like this + // data N 6 | parity M 10 | local L 2 + // d1 d2 d3 d4 d5 d6 p1 .. p5 p6 .. p10 l1 l2 + // idc 1 1 1 2 2 2 1 2 1 2 + // + //sorted d4 d5 d6 p6 .. p10 d1 d2 d3 p1 .. p5 + //read-1 [d4 p10] + //read-2 [d4 p10 d1] + //read-3 [d4 p10 d1 d2] + //... + //read-9 [d4 p5] + //failed + Get(ctx context.Context, w io.Writer, location access.Location, readSize, offset uint64) (func() error, error) + + // Delete delete all blobs in this location + Delete(ctx context.Context, location *access.Location) error + + // Admin returns internal admin interface. + Admin() interface{} +} + +type streamAdmin struct { + config StreamConfig + memPool *resourcepool.MemPool + controller controller.ClusterController +} + +// StreamConfig access stream handler config +type StreamConfig struct { + IDC string `json:"idc"` + + MaxBlobSize uint32 `json:"max_blob_size"` + DiskPunishIntervalS int `json:"disk_punish_interval_s"` + DiskTimeoutPunishIntervalS int `json:"disk_timeout_punish_interval_s"` + ServicePunishIntervalS int `json:"service_punish_interval_s"` + AllocRetryTimes int `json:"alloc_retry_times"` + AllocRetryIntervalMS int `json:"alloc_retry_interval_ms"` + EncoderEnableVerify bool `json:"encoder_enableverify"` + EncoderConcurrency int `json:"encoder_concurrency"` + MinReadShardsX int `json:"min_read_shards_x"` + ShardCrcDisabled bool `json:"shard_crc_disabled"` + + MemPoolSizeClasses map[int]int `json:"mem_pool_size_classes"` + + // CodeModesPutQuorums + // just for one AZ is down, cant write quorum in all AZs + CodeModesPutQuorums map[codemode.CodeMode]int `json:"code_mode_put_quorums"` + + ClusterConfig controller.ClusterConfig `json:"cluster_config"` + BlobnodeConfig blobnode.Config `json:"blobnode_config"` + ProxyConfig proxy.Config `json:"proxy_config"` + + // hystrix command config + AllocCommandConfig hystrix.CommandConfig `json:"alloc_command_config"` + RWCommandConfig hystrix.CommandConfig `json:"rw_command_config"` +} + +// discard unhealthy volume +type discardVid struct { + cid proto.ClusterID + codeMode codemode.CodeMode + vid proto.Vid +} + +type blobIdent struct { + cid proto.ClusterID + vid proto.Vid + bid proto.BlobID +} + +func (id *blobIdent) String() string { + return fmt.Sprintf("blob(%d %d %d)", id.cid, id.vid, id.bid) +} + +// Handler stream handler +type Handler struct { + memPool *resourcepool.MemPool + encoder map[codemode.CodeMode]ec.Encoder + clusterController controller.ClusterController + + blobnodeClient blobnode.StorageAPI + proxyClient proxy.Client + + allCodeModes CodeModePairs + maxObjectSize int64 + + discardVidChan chan discardVid + stopCh <-chan struct{} + + StreamConfig +} + +func confCheck(cfg *StreamConfig) { + if cfg.IDC == "" { + log.Fatal("idc config can not be null") + } + cfg.ClusterConfig.IDC = cfg.IDC + + if len(cfg.MemPoolSizeClasses) == 0 { + cfg.MemPoolSizeClasses = getDefaultMempoolSize() + } + + for mode, quorum := range cfg.CodeModesPutQuorums { + tactic := mode.Tactic() + if quorum < tactic.N+tactic.L+1 || quorum > mode.GetShardNum() { + log.Fatalf("invalid put quorum(%d) in codemode(%d): %+v", quorum, mode, tactic) + } + } + + defaulter.Equal(&cfg.MaxBlobSize, defaultMaxBlobSize) + defaulter.LessOrEqual(&cfg.DiskPunishIntervalS, defaultDiskPunishIntervalS) + defaulter.LessOrEqual(&cfg.DiskTimeoutPunishIntervalS, defaultDiskPunishIntervalS/10) + defaulter.LessOrEqual(&cfg.ServicePunishIntervalS, defaultServicePunishIntervalS) + defaulter.LessOrEqual(&cfg.AllocRetryTimes, defaultAllocRetryTimes) + if cfg.AllocRetryIntervalMS <= 100 { + cfg.AllocRetryIntervalMS = defaultAllocRetryIntervalMS + } + defaulter.LessOrEqual(&cfg.EncoderConcurrency, defaultEncoderConcurrency) + defaulter.LessOrEqual(&cfg.MinReadShardsX, defaultMinReadShardsX) + + defaulter.LessOrEqual(&cfg.ClusterConfig.CMClientConfig.Config.ClientTimeoutMs, defaultTimeoutClusterMgr) + defaulter.LessOrEqual(&cfg.BlobnodeConfig.ClientTimeoutMs, defaultTimeoutBlobnode) + defaulter.LessOrEqual(&cfg.ProxyConfig.ClientTimeoutMs, defaultTimeoutProxy) + + hc := cfg.AllocCommandConfig + defaulter.LessOrEqual(&hc.Timeout, defaultAllocatorTimeout) + defaulter.LessOrEqual(&hc.MaxConcurrentRequests, defaultAllocatorMaxConcurrentRequests) + defaulter.LessOrEqual(&hc.RequestVolumeThreshold, defaultAllocatorRequestVolumeThreshold) + defaulter.LessOrEqual(&hc.SleepWindow, defaultAllocatorSleepWindow) + defaulter.LessOrEqual(&hc.ErrorPercentThreshold, defaultAllocatorErrorPercentThreshold) + cfg.AllocCommandConfig = hc + + hc = cfg.RWCommandConfig + defaulter.LessOrEqual(&hc.Timeout, defaultBlobnodeTimeout) + defaulter.LessOrEqual(&hc.MaxConcurrentRequests, defaultBlobnodeMaxConcurrentRequests) + defaulter.LessOrEqual(&hc.RequestVolumeThreshold, defaultBlobnodeRequestVolumeThreshold) + defaulter.LessOrEqual(&hc.SleepWindow, defaultBlobnodeSleepWindow) + defaulter.LessOrEqual(&hc.ErrorPercentThreshold, defaultBlobnodeErrorPercentThreshold) + cfg.RWCommandConfig = hc +} + +// NewStreamHandler returns a stream handler +func NewStreamHandler(cfg *StreamConfig, kvClient *api.Client, stopCh <-chan struct{}) StreamHandler { + confCheck(cfg) + + clusterController, err := controller.NewClusterController(&cfg.ClusterConfig, kvClient) + if err != nil { + log.Fatalf("new cluster controller failed, err: %v", err) + } + + handler := &Handler{ + memPool: resourcepool.NewMemPool(cfg.MemPoolSizeClasses), + clusterController: clusterController, + + blobnodeClient: blobnode.New(&cfg.BlobnodeConfig), + proxyClient: proxy.New(&cfg.ProxyConfig), + + maxObjectSize: defaultMaxObjectSize, + StreamConfig: *cfg, + } + + rawCodeModePolicies, err := handler.clusterController.GetConfig(context.Background(), proto.CodeModeConfigKey) + if err != nil { + log.Fatal("get codemode policy from cluster manager failed, err: ", err) + } + codeModePolicies := make([]codemode.Policy, 0) + err = json.Unmarshal([]byte(rawCodeModePolicies), &codeModePolicies) + if err != nil { + log.Fatal("json decode codemode policy failed, err: ", err) + } + if len(codeModePolicies) <= 0 { + log.Fatal("invalid codemode policy raw: ", rawCodeModePolicies) + } + + allCodeModes := make(CodeModePairs) + encoders := make(map[codemode.CodeMode]ec.Encoder) + maxSize := int64(0) + for _, policy := range codeModePolicies { + if policy.MaxSize > maxSize { + maxSize = policy.MaxSize + } + codeMode := policy.ModeName.GetCodeMode() + tactic := codeMode.Tactic() + allCodeModes[codeMode] = CodeModePair{ + Policy: policy, + Tactic: tactic, + } + encoder, err := ec.NewEncoder(ec.Config{ + CodeMode: tactic, + EnableVerify: cfg.EncoderEnableVerify, + Concurrency: cfg.EncoderConcurrency, + }) + if err != nil { + log.Fatalf("new encoder failed, err: %v", err) + } + encoders[codeMode] = encoder + } + handler.allCodeModes = allCodeModes + handler.encoder = encoders + if maxSize < handler.maxObjectSize { + handler.maxObjectSize = maxSize + } + + hystrix.ConfigureCommand(allocCommand, cfg.AllocCommandConfig) + hystrix.ConfigureCommand(rwCommand, cfg.RWCommandConfig) + + handler.discardVidChan = make(chan discardVid, 8) + handler.stopCh = stopCh + handler.loopDiscardVids() + return handler +} + +// Delete delete all blobs in this location +func (h *Handler) Delete(ctx context.Context, location *access.Location) error { + span := trace.SpanFromContextSafe(ctx) + span.Debugf("to delete %+v", location) + return h.clearGarbage(ctx, location) +} + +// Admin returns internal admin interface. +func (h *Handler) Admin() interface{} { + return &streamAdmin{ + config: h.StreamConfig, + memPool: h.memPool, + controller: h.clusterController, + } +} + +func (h *Handler) sendRepairMsgBg(ctx context.Context, blob blobIdent, badIdxes []uint8) { + go func() { + h.sendRepairMsg(ctx, blob, badIdxes) + }() +} + +func (h *Handler) sendRepairMsg(ctx context.Context, blob blobIdent, badIdxes []uint8) { + span := trace.SpanFromContextSafe(ctx) + span.Infof("to repair %s indexes(%+v)", blob.String(), badIdxes) + + clusterID := blob.cid + serviceController, err := h.clusterController.GetServiceController(clusterID) + if err != nil { + span.Error(errors.Detail(err)) + return + } + reportUnhealth(clusterID, "repair.msg", "-", "-", "-") + + repairArgs := &proxy.ShardRepairArgs{ + ClusterID: clusterID, + Bid: blob.bid, + Vid: blob.vid, + BadIdxes: badIdxes[:], + Reason: "access-repair", + } + + if err := retry.Timed(3, 200).On(func() error { + host, err := serviceController.GetServiceHost(ctx, serviceProxy) + if err != nil { + span.Warn(err) + return err + } + err = h.proxyClient.SendShardRepairMsg(ctx, host, repairArgs) + if err != nil { + span.Warnf("send to %s repair message(%+v) %s", host, repairArgs, err.Error()) + serviceController.PunishServiceWithThreshold(ctx, serviceProxy, host, h.ServicePunishIntervalS) + reportUnhealth(clusterID, "punish", serviceProxy, host, "failed") + err = errors.Base(err, host) + } + return err + }); err != nil { + reportUnhealth(clusterID, "repair.msg", serviceProxy, "-", "failed") + span.Errorf("send repair message(%+v) failed %s", repairArgs, errors.Detail(err)) + return + } + + span.Infof("send repair message(%+v)", repairArgs) +} + +func (h *Handler) clearGarbage(ctx context.Context, location *access.Location) error { + span := trace.SpanFromContextSafe(ctx) + serviceController, err := h.clusterController.GetServiceController(location.ClusterID) + if err != nil { + span.Error(errors.Detail(err)) + return errors.Base(err, "clear location:", *location) + } + + blobs := location.Spread() + deleteArgs := &proxy.DeleteArgs{ + ClusterID: location.ClusterID, + Blobs: make([]proxy.BlobDelete, 0, len(blobs)), + } + + for _, blob := range blobs { + deleteArgs.Blobs = append(deleteArgs.Blobs, proxy.BlobDelete{ + Bid: blob.Bid, + Vid: blob.Vid, + }) + } + + var logMsg interface{} = location + if len(deleteArgs.Blobs) <= 20 { + logMsg = deleteArgs + } + if err := retry.Timed(3, 200).On(func() error { + host, err := serviceController.GetServiceHost(ctx, serviceProxy) + if err != nil { + span.Warn(err) + return err + } + err = h.proxyClient.SendDeleteMsg(ctx, host, deleteArgs) + if err != nil { + span.Warnf("send to %s delete message(%+v) %s", host, logMsg, err.Error()) + serviceController.PunishServiceWithThreshold(ctx, serviceProxy, host, h.ServicePunishIntervalS) + reportUnhealth(location.ClusterID, "punish", serviceProxy, host, "failed") + err = errors.Base(err, host) + } + return err + }); err != nil { + reportUnhealth(location.ClusterID, "delete.msg", serviceProxy, "-", "failed") + span.Errorf("send delete message(%+v) failed %s", logMsg, errors.Detail(err)) + return errors.Base(err, "send delete message:", logMsg) + } + + span.Infof("send delete message(%+v)", logMsg) + return nil +} + +// getVolume get volume info +func (h *Handler) getVolume(ctx context.Context, clusterID proto.ClusterID, vid proto.Vid, isCache bool) (*controller.VolumePhy, error) { + volumeGetter, err := h.clusterController.GetVolumeGetter(clusterID) + if err != nil { + return nil, err + } + + volume := volumeGetter.Get(ctx, vid, isCache) + if volume == nil { + return nil, errors.Newf("not found volume of (%d %d)", clusterID, vid) + } + + return volume, nil +} + +func (h *Handler) punishVolume(ctx context.Context, clusterID proto.ClusterID, vid proto.Vid, host, reason string) { + reportUnhealth(clusterID, "punish", "volume", host, reason) + if volumeGetter, err := h.clusterController.GetVolumeGetter(clusterID); err == nil { + volumeGetter.Punish(ctx, vid, h.DiskPunishIntervalS) + } +} + +func (h *Handler) punishDisk(ctx context.Context, clusterID proto.ClusterID, diskID proto.DiskID, host, reason string) { + reportUnhealth(clusterID, "punish", "disk", host, reason) + if serviceController, err := h.clusterController.GetServiceController(clusterID); err == nil { + serviceController.PunishDisk(ctx, diskID, h.DiskPunishIntervalS) + } +} + +func (h *Handler) punishDiskWith(ctx context.Context, clusterID proto.ClusterID, diskID proto.DiskID, host, reason string) { + reportUnhealth(clusterID, "punish", "diskwith", host, reason) + if serviceController, err := h.clusterController.GetServiceController(clusterID); err == nil { + serviceController.PunishDiskWithThreshold(ctx, diskID, h.DiskTimeoutPunishIntervalS) + } +} + +// blobCount blobSize > 0 is certain +func blobCount(size uint64, blobSize uint32) uint64 { + return (size + uint64(blobSize) - 1) / uint64(blobSize) +} + +func minU64(a, b uint64) uint64 { + if a <= b { + return a + } + return b +} + +func errorTimeout(err error) bool { + msg := err.Error() + return strings.Contains(msg, "Timeout") || strings.Contains(msg, "timeout") +} + +func errorConnectionRefused(err error) bool { + return strings.Contains(err.Error(), "connection refused") +} diff --git a/blobstore/access/stream_alloc.go b/blobstore/access/stream_alloc.go new file mode 100644 index 000000000..5c4d7e8d3 --- /dev/null +++ b/blobstore/access/stream_alloc.go @@ -0,0 +1,206 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "context" + "sync/atomic" + + "github.com/afex/hystrix-go/hystrix" + "github.com/cubefs/cubefs/blobstore/api/access" + "github.com/cubefs/cubefs/blobstore/api/proxy" + "github.com/cubefs/cubefs/blobstore/common/codemode" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/util/errors" + "github.com/cubefs/cubefs/blobstore/util/retry" +) + +var errAllocatePunishedVolume = errors.New("allocate punished volume") + +// Alloc access interface /alloc +// required: size, file size +// optional: blobSize > 0, alloc with blobSize +// assignClusterID > 0, assign to alloc in this cluster certainly +// codeMode > 0, alloc in this codemode +// return: a location of file +func (h *Handler) Alloc(ctx context.Context, size uint64, blobSize uint32, + assignClusterID proto.ClusterID, codeMode codemode.CodeMode) (*access.Location, error) { + span := trace.SpanFromContextSafe(ctx) + span.Debugf("alloc request with size:%d blobsize:%d cluster:%d codemode:%d", + size, blobSize, assignClusterID, codeMode) + + if int64(size) > h.maxObjectSize { + span.Info("exceed max object size", h.maxObjectSize) + return nil, errcode.ErrAccessExceedSize + } + + if blobSize == 0 { + blobSize = atomic.LoadUint32(&h.MaxBlobSize) + span.Debugf("fill blobsize:%d", blobSize) + } + + if codeMode == 0 { + codeMode = h.allCodeModes.SelectCodeMode(int64(size)) + span.Debugf("select codemode:%d", codeMode) + } + if !codeMode.IsValid() { + span.Infof("invalid codemode:%d", codeMode) + return nil, errcode.ErrIllegalArguments + } + + clusterID, blobs, err := h.allocFromAllocatorWithHystrix(ctx, codeMode, size, blobSize, assignClusterID) + if err != nil { + span.Error("alloc from proxy", errors.Detail(err)) + return nil, err + } + span.Debugf("allocated from %d %+v", clusterID, blobs) + + location := &access.Location{ + ClusterID: clusterID, + CodeMode: codeMode, + Size: size, + BlobSize: blobSize, + Blobs: blobs, + } + span.Debugf("alloc ok %+v", location) + return location, nil +} + +func (h *Handler) allocFromAllocatorWithHystrix(ctx context.Context, codeMode codemode.CodeMode, size uint64, blobSize uint32, + clusterID proto.ClusterID) (cid proto.ClusterID, bidRets []access.SliceInfo, err error) { + err = hystrix.Do(allocCommand, func() error { + cid, bidRets, err = h.allocFromAllocator(ctx, codeMode, size, blobSize, clusterID) + return err + }, nil) + return +} + +func (h *Handler) allocFromAllocator(ctx context.Context, codeMode codemode.CodeMode, size uint64, blobSize uint32, + clusterID proto.ClusterID) (proto.ClusterID, []access.SliceInfo, error) { + span := trace.SpanFromContextSafe(ctx) + + if blobSize == 0 { + blobSize = atomic.LoadUint32(&h.MaxBlobSize) + } + if clusterID == 0 { + clusterChosen, err := h.clusterController.ChooseOne() + if err != nil { + return 0, nil, err + } + clusterID = clusterChosen.ClusterID + } + + args := proxy.AllocVolsArgs{ + Fsize: size, + CodeMode: codeMode, + BidCount: blobCount(size, blobSize), + } + + var allocRets []proxy.AllocRet + var allocHost string + hostsSet := make(map[string]struct{}, 1) + if err := retry.ExponentialBackoff(h.AllocRetryTimes, uint32(h.AllocRetryIntervalMS)).On(func() error { + serviceController, err := h.clusterController.GetServiceController(clusterID) + if err != nil { + span.Warn(err) + return errors.Info(err, "get service controller", clusterID) + } + + var host string + for range [10]struct{}{} { + host, err = serviceController.GetServiceHost(ctx, serviceProxy) + if err != nil { + span.Warn(err) + return errors.Info(err, "get proxy host", clusterID) + } + if _, ok := hostsSet[host]; ok { + continue + } + hostsSet[host] = struct{}{} + break + } + allocHost = host + + allocRets, err = h.proxyClient.VolumeAlloc(ctx, host, &args) + if err != nil { + if errorTimeout(err) || errorConnectionRefused(err) { + span.Info("punish unreachable proxy host:", host) + reportUnhealth(clusterID, "punish", serviceProxy, host, "Timeout") + serviceController.PunishServiceWithThreshold(ctx, serviceProxy, host, h.ServicePunishIntervalS) + } + span.Warn(host, err) + return errors.Base(err, "alloc from proxy", host) + } + + // filter punished volume in allocating progress + for _, ret := range allocRets { + vInfo, err := h.getVolume(ctx, clusterID, ret.Vid, true) + if err != nil { + span.Warn(err) + return err + } + if vInfo.IsPunish { + // return err and retry allocate + err = errAllocatePunishedVolume + args.Excludes = append(args.Excludes, vInfo.Vid) + span.Warn("next retry exclude vid:", vInfo.Vid, err) + return err + } + } + + return nil + }); err != nil { + if err != errAllocatePunishedVolume { + reportUnhealth(clusterID, "allocate", "-", "-", "failed") + return 0, nil, err + } + // still write to storage if allocating punished volume + reportUnhealth(clusterID, "allocate", "-", "-", "punished") + } + + // cache vid in which allocator + for _, ret := range allocRets { + setCacheVidHost(clusterID, ret.Vid, allocHost) + } + + blobN := blobCount(size, blobSize) + blobs := make([]access.SliceInfo, 0, blobN) + for _, bidRet := range allocRets { + if blobN <= 0 { + break + } + + count := minU64(blobN, uint64(bidRet.BidEnd)-uint64(bidRet.BidStart)+1) + blobN -= count + + blobs = append(blobs, access.SliceInfo{ + MinBid: bidRet.BidStart, + Vid: bidRet.Vid, + Count: uint32(count), + }) + } + if blobN > 0 { + return 0, nil, errors.New("no enough blob ids from allocator") + } + + if uint32(len(blobs)) > access.MaxLocationBlobs { + span.Errorf("alloc exceed max blobs %d>%d", len(blobs), access.MaxLocationBlobs) + return 0, nil, errors.New("alloc exceed max blobs of location") + } + + return clusterID, blobs, nil +} diff --git a/blobstore/access/stream_alloc_test.go b/blobstore/access/stream_alloc_test.go new file mode 100644 index 000000000..df278b2bc --- /dev/null +++ b/blobstore/access/stream_alloc_test.go @@ -0,0 +1,70 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/common/codemode" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" +) + +func TestAccessStreamAllocBase(t *testing.T) { + ctx := ctxWithName("TestAccessStreamAllocBase") + // 4M blobsize + { + loc, err := streamer.Alloc(ctx(), 1<<30, 0, 0, 0) + require.NoError(t, err) + require.Equal(t, clusterID, loc.ClusterID) + require.Equal(t, codemode.EC6P6, loc.CodeMode) + require.Equal(t, uint64(1<<30), loc.Size) + require.Equal(t, uint32(1<<22), loc.BlobSize) + require.Equal(t, 2, len(loc.Blobs)) + require.Equal(t, uint32(1), loc.Blobs[0].Count) + require.Equal(t, uint32((1<<8)-1), loc.Blobs[1].Count) + } + { + loc, err := streamer.Alloc(ctx(), (1<<30)+1, 0, 0, 0) + require.NoError(t, err) + require.Equal(t, 2, len(loc.Blobs)) + require.Equal(t, uint32(1), loc.Blobs[0].Count) + require.Equal(t, uint32(1<<8), loc.Blobs[1].Count) + } + // 1M blobsize + { + loc, err := streamer.Alloc(ctx(), 1<<30, 1<<20, 0, 0) + require.NoError(t, err) + require.Equal(t, 2, len(loc.Blobs)) + require.Equal(t, uint32(1), loc.Blobs[0].Count) + require.Equal(t, uint32((1<<10)-1), loc.Blobs[1].Count) + } + // max size + 1 + { + _, err := streamer.Alloc(ctx(), uint64(defaultMaxObjectSize+1), 1<<20, 0, 0) + require.EqualError(t, errcode.ErrAccessExceedSize, err.Error()) + } + + { + // wait for service manager to reload + defer func() { + time.Sleep(time.Second * 2) + }() + _, err := streamer.Alloc(ctx(), allocTimeoutSize+1, 0, 0, 0) + require.Error(t, err) + } +} diff --git a/blobstore/access/stream_get.go b/blobstore/access/stream_get.go new file mode 100644 index 000000000..f4108d814 --- /dev/null +++ b/blobstore/access/stream_get.go @@ -0,0 +1,823 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "context" + "fmt" + "io" + "math/rand" + "sort" + "sync" + "time" + + "github.com/afex/hystrix-go/hystrix" + + "github.com/cubefs/cubefs/blobstore/access/controller" + "github.com/cubefs/cubefs/blobstore/api/access" + "github.com/cubefs/cubefs/blobstore/api/blobnode" + "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/cubefs/cubefs/blobstore/common/ec" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/util/errors" + "github.com/cubefs/cubefs/blobstore/util/retry" +) + +var ( + errNeedReconstructRead = errors.New("need to reconstruct read") + errCanceledReadShard = errors.New("canceled read shard") + errPunishedDisk = errors.New("punished disk") +) + +type blobGetArgs struct { + Vid proto.Vid + Bid proto.BlobID + BlobSize uint64 + Offset uint64 + ReadSize uint64 +} + +type shardData struct { + index int + status bool + buffer []byte +} + +type sortedVuid struct { + index int + vuid proto.Vuid + diskID proto.DiskID + host string +} + +type pipeBuffer struct { + err error + blob blobGetArgs + shards [][]byte +} + +// Get read file +// required: location, readSize +// optional: offset(default is 0) +// +// first return value is data transfer to copy data after argument checking +// +// Read data shards firstly, if blob size is small or read few bytes +// then ec reconstruct-read, try to reconstruct from N+X to N+M +// +// sorted N+X is, such as we use mode EC6P10L2, X=2 and Read from idc=2 +// shards like this +// data N 6 | parity M 10 | local L 2 +// d1 d2 d3 d4 d5 d6 p1 .. p5 p6 .. p10 l1 l2 +// idc 1 1 1 2 2 2 1 2 1 2 +// +//sorted d4 d5 d6 p6 .. p10 d1 d2 d3 p1 .. p5 +//read-1 [d4 p10] +//read-2 [d4 p10 d1] +//read-3 [d4 p10 d1 d2] +//... +//read-9 [d4 p5] +//failed +func (h *Handler) Get(ctx context.Context, w io.Writer, location access.Location, readSize, offset uint64) (func() error, error) { + span := trace.SpanFromContextSafe(ctx) + span.Debugf("get request cluster:%d size:%d offset:%d", location.ClusterID, readSize, offset) + + blobs, err := genLocationBlobs(&location, readSize, offset) + if err != nil { + span.Info("illegal argument", err) + return func() error { return nil }, errcode.ErrIllegalArguments + } + if len(blobs) == 0 { + return func() error { return nil }, nil + } + + clusterID := location.ClusterID + var serviceController controller.ServiceController + if err = retry.Timed(3, 200).On(func() error { + sc, err := h.clusterController.GetServiceController(clusterID) + if err != nil { + return err + } + serviceController = sc + return nil + }); err != nil { + span.Error("get service", errors.Detail(err)) + return func() error { return nil }, err + } + + return func() error { + getTime := new(timeReadWrite) + defer func() { + span.AppendRPCTrackLog([]string{getTime.String()}) + }() + + // try to read data shard only, + // if blobsize is small: all data is in the first shard, cos shards aligned by MinShardSize. + // read few bytes: read bytes less than quarter of blobsize, like Range:[0-1]. + if len(blobs) == 1 { + blob := blobs[0] + sizes, _ := ec.GetBufferSizes(int(blob.BlobSize), location.CodeMode.Tactic()) + if int(blob.BlobSize) <= sizes.ShardSize || blob.ReadSize < blob.BlobSize/4 { + span.Debugf("read data shard only readsize:%d blobsize:%d shardsize:%d", + blob.ReadSize, blob.BlobSize, sizes.ShardSize) + + reportDownload(clusterID, "Range", "-") + err := h.getDataShardOnly(ctx, getTime, w, serviceController, clusterID, blob) + if err != errNeedReconstructRead { + if err != nil { + span.Error("read data shard only", err) + reportDownload(clusterID, "Range", err.Error()) + } + return err + } + reportDownload(clusterID, "Range", err.Error()) + span.Info("read data shard only failed", err) + } + } + + // data stream flow: + // client <--copy-- pipeline <--swap-- readBlob <--copy-- blobnode + // + // Alloc N+M shard buffers here, and release after written to client. + // Replace not-empty buffers in readBlob, need release old-buffers in that function. + closeCh := make(chan struct{}) + pipeline := func() <-chan pipeBuffer { + ch := make(chan pipeBuffer, 1) + go func() { + defer close(ch) + + var ( + blobVolume *controller.VolumePhy + sortedVuids []sortedVuid + ) + for _, blob := range blobs { + var err error + if blobVolume == nil || blobVolume.Vid != blob.Vid { + blobVolume, err = h.getVolume(ctx, clusterID, blob.Vid, true) + if err != nil { + span.Error("get volume", err) + ch <- pipeBuffer{err: err} + return + } + + tactic := blobVolume.CodeMode.Tactic() + // do not use local shards + sortedVuids = genSortedVuidByIDC(ctx, serviceController, h.IDC, blobVolume.Units[:tactic.N+tactic.M]) + span.Debugf("to read blob(%d %d %d) with read-shard-x:%d active-shard-n:%d of data-n:%d party-n:%d", + clusterID, blob.Vid, blob.Bid, h.MinReadShardsX, len(sortedVuids), tactic.N, tactic.M) + if len(sortedVuids) < tactic.N { + err = fmt.Errorf("broken blob(%d %d %d)", clusterID, blob.Vid, blob.Bid) + span.Error(err) + ch <- pipeBuffer{err: err} + return + } + } + + codeMode := blobVolume.CodeMode + tactic := codeMode.Tactic() + sizes, _ := ec.GetBufferSizes(int(blob.BlobSize), tactic) + shardSize := sizes.ShardSize + + st := time.Now() + shards := make([][]byte, tactic.N+tactic.M) + for ii := range shards { + buf, _ := h.memPool.Alloc(shardSize) + shards[ii] = buf + } + getTime.IncA(time.Since(st)) + + err = h.readOneBlob(ctx, getTime, serviceController, clusterID, + blobVolume.Vid, codeMode, blob, sortedVuids, shards) + if err != nil { + span.Error("read one blob", blob.Bid, err) + for _, buf := range shards { + h.memPool.Put(buf) + } + ch <- pipeBuffer{err: err} + return + } + + select { + case <-closeCh: + for _, buf := range shards { + h.memPool.Put(buf) + } + return + case ch <- pipeBuffer{blob: blob, shards: shards}: + } + } + }() + + return ch + }() + + var err error + reportDownload(clusterID, "EC", "-") + for line := range pipeline { + if line.err != nil { + err = line.err + break + } + + startWrite := time.Now() + + idx := 0 + off := line.blob.Offset + toReadSize := line.blob.ReadSize + for toReadSize > 0 { + buf := line.shards[idx] + l := uint64(len(buf)) + if off >= l { + idx++ + off -= l + continue + } + + toRead := minU64(toReadSize, l-off) + if _, e := w.Write(buf[off : off+toRead]); e != nil { + err = errors.Info(e, "write to response") + break + } + idx++ + off = 0 + toReadSize -= toRead + } + + getTime.IncW(time.Since(startWrite)) + + for _, buf := range line.shards { + h.memPool.Put(buf) + } + if err != nil { + close(closeCh) + break + } + } + + // release buffer in pipeline if fail to write client + go func() { + for line := range pipeline { + for _, buf := range line.shards { + h.memPool.Put(buf) + } + } + }() + + if err != nil { + span.Error("get request error", err) + return err + } + return nil + }, nil +} + +// 1. try to min-read shards bytes +// 2. if failed try to read next shard to reconstruct +// 3. write the the right offset bytes to writer +func (h *Handler) readOneBlob(ctx context.Context, getTime *timeReadWrite, + serviceController controller.ServiceController, + clusterID proto.ClusterID, vid proto.Vid, codeMode codemode.CodeMode, + blob blobGetArgs, sortedVuids []sortedVuid, shards [][]byte) error { + span := trace.SpanFromContextSafe(ctx) + + tactic := codeMode.Tactic() + sizes, err := ec.GetBufferSizes(int(blob.BlobSize), tactic) + if err != nil { + return err + } + empties := emptyDataShardIndexes(sizes) + + dataN, dataParityN := tactic.N, tactic.N+tactic.M + minShardsRead := dataN + h.MinReadShardsX + if minShardsRead > len(sortedVuids) { + minShardsRead = len(sortedVuids) + } + shardSize := len(shards[0]) + + stopChan := make(chan struct{}) + nextChan := make(chan struct{}, len(sortedVuids)) + + shardPipe := func() <-chan shardData { + ch := make(chan shardData) + go func() { + wg := new(sync.WaitGroup) + defer func() { + wg.Wait() + close(ch) + }() + + for _, vuid := range sortedVuids[:minShardsRead] { + if _, ok := empties[vuid.index]; !ok { + wg.Add(1) + go func(vuid sortedVuid) { + ch <- h.readOneShard(ctx, serviceController, clusterID, vid, + shardSize, blob, vuid, stopChan) + wg.Done() + }(vuid) + } + } + + for _, vuid := range sortedVuids[minShardsRead:] { + if _, ok := empties[vuid.index]; ok { + continue + } + + select { + case <-stopChan: + return + case <-nextChan: + } + + wg.Add(1) + go func(vuid sortedVuid) { + ch <- h.readOneShard(ctx, serviceController, clusterID, vid, + shardSize, blob, vuid, stopChan) + wg.Done() + }(vuid) + } + }() + + return ch + }() + + received := make(map[int]bool, minShardsRead) + for idx := range empties { + received[idx] = true + h.memPool.Zero(shards[idx]) + } + + startRead := time.Now() + reconstructed := false + for shard := range shardPipe { + // swap shard buffer + if shard.status { + buf := shards[shard.index] + shards[shard.index] = shard.buffer + h.memPool.Put(buf) + } + + received[shard.index] = shard.status + if len(received) < dataN { + continue + } + + // bad data index + badIdx := make([]int, 0, 8) + for i := 0; i < dataN; i++ { + if succ, ok := received[i]; !ok || !succ { + badIdx = append(badIdx, i) + } + } + if len(badIdx) == 0 { + reconstructed = true + close(stopChan) + break + } + + // update bad parity index + for i := dataN; i < dataParityN; i++ { + if succ, ok := received[i]; !ok || !succ { + badIdx = append(badIdx, i) + } + } + + badShards := 0 + for _, succ := range received { + if !succ { + badShards++ + } + } + // it will not wait all the shards, cos has no enough shards to reconstruct + if badShards > dataParityN-dataN { + span.Infof("bid(%d) bad(%d) has no enough to reconstruct", blob.Bid, badShards) + close(stopChan) + break + } + + // has bad shards, but have enough shards to reconstruct + if len(received) >= dataN+badShards { + span.Debugf("bid(%d) ready to ec reconstruct data", blob.Bid) + err := h.encoder[codeMode].ReconstructData(shards, badIdx) + if err == nil { + reconstructed = true + close(stopChan) + break + } + span.Infof("bid(%d) ec reconstruct data error:%s", blob.Bid, err.Error()) + } + + if len(received) >= len(sortedVuids) { + close(stopChan) + break + } + nextChan <- struct{}{} + } + getTime.IncR(time.Since(startRead)) + + // release buffer of delayed shards + go func() { + for shard := range shardPipe { + if shard.status { + h.memPool.Put(shard.buffer) + } + } + }() + + if reconstructed { + return nil + } + return fmt.Errorf("broken blob(%d %d %d)", clusterID, blob.Vid, blob.Bid) +} + +func (h *Handler) readOneShard(ctx context.Context, serviceController controller.ServiceController, + clusterID proto.ClusterID, vid proto.Vid, shardSize int, + blob blobGetArgs, vuid sortedVuid, stopChan <-chan struct{}) shardData { + span := trace.SpanFromContextSafe(ctx) + shardResult := shardData{ + index: vuid.index, + status: false, + } + + args := blobnode.RangeGetShardArgs{ + GetShardArgs: blobnode.GetShardArgs{ + DiskID: vuid.diskID, + Vuid: vuid.vuid, + Bid: blob.Bid, + }, + Offset: 0, + Size: int64(shardSize), + } + + var ( + err error + body io.ReadCloser + ) + if hErr := hystrix.Do(rwCommand, func() error { + body, err = h.getOneShardFromHost(ctx, serviceController, vuid.host, vuid.diskID, args, + vuid.index, clusterID, vid, 3, stopChan) + if err != nil && (errorTimeout(err) || rpc.DetectStatusCode(err) == errcode.CodeOverload) { + return err + } + return nil + }, nil); hErr != nil { + span.Warnf("hystrix: read blob(%d %d %d) on blobnode(%d %d %s) ecidx(%d): %s", + clusterID, vid, blob.Bid, vuid.vuid, vuid.diskID, vuid.host, vuid.index, hErr.Error()) + return shardResult + } + + if err != nil { + if err == errPunishedDisk || err == errCanceledReadShard { + span.Debugf("read blob(%d %d %d) on blobnode(%d %d %s) ecidx(%d): %s", + clusterID, vid, blob.Bid, vuid.vuid, vuid.diskID, vuid.host, vuid.index, err.Error()) + return shardResult + } + span.Warnf("read blob(%d %d %d) on blobnode(%d %d %s) ecidx(%d): %s", + clusterID, vid, blob.Bid, vuid.vuid, vuid.diskID, vuid.host, vuid.index, errors.Detail(err)) + return shardResult + } + defer body.Close() + + buf, err := h.memPool.Alloc(shardSize) + if err != nil { + span.Warn(err) + return shardResult + } + + _, err = io.ReadFull(body, buf) + if err != nil { + h.memPool.Put(buf) + span.Warnf("read blob(%d %d %d) on blobnode(%d %d %s) ecidx(%d): %s", + clusterID, vid, blob.Bid, + vuid.vuid, vuid.diskID, vuid.host, vuid.index, err.Error()) + return shardResult + } + + shardResult.status = true + shardResult.buffer = buf + return shardResult +} + +func (h *Handler) getDataShardOnly(ctx context.Context, getTime *timeReadWrite, + w io.Writer, serviceController controller.ServiceController, + clusterID proto.ClusterID, blob blobGetArgs) error { + span := trace.SpanFromContextSafe(ctx) + if blob.ReadSize == 0 { + return nil + } + + blobVolume, err := h.getVolume(ctx, clusterID, blob.Vid, true) + if err != nil { + return err + } + tactic := blobVolume.CodeMode.Tactic() + + from, to := int(blob.Offset), int(blob.Offset+blob.ReadSize) + buffer, err := ec.NewRangeBuffer(int(blob.BlobSize), from, to, tactic, h.memPool) + if err != nil { + return err + } + defer buffer.Release() + + shardSize := buffer.ShardSize + firstShardIdx := int(blob.Offset) / shardSize + shardOffset := int(blob.Offset) % shardSize + + startRead := time.Now() + remainSize := blob.ReadSize + bufOffset := 0 + for i, shard := range blobVolume.Units[firstShardIdx:tactic.N] { + if remainSize <= 0 { + break + } + + toReadSize := minU64(remainSize, uint64(shardSize-shardOffset)) + args := blobnode.RangeGetShardArgs{ + GetShardArgs: blobnode.GetShardArgs{ + DiskID: shard.DiskID, + Vuid: shard.Vuid, + Bid: blob.Bid, + }, + Offset: int64(shardOffset), + Size: int64(toReadSize), + } + + body, err := h.getOneShardFromHost(ctx, serviceController, shard.Host, shard.DiskID, args, + firstShardIdx+i, clusterID, blob.Vid, 1, nil) + if err != nil { + span.Warnf("read blob(%d %d %d) on blobnode(%d %d %s) ecidx(%d): %s", + clusterID, blob.Vid, blob.Bid, + shard.Vuid, shard.DiskID, shard.Host, firstShardIdx+i, errors.Detail(err)) + return errNeedReconstructRead + } + defer body.Close() + + buf := buffer.DataBuf[bufOffset : bufOffset+int(toReadSize)] + _, err = io.ReadFull(body, buf) + if err != nil { + span.Warn(err) + return errNeedReconstructRead + } + + // reset next shard offset + shardOffset = 0 + remainSize -= toReadSize + bufOffset += int(toReadSize) + } + getTime.IncR(time.Since(startRead)) + + if remainSize > 0 { + return fmt.Errorf("no enough data to read %d", remainSize) + } + + startWrite := time.Now() + if _, err := w.Write(buffer.DataBuf[:int(blob.ReadSize)]); err != nil { + getTime.IncW(time.Since(startWrite)) + return errors.Info(err, "write to response") + } + getTime.IncW(time.Since(startWrite)) + + return nil +} + +// getOneShardFromHost get body of one shard +func (h *Handler) getOneShardFromHost(ctx context.Context, serviceController controller.ServiceController, + host string, diskID proto.DiskID, args blobnode.RangeGetShardArgs, // get shard param with host diskid + index int, clusterID proto.ClusterID, vid proto.Vid, // param to update volume cache + attempts int, cancelChan <-chan struct{}, // do not retry again if cancelChan was closed +) (io.ReadCloser, error) { + span := trace.SpanFromContextSafe(ctx) + + // skip punished disk + if diskHost, err := serviceController.GetDiskHost(ctx, diskID); err != nil { + return nil, err + } else if diskHost.Punished { + return nil, errPunishedDisk + } + + var ( + rbody io.ReadCloser + rerr error + ) + rerr = retry.ExponentialBackoff(attempts, 200).RuptOn(func() (bool, error) { + if cancelChan != nil { + select { + case <-cancelChan: + return true, errCanceledReadShard + default: + } + } + + // new child span to get from blobnode, we should finish it here. + spanChild, ctxChild := trace.StartSpanFromContextWithTraceID( + context.Background(), "GetFromBlobnode", span.TraceID()) + defer spanChild.Finish() + + body, _, err := h.blobnodeClient.RangeGetShard(ctxChild, host, &args) + if err == nil { + rbody = body + return true, nil + } + + code := rpc.DetectStatusCode(err) + switch code { + case errcode.CodeOverload: + return true, err + + // EIO and Readonly error, then we need to punish disk in local and no need to retry + case errcode.CodeDiskBroken, errcode.CodeVUIDReadonly: + h.punishDisk(ctx, clusterID, diskID, host, "BrokenOrRO") + span.Infof("punish disk:%d on:%s cos:blobnode/%d", diskID, host, code) + return true, fmt.Errorf("punished disk (%d %s)", diskID, host) + + // vuid not found means the reflection between vuid and diskID has change, + // should refresh the blob volume cache + case errcode.CodeDiskNotFound, errcode.CodeVuidNotFound: + span.Infof("volume info outdated disk %d on host %s", diskID, host) + + latestVolume, e := h.getVolume(ctx, clusterID, vid, false) + if e != nil { + span.Warnf("update volume info with no cache %d %d err: %s", clusterID, vid, e) + return false, err + } + newUnit := latestVolume.Units[index] + + newDiskID := newUnit.DiskID + if newDiskID != diskID { + hi, e := serviceController.GetDiskHost(ctx, newDiskID) + if e == nil && !hi.Punished { + span.Infof("update disk %d %d %d -> %d", clusterID, vid, diskID, newDiskID) + + host = hi.Host + diskID = newDiskID + args.GetShardArgs.DiskID = diskID + args.GetShardArgs.Vuid = newUnit.Vuid + return false, err + } + } + + h.punishDiskWith(ctx, clusterID, diskID, host, "NotFound") + span.Debugf("punish threshold disk:%d cos:blobnode/%d", diskID, code) + } + + // do not retry on timeout then punish threshold this disk + if errorTimeout(err) { + h.punishDiskWith(ctx, clusterID, diskID, host, "Timeout") + return true, err + } + span.Debugf("read from disk:%d blobnode/%s", diskID, err.Error()) + + err = errors.Base(err, fmt.Sprintf("get shard on (%d %s)", diskID, host)) + return false, err + }) + + return rbody, rerr +} + +func genLocationBlobs(location *access.Location, readSize uint64, offset uint64) ([]blobGetArgs, error) { + if offset+readSize > location.Size { + return nil, fmt.Errorf("FileSize:%d ReadSize:%d Offset:%d", location.Size, readSize, offset) + } + + blobSize := uint64(location.BlobSize) + if blobSize <= 0 { + return nil, fmt.Errorf("BlobSize:%d", blobSize) + } + + remainSize := readSize + firstBlobIdx := offset / blobSize + blobOffset := offset % blobSize + + idx := uint64(0) + blobs := make([]blobGetArgs, 0, 1+(readSize+blobOffset)/blobSize) + for _, blob := range location.Blobs { + currBlobID := blob.MinBid + + for ii := uint32(0); ii < blob.Count; ii++ { + if remainSize <= 0 { + return blobs, nil + } + + if idx >= firstBlobIdx { + toReadSize := minU64(remainSize, blobSize-blobOffset) + if toReadSize > 0 { + blobs = append(blobs, blobGetArgs{ + Vid: blob.Vid, + Bid: currBlobID, + BlobSize: minU64(location.Size-idx*blobSize, blobSize), // update the last blob size + Offset: blobOffset, + ReadSize: toReadSize, + }) + } + + // reset next blob offset + blobOffset = 0 + remainSize -= toReadSize + } + + currBlobID++ + idx++ + } + } + + if remainSize > 0 { + return nil, fmt.Errorf("no enough data to read %d", remainSize) + } + + return blobs, nil +} + +func genSortedVuidByIDC(ctx context.Context, serviceController controller.ServiceController, idc string, + vuidPhys []controller.Unit) []sortedVuid { + span := trace.SpanFromContextSafe(ctx) + + vuids := make([]sortedVuid, 0, len(vuidPhys)) + sortMap := make(map[int][]sortedVuid) + + for idx, phy := range vuidPhys { + var hostIDC *controller.HostIDC + if err := retry.ExponentialBackoff(2, 100).On(func() error { + hi, e := serviceController.GetDiskHost(context.Background(), phy.DiskID) + if e != nil { + return e + } + hostIDC = hi + return nil + }); err != nil { + span.Warnf("no host of disk(%d %d) %s", phy.Vuid, phy.DiskID, err.Error()) + continue + } + + dis := distance(idc, hostIDC.IDC, hostIDC.Punished) + if _, ok := sortMap[dis]; !ok { + sortMap[dis] = make([]sortedVuid, 0, 8) + } + sortMap[dis] = append(sortMap[dis], sortedVuid{ + index: idx, + vuid: phy.Vuid, + diskID: phy.DiskID, + host: phy.Host, + }) + } + + keys := make([]int, 0, len(sortMap)) + for dis := range sortMap { + keys = append(keys, dis) + } + sort.Ints(keys) + + for _, dis := range keys { + ids := sortMap[dis] + rand.Shuffle(len(ids), func(i, j int) { + ids[i], ids[j] = ids[j], ids[i] + }) + vuids = append(vuids, ids...) + if dis > 1 { + span.Debugf("distance: %d punished vuids: %+v", dis, ids) + } + } + + return vuids +} + +func distance(idc1, idc2 string, punished bool) int { + if punished { + if idc1 == idc2 { + return 2 + } + return 3 + } + if idc1 == idc2 { + return 0 + } + return 1 +} + +func emptyDataShardIndexes(sizes ec.BufferSizes) map[int]struct{} { + firstEmptyIdx := (sizes.DataSize + sizes.ShardSize - 1) / sizes.ShardSize + n := sizes.ECDataSize / sizes.ShardSize + if firstEmptyIdx >= n { + return make(map[int]struct{}) + } + + set := make(map[int]struct{}, n-firstEmptyIdx) + for i := firstEmptyIdx; i < n; i++ { + set[i] = struct{}{} + } + + return set +} diff --git a/blobstore/access/stream_get_test.go b/blobstore/access/stream_get_test.go new file mode 100644 index 000000000..ca08b2746 --- /dev/null +++ b/blobstore/access/stream_get_test.go @@ -0,0 +1,566 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "bytes" + "crypto/rand" + mrand "math/rand" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/api/access" + "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +func TestAccessStreamGetBase(t *testing.T) { + ctx := ctxWithName("TestAccessStreamGetBase") + // error + { + dataShards.clean() + data := []byte("x") + loc, err := streamer.Put(ctx(), bytes.NewReader(data), int64(len(data)), nil) + require.NoError(t, err) + + buff := bytes.NewBuffer(nil) + _, err = streamer.Get(ctx(), buff, *loc, 1, 1) + require.NotNil(t, err) + _, err = streamer.Get(ctx(), buff, *loc, 2, 0) + require.NotNil(t, err) + } + // 1 byte + { + dataShards.clean() + data := []byte("x") + loc, err := streamer.Put(ctx(), bytes.NewReader(data), int64(len(data)), nil) + require.NoError(t, err) + + buff := bytes.NewBuffer(nil) + transfer, err := streamer.Get(ctx(), buff, *loc, 1, 0) + require.NoError(t, err) + err = transfer() + require.NoError(t, err) + require.True(t, dataEqual(data, buff.Bytes())) + + _, err = streamer.Get(ctx(), nil, *loc, 0, 1) + require.NoError(t, err) + } + + vuidController.SetBNRealError(true) + defer func() { + vuidController.SetBNRealError(false) + }() + + cases := []struct { + size int + }{ + {12}, + {1 << 12}, + {(1 << 13) + 777}, + {1 << 22}, + {(1 << 22) + 1}, + {(1 << 22) + 1023}, + {1 << 23}, + {(1 << 23) + 1025}, + } + for _, cs := range cases { + dataShards.clean() + size := cs.size + data := make([]byte, size) + rand.Read(data) + loc, err := streamer.Put(ctx(), bytes.NewReader(data), int64(size), nil) + require.NoError(t, err) + + buff := bytes.NewBuffer(nil) + transfer, err := streamer.Get(ctx(), buff, *loc, uint64(size), 0) + require.NoError(t, err) + err = transfer() + require.NoError(t, err) + require.True(t, dataEqual(data, buff.Bytes())) + + // time wait the punished services + time.Sleep(time.Second * time.Duration(punishServiceS)) + } + + dataShards.clean() +} + +func TestAccessStreamGetBroken(t *testing.T) { + ctx := ctxWithName("TestAccessStreamGetBroken") + defer func() { + dataShards.clean() + for ii := 0; ii < len(allID); ii++ { + vuidController.Unbreak(proto.Vuid(allID[ii])) + } + vuidController.Break(1005) + }() + + dataShards.clean() + tactic := codemode.EC6P6.Tactic() + size := tactic.N * tactic.MinShardSize + data := make([]byte, size) + rand.Read(data) + // time wait the punished services + time.Sleep(time.Second * time.Duration(punishServiceS)) + loc, err := streamer.Put(ctx(), bytes.NewReader(data), int64(size), nil) + require.NoError(t, err) + + cases := []struct { + id proto.Vuid + hasError bool + }{ + {1001, false}, + {1002, false}, + {1003, false}, + {1004, false}, + {1005, false}, + {1006, false}, + {1007, true}, + {1008, true}, + } + + for _, cs := range cases { + vuidController.Break(cs.id) + + buff := bytes.NewBuffer(nil) + transfer, err := streamer.Get(ctx(), buff, *loc, uint64(size), 0) + require.Nil(t, err) + err = transfer() + if cs.hasError { + require.NotNil(t, err) + } else { + require.NoError(t, err) + require.True(t, dataEqual(data, buff.Bytes())) + } + } +} + +func TestAccessStreamGetOffset(t *testing.T) { + ctx := ctxWithName("TestAccessStreamGetOffset") + vuidController.Unbreak(1005) + defer vuidController.Break(1005) + + cases := []struct { + size int64 + offset uint64 + readSize uint64 + }{ + {12, 0, 12}, + {12, 1, 11}, + {12, 1, 1}, + {12, 1, 4}, + {12, 1, 7}, + {(1 << 22) + 1, 0, 1 << 22}, + {(1 << 22) + 1, 1 << 20, 1 << 20}, + {(1 << 22) + 1, 1 << 22, 1}, + {(1 << 22) + 1023, 0, 1}, + {(1 << 22) + 1023, 1 << 22, 1022}, + {(1 << 22) + 1023, 1 << 22, 1023}, + {12192823, 6799138, 908019}, + } + for _, cs := range cases { + dataShards.clean() + size := cs.size + data := make([]byte, size) + rand.Read(data) + loc, err := streamer.Put(ctx(), bytes.NewReader(data), size, nil) + require.NoError(t, err) + + buff := bytes.NewBuffer(nil) + transfer, _ := streamer.Get(ctx(), buff, *loc, cs.readSize, cs.offset) + err = transfer() + require.NoError(t, err) + require.True(t, dataEqual(data[cs.offset:cs.offset+cs.readSize], buff.Bytes())) + } + + dataShards.clean() +} + +func TestAccessStreamGetShardTimeout(t *testing.T) { + ctx := ctxWithName("TestAccessStreamGetShardTimeout") + dataShards.clean() + vuidController.Unbreak(1005) + defer func() { + vuidController.Break(1005) + dataShards.clean() + }() + + size := 1 << 22 + buff := make([]byte, size) + rand.Read(buff) + loc, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil) + require.NoError(t, err) + + // no delay when blocking one shard, cos MinReadShardsX = 1 + vuidController.Block(1001) + defer func() { + vuidController.Unblock(1001) + }() + { + startTime := time.Now() + transfer, _ := streamer.Get(ctx(), bytes.NewBuffer(nil), *loc, uint64(size), 0) + err := transfer() + require.NoError(t, err) + + duration := time.Since(startTime) + require.GreaterOrEqual(t, vuidController.duration, duration, "greater duration: ", duration) + } + + // delay one duration when blocking two shard, cos MinReadShardsX = 1 + vuidController.Block(1002) + defer func() { + vuidController.Unblock(1002) + }() + { + startTime := time.Now() + transfer, _ := streamer.Get(ctx(), bytes.NewBuffer(nil), *loc, uint64(size), 0) + err = transfer() + require.NoError(t, err) + + duration := time.Since(startTime) + minDuration := vuidController.duration + maxDuration := minDuration + minDuration/2 + t.Log(duration, minDuration, maxDuration) + require.LessOrEqual(t, minDuration, duration, "less duration: ", duration) + require.GreaterOrEqual(t, maxDuration, duration, "greater duration: ", duration) + } +} + +func TestAccessStreamGetShardBroken(t *testing.T) { + ctx := ctxWithName("TestAccessStreamGetShardBroken") + dataShards.clean() + vuidController.Unbreak(1005) + defer func() { + vuidController.Break(1005) + dataShards.clean() + }() + + size := 1 << 22 + buff := make([]byte, size) + rand.Read(buff) + loc, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil) + require.NoError(t, err) + + // no delay when blocking one shard, cos MinReadShardsX = 1 + // it will not wait the blocking shard, cos has no enough shards to reconstruct + vuidController.Block(1001) + for _, id := range allID[1:] { + vuidController.Break(proto.Vuid(id)) + } + defer func() { + vuidController.Unblock(1001) + for _, id := range allID[1:] { + vuidController.Unbreak(proto.Vuid(id)) + } + }() + { + startTime := time.Now() + transfer, err := streamer.Get(ctx(), bytes.NewBuffer(nil), *loc, uint64(size), 0) + require.NoError(t, err) + err = transfer() + require.Error(t, err) + + duration := time.Since(startTime) + require.GreaterOrEqual(t, vuidController.duration, duration, "greater duration: ", duration) + } +} + +func TestAccessStreamGetLocalIDC(t *testing.T) { + ctx := ctxWithName("TestAccessStreamGetLocalIDC") + dataShards.clean() + vuidController.Unbreak(1005) + defer func() { + vuidController.Break(1005) + dataShards.clean() + }() + + size := 1 << 22 + buff := make([]byte, size) + rand.Read(buff) + loc, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil) + require.NoError(t, err) + + // no delay when blocking other idc all shards + func() { + for _, id := range idcOtherID { + vuidController.Block(proto.Vuid(id)) + } + defer func() { + for _, id := range idcOtherID { + vuidController.Unblock(proto.Vuid(id)) + } + }() + startTime := time.Now() + transfer, _ := streamer.Get(ctx(), bytes.NewBuffer(nil), *loc, uint64(size), 0) + err = transfer() + require.NoError(t, err) + + duration := time.Since(startTime) + require.GreaterOrEqual(t, vuidController.duration, duration, "greater duration: ", duration) + }() + + // max delay one duration when other local has two blocked shard, cos MinReadShardsX = 1 + vuidController.Break(proto.Vuid(idcID[0])) + defer func() { + vuidController.Unbreak(proto.Vuid(idcID[0])) + }() + func() { + vuidController.Block(proto.Vuid(idcOtherID[0])) + vuidController.Block(proto.Vuid(idcOtherID[1])) + defer func() { + vuidController.Unblock(proto.Vuid(idcOtherID[0])) + vuidController.Unblock(proto.Vuid(idcOtherID[1])) + }() + startTime := time.Now() + transfer, _ := streamer.Get(ctx(), bytes.NewBuffer(nil), *loc, uint64(size), 0) + err = transfer() + require.NoError(t, err) + + duration := time.Since(startTime) + if duration < vuidController.duration { + return + } + minDuration := vuidController.duration + maxDuration := minDuration + minDuration/2 + t.Log(duration, minDuration, maxDuration) + require.LessOrEqual(t, minDuration, duration, "less duration: ", duration) + require.GreaterOrEqual(t, maxDuration, duration, "greater duration: ", duration) + }() +} + +func TestAccessStreamGetAligned(t *testing.T) { + ctx := ctxWithName("TestAccessStreamGetAligned") + defer func() { + dataShards.clean() + for ii := 0; ii < len(allID); ii++ { + vuidController.Unbreak(proto.Vuid(allID[ii])) + } + vuidController.Break(1005) + }() + vuidController.Unbreak(1005) + + shardSize := codemode.EC6P6.Tactic().MinShardSize + cases := []struct { + size int + goodShards int + succ bool + }{ + {1, 0, false}, + {1, 1, true}, + {1, 2, true}, + + {shardSize - 1, 1, true}, + {shardSize, 1, true}, + {shardSize + 1, 1, false}, + {shardSize + 1, 2, true}, + + {shardSize * 2, 1, false}, + {shardSize * 2, 2, true}, + {shardSize * 3, 2, false}, + {shardSize * 3, 3, true}, + {shardSize * 4, 3, false}, + {shardSize * 4, 4, true}, + {shardSize * 5, 4, false}, + {shardSize * 5, 5, true}, + {shardSize * 6, 5, false}, + {shardSize * 6, 6, true}, + } + + randomGoodShards := func(n int) { + for ii := 0; ii < len(allID); ii++ { + vuidController.Break(proto.Vuid(allID[ii])) + } + + shards := make([]int, 0, len(allID)) + shards = append(shards, allID[6:]...) + shards = append(shards, allID[:n]...) + + mrand.Shuffle(len(shards), func(i, j int) { + shards[i], shards[j] = shards[j], shards[i] + }) + for ii := 0; ii < n; ii++ { + vuidController.Unbreak(proto.Vuid(shards[ii])) + } + } + + for _, cs := range cases { + dataShards.clean() + + data := make([]byte, cs.size) + rand.Read(data) + loc, err := streamer.Put(ctx(), bytes.NewReader(data), int64(cs.size), nil) + require.NoError(t, err) + + // cos put shards asynchronously, should wait all shard written + time.Sleep(200 * time.Millisecond) + + randomGoodShards(cs.goodShards) + + buff := bytes.NewBuffer(nil) + transfer, err := streamer.Get(ctx(), buff, *loc, uint64(cs.size), 0) + require.NoError(t, err) + + err = transfer() + if cs.succ { + require.NoError(t, err) + require.True(t, dataEqual(data, buff.Bytes())) + } else { + require.Error(t, err) + } + + for ii := 0; ii < len(allID); ii++ { + vuidController.Unbreak(proto.Vuid(allID[ii])) + } + } +} + +func TestAccessStreamGenLocationBlobs(t *testing.T) { + firstSliceStart := proto.BlobID(100) + secondSliceStart := proto.BlobID(200) + + loc := access.Location{ + ClusterID: 0, + Size: 1024*4 + 37 + 1024*2, // 5 fine blobs and 2 missing blobs + BlobSize: 1024, + Blobs: []access.SliceInfo{ + { + MinBid: firstSliceStart, + Vid: proto.Vid(1001), + Count: 3, + }, + { + MinBid: secondSliceStart, + Vid: proto.Vid(2001), + Count: 2, + }, + }, + } + + type blobArgs = blobGetArgs + cases := []struct { + readSize, offset uint64 + err bool + checker func([]blobArgs) bool + }{ + {1024 * 7, 0, true, func(blobs []blobArgs) bool { return blobs == nil }}, + {1024 * 6, 38, true, func(blobs []blobArgs) bool { return blobs == nil }}, + {1024 * 5, 38, true, func(blobs []blobArgs) bool { return blobs == nil }}, + + {0, 0, false, func(blobs []blobArgs) bool { + return len(blobs) == 0 + }}, + {1, 0, false, func(blobs []blobArgs) bool { + return len(blobs) == 1 && blobs[0].Bid == firstSliceStart + }}, + {1024, 0, false, func(blobs []blobArgs) bool { + return len(blobs) == 1 && blobs[0].Bid == firstSliceStart + }}, + {1025, 0, false, func(blobs []blobArgs) bool { + return len(blobs) == 2 && blobs[1].Bid == firstSliceStart+1 + }}, + {1024*4 + 37, 0, false, func(blobs []blobArgs) bool { + return len(blobs) == 5 + }}, + + {0, 1024, false, func(blobs []blobArgs) bool { + return len(blobs) == 0 + }}, + {1, 1023, false, func(blobs []blobArgs) bool { + return len(blobs) == 1 && blobs[0].Bid == firstSliceStart + }}, + {1, 1024, false, func(blobs []blobArgs) bool { + return len(blobs) == 1 && blobs[0].Bid == firstSliceStart+1 + }}, + {1024, 1024, false, func(blobs []blobArgs) bool { + return len(blobs) == 1 && blobs[0].Bid == firstSliceStart+1 + }}, + {1024 * 2, 1024 + 1, false, func(blobs []blobArgs) bool { + return len(blobs) == 3 && blobs[0].Bid == firstSliceStart+1 && + blobs[0].Offset == 1 && blobs[0].ReadSize == 1023 && + blobs[1].Offset == 0 && blobs[1].ReadSize == 1024 && + blobs[2].Offset == 0 && blobs[2].ReadSize == 1 + }}, + + {1, 1024 * 4, false, func(blobs []blobArgs) bool { + return len(blobs) == 1 && blobs[0].Bid == secondSliceStart+1 && + blobs[0].Offset == 0 && blobs[0].ReadSize == 1 + }}, + {1, 1024*4 + 1, false, func(blobs []blobArgs) bool { + return len(blobs) == 1 && blobs[0].Bid == secondSliceStart+1 && + blobs[0].Offset == 1 && blobs[0].ReadSize == 1 + }}, + {36, 1024*4 + 1, false, func(blobs []blobArgs) bool { + return len(blobs) == 1 && blobs[0].Bid == secondSliceStart+1 && + blobs[0].Offset == 1 && blobs[0].ReadSize == 36 + }}, + } + + for _, cs := range cases { + blobs, err := genLocationBlobs(&loc, cs.readSize, cs.offset) + if cs.err { + require.True(t, err != nil) + } else { + require.True(t, err == nil) + } + require.True(t, cs.checker(blobs)) + } +} + +type writer struct { + buf []byte +} + +func (w *writer) Write(p []byte) (n int, err error) { + copy(w.buf, p) + return len(p), nil +} + +func BenchmarkAccessStreamGet(b *testing.B) { + ctx := ctxWithName("BenchmarkAccessStreamGet")() + vuidController.Unbreak(1005) + defer func() { + vuidController.Break(1005) + dataShards.clean() + }() + + cases := []struct { + name string + size int + }{ + {"1B", 1}, + {"1KB", 1 << 10}, + {"1MB", 1 << 20}, + {"4MB", 1 << 22}, + {"8MB", 1 << 23}, + } + + w := &writer{buf: make([]byte, 1<<23)} + for _, cs := range cases { + b.ResetTimer() + b.Run(cs.name, func(b *testing.B) { + loc, err := streamer.Put(ctx, newReader(cs.size), int64(cs.size), nil) + require.NoError(b, err) + + b.ResetTimer() + for ii := 0; ii <= b.N; ii++ { + transfer, _ := streamer.Get(ctx, w, *loc, uint64(cs.size), 0) + transfer() + } + }) + } +} diff --git a/blobstore/access/stream_loop.go b/blobstore/access/stream_loop.go new file mode 100644 index 000000000..ba79f8dae --- /dev/null +++ b/blobstore/access/stream_loop.go @@ -0,0 +1,121 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "context" + "time" + + "github.com/cubefs/cubefs/blobstore/api/proxy" + "github.com/cubefs/cubefs/blobstore/common/memcache" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/util/errors" +) + +type ( + vidHostKey struct { + cid proto.ClusterID + vid proto.Vid + } +) + +// cacheVidAllocator memory cache of vid in allocator host +var cacheVidAllocator *memcache.MemCache + +func init() { + mc, err := memcache.NewMemCache(context.Background(), 1<<15) + if err != nil { + panic(err) + } + cacheVidAllocator = mc +} + +func setCacheVidHost(cid proto.ClusterID, vid proto.Vid, host string) { + cacheVidAllocator.Set(vidHostKey{cid: cid, vid: vid}, host) +} + +func getCacheVidHost(cid proto.ClusterID, vid proto.Vid) (string, error) { + val := cacheVidAllocator.Get(vidHostKey{cid: cid, vid: vid}) + if val == nil { + return "", errors.Newf("not found host of (%d %d)", cid, vid) + } + host, ok := val.(string) + if !ok { + return "", errors.Newf("not string host of (%d %d)", cid, vid) + } + return host, nil +} + +func (h *Handler) loopDiscardVids() { + go func() { + cache := make(map[discardVid]struct{}) + + duration := time.Second * 3 + timer := time.NewTimer(duration) + defer timer.Stop() + + for { + flush := false + select { + case <-h.stopCh: + return + case dv := <-h.discardVidChan: + cache[dv] = struct{}{} + if len(cache) >= 8 { + flush = true + } + case <-timer.C: + if len(cache) > 0 { + flush = true + } + timer.Reset(duration) + } + + if !flush { + continue + } + + for dv := range cache { + go func(dv discardVid) { + h.tryDiscardVidOnAllocator(dv.cid, dv.vid, proxy.AllocVolsArgs{ + Fsize: 1, + BidCount: 1, + CodeMode: dv.codeMode, + Discards: []proto.Vid{dv.vid}, + }) + }(dv) + delete(cache, dv) + } + } + }() +} + +func (h *Handler) tryDiscardVidOnAllocator(cid proto.ClusterID, vid proto.Vid, args proxy.AllocVolsArgs) { + span, ctx := trace.StartSpanFromContext(context.Background(), "") + + host, err := getCacheVidHost(cid, vid) + if err != nil { + span.Warnf("to discard vids %+v : %v", args, err) + return + } + + span.Infof("to post on %s discard vids %+v", host, args) + + _, err = h.proxyClient.VolumeAlloc(ctx, host, &args) + if err != nil { + span.Warnf("post on %s discard vids %+v : %v", host, args, err) + } +} diff --git a/blobstore/access/stream_mock_test.go b/blobstore/access/stream_mock_test.go new file mode 100644 index 000000000..710b9957e --- /dev/null +++ b/blobstore/access/stream_mock_test.go @@ -0,0 +1,551 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +// github.com/cubefs/cubefs/blobstore/access/... module access interfaces +//go:generate mockgen -destination=./controller_mock_test.go -package=access -mock_names ClusterController=MockClusterController,ServiceController=MockServiceController,VolumeGetter=MockVolumeGetter github.com/cubefs/cubefs/blobstore/access/controller ClusterController,ServiceController,VolumeGetter +//go:generate mockgen -destination=./access_mock_test.go -package=access -mock_names StreamHandler=MockStreamHandler,Limiter=MockLimiter github.com/cubefs/cubefs/blobstore/access StreamHandler,Limiter + +import ( + "bytes" + "context" + "errors" + "fmt" + "hash/crc32" + "io" + "io/ioutil" + "math" + "math/rand" + "strconv" + "sync" + "testing" + "time" + + "github.com/afex/hystrix-go/hystrix" + "github.com/alicebob/miniredis/v2" + "github.com/golang/mock/gomock" + + "github.com/cubefs/cubefs/blobstore/access/controller" + "github.com/cubefs/cubefs/blobstore/api/blobnode" + "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/api/proxy" + "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/cubefs/cubefs/blobstore/common/ec" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/redis" + "github.com/cubefs/cubefs/blobstore/common/resourcepool" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/testing/mocks" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +var ( + errNotFound = errors.New("not found") + errAllocTimeout = errors.New("alloc timeout") + + allocTimeoutSize uint64 = 1 << 40 + punishServiceS = 1 + + idc = "test-idc" + idcOther = "test-idc-other" + allID = []int{1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012} + idcID = []int{1001, 1002, 1003, 1007, 1008, 1009} + idcOtherID = []int{1004, 1005, 1006, 1010, 1011, 1012} + + clusterID = proto.ClusterID(1) + volumeID = proto.Vid(1) + blobSize = 1 << 22 + + streamer *Handler + + memPool *resourcepool.MemPool + encoder map[codemode.CodeMode]ec.Encoder + proxyClient proxy.Client + + allCodeModes CodeModePairs + + redismr *miniredis.Miniredis + rediscli *redis.ClusterClient + + cmcli clustermgr.APIAccess + volumeGetter controller.VolumeGetter + serviceController controller.ServiceController + cc controller.ClusterController + + clusterInfo *clustermgr.ClusterInfo + dataVolume *clustermgr.VolumeInfo + dataAllocs []proxy.AllocRet + dataNodes map[string]clustermgr.ServiceInfo + dataDisks map[proto.DiskID]blobnode.DiskInfo + dataShards *shardsData + + vuidController *vuidControl + + putErrors = []errcode.Error{ + errcode.ErrDiskBroken, errcode.ErrReadonlyVUID, + errcode.ErrChunkNoSpace, + errcode.ErrNoSuchDisk, errcode.ErrNoSuchVuid, + } + getErrors = []errcode.Error{ + errcode.ErrOverload, + errcode.ErrDiskBroken, errcode.ErrReadonlyVUID, + errcode.ErrNoSuchDisk, errcode.ErrNoSuchVuid, + } +) + +type shardKey struct { + Vuid proto.Vuid + Bid proto.BlobID +} + +type shardsData struct { + mutex sync.RWMutex + data map[shardKey][]byte +} + +func (d *shardsData) clean() { + d.mutex.Lock() + for key := range d.data { + d.data[key] = d.data[key][:0] + } + d.mutex.Unlock() +} + +func (d *shardsData) get(vuid proto.Vuid, bid proto.BlobID) []byte { + key := shardKey{Vuid: vuid, Bid: bid} + d.mutex.RLock() + data := d.data[key] + buff := make([]byte, len(data)) + copy(buff, data) + d.mutex.RUnlock() + return buff +} + +func (d *shardsData) set(vuid proto.Vuid, bid proto.BlobID, b []byte) { + key := shardKey{Vuid: vuid, Bid: bid} + d.mutex.Lock() + old := d.data[key] + if cap(old) <= len(b) { + d.data[key] = make([]byte, len(b)) + } else { + d.data[key] = old[:len(b)] + } + copy(d.data[key], b) + d.mutex.Unlock() +} + +type vuidControl struct { + mutex sync.Mutex + broken map[proto.Vuid]bool + blocked map[proto.Vuid]bool + block func() + duration time.Duration + + isBNRealError bool // is return blobnode real error +} + +func (c *vuidControl) Break(id proto.Vuid) { + c.mutex.Lock() + c.broken[id] = true + c.mutex.Unlock() +} + +func (c *vuidControl) Unbreak(id proto.Vuid) { + c.mutex.Lock() + delete(c.broken, id) + c.mutex.Unlock() +} + +func (c *vuidControl) Isbroken(id proto.Vuid) bool { + c.mutex.Lock() + v, ok := c.broken[id] + c.mutex.Unlock() + return ok && v +} + +func (c *vuidControl) Block(id proto.Vuid) { + c.mutex.Lock() + c.blocked[id] = true + c.mutex.Unlock() +} + +func (c *vuidControl) Unblock(id proto.Vuid) { + c.mutex.Lock() + delete(c.blocked, id) + c.mutex.Unlock() +} + +func (c *vuidControl) Isblocked(id proto.Vuid) bool { + c.mutex.Lock() + v, ok := c.blocked[id] + c.mutex.Unlock() + return ok && v +} + +func (c *vuidControl) SetBNRealError(b bool) { + c.mutex.Lock() + c.isBNRealError = b + c.mutex.Unlock() +} + +func (c *vuidControl) IsBNRealError() bool { + c.mutex.Lock() + defer c.mutex.Unlock() + return c.isBNRealError +} + +func randBlobnodeRealError(errors []errcode.Error) error { + n := rand.Intn(1024) % len(errors) + return errors[n] +} + +var storageAPIRangeGetShard = func(ctx context.Context, host string, args *blobnode.RangeGetShardArgs) ( + body io.ReadCloser, shardCrc uint32, err error) { + if vuidController.Isbroken(args.Vuid) { + err = errors.New("get shard fake error") + if vuidController.IsBNRealError() { + err = randBlobnodeRealError(getErrors) + } + return + } + if vuidController.Isblocked(args.Vuid) { + vuidController.block() + if rand.Intn(2) == 0 { + err = errors.New("get shard timeout") + } else { + err = errors.New("get shard Timeout") + } + return + } + + buff := dataShards.get(args.Vuid, args.Bid) + if len(buff) == 0 { + return nil, 0, errNotFound + } + if len(buff) < int(args.Offset+args.Size) { + err = errors.New("get shard concurrently") + return + } + + buff = buff[int(args.Offset):int(args.Offset+args.Size)] + shardCrc = crc32.ChecksumIEEE(buff) + body = ioutil.NopCloser(bytes.NewReader(buff)) + return +} + +var storageAPIPutShard = func(ctx context.Context, host string, args *blobnode.PutShardArgs) ( + crc uint32, err error) { + if vuidController.Isbroken(args.Vuid) { + err = errors.New("put shard fake error") + if vuidController.IsBNRealError() { + err = randBlobnodeRealError(putErrors) + } + return + } + if vuidController.Isblocked(args.Vuid) { + vuidController.block() + err = errors.New("put shard timeout") + return + } + + buffer, _ := memPool.Alloc(int(args.Size)) + defer memPool.Put(buffer) + + buffer = buffer[:int(args.Size)] + _, err = io.ReadFull(args.Body, buffer) + if err != nil { + return + } + + crc = crc32.ChecksumIEEE(buffer) + dataShards.set(args.Vuid, args.Bid, buffer) + return +} + +func initMockData() { + dataAllocs = make([]proxy.AllocRet, 2) + dataAllocs[0] = proxy.AllocRet{ + BidStart: 10000, + BidEnd: 10000, + Vid: volumeID, + } + dataAllocs[1] = proxy.AllocRet{ + BidStart: 20000, + BidEnd: 50000, + Vid: volumeID, + } + + dataVolume = &clustermgr.VolumeInfo{ + VolumeInfoBase: clustermgr.VolumeInfoBase{ + Vid: volumeID, + CodeMode: codemode.EC6P6, + }, + Units: func() (units []clustermgr.Unit) { + for _, id := range allID { + units = append(units, clustermgr.Unit{ + Vuid: proto.Vuid(id), + DiskID: proto.DiskID(id), + Host: strconv.Itoa(id), + }) + } + return + }(), + } + + proxyNodes := make([]clustermgr.ServiceNode, 32) + for idx := range proxyNodes { + proxyNodes[idx] = clustermgr.ServiceNode{ + ClusterID: 1, + Name: serviceProxy, + Host: fmt.Sprintf("proxy-%d", idx), + Idc: idc, + } + } + + dataNodes = make(map[string]clustermgr.ServiceInfo) + dataNodes[serviceProxy] = clustermgr.ServiceInfo{ + Nodes: proxyNodes, + } + + dataDisks = make(map[proto.DiskID]blobnode.DiskInfo) + for _, id := range idcID { + dataDisks[proto.DiskID(id)] = blobnode.DiskInfo{ + ClusterID: clusterID, Idc: idc, Host: strconv.Itoa(id), + DiskHeartBeatInfo: blobnode.DiskHeartBeatInfo{DiskID: proto.DiskID(id)}, + } + } + for _, id := range idcOtherID { + dataDisks[proto.DiskID(id)] = blobnode.DiskInfo{ + ClusterID: clusterID, Idc: idcOther, Host: strconv.Itoa(id), + DiskHeartBeatInfo: blobnode.DiskHeartBeatInfo{DiskID: proto.DiskID(id)}, + } + } + + dataShards = &shardsData{ + data: make(map[shardKey][]byte, len(allID)), + } + dataShards.clean() + + ctr := gomock.NewController(&testing.T{}) + cli := mocks.NewMockClientAPI(ctr) + cli.EXPECT().GetService(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( + func(ctx context.Context, args clustermgr.GetServiceArgs) (clustermgr.ServiceInfo, error) { + if val, ok := dataNodes[args.Name]; ok { + return val, nil + } + return clustermgr.ServiceInfo{}, errNotFound + }) + cli.EXPECT().GetVolumeInfo(gomock.Any(), gomock.Any()).AnyTimes().Return(dataVolume, nil) + cli.EXPECT().DiskInfo(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( + func(ctx context.Context, id proto.DiskID) (*blobnode.DiskInfo, error) { + if val, ok := dataDisks[id]; ok { + return &val, nil + } + return nil, errNotFound + }) + cmcli = cli + + redismr, _ = miniredis.Run() + if rand.Int()%2 == 0 { + rediscli = redis.NewClusterClient(&redis.ClusterConfig{ + Addrs: []string{redismr.Addr()}, + }) + } + clusterInfo = &clustermgr.ClusterInfo{ + Region: "test-region", + ClusterID: clusterID, + Nodes: []string{"node-1", "node-2", "node-3"}, + } + serviceController, _ = controller.NewServiceController( + controller.ServiceConfig{ + ClusterID: clusterID, + IDC: idc, + ReloadSec: 1000, + }, cmcli) + volumeGetter, _ = controller.NewVolumeGetter(clusterID, cmcli, rediscli, 0) + + ctr = gomock.NewController(&testing.T{}) + c := NewMockClusterController(ctr) + c.EXPECT().Region().AnyTimes().Return("test-region") + c.EXPECT().ChooseOne().AnyTimes().Return(clusterInfo, nil) + c.EXPECT().GetServiceController(gomock.Any()).AnyTimes().Return(serviceController, nil) + c.EXPECT().GetVolumeGetter(gomock.Any()).AnyTimes().Return(volumeGetter, nil) + c.EXPECT().ChangeChooseAlg(gomock.Any()).AnyTimes().DoAndReturn( + func(alg controller.AlgChoose) error { + if alg < 10 { + return nil + } + return controller.ErrInvalidChooseAlg + }) + cc = c + + ctr = gomock.NewController(&testing.T{}) + allocCli := mocks.NewMockProxyClient(ctr) + allocCli.EXPECT().SendDeleteMsg(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil) + allocCli.EXPECT().SendShardRepairMsg(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil) + allocCli.EXPECT().VolumeAlloc(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( + func(ctx context.Context, host string, args *proxy.AllocVolsArgs) ([]proxy.AllocRet, error) { + if args.Fsize > allocTimeoutSize { + return nil, errAllocTimeout + } + return dataAllocs, nil + }) + proxyClient = allocCli +} + +func initPool() { + memPool = resourcepool.NewMemPool(getDefaultMempoolSize()) +} + +func initEncoder() { + coderEC6P6, _ := ec.NewEncoder(ec.Config{ + CodeMode: codemode.EC6P6.Tactic(), + EnableVerify: true, + }) + coderEC6P10L2, _ := ec.NewEncoder(ec.Config{ + CodeMode: codemode.EC6P10L2.Tactic(), + EnableVerify: true, + }) + coderEC15P12, _ := ec.NewEncoder(ec.Config{ + CodeMode: codemode.EC15P12.Tactic(), + EnableVerify: true, + }) + coderEC16P20L2, _ := ec.NewEncoder(ec.Config{ + CodeMode: codemode.EC16P20L2.Tactic(), + EnableVerify: true, + }) + encoder = map[codemode.CodeMode]ec.Encoder{ + codemode.EC6P6: coderEC6P6, + codemode.EC6P10L2: coderEC6P10L2, + codemode.EC15P12: coderEC15P12, + codemode.EC16P20L2: coderEC16P20L2, + } +} + +func initEC() { + allCodeModes = CodeModePairs{ + codemode.EC6P6: CodeModePair{ + Policy: codemode.Policy{ + ModeName: codemode.EC6P6.Name(), + MaxSize: math.MaxInt64, + Enable: true, + }, + Tactic: codemode.EC6P6.Tactic(), + }, + codemode.EC6P10L2: CodeModePair{ + Policy: codemode.Policy{ + ModeName: codemode.EC6P10L2.Name(), + MaxSize: -1, + }, + Tactic: codemode.EC6P10L2.Tactic(), + }, + codemode.EC15P12: CodeModePair{ + Policy: codemode.Policy{ + ModeName: codemode.EC15P12.Name(), + MaxSize: -1, + }, + Tactic: codemode.EC15P12.Tactic(), + }, + codemode.EC16P20L2: CodeModePair{ + Policy: codemode.Policy{ + ModeName: codemode.EC16P20L2.Name(), + MaxSize: -1, + }, + Tactic: codemode.EC16P20L2.Tactic(), + }, + } +} + +func initController() { + vuidController = &vuidControl{ + broken: make(map[proto.Vuid]bool), + blocked: make(map[proto.Vuid]bool), + block: func() { + time.Sleep(2 * time.Second) + }, + duration: 2 * time.Second, + isBNRealError: false, + } + // initialized broken 1005 + vuidController.Break(1005) +} + +func newMockStorageAPI() blobnode.StorageAPI { + ctr := gomock.NewController(&testing.T{}) + api := mocks.NewMockStorageAPI(ctr) + api.EXPECT().RangeGetShard(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes(). + DoAndReturn(storageAPIRangeGetShard) + api.EXPECT().PutShard(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes(). + DoAndReturn(storageAPIPutShard) + return api +} + +func init() { + rand.Seed(int64(time.Now().Nanosecond())) + log.SetOutputLevel(log.Lfatal) + + hystrix.ConfigureCommand(rwCommand, hystrix.CommandConfig{ + Timeout: 9000, + MaxConcurrentRequests: 9000, + ErrorPercentThreshold: 90, + }) + + initPool() + initEncoder() + initEC() + initMockData() + initController() + + streamer = &Handler{ + memPool: memPool, + encoder: encoder, + clusterController: cc, + + blobnodeClient: newMockStorageAPI(), + proxyClient: proxyClient, + + allCodeModes: allCodeModes, + maxObjectSize: defaultMaxObjectSize, + StreamConfig: StreamConfig{ + IDC: idc, + MaxBlobSize: uint32(blobSize), // 4M + DiskPunishIntervalS: punishServiceS, + ServicePunishIntervalS: punishServiceS, + AllocRetryTimes: 3, + AllocRetryIntervalMS: 3000, + MinReadShardsX: defaultMinReadShardsX, + }, + discardVidChan: make(chan discardVid, 8), + stopCh: make(chan struct{}), + } + streamer.loopDiscardVids() +} + +func ctxWithName(funcName string) func() context.Context { + return func() context.Context { + _, ctx := trace.StartSpanFromContextWithTraceID(context.Background(), funcName, funcName) + return ctx + } +} + +func getBufSizes(size int) ec.BufferSizes { + sizes, _ := ec.GetBufferSizes(size, codemode.EC6P6.Tactic()) + return sizes +} + +func dataEqual(exp, act []byte) bool { + return crc32.ChecksumIEEE(exp) == crc32.ChecksumIEEE(act) +} diff --git a/blobstore/access/stream_put.go b/blobstore/access/stream_put.go new file mode 100644 index 000000000..ce110a8cb --- /dev/null +++ b/blobstore/access/stream_put.go @@ -0,0 +1,442 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "bytes" + "context" + "fmt" + "hash/crc32" + "io" + "sync" + "sync/atomic" + "time" + + "github.com/afex/hystrix-go/hystrix" + + "github.com/cubefs/cubefs/blobstore/api/access" + "github.com/cubefs/cubefs/blobstore/api/blobnode" + "github.com/cubefs/cubefs/blobstore/common/ec" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/util/errors" + "github.com/cubefs/cubefs/blobstore/util/retry" +) + +// TODO: To Be Continue +// put empty shard to blobnode if file has been aligned. + +// Put put one object +// required: size, file size +// optional: hasher map to calculate hash.Hash +func (h *Handler) Put(ctx context.Context, rc io.Reader, size int64, + hasherMap access.HasherMap) (*access.Location, error) { + span := trace.SpanFromContextSafe(ctx) + span.Debugf("put request size:%d hashes:b(%b)", size, hasherMap.ToHashAlgorithm()) + + if size <= 0 { + return nil, errcode.ErrIllegalArguments + } + if size > h.maxObjectSize { + span.Info("exceed max object size", h.maxObjectSize) + return nil, errcode.ErrAccessExceedSize + } + + // 1.make hasher + if len(hasherMap) > 0 { + rc = io.TeeReader(rc, hasherMap.ToWriter()) + } + + // 2.choose cluster and alloc volume from allocator + selectedCodeMode := h.allCodeModes.SelectCodeMode(size) + span.Debugf("select codemode %d", selectedCodeMode) + + blobSize := atomic.LoadUint32(&h.MaxBlobSize) + clusterID, blobs, err := h.allocFromAllocatorWithHystrix(ctx, selectedCodeMode, uint64(size), blobSize, 0) + if err != nil { + span.Error("alloc failed", errors.Detail(err)) + return nil, err + } + span.Debugf("allocated from %d %+v", clusterID, blobs) + + // 3.read body and split, alloc from mem pool;ec encode and put into data node + limitReader := io.LimitReader(rc, int64(size)) + location := &access.Location{ + ClusterID: clusterID, + CodeMode: selectedCodeMode, + Size: uint64(size), + BlobSize: blobSize, + Blobs: blobs, + } + + uploadSucc := false + defer func() { + if !uploadSucc { + span.Infof("put failed clean location %+v", location) + if err := h.clearGarbage(ctx, location); err != nil { + span.Warn(errors.Detail(err)) + } + } + }() + + var buffer *ec.Buffer + putTime := new(timeReadWrite) + defer func() { + // release ec buffer which have not takeover + buffer.Release() + span.AppendRPCTrackLog([]string{putTime.String()}) + }() + + // concurrent buffer in per request + const concurrence = 4 + ready := make(chan struct{}, concurrence) + for range [concurrence]struct{}{} { + ready <- struct{}{} + } + + encoder := h.encoder[selectedCodeMode] + tactic := selectedCodeMode.Tactic() + for _, blob := range location.Spread() { + vid, bid, bsize := blob.Vid, blob.Bid, int(blob.Size) + + // new an empty ec buffer for per blob + var err error + st := time.Now() + buffer, err = ec.NewBuffer(bsize, tactic, h.memPool) + putTime.IncA(time.Since(st)) + if err != nil { + return nil, err + } + + readBuff := buffer.DataBuf[:bsize] + shards, err := encoder.Split(buffer.ECDataBuf) + if err != nil { + return nil, err + } + + startRead := time.Now() + n, err := io.ReadFull(limitReader, readBuff) + putTime.IncR(time.Since(startRead)) + if err != nil && err != io.EOF { + span.Infof("read blob data failed want:%d read:%d %s", bsize, n, err.Error()) + return nil, errcode.ErrAccessReadRequestBody + } + if n != bsize { + span.Infof("read blob less data want:%d but:%d", bsize, n) + return nil, errcode.ErrAccessReadRequestBody + } + + // ec encode + if err = encoder.Encode(shards); err != nil { + return nil, err + } + + blobident := blobIdent{clusterID, vid, bid} + span.Debug("to write", blobident) + + // takeover the buffer, release to pool in function writeToBlobnodes + takeoverBuffer := buffer + buffer = nil + <-ready + startWrite := time.Now() + err = h.writeToBlobnodesWithHystrix(ctx, blobident, shards, func() { + takeoverBuffer.Release() + ready <- struct{}{} + }) + putTime.IncW(time.Since(startWrite)) + if err != nil { + return nil, errors.Info(err, "write to blobnode failed") + } + } + + uploadSucc = true + return location, nil +} + +func (h *Handler) writeToBlobnodesWithHystrix(ctx context.Context, + blob blobIdent, shards [][]byte, callback func()) error { + safe := make(chan struct{}, 1) + err := hystrix.Do(rwCommand, func() error { + safe <- struct{}{} + return h.writeToBlobnodes(ctx, blob, shards, callback) + }, nil) + + select { + case <-safe: + default: + callback() // callback if fused by hystrix + } + return err +} + +type shardPutStatus struct { + index int + status bool +} + +// writeToBlobnodes write shards to blobnodes. +// takeover ec buffer release by callback. +// return if had quorum successful shards, then wait all shards in background. +func (h *Handler) writeToBlobnodes(ctx context.Context, + blob blobIdent, shards [][]byte, callback func()) (err error) { + span := trace.SpanFromContextSafe(ctx) + clusterID, vid, bid := blob.cid, blob.vid, blob.bid + + wg := &sync.WaitGroup{} + defer func() { + // waiting all shards done in background + go func() { + wg.Wait() + callback() + }() + }() + + volume, err := h.getVolume(ctx, clusterID, vid, true) + if err != nil { + return + } + serviceController, err := h.clusterController.GetServiceController(clusterID) + if err != nil { + return + } + + statusCh := make(chan shardPutStatus, len(volume.Units)) + tactic := volume.CodeMode.Tactic() + putQuorum := uint32(tactic.PutQuorum) + if num, ok := h.CodeModesPutQuorums[volume.CodeMode]; ok && num <= tactic.N+tactic.M { + putQuorum = uint32(num) + } + + // writtenNum ONLY apply on data and partiy shards + // TODO: count N and M in each AZ, + // decision ec data is recoverable or not. + maxWrittenIndex := tactic.N + tactic.M + writtenNum := uint32(0) + + wg.Add(len(volume.Units)) + for i, unitI := range volume.Units { + index, unit := i, unitI + + go func() { + status := shardPutStatus{index: index} + defer func() { + statusCh <- status + wg.Done() + }() + + diskID := unit.DiskID + args := &blobnode.PutShardArgs{ + DiskID: diskID, + Vuid: unit.Vuid, + Bid: bid, + Size: int64(len(shards[index])), + Type: blobnode.NormalIO, + } + + crcDisabled := h.ShardCrcDisabled + var crcOrigin uint32 + if !crcDisabled { + crcOrigin = crc32.ChecksumIEEE(shards[index]) + } + + // new child span to write to blobnode, we should finish it here. + spanChild, ctxChild := trace.StartSpanFromContextWithTraceID( + context.Background(), "WriteToBlobnode", span.TraceID()) + defer spanChild.Finish() + + RETRY: + hostInfo, err := serviceController.GetDiskHost(ctxChild, diskID) + if err != nil { + span.Error("get disk host failed", errors.Detail(err)) + return + } + // punished disk, ignore and return + if hostInfo.Punished { + span.Infof("ignore punished disk(%d %s) uvid(%d) ecidx(%02d) in idc(%s)", + diskID, hostInfo.Host, unit.Vuid, index, hostInfo.IDC) + return + } + host := hostInfo.Host + + var ( + writeErr error + needRetry bool + crc uint32 + ) + writeErr = retry.ExponentialBackoff(3, 200).RuptOn(func() (bool, error) { + args.Body = bytes.NewReader(shards[index]) + + crc, err = h.blobnodeClient.PutShard(ctxChild, host, args) + if err == nil { + if !crcDisabled && crc != crcOrigin { + return false, fmt.Errorf("crc mismatch 0x%x != 0x%x", crc, crcOrigin) + } + + needRetry = false + return true, nil + } + + code := rpc.DetectStatusCode(err) + switch code { + case errcode.CodeDiskBroken, errcode.CodeDiskNotFound, + errcode.CodeChunkNoSpace, errcode.CodeVUIDReadonly: + h.discardVidChan <- discardVid{ + cid: clusterID, + codeMode: volume.CodeMode, + vid: vid, + } + } + + switch code { + // EIO and Readonly error, then we need to punish disk in local and no necessary to retry + case errcode.CodeDiskBroken, errcode.CodeVUIDReadonly: + h.punishVolume(ctx, clusterID, vid, host, "BrokenOrRO") + h.punishDisk(ctx, clusterID, diskID, host, "BrokenOrRO") + span.Infof("punish disk:%d volume:%d cos:blobnode/%d", diskID, vid, code) + return true, err + + // chunk no space, we should punish this volume + case errcode.CodeChunkNoSpace: + h.punishVolume(ctx, clusterID, vid, host, "NoSpace") + span.Infof("punish volume:%d cos:blobnode/%d", vid, code) + return true, err + + // vuid not found means the reflection between vuid and diskID has change, we should refresh the volume + // disk not found means disk has been repaired or offline + case errcode.CodeDiskNotFound, errcode.CodeVuidNotFound: + latestVolume, e := h.getVolume(ctx, clusterID, vid, false) + if e != nil { + return true, errors.Base(err, "get volume with no cache failed").Detail(e) + } + + newUnit := latestVolume.Units[index] + if diskID != newUnit.DiskID { + diskID = newUnit.DiskID + unit = newUnit + args.DiskID = newUnit.DiskID + args.Vuid = newUnit.Vuid + + needRetry = true + return true, err + } + + h.punishVolume(ctx, clusterID, vid, host, "NotFound") + h.punishDisk(ctx, clusterID, diskID, host, "NotFound") + span.Infof("punish disk:%d volume:%d cos:blobnode/%d", diskID, vid, code) + return true, err + } + + // in timeout case and writtenNum is not satisfied with putQuorum, then should retry + if errorTimeout(err) && atomic.LoadUint32(&writtenNum) < putQuorum { + h.punishDiskWith(ctx, clusterID, diskID, host, "Timeout") + span.Info("connect timeout, need to punish threshold disk", diskID, host) + return false, err + } + + // others, do not retry this round + return true, err + }) + + if needRetry { + goto RETRY + } + if writeErr != nil { + span.Warnf("write %s on blobnode(%d %d %s) ecidx(%02d): %s", + blob.String(), args.Vuid, args.DiskID, hostInfo.Host, index, errors.Detail(writeErr)) + return + } + + if index < maxWrittenIndex { + atomic.AddUint32(&writtenNum, 1) + } + status.status = true + }() + } + + received := make(map[int]shardPutStatus, len(volume.Units)) + for len(received) < len(volume.Units) && atomic.LoadUint32(&writtenNum) < putQuorum { + st := <-statusCh + received[st.index] = st + } + + writeDone := make(chan struct{}, 1) + // write unaccomplished shard to repair queue + go func(writeDone <-chan struct{}) { + for len(received) < len(volume.Units) { + st := <-statusCh + received[st.index] = st + } + + if _, ok := <-writeDone; !ok { + return + } + + badIdxes := make([]uint8, 0) + for idx := range volume.Units { + if st, ok := received[idx]; ok && st.status { + continue + } + badIdxes = append(badIdxes, uint8(idx)) + } + if len(badIdxes) > 0 { + h.sendRepairMsgBg(ctx, blob, badIdxes) + } + }(writeDone) + + // return if had quorum successful shards + if atomic.LoadUint32(&writtenNum) >= putQuorum { + writeDone <- struct{}{} + return + } + + // It tolerate one az was down when we have 3 or more azs. + // But MUST make sure others azs data is all completed, + // And all data in the down az are failed. + if tactic.AZCount >= 3 { + allFine := 0 + allDown := 0 + + for _, azIndexes := range tactic.GetECLayoutByAZ() { + azFine := true + azDown := true + for _, idx := range azIndexes { + if st, ok := received[idx]; !ok || !st.status { + azFine = false + } else { + azDown = false + } + } + if azFine { + allFine++ + } + if azDown { + allDown++ + } + } + + span.Debugf("tolerate-multi-az-write (az-fine:%d az-down:%d az-all:%d)", allFine, allDown, tactic.AZCount) + if allFine == tactic.AZCount-1 && allDown == 1 { + span.Warnf("tolerate-multi-az-write (az-fine:%d az-down:%d az-all:%d) of %s", + allFine, allDown, tactic.AZCount, blob.String()) + writeDone <- struct{}{} + return + } + } + + close(writeDone) + err = fmt.Errorf("quorum write failed (%d < %d) of %s", writtenNum, putQuorum, blob.String()) + return +} diff --git a/blobstore/access/stream_put_test.go b/blobstore/access/stream_put_test.go new file mode 100644 index 000000000..c6dd5d2de --- /dev/null +++ b/blobstore/access/stream_put_test.go @@ -0,0 +1,349 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "bytes" + "crypto/md5" + "crypto/rand" + "crypto/sha1" + "crypto/sha256" + "encoding/hex" + "hash/crc32" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/api/access" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +func TestAccessStreamPutBase(t *testing.T) { + ctx := ctxWithName("TestAccessStreamPutBase") + vuidController.SetBNRealError(true) + defer func() { + vuidController.SetBNRealError(false) + }() + + // 0 + { + size := 0 + _, err := streamer.Put(ctx(), newReader(size), int64(size), nil) + require.Error(t, err) + } + // 1 byte + { + size := 1 + loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil) + require.NoError(t, err) + require.Equal(t, 1, len(loc.Blobs)) + require.Equal(t, uint32(1), loc.Blobs[0].Count) + // time wait the punished services + time.Sleep(time.Second * time.Duration(punishServiceS)) + } + // <4M + { + size := 1 << 18 + loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil) + require.NoError(t, err) + require.Equal(t, 1, len(loc.Blobs)) + require.Equal(t, uint32(1), loc.Blobs[0].Count) + time.Sleep(time.Second * time.Duration(punishServiceS)) + } + // 8M + 1k + { + size := (1 << 23) + 1024 + loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil) + require.NoError(t, err) + require.Equal(t, 2, len(loc.Blobs)) + require.Equal(t, uint32(2), loc.Blobs[1].Count) + time.Sleep(time.Second * time.Duration(punishServiceS)) + } + // max size + 1 + { + size := defaultMaxObjectSize + 1 + _, err := streamer.Put(ctx(), nil, int64(size), nil) + require.EqualError(t, errcode.ErrAccessExceedSize, err.Error()) + } + + dataShards.clean() +} + +func TestAccessStreamPutSum(t *testing.T) { + ctx := ctxWithName("TestAccessStreamPutSum") + sumChecker := func(data []byte) { + dataShards.clean() + + hasherMap := access.HasherMap{ + access.HashAlgDummy: access.HashAlgDummy.ToHasher(), + access.HashAlgCRC32: access.HashAlgCRC32.ToHasher(), + access.HashAlgMD5: access.HashAlgMD5.ToHasher(), + access.HashAlgSHA1: access.HashAlgSHA1.ToHasher(), + access.HashAlgSHA256: access.HashAlgSHA256.ToHasher(), + } + hashSumMap := make(access.HashSumMap, len(hasherMap)) + + _, err := streamer.Put(ctx(), bytes.NewReader(data), int64(len(data)), hasherMap) + require.NoError(t, err) + for alg, hasher := range hasherMap { + hashSumMap[alg] = hasher.Sum(nil) + } + + bytesMD5 := md5.Sum(data) + bytesSHA1 := sha1.Sum(data) + bytesSHA256 := sha256.Sum256(data) + + require.Equal(t, hashSumMap.GetSumVal(access.HashAlgCRC32), crc32.ChecksumIEEE(data)) + require.Equal(t, hashSumMap.GetSumVal(access.HashAlgMD5), sum2Str(bytesMD5[:])) + require.Equal(t, hashSumMap.GetSumVal(access.HashAlgSHA1), sum2Str(bytesSHA1[:])) + require.Equal(t, hashSumMap.GetSumVal(access.HashAlgSHA256), sum2Str(bytesSHA256[:])) + } + + shardsChecher := func(shardSize int, buff []byte, bid proto.BlobID) { + require.Equal(t, buff[:shardSize], dataShards.get(1001, bid)) + require.Equal(t, buff[shardSize:shardSize*2], dataShards.get(1002, bid)) + require.Equal(t, buff[shardSize*2:shardSize*3], dataShards.get(1003, bid)) + require.Equal(t, buff[shardSize*3:shardSize*4], dataShards.get(1004, bid)) + require.Equal(t, 0, len(dataShards.get(1005, 10000))) + require.Equal(t, buff[shardSize*5:shardSize*6], dataShards.get(1006, bid)) + require.NotEqual(t, ([]byte)(nil), dataShards.get(1007, bid)) + require.Equal(t, shardSize, len(dataShards.get(1007, bid))) + require.Equal(t, shardSize, len(dataShards.get(1012, bid))) + } + + // 1 byte + { + data := []byte("x") + sumChecker(data) + + shardSize := getBufSizes(len(data)).ShardSize + zeroShard := make([]byte, shardSize) + shard1 := make([]byte, shardSize) + copy(shard1, data) + + require.Equal(t, shard1, dataShards.get(1001, 10000)) + require.Equal(t, zeroShard, dataShards.get(1002, 10000)) + require.Equal(t, zeroShard, dataShards.get(1004, 10000)) + require.Equal(t, 0, len(dataShards.get(1005, 10000))) + require.NotEqual(t, ([]byte)(nil), dataShards.get(1007, 10000)) + } + // 7 bytes + { + data := []byte{0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6} + sumChecker(data) + + sizes := getBufSizes(len(data)) + buff := make([]byte, sizes.ECDataSize) + copy(buff, data) + shardsChecher(sizes.ShardSize, buff, 10000) + } + // 4K + 511B, aligned with MinShardSize + { + size := (1 << 12) + 511 + data := make([]byte, size) + rand.Read(data) + sumChecker(data) + + sizes := getBufSizes(len(data)) + buff := make([]byte, sizes.ECDataSize) + copy(buff, data) + shardsChecher(sizes.ShardSize, buff, 10000) + } + // 4M + { + size := 1 << 22 + data := make([]byte, size) + rand.Read(data) + sumChecker(data) + + sizes := getBufSizes(len(data)) + buff := make([]byte, sizes.ECDataSize) + copy(buff, data) + shardsChecher(sizes.ShardSize, buff, 10000) + } + // 4M + 1B + { + size := (1 << 22) + 1 + data := make([]byte, size) + rand.Read(data) + sumChecker(data) + + sizes := getBufSizes(blobSize) + buff := make([]byte, sizes.ECDataSize) + copy(buff, data[:blobSize]) + shardsChecher(sizes.ShardSize, buff, 10000) + + data = data[blobSize:] + sizes = getBufSizes(len(data)) + buff = make([]byte, sizes.ECDataSize) + copy(buff, data) + shardsChecher(sizes.ShardSize, buff, 20000) + } + // 6M + 1K + 7B + { + size := (1<<20)*6 + 1024 + 7 + data := make([]byte, size) + rand.Read(data) + sumChecker(data) + + sizes := getBufSizes(blobSize) + buff := make([]byte, sizes.ECDataSize) + copy(buff, data[:blobSize]) + shardsChecher(sizes.ShardSize, buff, 10000) + + data = data[blobSize:] + sizes = getBufSizes(len(data)) + buff = make([]byte, sizes.ECDataSize) + copy(buff, data) + shardsChecher(sizes.ShardSize, buff, 20000) + } + + dataShards.clean() +} + +func TestAccessStreamPutShardTimeout(t *testing.T) { + ctx := ctxWithName("TestAccessStreamPutShardTimeout") + dataShards.clean() + vuidController.Block(1001) + vuidController.Unbreak(1005) + defer func() { + vuidController.Unblock(1001) + vuidController.Unblock(1002) + vuidController.Break(1005) + dataShards.clean() + }() + + size := 1 << 22 + buff := make([]byte, size) + rand.Read(buff) + startTime := time.Now() + loc, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil) + require.NoError(t, err) + + // response immediately if had quorum shards + duration := time.Since(startTime) + require.GreaterOrEqual(t, time.Second/2, duration, "greater duration: ", duration) + + transfer, err := streamer.Get(ctx(), bytes.NewBuffer(nil), *loc, uint64(size), 0) + require.NoError(t, err) + err = transfer() + require.NoError(t, err) + + // wait all shards if no quorum shards + vuidController.Block(1002) + { + startTime := time.Now() + _, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil) + require.Error(t, err) + + duration := time.Since(startTime) + minDuration := vuidController.duration * 3 // retry ExponentialBackoff(3, 200) + maxDuration := minDuration + minDuration/2 + t.Log(duration, minDuration, maxDuration) + require.LessOrEqual(t, minDuration, duration, "less duration: ", duration) + require.GreaterOrEqual(t, maxDuration, duration, "greater duration: ", duration) + } +} + +func TestAccessStreamPutQuorum(t *testing.T) { + ctx := ctxWithName("TestAccessStreamPutQuorum") + defer func() { + dataShards.clean() + for ii := 0; ii < len(allID); ii++ { + vuidController.Unbreak(proto.Vuid(allID[ii])) + } + vuidController.Break(1005) + }() + + dataShards.clean() + vuidController.Unbreak(1005) + cases := []struct { + ids []proto.Vuid + hasError bool + }{ + // indexes by AZ + // [[0 1 6 7] [2 3 8 9] [4 5 10 11]] + {[]proto.Vuid{1001}, false}, + {[]proto.Vuid{1001, 1002}, true}, // az-0 instable + {[]proto.Vuid{1001, 1002, 1007, 1008}, false}, // az-0 down + {[]proto.Vuid{1003}, false}, + {[]proto.Vuid{1003, 1010}, true}, // az-1 instable + {[]proto.Vuid{1003, 1004, 1009, 1010}, false}, // az-1 down + {[]proto.Vuid{1005}, false}, + {[]proto.Vuid{1005, 1011}, true}, // az-2 instable + {[]proto.Vuid{1005, 1006, 1011, 1012}, false}, // az-2 down + {[]proto.Vuid{1001, 1011}, true}, + {[]proto.Vuid{1006, 1010}, true}, + {[]proto.Vuid{1008, 1010, 1012}, true}, + } + + size := 1 << 22 + buff := make([]byte, size) + rand.Read(buff) + for _, cs := range cases { + for _, id := range cs.ids { + vuidController.Break(id) + } + + _, err := streamer.Put(ctx(), bytes.NewReader(buff), int64(size), nil) + if cs.hasError { + require.NotNil(t, err) + } else { + require.NoError(t, err) + } + + time.Sleep(time.Second * time.Duration(punishServiceS)) + for _, id := range cs.ids { + vuidController.Unbreak(id) + } + } +} + +func BenchmarkAccessStreamPut(b *testing.B) { + ctx := ctxWithName("BenchmarkAccessStreamPut")() + vuidController.Unbreak(1005) + defer func() { + vuidController.Break(1005) + dataShards.clean() + }() + + cases := []struct { + name string + size int + }{ + {"1B", 1}, + {"1KB", 1 << 10}, + {"1MB", 1 << 20}, + {"4MB", 1 << 22}, + {"8MB", 1 << 23}, + } + + buff := make([]byte, 1<<23) + rand.Read(buff) + for _, cs := range cases { + b.ResetTimer() + b.Run(cs.name, func(b *testing.B) { + for ii := 0; ii <= b.N; ii++ { + streamer.Put(ctx, bytes.NewReader(buff[:cs.size]), int64(cs.size), nil) + } + }) + } +} + +func sum2Str(b []byte) string { + return hex.EncodeToString(b[:]) +} diff --git a/blobstore/access/stream_putat.go b/blobstore/access/stream_putat.go new file mode 100644 index 000000000..910bc3ff8 --- /dev/null +++ b/blobstore/access/stream_putat.go @@ -0,0 +1,102 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "context" + "io" + "time" + + "github.com/cubefs/cubefs/blobstore/api/access" + "github.com/cubefs/cubefs/blobstore/common/ec" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/trace" +) + +// PutAt access interface /putat, put one blob +// required: rc file reader +// required: clusterID VolumeID BlobID +// required: size, one blob size +// optional: hasherMap, computing hash +func (h *Handler) PutAt(ctx context.Context, rc io.Reader, + clusterID proto.ClusterID, vid proto.Vid, bid proto.BlobID, size int64, + hasherMap access.HasherMap) error { + span := trace.SpanFromContextSafe(ctx) + span.Debugf("putat request cluster:%d vid:%d bid:%d size:%d hashes:b(%b)", + clusterID, vid, bid, size, hasherMap.ToHashAlgorithm()) + + if len(hasherMap) > 0 { + rc = io.TeeReader(rc, hasherMap.ToWriter()) + } + + volume, err := h.getVolume(ctx, clusterID, vid, true) + if err != nil { + return err + } + encoder := h.encoder[volume.CodeMode] + + readSize := int(size) + st := time.Now() + buffer, err := ec.NewBuffer(readSize, volume.CodeMode.Tactic(), h.memPool) + if err != nil { + return err + } + + putTime := new(timeReadWrite) + putTime.IncA(time.Since(st)) + defer func() { + buffer.Release() + span.AppendRPCTrackLog([]string{putTime.String()}) + }() + + shards, err := encoder.Split(buffer.ECDataBuf) + if err != nil { + return err + } + + startRead := time.Now() + n, err := io.ReadFull(rc, buffer.DataBuf) + putTime.IncR(time.Since(startRead)) + if err != nil && err != io.EOF { + span.Info("read blob data from request body", err) + return errcode.ErrAccessReadRequestBody + } + if n != readSize { + span.Infof("want to read %d, but %d", readSize, n) + return errcode.ErrAccessReadRequestBody + } + + if err = encoder.Encode(shards); err != nil { + return err + } + + blobident := blobIdent{clusterID, vid, bid} + span.Debug("to write", blobident) + + takeoverBuffer := buffer + buffer = nil + startWrite := time.Now() + err = h.writeToBlobnodesWithHystrix(ctx, blobident, shards, func() { + takeoverBuffer.Release() + }) + putTime.IncW(time.Since(startWrite)) + if err != nil { + return err + } + + span.Debugf("putat done cluster:%d vid:%d bid:%d size:%d", clusterID, vid, bid, size) + return nil +} diff --git a/blobstore/access/stream_putat_test.go b/blobstore/access/stream_putat_test.go new file mode 100644 index 000000000..82d113cc4 --- /dev/null +++ b/blobstore/access/stream_putat_test.go @@ -0,0 +1,97 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "bytes" + "crypto/rand" + "hash/crc32" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/api/access" +) + +func TestAccessStreamPutAtBase(t *testing.T) { + ctx := ctxWithName("TestAccessStreamPutAtBase") + sumChecker := func(size int, data []byte) { + dataShards.clean() + + hasherMap := access.HasherMap{ + access.HashAlgCRC32: access.HashAlgCRC32.ToHasher(), + } + hashSumMap := make(access.HashSumMap, len(hasherMap)) + err := streamer.PutAt(ctx(), bytes.NewReader(data), clusterID, 1, 10000, int64(size), hasherMap) + require.Nil(t, err) + for alg, hasher := range hasherMap { + hashSumMap[alg] = hasher.Sum(nil) + } + require.Equal(t, hashSumMap.GetSumVal(access.HashAlgCRC32), crc32.ChecksumIEEE(data[:size])) + } + // 0 + { + size := 0 + err := streamer.PutAt(ctx(), newReader(size), clusterID, 1, 10000, int64(size), nil) + require.NotNil(t, err) + } + // 1 byte + { + size := 1 + data := make([]byte, size) + rand.Read(data) + sumChecker(size, data) + + shardSize := getBufSizes(len(data)).ShardSize + zeroShard := make([]byte, shardSize) + shard1 := make([]byte, shardSize) + copy(shard1, data) + + require.Equal(t, shard1, dataShards.get(1001, 10000)) + require.Equal(t, zeroShard, dataShards.get(1002, 10000)) + require.Equal(t, zeroShard, dataShards.get(1004, 10000)) + require.Equal(t, 0, len(dataShards.get(1005, 10000))) + require.NotEqual(t, ([]byte)(nil), dataShards.get(1007, 10000)) + } + // 4M per shard + { + size := (1 << 22) * 6 + data := make([]byte, size) + rand.Read(data) + sumChecker(size, data) + + require.Equal(t, 1<<22, len(dataShards.get(1001, 10000))) + } + // 4M + 1B, put 1B + { + size := (1 << 22) + 1 + data := make([]byte, size) + rand.Read(data) + sumChecker(1, data) + + shardSize := getBufSizes(1).ShardSize + require.Equal(t, shardSize, len(dataShards.get(1001, 10000))) + } + // 4M, error + { + dataShards.clean() + size := 1 << 22 + err := streamer.PutAt(ctx(), newReader(size), clusterID, 1, 10000, int64(size)+1, nil) + require.NotNil(t, err) + require.Equal(t, 0, len(dataShards.get(1001, 10000))) + } + + dataShards.clean() +} diff --git a/blobstore/access/stream_test.go b/blobstore/access/stream_test.go new file mode 100644 index 000000000..8c11c74c8 --- /dev/null +++ b/blobstore/access/stream_test.go @@ -0,0 +1,96 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "bytes" + "crypto/rand" + "io" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/access/controller" + "github.com/cubefs/cubefs/blobstore/common/codemode" +) + +func newReader(size int) io.Reader { + buff := make([]byte, size) + rand.Read(buff) + return bytes.NewReader(buff) +} + +func TestAccessStreamConfig(t *testing.T) { + cfg := StreamConfig{ + IDC: idcOther, + MemPoolSizeClasses: map[int]int{1024: 1}, + CodeModesPutQuorums: map[codemode.CodeMode]int{ + codemode.EC15P12: 16, + codemode.EC6P10L2: 18, + }, + } + confCheck(&cfg) + + require.Equal(t, idcOther, cfg.IDC) + require.Equal(t, map[int]int{1024: 1}, cfg.MemPoolSizeClasses) + require.Equal(t, defaultDiskPunishIntervalS, cfg.DiskPunishIntervalS) +} + +func TestAccessStreamNew(t *testing.T) { + require.Equal(t, idc, streamer.IDC) + + require.Panics(t, func() { + NewStreamHandler(&StreamConfig{IDC: "idc"}, nil, nil) + }) +} + +func TestAccessStreamDelete(t *testing.T) { + ctx := ctxWithName("TestAccessStreamDelete") + size := 1 << 18 + loc, err := streamer.Put(ctx(), newReader(size), int64(size), nil) + require.NoError(t, err) + + err = streamer.Delete(ctx(), loc) + require.NoError(t, err) + + dataShards.clean() +} + +func TestAccessStreamAdmin(t *testing.T) { + { + handler := Handler{} + sa := handler.Admin() + require.NotNil(t, sa) + + admin := sa.(*streamAdmin) + require.Nil(t, admin.memPool) + require.Nil(t, admin.controller) + } + { + sa := streamer.Admin() + require.NotNil(t, sa) + + admin := sa.(*streamAdmin) + require.NotNil(t, admin.memPool) + t.Log("mempool status:", admin.memPool.Status()) + + ctr := admin.controller + require.NotNil(t, ctr) + t.Log("region:", ctr.Region()) + require.Error(t, ctr.ChangeChooseAlg(controller.AlgChoose(100))) + require.NoError(t, ctr.ChangeChooseAlg(controller.AlgRandom)) + require.NoError(t, ctr.ChangeChooseAlg(controller.AlgAvailable)) + } +} diff --git a/blobstore/access/stream_time.go b/blobstore/access/stream_time.go new file mode 100644 index 000000000..301079181 --- /dev/null +++ b/blobstore/access/stream_time.go @@ -0,0 +1,47 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "fmt" + "sync/atomic" + "time" +) + +type timeReadWrite struct { + a int64 // alloc buffer + r int64 // PUT: read from client, GET: read from blobnode + w int64 // PUT: write to blobnode, GET: write to client +} + +func (t *timeReadWrite) IncA(dur time.Duration) { + atomic.AddInt64(&t.a, int64(dur)) +} + +func (t *timeReadWrite) IncR(dur time.Duration) { + atomic.AddInt64(&t.r, int64(dur)) +} + +func (t *timeReadWrite) IncW(dur time.Duration) { + atomic.AddInt64(&t.w, int64(dur)) +} + +// String within milliseconds +func (t *timeReadWrite) String() string { + a := atomic.LoadInt64(&t.a) / 1e6 + r := atomic.LoadInt64(&t.r) / 1e6 + w := atomic.LoadInt64(&t.w) / 1e6 + return fmt.Sprintf("a_%d_r_%d_w_%d", a, r, w) +} diff --git a/blobstore/api/access/client.go b/blobstore/api/access/client.go new file mode 100644 index 000000000..a8149ccb8 --- /dev/null +++ b/blobstore/api/access/client.go @@ -0,0 +1,809 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + + "github.com/hashicorp/consul/api" + "gopkg.in/natefinch/lumberjack.v2" + + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/resourcepool" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/util/defaulter" + "github.com/cubefs/cubefs/blobstore/util/log" + "github.com/cubefs/cubefs/blobstore/util/selector" + "github.com/cubefs/cubefs/blobstore/util/task" +) + +const ( + defaultMaxSizePutOnce int64 = 1 << 28 // 256MB + defaultMaxPartRetry int = 3 + defaultMaxHostRetry int = 3 + defaultPartConcurrence int = 4 + defaultServiceIntervalMs int64 = 5000 + defaultServiceName = "access" + + _cacheBufferPutOnce = 1 << 23 // 8M +) + +// RPCConnectMode self-defined rpc client connection config setting +type RPCConnectMode uint8 + +// timeout: [short - - - - - - - - -> long] +// quick --> general --> default --> slow --> nolimit +// speed: 40MB --> 20MB --> 10MB --> 4MB --> nolimit +const ( + DefaultConnMode RPCConnectMode = iota + QuickConnMode + GeneralConnMode + SlowConnMode + NoLimitConnMode +) + +func (mode RPCConnectMode) getConfig(speed float64, timeout, baseTimeout int64) rpc.Config { + getSpeed := func(defaultVal float64) float64 { + if speed > 0 { + return speed + } + return defaultVal + } + getBaseTimeout := func(defaultVal int64) int64 { + if baseTimeout > 0 { + return baseTimeout + } + return defaultVal + } + getTimeout := func(speed float64) int64 { + if timeout > 0 { + return timeout + } + return 5 * (1 << 30) * 1e3 / int64(speed*(1<<20)) + } + + config := rpc.Config{ + // the whole request and response timeout + ClientTimeoutMs: getTimeout(getSpeed(10)), + BodyBandwidthMBPs: getSpeed(10), + BodyBaseTimeoutMs: getBaseTimeout(30 * 1000), + Tc: rpc.TransportConfig{ + // dial timeout + DialTimeoutMs: 5 * 1000, + // response header timeout after send the request + ResponseHeaderTimeoutMs: 5 * 1000, + // IdleConnTimeout is the maximum amount of time an idle + // (keep-alive) connection will remain idle before closing + // itself.Zero means no limit. + IdleConnTimeoutMs: 30 * 1000, + + MaxIdleConns: 0, + MaxConnsPerHost: 2048, + MaxIdleConnsPerHost: 1024, + DisableCompression: true, + }, + } + + switch mode { + case QuickConnMode: + config.ClientTimeoutMs = getTimeout(getSpeed(40)) + config.BodyBandwidthMBPs = getSpeed(40) + config.BodyBaseTimeoutMs = getBaseTimeout(3 * 1000) + config.Tc.DialTimeoutMs = 2 * 1000 + config.Tc.ResponseHeaderTimeoutMs = 2 * 1000 + config.Tc.IdleConnTimeoutMs = 10 * 1000 + case GeneralConnMode: + config.ClientTimeoutMs = getTimeout(getSpeed(20)) + config.BodyBandwidthMBPs = getSpeed(20) + config.BodyBaseTimeoutMs = getBaseTimeout(10 * 1000) + config.Tc.DialTimeoutMs = 3 * 1000 + config.Tc.ResponseHeaderTimeoutMs = 3 * 1000 + config.Tc.IdleConnTimeoutMs = 30 * 1000 + case SlowConnMode: + config.ClientTimeoutMs = getTimeout(getSpeed(4)) + config.BodyBandwidthMBPs = getSpeed(4) + config.BodyBaseTimeoutMs = getBaseTimeout(120 * 1000) + config.Tc.DialTimeoutMs = 10 * 1000 + config.Tc.ResponseHeaderTimeoutMs = 10 * 1000 + config.Tc.IdleConnTimeoutMs = 60 * 1000 + case NoLimitConnMode: + config.ClientTimeoutMs = 0 + config.BodyBandwidthMBPs = getSpeed(0) + config.BodyBaseTimeoutMs = getBaseTimeout(0) + config.Tc.DialTimeoutMs = 0 + config.Tc.ResponseHeaderTimeoutMs = 0 + config.Tc.IdleConnTimeoutMs = 600 * 1000 + default: + } + + return config +} + +// Config access client config +type Config struct { + // ConnMode rpc connection timeout setting + ConnMode RPCConnectMode + // ClientTimeoutMs the whole request and response timeout + ClientTimeoutMs int64 + // BodyBandwidthMBPs reading body timeout, request or response + // timeout = ContentLength/BodyBandwidthMBPs + BodyBaseTimeoutMs + BodyBandwidthMBPs float64 + // BodyBaseTimeoutMs base timeout for read body + BodyBaseTimeoutMs int64 + + // Consul is consul config for discovering service + Consul ConsulConfig + // ServiceIntervalMs is interval ms for discovering service + ServiceIntervalMs int64 + // PriorityAddrs priority addrs of access service when retry + PriorityAddrs []string + // MaxSizePutOnce max size using once-put object interface + MaxSizePutOnce int64 + // MaxPartRetry max retry times when putting one part, 0 means forever + MaxPartRetry int + // MaxHostRetry max retry hosts of access service + MaxHostRetry int + // PartConcurrence concurrence of put parts + PartConcurrence int + + // RPCConfig user-defined rpc config + // All connections will use the config if it's not nil + // ConnMode will be ignored if rpc config is setting + RPCConfig *rpc.Config + + // LogLevel client output logging level. + LogLevel log.Level + + // Logger trace all logging to the logger if setting. + // It is an io.WriteCloser that writes to the specified filename. + // YOU should CLOSE it after you do not use the client anymore. + Logger *Logger +} + +// ConsulConfig alias of consul api.Config +// Fixup: client and sdk using the same config type +type ConsulConfig = api.Config + +// Logger alias of lumberjack Logger +// See more at: https://github.com/natefinch/lumberjack +type Logger = lumberjack.Logger + +// client access rpc client +type client struct { + config Config + consulClient *api.Client + selector selector.Selector + rpcClient rpc.Client +} + +// API access api for s3 +// To trace request id, the ctx is better WithRequestID(ctx, rid). +type API interface { + // Put object once if size is not greater than MaxSizePutOnce, otherwise put blobs one by one. + // return a location and map of hash summary bytes you excepted. + Put(ctx context.Context, args *PutArgs) (location Location, hashSumMap HashSumMap, err error) + // Get object, range is supported. + Get(ctx context.Context, args *GetArgs) (body io.ReadCloser, err error) + // Delete all blobs in these locations. + // return failed locations which have yet been deleted if error is not nil. + Delete(ctx context.Context, args *DeleteArgs) (failedLocations []Location, err error) +} + +var _ API = (*client)(nil) + +type hadReader struct { + isRead bool + reader io.Reader +} + +// Read the io.Reader had been read +func (r *hadReader) Read(p []byte) (n int, err error) { + r.isRead = true + return r.reader.Read(p) +} + +type noopBody struct{} + +var _ io.ReadCloser = (*noopBody)(nil) + +func (rc noopBody) Read(p []byte) (n int, err error) { return 0, io.EOF } +func (rc noopBody) Close() error { return nil } + +var memPool *resourcepool.MemPool + +func init() { + memPool = resourcepool.NewMemPool(map[int]int{ + 1 << 12: -1, + 1 << 14: -1, + 1 << 18: -1, + 1 << 20: -1, + 1 << 22: -1, + 1 << 23: -1, + 1 << 24: -1, + }) +} + +// New returns an access API +func New(cfg Config) (API, error) { + defaulter.LessOrEqual(&cfg.MaxSizePutOnce, defaultMaxSizePutOnce) + defaulter.Less(&cfg.MaxPartRetry, defaultMaxPartRetry) + defaulter.LessOrEqual(&cfg.MaxHostRetry, defaultMaxHostRetry) + defaulter.LessOrEqual(&cfg.PartConcurrence, defaultPartConcurrence) + if cfg.ServiceIntervalMs < 500 { + cfg.ServiceIntervalMs = defaultServiceIntervalMs + } + + var rpcClient rpc.Client + if cfg.RPCConfig != nil { + rpcClient = rpc.NewClient(cfg.RPCConfig) + } else { + rpcConfig := cfg.ConnMode.getConfig(cfg.BodyBandwidthMBPs, + cfg.ClientTimeoutMs, cfg.BodyBaseTimeoutMs) + rpcClient = rpc.NewClient(&rpcConfig) + } + + log.SetOutputLevel(cfg.LogLevel) + if cfg.Logger != nil { + log.SetOutput(cfg.Logger) + } + + consulConfig := api.Config(cfg.Consul) + consulClient, err := api.NewClient(&consulConfig) + if err != nil { + return nil, errcode.ErrAccessServiceDiscovery + } + + first := true + serviceName := defaultServiceName + hostGetter := func() ([]string, error) { + if first && len(cfg.PriorityAddrs) > 0 { + hosts := make([]string, len(cfg.PriorityAddrs)) + copy(hosts, cfg.PriorityAddrs[:]) + first = false + return hosts, nil + } + + services, _, err := consulClient.Health().Service(serviceName, "", true, nil) + if err != nil { + return nil, err + } + + hosts := make([]string, 0, len(services)) + for _, s := range services { + address := s.Service.Address + if address == "" { + address = s.Node.Address + } + hosts = append(hosts, fmt.Sprintf("http://%s:%d", address, s.Service.Port)) + } + + if len(hosts) == 0 { + return nil, fmt.Errorf("unavailable service") + } + + return hosts, nil + } + + hostSelector, err := selector.NewSelector(cfg.ServiceIntervalMs, hostGetter) + if err != nil { + return nil, errcode.ErrAccessServiceDiscovery + } + + return &client{ + config: cfg, + consulClient: consulClient, + selector: hostSelector, + rpcClient: rpcClient, + }, nil +} + +func (c *client) Put(ctx context.Context, args *PutArgs) (location Location, hashSumMap HashSumMap, err error) { + if args.Size == 0 { + hashSumMap := args.Hashes.ToHashSumMap() + for alg := range hashSumMap { + hashSumMap[alg] = alg.ToHasher().Sum(nil) + } + return Location{Blobs: make([]SliceInfo, 0)}, hashSumMap, nil + } + + ctx = withReqidContext(ctx) + if args.Size <= c.config.MaxSizePutOnce { + return c.putObject(ctx, args) + } + return c.putParts(ctx, args) +} + +func (c *client) putObject(ctx context.Context, args *PutArgs) (location Location, hashSumMap HashSumMap, err error) { + span := trace.SpanFromContextSafe(ctx) + + var ( + cached bool + buffer []byte + reader *hadReader + ) + if args.Size > 0 && args.Size <= _cacheBufferPutOnce { + cached = true + buffer, _ = memPool.Alloc(int(args.Size)) + buffer = buffer[:args.Size] + defer memPool.Put(buffer) + + _, err := io.ReadFull(args.Body, buffer) + if err != nil { + span.Error("read buffer from request", err) + return Location{}, nil, errcode.ErrAccessReadRequestBody + } + } else { + reader = &hadReader{ + isRead: false, + reader: args.Body, + } + } + + err = c.tryN(ctx, c.config.MaxHostRetry, func(host string) error { + var body io.Reader + if cached { + body = bytes.NewReader(buffer) + } else { + if reader.isRead { + span.Info("retry on other access host which had been read body") + return errcode.ErrAccessReadConflictBody + } + body = reader + } + + urlStr := fmt.Sprintf("%s/put?size=%d&hashes=%d", host, args.Size, args.Hashes) + req, e := http.NewRequest(http.MethodPut, urlStr, body) + if e != nil { + return e + } + + resp := &PutResp{} + e = c.rpcClient.DoWith(ctx, req, resp, rpc.WithCrcEncode()) + if e == nil { + location = resp.Location + hashSumMap = resp.HashSumMap + } + return e + }) + return +} + +type blobPart struct { + cid proto.ClusterID + vid proto.Vid + bid proto.BlobID + size int + index int + token string + buf []byte +} + +func (c *client) putPartsBatch(ctx context.Context, parts []blobPart) error { + tasks := make([]func() error, 0, len(parts)) + for _, part := range parts { + part := part + tasks = append(tasks, func() error { + return c.tryN(ctx, c.config.MaxHostRetry, func(host string) error { + urlStr := fmt.Sprintf("%s/putat?clusterid=%d&volumeid=%d&blobid=%d&size=%d&hashes=%d&token=%s", + host, part.cid, part.vid, part.bid, part.size, 0, part.token) + req, err := http.NewRequest(http.MethodPut, urlStr, bytes.NewReader(part.buf)) + if err != nil { + return err + } + resp := &PutAtResp{} + return c.rpcClient.DoWith(ctx, req, resp, rpc.WithCrcEncode()) + }) + }) + } + + if err := task.Run(context.Background(), tasks...); err != nil { + for _, part := range parts { + part := part + // asynchronously delete blob + go func() { + c.tryN(ctx, c.config.MaxHostRetry, func(host string) error { + urlStr := fmt.Sprintf("%s/deleteblob?clusterid=%d&volumeid=%d&blobid=%d&size=%d&token=%s", + host, part.cid, part.vid, part.bid, part.size, part.token) + req, err := http.NewRequest(http.MethodDelete, urlStr, nil) + if err != nil { + return err + } + return c.rpcClient.DoWith(ctx, req, nil) + }) + }() + } + return err + } + return nil +} + +func (c *client) putParts(ctx context.Context, args *PutArgs) (Location, HashSumMap, error) { + span := trace.SpanFromContextSafe(ctx) + + hashSumMap := args.Hashes.ToHashSumMap() + hasherMap := make(HasherMap, len(hashSumMap)) + for alg := range hashSumMap { + hasherMap[alg] = alg.ToHasher() + } + + reqBody := args.Body + if len(hasherMap) > 0 { + reqBody = io.TeeReader(args.Body, hasherMap.ToWriter()) + } + + var ( + loc Location + tokens []string + ) + + signArgs := SignArgs{} + success := false + defer func() { + if success { + return + } + + locations := signArgs.Locations[:] + if len(locations) > 1 { + signArgs.Location = loc.Copy() + signResp := &SignResp{} + if err := c.tryN(ctx, c.config.MaxHostRetry, func(host string) error { + return c.rpcClient.PostWith(ctx, fmt.Sprintf("%s/sign", host), signResp, signArgs) + }); err == nil { + locations = []Location{signResp.Location.Copy()} + } + } + if len(locations) > 0 { + if _, err := c.Delete(ctx, &DeleteArgs{Locations: locations}); err != nil { + span.Warnf("clean location '%+v' failed %s", locations, err.Error()) + } + } + }() + + // alloc + err := c.tryN(ctx, c.config.MaxHostRetry, func(host string) error { + allocResp := &AllocResp{} + if err := c.rpcClient.PostWith(ctx, fmt.Sprintf("%s/alloc", host), allocResp, AllocArgs{ + Size: uint64(args.Size), + }); err != nil { + return err + } + loc = allocResp.Location + tokens = allocResp.Tokens + return nil + }) + if err != nil { + return loc, nil, err + } + signArgs.Locations = append(signArgs.Locations, loc.Copy()) + + // buffer pipeline + closeCh := make(chan struct{}) + bufferPipe := func(size, blobSize int) <-chan []byte { + ch := make(chan []byte, c.config.PartConcurrence-1) + go func() { + for size > 0 { + toread := blobSize + if toread > size { + toread = size + } + + buf, _ := memPool.Alloc(toread) + buf = buf[:toread] + _, err := io.ReadFull(reqBody, buf) + if err != nil { + span.Error("read buffer from request", err) + memPool.Put(buf) + close(ch) + return + } + + select { + case <-closeCh: + memPool.Put(buf) + close(ch) + return + case ch <- buf: + } + + size -= toread + } + close(ch) + }() + return ch + }(int(loc.Size), int(loc.BlobSize)) + + defer func() { + // waiting pipeline close if has error + for buf := range bufferPipe { + if len(buf) > 0 { + memPool.Put(buf) + } + } + }() + + releaseBuffer := func(parts []blobPart) { + for _, part := range parts { + memPool.Put(part.buf) + } + } + + currBlobIdx := 0 + currBlobCount := uint32(0) + remainSize := loc.Size + restPartsLoc := loc + + index := -1 + readSize := 0 + for readSize < int(loc.Size) { + parts := make([]blobPart, 0, c.config.PartConcurrence) + + // waiting at least one blob + buf, ok := <-bufferPipe + if !ok && readSize < int(loc.Size) { + return Location{}, nil, errcode.ErrAccessReadRequestBody + } + index++ + readSize += len(buf) + parts = append(parts, blobPart{size: len(buf), index: index, buf: buf}) + + more := true + for more && len(parts) < c.config.PartConcurrence { + select { + case buf, ok := <-bufferPipe: + if !ok { + if readSize < int(loc.Size) { + releaseBuffer(parts) + return Location{}, nil, errcode.ErrAccessReadRequestBody + } + more = false + } else { + index++ + readSize += len(buf) + parts = append(parts, blobPart{size: len(buf), index: index, buf: buf}) + } + default: + more = false + } + } + + tryTimes := c.config.MaxPartRetry + for { + if uint32(len(loc.Blobs)) > MaxLocationBlobs { + releaseBuffer(parts) + close(closeCh) + return Location{}, nil, errcode.ErrUnexpected + } + + // feed new params + currIdx := currBlobIdx + currCount := currBlobCount + for i := range parts { + token := tokens[currIdx] + if restPartsLoc.Size > uint64(loc.BlobSize) && parts[i].size < int(loc.BlobSize) { + token = tokens[currIdx+1] + } + parts[i].token = token + parts[i].cid = loc.ClusterID + parts[i].vid = loc.Blobs[currIdx].Vid + parts[i].bid = loc.Blobs[currIdx].MinBid + proto.BlobID(currCount) + + currCount++ + if loc.Blobs[currIdx].Count == currCount { + currIdx++ + currCount = 0 + } + } + + err := c.putPartsBatch(ctx, parts) + if err == nil { + for _, part := range parts { + remainSize -= uint64(part.size) + currBlobCount++ + // next blobs + if loc.Blobs[currBlobIdx].Count == currBlobCount { + currBlobIdx++ + currBlobCount = 0 + } + } + + break + } + span.Warn("putat parts", err) + + if tryTimes > 0 { // has retry setting + if tryTimes == 1 { + releaseBuffer(parts) + close(closeCh) + span.Error("exceed the max retry limit", c.config.MaxPartRetry) + return Location{}, nil, errcode.ErrUnexpected + } + tryTimes-- + } + + var restPartsResp *AllocResp + // alloc the rest parts + err = c.tryN(ctx, c.config.MaxHostRetry, func(host string) error { + resp := &AllocResp{} + if err := c.rpcClient.PostWith(ctx, fmt.Sprintf("%s/alloc", host), resp, AllocArgs{ + Size: remainSize, + BlobSize: loc.BlobSize, + CodeMode: loc.CodeMode, + AssignClusterID: loc.ClusterID, + }); err != nil { + return err + } + if len(resp.Location.Blobs) > 0 { + if newVid := resp.Location.Blobs[0].Vid; newVid == loc.Blobs[currBlobIdx].Vid { + return fmt.Errorf("alloc the same vid %d", newVid) + } + } + restPartsResp = resp + return nil + }) + if err != nil { + releaseBuffer(parts) + close(closeCh) + span.Error("alloc another parts to put", err) + return Location{}, nil, errcode.ErrUnexpected + } + + restPartsLoc = restPartsResp.Location + signArgs.Locations = append(signArgs.Locations, restPartsLoc.Copy()) + + if currBlobCount > 0 { + loc.Blobs[currBlobIdx].Count = currBlobCount + currBlobIdx++ + } + loc.Blobs = append(loc.Blobs[:currBlobIdx], restPartsLoc.Blobs...) + tokens = append(tokens[:currBlobIdx], restPartsResp.Tokens...) + + currBlobCount = 0 + } + + releaseBuffer(parts) + } + + if len(signArgs.Locations) > 1 { + signArgs.Location = loc.Copy() + // sign + err = c.tryN(ctx, c.config.MaxHostRetry, func(host string) error { + signResp := &SignResp{} + if err := c.rpcClient.PostWith(ctx, fmt.Sprintf("%s/sign", host), signResp, signArgs); err != nil { + return err + } + loc = signResp.Location + return nil + }) + if err != nil { + span.Error("sign location with crc", err) + return Location{}, nil, errcode.ErrUnexpected + } + } + + for alg, hasher := range hasherMap { + hashSumMap[alg] = hasher.Sum(nil) + } + success = true + return loc, hashSumMap, nil +} + +func (c *client) Get(ctx context.Context, args *GetArgs) (body io.ReadCloser, err error) { + if !args.IsValid() { + return nil, errcode.ErrIllegalArguments + } + + ctx = withReqidContext(ctx) + if args.Location.Size == 0 || args.ReadSize == 0 { + return noopBody{}, nil + } + + err = c.tryN(ctx, c.config.MaxHostRetry, func(host string) error { + gotBody, e := func() (io.ReadCloser, error) { + resp, e := c.rpcClient.Post(ctx, fmt.Sprintf("%s/get", host), args) + if e != nil { + return nil, e + } + if resp.StatusCode >= 400 { + return nil, rpc.NewError(resp.StatusCode, "StatusCode", fmt.Errorf("code: %d", resp.StatusCode)) + } + return resp.Body, nil + }() + if e != nil { + return e + } + body = gotBody + return nil + }) + return +} + +func (c *client) Delete(ctx context.Context, args *DeleteArgs) ([]Location, error) { + if !args.IsValid() { + if args == nil { + return nil, errcode.ErrIllegalArguments + } + return args.Locations, errcode.ErrIllegalArguments + } + + ctx = withReqidContext(ctx) + locations := make([]Location, 0, len(args.Locations)) + for _, loc := range args.Locations { + if loc.Size > 0 { + locations = append(locations, loc.Copy()) + } + } + if len(locations) == 0 { + return nil, nil + } + + err := c.tryN(ctx, c.config.MaxHostRetry, func(host string) error { + // access response 2xx even if there has failed locations + deleteResp := &DeleteResp{} + if err := c.rpcClient.PostWith(ctx, fmt.Sprintf("%s/delete", host), deleteResp, + DeleteArgs{Locations: locations}); err != nil && rpc.DetectStatusCode(err) != http.StatusIMUsed { + return err + } + if len(deleteResp.FailedLocations) > 0 { + locations = deleteResp.FailedLocations[:] + return errcode.ErrUnexpected + } + return nil + }) + if err == nil { + return nil, nil + } + return locations, err +} + +func (c *client) tryN(ctx context.Context, n int, connector func(string) error) error { + span := trace.SpanFromContextSafe(ctx) + + hs := c.selector.GetRandomN(n) + hosts := append(c.config.PriorityAddrs[:], hs...) + if len(hosts) == 0 { + return errcode.ErrAccessServiceDiscovery + } + + var host string + for _, host = range hosts { + err := connector(host) + if err == nil { + return nil + } + + // has connected access node + if httpErr, ok := err.(rpc.HTTPError); ok { + // 500 need to retry next host + if httpErr.StatusCode() == http.StatusInternalServerError { + span.Warn("httpcode 500 need to try next, failed on", host, err) + continue + } + return err + } + + // cannot connected most probably + span.Warn("failed on", host, err) + } + + span.Error("try all hosts failed, recently on", host) + return errcode.ErrUnexpected +} diff --git a/blobstore/api/access/client_reqid.go b/blobstore/api/access/client_reqid.go new file mode 100644 index 000000000..442784ffa --- /dev/null +++ b/blobstore/api/access/client_reqid.go @@ -0,0 +1,74 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "context" + + "github.com/cubefs/cubefs/blobstore/common/trace" +) + +type ctxKey uint8 + +const ( + _operationName = "access_client" +) + +const ( + _ ctxKey = iota + reqidKey +) + +// WithRequestID trace request id in full life of the request +// The second parameter rid could be the one of type below: +// a string, +// an interface { String() string }, +// an interface { TraceID() string }, +// an interface { RequestID() string }, +func WithRequestID(ctx context.Context, rid interface{}) context.Context { + return context.WithValue(ctx, reqidKey, rid) +} + +func reqidFromContext(ctx context.Context) (string, bool) { + val := ctx.Value(reqidKey) + if val == nil { + return "", false + } + if rid, ok := val.(string); ok { + return rid, true + } + if rid, ok := val.(interface{ String() string }); ok { + return rid.String(), true + } + if rid, ok := val.(interface{ TraceID() string }); ok { + return rid.TraceID(), true + } + if rid, ok := val.(interface{ RequestID() string }); ok { + return rid.RequestID(), true + } + return "", false +} + +func withReqidContext(ctx context.Context) context.Context { + if rid, ok := reqidFromContext(ctx); ok { + _, ctx := trace.StartSpanFromContextWithTraceID(ctx, _operationName, rid) + return ctx + } + if span := trace.SpanFromContext(ctx); span != nil { + return ctx + } + _, ctx = trace.StartSpanFromContext(ctx, _operationName) + return ctx +} diff --git a/blobstore/api/access/client_test.go b/blobstore/api/access/client_test.go new file mode 100644 index 000000000..621f13231 --- /dev/null +++ b/blobstore/api/access/client_test.go @@ -0,0 +1,936 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access_test + +import ( + "bytes" + "context" + "crypto/rand" + "fmt" + "hash/crc32" + "io" + "io/ioutil" + mrand "math/rand" + "net/http" + "net/http/httptest" + "os" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/hashicorp/consul/api" + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/api/access" + errcode "github.com/cubefs/cubefs/blobstore/common/errors" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/trace" + "github.com/cubefs/cubefs/blobstore/common/uptoken" + "github.com/cubefs/cubefs/blobstore/util/bytespool" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +const ( + blobSize = 1 << 20 +) + +type dataCacheT struct { + mu sync.Mutex + data map[proto.BlobID][]byte +} + +func (c *dataCacheT) clean() { + c.mu.Lock() + c.data = make(map[proto.BlobID][]byte, len(c.data)) + c.mu.Unlock() +} + +func (c *dataCacheT) put(bid proto.BlobID, b []byte) { + c.mu.Lock() + c.data[bid] = b + c.mu.Unlock() +} + +func (c *dataCacheT) get(bid proto.BlobID) []byte { + c.mu.Lock() + defer c.mu.Unlock() + return c.data[bid] +} + +var ( + mockServer *httptest.Server + client access.API + dataCache *dataCacheT + services []*api.ServiceEntry + hostsApply []*api.ServiceEntry + tokenAlloc = []byte("token") + tokenPutat = []byte("token") + + partRandBroken = false +) + +func init() { + log.SetOutputLevel(log.Lfatal) + + t := time.Now() + services = make([]*api.ServiceEntry, 0, 2) + services = append(services, &api.ServiceEntry{ + Node: &api.Node{ + ID: "unreachable", + }, + Service: &api.AgentService{ + Service: "access", + Address: "127.0.0.1", + Port: 9997, + }, + }) + + dataCache = &dataCacheT{} + dataCache.clean() + + rpc.RegisterArgsParser(&access.PutArgs{}, "json") + rpc.RegisterArgsParser(&access.PutAtArgs{}, "json") + + handler := rpc.New() + handler.Handle(http.MethodGet, "/v1/health/service/access", handleService) + handler.Handle(http.MethodPost, "/alloc", handleAlloc, rpc.OptArgsBody()) + handler.Handle(http.MethodPut, "/put", handlePut, rpc.OptArgsQuery()) + handler.Handle(http.MethodPut, "/putat", handlePutAt, rpc.OptArgsQuery()) + handler.Handle(http.MethodPost, "/get", handleGet, rpc.OptArgsBody()) + handler.Handle(http.MethodPost, "/delete", handleDelete, rpc.OptArgsBody()) + handler.Handle(http.MethodPost, "/sign", handleSign, rpc.OptArgsBody()) + handler.Router.NotFound = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusOK) + }) + mockServer = httptest.NewServer(handler) + + u := strings.Split(mockServer.URL[7:], ":") + port, _ := strconv.Atoi(u[1]) + services = append(services, &api.ServiceEntry{ + Node: &api.Node{ + ID: "mockServer", + }, + Service: &api.AgentService{ + Service: "access", + Address: u[0], + Port: port, + }, + }) + hostsApply = services + + cfg := access.Config{} + cfg.Consul.Address = mockServer.URL[7:] // strip http:// + cfg.PriorityAddrs = []string{mockServer.URL} + cfg.ConnMode = access.QuickConnMode + cfg.MaxSizePutOnce = 1 << 20 + cfg.PartConcurrence = 2 + cfg.LogLevel = log.Lfatal + cli, err := access.New(cfg) + if err != nil { + panic(err) + } + client = cli + + mrand.Seed(int64(time.Since(t))) +} + +func handleService(c *rpc.Context) { + c.RespondJSON(hostsApply) +} + +func handleAlloc(c *rpc.Context) { + args := new(access.AllocArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + loc := access.Location{ + ClusterID: 1, + Size: args.Size, + BlobSize: blobSize, + Blobs: []access.SliceInfo{ + { + MinBid: proto.BlobID(mrand.Int()), + Vid: proto.Vid(mrand.Int()), + Count: uint32((args.Size + blobSize - 1) / blobSize), + }, + }, + } + // split to two blobs if large enough + if loc.Blobs[0].Count > 2 { + loc.Blobs[0].Count = 2 + loc.Blobs = append(loc.Blobs, []access.SliceInfo{ + { + MinBid: proto.BlobID(mrand.Int()), + Vid: proto.Vid(mrand.Int()), + Count: uint32((args.Size - 2*blobSize + blobSize - 1) / blobSize), + }, + }...) + } + // alloc the rest parts + if args.AssignClusterID > 0 { + loc.Blobs = []access.SliceInfo{ + { + MinBid: proto.BlobID(mrand.Int()), + Vid: proto.Vid(mrand.Int()), + Count: uint32((args.Size + blobSize - 1) / blobSize), + }, + } + } + + tokens := make([]string, 0, len(loc.Blobs)+1) + + hasMultiBlobs := loc.Size >= uint64(loc.BlobSize) + lastSize := uint32(loc.Size % uint64(loc.BlobSize)) + for idx, blob := range loc.Blobs { + // returns one token if size < blobsize + if hasMultiBlobs { + count := blob.Count + if idx == len(loc.Blobs)-1 && lastSize > 0 { + count-- + } + tokens = append(tokens, uptoken.EncodeToken(uptoken.NewUploadToken(loc.ClusterID, + blob.Vid, blob.MinBid, count, + loc.BlobSize, 0, tokenAlloc[:]))) + } + + // token of the last blob + if idx == len(loc.Blobs)-1 && lastSize > 0 { + tokens = append(tokens, uptoken.EncodeToken(uptoken.NewUploadToken(loc.ClusterID, + blob.Vid, blob.MinBid+proto.BlobID(blob.Count)-1, 1, + lastSize, 0, tokenAlloc[:]))) + } + } + + fillCrc(&loc) + c.RespondJSON(access.AllocResp{ + Location: loc, + Tokens: tokens, + }) +} + +func handlePut(c *rpc.Context) { + args := new(access.PutArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + // just for testing timeout + if args.Size < 0 { + time.Sleep(30 * time.Second) + c.RespondStatus(http.StatusForbidden) + return + } + + buf := make([]byte, args.Size) + io.ReadFull(c.Request.Body, buf) + dataCache.put(0, buf) + + hashSumMap := args.Hashes.ToHashSumMap() + for alg := range hashSumMap { + hasher := alg.ToHasher() + hasher.Write(buf) + hashSumMap[alg] = hasher.Sum(nil) + } + + loc := access.Location{Size: uint64(args.Size)} + fillCrc(&loc) + c.RespondJSON(access.PutResp{ + Location: loc, + HashSumMap: hashSumMap, + }) +} + +func handlePutAt(c *rpc.Context) { + args := new(access.PutAtArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + if partRandBroken && args.Blobid%3 == 0 { // random broken + c.RespondStatus(http.StatusForbidden) + return + } + + token := uptoken.DecodeToken(args.Token) + if !token.IsValid(args.ClusterID, args.Vid, args.Blobid, uint32(args.Size), tokenPutat[:]) { + c.RespondStatus(http.StatusForbidden) + return + } + + blobBuf := make([]byte, args.Size) + io.ReadFull(c.Request.Body, blobBuf) + + hashSumMap := args.Hashes.ToHashSumMap() + for alg := range hashSumMap { + hasher := alg.ToHasher() + hasher.Write(blobBuf) + hashSumMap[alg] = hasher.Sum(nil) + } + + dataCache.put(args.Blobid, blobBuf) + c.RespondJSON(access.PutAtResp{HashSumMap: hashSumMap}) +} + +func handleGet(c *rpc.Context) { + args := new(access.GetArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + if !verifyCrc(&args.Location) { + c.RespondStatus(http.StatusForbidden) + return + } + + c.Writer.Header().Set(rpc.HeaderContentLength, strconv.Itoa(int(args.ReadSize))) + c.RespondStatus(http.StatusOK) + if buf := dataCache.get(0); len(buf) > 0 { + c.Writer.Write(buf) + } else { + for _, blob := range args.Location.Spread() { + c.Writer.Write(dataCache.get(blob.Bid)) + } + } +} + +func handleDelete(c *rpc.Context) { + args := new(access.DeleteArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + if !args.IsValid() { + c.RespondStatus(http.StatusBadRequest) + return + } + for _, loc := range args.Locations { + if !verifyCrc(&loc) { + c.RespondStatus(http.StatusBadRequest) + return + } + } + + if len(args.Locations) > 0 && len(args.Locations)%2 == 0 { + locs := args.Locations[:] + c.RespondStatusData(http.StatusIMUsed, access.DeleteResp{FailedLocations: locs}) + return + } + c.RespondJSON(access.DeleteResp{}) +} + +func handleSign(c *rpc.Context) { + args := new(access.SignArgs) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + + if err := signCrc(&args.Location, args.Locations); err != nil { + c.RespondStatus(http.StatusForbidden) + return + } + c.RespondJSON(access.SignResp{Location: args.Location}) +} + +func calcCrc(loc *access.Location) (uint32, error) { + crcWriter := crc32.New(crc32.IEEETable) + + buf := bytespool.Alloc(1024) + defer bytespool.Free(buf) + + n := loc.Encode2(buf) + if n < 4 { + return 0, fmt.Errorf("no enough bytes(%d) fill into buf", n) + } + + if _, err := crcWriter.Write(buf[4:n]); err != nil { + return 0, fmt.Errorf("fill crc %s", err.Error()) + } + + return crcWriter.Sum32(), nil +} + +func fillCrc(loc *access.Location) error { + crc, err := calcCrc(loc) + if err != nil { + return err + } + loc.Crc = crc + return nil +} + +func verifyCrc(loc *access.Location) bool { + crc, err := calcCrc(loc) + if err != nil { + return false + } + return loc.Crc == crc +} + +func signCrc(loc *access.Location, locs []access.Location) error { + first := locs[0] + bids := make(map[proto.BlobID]struct{}, 64) + + if loc.ClusterID != first.ClusterID || + loc.CodeMode != first.CodeMode || + loc.BlobSize != first.BlobSize { + return fmt.Errorf("not equal in constant field") + } + + for _, l := range locs { + if !verifyCrc(&l) { + return fmt.Errorf("not equal in crc %d", l.Crc) + } + + // assert + if l.ClusterID != first.ClusterID || + l.CodeMode != first.CodeMode || + l.BlobSize != first.BlobSize { + return fmt.Errorf("not equal in constant field") + } + + for _, blob := range l.Blobs { + for c := 0; c < int(blob.Count); c++ { + bids[blob.MinBid+proto.BlobID(c)] = struct{}{} + } + } + } + + for _, blob := range loc.Blobs { + for c := 0; c < int(blob.Count); c++ { + bid := blob.MinBid + proto.BlobID(c) + if _, ok := bids[bid]; !ok { + return fmt.Errorf("not equal in blob_id(%d)", bid) + } + } + } + + return fillCrc(loc) +} + +type stringid struct{ id string } + +func (s stringid) String() string { return s.id } + +type traceid struct{ id string } + +func (t traceid) TraceID() string { return t.id } + +type requestid struct{ id string } + +func (r requestid) RequestID() string { return r.id } + +func randCtx() context.Context { + ctx := context.Background() + + switch mrand.Int31() % 7 { + case 0: + return ctx + case 1: + return access.WithRequestID(ctx, nil) + case 2: + return access.WithRequestID(ctx, "TestAccessClient-string") + case 3: + return access.WithRequestID(ctx, stringid{"TestAccessClient-String"}) + case 4: + return access.WithRequestID(ctx, traceid{"TestAccessClient-TraceID"}) + case 5: + return access.WithRequestID(ctx, requestid{"TestAccessClient-RequestID"}) + case 6: + return access.WithRequestID(ctx, struct{}{}) + default: + } + + _, ctx = trace.StartSpanFromContext(ctx, "TestAccessClient") + return ctx +} + +func TestAccessClientConnectionMode(t *testing.T) { + cases := []struct { + mode access.RPCConnectMode + size int64 + }{ + {0, 0}, + {0, -1}, + {access.QuickConnMode, 1 << 10}, + {access.GeneralConnMode, 1 << 10}, + {access.SlowConnMode, 1 << 10}, + {access.NoLimitConnMode, 1 << 10}, + {access.DefaultConnMode, 1 << 10}, + {100, 1 << 10}, + {0xff, 100}, + } + + for _, cs := range cases { + cfg := access.Config{} + cfg.Consul.Address = mockServer.URL[7:] + cfg.MaxSizePutOnce = cs.size + cfg.ConnMode = cs.mode + cfg.LogLevel = log.Lfatal + cli, err := access.New(cfg) + require.NoError(t, err) + + if cs.size <= 0 { + continue + } + loc := access.Location{Size: uint64(mrand.Int63n(cs.size))} + fillCrc(&loc) + _, err = cli.Delete(randCtx(), &access.DeleteArgs{ + Locations: []access.Location{loc}, + }) + require.NoError(t, err) + } +} + +func TestAccessClientPutGet(t *testing.T) { + cases := []struct { + size int + }{ + {0}, + {1}, + {1 << 6}, + {1 << 10}, + {1 << 19}, + {(1 << 19) + 1}, + {1 << 20}, + } + for _, cs := range cases { + buff := make([]byte, cs.size) + rand.Read(buff) + crcExpected := crc32.ChecksumIEEE(buff) + args := access.PutArgs{ + Size: int64(cs.size), + Hashes: access.HashAlgCRC32, + Body: bytes.NewBuffer(buff), + } + + loc, hashSumMap, err := client.Put(randCtx(), &args) + crc, _ := hashSumMap.GetSum(access.HashAlgCRC32) + require.NoError(t, err) + require.Equal(t, crcExpected, crc) + + body, err := client.Get(randCtx(), &access.GetArgs{Location: loc, ReadSize: uint64(cs.size)}) + require.NoError(t, err) + defer body.Close() + + io.ReadFull(body, buff) + require.Equal(t, crcExpected, crc32.ChecksumIEEE(buff)) + } +} + +func TestAccessClientPutAtBase(t *testing.T) { + cfg := access.Config{} + cfg.Consul.Address = mockServer.URL[7:] + cfg.MaxSizePutOnce = 1 << 20 + cfg.PartConcurrence = 2 + cfg.LogLevel = log.Lfatal + client, err := access.New(cfg) + require.NoError(t, err) + + cases := []struct { + size int + }{ + {0}, + {1}, + {1 << 10}, + {(1 << 19) + 1}, + {1 << 20}, + {1<<20 + 1023}, + {1<<23 + 1023}, + {1 << 24}, + } + for _, cs := range cases { + dataCache.clean() + + buff := make([]byte, cs.size) + rand.Read(buff) + crcExpected := crc32.ChecksumIEEE(buff) + args := access.PutArgs{ + Size: int64(cs.size), + Hashes: access.HashAlgCRC32, + Body: bytes.NewBuffer(buff), + } + + loc, hashSumMap, err := client.Put(randCtx(), &args) + crc, _ := hashSumMap.GetSum(access.HashAlgCRC32) + require.NoError(t, err) + require.Equal(t, crcExpected, crc) + + body, err := client.Get(randCtx(), &access.GetArgs{Location: loc, ReadSize: uint64(cs.size)}) + require.NoError(t, err) + defer body.Close() + + io.ReadFull(body, buff) + require.Equal(t, crcExpected, crc32.ChecksumIEEE(buff)) + } +} + +func TestAccessClientPutAtMerge(t *testing.T) { + cfg := access.Config{} + cfg.Consul.Address = mockServer.URL[7:] + cfg.MaxSizePutOnce = 1 << 20 + cfg.PartConcurrence = 2 + cfg.LogLevel = log.Lfatal + client, err := access.New(cfg) + require.NoError(t, err) + + partRandBroken = true + defer func() { + partRandBroken = false + }() + dataCache.clean() + + cases := []struct { + size int + }{ + {1<<21 + 77777}, + {1<<21 + 77777}, + {1<<21 + 77777}, + {1 << 22}, + {1 << 22}, + {1 << 22}, + } + for _, cs := range cases { + dataCache.clean() + + size := cs.size + buff := make([]byte, size) + rand.Read(buff) + crcExpected := crc32.ChecksumIEEE(buff) + args := access.PutArgs{ + Size: int64(size), + Hashes: access.HashAlgCRC32, + Body: bytes.NewBuffer(buff), + } + + loc, hashSumMap, err := client.Put(randCtx(), &args) + crc, _ := hashSumMap.GetSum(access.HashAlgCRC32) + require.NoError(t, err) + require.Equal(t, crcExpected, crc) + + body, err := client.Get(randCtx(), &access.GetArgs{Location: loc, ReadSize: uint64(size)}) + require.NoError(t, err) + defer body.Close() + + io.ReadFull(body, buff) + require.Equal(t, crcExpected, crc32.ChecksumIEEE(buff)) + } +} + +func TestAccessClientPutMaxBlobsLength(t *testing.T) { + cfg := access.Config{} + cfg.Consul.Address = mockServer.URL[7:] + cfg.MaxSizePutOnce = 1 << 20 + cfg.PartConcurrence = 2 + cfg.LogLevel = log.Lfatal + client, err := access.New(cfg) + require.NoError(t, err) + + partRandBroken = true + defer func() { + partRandBroken = false + }() + + cases := []struct { + size int + err error + }{ + {1 << 20, nil}, + {(1 << 21) + 1023, nil}, + {1 << 25, errcode.ErrUnexpected}, + } + for _, cs := range cases { + dataCache.clean() + + buff := make([]byte, cs.size) + rand.Read(buff) + args := access.PutArgs{ + Size: int64(cs.size), + Hashes: access.HashAlgDummy, + Body: bytes.NewBuffer(buff), + } + + _, _, err := client.Put(randCtx(), &args) + require.ErrorIs(t, cs.err, err) + } +} + +func linearTimeoutMs(baseSec, size, speedMBps float64) int64 { + const alignMs = 500 + spentSec := size / 1024.0 / 1024.0 / speedMBps + timeoutMs := int64((baseSec + spentSec) * 1000) + if timeoutMs < 0 { + timeoutMs = 0 + } + if ms := timeoutMs % alignMs; ms > 0 { + timeoutMs += (alignMs - ms) + } + return timeoutMs +} + +func TestAccessClientPutTimeout(t *testing.T) { + cfg := access.Config{} + cfg.Consul.Address = mockServer.URL[7:] + cfg.LogLevel = log.Lfatal + + mb := int(1 << 20) + ms := time.Millisecond + + cases := []struct { + mode access.RPCConnectMode + size int + minMs time.Duration + maxMs time.Duration + }{ + // QuickConnMode 3s + size / 40, dial and response 2s + {access.QuickConnMode, mb * -119, ms * 0, ms * 600}, + {access.QuickConnMode, mb * -100, ms * 500, ms * 1000}, + {access.QuickConnMode, mb * -80, ms * 1000, ms * 1500}, + {access.QuickConnMode, mb * -1, ms * 2000, ms * 2500}, + + // DefaultConnMode 30s + size / 10, dial and response 5s + {access.DefaultConnMode, mb * -299, ms * 0, ms * 600}, + {access.DefaultConnMode, mb * -280, ms * 2000, ms * 2500}, + {access.DefaultConnMode, mb * -270, ms * 3000, ms * 3500}, + {access.DefaultConnMode, mb * -1, ms * 5000, ms * 5500}, + } + for _, cs := range cases { + cfg.ConnMode = cs.mode + switch cs.mode { + case access.QuickConnMode: + cfg.ClientTimeoutMs = linearTimeoutMs(3, float64(cs.size), 40) + case access.DefaultConnMode: + cfg.ClientTimeoutMs = linearTimeoutMs(30, float64(cs.size), 10) + } + client, err := access.New(cfg) + require.NoError(t, err) + + buff := make([]byte, 0) + args := access.PutArgs{ + Size: int64(cs.size), + Body: bytes.NewBuffer(buff), + } + + startTime := time.Now() + _, _, err = client.Put(randCtx(), &args) + require.Error(t, err) + duration := time.Since(startTime) + require.GreaterOrEqual(t, cs.maxMs, duration, "greater duration: ", duration) + require.LessOrEqual(t, cs.minMs, duration, "less duration: ", duration) + } +} + +func TestAccessClientDelete(t *testing.T) { + { + locs, err := client.Delete(randCtx(), nil) + require.Nil(t, locs) + require.ErrorIs(t, errcode.ErrIllegalArguments, err) + } + { + locs, err := client.Delete(randCtx(), &access.DeleteArgs{}) + require.Nil(t, locs) + require.ErrorIs(t, errcode.ErrIllegalArguments, err) + } + { + locs, err := client.Delete(randCtx(), &access.DeleteArgs{ + Locations: make([]access.Location, 1), + }) + require.Nil(t, locs) + require.NoError(t, err) + } + { + args := &access.DeleteArgs{ + Locations: make([]access.Location, 1000), + } + _, err := client.Delete(randCtx(), args) + require.NoError(t, err) + } + { + args := &access.DeleteArgs{ + Locations: make([]access.Location, access.MaxDeleteLocations+1), + } + locs, err := client.Delete(randCtx(), args) + require.Equal(t, args.Locations, locs) + require.ErrorIs(t, errcode.ErrIllegalArguments, err) + } + { + loc := access.Location{Size: 100, Blobs: make([]access.SliceInfo, 0)} + fillCrc(&loc) + args := &access.DeleteArgs{ + Locations: make([]access.Location, 0, access.MaxDeleteLocations), + } + for i := 1; i < access.MaxDeleteLocations/10; i++ { + args.Locations = append(args.Locations, loc) + locs, err := client.Delete(randCtx(), args) + if i%2 == 0 { + require.Equal(t, args.Locations, locs) + require.Equal(t, errcode.ErrUnexpected, err) + } else { + require.NoError(t, err) + } + } + } +} + +func TestAccessClientRequestBody(t *testing.T) { + cfg := access.Config{} + cfg.Consul.Address = mockServer.URL[7:] + cfg.MaxSizePutOnce = 1 << 20 + cfg.PartConcurrence = 2 + cfg.LogLevel = log.Lfatal + cfg.PriorityAddrs = []string{mockServer.URL} + client, err := access.New(cfg) + require.NoError(t, err) + + cases := []struct { + size int + }{ + {0}, + {1}, + {1 << 10}, + {(1 << 19) + 1}, + {1 << 20}, + {1<<20 + 1023}, + {1<<23 + 1023}, + {1 << 24}, + } + for _, cs := range cases { + dataCache.clean() + + buff := make([]byte, cs.size) + rand.Read(buff) + args := access.PutArgs{ + Size: int64(cs.size) + 1, + Body: bytes.NewBuffer(buff), + } + + _, _, err := client.Put(randCtx(), &args) + if cs.size < int(cfg.MaxSizePutOnce) { + if cs.size <= 1<<22 { + require.Equal(t, errcode.ErrAccessReadRequestBody, err) + } else { + require.Equal(t, errcode.ErrAccessReadConflictBody, err) + } + } else { + require.Equal(t, errcode.ErrAccessReadRequestBody, err) + } + } +} + +func TestAccessClientPutAtToken(t *testing.T) { + tokenKey := tokenPutat + defer func() { + tokenPutat = tokenKey + }() + + cfg := access.Config{} + cfg.Consul.Address = mockServer.URL[7:] + cfg.MaxSizePutOnce = 1 << 20 + cfg.PartConcurrence = 1 + cfg.MaxPartRetry = -1 + cfg.LogLevel = log.Lfatal + client, err := access.New(cfg) + require.NoError(t, err) + + cases := []struct { + keyLen int + }{ + {0}, + {1}, + {20}, + {100}, + } + buff := make([]byte, 1<<21) + rand.Read(buff) + for _, cs := range cases { + tokenPutat = make([]byte, cs.keyLen) + rand.Read(tokenPutat) + args := access.PutArgs{ + Size: int64(1 << 21), + Body: bytes.NewBuffer(buff), + } + _, _, err := client.Put(randCtx(), &args) + require.ErrorIs(t, errcode.ErrUnexpected, err) + } +} + +func TestAccessClientRPCConfig(t *testing.T) { + cfg := access.Config{} + cfg.RPCConfig = &rpc.Config{} + cfg.LogLevel = log.Lfatal + cfg.Consul.Address = mockServer.URL[7:] + cfg.PriorityAddrs = []string{mockServer.URL} + client, err := access.New(cfg) + require.NoError(t, err) + cases := []struct { + size int + }{ + {0}, + {1}, + {1 << 6}, + {1 << 10}, + } + for _, cs := range cases { + buff := make([]byte, cs.size) + rand.Read(buff) + args := access.PutArgs{ + Size: int64(cs.size), + Body: bytes.NewBuffer(buff), + } + loc, _, err := client.Put(randCtx(), &args) + require.NoError(t, err) + _, err = client.Get(randCtx(), &access.GetArgs{Location: loc, ReadSize: uint64(cs.size)}) + require.NoError(t, err) + } +} + +func TestAccessClientLogger(t *testing.T) { + file, err := ioutil.TempFile(os.TempDir(), "TestAccessClientLogger") + require.NoError(t, err) + require.NoError(t, file.Close()) + defer func() { + os.Remove(file.Name()) + }() + + cfg := access.Config{ + LogLevel: log.Lfatal, + Logger: &access.Logger{ + Filename: file.Name(), + }, + } + cfg.Consul.Address = mockServer.URL[7:] + client, err := access.New(cfg) + require.NoError(t, err) + defer func() { + cfg.Logger.Close() + }() + + size := 1024 + args := access.PutArgs{ + Size: int64(size), + Body: bytes.NewBuffer(make([]byte, size-1)), + } + _, _, err = client.Put(randCtx(), &args) + require.Error(t, err) +} diff --git a/blobstore/api/access/proto.go b/blobstore/api/access/proto.go new file mode 100644 index 000000000..ad06c7753 --- /dev/null +++ b/blobstore/api/access/proto.go @@ -0,0 +1,632 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "fmt" + "hash" + "hash/crc32" + "io" + + "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +// HashAlgorithm hash.Hash algorithm when uploading data +type HashAlgorithm uint8 + +// defined hash algorithm +const ( + HashAlgDummy HashAlgorithm = 1 << iota + HashAlgCRC32 // crc32 with IEEE + HashAlgMD5 // md5 + HashAlgSHA1 // sha1 + HashAlgSHA256 // sha256 +) + +const ( + // HashSize dummy hash size + HashSize = 0 + + // MaxLocationBlobs max blobs length in Location + MaxLocationBlobs uint32 = 4 + // MaxDeleteLocations max locations of delete request + MaxDeleteLocations int = 1024 + // MaxBlobSize max blob size for allocation + MaxBlobSize uint32 = 1 << 25 // 32MB +) + +type dummyHash struct{} + +var _ hash.Hash = (*dummyHash)(nil) + +// implements hash.Hash +func (d dummyHash) Write(p []byte) (n int, err error) { return len(p), nil } +func (d dummyHash) Sum(b []byte) []byte { return []byte{} } +func (d dummyHash) Reset() {} +func (d dummyHash) Size() int { return 0 } +func (d dummyHash) BlockSize() int { return 0 } + +// ToHasher returns a new hash.Hash computing checksum +// the value of algorithm should be one of HashAlg* +func (alg HashAlgorithm) ToHasher() hash.Hash { + switch alg { + case HashAlgCRC32: + return crc32.NewIEEE() + case HashAlgMD5: + return md5.New() + case HashAlgSHA1: + return sha1.New() + case HashAlgSHA256: + return sha256.New() + default: + return dummyHash{} + } +} + +// ToHashSumMap returns a new HashSumMap, decode from rpc url argument +func (alg HashAlgorithm) ToHashSumMap() HashSumMap { + h := make(HashSumMap) + for _, a := range []HashAlgorithm{ + HashAlgDummy, + HashAlgCRC32, + HashAlgMD5, + HashAlgSHA1, + HashAlgSHA256, + } { + if alg&a == a { + h[a] = nil + } + } + return h +} + +// HasherMap map hasher of HashAlgorithm +type HasherMap map[HashAlgorithm]hash.Hash + +// ToHashAlgorithm returns HashAlgorithm +func (h HasherMap) ToHashAlgorithm() HashAlgorithm { + alg := HashAlgorithm(0) + for k := range h { + alg |= k + } + return alg +} + +// ToWriter returns io writer +func (h HasherMap) ToWriter() io.Writer { + writers := make([]io.Writer, 0, len(h)) + for _, hasher := range h { + writers = append(writers, hasher) + } + return io.MultiWriter(writers...) +} + +// HashSumMap save checksum in rpc calls +type HashSumMap map[HashAlgorithm][]byte + +// GetSum get checksum value and ok via HashAlgorithm +// HashAlgDummy returns nil, bool +// HashAlgCRC32 returns uint32, bool +// HashAlgMD5 returns string(32), bool +// HashAlgSHA1 returns string(40), bool +// HashAlgSHA256 returns string(64), bool +func (h HashSumMap) GetSum(key HashAlgorithm) (interface{}, bool) { + b, ok := h[key] + if !ok { + return nil, false + } + + switch key { + case HashAlgCRC32: + if len(b) != crc32.Size { + return nil, false + } + return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24, true + case HashAlgMD5: + if len(b) != md5.Size { + return nil, false + } + return hex.EncodeToString(b[:]), true + case HashAlgSHA1: + if len(b) != sha1.Size { + return nil, false + } + return hex.EncodeToString(b[:]), true + case HashAlgSHA256: + if len(b) != sha256.Size { + return nil, false + } + return hex.EncodeToString(b[:]), true + default: + if len(b) != HashSize { + return nil, false + } + return nil, true + } +} + +// GetSumVal get checksum only value via HashAlgorithm +func (h HashSumMap) GetSumVal(key HashAlgorithm) interface{} { + val, _ := h.GetSum(key) + return val +} + +// ToHashAlgorithm returns HashAlgorithm, encode to rpc url argument +func (h HashSumMap) ToHashAlgorithm() HashAlgorithm { + alg := HashAlgorithm(0) + for k := range h { + alg |= k + } + return alg +} + +// All returns readable checksum +func (h HashSumMap) All() map[string]interface{} { + m := make(map[string]interface{}) + for a, name := range map[HashAlgorithm]string{ + HashAlgCRC32: "crc32", + HashAlgMD5: "md5", + HashAlgSHA1: "sha1", + HashAlgSHA256: "sha256", + } { + if val, ok := h.GetSum(a); ok { + m[name] = val + } + } + return m +} + +// Location file location, 4 + 1 + 8 + 4 + 4 + len*16 bytes +// | | +// | ClusterID(4) | CodeMode(1) | +// | Size(8) | +// | BlobSize(4) | Crc(4) | +// | len*SliceInfo(16) | +// +// ClusterID which cluster file is in +// CodeMode is ec encode mode, see defined in "common/lib/codemode" +// Size is file size +// BlobSize is every blob's size but the last one which's size=(Size mod BlobSize) +// Crc is the checksum, change anything of the location, crc will mismatch +// Blobs all blob information +type Location struct { + _ [0]byte + ClusterID proto.ClusterID `json:"cluster_id"` + CodeMode codemode.CodeMode `json:"code_mode"` + Size uint64 `json:"size"` + BlobSize uint32 `json:"blob_size"` + Crc uint32 `json:"crc"` + Blobs []SliceInfo `json:"blobs"` +} + +// SliceInfo blobs info, 8 + 4 + 4 bytes +// +// MinBid is the first blob id +// Vid is which volume all blobs in +// Count is num of consecutive blob ids, count=1 just has one blob +// +// blob ids = [MinBid, MinBid+count) +type SliceInfo struct { + _ [0]byte + MinBid proto.BlobID `json:"min_bid"` + Vid proto.Vid `json:"vid"` + Count uint32 `json:"count"` +} + +// Blob is one piece of data in a location +// +// Bid is the blob id +// Vid is which volume the blob in +// Size is real size of the blob +type Blob struct { + Bid proto.BlobID + Vid proto.Vid + Size uint32 +} + +// Copy returns a new same Location +func (loc *Location) Copy() Location { + dst := Location{ + ClusterID: loc.ClusterID, + CodeMode: loc.CodeMode, + Size: loc.Size, + BlobSize: loc.BlobSize, + Crc: loc.Crc, + Blobs: make([]SliceInfo, len(loc.Blobs)), + } + copy(dst.Blobs, loc.Blobs) + return dst +} + +// Encode transfer Location to slice byte +// Returns the buf created by me +// (n) means max-n bytes +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// | field | crc | clusterid | codemode | size | blobsize | +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// | n-bytes | 4 | uvarint(5) | 1 | uvarint(10) | uvarint(5) | +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// 25 + (5){len(blobs)} + len(Blobs) * 20 +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// | blobs | minbid | vid | count | ... | +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// | n-bytes | (10) | (5) | (5) | (20) | (20) | ... | +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +func (loc *Location) Encode() []byte { + if loc == nil { + return nil + } + n := 25 + 5 + len(loc.Blobs)*20 + buf := make([]byte, n) + n = loc.Encode2(buf) + return buf[:n] +} + +// Encode2 transfer Location to the buf, the buf reuse by yourself +// Returns the number of bytes read +// If the buffer is too small, Encode2 will panic +func (loc *Location) Encode2(buf []byte) int { + if loc == nil { + return 0 + } + + n := 0 + binary.BigEndian.PutUint32(buf[n:], loc.Crc) + n += 4 + n += binary.PutUvarint(buf[n:], uint64(loc.ClusterID)) + buf[n] = byte(loc.CodeMode) + n++ + n += binary.PutUvarint(buf[n:], uint64(loc.Size)) + n += binary.PutUvarint(buf[n:], uint64(loc.BlobSize)) + + n += binary.PutUvarint(buf[n:], uint64(len(loc.Blobs))) + for _, blob := range loc.Blobs { + n += binary.PutUvarint(buf[n:], uint64(blob.MinBid)) + n += binary.PutUvarint(buf[n:], uint64(blob.Vid)) + n += binary.PutUvarint(buf[n:], uint64(blob.Count)) + } + + return n +} + +// Decode parse location from buf +// Returns the number of bytes read +// Error is not nil when parsing failed +func (loc *Location) Decode(buf []byte) (int, error) { + if loc == nil { + return 0, fmt.Errorf("location receiver is nil") + } + + location, n, err := DecodeLocation(buf) + if err != nil { + return n, err + } + + *loc = location + return n, nil +} + +// ToString transfer location to hex string +func (loc *Location) ToString() string { + return loc.HexString() +} + +// HexString transfer location to hex string +func (loc *Location) HexString() string { + return hex.EncodeToString(loc.Encode()) +} + +// Base64String transfer location to base64 string +func (loc *Location) Base64String() string { + return base64.StdEncoding.EncodeToString(loc.Encode()) +} + +// Spread location blobs to slice +func (loc *Location) Spread() []Blob { + count := 0 + for _, blob := range loc.Blobs { + count += int(blob.Count) + } + + blobs := make([]Blob, 0, count) + for _, blob := range loc.Blobs { + for offset := uint32(0); offset < blob.Count; offset++ { + blobs = append(blobs, Blob{ + Bid: blob.MinBid + proto.BlobID(offset), + Vid: blob.Vid, + Size: loc.BlobSize, + }) + } + } + if len(blobs) > 0 && loc.BlobSize > 0 { + if lastSize := loc.Size % uint64(loc.BlobSize); lastSize > 0 { + blobs[len(blobs)-1].Size = uint32(lastSize) + } + } + return blobs +} + +// DecodeLocation parse location from buf +// Returns Location and the number of bytes read +// Error is not nil when parsing failed +func DecodeLocation(buf []byte) (Location, int, error) { + var ( + loc Location + n int + + val uint64 + nn int + ) + next := func() (uint64, int) { + val, nn := binary.Uvarint(buf) + if nn <= 0 { + return 0, nn + } + n += nn + buf = buf[nn:] + return val, nn + } + + if len(buf) < 4 { + return loc, n, fmt.Errorf("bytes crc %d", len(buf)) + } + loc.Crc = binary.BigEndian.Uint32(buf) + n += 4 + buf = buf[4:] + + if val, nn = next(); nn <= 0 { + return loc, n, fmt.Errorf("bytes cluster_id %d", nn) + } + loc.ClusterID = proto.ClusterID(val) + + if len(buf) < 1 { + return loc, n, fmt.Errorf("bytes codemode %d", len(buf)) + } + loc.CodeMode = codemode.CodeMode(buf[0]) + n++ + buf = buf[1:] + + if val, nn = next(); nn <= 0 { + return loc, n, fmt.Errorf("bytes size %d", nn) + } + loc.Size = val + + if val, nn = next(); nn <= 0 { + return loc, n, fmt.Errorf("bytes blob_size %d", nn) + } + loc.BlobSize = uint32(val) + + if val, nn = next(); nn <= 0 { + return loc, n, fmt.Errorf("bytes length blobs %d", nn) + } + length := int(val) + + if length > 0 { + loc.Blobs = make([]SliceInfo, 0, length) + } + for index := 0; index < length; index++ { + var blob SliceInfo + if val, nn = next(); nn <= 0 { + return loc, n, fmt.Errorf("bytes %dth-blob min_bid %d", index, nn) + } + blob.MinBid = proto.BlobID(val) + + if val, nn = next(); nn <= 0 { + return loc, n, fmt.Errorf("bytes %dth-blob vid %d", index, nn) + } + blob.Vid = proto.Vid(val) + + if val, nn = next(); nn <= 0 { + return loc, n, fmt.Errorf("bytes %dth-blob count %d", index, nn) + } + blob.Count = uint32(val) + + loc.Blobs = append(loc.Blobs, blob) + } + + return loc, n, nil +} + +// DecodeLocationFrom decode location from hex string +func DecodeLocationFrom(s string) (Location, error) { + return DecodeLocationFromHex(s) +} + +// DecodeLocationFromHex decode location from hex string +func DecodeLocationFromHex(s string) (Location, error) { + var loc Location + src, err := hex.DecodeString(s) + if err != nil { + return loc, err + } + _, err = loc.Decode(src) + if err != nil { + return loc, err + } + return loc, nil +} + +// DecodeLocationFromBase64 decode location from base64 string +func DecodeLocationFromBase64(s string) (Location, error) { + var loc Location + src, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return loc, err + } + _, err = loc.Decode(src) + if err != nil { + return loc, err + } + return loc, nil +} + +// PutArgs for service /put +// Hashes means how to calculate check sum, +// HashAlgCRC32 | HashAlgMD5 equal 2 + 4 = 6 +type PutArgs struct { + Size int64 `json:"size"` + Hashes HashAlgorithm `json:"hashes,omitempty"` + Body io.Reader `json:"-"` +} + +// IsValid is valid put args +func (args *PutArgs) IsValid() bool { + if args == nil { + return false + } + return args.Size > 0 +} + +// PutResp put response result +type PutResp struct { + Location Location `json:"location"` + HashSumMap HashSumMap `json:"hashsum"` +} + +// PutAtArgs for service /putat +type PutAtArgs struct { + ClusterID proto.ClusterID `json:"clusterid"` + Vid proto.Vid `json:"volumeid"` + Blobid proto.BlobID `json:"blobid"` + Size int64 `json:"size"` + Hashes HashAlgorithm `json:"hashes,omitempty"` + Token string `json:"token"` + Body io.Reader `json:"-"` +} + +// IsValid is valid putat args +func (args *PutAtArgs) IsValid() bool { + if args == nil { + return false + } + return args.ClusterID > proto.ClusterID(0) && + args.Vid > proto.Vid(0) && + args.Blobid > proto.BlobID(0) && + args.Size > 0 +} + +// PutAtResp putat response result +type PutAtResp struct { + HashSumMap HashSumMap `json:"hashsum"` +} + +// AllocArgs for service /alloc +type AllocArgs struct { + Size uint64 `json:"size"` + BlobSize uint32 `json:"blob_size"` + AssignClusterID proto.ClusterID `json:"assign_cluster_id"` + CodeMode codemode.CodeMode `json:"code_mode"` +} + +// IsValid is valid alloc args +func (args *AllocArgs) IsValid() bool { + if args == nil { + return false + } + if args.AssignClusterID > 0 { + return args.Size > 0 && args.BlobSize > 0 && args.BlobSize <= MaxBlobSize && + args.CodeMode.IsValid() + } + return args.Size > 0 && args.BlobSize <= MaxBlobSize +} + +// AllocResp alloc response result with tokens +// if size mod blobsize == 0, length of tokens equal length of location blobs +// otherwise additional token for the last blob uploading +type AllocResp struct { + Location Location `json:"location"` + Tokens []string `json:"tokens"` +} + +// GetArgs for service /get +type GetArgs struct { + Location Location `json:"location"` + Offset uint64 `json:"offset"` + ReadSize uint64 `json:"read_size"` +} + +// IsValid is valid get args +func (args *GetArgs) IsValid() bool { + if args == nil { + return false + } + return args.Offset+args.ReadSize <= args.Location.Size +} + +// DeleteArgs for service /delete +type DeleteArgs struct { + Locations []Location `json:"locations"` +} + +// IsValid is valid delete args +func (args *DeleteArgs) IsValid() bool { + if args == nil { + return false + } + return len(args.Locations) > 0 && len(args.Locations) <= MaxDeleteLocations +} + +// DeleteResp delete response with failed locations +type DeleteResp struct { + FailedLocations []Location `json:"failed_locations,omitempty"` +} + +// DeleteBlobArgs for service /deleteblob +type DeleteBlobArgs struct { + ClusterID proto.ClusterID `json:"clusterid"` + Vid proto.Vid `json:"volumeid"` + Blobid proto.BlobID `json:"blobid"` + Size int64 `json:"size"` + Token string `json:"token"` +} + +// IsValid is valid delete blob args +func (args *DeleteBlobArgs) IsValid() bool { + if args == nil { + return false + } + return args.ClusterID > proto.ClusterID(0) && + args.Vid > proto.Vid(0) && + args.Blobid > proto.BlobID(0) && + args.Size > 0 +} + +// SignArgs for service /sign +// Locations are signed location getting from /alloc +// Location is to be signed location which merged by yourself +type SignArgs struct { + Locations []Location `json:"locations"` + Location Location `json:"location"` +} + +// IsValid is valid sign args +func (args *SignArgs) IsValid() bool { + if args == nil { + return false + } + return len(args.Locations) > 0 +} + +// SignResp sign response location with crc +type SignResp struct { + Location Location `json:"location"` +} diff --git a/blobstore/api/access/proto_test.go b/blobstore/api/access/proto_test.go new file mode 100644 index 000000000..06d3556e2 --- /dev/null +++ b/blobstore/api/access/proto_test.go @@ -0,0 +1,566 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access_test + +import ( + "crypto/md5" + "crypto/rand" + "crypto/sha1" + "crypto/sha256" + "hash/crc32" + "math" + mrand "math/rand" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/api/access" + "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +func TestHashAlgorithm(t *testing.T) { + cases := []struct { + alg access.HashAlgorithm + size int + sumLen int + }{ + {0, 0, access.HashSize}, + {0b0, 1 << 10, access.HashSize}, + {access.HashAlgorithm(0), 1 << 14, access.HashSize}, + + {1, 0, access.HashSize}, + {0b1, 1 << 10, access.HashSize}, + {access.HashAlgDummy, 1 << 14, access.HashSize}, + + {2, 0, crc32.Size}, + {0b10, 1 << 10, crc32.Size}, + {access.HashAlgCRC32, 1 << 14, crc32.Size}, + + {4, 0, md5.Size}, + {0b100, 1 << 10, md5.Size}, + {access.HashAlgMD5, 1 << 14, md5.Size}, + + {8, 0, sha1.Size}, + {0b1000, 1 << 10, sha1.Size}, + {access.HashAlgSHA1, 1 << 14, sha1.Size}, + + {16, 0, sha256.Size}, + {0b10000, 1 << 10, sha256.Size}, + {access.HashAlgSHA256, 1 << 14, sha256.Size}, + + {100, 0, access.HashSize}, + {0b1111111, 1 << 10, access.HashSize}, + {access.HashAlgDummy, 1 << 14, access.HashSize}, + } + for _, cs := range cases { + buffer := make([]byte, cs.size) + rand.Read(buffer) + + hasher := cs.alg.ToHasher() + _, err := hasher.Write(buffer) + require.NoError(t, err) + require.Equal(t, cs.sumLen, len(hasher.Sum(nil))) + } +} + +func TestHasherMapWriter(t *testing.T) { + hasherMap := access.HasherMap{ + access.HashAlgCRC32: access.HashAlgCRC32.ToHasher(), + access.HashAlgMD5: access.HashAlgMD5.ToHasher(), + access.HashAlgSHA1: access.HashAlgSHA1.ToHasher(), + } + writer := hasherMap.ToWriter() + writer.Write([]byte("foo x bar")) + require.Equal(t, []byte{0x5a, 0xc7, 0x36, 0x25}, hasherMap[access.HashAlgCRC32].Sum(nil)) + require.Equal(t, []byte{ + 0x7c, 0xb4, 0x1e, 0x1a, 0x38, 0xad, 0x8b, 0xd9, + 0x5a, 0x49, 0x2d, 0xb6, 0x29, 0x30, 0x62, 0x8a, + }, hasherMap[access.HashAlgMD5].Sum(nil)) + require.Equal(t, []byte{ + 0x75, 0xe8, 0xe5, 0x0, 0x5b, 0xaa, 0x5c, 0xd, 0x40, 0x44, 0x1c, 0x42, + 0xe5, 0xf6, 0xb4, 0x9a, 0xf, 0x80, 0x80, 0x11, + }, hasherMap[access.HashAlgSHA1].Sum(nil)) +} + +func TestHashSumMapGetSum(t *testing.T) { + crc32Hash := crc32.NewIEEE() + crc32Hash.Write([]byte("crc32")) + bytesCRC32 := crc32Hash.Sum(nil) + bytesMD5 := md5.Sum([]byte("md5")) + bytesSHA1 := sha1.Sum([]byte("sha1")) + bytesSHA256 := sha256.Sum256([]byte("sha256")) + + cases := []struct { + key string + m access.HashSumMap + alg access.HashAlgorithm + val interface{} + ok bool + }{ + { + "dummy", + access.HashSumMap{}, + access.HashAlgDummy, nil, false, + }, + { + "dummy", + access.HashSumMap{access.HashAlgDummy: []byte{0x11}}, + access.HashAlgDummy, nil, false, + }, + { + "dummy", + access.HashSumMap{access.HashAlgDummy: []byte{}}, + access.HashAlgDummy, nil, true, + }, + + { + "crc32", + access.HashSumMap{access.HashAlgCRC32: []byte{}}, + access.HashAlgCRC32, nil, false, + }, + { + "crc32", + access.HashSumMap{access.HashAlgCRC32: []byte{}}, + access.HashAlgCRC32, nil, false, + }, + { + "crc32", + access.HashSumMap{access.HashAlgCRC32: bytesCRC32}, + access.HashAlgCRC32, crc32.ChecksumIEEE([]byte("crc32")), true, + }, + + { + "md5", + access.HashSumMap{access.HashAlgMD5: []byte{}}, + access.HashAlgMD5, nil, false, + }, + { + "md5", + access.HashSumMap{access.HashAlgMD5: []byte{0x11}}, + access.HashAlgMD5, nil, false, + }, + { + "md5", + access.HashSumMap{access.HashAlgMD5: bytesMD5[:]}, + access.HashAlgMD5, "1bc29b36f623ba82aaf6724fd3b16718", true, + }, + + { + "sha1", + access.HashSumMap{access.HashAlgSHA1: []byte{}}, + access.HashAlgSHA1, nil, false, + }, + { + "sha1", + access.HashSumMap{access.HashAlgSHA1: []byte{0x00, 0xff}}, + access.HashAlgSHA1, nil, false, + }, + { + "sha1", + access.HashSumMap{access.HashAlgSHA1: bytesSHA1[:]}, + access.HashAlgSHA1, "415ab40ae9b7cc4e66d6769cb2c08106e8293b48", true, + }, + + { + "sha256", + access.HashSumMap{}, + access.HashAlgSHA256, nil, false, + }, + { + "sha256", + access.HashSumMap{access.HashAlgSHA256: nil}, + access.HashAlgSHA256, nil, false, + }, + { + "sha256", + access.HashSumMap{access.HashAlgSHA256: bytesSHA256[:]}, + access.HashAlgSHA256, + "5d5b09f6dcb2d53a5fffc60c4ac0d55fabdf556069d6631545f42aa6e3500f2e", true, + }, + + { + "multi", + access.HashSumMap{access.HashAlgSHA1: bytesSHA1[:]}, + access.HashAlgSHA256, nil, false, + }, + { + "multi", + access.HashSumMap{access.HashAlgSHA1: bytesSHA1[:]}, + access.HashAlgMD5, nil, false, + }, + } + + for _, cs := range cases { + val, ok := cs.m.GetSum(cs.alg) + require.Equal(t, cs.ok, ok, cs.key) + require.Equal(t, cs.val, val, cs.key) + + val = cs.m.GetSumVal(cs.alg) + require.Equal(t, cs.val, val, cs.key) + + hasherMap := make(access.HasherMap) + for alg := range cs.m { + hasherMap[alg] = nil + } + require.Equal(t, cs.m.ToHashAlgorithm(), hasherMap.ToHashAlgorithm()) + + cs.m.All() + } +} + +func TestHashAlgorithm2HashSumMap(t *testing.T) { + cases := []struct { + m access.HashSumMap + h access.HashAlgorithm + }{ + {access.HashSumMap{}, 0}, + {access.HashSumMap{ + access.HashAlgDummy: nil, + }, 0b1}, + {access.HashSumMap{ + access.HashAlgDummy: nil, + access.HashAlgCRC32: nil, + access.HashAlgMD5: nil, + access.HashAlgSHA1: nil, + access.HashAlgSHA256: nil, + }, 0b11111}, + {access.HashSumMap{ + access.HashAlgDummy: nil, + access.HashAlgCRC32: nil, + access.HashAlgMD5: nil, + access.HashAlgSHA1: nil, + access.HashAlgSHA256: nil, + }, 0b00011111}, + {access.HashSumMap{ + access.HashAlgDummy: nil, + access.HashAlgSHA1: nil, + access.HashAlgSHA256: nil, + }, 0b11001}, + } + for _, cs := range cases { + h := cs.m.ToHashAlgorithm() + require.Equal(t, cs.h, h) + m := h.ToHashSumMap() + require.Equal(t, cs.m, m) + } +} + +func TestLocationEncodeDecodeNil(t *testing.T) { + var loc *access.Location + require.Nil(t, loc.Encode()) + require.Equal(t, 0, loc.Encode2(nil)) + require.Equal(t, "", loc.ToString()) + require.Equal(t, "", loc.HexString()) + require.Equal(t, "", loc.Base64String()) + + n, err := loc.Decode(nil) + require.Error(t, err) + require.Equal(t, 0, n) + + locx, n, err := access.DecodeLocation(nil) + require.Error(t, err) + require.Equal(t, 0, n) + require.Equal(t, access.Location{}, locx) + + locx, err = access.DecodeLocationFrom("") + require.Error(t, err) + require.Equal(t, access.Location{}, locx) + + locx, err = access.DecodeLocationFrom("xxx") + require.Error(t, err) + require.Equal(t, access.Location{}, locx) + + locx, err = access.DecodeLocationFromHex("xxx") + require.Error(t, err) + require.Equal(t, access.Location{}, locx) + + locx, err = access.DecodeLocationFromBase64("xxx") + require.Error(t, err) + require.Equal(t, access.Location{}, locx) +} + +func TestLocationEncodeDecode(t *testing.T) { + for ii := 0; ii < 100; ii++ { + loc := &access.Location{ + ClusterID: proto.ClusterID(mrand.Uint32()), + CodeMode: codemode.CodeMode(mrand.Intn(0xff)), + Size: mrand.Uint64(), + BlobSize: mrand.Uint32(), + Crc: mrand.Uint32(), + } + + num := mrand.Intn(5) + for i := 0; i < num; i++ { + loc.Blobs = append(loc.Blobs, access.SliceInfo{ + MinBid: proto.BlobID(mrand.Uint64()), + Vid: proto.Vid(mrand.Uint32()), + Count: mrand.Uint32(), + }) + } + + buf := loc.Encode() + bufx := make([]byte, len(buf)) + n := loc.Encode2(bufx) + require.Equal(t, len(buf), n) + require.Equal(t, buf, bufx) + + require.Panics(t, func() { loc.Encode2(nil) }) + require.Panics(t, func() { loc.Encode2(bufx[:3]) }) + require.Panics(t, func() { loc.Encode2(bufx[:n/2]) }) + require.Panics(t, func() { loc.Encode2(bufx[:n-1]) }) + + locx := access.Location{} + locx.Decode(bufx) + require.Equal(t, loc.ToString(), locx.ToString()) + require.Equal(t, loc.HexString(), locx.HexString()) + require.Equal(t, loc.Base64String(), locx.Base64String()) + + str := loc.ToString() + locx, err := access.DecodeLocationFrom(str) + require.NoError(t, err) + require.Equal(t, *loc, locx) + + str = loc.HexString() + locx, err = access.DecodeLocationFromHex(str) + require.NoError(t, err) + require.Equal(t, *loc, locx) + + str = loc.Base64String() + locx, err = access.DecodeLocationFromBase64(str) + require.NoError(t, err) + require.Equal(t, *loc, locx) + } +} + +func TestLocationDecodeError(t *testing.T) { + loc := &access.Location{ + ClusterID: proto.ClusterID(math.MaxUint32), + CodeMode: codemode.CodeMode(math.MaxInt8), + Size: math.MaxUint64, + BlobSize: math.MaxUint32, + Crc: math.MaxUint32, + } + + buf := loc.Encode() + require.Equal(t, 25+1, len(buf)) + for _, n := range []int{3, 8, 9, 19, 24} { + _, _, err := access.DecodeLocation(buf[:n]) + require.Error(t, err) + t.Log(err) + } + + loc.Blobs = append(loc.Blobs, access.SliceInfo{ + MinBid: proto.BlobID(math.MaxUint64), + Vid: proto.Vid(math.MaxUint32), + Count: math.MaxUint32, + }) + + buf = loc.Encode() + require.Equal(t, 25+1+20, len(buf)) + for _, n := range []int{25, 35, 40, 45} { + _, _, err := access.DecodeLocation(buf[:n]) + require.Error(t, err) + t.Log(err) + } +} + +func TestLocationSpread(t *testing.T) { + { + var loc access.Location + blobs := loc.Spread() + require.NotNil(t, blobs) + require.Equal(t, 0, len(blobs)) + } + { + loc := &access.Location{ + Size: 10, + BlobSize: 1 << 22, + Blobs: []access.SliceInfo{{ + MinBid: 100, + Vid: 4, + Count: 1, + }}, + } + blobs := loc.Spread() + require.Equal(t, 1, len(blobs)) + require.Equal(t, proto.BlobID(100), blobs[0].Bid) + require.Equal(t, proto.Vid(4), blobs[0].Vid) + require.Equal(t, uint32(10), blobs[0].Size) + } + { + loc := &access.Location{ + Size: (1 << 22) + 10, + BlobSize: 1 << 22, + Blobs: []access.SliceInfo{{ + MinBid: 100, + Vid: 4, + Count: 2, + }}, + } + blobs := loc.Spread() + require.Equal(t, 2, len(blobs)) + require.Equal(t, proto.BlobID(100), blobs[0].Bid) + require.Equal(t, proto.Vid(4), blobs[0].Vid) + require.Equal(t, uint32(1<<22), blobs[0].Size) + require.Equal(t, proto.BlobID(101), blobs[1].Bid) + require.Equal(t, proto.Vid(4), blobs[1].Vid) + require.Equal(t, uint32(10), blobs[1].Size) + } + { + loc := &access.Location{ + Size: 1 << 23, + BlobSize: 1 << 22, + Blobs: []access.SliceInfo{{ + MinBid: 100, + Vid: 4, + Count: 1, + }, { + MinBid: 200, + Vid: 4, + Count: 1, + }}, + } + blobs := loc.Spread() + require.Equal(t, 2, len(blobs)) + require.Equal(t, proto.BlobID(100), blobs[0].Bid) + require.Equal(t, proto.Vid(4), blobs[0].Vid) + require.Equal(t, uint32(1<<22), blobs[0].Size) + require.Equal(t, proto.BlobID(200), blobs[1].Bid) + require.Equal(t, proto.Vid(4), blobs[1].Vid) + require.Equal(t, uint32(1<<22), blobs[1].Size) + } +} + +func TestPutArgs(t *testing.T) { + cases := []struct { + size int64 + valid bool + }{ + {-(1 << 20), false}, + {-1, false}, + {0, false}, + {1, true}, + {100, true}, + {1 << 32, true}, + } + for _, cs := range cases { + args := access.PutArgs{Size: cs.size} + require.Equal(t, cs.valid, args.IsValid()) + } +} + +func TestPutAtArgs(t *testing.T) { + cases := []struct { + cid proto.ClusterID + vid proto.Vid + bid proto.BlobID + size int64 + valid bool + }{ + {0, 0, 0, -1, false}, + {0, 0, 0, 0, false}, + {0, 0, 0, 1, false}, + {1, 0, 0, 1, false}, + {1, 1, 0, 1, false}, + {1, 1, 1, 1, true}, + } + for _, cs := range cases { + args := access.PutAtArgs{ + ClusterID: cs.cid, + Vid: cs.vid, + Blobid: cs.bid, + Size: cs.size, + } + require.Equal(t, cs.valid, args.IsValid()) + } +} + +func TestAllocArgs(t *testing.T) { + cases := []struct { + size uint64 + cid proto.ClusterID + blobSize uint32 + cm codemode.CodeMode + valid bool + }{ + {0, 0, 0, 0, false}, + {0, 0, 0, 1, false}, + {1, 0, 0, 0, true}, + {1, 1, 0, 0, false}, + {1, 1, 1, 0, false}, + {1, 1, 1, 1, true}, + {1, 1, 1, 3, true}, + {1, 1, 1, 0xff, false}, + {1, 1, 0, 2, false}, + {1, 0, access.MaxBlobSize, 0, true}, + {1, 1, access.MaxBlobSize, 2, true}, + {1, 0, access.MaxBlobSize + 1, 0, false}, + {1, 1, access.MaxBlobSize + 1, 2, false}, + } + for _, cs := range cases { + args := access.AllocArgs{ + Size: cs.size, + BlobSize: cs.blobSize, + CodeMode: cs.cm, + AssignClusterID: cs.cid, + } + require.Equal(t, cs.valid, args.IsValid()) + } +} + +func TestGetArgs(t *testing.T) { + args := access.GetArgs{} + require.True(t, args.IsValid()) +} + +func TestDeleteArgs(t *testing.T) { + args := access.DeleteArgs{} + require.False(t, args.IsValid()) + require.False(t, (*access.DeleteArgs)(nil).IsValid()) + args.Locations = []access.Location{{}} + require.True(t, args.IsValid()) + args.Locations = make([]access.Location, access.MaxDeleteLocations) + require.True(t, args.IsValid()) + args.Locations = make([]access.Location, access.MaxDeleteLocations+1) + require.False(t, args.IsValid()) +} + +func TestDeleteBlobArgs(t *testing.T) { + args := access.DeleteBlobArgs{} + require.False(t, args.IsValid()) + require.False(t, (*access.DeleteBlobArgs)(nil).IsValid()) + args.ClusterID = 1 + require.False(t, args.IsValid()) + args.Blobid = 1 + require.False(t, args.IsValid()) + args.Vid = 1 + require.False(t, args.IsValid()) + args.Size = 1 + require.True(t, args.IsValid()) +} + +func TestSignArgs(t *testing.T) { + args := access.SignArgs{} + require.False(t, args.IsValid()) + require.False(t, (*access.DeleteArgs)(nil).IsValid()) + args.Locations = []access.Location{{}} + require.True(t, args.IsValid()) +} + +func init() { + mrand.Seed(time.Now().UnixNano()) +} diff --git a/blobstore/blobstore.go b/blobstore/blobstore.go new file mode 100644 index 000000000..565eb601f --- /dev/null +++ b/blobstore/blobstore.go @@ -0,0 +1,27 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package blobstore + +// go vet +//go:generate go vet ./... + +// code formats with 'gofumpt' at version v0.2.1 +// go install mvdan.cc/gofumpt@v0.2.1 +//go:generate gofumpt -l -w . +//go:generate git diff --exit-code + +// code golangci lint with 'golangci-lint' version v1.43.0 +// go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.43.0 +//go:generate golangci-lint run --issues-exit-code=1 -D errcheck -E bodyclose ./... diff --git a/blobstore/cli/access/access.go b/blobstore/cli/access/access.go new file mode 100644 index 000000000..0f765ce5f --- /dev/null +++ b/blobstore/cli/access/access.go @@ -0,0 +1,157 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "fmt" + "os" + + "github.com/desertbit/grumble" + "github.com/hashicorp/consul/api" + + "github.com/cubefs/cubefs/blobstore/api/access" + "github.com/cubefs/cubefs/blobstore/cli/common" + "github.com/cubefs/cubefs/blobstore/cli/common/cfmt" + "github.com/cubefs/cubefs/blobstore/cli/common/flags" + "github.com/cubefs/cubefs/blobstore/cli/config" +) + +func newAccessClient() (access.API, error) { + return access.New(access.Config{ + ConnMode: access.RPCConnectMode(config.AccessConnMode()), + Consul: access.ConsulConfig{Address: config.AccessConsulAddr()}, + PriorityAddrs: config.AccessPriorityAddrs(), + MaxSizePutOnce: config.AccessMaxSizePutOnce(), + MaxPartRetry: config.AccessMaxPartRetry(), + MaxHostRetry: config.AccessMaxHostRetry(), + + ServiceIntervalMs: config.AccessServiceIntervalMs(), + }) +} + +func newConsulClient() (*api.Client, error) { + conf := api.DefaultConfig() + conf.Address = config.AccessConsulAddr() + return api.NewClient(conf) +} + +func readLocation(f grumble.FlagMap) (loc access.Location, err error) { + if f.String("location") != "" { + loc, err = cfmt.ParseLocation(f.String("location")) + return + } + + filepath := f.String("locationpath") + if filepath == "" { + err = fmt.Errorf("no location and locationpath setting") + return + } + + file, err := os.Open(filepath) + if err != nil { + return + } + defer file.Close() + + err = common.NewDecoder(file).Decode(&loc) + return +} + +func accessFlags(f *grumble.Flags) { + // readable + flags.VerboseRegister(f) + + // location + f.String("p", "locationpath", "", "location file path") + f.String("l", "location", "", "location string by [json|hex|base64]") +} + +// Register register access +func Register(app *grumble.App) { + accessCommand := &grumble.Command{ + Name: "access", + Help: "access tools", + LongHelp: "blobstore access api tools", + Run: func(c *grumble.Context) error { + cli, err := newConsulClient() + if err != nil { + return err + } + services, _, err := cli.Health().Service("access", "", true, + &api.QueryOptions{RequireConsistent: true}) + if err != nil { + return err + } + fmt.Println("discovery access services on") + fmt.Println(common.Readable(services)) + fmt.Println() + return nil + }, + } + app.AddCommand(accessCommand) + + accessCommand.AddCommand(&grumble.Command{ + Name: "put", + Help: "put file", + Run: putFile, + Flags: func(f *grumble.Flags) { + accessFlags(f) + f.String("d", "data", "", "raw data body") + f.String("f", "filepath", "", "put file path") + f.Int64("", "size", 0, "put file size, 0 means file size") + f.Uint("", "hashes", 0, "put file hashes") + }, + }) + accessCommand.AddCommand(&grumble.Command{ + Name: "get", + Help: "get file", + Run: getFile, + Flags: func(f *grumble.Flags) { + accessFlags(f) + f.String("f", "filepath", "", "save file path") + f.Uint64("", "offset", 0, "get file offset") + f.Uint64("", "readsize", 0, "get file read size, 0 means file size") + }, + }) + accessCommand.AddCommand(&grumble.Command{ + Name: "del", + Help: "del file", + Run: delFile, + Flags: func(f *grumble.Flags) { + accessFlags(f) + }, + }) + accessCommand.AddCommand(&grumble.Command{ + Name: "cluster", + Help: "show cluster", + LongHelp: "show cluster in [region]", + Run: showClusters, + Args: func(a *grumble.Args) { + a.String("region", "show cluster of region", grumble.Default("")) + }, + }) + accessCommand.AddCommand(&grumble.Command{ + Name: "ec", + Help: "show ec buffer size", + LongHelp: "show ec buffer size with [blobsize]", + Run: showECbuffer, + Args: func(a *grumble.Args) { + a.Int("blobsize", "show ec buffer with blobsize", grumble.Default(0)) + }, + Flags: func(f *grumble.Flags) { + f.IntL("codemode", 0, "on special codemode") + }, + }) +} diff --git a/blobstore/cli/access/cluster.go b/blobstore/cli/access/cluster.go new file mode 100644 index 000000000..375135854 --- /dev/null +++ b/blobstore/cli/access/cluster.go @@ -0,0 +1,94 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "fmt" + "strings" + + "github.com/desertbit/grumble" + "github.com/dustin/go-humanize" + "github.com/fatih/color" + + cmapi "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/cli/common" + "github.com/cubefs/cubefs/blobstore/cli/common/cfmt" +) + +func showClusters(c *grumble.Context) error { + cli, err := newConsulClient() + if err != nil { + return err + } + + var regions []string + if region := c.Args.String("region"); region != "" { + regions = []string{region} + } else { + pairs, _, err := cli.KV().List("/ebs/", nil) + if err != nil { + return err + } + for _, pair := range pairs { + paths := strings.Split(strings.Trim(pair.Key, "/"), "/") + region := paths[1] + has := false + for _, r := range regions { + if r == region { + has = true + break + } + } + if !has { + regions = append(regions, region) + } + } + fmt.Printf("found %s regions\n\n", color.GreenString("%d", len(regions))) + } + + for _, region := range regions { + fmt.Println("to list region:", color.RedString("%s", region)) + path := cmapi.GetConsulClusterPath(region) + pairs, _, err := cli.KV().List(path, nil) + if err != nil { + fmt.Println("\terror:", err) + continue + } + + capacity := int64(1) + available := int64(1) + for _, pair := range pairs { + clusterInfo := &cmapi.ClusterInfo{} + err := common.Unmarshal(pair.Value, clusterInfo) + if err != nil { + fmt.Println("\terror:", err) + continue + } + + capacity += clusterInfo.Capacity + available += clusterInfo.Available + + clusterInfo.Capacity++ + clusterInfo.Available++ + fmt.Println(cfmt.ClusterInfoJoin(clusterInfo, "\t")) + fmt.Println() + } + fmt.Printf(" space in region: %s (%s / %s)\n", region, + common.ColorizeInt64(-available, capacity).Sprint(humanize.IBytes(uint64(available))), + humanize.IBytes(uint64(capacity))) + } + + return nil +} diff --git a/blobstore/cli/access/del.go b/blobstore/cli/access/del.go new file mode 100644 index 000000000..c7471dcc2 --- /dev/null +++ b/blobstore/cli/access/del.go @@ -0,0 +1,61 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "fmt" + + "github.com/desertbit/grumble" + + "github.com/cubefs/cubefs/blobstore/api/access" + "github.com/cubefs/cubefs/blobstore/cli/common" + "github.com/cubefs/cubefs/blobstore/cli/common/cfmt" + "github.com/cubefs/cubefs/blobstore/cli/common/flags" + "github.com/cubefs/cubefs/blobstore/cli/config" +) + +func delFile(c *grumble.Context) error { + client, err := newAccessClient() + if err != nil { + return err + } + + location, err := readLocation(c.Flags) + if err != nil { + return err + } + + fmt.Printf("del location json : %s\n", common.RawString(location)) + if flags.Verbose(c.Flags) || config.Verbose() { + fmt.Println("del location readable:") + fmt.Println(cfmt.LocationJoin(&location, "\t")) + } else { + fmt.Printf("del location verbose : %+v\n", location) + } + + if !common.Confirm("to delete?") { + return nil + } + deleteArgs := &access.DeleteArgs{ + Locations: []access.Location{location}, + } + _, err = client.Delete(common.CmdContext(), deleteArgs) + if err != nil { + return err + } + + fmt.Println("DELETE OK!") + return nil +} diff --git a/blobstore/cli/access/ec.go b/blobstore/cli/access/ec.go new file mode 100644 index 000000000..d88b2003a --- /dev/null +++ b/blobstore/cli/access/ec.go @@ -0,0 +1,135 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "fmt" + "strings" + + "github.com/desertbit/grumble" + "github.com/dustin/go-humanize" + "github.com/fatih/color" + + "github.com/cubefs/cubefs/blobstore/cli/common" + "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/cubefs/cubefs/blobstore/common/ec" +) + +func showECbuffer(c *grumble.Context) error { + kb := 1 << 10 + mb := 1 << 20 + perPage := 7 + + blobSize := c.Args.Int("blobsize") + if blobSize > 64*mb { + return fmt.Errorf("too large blobsize %d", blobSize) + } + + var modes, all []codemode.CodeMode + all = codemode.GetAllCodeModes() + mode := c.Flags.Int("codemode") + for _, m := range all { + if m == codemode.CodeMode(mode) { + modes = []codemode.CodeMode{m} + break + } + } + if len(modes) == 0 { + modes = all + } + + for len(modes) > 0 { + length := len(modes) + hasNextPage := length > perPage + if length > perPage { + length = perPage + } + + showModes := modes[:length] + showEcbufferSize(-1, common.Optimal, showModes) + if blobSize > 0 { + showEcbufferSize(blobSize, common.Loaded, showModes) + if hasNextPage { + fmt.Println() + } + modes = modes[length:] + continue + } + + alterColor := common.NewAlternateColor(3) + for _, size := range []int{ + 1, + kb * 2, + kb * 12, + kb * 64, + kb * 512, + mb * 1, + mb * 2, + mb * 4, + mb * 8, + mb * 16, + } { + showEcbufferSize(size, alterColor.Next(), showModes) + } + if hasNextPage { + fmt.Println() + } + modes = modes[length:] + } + + return nil +} + +func showEcbufferSize(size int, colorFmt *color.Color, modes []codemode.CodeMode) { + if size == -1 { + colorFmt.Printf("|%s|", center("blobsize")) + for _, mode := range modes { + colorFmt.Printf("%s|", center(mode.String())) + } + fmt.Println() + return + } + + colorFmt.Printf("|%s|", center(size2Str(size))) + for _, mode := range modes { + sizes, _ := ec.GetBufferSizes(size, mode.Tactic()) + colorFmt.Printf("%s|", center(size2Str(sizes.ECSize))) + } + fmt.Println() + + colorFmt.Printf("|%s|", center("shard size")) + for _, mode := range modes { + sizes, _ := ec.GetBufferSizes(size, mode.Tactic()) + colorFmt.Printf("%s|", center(size2Str(sizes.ShardSize))) + } + fmt.Println() +} + +func center(s string) string { + all := 19 + l := (all - len(s)) / 2 + if l < 0 { + l = 0 + } + r := all - len(s) - l + if r < 0 { + r = 0 + } + return fmt.Sprintf("%s%s%s", strings.Repeat(" ", l), s, strings.Repeat(" ", r)) +} + +func size2Str(size int) string { + return fmt.Sprintf("%d(%s)", size, humanize.IBytes(uint64(size))) +} diff --git a/blobstore/cli/access/get.go b/blobstore/cli/access/get.go new file mode 100644 index 000000000..f823e2c94 --- /dev/null +++ b/blobstore/cli/access/get.go @@ -0,0 +1,110 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "time" + + "github.com/desertbit/grumble" + + "github.com/cubefs/cubefs/blobstore/api/access" + "github.com/cubefs/cubefs/blobstore/cli/common" + "github.com/cubefs/cubefs/blobstore/cli/common/cfmt" + "github.com/cubefs/cubefs/blobstore/cli/common/flags" + "github.com/cubefs/cubefs/blobstore/cli/config" +) + +func getFile(c *grumble.Context) error { + client, err := newAccessClient() + if err != nil { + return err + } + + location, err := readLocation(c.Flags) + if err != nil { + return err + } + + fmt.Printf("get location json : %s\n", common.RawString(location)) + if flags.Verbose(c.Flags) || config.Verbose() { + fmt.Println("get location readable:") + fmt.Println(cfmt.LocationJoin(&location, "\t")) + } else { + fmt.Printf("get location verbose : %+v\n", location) + } + + size := c.Flags.Uint64("readsize") + if size == 0 { + size = location.Size - c.Flags.Uint64("offset") + } + + r, err := client.Get(common.CmdContext(), &access.GetArgs{ + Location: location, + Offset: c.Flags.Uint64("offset"), + ReadSize: size, + }) + if err != nil { + return err + } + defer r.Close() + + reader := common.NewPReader(int(size), r) + defer reader.Close() + + reader.LineBar(50) + + filepath := c.Flags.String("filepath") + if filepath == "" { + var w io.Writer + buffer := bytes.NewBuffer(nil) + // > 4K + if size > 1<<12 { + fmt.Printf("data is too long %d > %d\n", size, 1<<12) + w = ioutil.Discard + } else { + w = buffer + } + + _, err = io.CopyN(w, reader, int64(size)) + if err != nil { + return fmt.Errorf("downloading %s", err.Error()) + } + + data := buffer.Bytes() + if len(data) > 0 { + time.Sleep(10 * time.Millisecond) + fmt.Printf("raw data %d: '%s'\n", len(data), string(data)) + } + return nil + } + + file, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return fmt.Errorf("open file %s : %s", filepath, err.Error()) + } + defer file.Close() + + _, err = io.CopyN(file, reader, int64(size)) + if err != nil { + return fmt.Errorf("downloading to %s : %s", filepath, err.Error()) + } + + return nil +} diff --git a/blobstore/cli/access/put.go b/blobstore/cli/access/put.go new file mode 100644 index 000000000..6d812f468 --- /dev/null +++ b/blobstore/cli/access/put.go @@ -0,0 +1,102 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package access + +import ( + "bytes" + "fmt" + "os" + + "github.com/desertbit/grumble" + + "github.com/cubefs/cubefs/blobstore/api/access" + "github.com/cubefs/cubefs/blobstore/cli/common" + "github.com/cubefs/cubefs/blobstore/cli/common/cfmt" + "github.com/cubefs/cubefs/blobstore/cli/common/flags" + "github.com/cubefs/cubefs/blobstore/cli/config" +) + +func putFile(c *grumble.Context) error { + client, err := newAccessClient() + if err != nil { + return err + } + + var ( + reader *common.PReader + size int64 + ) + raw := c.Flags.String("data") + if len(raw) > 0 { + data := []byte(raw) + size = int64(len(data)) + reader = common.NewPReader(int(size), bytes.NewReader(data)) + + } else { + filepath := c.Flags.String("filepath") + if filepath == "" { + return fmt.Errorf("no filepath setting") + } + + file, err := os.Open(c.Flags.String("filepath")) + if err != nil { + return fmt.Errorf("open file %s : %s", filepath, err.Error()) + } + defer file.Close() + + size = c.Flags.Int64("size") + if size == 0 { + st, _ := os.Stat(filepath) + size = st.Size() + } + reader = common.NewPReader(int(size), file) + } + defer reader.Close() + + reader.LineBar(50) + + putHashes := access.HashAlgorithm(c.Flags.Uint("hashes")) + fmt.Printf("to upload size:%d hash:b(%b)\n", size, putHashes) + + location, hashes, err := client.Put(common.CmdContext(), &access.PutArgs{ + Size: size, + Hashes: putHashes, + Body: reader, + }) + if err != nil { + return err + } + + fmt.Println(cfmt.HashSumMapJoin(hashes, "\t")) + fmt.Printf("put location json : %s\n", common.RawString(location)) + if flags.Verbose(c.Flags) || config.Verbose() { + fmt.Println("put location readable:") + fmt.Println(cfmt.LocationJoin(&location, "\t")) + } else { + fmt.Printf("put location verbose : %+v\n", location) + } + + locPath := c.Flags.String("locationpath") + if locPath != "" { + f, err := os.OpenFile(locPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return fmt.Errorf("open file %s : %s", locPath, err.Error()) + } + defer f.Close() + return common.NewEncoder(f).Encode(location) + } + + return nil +} diff --git a/blobstore/cli/app.go b/blobstore/cli/app.go new file mode 100644 index 000000000..2139329e9 --- /dev/null +++ b/blobstore/cli/app.go @@ -0,0 +1,107 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cli + +import ( + "os" + "os/user" + "time" + + "github.com/desertbit/grumble" + "github.com/fatih/color" + + "github.com/cubefs/cubefs/blobstore/cli/access" + "github.com/cubefs/cubefs/blobstore/cli/clustermgr" + "github.com/cubefs/cubefs/blobstore/cli/common" + "github.com/cubefs/cubefs/blobstore/cli/common/flags" + "github.com/cubefs/cubefs/blobstore/cli/config" +) + +// App blobstore command app +var App = grumble.New(&grumble.Config{ + Name: "blobstore manager", + Description: "A command manager of blobstore", + HistoryFile: "/tmp/.blobstore_cli.history", + HistoryLimit: 10000, + ErrorColor: color.New(color.FgRed, color.Bold, color.Faint), + HelpHeadlineColor: color.New(color.FgGreen), + HelpHeadlineUnderline: false, + HelpSubCommands: true, + Prompt: "BS $> ", + PromptColor: color.New(color.FgBlue, color.Bold), + PromptRuntime: func() func() string { + username := "NO-USER" + hostname := "NO-HOSTNAME" + if user, err := user.Current(); err == nil { + username = user.Username + } + if host, _ := os.Hostname(); host != "" { + hostname = host + } + + return func() string { + now := time.Now() + mins := (now.Hour() * 60) + now.Minute() + curr := mins * 100 / (60 * 24) + tStr := now.Format("01-02 15:04:05.000") + return color.New().Sprintf("%s %s%s %s@%s %s ", + color.New(color.FgBlue, color.Italic).Sprint("BS"), + color.New(color.FgMagenta, color.Faint).Sprintf("[%s]", common.BoldBar(curr)), + color.New(color.FgMagenta, color.Faint).Sprintf("[%s]", tStr), + color.New(color.FgGreen, color.Bold).Sprint(username), + color.New(color.FgYellow, color.Bold).Sprint(hostname), + color.New(color.FgBlack, color.BgHiYellow).Sprint(" $> "), + ) + } + }(), + Flags: func(f *grumble.Flags) { + flags.ConfigRegister(f) + flags.VerboseRegister(f) + flags.VverboseRegister(f) + }, +}) + +func init() { + App.OnInit(func(a *grumble.App, fm grumble.FlagMap) error { + if path := flags.Config(fm); path != "" { + config.LoadConfig(path) + } + if flags.Verbose(fm) { + config.Set("Flag-Verbose", true) + } + if flags.Vverbose(fm) { + config.Set("Flag-Vverbose", true) + } + return nil + }) + + App.SetPrintASCIILogo(func(a *grumble.App) { + a.Println(` _______ _______ _______ _______ _______ _______ _______ _______ _______ _______ _______ _______ `) + a.Println(`|\ /|\ /|\ /|\ /|\ /|\ /|\ /|\ /|\ /| |\ /|\ /|\ /|`) + a.Println(`| +---+ | +---+ | +---+ | +---+ | +---+ | +---+ | +---+ | +---+ | +---+ | | +---+ | +---+ | +---+ |`) + a.Println(`| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |`) + a.Println(`| |b | | |l | | |o | | |b | | |s | | |t | | |o | | |r | | |e | | | |c | | |l | | |i | |`) + a.Println(`| +---+ | +---+ | +---+ | +---+ | +---+ | +---+ | +---+ | +---+ | +---+ | | +---+ | +---+ | +---+ |`) + a.Println(`|/_____\|/_____\|/_____\|/_____\|/_____\|/_____\|/_____\|/_____\|/_____\| |/_____\|/_____\|/_____\|`) + a.Println() + }) + + registerHistory(App) + registerConfig(App) + registerUtil(App) + + access.Register(App) + clustermgr.Register(App) +} diff --git a/blobstore/cli/cli/cli.conf b/blobstore/cli/cli/cli.conf new file mode 100644 index 000000000..25c2dfa79 --- /dev/null +++ b/blobstore/cli/cli/cli.conf @@ -0,0 +1,25 @@ +{ + "access": { + "conn_mode": 4, + "consul_addr": "http://127.0.0.1:8500", + "max_host_retry": 0, + "max_part_retry": 0, + "max_size_put_once": 0, + "priority_addrs": [ + "http://localhost:9500", + "http://127.0.0.1:9500" + ], + "service_interval_ms": 0 + }, + "redis_addrs": [ + "127.0.0.1:6379" + ], + "redis_user": "", + "redis_pass": "", + "cm_addrs": [ + "http://localhost:9998", + "http://127.0.0.1:9998" + ], + "verbose": false, + "vverbose": false +} diff --git a/blobstore/cli/cli/main.go b/blobstore/cli/cli/main.go new file mode 100644 index 000000000..b5395af6b --- /dev/null +++ b/blobstore/cli/cli/main.go @@ -0,0 +1,25 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package main + +import ( + "github.com/desertbit/grumble" + + "github.com/cubefs/cubefs/blobstore/cli" +) + +func main() { + grumble.Main(cli.App) +} diff --git a/blobstore/cli/clustermgr/clustermgr.go b/blobstore/cli/clustermgr/clustermgr.go new file mode 100644 index 000000000..e1ba94b77 --- /dev/null +++ b/blobstore/cli/clustermgr/clustermgr.go @@ -0,0 +1,107 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package clustermgr + +import ( + "fmt" + "strings" + + "github.com/desertbit/grumble" + + "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/cli/common" + "github.com/cubefs/cubefs/blobstore/cli/config" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/rpc/auth" +) + +func newCMClient(secret string, hosts ...string) *clustermgr.Client { + if len(hosts) == 0 { + hosts = config.ClusterMgrAddrs() + } + if secret == "" { + secret = config.ClusterMgrSecret() + } + enableAuth := false + if secret != "" { + enableAuth = true + } + return clustermgr.New(&clustermgr.Config{ + LbConfig: rpc.LbConfig{ + Hosts: hosts, + Config: rpc.Config{ + Tc: rpc.TransportConfig{ + Auth: auth.Config{ + EnableAuth: enableAuth, + Secret: secret, + }, + }, + }, + }, + }) +} + +func clusterFlags(f *grumble.Flags) { + f.StringL("host", "", "specific clustermgr host") + f.StringL("secret", "", "specific clustermgr secret") +} + +func specificHosts(f grumble.FlagMap) []string { + var hosts []string + for _, host := range strings.Split(f.String("host"), ",") { + host = strings.TrimSpace(host) + if host == "" { + continue + } + if !strings.HasPrefix(host, "http") { + host = "http://" + host + } + hosts = append(hosts, host) + } + return hosts +} + +// Register register cm +func Register(app *grumble.App) { + cmCommand := &grumble.Command{ + Name: "cm", + Help: "cluster manager tools", + } + app.AddCommand(cmCommand) + + addCmdConfig(cmCommand) + addCmdService(cmCommand) + addCmdWalParse(cmCommand) + addCmdVolume(cmCommand) + addCmdListAllDB(cmCommand) + addCmdDisk(cmCommand) + + cmCommand.AddCommand(&grumble.Command{ + Name: "stat", + Help: "show stat of clustermgr", + Flags: func(f *grumble.Flags) { + clusterFlags(f) + }, + Run: func(c *grumble.Context) error { + cli := newCMClient(c.Flags.String("secret"), specificHosts(c.Flags)...) + stat, err := cli.Stat(common.CmdContext()) + if err != nil { + return err + } + fmt.Println(common.Readable(stat)) + return nil + }, + }) +} diff --git a/blobstore/cli/clustermgr/config.go b/blobstore/cli/clustermgr/config.go new file mode 100644 index 000000000..d83c81430 --- /dev/null +++ b/blobstore/cli/clustermgr/config.go @@ -0,0 +1,151 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package clustermgr + +import ( + "fmt" + "net/http" + "sort" + + "github.com/desertbit/grumble" + + "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/cli/common" + "github.com/cubefs/cubefs/blobstore/cli/common/flags" + "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/cubefs/cubefs/blobstore/common/rpc" +) + +func cmdGetConfig(c *grumble.Context) error { + cli, ctx := newCMClient(c.Flags.String("secret"), specificHosts(c.Flags)...), common.CmdContext() + + verbose := flags.Verbose(c.Flags) + key := c.Args.String("key") + if key != "" { + value, err := cli.GetConfig(ctx, key) + if err != nil { + return err + } + showConfig(key, value, verbose) + return nil + } + + all, err := cli.ListConfig(ctx) + if err != nil { + return err + } + keys := make([]string, 0, len(all.Configs)) + for key := range all.Configs { + keys = append(keys, key) + } + sort.Strings(keys) + + for _, key := range keys { + showConfig(key, all.Configs[key], verbose) + } + return nil +} + +func showConfig(key, value string, verbose bool) { + if verbose && key == "code_mode" { + policies := make([]codemode.Policy, 0) + if common.Unmarshal([]byte(value), &policies) == nil { + value = "\n" + common.Readable(policies) + } + } + fmt.Println(" ", key, ":", value) +} + +func addCmdConfig(cmd *grumble.Command) { + configCommand := &grumble.Command{ + Name: "config", + Help: "config tools", + LongHelp: "config tools for clustermgr", + } + cmd.AddCommand(configCommand) + + configCommand.AddCommand(&grumble.Command{ + Name: "get", + Help: "show config of [key]", + Run: cmdGetConfig, + Args: func(a *grumble.Args) { + a.String("key", "config key", grumble.Default("")) + }, + Flags: func(f *grumble.Flags) { + flags.VerboseRegister(f) + clusterFlags(f) + }, + }) + configCommand.AddCommand(&grumble.Command{ + Name: "set", + Help: "set config of key", + Args: func(a *grumble.Args) { + a.String("key", "config key") + a.String("value", "config value") + }, + Flags: func(f *grumble.Flags) { + clusterFlags(f) + }, + Run: func(c *grumble.Context) error { + key := c.Args.String("key") + value := c.Args.String("value") + + cli, ctx := newCMClient(c.Flags.String("secret"), specificHosts(c.Flags)...), common.CmdContext() + oldV, err := cli.GetConfig(ctx, key) + if err != nil { + if rpc.DetectStatusCode(err) != http.StatusNotFound || + !common.Confirm("to set new key: "+common.Loaded.Sprint(key)+" ?") { + return err + } + } + showConfig(key, oldV, true) + + if common.Confirm(fmt.Sprintf( + "to set key: `%s` from `%s` --> `%s` ?", common.Loaded.Sprint(key), + common.Danger.Sprint(oldV), common.Normal.Sprint(value))) { + return cli.SetConfig(ctx, &clustermgr.ConfigSetArgs{ + Key: key, + Value: value, + }) + } + return nil + }, + }) + configCommand.AddCommand(&grumble.Command{ + Name: "del", + Help: "del config of key", + Args: func(a *grumble.Args) { + a.String("key", "config key") + }, + Flags: func(f *grumble.Flags) { + clusterFlags(f) + }, + Run: func(c *grumble.Context) error { + key := c.Args.String("key") + + cli, ctx := newCMClient(c.Flags.String("secret"), specificHosts(c.Flags)...), common.CmdContext() + oldV, err := cli.GetConfig(ctx, key) + if err != nil { + return err + } + showConfig(key, oldV, true) + + if common.Confirm("to del key: " + common.Loaded.Sprint(key) + " ?") { + return cli.DeleteConfig(ctx, key) + } + return nil + }, + }) +} diff --git a/blobstore/cli/clustermgr/diskmgr.go b/blobstore/cli/clustermgr/diskmgr.go new file mode 100644 index 000000000..a1a4316f9 --- /dev/null +++ b/blobstore/cli/clustermgr/diskmgr.go @@ -0,0 +1,203 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package clustermgr + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/desertbit/grumble" + + "github.com/cubefs/cubefs/blobstore/api/blobnode" + "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/cli/common" + "github.com/cubefs/cubefs/blobstore/cli/common/args" + "github.com/cubefs/cubefs/blobstore/cli/common/cfmt" + "github.com/cubefs/cubefs/blobstore/cli/common/flags" + "github.com/cubefs/cubefs/blobstore/cli/config" + "github.com/cubefs/cubefs/blobstore/clustermgr/persistence/normaldb" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/util/errors" +) + +func addCmdDisk(cmd *grumble.Command) { + command := &grumble.Command{ + Name: "disk", + Help: "disk tools", + LongHelp: "disk tools for clustermgr", + } + cmd.AddCommand(command) + + command.AddCommand(&grumble.Command{ + Name: "get", + Help: "show disk ", + Run: cmdGetDisk, + Args: func(a *grumble.Args) { + args.DiskIDRegister(a) + }, + Flags: func(f *grumble.Flags) { + flags.VerboseRegister(f) + clusterFlags(f) + }, + }) + + command.AddCommand(&grumble.Command{ + Name: "listDisk", + Help: "show disks", + Run: cmdListDisks, + Flags: func(f *grumble.Flags) { + flags.VverboseRegister(f) + flags.VerboseRegister(f) + clusterFlags(f) + + f.UintL("status", 0, "list disk status") + f.Int64L("marker", 0, "list disk marker") + f.IntL("count", 0, "list disk count") + }, + }) + + command.AddCommand(&grumble.Command{ + Name: "updateDisk", + Help: "update disk info in db", + Run: cmdUpdateDisk, + Args: func(a *grumble.Args) { + args.DiskIDRegister(a) + a.String("dbPath", "normal db path") + a.String("diskInfo", "modify disk info data") + }, + Flags: func(f *grumble.Flags) { + flags.VerboseRegister(f) + clusterFlags(f) + }, + }) +} + +func cmdGetDisk(c *grumble.Context) error { + cmClient := newCMClient(c.Flags.String("secret"), specificHosts(c.Flags)...) + ctx := common.CmdContext() + disk, err := cmClient.DiskInfo(ctx, args.DiskID(c.Args)) + if err != nil { + return err + } + if config.Verbose() || flags.Verbose(c.Flags) { + fmt.Println(cfmt.DiskInfoJoinV(disk, "")) + } else { + fmt.Println(disk) + } + return nil +} + +func cmdListDisks(c *grumble.Context) error { + cmClient := newCMClient(c.Flags.String("secret"), specificHosts(c.Flags)...) + ctx := common.CmdContext() + + args := &clustermgr.ListOptionArgs{ + Status: proto.DiskStatus(c.Flags.Uint("status")), + Marker: proto.DiskID(c.Flags.Int64("marker")), + Count: c.Flags.Int("count"), + } + if args.Marker <= proto.InvalidDiskID { + args.Marker = proto.DiskID(1) + } + + verbose := config.Verbose() || flags.Verbose(c.Flags) + vv := flags.Vverbose(c.Flags) + next := true + num := 0 + ac := common.NewAlternateColor(3) + for next && args.Marker > proto.InvalidDiskID { + disks, err := cmClient.ListDisk(ctx, args) + if err != nil { + return err + } + + for _, disk := range disks.Disks { + num++ + if verbose || vv { + fmt.Printf("%4d. %s\n", num, strings.Repeat("- ", 60)) + if vv { + ac.Next().Println(cfmt.DiskInfoJoinV(disk, " ")) + } else { + ac.Next().Println(cfmt.DiskInfoJoin(disk, " ")) + } + } else { + ac.Next().Printf("%4d. %v\n", num, disk) + } + } + + if disks.Marker == proto.InvalidDiskID || len(disks.Disks) < args.Count { + next = false + } else { + args.Marker = disks.Marker + fmt.Println() + next = common.Confirm("list next page?") + } + } + return nil +} + +func cmdUpdateDisk(c *grumble.Context) error { + diskid := args.DiskID(c.Args) + dbPath := c.Args.String("dbPath") + data := c.Args.String("diskInfo") + if diskid <= 0 || dbPath == "" || data == "" { + return errors.New("invalid common args") + } + diskInfo := &blobnode.DiskInfo{} + err := json.Unmarshal([]byte(data), diskInfo) + if err != nil { + return err + } + db, err := openNormalDB(dbPath, false) + if err != nil { + return err + } + defer db.Close() + tbl, err := openDiskTable(db) + if err != nil { + return err + } + diskRec, err := tbl.GetDisk(proto.DiskID(diskid)) + if err != nil { + return err + } + if diskInfo.MaxChunkCnt > 0 { + diskRec.MaxChunkCnt = diskInfo.MaxChunkCnt + } + if diskInfo.FreeChunkCnt > 0 { + diskRec.FreeChunkCnt = diskInfo.FreeChunkCnt + } + if diskInfo.UsedChunkCnt > 0 { + diskRec.UsedChunkCnt = diskInfo.UsedChunkCnt + } + if diskInfo.Status > 0 { + diskRec.Status = diskInfo.Status + } + + if !common.Confirm("to change?\n") { + return nil + } + + return tbl.AddDisk(diskRec) +} + +func openDiskTable(db *normaldb.NormalDB) (*normaldb.DiskTable, error) { + tbl, err := normaldb.OpenDiskTable(db, true) + if err != nil { + return nil, err + } + return tbl, nil +} diff --git a/blobstore/cli/clustermgr/list_all.go b/blobstore/cli/clustermgr/list_all.go new file mode 100644 index 000000000..1e674e409 --- /dev/null +++ b/blobstore/cli/clustermgr/list_all.go @@ -0,0 +1,246 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package clustermgr + +import ( + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/cubefs/cubefs/blobstore/common/kvstore" + + "github.com/desertbit/grumble" + + "github.com/cubefs/cubefs/blobstore/clustermgr/persistence/normaldb" + "github.com/cubefs/cubefs/blobstore/clustermgr/persistence/volumedb" +) + +type volInfo struct { + volumedb.VolumeRecord + units []volumedb.VolumeUnitRecord + token volumedb.TokenRecord +} + +type serviceNode struct { + ClusterID uint64 `json:"cluster_id"` + Name string `json:"moduleName"` + Host string `json:"host"` + Idc string `json:"idc"` + Timeout int `json:"timeout"` + Expires time.Time `json:"expires"` +} + +func addCmdListAllDB(cmd *grumble.Command) { + command := &grumble.Command{ + Name: "listAllDB", + Help: "list all db tools", + LongHelp: "list all db tools for clustermgr", + Run: cmdListAllDB, + Args: func(a *grumble.Args) { + a.String("volumeDBPath", "wal log filename") + a.String("normalDBPath", "wal log path") + }, + } + cmd.AddCommand(command) +} + +func cmdListAllDB(c *grumble.Context) error { + volumeDBPath := c.Args.String("volumeDBPath") + normalDBPath := c.Args.String("normalDBPath") + if volumeDBPath == "" || normalDBPath == "" { + return errors.New("invalid command arguments") + } + + volumeDB, err := openVolumeDB(volumeDBPath, false) + if err != nil { + return err + } + defer volumeDB.Close() + normalDB, err := openNormalDB(normalDBPath, false) + if err != nil { + return err + } + defer normalDB.Close() + + volumeTbl, err := volumedb.OpenVolumeTable(volumeDB) + if err != nil { + return err + } + fmt.Println("list volumes: ") + err = listAllVolumes(volumeTbl) + if err != nil { + return err + } + fmt.Println() + + diskTbl, err := normaldb.OpenDiskTable(normalDB, true) + if err != nil { + return err + } + fmt.Println("list disk: ") + err = listAllDisks(diskTbl) + if err != nil { + return err + } + fmt.Println() + + configTbl, err := normaldb.OpenConfigTable(normalDB) + if err != nil { + return err + } + fmt.Println("list config: ") + err = listAllConfigs(configTbl) + if err != nil { + return err + } + fmt.Println() + + dropTbl, err := normaldb.OpenDroppedDiskTable(normalDB) + if err != nil { + return err + } + fmt.Println("list dropping disk: ") + err = listAllDroppingDisks(dropTbl) + if err != nil { + return err + } + fmt.Println() + + scopeTbl, err := normaldb.OpenScopeTable(normalDB) + if err != nil { + return err + } + fmt.Println("list scope: ") + err = listAllScopes(scopeTbl) + if err != nil { + return err + } + fmt.Println() + + serviceTbl := normaldb.OpenServiceTable(normalDB) + if err != nil { + return err + } + fmt.Println("list service: ") + err = listAllServices(serviceTbl) + if err != nil { + return err + } + fmt.Println() + + return nil +} + +func openNormalDB(path string, readonly bool) (*normaldb.NormalDB, error) { + db, err := normaldb.OpenNormalDB(path, false, func(option *kvstore.RocksDBOption) { + option.ReadOnly = readonly + }) + if err != nil { + return nil, fmt.Errorf("open db failed, err: %s", err.Error()) + } + return db, nil +} + +func listAllVolumes(volumeTbl *volumedb.VolumeTable) error { + return volumeTbl.RangeVolumeRecord(func(volRecord *volumedb.VolumeRecord) error { + volInfo := volInfo{ + VolumeRecord: *volRecord, + units: make([]volumedb.VolumeUnitRecord, 0), + } + + for _, vuidPrefix := range volRecord.VuidPrefixs { + unitRecord, err := volumeTbl.GetVolumeUnit(vuidPrefix) + if err != nil { + return fmt.Errorf("get volume unit failed: %s", err.Error()) + } + volInfo.units = append(volInfo.units, *unitRecord) + } + + token, err := volumeTbl.GetToken(volRecord.Vid) + if err != nil { + return fmt.Errorf("get volume token failed: %s", err.Error()) + } + if token != nil { + volInfo.token = *token + } + + data, err := json.Marshal(volInfo) + if err != nil { + return fmt.Errorf("json marshal failed, err: %s", err.Error()) + } + fmt.Println(string(data)) + return nil + }) +} + +func listAllDisks(tbl *normaldb.DiskTable) error { + list, err := tbl.GetAllDisks() + if err != nil { + return fmt.Errorf("list disk failed, err: %s", err.Error()) + } + for i := range list { + data, err := json.Marshal(list[i]) + if err != nil { + return fmt.Errorf("json marshal failed, err: %s", err.Error()) + } + fmt.Println(string(data)) + } + return nil +} + +func listAllConfigs(tbl *normaldb.ConfigTable) error { + list, err := tbl.List() + if err != nil { + return fmt.Errorf("list disk failed, err: %s", err.Error()) + } + fmt.Println(list) + return nil +} + +func listAllDroppingDisks(tbl *normaldb.DroppedDiskTable) error { + list, err := tbl.GetAllDroppingDisk() + if err != nil { + return fmt.Errorf("list dropping disk failed, err: %s", err.Error()) + } + fmt.Println(list) + return nil +} + +func listAllScopes(tbl *normaldb.ScopeTable) error { + scopes, err := tbl.Load() + if err != nil { + return fmt.Errorf("list scopes failed, err: %s", err.Error()) + } + fmt.Println(scopes) + return nil +} + +func listAllServices(tbl *normaldb.ServiceTable) error { + var retErr error + err := tbl.Range(func(key, val []byte) bool { + node := &serviceNode{} + if err := json.Unmarshal(val, &node); err != nil { + retErr = fmt.Errorf("json unmarshal one service from db failed, err: %s", err.Error()) + return false + } + fmt.Println(string(val)) + return true + }) + if err != nil { + retErr = err + } + return retErr +} diff --git a/blobstore/cli/clustermgr/raft_wal.go b/blobstore/cli/clustermgr/raft_wal.go new file mode 100644 index 000000000..aa9a4315c --- /dev/null +++ b/blobstore/cli/clustermgr/raft_wal.go @@ -0,0 +1,172 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package clustermgr + +import ( + "encoding/binary" + "errors" + "fmt" + "hash/crc32" + "io" + "io/ioutil" + "log" + "os" + + "github.com/desertbit/grumble" + pb "go.etcd.io/etcd/raft/v3/raftpb" + + "github.com/cubefs/cubefs/blobstore/clustermgr/base" + "github.com/cubefs/cubefs/blobstore/clustermgr/volumemgr" +) + +var logger *log.Logger + +func addCmdWalParse(cmd *grumble.Command) { + command := &grumble.Command{ + Name: "wal", + Help: "wal tools", + LongHelp: "wal tools for clustermgr", + } + cmd.AddCommand(command) + + command.AddCommand(&grumble.Command{ + Name: "parse", + Help: "parse wal file or path", + Run: cmdWalParse, + Args: func(a *grumble.Args) { + a.String("filename", "wal log filename", grumble.Default("")) + a.String("path", "wal log path", grumble.Default("")) + a.String("out", "pase result output", grumble.Default("")) + }, + }) +} + +func cmdWalParse(c *grumble.Context) error { + filename := c.Args.String("filename") + path := c.Args.String("path") + out := c.Args.String("out") + + if filename == "" && path == "" { + return errors.New("wal log file path can't be null") + } + if out != "" { + f, err := os.OpenFile(out, os.O_CREATE|os.O_APPEND, 0o755) + if err != nil { + return err + } + logger = log.New(f, "", log.LstdFlags) + } + + if filename != "" { + parse(filename) + return nil + } + fis, err := ioutil.ReadDir(path) + if err != nil { + return err + } + for i := range fis { + if fis[i].IsDir() { + continue + } + parse(path + fis[i].Name()) + } + return nil +} + +func parse(filename string) { + f, err := os.Open(filename) + if err != nil { + printf("open %s error: %v", filename, err) + return + } + + u64Buf := make([]byte, 8) + var offset int64 + for { + n, err := io.ReadFull(f, u64Buf) + if err != nil { + if err != io.EOF { + printf("read data len offset=%d error: %v\n", offset, err) + } + break + } + offset = offset + int64(n) + dataLen := binary.BigEndian.Uint64(u64Buf) + crc := crc32.NewIEEE() + tr := io.TeeReader(f, crc) + data := make([]byte, dataLen+4) + n, err = io.ReadFull(tr, data[0:int(dataLen)]) + if err != nil { + printf("read data offset=%d error: %v\n", offset, err) + break + } + offset = offset + int64(n) + n, err = io.ReadFull(f, data[dataLen:]) + if err != nil { + printf("read data crc offset=%d error: %v\n", offset, err) + break + } + offset = offset + int64(n) + + // decode crc + recType := data[0] + checksum := binary.BigEndian.Uint32(data[dataLen:]) + data = data[1:dataLen] + // checksum + if checksum != crc.Sum32() { + printf("error checksum=%d\n", checksum) + break + } + + if recType != 1 { + fmt.Println("type: ", recType) + break + } + + var entry pb.Entry + if err = entry.Unmarshal(data); err != nil { + printf("unmarshal error: %v\n", err) + break + } + if entry.Type == pb.EntryConfChange { + var cc pb.ConfChange + cc.Unmarshal(entry.Data) + printf("term=%d index=%d ConfChange=%s\n", entry.Term, entry.Index, cc.String()) + } else { + if len(entry.Data) == 0 { + printf("term=%d index=%d\n", entry.Term, entry.Index) + } else { + data := entry.Data[8:] + proposeInfo := base.DecodeProposeInfo(data) + if proposeInfo.Module == "VolumeMgr" && proposeInfo.OperType == volumemgr.OperTypeChunkReport { + printf("term=%d index=%d module=%s opertype=%d data=%v \n", + entry.Term, entry.Index, proposeInfo.Module, proposeInfo.OperType, proposeInfo.Data) + } else { + printf("term=%d index=%d module=%s opertype=%d data=%s \n", + entry.Term, entry.Index, proposeInfo.Module, proposeInfo.OperType, string(proposeInfo.Data)) + } + } + } + } +} + +func printf(format string, opts ...interface{}) { + if logger != nil { + logger.Printf(format, opts...) + } else { + fmt.Printf(format, opts...) + } +} diff --git a/blobstore/cli/clustermgr/service.go b/blobstore/cli/clustermgr/service.go new file mode 100644 index 000000000..e830eb7e0 --- /dev/null +++ b/blobstore/cli/clustermgr/service.go @@ -0,0 +1,68 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package clustermgr + +import ( + "fmt" + + "github.com/desertbit/grumble" + + "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/cli/common" + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +func cmdGetService(c *grumble.Context) error { + cli, ctx := newCMClient(c.Flags.String("secret"), specificHosts(c.Flags)...), common.CmdContext() + + names := []string{ + proto.ServiceNameProxy, + proto.ServiceNameBlobNode, + } + name := c.Args.String("name") + if name != "" { + names = []string{name} + } + + for _, name := range names { + fmt.Println("Service of", name, ":") + info, err := cli.GetService(ctx, clustermgr.GetServiceArgs{ + Name: name, + }) + if err != nil { + return err + } + fmt.Println(common.Readable(info)) + fmt.Println() + } + + return nil +} + +func addCmdService(cmd *grumble.Command) { + serviceCommand := &grumble.Command{ + Name: "service", + Help: "service tools", + LongHelp: "service tools for clustermgr [name]", + Run: cmdGetService, + Args: func(a *grumble.Args) { + a.String("name", "service name", grumble.Default("")) + }, + Flags: func(f *grumble.Flags) { + clusterFlags(f) + }, + } + cmd.AddCommand(serviceCommand) +} diff --git a/blobstore/cli/clustermgr/volumemgr.go b/blobstore/cli/clustermgr/volumemgr.go new file mode 100644 index 000000000..5a6fd4018 --- /dev/null +++ b/blobstore/cli/clustermgr/volumemgr.go @@ -0,0 +1,251 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package clustermgr + +import ( + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/desertbit/grumble" + + "github.com/cubefs/cubefs/blobstore/api/blobnode" + "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/cli/common" + "github.com/cubefs/cubefs/blobstore/cli/common/args" + "github.com/cubefs/cubefs/blobstore/cli/common/cfmt" + "github.com/cubefs/cubefs/blobstore/cli/common/flags" + "github.com/cubefs/cubefs/blobstore/clustermgr/persistence/volumedb" + "github.com/cubefs/cubefs/blobstore/common/kvstore" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/util/task" +) + +func addCmdVolume(cmd *grumble.Command) { + command := &grumble.Command{ + Name: "volume", + Help: "volume tools", + LongHelp: "volume tools for clustermgr", + } + cmd.AddCommand(command) + + command.AddCommand(&grumble.Command{ + Name: "listVolumeUnits", + Help: "show volume units", + Run: cmdListVolumeUnits, + Args: func(a *grumble.Args) { + args.VidRegister(a) + }, + Flags: func(f *grumble.Flags) { + flags.VerboseRegister(f) + clusterFlags(f) + }, + }) + + command.AddCommand(&grumble.Command{ + Name: "updateVolume", + Help: "update volume info", + Run: cmdUpdateVolume, + Args: func(a *grumble.Args) { + args.VidRegister(a) + a.String("dbPath", "volume db path") + a.String("volumeInfo", "modify volume info, "+ + "only support for codemode/status/total/used/free") + }, + Flags: func(f *grumble.Flags) { + flags.VerboseRegister(f) + clusterFlags(f) + }, + }) + + command.AddCommand(&grumble.Command{ + Name: "updateVolumeUnit", + Help: "update volume unit", + Run: cmdUpdateVolumeUnit, + Args: func(a *grumble.Args) { + args.VuidRegister(a) + a.String("dbPath", "volume db path") + a.String("unit", "modify volume unit, "+ + "only support for diskid/epoch/nexEpoch/compacting") + }, + Flags: func(f *grumble.Flags) { + flags.VerboseRegister(f) + clusterFlags(f) + }, + }) +} + +func cmdUpdateVolume(c *grumble.Context) error { + vid := args.Vid(c.Args) + dbPath := c.Args.String("dbPath") + data := c.Args.String("volumeInfo") + if vid == 0 || dbPath == "" || data == "" { + return errors.New("invalid command arguments") + } + + modifyInfo := &clustermgr.VolumeInfo{} + err := json.Unmarshal([]byte(data), modifyInfo) + if err != nil { + return err + } + db, err := openVolumeDB(dbPath, false) + if err != nil { + return err + } + defer db.Close() + tbl, err := openVolumeTable(db) + if err != nil { + return err + } + srcInfo, err := tbl.GetVolume(proto.Vid(vid)) + if err != nil { + return err + } + + if modifyInfo.Status.IsValid() { + srcInfo.Status = modifyInfo.Status + } + if modifyInfo.CodeMode.IsValid() { + srcInfo.CodeMode = modifyInfo.CodeMode + } + if modifyInfo.Total > 0 { + srcInfo.Total = modifyInfo.Total + } + if modifyInfo.Used > 0 { + srcInfo.Used = modifyInfo.Used + } + if modifyInfo.Free > 0 { + srcInfo.Free = modifyInfo.Free + } + + if !common.Confirm("to change?\n" + cfmt.VolumeInfoJoin(modifyInfo, "")) { + return nil + } + return tbl.PutVolumeRecord(srcInfo) +} + +func cmdListVolumeUnits(c *grumble.Context) error { + cmClient := newCMClient(c.Flags.String("secret"), specificHosts(c.Flags)...) + ctx := common.CmdContext() + volumeInfo, err := cmClient.GetVolumeInfo(ctx, + &clustermgr.GetVolumeArgs{Vid: args.Vid(c.Args)}) + if err != nil { + return err + } + + fmt.Println("volume info:") + fmt.Println(cfmt.VolumeInfoJoin(volumeInfo, "\t")) + + verbose := flags.Verbose(c.Flags) + + dnConfig := blobnode.Config{} + dnConfig.ClientTimeoutMs = 3000 + blobnodeCli := blobnode.New(&dnConfig) + + n := len(volumeInfo.Units) + loader := common.Loader(n) + + taskArgs := make([]interface{}, n) + for i, arg := range volumeInfo.Units { + taskArgs[i] = arg + } + chunkInfos := make([]string, n) + task.C(func(i int, taskArgs interface{}) { + unit := taskArgs.(clustermgr.Unit) + info, err := blobnodeCli.StatChunk(ctx, unit.Host, + &blobnode.StatChunkArgs{ + DiskID: unit.DiskID, + Vuid: unit.Vuid, + }) + if err != nil { + chunkInfos[i] = fmt.Sprintf("ERROR: %s %d %s", unit.Host, unit.Vuid, err.Error()) + } else if verbose { + chunkInfos[i] = cfmt.ChunkInfoJoin(info, "\t") + } else { + chunkInfos[i] = fmt.Sprint(info) + } + loader <- 1 + }, taskArgs) + time.Sleep(100 * time.Millisecond) + + fmt.Println("chunks:") + for i, info := range chunkInfos { + if verbose { + fmt.Println("chunk info:", i) + } + fmt.Println(info) + } + + return nil +} + +func cmdUpdateVolumeUnit(c *grumble.Context) error { + vuid := args.Vuid(c.Args) + dbPath := c.Args.String("dbPath") + data := c.Args.String("unitInfo") + if vuid == 0 || dbPath == "" || data == "" { + return errors.New("invalid command arguments") + } + + modifyInfo := &clustermgr.AdminUpdateUnitArgs{} + err := json.Unmarshal([]byte(data), modifyInfo) + if err != nil { + return err + } + db, err := openVolumeDB(dbPath, false) + if err != nil { + return err + } + defer db.Close() + tbl, err := openVolumeTable(db) + if err != nil { + return err + } + srcInfo, err := tbl.GetVolumeUnit(vuid.VuidPrefix()) + if err != nil { + return err + } + if modifyInfo.DiskID > 0 { + srcInfo.DiskID = modifyInfo.DiskID + } + if proto.IsValidEpoch(modifyInfo.Epoch) { + srcInfo.Epoch = modifyInfo.Epoch + } + if proto.IsValidEpoch(modifyInfo.NextEpoch) { + srcInfo.NextEpoch = modifyInfo.NextEpoch + } + srcInfo.Compacting = modifyInfo.Compacting + + return tbl.PutVolumeUnit(vuid.VuidPrefix(), srcInfo) +} + +func openVolumeDB(path string, readonly bool) (*volumedb.VolumeDB, error) { + db, err := volumedb.Open(path, false, func(option *kvstore.RocksDBOption) { + option.ReadOnly = readonly + }) + if err != nil { + return nil, fmt.Errorf("open db failed, err: %s", err.Error()) + } + return db, nil +} + +func openVolumeTable(db *volumedb.VolumeDB) (*volumedb.VolumeTable, error) { + tbl, err := volumedb.OpenVolumeTable(db) + if err != nil { + return nil, fmt.Errorf("open volume table failed, err: %s", err.Error()) + } + return tbl, nil +} diff --git a/blobstore/cli/common/args/args.go b/blobstore/cli/common/args/args.go new file mode 100644 index 000000000..1ec8b5ad7 --- /dev/null +++ b/blobstore/cli/common/args/args.go @@ -0,0 +1,56 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package args + +import ( + "github.com/desertbit/grumble" + + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +// Line lines +func Line(a *grumble.Args) { + a.Int("line", "show lines at the tail", grumble.Default(40)) +} + +// VidRegister vid +func VidRegister(a *grumble.Args, opts ...grumble.ArgOption) { + a.Uint64("vid", "volume id", opts...) +} + +// Vid returns vid +func Vid(a grumble.ArgMap) proto.Vid { + return proto.Vid(a.Uint64("vid")) +} + +// DiskIDRegister disk id +func DiskIDRegister(a *grumble.Args, opts ...grumble.ArgOption) { + a.Uint64("diskID", "disk id", opts...) +} + +// DiskID returns disk id +func DiskID(a grumble.ArgMap) proto.DiskID { + return proto.DiskID(a.Uint64("diskID")) +} + +// VuidRegister vuid +func VuidRegister(a *grumble.Args, opts ...grumble.ArgOption) { + a.Uint64("vuid", "vuid", opts...) +} + +// Vuid returns vuid +func Vuid(a grumble.ArgMap) proto.Vuid { + return proto.Vuid(a.Uint64("vuid")) +} diff --git a/blobstore/cli/common/cfmt/access.go b/blobstore/cli/common/cfmt/access.go new file mode 100644 index 000000000..8fc1ca5fc --- /dev/null +++ b/blobstore/cli/common/cfmt/access.go @@ -0,0 +1,96 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cfmt + +import ( + "fmt" + + "github.com/dustin/go-humanize" + + "github.com/cubefs/cubefs/blobstore/api/access" + "github.com/cubefs/cubefs/blobstore/cli/common" +) + +// ParseLocation parst location from json or string +func ParseLocation(jsonORstr string) (access.Location, error) { + var loc access.Location + var err error + if err = common.Unmarshal([]byte(jsonORstr), &loc); err == nil { + return loc, nil + } + if loc, err = access.DecodeLocationFromHex(jsonORstr); err == nil { + return loc, nil + } + if loc, err = access.DecodeLocationFromBase64(jsonORstr); err == nil { + return loc, nil + } + return loc, fmt.Errorf("invalid (%s) %s", jsonORstr, err.Error()) +} + +// LocationJoin join line into string +func LocationJoin(loc *access.Location, prefix string) string { + return joinWithPrefix(prefix, LocationF(loc)) +} + +// LocationF fmt pointer of Location +func LocationF(loc *access.Location) (vals []string) { + if loc == nil { + return []string{" "} + } + vals = make([]string, 0, 8) + vals = append(vals, []string{ + fmt.Sprintf("Crc : %-12d (0x%x)", loc.Crc, loc.Crc), + fmt.Sprintf("ClusterID : %d", loc.ClusterID), + fmt.Sprintf("CodeMode : %-12d (%s)", loc.CodeMode, loc.CodeMode.String()), + fmt.Sprintf("Size : %-12d (%s)", loc.Size, humanize.IBytes(loc.Size)), + fmt.Sprintf("BlobSize : %-12d (%s)", loc.BlobSize, humanize.IBytes(uint64(loc.BlobSize))), + fmt.Sprintf("Blobs: (%d) [", len(loc.Blobs)), + }...) + for idx, blob := range loc.Blobs { + vals = append(vals, fmt.Sprintf(" >:%3d| MinBid: %-20d Vid: %-10d Count: %-10d", + idx, blob.MinBid, blob.Vid, blob.Count)) + } + vals = append(vals, "]") + vals = append(vals, fmt.Sprintf("--> Encode: %d of %d bytes", len(loc.Encode()), 21+16*len(loc.Blobs))) + vals = append(vals, fmt.Sprintf("--> Hex : %s", loc.HexString())) + vals = append(vals, fmt.Sprintf("--> Base64: %s", loc.Base64String())) + return +} + +// HashSumMapJoin join line into string +func HashSumMapJoin(hashes access.HashSumMap, prefix string) string { + return joinWithPrefix(prefix, HashSumMapF(hashes)) +} + +// HashSumMapF fmt HashSumMap +func HashSumMapF(hashes access.HashSumMap) (vals []string) { + if v, ok := hashes.GetSum(access.HashAlgCRC32); ok { + val := v.(uint32) + vals = append(vals, fmt.Sprintf("CRC32 : %d (%x)", val, val)) + } + if v, ok := hashes.GetSum(access.HashAlgMD5); ok { + val := v.(string) + vals = append(vals, fmt.Sprintf("MD5 : %s", val)) + } + if v, ok := hashes.GetSum(access.HashAlgSHA1); ok { + val := v.(string) + vals = append(vals, fmt.Sprintf("SHA1 : %s", val)) + } + if v, ok := hashes.GetSum(access.HashAlgSHA256); ok { + val := v.(string) + vals = append(vals, fmt.Sprintf("SHA256: %s", val)) + } + return +} diff --git a/blobstore/cli/common/cfmt/access_test.go b/blobstore/cli/common/cfmt/access_test.go new file mode 100644 index 000000000..61f5efde1 --- /dev/null +++ b/blobstore/cli/common/cfmt/access_test.go @@ -0,0 +1,113 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cfmt_test + +import ( + "crypto/rand" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/api/access" + "github.com/cubefs/cubefs/blobstore/cli/common" + "github.com/cubefs/cubefs/blobstore/cli/common/cfmt" +) + +func TestParseLocation(t *testing.T) { + loc := access.Location{ + ClusterID: 1, + CodeMode: 3, + Size: 19213422425, + BlobSize: 1 << 22, + Crc: 1 << 31, + Blobs: []access.SliceInfo{ + {MinBid: 0x199, Vid: 100020, Count: 0}, + {MinBid: 10, Vid: 20, Count: 300}, + {MinBid: 14225224, Vid: 0xffffffff, Count: 1 << 31}, + }, + } + + locx, err := cfmt.ParseLocation("") + require.Error(t, err) + + _, err = cfmt.ParseLocation("xxxx") + require.Error(t, err) + + locx, err = cfmt.ParseLocation("{}") + require.NoError(t, err) + require.Equal(t, access.Location{}, locx) + + locx, err = cfmt.ParseLocation(loc.ToString()) + require.NoError(t, err) + require.Equal(t, loc, locx) + + locx, err = cfmt.ParseLocation(loc.Base64String()) + require.NoError(t, err) + require.Equal(t, loc, locx) + + b, _ := common.Marshal(loc) + locx, err = cfmt.ParseLocation(string(b)) + require.NoError(t, err) + require.Equal(t, loc, locx) +} + +func TestLocation(t *testing.T) { + loc := access.Location{ + ClusterID: 1, + CodeMode: 3, + Size: 19213422425, + BlobSize: 1 << 22, + Crc: 1 << 31, + Blobs: []access.SliceInfo{ + {MinBid: 0x199, Vid: 100020, Count: 0}, + {MinBid: 10, Vid: 20, Count: 300}, + {MinBid: 14225224, Vid: 0xffffffff, Count: 1 << 31}, + }, + } + printLine() + for _, line := range cfmt.LocationF(&loc) { + fmt.Println(line) + } + printLine() + fmt.Println(cfmt.LocationJoin(&loc, "\t-->\t")) + printLine() + fmt.Println(cfmt.LocationJoin(nil, "\t--> ")) + printLine() +} + +func TestHashSumMap(t *testing.T) { + hashes := make(access.HashSumMap) + algs := access.HashAlgorithm(0xff - 1) + for alg := range algs.ToHashSumMap() { + hasher := alg.ToHasher() + buf := make([]byte, 1024) + rand.Read(buf) + hasher.Write(buf) + hashes[alg] = hasher.Sum(nil) + } + printLine() + for _, printLine := range cfmt.HashSumMapF(hashes) { + fmt.Println(printLine) + } + printLine() + fmt.Println(cfmt.HashSumMapJoin(hashes, "xxx | --> ")) + printLine() +} + +func printLine() { + fmt.Println(strings.Repeat("-", 100)) +} diff --git a/blobstore/cli/common/cfmt/blobnode.go b/blobstore/cli/common/cfmt/blobnode.go new file mode 100644 index 000000000..037d9686e --- /dev/null +++ b/blobstore/cli/common/cfmt/blobnode.go @@ -0,0 +1,132 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cfmt + +import ( + "fmt" + "time" + + "github.com/dustin/go-humanize" + "github.com/fatih/color" + + "github.com/cubefs/cubefs/blobstore/api/blobnode" +) + +var chunkStatus2Str = map[blobnode.ChunkStatus]string{ + blobnode.ChunkStatusNormal: "normal", + blobnode.ChunkStatusReadOnly: "readonly", + blobnode.ChunkStatusRelease: "release", +} + +// ChunkidF chunk id +func ChunkidF(id blobnode.ChunkId) string { + t := id.UnixTime() + c := color.New(color.Faint, color.Italic) + return fmt.Sprintf("%s (V:%20d T:%20d %s)", id.String(), id.VolumeUnitId(), + t, c.Sprint(time.Unix(0, int64(t)).Format("2006-01-02 15:04:05.000")), + ) +} + +// ChunkInfoJoin chunk info +func ChunkInfoJoin(info *blobnode.ChunkInfo, prefix string) string { + return joinWithPrefix(prefix, ChunkInfoF(info)) +} + +// ChunkInfoF chunk info +func ChunkInfoF(info *blobnode.ChunkInfo) []string { + if info == nil { + return []string{" "} + } + return []string{ + fmt.Sprintf("ID : %s", ChunkidF(info.Id)), + fmt.Sprintf("Vuid : %s", VuidCF(info.Vuid)), + fmt.Sprintf("DiskID : %-10d", info.DiskID), + fmt.Sprintf("Status : %d (%s)", info.Status, chunkStatus2Str[info.Status]), + fmt.Sprintf("Compact: %v", info.Compacting), + fmt.Sprintf("Size : %-24s", humanIBytes(info.Size)), + fmt.Sprintf("Total : %-24s Used: %-24s Free: %-24s", + humanIBytes(info.Total), humanIBytes(info.Used), humanIBytes(info.Free)), + } +} + +// DiskHeartBeatInfoJoin disk heartbeat info +func DiskHeartBeatInfoJoin(info *blobnode.DiskHeartBeatInfo, prefix string) string { + return joinWithPrefix(prefix, DiskHeartBeatInfoF(info)) +} + +// DiskHeartBeatInfoF disk heartbeat info +func DiskHeartBeatInfoF(info *blobnode.DiskHeartBeatInfo) []string { + if info == nil { + return []string{" "} + } + return []string{ + fmt.Sprintf("DiskID: %-12d | MaxN: %-8d | UsedN: %-8d | FreeN: %-8d", + info.DiskID, info.MaxChunkCnt, info.UsedChunkCnt, info.FreeChunkCnt), + fmt.Sprintf("Size : %-24s | Used: %-24s | Free: %-24s", + humanIBytes(info.Size), humanIBytes(info.Used), humanIBytes(info.Free)), + } +} + +// DiskInfoJoin disk info +func DiskInfoJoin(info *blobnode.DiskInfo, prefix string) string { + return joinWithPrefix(prefix, DiskInfoF(info)) +} + +// DiskInfoF disk info +func DiskInfoF(info *blobnode.DiskInfo) []string { + if info == nil { + return []string{" "} + } + return append(DiskHeartBeatInfoF(&info.DiskHeartBeatInfo), + []string{ + fmt.Sprintf("ClusterID : %-4d | Readonly: %-6v | IDC: %-12s | Rack: %s", + info.ClusterID, info.Readonly, info.Idc, info.Rack), + fmt.Sprintf("Status : %-10s(%d) | Host: %-30s | Path: %s", + info.Status, info.Status, info.Host, info.Path), + fmt.Sprintf("CreateAt: %s (%s) | LastUpdateAt: %s (%s)", + info.CreateAt.Format(time.RFC822), humanize.Time(info.CreateAt), + info.LastUpdateAt.Format(time.RFC822), humanize.Time(info.LastUpdateAt)), + }...) +} + +// DiskInfoJoinV disk info verbose +func DiskInfoJoinV(info *blobnode.DiskInfo, prefix string) string { + return joinWithPrefix(prefix, DiskInfoFV(info)) +} + +// DiskInfoFV disk info +func DiskInfoFV(info *blobnode.DiskInfo) []string { + if info == nil { + return []string{" "} + } + return []string{ + fmt.Sprint("DiskID : ", info.DiskID), + fmt.Sprint("Readonly : ", info.Readonly), + fmt.Sprintf("Status : %s(%d) ", info.Status, info.Status), + fmt.Sprintf("Chunk : MaxN: %-18d | UsedN: %-23d | FreeN: %-24d", + info.MaxChunkCnt, info.UsedChunkCnt, info.FreeChunkCnt), + fmt.Sprintf("Size : %-24s | Used: %-24s | Free: %-24s", + humanIBytes(info.Size), humanIBytes(info.Used), humanIBytes(info.Free)), + fmt.Sprint("ClusterID: ", info.ClusterID), + fmt.Sprint("IDC : ", info.Idc), + fmt.Sprint("Rack : ", info.Rack), + fmt.Sprint("Host : ", info.Host), + fmt.Sprint("Path : ", info.Path), + fmt.Sprintf("CreateAt : %s (%s)", + info.CreateAt.Format(time.RFC822), humanize.Time(info.CreateAt)), + fmt.Sprintf("UpdateAt : %s (%s)", + info.LastUpdateAt.Format(time.RFC822), humanize.Time(info.LastUpdateAt)), + } +} diff --git a/blobstore/cli/common/cfmt/blobnode_test.go b/blobstore/cli/common/cfmt/blobnode_test.go new file mode 100644 index 000000000..216ba64c2 --- /dev/null +++ b/blobstore/cli/common/cfmt/blobnode_test.go @@ -0,0 +1,110 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cfmt_test + +import ( + "fmt" + "testing" + "time" + + "github.com/cubefs/cubefs/blobstore/api/blobnode" + "github.com/cubefs/cubefs/blobstore/cli/common/cfmt" + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +func TestChunkInfo(t *testing.T) { + val := blobnode.ChunkInfo{ + Id: blobnode.NewChunkId(4191518780817409), + Vuid: 4191518780817409, + DiskID: 0xff, + Total: 1 << 35, + Used: 1 << 34, + Free: 100, + Size: 1 << 32, + Status: blobnode.ChunkStatusReadOnly, + Compacting: false, + } + printLine() + for _, line := range cfmt.ChunkInfoF(&val) { + fmt.Println(line) + } + printLine() + fmt.Println(cfmt.ChunkInfoJoin(&val, "\t--> ")) + printLine() + fmt.Println(cfmt.ChunkInfoJoin(nil, "\t--> ")) + printLine() +} + +func TestDiskHeartBeatInfo(t *testing.T) { + val := blobnode.DiskHeartBeatInfo{ + DiskID: 12342522, + Used: 10220055555, + Free: 9483500, + Size: 10 * (1 << 40), + MaxChunkCnt: 9999, + FreeChunkCnt: 880, + UsedChunkCnt: 82, + } + printLine() + for _, line := range cfmt.DiskHeartBeatInfoF(&val) { + fmt.Println(line) + } + printLine() + fmt.Println(cfmt.DiskHeartBeatInfoJoin(&val, "\t--> ")) + printLine() + fmt.Println(cfmt.DiskHeartBeatInfoJoin(nil, "\t--> ")) + printLine() +} + +func TestDiskInfo(t *testing.T) { + val := blobnode.DiskInfo{ + ClusterID: 0xf2, + Idc: "idc--xx", + Rack: "rack-10", + Host: "http://255.255.255.255:99999", + Path: "/path/to/disk", + Status: proto.DiskStatusRepaired, + Readonly: false, + CreateAt: time.Now().Add(-time.Hour * 1020), + LastUpdateAt: time.Now().Add(-time.Hour * 224), + DiskHeartBeatInfo: blobnode.DiskHeartBeatInfo{ + DiskID: 12342522, + Used: 10220055555, + Free: 9483500, + Size: 10 * (1 << 40), + MaxChunkCnt: 9999, + FreeChunkCnt: 880, + UsedChunkCnt: 82, + }, + } + printLine() + for _, line := range cfmt.DiskInfoF(&val) { + fmt.Println(line) + } + printLine() + fmt.Println(cfmt.DiskInfoJoin(&val, "\t--> ")) + printLine() + fmt.Println(cfmt.DiskInfoJoin(nil, "\t--> ")) + printLine() + + for _, line := range cfmt.DiskInfoFV(&val) { + fmt.Println(line) + } + printLine() + fmt.Println(cfmt.DiskInfoJoinV(&val, "\t--> ")) + printLine() + fmt.Println(cfmt.DiskInfoJoinV(nil, "\t--> ")) + printLine() +} diff --git a/blobstore/cli/common/cfmt/cfmt.go b/blobstore/cli/common/cfmt/cfmt.go new file mode 100644 index 000000000..d61812511 --- /dev/null +++ b/blobstore/cli/common/cfmt/cfmt.go @@ -0,0 +1,43 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +// Package cfmt provides fmt string for all struct in blobstore +// +// function *F make the pointer struct value to []string +// function *Join []string into string with profix each line, sep is '\n' +// +package cfmt + +import ( + "fmt" + "strconv" + "strings" + + "github.com/dustin/go-humanize" +) + +func joinWithPrefix(prefix string, vals []string) string { + if len(prefix) > 0 { + for idx := range vals { + vals[idx] = prefix + vals[idx] + } + } + return strings.Join(vals, "\n") +} + +func humanIBytes(size interface{}) string { + str := fmt.Sprintf("%v", size) + s, _ := strconv.Atoi(str) + return fmt.Sprintf("%s (%s)", str, humanize.IBytes(uint64(s))) +} diff --git a/blobstore/cli/common/cfmt/cluster.go b/blobstore/cli/common/cfmt/cluster.go new file mode 100644 index 000000000..6fafd17f3 --- /dev/null +++ b/blobstore/cli/common/cfmt/cluster.go @@ -0,0 +1,83 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cfmt + +import ( + "fmt" + + "github.com/dustin/go-humanize" + + "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/cli/common" +) + +// VolumeInfoJoin volume info +func VolumeInfoJoin(vol *clustermgr.VolumeInfo, prefix string) string { + return joinWithPrefix(prefix, VolumeInfoF(vol)) +} + +// VolumeInfoF volume info +func VolumeInfoF(vol *clustermgr.VolumeInfo) []string { + if vol == nil { + return []string{" "} + } + usedC := common.ColorizeUint64(vol.Used, vol.Total) + freeC := common.ColorizeUint64Free(vol.Free, vol.Total) + vals := make([]string, 0, 32) + vals = append(vals, []string{ + fmt.Sprintf("Vid : %d", vol.Vid), + fmt.Sprintf("CodeMode : %-12d (%s)", vol.CodeMode, vol.CodeMode.String()), + fmt.Sprintf("Status : %-12d (%s)", vol.Status, vol.Status.String()), + fmt.Sprintf("HealthScore: %d", vol.HealthScore), + fmt.Sprintf("Total : %-16d (%s)", vol.Total, humanize.IBytes(vol.Total)), + fmt.Sprintf("Free : %-16d (%s)", vol.Free, freeC.Sprint(humanize.IBytes(vol.Free))), + fmt.Sprintf("Used : %-16d (%s)", vol.Used, usedC.Sprint(humanize.IBytes(vol.Used))), + fmt.Sprintf("Uints: (%d) [", len(vol.Units)), + }...) + alterColor := common.NewAlternateColor(3) + for idx, unit := range vol.Units { + vals = append(vals, alterColor.Next().Sprintf(" >:%3d| Vuid: %s | DiskID: %-10d | Host: %s", + idx, VuidF(unit.Vuid), unit.DiskID, unit.Host)) + } + vals = append(vals, "]") + return vals +} + +// ClusterInfoJoin cluster info +func ClusterInfoJoin(info *clustermgr.ClusterInfo, prefix string) string { + return joinWithPrefix(prefix, ClusterInfoF(info)) +} + +// ClusterInfoF cluster info +func ClusterInfoF(info *clustermgr.ClusterInfo) []string { + if info == nil { + return []string{" "} + } + avaiC := common.ColorizeInt64(-info.Available, info.Capacity) + vals := make([]string, 0, 8) + vals = append(vals, []string{ + fmt.Sprintf("Region : %s", info.Region), + fmt.Sprintf("ClusterID : %d", info.ClusterID), + fmt.Sprintf("Readonly : %v", info.Readonly), + fmt.Sprintf("Capacity : %-16d (%s)", info.Capacity, humanize.IBytes(uint64(info.Capacity))), + fmt.Sprintf("Available : %-16d (%s)", info.Available, avaiC.Sprint(humanize.IBytes(uint64(info.Available)))), + fmt.Sprintf("Nodes: (%d) [", len(info.Nodes)), + }...) + for _, node := range info.Nodes { + vals = append(vals, fmt.Sprintf(" - %s", node)) + } + vals = append(vals, "]") + return vals +} diff --git a/blobstore/cli/common/cfmt/cluster_test.go b/blobstore/cli/common/cfmt/cluster_test.go new file mode 100644 index 000000000..0cf711ad8 --- /dev/null +++ b/blobstore/cli/common/cfmt/cluster_test.go @@ -0,0 +1,76 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cfmt_test + +import ( + "fmt" + "math/rand" + "testing" + + "github.com/cubefs/cubefs/blobstore/api/clustermgr" + "github.com/cubefs/cubefs/blobstore/cli/common/cfmt" + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +func TestVolumeInfo(t *testing.T) { + val := clustermgr.VolumeInfo{ + VolumeInfoBase: clustermgr.VolumeInfoBase{ + Vid: 101313, + CodeMode: 2, + Status: 2, + HealthScore: -1, + Total: 1 << 50, + Free: 1 << 32, + Used: 1 << 45, + }, + Units: make([]clustermgr.Unit, 32), + } + for i := 0; i < len(val.Units); i++ { + val.Units[i] = clustermgr.Unit{ + Vuid: proto.Vuid(rand.Uint64()), + DiskID: proto.DiskID(rand.Uint32()), + Host: fmt.Sprintf("host-%d", i), + } + } + printLine() + for _, line := range cfmt.VolumeInfoF(&val) { + fmt.Println(line) + } + printLine() + fmt.Println(cfmt.VolumeInfoJoin(&val, "\t--> ")) + printLine() + fmt.Println(cfmt.VolumeInfoJoin(nil, "\t--> ")) + printLine() +} + +func TestClusterInfo(t *testing.T) { + val := clustermgr.ClusterInfo{ + Region: "foo-region", + ClusterID: 0xffff, + Capacity: 1 << 40, + Available: 1 << 38, + Readonly: false, + Nodes: []string{"node-1", "node-2", "node-xxx"}, + } + printLine() + for _, line := range cfmt.ClusterInfoF(&val) { + fmt.Println(line) + } + printLine() + fmt.Println(cfmt.ClusterInfoJoin(&val, "\t--> ")) + printLine() + fmt.Println(cfmt.ClusterInfoJoin(nil, "\t--> ")) + printLine() +} diff --git a/blobstore/cli/common/cfmt/proto.go b/blobstore/cli/common/cfmt/proto.go new file mode 100644 index 000000000..fbdc7fa11 --- /dev/null +++ b/blobstore/cli/common/cfmt/proto.go @@ -0,0 +1,40 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cfmt + +import ( + "fmt" + + "github.com/fatih/color" + + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +// VuidF vuid fmt +func VuidF(vuid proto.Vuid) string { + return fmt.Sprintf("%-20d (V:%10d I:%3d E:%10d)", vuid, + vuid.Vid(), vuid.Index(), vuid.Epoch(), + ) +} + +// VuidCF vuid fmt with color +func VuidCF(vuid proto.Vuid) string { + c := color.New(color.Faint, color.Italic) + return fmt.Sprintf("%-20d (%s%10d %s%3d %s%10d)", vuid, + c.Sprint("V:"), vuid.Vid(), + c.Sprint("I:"), vuid.Index(), + c.Sprint("E:"), vuid.Epoch(), + ) +} diff --git a/blobstore/cli/common/color.go b/blobstore/cli/common/color.go new file mode 100644 index 000000000..97ed1b2e8 --- /dev/null +++ b/blobstore/cli/common/color.go @@ -0,0 +1,144 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package common + +import ( + "github.com/fatih/color" +) + +// colorize defined +var ( + Optimal = color.New(color.FgHiWhite) + Normal = color.New(color.FgHiGreen) + Loaded = color.New(color.FgHiBlue) + Warn = color.New(color.FgHiYellow) + Danger = color.New(color.FgHiRed) + Dead = color.New(color.FgBlack, color.BgCyan) + + alternateColor = []*color.Color{ + color.New(color.FgHiWhite), + color.New(color.FgWhite, color.Faint), + color.New(color.FgHiBlue), + color.New(color.FgBlue, color.Faint), + color.New(color.FgHiGreen), + color.New(color.FgGreen, color.Faint), + } + maxAlternateColor = len(alternateColor) +) + +// AlternateColor alternate color formatter +type AlternateColor struct { + index int + n int + colors []*color.Color +} + +// NewAlternateColor returns an alternate color formatter +func NewAlternateColor(n int) *AlternateColor { + if n <= 0 { + n = 1 + } + if n > maxAlternateColor { + n = maxAlternateColor + } + return &AlternateColor{ + index: 0, + n: n, + colors: alternateColor[:n], + } +} + +// Next returns next color +func (a *AlternateColor) Next() *color.Color { + c := a.colors[a.index] + a.index = (a.index + 1) % a.n + return c +} + +// Colorize by int [-100, 100] percent +// Danger --> Optimal --> Danger +func Colorize(percent int) *color.Color { + if percent < -100 { + percent = -100 + } + if percent > 100 { + percent = 100 + } + if percent < 0 { + percent = 100 + percent + } + + switch { + case percent < 20: + return Optimal + case percent < 40: + return Normal + case percent < 60: + return Loaded + case percent < 85: + return Warn + case percent < 97: + return Danger + default: + return Dead + } +} + +// ColorizeFloat color by float64 [-1.0, 1.0] ratio +func ColorizeFloat(ratio float64) *color.Color { + return Colorize(int(ratio * 100)) +} + +func percentIfFree(free bool, percent int) int { + if free && percent == 0 { + return -1 + } + return percent +} + +// ColorizeInt by int +func ColorizeInt(used, total int) *color.Color { + return Colorize(percentIfFree(used < 0, used*100/total)) +} + +// ColorizeInt32 by int32 +func ColorizeInt32(used, total int32) *color.Color { + return Colorize(percentIfFree(used < 0, int(used*100/total))) +} + +// ColorizeInt64 by int64 +func ColorizeInt64(used, total int64) *color.Color { + return Colorize(percentIfFree(used < 0, int(used*100/total))) +} + +// ColorizeUint32 by uint32 +func ColorizeUint32(used, total uint32) *color.Color { + return Colorize(int(used * 100 / total)) +} + +// ColorizeUint64 by uint64 +func ColorizeUint64(used, total uint64) *color.Color { + return Colorize(int(used * 100 / total)) +} + +// ColorizeUint32Free free by uint32 +func ColorizeUint32Free(free, total uint32) *color.Color { + return Colorize(percentIfFree(true, -int(free*100/total))) +} + +// ColorizeUint64Free free by uint64 +func ColorizeUint64Free(free, total uint64) *color.Color { + return Colorize(percentIfFree(true, -int(free*100/total))) +} diff --git a/blobstore/cli/common/color_test.go b/blobstore/cli/common/color_test.go new file mode 100644 index 000000000..d5c9bd5b6 --- /dev/null +++ b/blobstore/cli/common/color_test.go @@ -0,0 +1,101 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package common_test + +import ( + "fmt" + "math/rand" + "testing" + "time" + + "github.com/cubefs/cubefs/blobstore/cli/common" +) + +func TestCmdCommonColorAlternate(t *testing.T) { + lines := []string{ + "first-line", + "second-line", + "3th-line", + "4th-line", + "5th-line", + "6th-line", + "7th-line", + } + for _, n := range []int{-1, 0, 1, 2, 3, 4, 5, 6, 10} { + alterColor := common.NewAlternateColor(n) + fmt.Printf("%3d : ", n) + for _, line := range lines { + alterColor.Next().Print(line + " ") + } + fmt.Println() + } +} + +func TestCmdCommonColorRatio(t *testing.T) { + cases := []struct { + rand int + offset int + s string + }{ + {1000, -1100, "White [Optimal]"}, + {20, 0, "white [Optimal]"}, + {20, 20, "green [Normal]"}, + {20, 40, "blue [Loaded]"}, + {25, 60, "yellow [Warn]"}, + {12, 85, "red [Danger]"}, + {3, 97, "highlight [Dead]"}, + {1000, 100, "highlight [Dead]"}, + } + + rand.Seed(time.Now().Unix()) + for _, cs := range cases { + for i := 0; i < 10; i++ { + percent := rand.Intn(cs.rand) + cs.offset + c := common.Colorize(percent) + c.Printf("used(%d) %s ", percent, cs.s) + if cs.rand < 1000 { + c = common.Colorize(percent - 100) + c.Printf("| free(%d) %s", percent-100, cs.s) + } + fmt.Println() + } + fmt.Println() + } + + for i := 0; i < 10; i++ { + f := rand.Float64() + c := common.ColorizeFloat(f) + c.Printf("used(%.3f) unknow ", f) + c = common.ColorizeFloat(-f) + c.Printf("free(%.3f) unknow ", -f) + fmt.Println() + } + fmt.Println() + + common.ColorizeInt(0, 100).Printf("ColorizeInt --> used(%d/%d) white\n", 0, 100) + common.ColorizeInt(-1, 1000).Printf("ColorizeInt --> free(%d/%d) dead", -1, 10000) + fmt.Println() + + common.ColorizeInt(80, 100).Printf("ColorizeInt --> used(%d/%d) yellow\n", 80, 100) + common.ColorizeInt(-80, 100).Printf("ColorizeInt --> free(%d/%d) green\n", 80, 100) + common.ColorizeInt32(80, 100).Printf("ColorizeInt32 --> used(%d/%d) yellow\n", 80, 100) + common.ColorizeInt32(-80, 100).Printf("ColorizeInt32 --> free(%d/%d) green\n", 80, 100) + common.ColorizeInt64(80, 100).Printf("ColorizeInt64 --> used(%d/%d) yellow\n", 80, 100) + common.ColorizeInt64(-80, 100).Printf("ColorizeInt64 --> free(%d/%d) green\n", 80, 100) + common.ColorizeUint32(80, 100).Printf("ColorizeUint32 --> used(%d/%d) yellow\n", 80, 100) + common.ColorizeUint32Free(80, 100).Printf("ColorizeUint32Free --> free(%d/%d) green\n", 80, 100) + common.ColorizeUint64(80, 100).Printf("ColorizeUint64 --> used(%d/%d) yellow\n", 80, 100) + common.ColorizeUint64Free(80, 100).Printf("ColorizeUint64Free --> free(%d/%d) green\n", 80, 100) +} diff --git a/blobstore/cli/common/common.go b/blobstore/cli/common/common.go new file mode 100644 index 000000000..784fe16bd --- /dev/null +++ b/blobstore/cli/common/common.go @@ -0,0 +1,55 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package common + +import ( + "context" + "fmt" + "strings" + + "github.com/cubefs/cubefs/blobstore/common/trace" +) + +// CmdContext returns context with special reqest id tag +func CmdContext() context.Context { + _, ctx := trace.StartSpanFromContext(context.Background(), "from-cmd") + return ctx +} + +// Confirm ask for confirmation +// case insensitive 'y', 'yes', means yes +// case insensitive 'n', 'no', means no +// otherwise ask again +func Confirm(s string) bool { + for { + fmt.Printf("%s [Y / N]: ", s) + + var response string + _, err := fmt.Scanln(&response) + if err != nil { + fmt.Println("Error:", err) + return false + } + + response = strings.ToLower(strings.TrimSpace(response)) + switch response { + case "y", "yes": + return true + case "n", "no": + return false + default: + } + } +} diff --git a/blobstore/cli/common/common_test.go b/blobstore/cli/common/common_test.go new file mode 100644 index 000000000..397820194 --- /dev/null +++ b/blobstore/cli/common/common_test.go @@ -0,0 +1,36 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package common_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/cli/common" + "github.com/cubefs/cubefs/blobstore/common/trace" +) + +func TestCmdCommonContext(t *testing.T) { + for ii := 0; ii < 10; ii++ { + span := trace.SpanFromContext(common.CmdContext()) + t.Log(span) + span.Info("infoooooooooooooooooo bar") + } +} + +func TestCmdCommonConfirm(t *testing.T) { + require.False(t, common.Confirm("test confirm?")) +} diff --git a/blobstore/cli/common/flags/flags.go b/blobstore/cli/common/flags/flags.go new file mode 100644 index 000000000..b0e0c4593 --- /dev/null +++ b/blobstore/cli/common/flags/flags.go @@ -0,0 +1,57 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package flags + +import "github.com/desertbit/grumble" + +// VerboseRegister enable verbose mode +func VerboseRegister(f *grumble.Flags) { + f.Bool("v", "verbose", false, "enable verbose mode") +} + +// Verbose enable verbose mode +func Verbose(f grumble.FlagMap) bool { + return f.Bool("verbose") +} + +// VverboseRegister enable verbose verbose mode +func VverboseRegister(f *grumble.Flags) { + f.BoolL("vv", false, "enable verbose verbose mode") +} + +// Vverbose enable verbose verbose mode +func Vverbose(f grumble.FlagMap) bool { + return f.Bool("vv") +} + +// ConfigRegister config path +func ConfigRegister(f *grumble.Flags) { + f.String("c", "config", "", "config path") +} + +// Config config path +func Config(f grumble.FlagMap) string { + return f.String("config") +} + +// FilePathRegister file path, relative path is ok +func FilePathRegister(f *grumble.Flags) { + f.String("f", "filepath", "", "file path") +} + +// FilePath file path, relative path is ok +func FilePath(f grumble.FlagMap) string { + return f.String("filepath") +} diff --git a/blobstore/cli/common/json.go b/blobstore/cli/common/json.go new file mode 100644 index 000000000..df04ff837 --- /dev/null +++ b/blobstore/cli/common/json.go @@ -0,0 +1,56 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package common + +import ( + "encoding/json" + "fmt" + "io" +) + +// Marshal alias of json.Marshal +func Marshal(v interface{}) ([]byte, error) { + return json.Marshal(v) +} + +// NewEncoder alias of json.NewEncoder +func NewEncoder(w io.Writer) *json.Encoder { + return json.NewEncoder(w) +} + +// Unmarshal alias of json.Unmarshal +func Unmarshal(data []byte, val interface{}) error { + return json.Unmarshal(data, val) +} + +// NewDecoder alias of json.NewDecoder +func NewDecoder(r io.Reader) *json.Decoder { + return json.NewDecoder(r) +} + +// RawString returns json string +func RawString(v interface{}) string { + data, _ := Marshal(v) + return string(data) +} + +// Readable print value of json +func Readable(v interface{}) string { + data, err := json.MarshalIndent(v, "", " ") + if err != nil { + return fmt.Sprint("Error: ", err) + } + return string(data) +} diff --git a/blobstore/cli/common/json_test.go b/blobstore/cli/common/json_test.go new file mode 100644 index 000000000..08b63b83c --- /dev/null +++ b/blobstore/cli/common/json_test.go @@ -0,0 +1,54 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package common_test + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/cli/common" +) + +type typeT struct { + Int int `json:"int"` + Bool bool `json:"bool"` + String string `json:"string"` + Slice []float32 `json:"slice"` + Map map[int]string `json:"map"` +} + +func TestCmdCommonJSONBase(t *testing.T) { + value := typeT{ + Int: 10, + Bool: true, + String: "test json", + Slice: []float32{0.02, 1.111}, + Map: map[int]string{-1: "-1", 100: "100"}, + } + + data, err := common.Marshal(value) + require.NoError(t, err) + + var val typeT + err = common.Unmarshal(data, &val) + require.NoError(t, err) + + require.Equal(t, value, val) + + fmt.Println("Raw JSON:", common.RawString(value)) + fmt.Println("Readable JSON:", common.Readable(value)) +} diff --git a/blobstore/cli/common/progress.go b/blobstore/cli/common/progress.go new file mode 100644 index 000000000..a70c8a940 --- /dev/null +++ b/blobstore/cli/common/progress.go @@ -0,0 +1,229 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package common + +import ( + "fmt" + "io" + "strings" + "sync" + + "github.com/fatih/color" +) + +var boldRunes = []rune(" ▏▎▍▌▋▋▊▉█") + +// BoldBar returns bold progress bar like: +// [███▍ ] +// +// curr is [0 - 100] +func BoldBar(curr int) string { + if curr == 100 { + return "██████████" + } + return fmt.Sprintf("%s%*c", strings.Repeat("█", curr/10), curr/10-10, boldRunes[curr%10]) +} + +// LineBar returns line progress bar like: +// [===============================================> ] +// +// curr is [0 - 100] +// width is the length of bar +func LineBar(curr, width int) string { + n := 100 / width + if n <= 0 { + n = 1 + } + + if curr == 100 { + return strings.Repeat("=", 100/n) + } + return fmt.Sprintf("%s>%s", strings.Repeat("=", curr/n), strings.Repeat(" ", (100-curr-1)/n)) +} + +// Loader loader bar +func Loader(all int) chan<- int { + ch := make(chan int) + + go func() { + loaded := 0 + for { + n, ok := <-ch + if !ok { + fmt.Println() + return + } + loaded += n + curr := loaded * 100 / all + fmt.Printf("\rLoading: [%s] %s/%s", BoldBar(curr), + color.GreenString("%-5d", loaded), + color.RedString("%5d", all)) + if loaded >= all { + fmt.Println() + return + } + } + }() + + return ch +} + +// PReader progress bar for io.Reader +// example: +// reader := NewPReader(1024, r) +// defer reader.Close() +// reader.LineBar() +// +// // balabala ... +type PReader struct { + total int + r io.Reader + + readCh chan int + closeCh chan struct{} + once sync.Once +} + +// NewPReader returns a progress reader +func NewPReader(total int, r io.Reader) *PReader { + return &PReader{ + total: total, + r: io.LimitReader(r, int64(total)), + + readCh: make(chan int, 1), + closeCh: make(chan struct{}), + } +} + +func (r *PReader) Read(p []byte) (n int, err error) { + n, err = r.r.Read(p) + if n > 0 { + r.readCh <- n + } + return +} + +func (r *PReader) bar(width int, printFunc func(curr int)) { + go func() { + read := 0 + for { + select { + case n := <-r.readCh: + read += n + curr := read * 100 / r.total + printFunc(curr) + if read >= r.total { + fmt.Println() + return + } + case <-r.closeCh: + fmt.Println() + return + } + } + }() +} + +// LineBar line bar +func (r *PReader) LineBar(width int) { + r.bar(width, func(curr int) { + fmt.Printf("\rReading: [%s] %s%%", LineBar(curr, width), color.GreenString("%3d", curr)) + }) +} + +// BoldBar bold bar +func (r *PReader) BoldBar() { + r.bar(0, func(curr int) { + fmt.Printf("\rReading: [%s] %s%%", BoldBar(curr), color.GreenString("%3d", curr)) + }) +} + +// Close the bar +func (r *PReader) Close() { + r.once.Do(func() { + close(r.closeCh) + }) +} + +// PWriter progress bar for io.Writer +type PWriter struct { + total int + w io.Writer + + writeCh chan int + closeCh chan struct{} + once sync.Once +} + +// NewPWriter returns a progress writer +func NewPWriter(total int, w io.Writer) *PWriter { + return &PWriter{ + total: total, + w: w, + + writeCh: make(chan int, 1), + closeCh: make(chan struct{}), + } +} + +func (w *PWriter) Write(p []byte) (n int, err error) { + n, err = w.w.Write(p) + if n > 0 { + w.writeCh <- n + } + return +} + +func (w *PWriter) bar(width int, printFunc func(curr int)) { + go func() { + written := 0 + for { + select { + case n := <-w.writeCh: + written += n + curr := written * 100 / w.total + printFunc(curr) + if written >= w.total { + fmt.Println() + return + } + case <-w.closeCh: + fmt.Println() + return + } + } + }() +} + +// LineBar line bar +func (w *PWriter) LineBar(width int) { + w.bar(width, func(curr int) { + fmt.Printf("\rWriting: [%s] %s%%", LineBar(curr, width), color.GreenString("%3d", curr)) + }) +} + +// BoldBar bold bar +func (w *PWriter) BoldBar() { + w.bar(0, func(curr int) { + fmt.Printf("\rWriting: [%s] %s%%", BoldBar(curr), color.GreenString("%3d", curr)) + }) +} + +// Close the bar +func (w *PWriter) Close() { + w.once.Do(func() { + close(w.closeCh) + }) +} diff --git a/blobstore/cli/common/progress_test.go b/blobstore/cli/common/progress_test.go new file mode 100644 index 000000000..8d024868b --- /dev/null +++ b/blobstore/cli/common/progress_test.go @@ -0,0 +1,168 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package common_test + +import ( + "fmt" + "math/rand" + "testing" + "time" + + "github.com/cubefs/cubefs/blobstore/cli/common" +) + +func TestCmdCommonProgressBoldBar(t *testing.T) { + for curr := 0; curr <= 100; curr++ { + fmt.Printf("\r[%s] %d%%", common.BoldBar(curr), curr) + time.Sleep(20 * time.Millisecond) + } + fmt.Println() +} + +func TestCmdCommonProgressLineBar(t *testing.T) { + for curr := 0; curr <= 100; curr++ { + fmt.Printf("\r[%s] %d%%", common.LineBar(curr, 10), curr) + time.Sleep(10 * time.Millisecond) + } + fmt.Println() + + for curr := 0; curr <= 100; curr++ { + fmt.Printf("\r[%s] %d%%", common.LineBar(curr, 50), curr) + time.Sleep(10 * time.Millisecond) + } + fmt.Println() + + for curr := 0; curr <= 100; curr++ { + fmt.Printf("\r[%s] %d%%", common.LineBar(curr, 100), curr) + time.Sleep(10 * time.Millisecond) + } + fmt.Println() +} + +func TestCmdCommonProgressLoader(t *testing.T) { + alls := []int{ + 1, + 7, + 17, + 20, + 100, + 19999, + } + + for _, all := range alls { + ch := common.Loader(all) + n := (all + 9) / 10 + loaded := 0 + for ii := 0; ii < 10; ii++ { + if loaded >= all { + break + } + if loaded+n > all { + ch <- (all - loaded) + } else { + ch <- n + } + loaded += n + time.Sleep(100 * time.Millisecond) + } + close(ch) + } +} + +type mockReader struct{} + +func (r *mockReader) Read(p []byte) (n int, err error) { + time.Sleep(10 * time.Millisecond) + return len(p), nil +} + +func TestCmdCommonProgressReader(t *testing.T) { + sizes := []int{ + 1, + 1 << 10, + 10250, + 1 << 20, + 1 << 30, + } + + rand.Seed(time.Now().Unix()) + for _, size := range sizes { + reader := common.NewPReader(size, &mockReader{}) + reader.LineBar((int(rand.Int31n(9)+1) * 10)) + l := (size / 100) + 1 + // 102 times + for ii := 0; ii <= 101; ii++ { + _, _ = reader.Read(make([]byte, l)) + } + reader.Close() + } + + for _, size := range sizes { + reader := common.NewPReader(size, &mockReader{}) + reader.BoldBar() + l := (size / 100) + 1 + for ii := 0; ii <= 101; ii++ { + _, _ = reader.Read(make([]byte, l)) + } + reader.Close() + } +} + +type mockWriter struct { + total int + written int +} + +func (w *mockWriter) Write(p []byte) (n int, err error) { + if w.written >= w.total { + return 0, nil + } + w.written += len(p) + time.Sleep(10 * time.Millisecond) + return len(p), nil +} + +func TestCmdCommonProgressWriter(t *testing.T) { + sizes := []int{ + 1, + 1 << 10, + 10250, + 1 << 20, + 1 << 30, + } + + rand.Seed(time.Now().Unix()) + for _, size := range sizes { + writer := common.NewPWriter(size, &mockWriter{total: size}) + writer.LineBar((int(rand.Int31n(9)+1) * 10)) + l := (size / 100) + 1 + // 102 times + for ii := 0; ii <= 101; ii++ { + _, _ = writer.Write(make([]byte, l)) + } + + writer.Close() + } + + for _, size := range sizes { + writer := common.NewPWriter(size, &mockWriter{total: size}) + writer.BoldBar() + l := (size / 100) + 1 + for ii := 0; ii <= 101; ii++ { + _, _ = writer.Write(make([]byte, l)) + } + writer.Close() + } +} diff --git a/blobstore/cli/config.go b/blobstore/cli/config.go new file mode 100644 index 000000000..bb3c028d9 --- /dev/null +++ b/blobstore/cli/config.go @@ -0,0 +1,99 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cli + +import ( + "fmt" + "sort" + + "github.com/desertbit/grumble" + "github.com/fatih/color" + + "github.com/cubefs/cubefs/blobstore/cli/config" +) + +func cmdConfigGet(c *grumble.Context) error { + fmt.Println("configs:") + + all := config.All() + keys := c.Args.StringList("keys") + if len(keys) == 0 { + keys = make([]string, 0, len(all)) + for key := range all { + keys = append(keys, key) + } + } + sort.Strings(keys) + + for _, key := range keys { + fmt.Printf("\t%s --> %s\n", color.RedString("%-30s", key), color.GreenString("%+v", all[key])) + } + return nil +} + +func registerConfig(app *grumble.App) { + configCommand := &grumble.Command{ + Name: "config", + Help: "manager memory cache of config", + } + app.AddCommand(configCommand) + + configCommand.AddCommand(&grumble.Command{ + Name: "type", + Help: "print type in cache", + Run: func(c *grumble.Context) error { + config.PrintType() + return nil + }, + }) + configCommand.AddCommand(&grumble.Command{ + Name: "get", + Help: "get config in cache", + LongHelp: "show all caches if no args", + Args: func(a *grumble.Args) { + a.StringList("keys", "config keys", grumble.Default([]string{}), grumble.Max(100)) + }, + Run: cmdConfigGet, + }) + configCommand.AddCommand(&grumble.Command{ + Name: "set", + Help: "set config to cache", + LongHelp: "set config to cache, value concat with ',' if []string", + Args: func(a *grumble.Args) { + a.String("key", "config key") + a.String("value", "config value, concat with ',' if []string") + }, + Run: func(c *grumble.Context) error { + key, val := c.Args.String("key"), c.Args.String("value") + fmt.Println("To set Key:", color.RedString("%s", key), "Value:", color.GreenString("%s", val)) + config.SetFrom(key, val) + return nil + }, + }) + configCommand.AddCommand(&grumble.Command{ + Name: "del", + Help: "del config of keys", + Args: func(a *grumble.Args) { + a.StringList("keys", "config keys", grumble.Default([]string{}), grumble.Max(100)) + }, + Run: func(c *grumble.Context) error { + for _, key := range c.Args.StringList("keys") { + fmt.Println("To del Key:", color.RedString("%s", key)) + config.Del(key) + } + return nil + }, + }) +} diff --git a/blobstore/cli/config/cache.go b/blobstore/cli/config/cache.go new file mode 100644 index 000000000..97722fc70 --- /dev/null +++ b/blobstore/cli/config/cache.go @@ -0,0 +1,82 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package config + +var cacher = make(map[string]interface{}) + +// All returns all cache +func All() map[string]interface{} { + return cacher +} + +// Set cache setter +func Set(key string, value interface{}) { + cacher[key] = value +} + +// SetFrom value from string +func SetFrom(key, value string) { + if valuer, ok := keyValuer[key]; ok { + Set(key, valuer.Valuer(value)) + } else { + Set(key, value) + } +} + +// Del del cache of key +func Del(key string) { + delete(cacher, key) +} + +// Get cache getter +func Get(key string) interface{} { + return cacher[key] +} + +// Verbose returns app verbose +func Verbose() bool { + if val := Get("Flag-Verbose"); val != nil { + if v, ok := val.(bool); ok { + return v + } + } + return false +} + +// Vverbose returns app verbose verbose +func Vverbose() bool { + if val := Get("Flag-Vverbose"); val != nil { + if v, ok := val.(bool); ok { + return v + } + } + return false +} + +func RedisAddrs() []string { return Get("Key-RedisAddrs").([]string) } +func RedisUser() string { return Get("Key-RedisUser").(string) } +func RedisPass() string { return Get("Key-RedisPass").(string) } + +// ClusterMgrAddrs returns cluster manager addrs +func ClusterMgrAddrs() []string { return Get("Key-ClusterMgrAddrs").([]string) } +func ClusterMgrSecret() string { return Get("Key-ClusterMgrSecret").(string) } + +func AccessConnMode() uint8 { return Get("Key-Access-ConnMode").(uint8) } +func AccessConsulAddr() string { return Get("Key-Access-ConsulAddr").(string) } +func AccessServiceIntervalMs() int64 { return Get("Key-Access-ServiceIntervalMs").(int64) } +func AccessPriorityAddrs() []string { return Get("Key-Access-PriorityAddrs").([]string) } +func AccessMaxSizePutOnce() int64 { return Get("Key-Access-MaxSizePutOnce").(int64) } +func AccessMaxPartRetry() int { return Get("Key-Access-MaxPartRetry").(int) } +func AccessMaxHostRetry() int { return Get("Key-Access-MaxHostRetry").(int) } diff --git a/blobstore/cli/config/config.go b/blobstore/cli/config/config.go new file mode 100644 index 000000000..41bf527f7 --- /dev/null +++ b/blobstore/cli/config/config.go @@ -0,0 +1,171 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package config + +import ( + "fmt" + "os" + "reflect" + "sort" + "strconv" + "strings" + + "github.com/cubefs/cubefs/blobstore/cli/common" +) + +// Config config in file +type Config struct { + Verbose bool `json:"verbose" cache:"Flag-Verbose" help:"enable verbose mode"` + Vverbose bool `json:"vverbose" cache:"Flag-Vverbose" help:"enable verbose verbose mode"` + + RedisAddrs []string `json:"redis_addrs" cache:"Key-RedisAddrs" help:"redis addrs"` + RedisUser string `json:"redis_user" cache:"Key-RedisUser" help:"redis username"` + RedisPass string `json:"redis_pass" cache:"Key-RedisPass" help:"redis password"` + + ClusterMgrAddrs []string `json:"cm_addrs" cache:"Key-ClusterMgrAddrs" help:"cluster manager addrs"` + ClusterMgrSecret string `json:"cm_secret" cache:"Key-ClusterMgrSecret" help:"cluster manager secret"` + + Access struct { // see more in api/access/client.go + ConnMode uint8 `json:"conn_mode" cache:"Key-Access-ConnMode" help:"connection mode, 4 means no timeout"` + ConsulAddr string `json:"consul_addr" cache:"Key-Access-ConsulAddr" help:"consul address"` + ServiceIntervalMs int64 `json:"service_interval_ms" cache:"Key-Access-ServiceIntervalMs" help:"service interval ms"` + PriorityAddrs []string `json:"priority_addrs" cache:"Key-Access-PriorityAddrs" help:"priority addresses to try"` + MaxSizePutOnce int64 `json:"max_size_put_once" cache:"Key-Access-MaxSizePutOnce" help:"max size put once"` + MaxPartRetry int `json:"max_part_retry" cache:"Key-Access-MaxPartRetry" help:"max times to retry part"` + MaxHostRetry int `json:"max_host_retry" cache:"Key-Access-MaxHostRetry" help:"max times to retry host"` + } `json:"access"` +} + +func load(conf *Config) { + cacheSetter := func(elemT reflect.Type, elemV reflect.Value) { + for idx := 0; idx < elemT.NumField(); idx++ { + field := elemT.Field(idx) + value := elemV.Field(idx) + if key := field.Tag.Get("cache"); key != "" { + keyValuer[key] = typeValuer{Type: field.Type.Kind().String(), Valuer: valuer(field.Type.Kind())} + Set(key, value.Interface()) + } + } + } + + cacheSetter(reflect.TypeOf(conf).Elem(), reflect.ValueOf(conf).Elem()) + + confAccess := &conf.Access + cacheSetter(reflect.TypeOf(confAccess).Elem(), reflect.ValueOf(confAccess).Elem()) +} + +// LoadConfig load config from path +func LoadConfig(path string) { + var conf *Config + f, err := os.Open(path) + if err != nil { + panic(err) + } + if err := common.NewDecoder(f).Decode(&conf); err != nil { + panic(err) + } + load(conf) +} + +type typeValuer struct { + Type string + Valuer func(string) interface{} +} + +var ( + keyValuer = make(map[string]typeValuer) + + valuer = func(t reflect.Kind) func(string) interface{} { + switch t { + case reflect.Bool: + return func(val string) interface{} { v, _ := strconv.ParseBool(val); return v } + + case reflect.Int: + return func(val string) interface{} { v, _ := strconv.Atoi(val); return int(v) } + case reflect.Int8: + return func(val string) interface{} { v, _ := strconv.Atoi(val); return int8(v) } + case reflect.Int16: + return func(val string) interface{} { v, _ := strconv.Atoi(val); return int16(v) } + case reflect.Int32: + return func(val string) interface{} { v, _ := strconv.Atoi(val); return int32(v) } + case reflect.Int64: + return func(val string) interface{} { v, _ := strconv.Atoi(val); return int64(v) } + + case reflect.Uint: + return func(val string) interface{} { v, _ := strconv.Atoi(val); return uint(v) } + case reflect.Uint8: + return func(val string) interface{} { v, _ := strconv.Atoi(val); return uint8(v) } + case reflect.Uint16: + return func(val string) interface{} { v, _ := strconv.Atoi(val); return uint16(v) } + case reflect.Uint32: + return func(val string) interface{} { v, _ := strconv.Atoi(val); return uint32(v) } + case reflect.Uint64: + return func(val string) interface{} { v, _ := strconv.Atoi(val); return uint64(v) } + + case reflect.Float32: + return func(val string) interface{} { v, _ := strconv.ParseFloat(val, 32); return float32(v) } + case reflect.Float64: + return func(val string) interface{} { v, _ := strconv.ParseFloat(val, 64); return float64(v) } + + case reflect.String: + return func(val string) interface{} { return val } + case reflect.Slice: // []string + return func(val string) interface{} { + if val == "" { + return []string{} + } + return strings.Split(val, ",") + } + + case reflect.Map: // map[string]string + return func(val string) interface{} { + m := make(map[string]string) + if val == "" { + return m + } + kvs := strings.Split(val, ",") + for _, kvStr := range kvs { + kv := strings.SplitN(kvStr, ":", 2) + k, v := strings.TrimSpace(kv[0]), "" + if len(kv) > 1 { + v = strings.TrimSpace(kv[1]) + } + m[k] = v + } + return m + } + default: + panic(fmt.Sprintf("unknown type %v", t)) + } + } +) + +// PrintType print type of keyValuer +func PrintType() { + keys := make([]string, 0, len(keyValuer)) + for key := range keyValuer { + keys = append(keys, key) + } + sort.Strings(keys) + fmt.Println("Init Cache Keys:") + for _, key := range keys { + fmt.Printf("\t| %-30s | %-10s |\n", key, keyValuer[key].Type) + } +} + +func init() { + conf := &Config{} + load(conf) +} diff --git a/blobstore/cli/history.go b/blobstore/cli/history.go new file mode 100644 index 000000000..a923a121b --- /dev/null +++ b/blobstore/cli/history.go @@ -0,0 +1,75 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cli + +import ( + "bufio" + "fmt" + "io" + "os" + + "github.com/desertbit/grumble" + "github.com/fatih/color" +) + +func cmdHistory(path string) func(c *grumble.Context) error { + hisColor := color.New(color.FgHiMagenta) + return func(c *grumble.Context) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + l := c.Args.Int("line") + lines := make([]string, 0, l) + + reader := bufio.NewReader(file) + n := 1 + for { + line, _, err := reader.ReadLine() + if err == io.EOF { + break + } + if err != nil { + return err + } + + lines = append(lines, fmt.Sprintf("%5.d: %s", n, line)) + n++ + } + if len(lines) > l { + lines = lines[len(lines)-l:] + } + + for _, line := range lines { + hisColor.Println(line) + } + return nil + } +} + +func registerHistory(app *grumble.App) { + historyCommand := &grumble.Command{ + Name: "history", + Help: "show history commands", + LongHelp: "show history commands in file " + app.Config().HistoryFile, + Args: func(a *grumble.Args) { + a.Int("line", "show lines at the tail", grumble.Default(40)) + }, + Run: cmdHistory(app.Config().HistoryFile), + } + app.AddCommand(historyCommand) +} diff --git a/blobstore/cli/util.go b/blobstore/cli/util.go new file mode 100644 index 000000000..3f25b2d67 --- /dev/null +++ b/blobstore/cli/util.go @@ -0,0 +1,269 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cli + +import ( + "encoding/binary" + "encoding/hex" + "fmt" + "strconv" + "strings" + "time" + + "github.com/desertbit/grumble" + "github.com/dustin/go-humanize" + "github.com/fatih/color" + + "github.com/cubefs/cubefs/blobstore/access/controller" + "github.com/cubefs/cubefs/blobstore/api/access" + "github.com/cubefs/cubefs/blobstore/cli/common" + "github.com/cubefs/cubefs/blobstore/cli/common/args" + "github.com/cubefs/cubefs/blobstore/cli/common/cfmt" + "github.com/cubefs/cubefs/blobstore/cli/config" + "github.com/cubefs/cubefs/blobstore/common/redis" + "github.com/cubefs/cubefs/blobstore/common/uptoken" +) + +func newRedisCli(addrs ...string) *redis.ClusterClient { + if len(addrs) == 0 || (len(addrs) == 1 && addrs[0] == "") { + addrs = config.RedisAddrs() + } + return redis.NewClusterClient(&redis.ClusterConfig{ + Addrs: addrs, + Username: config.RedisUser(), + Password: config.RedisPass(), + }) +} + +func cmdTime(c *grumble.Context) error { + unix := c.Args.String("unix") + format := c.Args.String("format") + + if unix == "" { + t := time.Now() + fmt.Printf("timestamp = %s (seconds = %d nanosecs = %d) \n\t--> format: %s (%s)\n\n", + color.RedString("%d", t.UnixNano()), t.Unix(), t.Nanosecond(), + color.GreenString("%s", t.Format(time.RFC3339Nano)), humanize.Time(t)) + return nil + } + + if format == "" { + format = time.RFC3339Nano + } + + unix += strings.Repeat("0", 19) + sec, _ := strconv.ParseInt(unix[:10], 10, 64) + nsec, _ := strconv.ParseInt(unix[10:19], 10, 64) + + t := time.Unix(sec, nsec) + fmt.Printf("timestamp = %s (seconds = %d nanosecs = %d) \n\t--> format: %s (%s)\n\n", + color.RedString("%s", c.Args.String("unix")), sec, nsec, + color.GreenString("%s", t.Format(format)), humanize.Time(t)) + return nil +} + +func cmdToken(c *grumble.Context) error { + tokenStr := c.Args.String("token") + token := uptoken.DecodeToken(tokenStr) + err := fmt.Errorf("invalid token: %s", tokenStr) + + data := token.Data[:] + offset := 8 + next := func() (uint64, bool) { + val, n := binary.Uvarint(data[offset:]) + if n <= 0 { + return 0, false + } + offset += n + return val, true + } + + var ( + minBid, count, expiredTime uint64 + ok bool + ) + if minBid, ok = next(); !ok { + return err + } + if count, ok = next(); !ok { + return err + } + if expiredTime, ok = next(); !ok { + return err + } + + fmt.Println("Checksum: ", strings.ToUpper(hex.EncodeToString(data[0:8]))) + fmt.Println("MinBid : ", minBid) + fmt.Println("Count : ", count) + + t := time.Unix(int64(expiredTime), 9) + if time.Since(t) < 0 { + fmt.Printf("Time : %d (%s) (%s)", expiredTime, + common.Normal.Sprint(t.Format(time.RFC3339)), humanize.Time(t)) + } else { + fmt.Printf("Time : %d (%s) (%s)", expiredTime, + common.Danger.Sprint(t.Format(time.RFC3339)), humanize.Time(t)) + } + fmt.Println() + + return nil +} + +func registerUtil(app *grumble.App) { + utilCommand := &grumble.Command{ + Name: "util", + Help: "util commands", + LongHelp: "util commands, parse everything", + } + app.AddCommand(utilCommand) + + utilCommand.AddCommand(&grumble.Command{ + Name: "time", + Help: "time format [unix] [format]", + LongHelp: "time format, show now if no argument", + Run: cmdTime, + Args: func(a *grumble.Args) { + a.String("unix", "unix timestamp", grumble.Default("")) + a.String("format", "format for timestamp", grumble.Default("")) + }, + }) + utilCommand.AddCommand(&grumble.Command{ + Name: "vuid", + Help: "parse vuid ", + Args: func(a *grumble.Args) { + args.VuidRegister(a) + }, + Run: func(c *grumble.Context) error { + fmt.Println("Parse VUID: ", cfmt.VuidCF(args.Vuid(c.Args))) + return nil + }, + }) + utilCommand.AddCommand(&grumble.Command{ + Name: "token", + Help: "parse token ", + Run: cmdToken, + Args: func(a *grumble.Args) { + a.String("token", "token of putat") + }, + }) + + utilCommand.AddCommand(&grumble.Command{ + Name: "location", + Help: "parse location <[json|hex|base64]>", + LongHelp: "parse location, or decode from string", + Args: func(a *grumble.Args) { + a.String("jsonORstr", "location json or location string") + }, + Run: func(c *grumble.Context) error { + jsonORstr := c.Args.String("jsonORstr") + loc, err := cfmt.ParseLocation(jsonORstr) + if err == nil { + fmt.Println(cfmt.LocationJoin(&loc, "")) + return nil + } + + src, err := hex.DecodeString(jsonORstr) + if err != nil { + return fmt.Errorf("invalid (%s) %s", jsonORstr, err.Error()) + } + loc, n, err := access.DecodeLocation(src) + if err != nil { + fmt.Printf("has read bytes %d / %d\n", n, len(src)) + fmt.Println(cfmt.LocationJoin(&loc, "")) + return fmt.Errorf("invalid (%s) %s", jsonORstr, err.Error()) + } + + fmt.Println(cfmt.LocationJoin(&loc, "")) + return nil + }, + }) + + redisCommand := &grumble.Command{ + Name: "redis", + Help: "redis tools", + Run: func(c *grumble.Context) error { + fmt.Println("redis-addrs:", config.RedisAddrs()) + fmt.Println("redis-user :", config.RedisUser()) + fmt.Println("redis-pass :", config.RedisPass()) + fmt.Println() + fmt.Println("access volume prefix with access/volume/{cid}/{vid}") + return nil + }, + } + utilCommand.AddCommand(redisCommand) + redisCommand.AddCommand(&grumble.Command{ + Name: "get", + Help: "redis get ", + LongHelp: "redis get, access prefix access/volume/", + Args: func(a *grumble.Args) { + a.String("key", "redis key") + }, + Flags: func(f *grumble.Flags) { + f.StringL("addr", "", "redis addr") + }, + Run: func(c *grumble.Context) error { + key := c.Args.String("key") + cli := newRedisCli(c.Flags.String("addr")) + + fmt.Println("Get Key:", key) + if strings.HasPrefix(key, "access/volume/") { + var val controller.VolumePhy + if err := cli.Get(common.CmdContext(), key, &val); err != nil { + return err + } + fmt.Println(common.Readable(val)) + return nil + } + + b, err := cli.ClusterClient.Get(common.CmdContext(), key).Bytes() + if err != nil { + return err + } + fmt.Println(string(b)) + return nil + }, + }) + redisCommand.AddCommand(&grumble.Command{ + Name: "set", + Help: "redis set [value] [expiration]", + LongHelp: "redis set key, delete key if value is empty, expiration is duration string", + Args: func(a *grumble.Args) { + a.String("key", "redis key") + a.String("value", "redis value", grumble.Default("")) + a.String("expiration", "redis key expiration, duration string", + grumble.Default("0")) + }, + Flags: func(f *grumble.Flags) { + f.StringL("addr", "", "redis addr") + }, + Run: func(c *grumble.Context) error { + key := c.Args.String("key") + value := c.Args.String("value") + cli := newRedisCli(c.Flags.String("addr")) + + if value == "" { + fmt.Println("Del Key:", common.Loaded.Sprint(key)) + return cli.ClusterClient.Del(common.CmdContext(), key).Err() + } + expiration, err := time.ParseDuration(c.Args.String("expiration")) + if err != nil { + return err + } + + fmt.Println("Set Key:", common.Loaded.Sprint(key), "expiration:", expiration) + return cli.Set(common.CmdContext(), key, value, expiration) + }, + }) +} diff --git a/blobstore/cmd/access/access.conf b/blobstore/cmd/access/access.conf new file mode 100644 index 000000000..e4103306a --- /dev/null +++ b/blobstore/cmd/access/access.conf @@ -0,0 +1,21 @@ +{ + "bind_addr": ":9500", + "auditlog": { + "logdir": "/tmp/access/" + }, + "log": { + "level": "info", + "filename": "/tmp/logs/access.log" + }, + "consul_agent_addr": "127.0.0.1:8500", + "service_register": { + "consul_addr": "127.0.0.1:8500", + "service_ip": "127.0.0.1" + }, + "stream": { + "idc": "z0", + "cluster_config": { + "region": "test-region" + } + } +} diff --git a/blobstore/cmd/access/access.conf.default b/blobstore/cmd/access/access.conf.default new file mode 100644 index 000000000..7ea74932d --- /dev/null +++ b/blobstore/cmd/access/access.conf.default @@ -0,0 +1,112 @@ +{ + "max_procs": 0, + "shutdown_timeout_s": 30, + "log": { + "level": "debug", + "filename": "/tmp/access.log", + "maxsize": 1024, + "maxage": 7, + "maxbackups": 7 + }, + "auditlog": { + "chunkbits": 29, + "logdir": "/tmp/access/", + "backup": 0, + "rotate_new": false + }, + "bind_addr": ":9500", + "consul_agent_addr": "127.0.0.1:8500", + "service_register": { + "consul_addr": "127.0.0.1:8500", + "node": "nodename", + "service_ip": "service_ip" + }, + "limit": { + "name_rps": { + "alloc": 0, + "put": 0, + "putat": 0, + "get": 0, + "delete": 0, + "sign": 0 + }, + "reader_mbps": 0, + "writer_mbps": 0 + }, + "stream": { + "idc": "idc", + "max_blob_size": 4194304, + "mem_pool_size_classes": { + "2048": 81920, + "24576": 40960, + "65536": 40960, + "524288": 20480, + "2097152": 10240, + "8389632": 4096, + "16777216": 1024, + "33554432": 512, + "67108864": 64 + }, + "code_mode_put_quorums": { + "1": 24, + "2": 11, + "3": 34, + "4": 14 + }, + "alloc_retry_interval_ms": 100, + "alloc_retry_times": 3, + "encoder_concurrency": 1000, + "encoder_enableverify": true, + "min_read_shards_x": 1, + "shard_crc_disabled": false, + "cluster_config": { + "region": "region", + "region_magic": "", + "cluster_reload_secs": 3, + "clustermgr_client_config": { + "client_timeout_ms": 3000, + "hosts": [], + "transport_config": { + "auth": { + "enable_auth": true, + "secret": "secret key" + }, + "dial_timeout_ms": 2000 + } + }, + "redis_client_config": { + "addrs": [ + "127.0.0.1:xxx" + ], + "password": "pass", + "username": "user" + }, + "service_punish_threshold": 3, + "service_punish_valid_interval_s": 30, + "service_reload_secs": 3 + }, + "disk_punish_interval_s": 60, + "disk_timeout_punish_interval_s": 6, + "service_punish_interval_s": 60, + "blobnode_config": { + "client_timeout_ms": 10000 + }, + "proxy_config": { + "client_timeout_ms": 5000 + }, + "alloc_command_config": { + "error_percent_threshold": 50, + "max_concurrent_requests": 10240, + "request_volume_threshold": 100, + "sleep_window": 2000, + "timeout": 30000 + }, + "rw_command_config": { + "error_percent_threshold": 80, + "max_concurrent_requests": 102400, + "request_volume_threshold": 1000, + "sleep_window": 5000, + "timeout": 600000 + } + } +} diff --git a/blobstore/cmd/access/main.go b/blobstore/cmd/access/main.go new file mode 100644 index 000000000..c18c8bd0c --- /dev/null +++ b/blobstore/cmd/access/main.go @@ -0,0 +1,27 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package main + +import ( + "os" + + "github.com/cubefs/cubefs/blobstore/cmd" + + _ "github.com/cubefs/cubefs/blobstore/access" +) + +func main() { + cmd.Main(os.Args) +} diff --git a/blobstore/common/codemode/codemode.go b/blobstore/common/codemode/codemode.go new file mode 100644 index 000000000..889400b1a --- /dev/null +++ b/blobstore/common/codemode/codemode.go @@ -0,0 +1,337 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package codemode + +import "fmt" + +// CodeMode EC encode and decode mode +type CodeMode uint8 +type CodeModeName string + +// pre-defined mode +const ( + EC15P12 CodeMode = 1 + EC6P6 CodeMode = 2 + EC16P20L2 CodeMode = 3 + EC6P10L2 CodeMode = 4 + EC6P3L3 CodeMode = 5 + EC6P6Align0 CodeMode = 6 + EC6P6Align512 CodeMode = 7 + EC4P4L2 CodeMode = 8 + EC12P4 CodeMode = 9 + EC16P4 CodeMode = 10 + EC3P3 CodeMode = 11 + EC10P4 CodeMode = 12 + EC6P3 CodeMode = 13 +) + +// Note: Don't modify it unless you know very well how codemode works. +const ( + // align size per shard + alignSize0B = 0 // 0B + alignSize512B = 512 // 512B + alignSize2KB = 2048 // 2KB +) + +// The tactic is fixed pairing with one codemode. +// Add a new codemode if you want other features. +var constCodeModeTactic = map[CodeMode]Tactic{ + EC15P12: {N: 15, M: 12, L: 0, AZCount: 3, PutQuorum: 24, GetQuorum: 0, MinShardSize: alignSize2KB}, + EC6P6: {N: 6, M: 6, L: 0, AZCount: 3, PutQuorum: 11, GetQuorum: 0, MinShardSize: alignSize2KB}, + EC16P20L2: {N: 16, M: 20, L: 2, AZCount: 2, PutQuorum: 34, GetQuorum: 0, MinShardSize: alignSize2KB}, + EC6P10L2: {N: 6, M: 10, L: 2, AZCount: 2, PutQuorum: 14, GetQuorum: 0, MinShardSize: alignSize2KB}, + + // single az + EC12P4: {N: 12, M: 4, L: 0, AZCount: 1, PutQuorum: 15, GetQuorum: 0, MinShardSize: alignSize2KB}, + EC16P4: {N: 16, M: 4, L: 0, AZCount: 1, PutQuorum: 19, GetQuorum: 0, MinShardSize: alignSize2KB}, + EC3P3: {N: 3, M: 3, L: 0, AZCount: 1, PutQuorum: 5, GetQuorum: 0, MinShardSize: alignSize2KB}, + EC10P4: {N: 10, M: 4, L: 0, AZCount: 1, PutQuorum: 13, GetQuorum: 0, MinShardSize: alignSize2KB}, + EC6P3: {N: 6, M: 3, L: 0, AZCount: 1, PutQuorum: 8, GetQuorum: 0, MinShardSize: alignSize2KB}, + // for env test + EC6P3L3: {N: 6, M: 3, L: 3, AZCount: 3, PutQuorum: 9, GetQuorum: 0, MinShardSize: alignSize2KB}, + EC6P6Align0: {N: 6, M: 6, L: 0, AZCount: 3, PutQuorum: 11, GetQuorum: 0, MinShardSize: alignSize0B}, + EC6P6Align512: {N: 6, M: 6, L: 0, AZCount: 3, PutQuorum: 11, GetQuorum: 0, MinShardSize: alignSize512B}, + EC4P4L2: {N: 4, M: 4, L: 2, AZCount: 2, PutQuorum: 6, GetQuorum: 0, MinShardSize: alignSize2KB}, +} + +var constName2CodeMode = map[CodeModeName]CodeMode{ + "EC15P12": EC15P12, + "EC6P6": EC6P6, + "EC16P20L2": EC16P20L2, + "EC6P10L2": EC6P10L2, + "EC6P3L3": EC6P3L3, + "EC6P6Align0": EC6P6Align0, + "EC6P6Align512": EC6P6Align512, + "EC4P4L2": EC4P4L2, + "EC12P4": EC12P4, + "EC16P4": EC16P4, + "EC3P3": EC3P3, + "EC10P4": EC10P4, + "EC6P3": EC6P3, +} + +var constCodeMode2Name = map[CodeMode]CodeModeName{ + EC15P12: "EC15P12", + EC6P6: "EC6P6", + EC16P20L2: "EC16P20L2", + EC6P10L2: "EC6P10L2", + EC6P3L3: "EC6P3L3", + EC6P6Align0: "EC6P6Align0", + EC6P6Align512: "EC6P6Align512", + EC4P4L2: "EC4P4L2", + EC12P4: "EC12P4", + EC16P4: "EC16P4", + EC3P3: "EC3P3", + EC10P4: "EC10P4", + EC6P3: "EC6P3", +} + +//vol layout ep:EC6P10L2 +//|----N------|--------M----------------|--L--| +//|0,1,2,3,4,5|6,7,8,9,10,11,12,13,14,15|16,17| + +// global stripe:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15], n=6 m=10 +// two local stripes: +// local stripe1:[0,1,2, 6, 7, 8, 9,10, 16] n=8 m=1 +// local stripe2:[3,4,5, 11,12,13,14,15, 17] n=8 m=1 + +// Tactic constant strategy of one CodeMode +type Tactic struct { + N int + M int + // local parity count + L int + // the count of AZ, access use this for split data shards and parity shards + AZCount int + + // PutQuorum write quorum, + // MUST make sure that ec data is recoverable if one AZ was down + // We SHOULD ignore the local shards + // (N + M) / AZCount + N <= PutQuorum <= M + N + PutQuorum int + + // get quorum config + GetQuorum int + + // MinShardSize min size per shard, fill data into shards 0-N continuously, + // align with zero bytes if data size less than MinShardSize*N + // + // length of data less than MinShardSize*N, size of per shard = MinShardSize + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // | data | align zero bytes | + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // | 0 | 1 | 2 | .... | N | + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // + // length of data more than MinShardSize*N, size of per shard = ceil(len(data)/N) + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // | data |padding| + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // | 0 | 1 | 2 | .... | N | + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + MinShardSize int +} + +func init() { + // assert all codemode + for _, pair := range []struct { + Mode CodeMode + Size int + }{ + {Mode: EC15P12, Size: alignSize2KB}, + {Mode: EC6P6, Size: alignSize2KB}, + {Mode: EC16P20L2, Size: alignSize2KB}, + {Mode: EC6P10L2, Size: alignSize2KB}, + + {Mode: EC6P3L3, Size: alignSize2KB}, + {Mode: EC6P6Align0, Size: alignSize0B}, + {Mode: EC6P6Align512, Size: alignSize512B}, + } { + tactic := pair.Mode.Tactic() + if !tactic.IsValid() { + panic(fmt.Sprintf("Invalid codemode:%d Tactic:%+v", pair.Mode, tactic)) + } + + min := tactic.N + (tactic.N+tactic.M)/tactic.AZCount + max := tactic.N + tactic.M + if tactic.PutQuorum < min || tactic.PutQuorum > max { + panic(fmt.Sprintf("Invalid codemode:%d PutQuorum:%d([%d,%d])", pair.Mode, + tactic.PutQuorum, min, max)) + } + + if tactic.MinShardSize != pair.Size { + panic(fmt.Sprintf("Invalid codemode:%d MinShardSize:%d(%d)", pair.Mode, + tactic.MinShardSize, pair.Size)) + } + } +} + +// Tactic returns its constant tactic +func (c CodeMode) Tactic() Tactic { + if tactic, ok := constCodeModeTactic[c]; ok { + return tactic + } + panic(fmt.Sprintf("Invalid codemode:%d", c)) +} + +func (c CodeMode) GetShardNum() int { + tactic := c.Tactic() + return tactic.L + tactic.M + tactic.N +} + +// Name turn the CodeMode to CodeModeName +func (c CodeMode) Name() CodeModeName { + if name, ok := constCodeMode2Name[c]; ok { + return name + } + panic(fmt.Sprintf("codemode: %d is invalid", c)) +} + +// String turn the CodeMode to string +func (c CodeMode) String() string { + if name, ok := constCodeMode2Name[c]; ok { + return string(name) + } + return "" +} + +// IsValid check the CodeMode is valid +func (c CodeMode) IsValid() bool { + if _, ok := constCodeMode2Name[c]; ok { + return ok + } + return false +} + +// GetCodeMode get the code mode by name +func (cn CodeModeName) GetCodeMode() CodeMode { + if code, ok := constName2CodeMode[cn]; ok { + return code + } + panic(fmt.Sprintf("codemode: %s is invalid", cn)) +} + +// IsValid check the CodeMode is valid by Name +func (cn CodeModeName) IsValid() bool { + if _, ok := constName2CodeMode[cn]; ok { + return ok + } + return false +} + +// Tactic get tactic by code mode name +func (cn CodeModeName) Tactic() Tactic { + return cn.GetCodeMode().Tactic() +} + +// IsValid ec tactic valid or not +func (c *Tactic) IsValid() bool { + return c.N > 0 && c.M > 0 && c.L >= 0 && c.AZCount > 0 && + c.PutQuorum > 0 && c.GetQuorum >= 0 && c.MinShardSize >= 0 && + c.N%c.AZCount == 0 && c.M%c.AZCount == 0 && c.L%c.AZCount == 0 +} + +// GetECLayoutByAZ ec layout by AZ +func (c *Tactic) GetECLayoutByAZ() (azStripes [][]int) { + azStripes = make([][]int, c.AZCount) + n, m, l := c.N/c.AZCount, c.M/c.AZCount, c.L/c.AZCount + for idx := range azStripes { + stripe := make([]int, 0, n+m+l) + for i := 0; i < n; i++ { + stripe = append(stripe, idx*n+i) + } + for i := 0; i < m; i++ { + stripe = append(stripe, c.N+idx*m+i) + } + for i := 0; i < l; i++ { + stripe = append(stripe, c.N+c.M+idx*l+i) + } + azStripes[idx] = stripe + } + return azStripes +} + +// GlobalStripe returns initial stripe return name.GetCodeMode().Tactic() +func (c *Tactic) GlobalStripe() (indexes []int, n, m int) { + indexes = make([]int, c.N+c.M) + for i := 0; i < c.N+c.M; i++ { + indexes[i] = i + } + return indexes, c.N, c.M +} + +// AllLocalStripe returns all local stripes +func (c *Tactic) AllLocalStripe() (stripes [][]int, n, m int) { + if c.L == 0 { + return + } + + n, m, l := c.N/c.AZCount, c.M/c.AZCount, c.L/c.AZCount + return c.GetECLayoutByAZ(), n + m, l +} + +// LocalStripe get local stripe by index +func (c *Tactic) LocalStripe(index int) (localStripe []int, n, m int) { + if c.L == 0 { + return nil, 0, 0 + } + + n, m, l := c.N/c.AZCount, c.M/c.AZCount, c.L/c.AZCount + var azIdx int + if index < c.N { + azIdx = index / n + } else if index < c.N+c.M { + azIdx = (index - c.N) / m + } else if index < c.N+c.M+c.L { + azIdx = (index - c.N - c.M) / l + } else { + return nil, 0, 0 + } + + return c.LocalStripeInAZ(azIdx) +} + +// LocalStripeInAZ get local stripe in az index +func (c *Tactic) LocalStripeInAZ(azIndex int) (localStripe []int, n, m int) { + if c.L == 0 { + return nil, 0, 0 + } + + n, m, l := c.N/c.AZCount, c.M/c.AZCount, c.L/c.AZCount + azStripes := c.GetECLayoutByAZ() + if azIndex < 0 || azIndex >= len(azStripes) { + return nil, 0, 0 + } + return azStripes[azIndex][:], n + m, l +} + +// GetAllCodeModes get all the available CodeModes +func GetAllCodeModes() []CodeMode { + return []CodeMode{ + EC15P12, + EC6P6, + EC16P20L2, + EC6P10L2, + EC6P3L3, + EC6P6Align0, + EC6P6Align512, + EC4P4L2, + EC12P4, + EC16P4, + EC3P3, + EC10P4, + EC6P3, + } +} diff --git a/blobstore/common/codemode/codemode_test.go b/blobstore/common/codemode/codemode_test.go new file mode 100644 index 000000000..e73312d58 --- /dev/null +++ b/blobstore/common/codemode/codemode_test.go @@ -0,0 +1,240 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package codemode + +import ( + "math" + "testing" + + "github.com/stretchr/testify/assert" +) + +var ( + ec6P10L2Stripes = [][]int{ + {0, 1, 2, 6, 7, 8, 9, 10, 16}, + {3, 4, 5, 11, 12, 13, 14, 15, 17}, + } + ec16P20L2Stripes = [][]int{ + {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 36}, + {8, 9, 10, 11, 12, 13, 14, 15, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37}, + } +) + +func TestCodeModeBase(t *testing.T) { + codeMode1 := EC15P12.Tactic() + assert.Equal(t, codeMode1.MinShardSize, 2048) + indexes, n, m := (&codeMode1).GlobalStripe() + assert.Equal(t, codeMode1.N, n) + assert.Equal(t, codeMode1.M, m) + assert.True(t, codeMode1.IsValid()) + expectedIndex := make([]int, 0) + for i := 0; i < codeMode1.N+codeMode1.M; i++ { + expectedIndex = append(expectedIndex, i) + } + assert.Equal(t, expectedIndex, indexes) + + codeMode2 := EC6P10L2.Tactic() + assert.Equal(t, codeMode2.MinShardSize, 2048) + indexes, n, m = (&codeMode2).GlobalStripe() + assert.Equal(t, codeMode2.N, n) + assert.Equal(t, codeMode2.M, m) + assert.True(t, codeMode2.IsValid()) + expectedIndex = make([]int, 0) + for i := 0; i < codeMode2.N+codeMode2.M; i++ { + expectedIndex = append(expectedIndex, i) + } + assert.Equal(t, expectedIndex, indexes) +} + +func TestCodeModeGetTactic(t *testing.T) { + cases := []struct { + mode CodeMode + isPanic bool + }{ + {0, true}, + {1, false}, + {4, false}, + {(1 << 8) - 1, true}, + {math.MaxInt8, true}, + } + + for _, cs := range cases { + if cs.isPanic { + assert.Panics(t, func() { cs.mode.Tactic() }) + } else { + assert.NotPanics(t, func() { cs.mode.Tactic() }) + } + } +} + +func TestGetLayoutByAZ(t *testing.T) { + codeMode1 := EC15P12.Tactic() + indexes := (&codeMode1).GetECLayoutByAZ() + assert.Equal(t, 3, len(indexes)) + + for i := range indexes { + assert.Equal(t, 9, len(indexes[i])) + } + + codeMode2 := EC6P10L2.Tactic() + indexes = (&codeMode2).GetECLayoutByAZ() + assert.Equal(t, 2, len(indexes)) + + assert.Equal(t, ec6P10L2Stripes[0], indexes[0]) + assert.Equal(t, ec6P10L2Stripes[1], indexes[1]) + + { + codeMode := EC12P4.Tactic() + indexes := codeMode.GetECLayoutByAZ() + assert.Equal(t, 1, len(indexes)) + + for i := range indexes { + assert.Equal(t, codeMode.N+codeMode.M+codeMode.L, len(indexes[i])) + } + } +} + +func TestGlobalStripe(t *testing.T) { + cases := []struct { + mode CodeMode + n int + }{ + {EC15P12, 27}, + {EC6P6, 12}, + {EC16P20L2, 36}, + {EC6P10L2, 16}, + {EC12P4, 16}, + {EC16P4, 20}, + } + for _, cs := range cases { + tactic := cs.mode.Tactic() + stripe, n, m := tactic.GlobalStripe() + assert.Equal(t, cs.n, len(stripe)) + assert.Equal(t, tactic.N, n) + assert.Equal(t, tactic.M, m) + } +} + +func TestAllLocalStripe(t *testing.T) { + cases := []struct { + mode CodeMode + stripes [][]int + n int + m int + }{ + {EC6P6, nil, 0, 0}, + {EC6P10L2, ec6P10L2Stripes, 8, 1}, + {EC16P20L2, ec16P20L2Stripes, 18, 1}, + } + for _, cs := range cases { + tactic := cs.mode.Tactic() + stripes, n, m := tactic.AllLocalStripe() + assert.Equal(t, cs.stripes, stripes) + assert.Equal(t, cs.n, n) + assert.Equal(t, cs.m, m) + } +} + +func TestLocalStripe(t *testing.T) { + cases := []struct { + mode CodeMode + index int + stripe []int + n int + m int + }{ + {EC6P6, 0, nil, 0, 0}, + {EC6P6, 1, nil, 0, 0}, + {EC6P6, 4, nil, 0, 0}, + {EC6P6, 100, nil, 0, 0}, + + {EC6P10L2, 0, ec6P10L2Stripes[0], 8, 1}, + {EC6P10L2, 1, ec6P10L2Stripes[0], 8, 1}, + {EC6P10L2, 16, ec6P10L2Stripes[0], 8, 1}, + {EC6P10L2, 3, ec6P10L2Stripes[1], 8, 1}, + {EC6P10L2, 11, ec6P10L2Stripes[1], 8, 1}, + {EC6P10L2, 17, ec6P10L2Stripes[1], 8, 1}, + {EC6P10L2, 18, nil, 0, 0}, + + {EC16P20L2, 0, ec16P20L2Stripes[0], 18, 1}, + {EC16P20L2, 18, ec16P20L2Stripes[0], 18, 1}, + {EC16P20L2, 36, ec16P20L2Stripes[0], 18, 1}, + {EC16P20L2, 8, ec16P20L2Stripes[1], 18, 1}, + {EC16P20L2, 35, ec16P20L2Stripes[1], 18, 1}, + {EC16P20L2, 37, ec16P20L2Stripes[1], 18, 1}, + {EC16P20L2, 38, nil, 0, 0}, + } + for _, cs := range cases { + tactic := cs.mode.Tactic() + stripe, n, m := tactic.LocalStripe(cs.index) + assert.Equal(t, cs.stripe, stripe) + assert.Equal(t, cs.n, n) + assert.Equal(t, cs.m, m) + } +} + +func TestLocalStripeInAZ(t *testing.T) { + cases := []struct { + mode CodeMode + azIndex int + stripe []int + n int + m int + }{ + {EC6P6, 0, nil, 0, 0}, + {EC6P6, 1, nil, 0, 0}, + {EC6P6, 4, nil, 0, 0}, + {EC6P6, 100, nil, 0, 0}, + + {EC6P10L2, 0, ec6P10L2Stripes[0], 8, 1}, + {EC6P10L2, 1, ec6P10L2Stripes[1], 8, 1}, + {EC6P10L2, 2, nil, 0, 0}, + + {EC16P20L2, 0, ec16P20L2Stripes[0], 18, 1}, + {EC16P20L2, 1, ec16P20L2Stripes[1], 18, 1}, + {EC16P20L2, 2, nil, 0, 0}, + } + for _, cs := range cases { + tactic := cs.mode.Tactic() + stripe, n, m := tactic.LocalStripeInAZ(cs.azIndex) + assert.Equal(t, cs.stripe, stripe) + assert.Equal(t, cs.n, n) + assert.Equal(t, cs.m, m) + } +} + +func BenchmarkGlobalStripe(b *testing.B) { + t := EC16P20L2.Tactic() + tactic := &t + for ii := 0; ii < b.N; ii++ { + tactic.GlobalStripe() + } +} + +func BenchmarkGetECLayoutByAZ(b *testing.B) { + t := EC16P20L2.Tactic() + tactic := &t + for ii := 0; ii < b.N; ii++ { + tactic.GetECLayoutByAZ() + } +} + +func BenchmarkLocalStripe(b *testing.B) { + t := EC16P20L2.Tactic() + tactic := &t + for ii := 0; ii < b.N; ii++ { + tactic.LocalStripe(37) + } +} diff --git a/blobstore/common/codemode/policy.go b/blobstore/common/codemode/policy.go new file mode 100644 index 000000000..c8c3cf4b7 --- /dev/null +++ b/blobstore/common/codemode/policy.go @@ -0,0 +1,30 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package codemode + +// Policy will be used to adjust code mode's upload range or code mode's volume ratio and so on +type Policy struct { + ModeName CodeModeName `json:"mode_name"` + // min size of object, access use this to put object + MinSize int64 `json:"min_size"` + // max size of object. access use this to put object + MaxSize int64 `json:"max_size"` + // code mode's cluster space size ratio, clusterMgr should use this to create specified num of volume + SizeRatio float64 `json:"size_ratio"` + // enable means this kind of code mode enable or not + // access/allocator will ignore this kind of code mode's allocation when enable is false + // clustermgr will ignore this kind of code mode's creation when enable is false + Enable bool `json:"enable"` +} diff --git a/blobstore/common/counter/counter.go b/blobstore/common/counter/counter.go new file mode 100644 index 000000000..dad401345 --- /dev/null +++ b/blobstore/common/counter/counter.go @@ -0,0 +1,72 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package counter + +import ( + "sync/atomic" + + "github.com/benbjohnson/clock" +) + +const ( + // SLOT counter resulting stored. + SLOT int64 = 20 + + // interval default is one minute. + interval int64 = 60 +) + +// time clock just for testing mock. +var time = clock.New() + +// Counter resulting count in default interval period. +// It is thread safe. +type Counter struct { + counts [SLOT]int64 + timestamps [SLOT]int64 +} + +// Add counts once in now. +func (c *Counter) Add() { + c.AddN(1) +} + +// AddN adds n in now. +func (c *Counter) AddN(n int) { + now := time.Now().Unix() / interval + index := now % SLOT + + if ts := atomic.LoadInt64(&c.timestamps[index]); ts == now { + atomic.AddInt64(&c.counts[index], int64(n)) + } else { + atomic.StoreInt64(&c.counts[index], int64(n)) + atomic.StoreInt64(&c.timestamps[index], now) + } +} + +// Show returns result from oldest to newest. +func (c *Counter) Show() (counts [SLOT]int) { + now := time.Now().Unix() / interval + index := now % SLOT + + validTs := now - SLOT + for ii := int64(0); ii < SLOT; ii++ { + idx := (index + 1 + ii) % SLOT + if ts := atomic.LoadInt64(&c.timestamps[idx]); ts > validTs { + counts[ii] = int(atomic.LoadInt64(&c.counts[idx])) + } + } + return +} diff --git a/blobstore/common/counter/counter_test.go b/blobstore/common/counter/counter_test.go new file mode 100644 index 000000000..e9cf1d4f4 --- /dev/null +++ b/blobstore/common/counter/counter_test.go @@ -0,0 +1,109 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package counter + +import ( + "sync" + "testing" + + "github.com/benbjohnson/clock" + "github.com/stretchr/testify/require" +) + +const second clock.Duration = 1e9 + +var mockTime *clock.Mock + +func init() { + mockTime = clock.NewMock() + mockTime.Add(1483950782 * second) + time = mockTime +} + +func TestCounterBase(t *testing.T) { + var c Counter + var exp [SLOT]int + require.Equal(t, exp, c.Show()) + + c.Add() + exp[SLOT-1] = 1 + require.Equal(t, exp, c.Show()) + + for ii := int64(0); ii < SLOT; ii++ { + mockTime.Add(clock.Duration(interval) * second) + c.AddN(int(ii)) + copy(exp[:], exp[1:]) + exp[SLOT-1] = int(ii) + require.Equal(t, exp, c.Show()) + } + + for range [10]struct{}{} { + c := &Counter{} + c.Add() + mockTime.Add(clock.Duration(interval) * second) + exp := [SLOT]int{} + exp[SLOT-2] = 1 + require.Equal(t, exp, c.Show()) + + c.Add() + c.Add() + exp[SLOT-1] = 2 + require.Equal(t, exp, c.Show()) + + mockTime.Add(clock.Duration(SLOT-1) * clock.Duration(interval) * second) + exp = [SLOT]int{} + exp[0] = 2 + require.Equal(t, exp, c.Show()) + + c.Add() + exp[0] = 2 + exp[SLOT-1] = 1 + require.Equal(t, exp, c.Show()) + } +} + +func TestCounterSafe(t *testing.T) { + var c Counter + var wg sync.WaitGroup + wg.Add(1000) + for idx := range [1000]struct{}{} { + go func(ii int) { + mockTime.Add(clock.Duration(ii) * second) + c.AddN(ii) + if ii%3 == 0 { + c.Show() + } + wg.Done() + }(idx) + } + wg.Wait() +} + +func BenchmarkCounterAdd(b *testing.B) { + var c Counter + for ii := 0; ii < b.N; ii++ { + c.Add() + } +} + +func BenchmarkCounterShow(b *testing.B) { + var c Counter + c.AddN(100) + mockTime.Add(100 * second) + b.ResetTimer() + for ii := 0; ii < b.N; ii++ { + c.Show() + } +} diff --git a/blobstore/common/crc32block/block.go b/blobstore/common/crc32block/block.go new file mode 100644 index 000000000..f9186acee --- /dev/null +++ b/blobstore/common/crc32block/block.go @@ -0,0 +1,49 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package crc32block + +import ( + "encoding/binary" + "hash/crc32" +) + +const ( + defaultCrc32BlockSize = 64 * 1024 +) + +var gBlockSize int64 = defaultCrc32BlockSize + +type blockUnit []byte + +func (b blockUnit) length() int { + return len(b) +} + +func (b blockUnit) payload() int { + return len(b) - crc32Len +} + +func (b blockUnit) check() (err error) { + payloadCrc := crc32.ChecksumIEEE(b[crc32Len:]) + if binary.LittleEndian.Uint32(b) != payloadCrc { + return ErrMismatchedCrc + } + return nil +} + +func (b blockUnit) writeCrc() { + payloadCrc := crc32.ChecksumIEEE(b[crc32Len:]) + binary.LittleEndian.PutUint32(b, payloadCrc) +} diff --git a/blobstore/common/crc32block/decode.go b/blobstore/common/crc32block/decode.go new file mode 100644 index 000000000..ce30d5222 --- /dev/null +++ b/blobstore/common/crc32block/decode.go @@ -0,0 +1,216 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package crc32block + +import ( + "bufio" + "io" + "io/ioutil" +) + +/* +journal record: + + |-----block-----|------block-----|---block---| + +block: + + |--crc--|------payload -----| +*/ + +type Decoder struct { + from io.ReaderAt // read from + off int64 // offset. readonly + limit int64 // size limit, readonly + bufSize int64 // for speed + block blockUnit // block buffer +} + +type decoderReader struct { + reader io.Reader // + block []byte // + i, j int // block[i:j] + err error // +} + +type rangeReader struct { + r io.Reader + limit int64 + skip int64 + skiped bool +} + +type blockReader struct { + reader io.Reader // + block blockUnit // + i, j int // block[i:j] is unread portion of the current block's payload. + remain int64 // + err error // +} + +func (br *blockReader) Read(p []byte) (n int, err error) { + if br.err != nil { + return 0, br.err + } + + for br.i == br.j { + if br.remain == 0 { + return n, io.EOF + } + br.err = br.nextBlock() + if br.err != nil { + return 0, br.err + } + } + + n = copy(p, br.block[br.i:br.j]) + + br.i += n + br.remain -= int64(n) + return n, nil +} + +func (br *blockReader) nextBlock() (err error) { + blockLen := int64(len(br.block)) + blockPayloadLen := int64(blockLen - crc32Len) + + want := blockLen + if br.remain < blockPayloadLen { + want = br.remain + crc32Len + } + + _, err = io.ReadFull(br.reader, br.block[:want]) + if err != nil { + br.err = err + return br.err + } + + if err = blockUnit(br.block[:want]).check(); err != nil { + return err + } + + br.i = crc32Len + br.j = int(want) + + return nil +} + +func (r *rangeReader) Read(p []byte) (n int, err error) { + if !r.skiped { + _, err := io.CopyN(ioutil.Discard, r.r, r.skip) + if err != nil { + return 0, err + } + r.skiped = true + r.r = io.LimitReader(r.r, r.limit) + } + return r.r.Read(p) +} + +func (dec *Decoder) Reader(from, to int64) (r io.Reader, err error) { + blockLen := int64(dec.block.length()) + blockPayloadLen := int64(dec.block.payload()) + + blockOff := (from / blockPayloadLen) * blockLen + encodedSize := EncodeSize(dec.limit, blockLen) - blockOff + + // raw reader + r = io.NewSectionReader(dec.from, dec.off+blockOff, encodedSize) + + // buffer + r = bufio.NewReaderSize(r, int(dec.bufSize)) + + // decode reader + r = NewBlockReader(r, DecodeSize(encodedSize, blockLen), dec.block) + + // range reader + r = &rangeReader{ + r: r, + limit: to - from, + skip: from % blockPayloadLen, + } + + return r, nil +} + +func (r *decoderReader) Read(b []byte) (n int, err error) { + if r.err != nil { + return 0, r.err + } + + for len(b) > 0 { + if r.i == r.j { + if r.err = r.nextBlock(); r.err != nil { + if n > 0 { + return n, nil + } + return n, r.err + } + } + + readn := copy(b, r.block[r.i:r.j]) + r.i += readn + + b = b[readn:] + n += readn + } + return +} + +func (r *decoderReader) nextBlock() (err error) { + n, err := readFullOrToEnd(r.reader, r.block) + if err != nil { + return + } + + if n <= crc32Len { + return ErrMismatchedCrc + } + + if err = blockUnit(r.block[:n]).check(); err != nil { + return ErrMismatchedCrc + } + + r.i, r.j = crc32Len, n + + return nil +} + +func NewBlockReader(r io.Reader, limit int64, block []byte) *blockReader { + if block == nil || !isValidBlockLen(int64(len(block))) { + panic(ErrInvalidBlock) + } + return &blockReader{reader: r, remain: limit, block: block} +} + +// NewDecoderReader returns io.Reader +// +// Deprecated: no reused buffer, use NewBodyDecoder to instead. +func NewDecoderReader(in io.Reader) io.Reader { + chunk := make([]byte, defaultCrc32BlockSize) + return &decoderReader{block: chunk, err: nil, reader: in} +} + +func NewDecoderWithBlock(r io.ReaderAt, off int64, size int64, block []byte, bufferSize int64) (dec *Decoder, err error) { + if block == nil || !isValidBlockLen(int64(len(block))) { + return nil, ErrInvalidBlock + } + return &Decoder{from: r, off: off, block: block, limit: size, bufSize: bufferSize}, nil +} + +func NewDecoder(r io.ReaderAt, off int64, size int64) (dec *Decoder, err error) { + block := make([]byte, defaultCrc32BlockSize) + return NewDecoderWithBlock(r, off, size, block, defaultCrc32BlockSize) +} diff --git a/blobstore/common/crc32block/decode_test.go b/blobstore/common/crc32block/decode_test.go new file mode 100644 index 000000000..11fb52166 --- /dev/null +++ b/blobstore/common/crc32block/decode_test.go @@ -0,0 +1,53 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package crc32block + +import ( + "bytes" + "hash/crc32" + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSimpleEncoderDecoder(t *testing.T) { + size1 := int64(64*1024 - 4) + size2 := int64(64 * 1024) + size3 := int64(64*1024 + 4) + size4 := int64(1024 * 1024) + size5 := int64(0) + data := []struct { + bodysize int64 + }{ + {bodysize: size1}, + {bodysize: size2}, + {bodysize: size3}, + {bodysize: size4}, + {bodysize: size5}, + } + + for index, d := range data { + b1 := genRandData(d.bodysize) + body := bytes.NewReader(b1) + enc := NewEncoderReader(body) + + dec := NewDecoderReader(enc) + b2 := make([]byte, d.bodysize) + _, err := io.ReadFull(dec, b2) + require.NoError(t, err, "index is %v", index) + require.Equal(t, crc32.ChecksumIEEE(b1), crc32.ChecksumIEEE(b2), "index is %v", index) + } +} diff --git a/blobstore/common/crc32block/encode.go b/blobstore/common/crc32block/encode.go new file mode 100644 index 000000000..c84b05022 --- /dev/null +++ b/blobstore/common/crc32block/encode.go @@ -0,0 +1,172 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package crc32block + +import ( + "io" +) + +type ReaderError struct { + error +} + +type WriterError struct { + error +} + +type Encoder struct { + block blockUnit // block buffer +} + +type limitEncoderReader struct { + reader io.Reader + block blockUnit + remain int64 + i, j int + err error +} + +type encoderReader struct { + reader io.Reader // + block blockUnit // + i, j int // block[i:j] + err error // +} + +func (enc *Encoder) Encode(from io.Reader, limitSize int64, to io.Writer) (n int64, err error) { + if !isValidBlockLen(int64(enc.block.length())) { + panic(ErrInvalidBlock) + } + + encSize := EncodeSize(limitSize, int64(enc.block.length())) + + reader := &limitEncoderReader{reader: from, block: enc.block, remain: limitSize} + + return io.CopyN(to, reader, encSize) +} + +func (r *limitEncoderReader) Read(b []byte) (n int, err error) { + if r.err != nil { + return 0, r.err + } + + for len(b) > 0 { + if r.i == r.j { + if r.remain == 0 { + return n, io.EOF + } + if r.err = r.nextBlock(); r.err != nil { + if n > 0 { + return n, nil + } + return n, r.err + } + } + + readn := copy(b, r.block[r.i:r.j]) + r.i += readn + + b = b[readn:] + n += readn + } + return +} + +func (r *limitEncoderReader) nextBlock() (err error) { + blockPayloadLen := r.block.payload() + + needn := blockPayloadLen + if r.remain < int64(blockPayloadLen) { + needn = int(r.remain) + } + + block := blockUnit(r.block[:crc32Len+needn]) + + n, err := io.ReadFull(r.reader, block[crc32Len:]) + if err != nil { + return ReaderError{err} + } + + r.i = 0 + r.j = crc32Len + n + + blockUnit(r.block[r.i:r.j]).writeCrc() + r.remain -= int64(block.payload()) + + return nil +} + +func (r *encoderReader) Read(b []byte) (n int, err error) { + if r.err != nil { + return 0, r.err + } + + for len(b) > 0 { + if r.i == r.j { + if r.err = r.nextBlock(); r.err != nil { + if n > 0 { + return n, nil + } + return n, r.err + } + } + + readn := copy(b, r.block[r.i:r.j]) + r.i += readn + + b = b[readn:] + n += readn + } + return +} + +func (r *encoderReader) nextBlock() (err error) { + n, err := readFullOrToEnd(r.reader, r.block[crc32Len:]) + if err != nil { + return err + } + + r.i = 0 + r.j = crc32Len + n + + blockUnit(r.block[r.i:r.j]).writeCrc() + + return nil +} + +func NewEncoder(block []byte) (enc *Encoder, err error) { + if block != nil && !isValidBlockLen(int64(len(block))) { + return nil, ErrInvalidBlock + } + if block == nil { + block = make([]byte, defaultCrc32BlockSize) + } + + return &Encoder{block: block}, nil +} + +// NewEncoderReader returns io.Reader +// +// Deprecated: no reused buffer, use NewBodyEncoder to instead. +func NewEncoderReader(r io.Reader) io.Reader { + block := make([]byte, defaultCrc32BlockSize) + return &encoderReader{block: block, reader: r} +} + +func NewLimitEncoderReader(r io.Reader, limitSize int64) (enc *limitEncoderReader) { + block := make([]byte, defaultCrc32BlockSize) + enc = &limitEncoderReader{reader: r, block: block, remain: limitSize} + return +} diff --git a/blobstore/common/crc32block/encode_test.go b/blobstore/common/crc32block/encode_test.go new file mode 100644 index 000000000..3c283e4ad --- /dev/null +++ b/blobstore/common/crc32block/encode_test.go @@ -0,0 +1,260 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package crc32block + +import ( + "bytes" + "crypto/rand" + "io" + "io/ioutil" + "testing" + + "github.com/stretchr/testify/require" +) + +const ( + _KiB = int64(1024) +) + +func genRandData(size int64) []byte { + b := make([]byte, size) + rand.Read(b) + return b +} + +func TestEncodeData(t *testing.T) { + data := genRandData(128*1024 + 80) + fsize := int64(len(data)) // 128k+80 + r := bytes.NewReader(data) + w := bytes.NewBuffer(nil) + + encoder, err := NewEncoder(nil) + require.NoError(t, err) + require.NotNil(t, encoder) + + // encode + n, err := encoder.Encode(r, fsize, w) + require.NoError(t, err) + require.Equal(t, n, EncodeSize(int64(fsize), defaultCrc32BlockSize)) +} + +func TestDecodeData(t *testing.T) { + data := genRandData(128*1024 + 80) + fsize := int64(len(data)) // 128k+80 + r := bytes.NewReader(data) + w := bytes.NewBuffer(nil) + + encoder, err := NewEncoder(nil) + require.NoError(t, err) + require.NotNil(t, encoder) + + // encode + n, err := encoder.Encode(r, fsize, w) + require.NoError(t, err) + require.Equal(t, n, EncodeSize(fsize, defaultCrc32BlockSize)) + + // decode + testDatas := []struct { + off int64 + from int64 + to int64 + fsize int64 + padding int64 + }{ + {from: 0, to: 0, fsize: 0}, + {from: fsize, to: fsize, fsize: fsize}, + {from: 0, to: fsize, fsize: fsize}, + + {from: 64*_KiB - 4, to: fsize, fsize: fsize}, + {from: 64*_KiB + 0, to: fsize, fsize: fsize}, + {from: 64*_KiB + 4, to: fsize, fsize: fsize}, + {from: 64*_KiB + 5, to: fsize, fsize: fsize}, + + {from: 64*_KiB - 4, to: fsize - 1, fsize: fsize}, + {from: 64*_KiB + 0, to: fsize - 1, fsize: fsize}, + {from: 64*_KiB + 4, to: fsize - 1, fsize: fsize}, + {from: 64*_KiB + 5, to: fsize - 1, fsize: fsize}, + + {from: 64*_KiB + 4, to: fsize - 64*_KiB, fsize: fsize}, + {from: 64*_KiB + 4, to: fsize - 64*_KiB - 4, fsize: fsize}, + {from: 64*_KiB + 4, to: fsize - 64*_KiB - 5, fsize: fsize}, + {from: 0, to: fsize - 64*_KiB - 4 - 64*_KiB - 4, fsize: fsize}, + } + + for _, tb := range testDatas { + disk := append(make([]byte, tb.off), w.Bytes()...) + disk = append(disk, make([]byte, tb.padding)...) + + decoder, err := NewDecoder(bytes.NewReader(disk), int64(tb.off), tb.fsize) + require.NoError(t, err) + require.NotNil(t, decoder) + + reader, err := decoder.Reader(tb.from, tb.to) + require.NoError(t, err) + require.NotNil(t, reader) + + cont, err := ioutil.ReadAll(reader) + require.NoError(t, err) + require.Equal(t, data[tb.from:tb.to], cont) + } +} + +func TestLimitEncoderReader(t *testing.T) { + data := genRandData(128*1024 + 80) + fsize := int64(len(data)) // 128k+80 + r := bytes.NewReader(data) + w := bytes.NewBuffer(nil) + + encReader := NewLimitEncoderReader(r, fsize) + io.Copy(w, encReader) + + // decode + testDatas := []struct { + off int64 + from int64 + to int64 + fsize int64 + padding int64 + }{ + {from: 0, to: 0, fsize: 0}, + {from: fsize, to: fsize, fsize: fsize}, + {from: 0, to: fsize, fsize: fsize}, + + {from: 0, to: 64, fsize: fsize}, + {from: 64*_KiB - 4, to: fsize, fsize: fsize}, + {from: 64*_KiB + 0, to: fsize, fsize: fsize}, + {from: 64*_KiB + 4, to: fsize, fsize: fsize}, + {from: 64*_KiB + 5, to: fsize, fsize: fsize}, + + {from: 64*_KiB - 4, to: fsize - 1, fsize: fsize}, + {from: 64*_KiB + 0, to: fsize - 1, fsize: fsize}, + {from: 64*_KiB + 4, to: fsize - 1, fsize: fsize}, + {from: 64*_KiB + 5, to: fsize - 1, fsize: fsize}, + + {from: 64*_KiB + 4, to: fsize - 64*_KiB, fsize: fsize}, + {from: 64*_KiB + 4, to: fsize - 64*_KiB - 4, fsize: fsize}, + {from: 64*_KiB + 4, to: fsize - 64*_KiB - 5, fsize: fsize}, + {from: 0, to: fsize - 64*_KiB - 4 - 64*_KiB - 4, fsize: fsize}, + } + + for _, tb := range testDatas { + disk := append(make([]byte, tb.off), w.Bytes()...) + disk = append(disk, make([]byte, tb.padding)...) + + decoder, err := NewDecoder(bytes.NewReader(disk), int64(tb.off), tb.fsize) + require.NoError(t, err) + require.NotNil(t, decoder) + + reader, err := decoder.Reader(tb.from, tb.to) + require.NoError(t, err) + require.NotNil(t, reader) + + cont, err := ioutil.ReadAll(reader) + require.NoError(t, err) + require.Equal(t, data[tb.from:tb.to], cont) + } +} + +func TestLimitEncoderReader2(t *testing.T) { + testDatas := []struct { + offset int64 + fsize int64 + from, to int64 + padding int64 + err error + }{ + {offset: 0, fsize: 1, from: 0, to: 1}, + {offset: 0, fsize: 64*_KiB - 4, from: 0, to: 64}, + {offset: 0, fsize: 64*_KiB - 4, from: 0, to: 64*_KiB - 4}, + {offset: 10, fsize: 64 * _KiB, from: 0, to: 64*_KiB - 4}, + {offset: 0, fsize: 64 * _KiB, from: 0, to: 64 * _KiB}, + {offset: 10, fsize: 64 * _KiB, from: 0, to: 64 * _KiB}, + {offset: 10, fsize: 64*_KiB + 5, from: 0, to: 64 * _KiB}, + + {offset: 10, fsize: 128 * _KiB, from: 0, to: 128 * _KiB}, + {offset: 10, fsize: 128 * _KiB, from: 128*_KiB - 4, to: 128 * _KiB}, + {offset: 10, fsize: 128 * _KiB, from: 64*_KiB - 4, to: 64 * _KiB}, + {offset: 10, fsize: 128 * _KiB, from: 64*_KiB + 4, to: 64*_KiB + 4}, + {offset: 10, fsize: 128 * _KiB, from: 64*_KiB + 4, to: 64*_KiB + 8}, + {offset: 10, fsize: 128 * _KiB, from: 64*_KiB + 7, to: 64*_KiB + 10}, + {offset: 10, fsize: 128 * _KiB, from: 64*_KiB - 4, to: 64*_KiB - 1}, + {offset: 10, fsize: 128 * _KiB, from: 4, to: 64*_KiB - 1}, + {offset: 10, fsize: 128 * _KiB, from: 4, to: 64*_KiB - 4}, + {offset: 10, fsize: 128 * _KiB, from: 4, to: 64 * _KiB}, + {offset: 10, fsize: 128 * _KiB, from: 64 * _KiB, to: 128 * _KiB}, + {offset: 10, fsize: 128*_KiB + 7, from: 64 * _KiB, to: 64 * _KiB}, + {offset: 10, fsize: 128*_KiB + 7, from: 64*_KiB + 4, to: 128 * _KiB}, + {offset: 10, fsize: 128*_KiB + 7, from: 64*_KiB - 4, to: 128*_KiB + 7}, + } + + for _, tb := range testDatas { + // encoder + data := genRandData(tb.fsize) + fsize := int64(len(data)) + r := bytes.NewReader(data) + w := bytes.NewBuffer(nil) + + encReader := NewLimitEncoderReader(r, fsize) + io.Copy(w, encReader) + + // decoder + content := append(make([]byte, tb.offset), w.Bytes()...) + content = append(content, make([]byte, tb.padding)...) + + decoder, err := NewDecoder(bytes.NewReader(content), int64(tb.offset), tb.fsize) + require.NoError(t, err) + require.NotNil(t, decoder) + + reader, err := decoder.Reader(tb.from, tb.to) + require.NoError(t, err) + require.NotNil(t, reader) + + cont, err := ioutil.ReadAll(reader) + require.NoError(t, err) + require.Equal(t, data[tb.from:tb.to], cont) + } +} + +func TestLimitEncoderReaderFailed(t *testing.T) { + // encoder + data := genRandData(64 * _KiB) + fsize := int64(len(data)) + r := bytes.NewReader(data) + w := bytes.NewBuffer(nil) + + enc, err := NewEncoder(make([]byte, 1)) + require.Error(t, err) + require.Nil(t, enc) + + enc, err = NewEncoder(nil) + require.NoError(t, err) + require.NotNil(t, enc) + + n, err := enc.Encode(r, fsize, w) + require.NoError(t, err) + require.Equal(t, n, EncodeSize(fsize, defaultCrc32BlockSize)) + + r = bytes.NewReader(data) + _, err = enc.Encode(r, fsize+1, ioutil.Discard) + require.Error(t, err) + _, ok := err.(ReaderError) + require.Equal(t, ok, true) +} + +func BenchmarkEncode(b *testing.B) { + benchmarkCode(b, func(r io.Reader) io.ReadCloser { + return io.NopCloser(NewEncoderReader(r)) + }) +} diff --git a/blobstore/common/crc32block/request_body.go b/blobstore/common/crc32block/request_body.go new file mode 100644 index 000000000..0e41b3c10 --- /dev/null +++ b/blobstore/common/crc32block/request_body.go @@ -0,0 +1,203 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package crc32block + +import ( + "io" + "sync" + + "github.com/cubefs/cubefs/blobstore/util/bytespool" +) + +// RequestBody is implemented http request's body. +// always io.ReadCloser. +// +// For client requests, The HTTP Client's Transport is +// responsible for calling the Close method. Necessarily call +// the Close method if the body's life-cycle control by yourself. +// +// For server requests, the Server will close the request body. +// The ServeHTTP Handler does not need to. +// +// The Body must allow Read to be called concurrently with Close. +// In particular, calling Close should unblock a Read waiting +// for input. +type RequestBody interface { + io.ReadCloser + + // CodeSize returns encoded whole body size for encoding, + // or origin body size for decoding. + CodeSize(int64) int64 +} + +type requestBody struct { + encode bool + offset int + err error + block blockUnit + rc io.ReadCloser + + blockLock chan struct{} // safely free the block + closeCh chan struct{} + closeOnce sync.Once +} + +func (r *requestBody) Read(p []byte) (n int, err error) { + if r.err != nil { + return 0, r.err + } + + for len(p) > 0 { + if r.offset < 0 || r.offset == r.block.length() { + if r.err = r.nextBlock(); r.err != nil { + if n > 0 { + return n, nil + } + return n, r.err + } + } + + read := copy(p, r.block[r.offset:]) + r.offset += read + + p = p[read:] + n += read + } + return n, nil +} + +func (r *requestBody) nextBlock() error { + var ( + n int + err error + block blockUnit + ) + if r.encode { + block = r.block[crc32Len:] + } else { + block = r.block + } + + readCh := make(chan struct{}) + go func() { + if _, ok := <-r.blockLock; !ok { + // closed + return + } + n, err = readFullOrToEnd(r.rc, block) + close(readCh) + r.blockLock <- struct{}{} + }() + + select { + case <-r.closeCh: + return ErrReadOnClosed + case <-readCh: + } + if err != nil { + return err + } + + if r.encode { + r.offset = 0 + r.block = r.block[:crc32Len+n] + r.block.writeCrc() + return nil + } + + if n <= crc32Len { + return ErrMismatchedCrc + } + + r.offset = crc32Len + r.block = r.block[:n] + if err = r.block.check(); err != nil { + return ErrMismatchedCrc + } + return nil +} + +func (r *requestBody) Close() error { + r.closeOnce.Do(func() { + block := r.block + r.block = nil + close(r.closeCh) + + go func(buf []byte) { + <-r.blockLock + close(r.blockLock) + bytespool.Free(buf) + }(block) + }) + return r.rc.Close() +} + +func (r *requestBody) CodeSize(size int64) int64 { + if r.encode { + return EncodeSize(size, int64(r.block.length())) + } + return DecodeSize(size, int64(r.block.length())) +} + +type codeSizeBody struct { + encode bool + blockLength int64 +} + +func (c *codeSizeBody) Read(p []byte) (n int, err error) { return 0, io.EOF } +func (c *codeSizeBody) Close() error { return nil } +func (c *codeSizeBody) CodeSize(size int64) int64 { + if c.encode { + return EncodeSize(size, c.blockLength) + } + return DecodeSize(size, c.blockLength) +} + +// TODO: using resourcepool's chan-pool if block size greater than 64K. +func newRequestBody(rc io.ReadCloser, encode bool) RequestBody { + if rc == nil { + return &codeSizeBody{ + encode: encode, + blockLength: gBlockSize, + } + } + + lock := make(chan struct{}, 1) + lock <- struct{}{} + return &requestBody{ + encode: encode, + block: bytespool.Alloc(int(gBlockSize)), + offset: -1, + rc: rc, + blockLock: lock, + closeCh: make(chan struct{}), + } +} + +// NewBodyEncoder returns encoder with crc32. +// +// If rc == nil, the encoder is called just with CodeSize, +// you need not to Close it at all. +func NewBodyEncoder(rc io.ReadCloser) RequestBody { + return newRequestBody(rc, true) +} + +// NewBodyDecoder returns decoder with crc32. +// +// If rc == nil, the decoder is called just with CodeSize, +// you need not to Close it at all. +func NewBodyDecoder(rc io.ReadCloser) RequestBody { + return newRequestBody(rc, false) +} diff --git a/blobstore/common/crc32block/request_body_test.go b/blobstore/common/crc32block/request_body_test.go new file mode 100644 index 000000000..e7c181940 --- /dev/null +++ b/blobstore/common/crc32block/request_body_test.go @@ -0,0 +1,207 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package crc32block + +import ( + "bytes" + "hash/crc32" + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBodyEncoderDecoder(t *testing.T) { + cases := []int64{ + 0, + 64*1024 - 4, + 64 * 1024, + 64*1024 + 4, + 1024 * 1024, + } + for idx, size := range cases { + wb := genRandData(size) + { + encoder := NewBodyEncoder(io.NopCloser(bytes.NewReader(wb))) + defer encoder.Close() + decoder := NewBodyDecoder(encoder) + defer decoder.Close() + require.Equal(t, size, decoder.CodeSize(encoder.CodeSize(size))) + + rb := make([]byte, size) + _, err := io.ReadFull(decoder, rb) + require.NoError(t, err, "index is %d", idx) + require.Equal(t, crc32.ChecksumIEEE(wb), crc32.ChecksumIEEE(rb), "index is %d", idx) + } + { + encoder := NewBodyEncoder(io.NopCloser(bytes.NewReader(wb))) + defer encoder.Close() + decoder := NewDecoderReader(encoder) + require.Equal(t, size, DecodeSizeWithDefualtBlock(encoder.CodeSize(size))) + + rb := make([]byte, size) + _, err := io.ReadFull(decoder, rb) + require.NoError(t, err, "index is %d", idx) + require.Equal(t, crc32.ChecksumIEEE(wb), crc32.ChecksumIEEE(rb), "index is %d", idx) + } + { + encoder := NewEncoderReader(bytes.NewReader(wb)) + decoder := NewBodyDecoder(io.NopCloser(encoder)) + defer decoder.Close() + require.Equal(t, size, decoder.CodeSize(EncodeSizeWithDefualtBlock(size))) + + rb := make([]byte, size) + _, err := io.ReadFull(decoder, rb) + require.NoError(t, err, "index is %d", idx) + require.Equal(t, crc32.ChecksumIEEE(wb), crc32.ChecksumIEEE(rb), "index is %d", idx) + } + } +} + +func TestNilEncoderDecoder(t *testing.T) { + cases := []int64{ + 0, + 64*1024 - 4, + 64 * 1024, + 64*1024 + 4, + 1024 * 1024, + } + for _, size := range cases { + { + encoder := NewBodyEncoder(nil) + defer encoder.Close() + decoder := NewBodyDecoder(nil) + require.Equal(t, size, decoder.CodeSize(encoder.CodeSize(size))) + + rb := make([]byte, 1024) + _, err := io.ReadFull(decoder, rb) + require.Error(t, err) + } + } +} + +type brokenReader struct { + io.ReadCloser +} + +func (r brokenReader) Read(p []byte) (n int, err error) { + n, err = r.ReadCloser.Read(p) + p[10]++ + return +} + +func TestBodyDecoderMissmatch(t *testing.T) { + var size int64 = 1 << 12 + wb := genRandData(size) + + encoder := NewBodyEncoder(io.NopCloser(bytes.NewReader(wb))) + defer encoder.Close() + decoder := NewBodyDecoder(brokenReader{encoder}) + defer decoder.Close() + require.Equal(t, size, decoder.CodeSize(encoder.CodeSize(size))) + + rb := make([]byte, size) + _, err := io.ReadFull(decoder, rb) + require.ErrorIs(t, ErrMismatchedCrc, err) +} + +type closedReader struct { + io.ReadCloser + waiter chan struct{} +} + +func (r *closedReader) Read(p []byte) (n int, err error) { + <-r.waiter + n, err = r.ReadCloser.Read(p) + return +} + +func TestBodyReadClosed(t *testing.T) { + var size int64 = 1 << 12 + wb := genRandData(size) + + encoder := NewBodyEncoder(io.NopCloser(bytes.NewReader(wb))) + defer encoder.Close() + rc := &closedReader{ + ReadCloser: encoder, + waiter: make(chan struct{}), + } + decoder := NewBodyDecoder(rc) + defer decoder.Close() + require.Equal(t, size, decoder.CodeSize(encoder.CodeSize(size))) + + decoder.Close() + rb := make([]byte, size) + _, err := io.ReadFull(decoder, rb) + require.ErrorIs(t, ErrReadOnClosed, err) +} + +func benchmarkCode(b *testing.B, newFunc func(io.Reader) io.ReadCloser) { + const kb = 1 << 10 + const mb = 1 << 20 + + cases := []struct { + name string + size int64 + }{ + {"1__B", 1}, + {"128B", 128}, + {"2__K", kb * 2}, + {"64_K", kb * 64}, + {"512K", kb * 512}, + {"1__M", mb}, + {"2__M", mb * 2}, + } + + for _, cs := range cases { + b.ResetTimer() + b.Run(cs.name, func(b *testing.B) { + src := make([]byte, cs.size) + b.ResetTimer() + for ii := 0; ii <= b.N; ii++ { + rc := newFunc(bytes.NewBuffer(src)) + io.Copy(io.Discard, rc) + rc.Close() + } + }) + } +} + +func BenchmarkBodyEncodeOnly(b *testing.B) { + benchmarkCode(b, func(r io.Reader) io.ReadCloser { + return NewBodyEncoder(io.NopCloser(r)) + }) +} + +type coder struct { + en, de RequestBody +} + +func (c *coder) Read(p []byte) (n int, err error) { + return c.de.Read(p) +} + +func (c *coder) Close() error { + c.en.Close() + return c.de.Close() +} + +func BenchmarkBodyEncodeDecode(b *testing.B) { + benchmarkCode(b, func(r io.Reader) io.ReadCloser { + encoder := NewBodyEncoder(io.NopCloser(r)) + decoder := NewBodyDecoder(encoder) + return &coder{encoder, decoder} + }) +} diff --git a/blobstore/common/crc32block/util.go b/blobstore/common/crc32block/util.go new file mode 100644 index 000000000..8bf6683db --- /dev/null +++ b/blobstore/common/crc32block/util.go @@ -0,0 +1,87 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package crc32block + +import ( + "errors" + "io" +) + +const ( + crc32Len = 4 + baseBlockBit = 12 + baseBlockLen = (1 << baseBlockBit) +) + +var ( + ErrInvalidBlock = errors.New("crc32block: invalid block buffer") + ErrMismatchedCrc = errors.New("crc32block: mismatched checksum") + ErrReadOnClosed = errors.New("crc32block: read on closed") +) + +func isValidBlockLen(blockLen int64) bool { + return blockLen > 0 && blockLen%baseBlockLen == 0 +} + +func blockPayload(blockLen int64) int64 { + return blockLen - crc32Len +} + +// SetBlockSize set default block size +func SetBlockSize(blockSize int64) { + if !isValidBlockLen(blockSize) { + panic(ErrInvalidBlock) + } + gBlockSize = blockSize +} + +func EncodeSize(size int64, blockLen int64) int64 { + if !isValidBlockLen(blockLen) { + panic(ErrInvalidBlock) + } + payload := blockPayload(blockLen) + blockCnt := (size + (payload - 1)) / payload + return size + 4*blockCnt +} + +func DecodeSize(totalSize int64, blockLen int64) int64 { + if !isValidBlockLen(blockLen) { + panic(ErrInvalidBlock) + } + blockCnt := (totalSize + (blockLen - 1)) / blockLen + return totalSize - 4*blockCnt +} + +func EncodeSizeWithDefualtBlock(size int64) int64 { + return EncodeSize(size, defaultCrc32BlockSize) +} + +func DecodeSizeWithDefualtBlock(size int64) int64 { + return DecodeSize(size, defaultCrc32BlockSize) +} + +func readFullOrToEnd(r io.Reader, buffer []byte) (n int, err error) { + nn, size := 0, len(buffer) + + for n < size && err == nil { + nn, err = r.Read(buffer[n:]) + n += nn + if n != 0 && err == io.EOF { + return n, nil + } + } + + return n, err +} diff --git a/blobstore/common/crc32block/util_test.go b/blobstore/common/crc32block/util_test.go new file mode 100644 index 000000000..cf16e3d50 --- /dev/null +++ b/blobstore/common/crc32block/util_test.go @@ -0,0 +1,63 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package crc32block + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDecodeSize(t *testing.T) { + datas := []struct { + blockLen int64 + fsize int64 + encodeSize int64 + }{ + {blockLen: 64 * 1024, fsize: 1, encodeSize: 5}, + {blockLen: 64 * 1024, fsize: 64*1024 - 4, encodeSize: 64 * 1024}, + {blockLen: 64 * 1024, fsize: 64 * 1024, encodeSize: 64*1024 + 8}, + } + + for _, dt := range datas { + require.Equal(t, int64(dt.encodeSize), EncodeSize(dt.fsize, dt.blockLen)) + require.Equal(t, int64(dt.fsize), DecodeSize(dt.encodeSize, dt.blockLen)) + } +} + +func TestSetBlockSize(t *testing.T) { + for _, size := range []int64{-100, -1, 0, baseBlockLen - 1, baseBlockLen + 1} { + require.Panics(t, func() { SetBlockSize(size) }) + } + + defer func() { + SetBlockSize(defaultCrc32BlockSize) + }() + + datas := []struct { + blockLen int64 + fsize int64 + encodeSize int64 + }{ + {blockLen: 1 << 12, fsize: 1, encodeSize: 5}, + {blockLen: 1 << 20, fsize: 64*1024 - 4, encodeSize: 64 * 1024}, + {blockLen: 64 * 1024, fsize: 64 * 1024, encodeSize: 64*1024 + 8}, + } + for _, dt := range datas { + SetBlockSize(dt.blockLen) + require.Equal(t, dt.encodeSize, NewBodyEncoder(nil).CodeSize(dt.fsize)) + require.Equal(t, dt.fsize, NewBodyDecoder(nil).CodeSize(dt.encodeSize)) + } +} diff --git a/blobstore/common/ec/buf.go b/blobstore/common/ec/buf.go new file mode 100644 index 000000000..f57b60b13 --- /dev/null +++ b/blobstore/common/ec/buf.go @@ -0,0 +1,208 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package ec + +import ( + "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/cubefs/cubefs/blobstore/common/resourcepool" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +// Buffer one ec blob's reused buffer +// Manually manage the DataBuf in Ranged mode, do not Split it +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// | data | align bytes | partiy | local | +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// | DataBuf | +// |<--DataSize->| +// - - - - - - - - - - - - - - - - - - +// | ECDataBuf | +// |<-- ECDataSize -->| +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// | ECBuf | +// |<--- ECSize --->| +type Buffer struct { + tactic codemode.Tactic + pool *resourcepool.MemPool + + // DataBuf real data buffer + DataBuf []byte + // ECDataBuf ec data buffer to Split, + // is nil if Buffer is Ranged mode + ECDataBuf []byte + // BufferSizes all sizes + BufferSizes +} + +// BufferSizes all sizes +// ECSize is sum of all ec shards size, +// not equal the capacity of DataBuf in Ranged mode +type BufferSizes struct { + ShardSize int // per shard size + DataSize int // real data size + ECDataSize int // ec data size only with data + ECSize int // ec buffer size with partiy and local shards + From int // ranged from + To int // ranged to +} + +func isOutOfRange(dataSize, from, to int) bool { + return dataSize <= 0 || to < from || + from < 0 || from > dataSize || + to < 0 || to > dataSize +} + +func newBuffer(dataSize, from, to int, tactic codemode.Tactic, pool *resourcepool.MemPool, hasParity bool) (*Buffer, error) { + if isOutOfRange(dataSize, from, to) { + return nil, ErrShortData + } + + shardN := tactic.N + if shardN <= 0 { + return nil, ErrInvalidCodeMode + } + + shardSize := (dataSize + shardN - 1) / shardN + // align per shard with tactic MinShardSize + if shardSize < tactic.MinShardSize { + shardSize = tactic.MinShardSize + } + + ecDataSize := shardSize * tactic.N + ecSize := shardSize * (tactic.N + tactic.M + tactic.L) + + var ( + err error + + buf []byte + dataBuf []byte + ecDataBuf []byte + ) + if pool != nil { + size := ecSize + if !hasParity { + size = to - from + } + + buf, err = pool.Get(size) + if err == resourcepool.ErrNoSuitableSizeClass { + log.Warn(err, "for", size, "try to alloc bytes") + buf, err = pool.Alloc(size) + } + if err != nil { + return nil, err + } + + if !hasParity { + dataBuf = buf[:size] + ecDataBuf = nil + } else { + // zero the padding bytes of data section + pool.Zero(buf[dataSize:ecDataSize]) + dataBuf = buf[:dataSize] + ecDataBuf = buf[:ecDataSize] + } + } + + return &Buffer{ + tactic: tactic, + pool: pool, + DataBuf: dataBuf, + ECDataBuf: ecDataBuf, + BufferSizes: BufferSizes{ + ShardSize: shardSize, + DataSize: dataSize, + ECDataSize: ecDataSize, + ECSize: ecSize, + From: from, + To: to, + }, + }, nil +} + +// NewBuffer new ec buffer with data size and ec mode +func NewBuffer(dataSize int, tactic codemode.Tactic, pool *resourcepool.MemPool) (*Buffer, error) { + return newBuffer(dataSize, 0, dataSize, tactic, pool, true) +} + +// NewRangeBuffer ranged buffer with least data size +func NewRangeBuffer(dataSize, from, to int, tactic codemode.Tactic, pool *resourcepool.MemPool) (*Buffer, error) { + return newBuffer(dataSize, from, to, tactic, pool, false) +} + +// GetBufferSizes calculate ec buffer sizes +func GetBufferSizes(dataSize int, tactic codemode.Tactic) (BufferSizes, error) { + buf, err := NewBuffer(dataSize, tactic, nil) + if err != nil { + return BufferSizes{}, err + } + return buf.BufferSizes, nil +} + +// Resize re-calculate ec buffer, alloc a new buffer if oversize +func (b *Buffer) Resize(dataSize int) error { + if dataSize == b.BufferSizes.DataSize { + return nil + } + + sizes, err := GetBufferSizes(dataSize, b.tactic) + if err != nil { + return err + } + + // buffer is enough + if sizes.ECSize <= cap(b.DataBuf) { + buf := b.DataBuf[:cap(b.DataBuf)] + // zero the padding bytes of data section + b.pool.Zero(buf[sizes.DataSize:sizes.ECDataSize]) + + b.DataBuf = buf[:sizes.DataSize] + b.ECDataBuf = buf[:sizes.ECDataSize] + b.BufferSizes = sizes + return nil + } + + newb, err := NewBuffer(dataSize, b.tactic, b.pool) + if err != nil { + return err + } + + b.Release() + *b = *newb + return nil +} + +// Release recycles the buffer into pool +func (b *Buffer) Release() error { + if b == nil { + return nil + } + if b.DataBuf == nil { + b.pool = nil + return nil + } + + buf := b.DataBuf + b.DataBuf = nil + b.ECDataBuf = nil + + pool := b.pool + b.pool = nil + if pool != nil { + return pool.Put(buf) + } + + return nil +} diff --git a/blobstore/common/ec/buf_test.go b/blobstore/common/ec/buf_test.go new file mode 100644 index 000000000..c41dadd6d --- /dev/null +++ b/blobstore/common/ec/buf_test.go @@ -0,0 +1,206 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package ec_test + +import ( + "crypto/rand" + mrand "math/rand" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/cubefs/cubefs/blobstore/common/ec" + rp "github.com/cubefs/cubefs/blobstore/common/resourcepool" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +const ( + kb = 1 << 10 + kb4 = 1 << 12 + kb64 = 1 << 16 + kb512 = 1 << 19 + mb = 1 << 20 +) + +var memPool = rp.NewMemPool(map[int]int{kb64: 1, mb: 1}) + +func init() { + mrand.Seed(time.Now().Unix()) + log.SetOutputLevel(log.Lfatal) +} + +func TestNewBuffer(t *testing.T) { + cm := codemode.EC6P6.Tactic() + buffer, err := ec.NewBuffer(kb, cm, memPool) + require.NoError(t, err) + defer buffer.Release() + require.Equal(t, kb, len(buffer.DataBuf)) + require.Equal(t, kb64, cap(buffer.DataBuf)) + require.Equal(t, cap(buffer.ECDataBuf), cap(buffer.DataBuf)) + + // mb pool + cm = codemode.EC16P20L2.Tactic() + buffer, err = ec.NewBuffer(kb64, cm, memPool) + require.NoError(t, err) + defer buffer.Release() + require.Equal(t, kb64, len(buffer.DataBuf)) + require.Equal(t, mb, cap(buffer.DataBuf)) + require.Equal(t, cap(buffer.ECDataBuf), cap(buffer.DataBuf)) + + // alloc new bytes when no suitable pool, do not put back + buffer, err = ec.NewBuffer(kb512, cm, memPool) + shardSize := (kb512 + cm.N - 1) / cm.N + require.NoError(t, err) + require.Equal(t, kb512, len(buffer.DataBuf)) + require.Equal(t, shardSize*(cm.N+cm.M+cm.L), cap(buffer.DataBuf)) + require.Equal(t, shardSize*cm.N, len(buffer.ECDataBuf)) + require.Equal(t, shardSize*(cm.N+cm.M+cm.L), cap(buffer.ECDataBuf)) +} + +func TestNewRangedBuffer(t *testing.T) { + cm := codemode.EC6P6.Tactic() + buffer, err := ec.NewBuffer(kb, cm, memPool) + require.NoError(t, err) + require.Equal(t, kb, len(buffer.DataBuf)) + require.NotNil(t, buffer.ECDataBuf) + require.Equal(t, kb64, cap(buffer.DataBuf)) + buffer.Release() + + // kb pool in ranged + cm = codemode.EC16P20L2.Tactic() + buffer, err = ec.NewRangeBuffer(kb4, kb, 2*kb+2, cm, memPool) + require.NoError(t, err) + require.Equal(t, kb+2, len(buffer.DataBuf)) + require.Nil(t, buffer.ECDataBuf) + require.Equal(t, kb64, cap(buffer.DataBuf)) + buffer.Release() +} + +func TestGetBufferSize(t *testing.T) { + cm := codemode.EC6P6.Tactic() + sizes, err := ec.GetBufferSizes(kb, cm) + shardSize := (kb + cm.N - 1) / cm.N + if shardSize < cm.MinShardSize { + shardSize = cm.MinShardSize + } + require.NoError(t, err) + require.Equal(t, shardSize, sizes.ShardSize) + require.Equal(t, kb, sizes.DataSize) + require.Equal(t, shardSize*cm.N, sizes.ECDataSize) + require.Equal(t, shardSize*(cm.N+cm.M+cm.L), sizes.ECSize) + + cm = codemode.EC16P20L2.Tactic() + sizes, err = ec.GetBufferSizes(kb512, cm) + shardSize = (kb512 + cm.N - 1) / cm.N + if shardSize < cm.MinShardSize { + shardSize = cm.MinShardSize + } + require.NoError(t, err) + require.Equal(t, shardSize, sizes.ShardSize) + require.Equal(t, kb512, sizes.DataSize) + require.Equal(t, shardSize*cm.N, sizes.ECDataSize) + require.Equal(t, shardSize*(cm.N+cm.M+cm.L), sizes.ECSize) + + _, err = ec.GetBufferSizes(0, cm) + require.ErrorIs(t, ec.ErrShortData, err) + _, err = ec.GetBufferSizes(-1, cm) + require.ErrorIs(t, ec.ErrShortData, err) +} + +func TestBufferResize(t *testing.T) { + cm := codemode.EC6P6.Tactic() + buffer, err := ec.NewBuffer(kb, cm, memPool) + require.NoError(t, err) + defer func() { + buffer.Release() + }() + require.Equal(t, kb, len(buffer.DataBuf)) + require.Equal(t, kb64, cap(buffer.DataBuf)) + + // // pool limited + // _, err = ec.NewBuffer(kb, cm, memPool) + // require.ErrorIs(t, rp.ErrPoolLimit, err) + + err = buffer.Resize(kb + 512) + require.NoError(t, err) + require.Equal(t, kb+512, len(buffer.DataBuf)) + require.Equal(t, kb64, cap(buffer.DataBuf)) + + // // pool limited + // _, err = ec.NewBuffer(kb, cm, memPool) + // require.ErrorIs(t, rp.ErrPoolLimit, err) + + // mb pool, release kb64 pool + err = buffer.Resize(kb64) + require.NoError(t, err) + require.Equal(t, kb64, len(buffer.DataBuf)) + require.Equal(t, mb, cap(buffer.DataBuf)) + + // old kb64 pool was released + buff, err := ec.NewBuffer(kb, cm, memPool) + require.NoError(t, err) + buff.Release() +} + +func TestBufferRelease(t *testing.T) { + { + var buffer *ec.Buffer + require.NoError(t, buffer.Release()) + } + { + buffer := &ec.Buffer{} + require.NoError(t, buffer.Release()) + } + { + buffer, err := ec.NewBuffer(kb, codemode.EC6P6.Tactic(), memPool) + require.NoError(t, err) + require.NoError(t, buffer.Release()) + } +} + +func TestBufferDataPadding(t *testing.T) { + // new a mempool while resize will alloc oversize buffer + memPool := rp.NewMemPool(map[int]int{kb64: 1, mb: 1}) + // random read + for _, size := range []int{kb64, mb} { + buf, err := memPool.Get(size) + require.NoError(t, err) + rand.Read(buf) + memPool.Put(buf) + } + + zero := make([]byte, mb) + + cm := codemode.EC6P6.Tactic() + buffer, err := ec.NewBuffer(kb, cm, memPool) + require.NoError(t, err) + defer func() { + buffer.Release() + }() + + buf := buffer.DataBuf[:cap(buffer.DataBuf)] + require.Equal(t, zero[:buffer.ShardSize*cm.N-buffer.DataSize], + buf[buffer.DataSize:buffer.ShardSize*cm.N]) + + for ii := 0; ii < 100; ii++ { + err = buffer.Resize(mrand.Intn(mb*6) + 1) + require.NoError(t, err) + buf = buffer.DataBuf[:cap(buffer.DataBuf)] + require.Equal(t, zero[:buffer.ShardSize*cm.N-buffer.DataSize], + buf[buffer.DataSize:buffer.ShardSize*cm.N]) + } +} diff --git a/blobstore/common/ec/doc.go b/blobstore/common/ec/doc.go new file mode 100644 index 000000000..e161498b6 --- /dev/null +++ b/blobstore/common/ec/doc.go @@ -0,0 +1,55 @@ +// Copyright 2022 The CubeFS Authors. +// +// Note: +// 1. Do not use after releasing it. +// 2. You need to ensure its safety yourself before releasing it. +// 3. You should know that its pointer would have changed after Resize. +// 4. Manually manage the DataBuf in Ranged mode, do not Split it. +// +// MinShardSize min size per shard, fill data into shards 0-N continuously, +// align with zero bytes if data size less than MinShardSize*N +// +// Length of real data less than MinShardSize*N, ShardSize = MinShardSize. +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// | data | align bytes | partiy | local | +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// | 0 | 1 | .... | N | N+1 | ... | N+M | N+M+L | +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// +// Length more than MinShardSize*N, ShardSize = ceil(len(data)/N) +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// | data |padding| partiy | local | +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// | 0 | 1 | .... | N | N+1 | ... | N+M | N+M+L | +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// +// +// Example: +// +// import ( +// "github.com/cubefs/cubefs/blobstore/common/codemode" +// "github.com/cubefs/cubefs/blobstore/common/resourcepool" +// ) +// func xxx() { +// pool := resourcepool.NewMemPool(nil) +// codeModeInfo := codemode.GetTactic(codemode.EC15p12) +// buffer, err := NewBuffer(1024, codeModeInfo, pool) +// if err != nil { +// // ... +// } +// // release +// defer buffer.Release() +// +// // release it in an anonymous func if you will resize +// defer func() { +// buffer.Release() +// }() +// +// io.Copy(buffer.DataBuf, io.Reader) +// shards := ec.Split(buffer.ECDataBuf) +// // ... +// +// buffer.Resize(10240) +// // ... +// } +package ec diff --git a/blobstore/common/ec/encoder.go b/blobstore/common/ec/encoder.go new file mode 100644 index 000000000..7111857b9 --- /dev/null +++ b/blobstore/common/ec/encoder.go @@ -0,0 +1,188 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package ec + +import ( + "errors" + "io" + + "github.com/klauspost/reedsolomon" + + "github.com/cubefs/cubefs/blobstore/common/codemode" + "github.com/cubefs/cubefs/blobstore/util/limit" + "github.com/cubefs/cubefs/blobstore/util/limit/count" +) + +const ( + defaultConcurrency = 100 +) + +// errors +var ( + ErrShortData = errors.New("short data") + ErrInvalidCodeMode = errors.New("invalid code mode") + ErrVerify = errors.New("shards verify failed") + ErrInvalidShards = errors.New("invalid shards") +) + +// Encoder normal ec encoder, implements all these functions +type Encoder interface { + // encode source data into shards, whatever normal ec or LRC + Encode(shards [][]byte) error + // reconstruct all missing shards, you should assign the missing or bad idx in shards + Reconstruct(shards [][]byte, badIdx []int) error + // only reconstruct data shards, you should assign the missing or bad idx in shards + ReconstructData(shards [][]byte, badIdx []int) error + // split source data into adapted shards size + Split(data []byte) ([][]byte, error) + // get data shards(No-Copy) + GetDataShards(shards [][]byte) [][]byte + // get parity shards(No-Copy) + GetParityShards(shards [][]byte) [][]byte + // get local shards(LRC model, No-Copy) + GetLocalShards(shards [][]byte) [][]byte + // get shards in an idc + GetShardsInIdc(shards [][]byte, idx int) [][]byte + // output source data into dst(io.Writer) + Join(dst io.Writer, shards [][]byte, outSize int) error + // verify parity shards with data shards + Verify(shards [][]byte) (bool, error) +} + +// Config ec encoder config +type Config struct { + CodeMode codemode.Tactic + EnableVerify bool + Concurrency int +} + +type encoder struct { + Config + pool limit.Limiter // concurrency pool + engine reedsolomon.Encoder +} + +// NewEncoder return an encoder which support normal EC or LRC +func NewEncoder(cfg Config) (Encoder, error) { + if !cfg.CodeMode.IsValid() { + return nil, ErrInvalidCodeMode + } + if cfg.Concurrency <= 0 { + cfg.Concurrency = defaultConcurrency + } + + engine, err := reedsolomon.New(cfg.CodeMode.N, cfg.CodeMode.M) + if err != nil { + return nil, err + } + pool := count.NewBlockingCount(cfg.Concurrency) + + if cfg.CodeMode.L != 0 { + localN := (cfg.CodeMode.N + cfg.CodeMode.M) / cfg.CodeMode.AZCount + localM := cfg.CodeMode.L / cfg.CodeMode.AZCount + localEngine, err := reedsolomon.New(localN, localM) + if err != nil { + return nil, err + } + return &lrcEncoder{ + Config: cfg, + pool: pool, + engine: engine, + localEngine: localEngine, + }, nil + } + + return &encoder{ + Config: cfg, + pool: pool, + engine: engine, + }, nil +} + +func (e *encoder) Encode(shards [][]byte) error { + e.pool.Acquire() + defer e.pool.Release() + + if err := e.engine.Encode(shards); err != nil { + return err + } + if e.EnableVerify { + ok, err := e.engine.Verify(shards) + if err != nil { + return err + } + if !ok { + return ErrVerify + } + } + return nil +} + +func (e *encoder) Verify(shards [][]byte) (bool, error) { + e.pool.Acquire() + defer e.pool.Release() + return e.engine.Verify(shards) +} + +func (e *encoder) Reconstruct(shards [][]byte, badIdx []int) error { + initBadShards(shards, badIdx) + e.pool.Acquire() + defer e.pool.Release() + return e.engine.Reconstruct(shards) +} + +func (e *encoder) ReconstructData(shards [][]byte, badIdx []int) error { + initBadShards(shards, badIdx) + e.pool.Acquire() + defer e.pool.Release() + return e.engine.ReconstructData(shards) +} + +func (e *encoder) Split(data []byte) ([][]byte, error) { + return e.engine.Split(data) +} + +func (e *encoder) GetDataShards(shards [][]byte) [][]byte { + return shards[:e.CodeMode.N] +} + +func (e *encoder) GetParityShards(shards [][]byte) [][]byte { + return shards[e.CodeMode.N:] +} + +func (e *encoder) GetLocalShards(shards [][]byte) [][]byte { + return nil +} + +func (e *encoder) GetShardsInIdc(shards [][]byte, idx int) [][]byte { + n, m := e.CodeMode.N, e.CodeMode.M + idcCnt := e.CodeMode.AZCount + + localN, localM := n/idcCnt, m/idcCnt + + return append(shards[idx*localN:(idx+1)*localN], shards[n+localM*idx:n+localM*(idx+1)]...) +} + +func (e *encoder) Join(dst io.Writer, shards [][]byte, outSize int) error { + return e.engine.Join(dst, shards, outSize) +} + +func initBadShards(shards [][]byte, badIdx []int) { + for _, i := range badIdx { + if shards[i] != nil && len(shards[i]) != 0 && cap(shards[i]) > 0 { + shards[i] = shards[i][:0] + } + } +} diff --git a/blobstore/common/ec/encoder_test.go b/blobstore/common/ec/encoder_test.go new file mode 100644 index 000000000..d113fad9d --- /dev/null +++ b/blobstore/common/ec/encoder_test.go @@ -0,0 +1,199 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package ec + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/cubefs/cubefs/blobstore/common/codemode" +) + +var srcData = []byte("Hello world") + +func TestEncoderNew(t *testing.T) { + { + _, err := NewEncoder(Config{CodeMode: codemode.Tactic{}}) + assert.ErrorIs(t, err, ErrInvalidCodeMode) + } + { + _, err := NewEncoder(Config{CodeMode: codemode.EC15P12.Tactic()}) + assert.NoError(t, err) + _, err = NewEncoder(Config{CodeMode: codemode.EC16P20L2.Tactic()}) + assert.NoError(t, err) + } +} + +func TestEncoder(t *testing.T) { + cfg := Config{ + CodeMode: codemode.EC15P12.Tactic(), + EnableVerify: true, + Concurrency: 10, + } + encoder, err := NewEncoder(cfg) + assert.NoError(t, err) + + // source data split + shards, err := encoder.Split(srcData) + assert.NoError(t, err) + + // encode data + err = encoder.Encode(shards) + assert.NoError(t, err) + wbuff := bytes.NewBuffer(make([]byte, 0)) + err = encoder.Join(wbuff, shards, len(srcData)) + assert.NoError(t, err) + assert.Equal(t, srcData, wbuff.Bytes()) + + dataShards := encoder.GetDataShards(shards) + // set one data shards broken + for i := range dataShards[0] { + dataShards[0][i] = 222 + } + // reconstruct data and check + err = encoder.ReconstructData(shards, []int{0}) + assert.NoError(t, err) + wbuff = bytes.NewBuffer(make([]byte, 0)) + err = encoder.Join(wbuff, shards, len(srcData)) + assert.NoError(t, err) + assert.Equal(t, srcData, wbuff.Bytes()) + + // reconstruct shard and check + parityShards := encoder.GetParityShards(shards) + for i := range parityShards[1] { + parityShards[1][i] = 11 + } + err = encoder.Reconstruct(shards, []int{cfg.CodeMode.N + 1}) + assert.NoError(t, err) + ok, err := encoder.Verify(shards) + assert.NoError(t, err) + assert.True(t, ok) + wbuff = bytes.NewBuffer(make([]byte, 0)) + err = encoder.Join(wbuff, shards, len(srcData)) + assert.NoError(t, err) + assert.Equal(t, srcData, wbuff.Bytes()) + + ls := encoder.GetLocalShards(shards) + assert.Equal(t, 0, len(ls)) + si := encoder.GetShardsInIdc(shards, 0) + assert.Equal(t, (cfg.CodeMode.N+cfg.CodeMode.M)/3, len(si)) +} + +func TestLrcEncoder(t *testing.T) { + cfg := Config{ + CodeMode: codemode.EC6P10L2.Tactic(), + EnableVerify: true, + } + encoder, err := NewEncoder(cfg) + assert.NoError(t, err) + + // source data split + shards, err := encoder.Split(srcData) + assert.NoError(t, err) + { + enoughBuff := make([]byte, 1<<10) + copy(enoughBuff, srcData) + enoughBuff = enoughBuff[:len(srcData)] + _, err := encoder.Split(enoughBuff) + assert.NoError(t, err) + } + + invalidShards := shards[:len(shards)-1] + assert.ErrorIs(t, encoder.Encode(invalidShards), ErrInvalidShards) + assert.ErrorIs(t, encoder.Encode(nil), ErrInvalidShards) + + // encode data + err = encoder.Encode(shards) + assert.NoError(t, err) + wbuff := bytes.NewBuffer(make([]byte, 0)) + err = encoder.Join(wbuff, shards, len(srcData)) + assert.NoError(t, err) + assert.Equal(t, srcData, wbuff.Bytes()) + + dataShards := encoder.GetDataShards(shards) + // set one data shard broken + for i := range dataShards[0] { + dataShards[0][i] = 222 + } + // reconstruct data and check + err = encoder.ReconstructData(shards, []int{0}) + assert.NoError(t, err) + wbuff = bytes.NewBuffer(make([]byte, 0)) + err = encoder.Join(wbuff, shards, len(srcData)) + assert.NoError(t, err) + assert.Equal(t, srcData, wbuff.Bytes()) + + // Local reconstruct shard and check + localShardsInIdc := encoder.GetShardsInIdc(shards, 0) + for idx := 0; idx < len(localShardsInIdc); idx++ { + // set wrong data + for i := range localShardsInIdc[idx] { + localShardsInIdc[idx][i] = 11 + } + // check must be false when a shard broken + ok, err := encoder.Verify(shards) + assert.NoError(t, err) + assert.False(t, ok) + + err = encoder.Reconstruct(localShardsInIdc, []int{idx}) + assert.NoError(t, err) + ok, err = encoder.Verify(shards) + assert.NoError(t, err) + assert.True(t, ok) + } + + // global reconstruct shard and check + dataShards = encoder.GetDataShards(shards) + parityShards := encoder.GetParityShards(shards) + badIdxs := make([]int, 0) + for i := 0; i < cfg.CodeMode.M; i++ { + if i%2 == 0 { + badIdxs = append(badIdxs, i) + // set wrong data + if i < len(dataShards) { + for j := range dataShards[i] { + dataShards[i][j] = 222 + } + } + } else { + badIdxs = append(badIdxs, cfg.CodeMode.N+i) + // set wrong data + for j := range parityShards[i] { + parityShards[i][j] = 222 + } + } + } + // add a local broken + for j := range shards[cfg.CodeMode.N+cfg.CodeMode.M+1] { + shards[cfg.CodeMode.N+cfg.CodeMode.M+1][j] = 222 + } + badIdxs = append(badIdxs, cfg.CodeMode.N+cfg.CodeMode.M+1) + err = encoder.Reconstruct(shards, badIdxs) + assert.NoError(t, err) + ok, err := encoder.Verify(shards) + assert.NoError(t, err) + assert.True(t, ok) + wbuff = bytes.NewBuffer(make([]byte, 0)) + err = encoder.Join(wbuff, shards, len(srcData)) + assert.NoError(t, err) + assert.Equal(t, srcData, wbuff.Bytes()) + + ls := encoder.GetLocalShards(shards) + assert.Equal(t, cfg.CodeMode.L, len(ls)) + si := encoder.GetShardsInIdc(shards, 0) + assert.Equal(t, (cfg.CodeMode.N+cfg.CodeMode.M+cfg.CodeMode.L)/cfg.CodeMode.AZCount, len(si)) +} diff --git a/blobstore/common/ec/lrcencoder.go b/blobstore/common/ec/lrcencoder.go new file mode 100644 index 000000000..cf52e4b19 --- /dev/null +++ b/blobstore/common/ec/lrcencoder.go @@ -0,0 +1,236 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package ec + +import ( + "context" + "io" + + "github.com/klauspost/reedsolomon" + + "github.com/cubefs/cubefs/blobstore/util/errors" + "github.com/cubefs/cubefs/blobstore/util/limit" + "github.com/cubefs/cubefs/blobstore/util/task" +) + +type lrcEncoder struct { + Config + pool limit.Limiter // concurrency pool + engine reedsolomon.Encoder + localEngine reedsolomon.Encoder +} + +func (e *lrcEncoder) Encode(shards [][]byte) error { + if len(shards) != (e.CodeMode.N + e.CodeMode.M + e.CodeMode.L) { + return ErrInvalidShards + } + e.pool.Acquire() + defer e.pool.Release() + + // firstly, do global ec encode + if err := e.engine.Encode(shards[:e.CodeMode.N+e.CodeMode.M]); err != nil { + return errors.Info(err, "lrcEncoder.Encode global failed") + } + if e.EnableVerify { + ok, err := e.engine.Verify(shards[:e.CodeMode.N+e.CodeMode.M]) + if err != nil { + return errors.Info(err, "lrcEncoder.Encode global verify failed") + } + if !ok { + return ErrVerify + } + } + + tasks := make([]func() error, 0, e.CodeMode.AZCount) + // secondly, do local ec encode + for i := 0; i < e.CodeMode.AZCount; i++ { + localShards := e.GetShardsInIdc(shards, i) + tasks = append(tasks, func() error { + if err := e.localEngine.Encode(localShards); err != nil { + return errors.Info(err, "lrcEncoder.Encode local failed") + } + if e.EnableVerify { + ok, err := e.localEngine.Verify(localShards) + if err != nil { + return errors.Info(err, "lrcEncoder.Encode local verify failed") + } + if !ok { + return ErrVerify + } + } + return nil + }) + } + if err := task.Run(context.Background(), tasks...); err != nil { + return err + } + + return nil +} + +type verifyError struct { + error + verified bool +} + +func (e *lrcEncoder) Verify(shards [][]byte) (bool, error) { + e.pool.Acquire() + defer e.pool.Release() + + ok, err := e.engine.Verify(shards[:e.CodeMode.N+e.CodeMode.M]) + if !ok || err != nil { + if err != nil { + err = errors.Info(err, "lrcEncoder.Verify global shards failed") + } + return ok, err + } + + tasks := make([]func() error, 0, e.CodeMode.AZCount) + for i := 0; i < e.CodeMode.AZCount; i++ { + localShards := e.GetShardsInIdc(shards, i) + tasks = append(tasks, func() error { + ok, err := e.localEngine.Verify(localShards) + if !ok || err != nil { + if err != nil { + err = errors.Info(err, "lrcEncoder.Verify local shards failed") + } + return verifyError{error: err, verified: ok} + } + return nil + }) + } + if err := task.Run(context.Background(), tasks...); err != nil { + if verifyErr, succ := err.(verifyError); succ { + return verifyErr.verified, verifyErr.error + } + return false, err + } + + return true, nil +} + +func (e *lrcEncoder) Reconstruct(shards [][]byte, badIdx []int) error { + globalBadIdx := make([]int, 0) + for _, i := range badIdx { + if i < e.CodeMode.N+e.CodeMode.M { + globalBadIdx = append(globalBadIdx, i) + } + } + initBadShards(shards, globalBadIdx) + e.pool.Acquire() + defer e.pool.Release() + + // use local ec reconstruct, saving network bandwidth + if len(shards) == (e.CodeMode.N+e.CodeMode.M+e.CodeMode.L)/e.CodeMode.AZCount { + if err := e.localEngine.Reconstruct(shards); err != nil { + return errors.Info(err, "lrcEncoder.Reconstruct local ec reconstruct failed") + } + return nil + } + + // can't reconstruct from local ec + // firstly, use global ec reconstruct + if err := e.engine.Reconstruct(shards[:e.CodeMode.N+e.CodeMode.M]); err != nil { + return errors.Info(err, "lrcEncoder.Reconstruct global ec reconstruct failed") + } + + // secondly, check if need to reconstruct the local shards + localRestructs := make(map[int][]int) + for _, i := range badIdx { + if i >= (e.CodeMode.N + e.CodeMode.M) { + idcIdx := (i - e.CodeMode.N - e.CodeMode.M) * e.CodeMode.AZCount / e.CodeMode.L + badIdx := idcIdx + (e.CodeMode.N+e.CodeMode.M)/e.CodeMode.AZCount - 1 + if _, ok := localRestructs[idcIdx]; !ok { + localRestructs[idcIdx] = make([]int, 0) + } + localRestructs[idcIdx] = append(localRestructs[idcIdx], badIdx) + } + } + + tasks := make([]func() error, 0, len(localRestructs)) + for idx, badIdx := range localRestructs { + localShards := e.GetShardsInIdc(shards, idx) + initBadShards(localShards, badIdx) + tasks = append(tasks, func() error { + return e.localEngine.Reconstruct(localShards) + }) + } + if err := task.Run(context.Background(), tasks...); err != nil { + return errors.Info(err, "lrcEncoder.Reconstruct local ec reconstruct after global ec failed") + } + return nil +} + +func (e *lrcEncoder) ReconstructData(shards [][]byte, badIdx []int) error { + initBadShards(shards, badIdx) + shards = shards[:e.CodeMode.N+e.CodeMode.M] + e.pool.Acquire() + defer e.pool.Release() + return e.engine.ReconstructData(shards) +} + +func (e *lrcEncoder) Split(data []byte) ([][]byte, error) { + shards, err := e.engine.Split(data) + if err != nil { + return nil, err + } + shardN, shardLen := len(shards), len(shards[0]) + if cap(data) >= (e.CodeMode.L+shardN)*shardLen { + if cap(data) > len(data) { + data = data[:cap(data)] + } + for i := 0; i < e.CodeMode.L; i++ { + shards = append(shards, data[(shardN+i)*shardLen:(shardN+i+1)*shardLen]) + } + } else { + for i := 0; i < e.CodeMode.L; i++ { + shards = append(shards, make([]byte, shardLen)) + } + } + return shards, nil +} + +func (e *lrcEncoder) GetDataShards(shards [][]byte) [][]byte { + return shards[:e.CodeMode.N] +} + +func (e *lrcEncoder) GetParityShards(shards [][]byte) [][]byte { + return shards[e.CodeMode.N : e.CodeMode.N+e.CodeMode.M] +} + +func (e *lrcEncoder) GetLocalShards(shards [][]byte) [][]byte { + return shards[e.CodeMode.N+e.CodeMode.M:] +} + +func (e *lrcEncoder) GetShardsInIdc(shards [][]byte, idx int) [][]byte { + n, m, l := e.CodeMode.N, e.CodeMode.M, e.CodeMode.L + idcCnt := e.CodeMode.AZCount + localN, localM, localL := n/idcCnt, m/idcCnt, l/idcCnt + + localShardsInIdc := make([][]byte, localN+localM+localL) + + shardsN := shards[idx*localN : (idx+1)*localN] + shardsM := shards[(n + localM*idx):(n + localM*(idx+1))] + shardsL := shards[(n + m + localL*idx):(n + m + localL*(idx+1))] + + copy(localShardsInIdc, shardsN) + copy(localShardsInIdc[localN:], shardsM) + copy(localShardsInIdc[localN+localM:], shardsL) + return localShardsInIdc +} + +func (e *lrcEncoder) Join(dst io.Writer, shards [][]byte, outSize int) error { + return e.engine.Join(dst, shards[:(e.CodeMode.N+e.CodeMode.M)], outSize) +} diff --git a/blobstore/common/memcache/memcache.go b/blobstore/common/memcache/memcache.go new file mode 100644 index 000000000..b95f7b2da --- /dev/null +++ b/blobstore/common/memcache/memcache.go @@ -0,0 +1,52 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package memcache + +import ( + "context" + + lru "github.com/hashicorp/golang-lru" +) + +// MemCache memory cache with lru-2q +// ctx with logging +type MemCache struct { + ctx context.Context + cache *lru.TwoQueueCache +} + +// NewMemCache new memory cache with capacity size +func NewMemCache(ctx context.Context, size int) (*MemCache, error) { + cache, err := lru.New2Q(size) + if err != nil { + return nil, err + } + return &MemCache{ctx: ctx, cache: cache}, nil +} + +// Get get by key, return nil if key doesn't exist or value is nil +func (mc *MemCache) Get(key interface{}) interface{} { + value, ok := mc.cache.Get(key) + if !ok { + return nil + } + + return value +} + +// Set set to key with value, delete key if value is nil +func (mc *MemCache) Set(key interface{}, value interface{}) { + mc.cache.Add(key, value) +} diff --git a/blobstore/common/memcache/memcache_test.go b/blobstore/common/memcache/memcache_test.go new file mode 100644 index 000000000..0450552d0 --- /dev/null +++ b/blobstore/common/memcache/memcache_test.go @@ -0,0 +1,82 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package memcache_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/common/memcache" +) + +func TestNewMemCache(t *testing.T) { + _, err := memcache.NewMemCache(context.TODO(), 1024) + require.Nil(t, err) +} + +func TestMemCacheGetSet(t *testing.T) { + mc, err := memcache.NewMemCache(context.TODO(), 4) + require.Nil(t, err) + + key, value := "foo", "bar" + mc.Set(key, value) + + v := mc.Get(key) + val, ok := v.(string) + require.True(t, ok) + require.Equal(t, value, val) +} + +func TestMemCacheDel(t *testing.T) { + mc, err := memcache.NewMemCache(context.TODO(), 4) + require.Nil(t, err) + + key, value := "foo", "bar" + mc.Set(key, value) + + v := mc.Get(key) + val, ok := v.(string) + require.True(t, ok) + require.Equal(t, value, val) + + mc.Set(key, nil) + v = mc.Get(key) + require.Nil(t, v) +} + +func BenchmarkNewMemCache(b *testing.B) { + for ii := 0; ii < b.N; ii++ { + memcache.NewMemCache(context.TODO(), 4) + } +} + +func BenchmarkMemCacheGet(b *testing.B) { + mc, _ := memcache.NewMemCache(context.TODO(), 4) + key, value := "foo", "bar" + mc.Set(key, value) + for ii := 0; ii < b.N; ii++ { + mc.Get(key) + } +} + +func BenchmarkMemCacheSet(b *testing.B) { + mc, _ := memcache.NewMemCache(context.TODO(), 4) + key, value := "foo", "bar" + for ii := 0; ii < b.N; ii++ { + mc.Set(key, value) + } +} diff --git a/blobstore/common/profile/doc.go b/blobstore/common/profile/doc.go new file mode 100644 index 000000000..e425d4cdc --- /dev/null +++ b/blobstore/common/profile/doc.go @@ -0,0 +1,44 @@ +// Copyright 2022 The CubeFS Authors. +// application memory data controller and metric exporter in localhost. +// +// With tag `noprofile` to avoid generating scripts. +// +// Note: +// 1. You can register http handler to profile multiplexer to control. +// 2. You may use environment to set variables, like `bind_addr`, `metric_exporter_labels`. +// 3. You cannot access the localhost service before profile initialized. +// +// Example: +// +// package main +// +// import ( +// "net/http" +// +// // 1. you can using default handler in profile defined +// // 2. you can register handler to profile in other module +// "github.com/cubefs/cubefs/blobstore/common/profile" +// "github.com/cubefs/cubefs/blobstore/common/rpc" +// +// ) +// +// func registerHandler() { +// profile.HandleFunc("GET","/myself/controller", func(ctx *rpc.Context) { +// // todo something +// }, rpc.ServerOption...) +// } +// +// func main() { +// ph := profile.NewProfileHandler("127.0.0.1:8888") +// httpServer := &http.Server{ +// Addr: "127.0.0.1:8888", +// Handler: rpc.MiddlewareHandlerWith(rpc.DefaultRouter, ph), +// } +// log.Info("Server is running at", "127.0.0.1:8888") +// go func() { +// err = httpServer.ListenAndServe() +// require.NoError(t, err) +// }() +// registerHandler() +// } +package profile diff --git a/blobstore/common/profile/gen.go b/blobstore/common/profile/gen.go new file mode 100644 index 000000000..b34761d1c --- /dev/null +++ b/blobstore/common/profile/gen.go @@ -0,0 +1,119 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package profile + +import ( + "fmt" + "log" + "os" + "path/filepath" + "runtime/pprof" + "strings" + "time" + + "gopkg.in/yaml.v2" +) + +// MEConfig metric config with yaml +type MEConfig struct { + URL string `yaml:"url"` + Labels map[string]string `yaml:"labels"` +} + +func genMetricExporter() { + fn := os.Args[0] + suffixMetricExporter + f, err := os.OpenFile(fn, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + log.Println(err) + return + } + defer f.Close() + + labels := map[string]string{"scrapejob": "profile"} + fields := strings.Split(os.Getenv(envMetricExporterLabels), ",") + for _, item := range fields { + s := strings.Split(item, "=") + if len(s) != 2 { + continue + } + labels[s[0]] = s[1] + } + + content, err := yaml.Marshal(MEConfig{ + URL: profileAddr + "/metrics", + Labels: labels, + }) + if err != nil { + log.Println(err) + return + } + + f.Write(content) + f.Sync() +} + +func genDumpScript() { + fn := os.Args[0] + suffixDumpScript + f, err := os.OpenFile(fn, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o755) + if err != nil { + log.Println(err) + return + } + defer f.Close() + + fmt.Fprint(f, "#!/bin/sh\n") + fmt.Fprint(f, "echo $(date) dumping runtime status\n") + fmt.Fprint(f, "set -x\n") + fmt.Fprintf(f, "DIR=profile/$(date '+%%Y%%m%%d%%H%%M%%S')\n") + fmt.Fprintf(f, "PREFIX=$DIR/%s_%d_%s_\n", + filepath.Base(os.Args[0]), os.Getpid(), time.Now().Format("20060102_150405")) + fmt.Fprint(f, "mkdir -p ${DIR}\n") + fmt.Fprintf(f, "curl -sS '%s/metrics' -o ${PREFIX}metrics\n", profileAddr) + fmt.Fprintf(f, "curl -sS '%s/debug/vars?seconds=5' -o ${PREFIX}vars\n", profileAddr) + for _, p := range pprof.Profiles() { + name := p.Name() + switch name { + case "heap", "allocs": + fmt.Fprintf(f, "curl -sS '%s/debug/pprof/%s' -o ${PREFIX}%s\n", + profileAddr, name, name) + case "goroutine": + fmt.Fprintf(f, "curl -sS '%s/debug/pprof/%s?debug=1' -o ${PREFIX}%s_debug_1\n", + profileAddr, name, name) + fmt.Fprintf(f, "curl -sS '%s/debug/pprof/%s?debug=2' -o ${PREFIX}%s_debug_2\n", + profileAddr, name, name) + fallthrough + default: + fmt.Fprintf(f, "curl -sS '%s/debug/pprof/%s?seconds=5' -o ${PREFIX}%s\n", + profileAddr, name, name) + } + } + fmt.Fprintf(f, "curl -sS '%s/debug/pprof/profile?seconds=5' -o ${PREFIX}profile\n", profileAddr) + fmt.Fprintf(f, "curl -sS '%s/debug/pprof/trace?seconds=5' -o ${PREFIX}trace\n", profileAddr) + fmt.Fprint(f, "tar -cvzf ${DIR}.tar.gz ${DIR}\n") + f.Sync() +} + +func genListenAddr() { + fn := os.Args[0] + suffixListenAddr + f, err := os.OpenFile(fn, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + log.Println(err) + return + } + defer f.Close() + + f.WriteString(strings.TrimPrefix(profileAddr, "http://")) + f.Sync() +} diff --git a/blobstore/common/profile/profile.go b/blobstore/common/profile/profile.go new file mode 100644 index 000000000..5d1f10aca --- /dev/null +++ b/blobstore/common/profile/profile.go @@ -0,0 +1,228 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package profile + +import ( + "expvar" + "fmt" + "net/http" + "net/http/pprof" + "os" + "runtime" + "sort" + "strings" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + + "github.com/cubefs/cubefs/blobstore/common/rpc" +) + +const ( + envMetricExporterLabels = "profile_metric_exporter_labels" + + suffixListenAddr = "_profile_listen_addr" + suffixMetricExporter = "_profile_metric_exporter.yaml" + suffixDumpScript = "_profile_dump.sh" +) + +var ( + profileAddr string + httpRouter = rpc.New() + startTime = time.Now() + + uptime = prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: "profile", + Subsystem: "process", + Name: "uptime_seconds", + Help: "Process uptime seconds.", + }, func() (v float64) { + return time.Since(startTime).Seconds() + }) + + gomaxprocs = prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: "profile", + Subsystem: "process", + Name: "maxprocs", + Help: "Go runtime.GOMAXPROCS.", + }, func() (v float64) { + return float64(runtime.GOMAXPROCS(-1)) + }) + + router = new(route) +) + +type route struct { + mu sync.Mutex + + vars []string + pprof []string + metrics []string + users []string +} + +func (r *route) AddUserPath(path string) { + r.mu.Lock() + r.users = append(r.users, path) + sort.Strings(r.users) + r.mu.Unlock() +} + +func (r *route) String() string { + r.mu.Lock() + sb := strings.Builder{} + sb.WriteString("usage:\n\t/\n\n") + sb.WriteString("vars:\n") + sb.WriteString(fmt.Sprintf("\t%s\n\n", strings.Join(r.vars, "\n\t"))) + sb.WriteString("pprof:\n") + sb.WriteString(fmt.Sprintf("\t%s\n\n", strings.Join(r.pprof, "\n\t"))) + sb.WriteString("metrics:\n") + sb.WriteString(fmt.Sprintf("\t%s\n\n", strings.Join(r.metrics, "\n\t"))) + if len(r.users) > 0 { + sb.WriteString("users:\n") + sb.WriteString(fmt.Sprintf("\t%s\n\n", strings.Join(r.users, "\n\t"))) + } + r.mu.Unlock() + return sb.String() +} + +type profileHandler struct{} + +func (ph *profileHandler) Handler(w http.ResponseWriter, r *http.Request, f func(http.ResponseWriter, *http.Request)) { + path := r.URL.Path + addr := r.RemoteAddr + handle, _, _ := httpRouter.Router.Lookup(r.Method, path) + if handle != nil { + // to support local host, need not authentication + if strings.HasPrefix(addr, "localhost") || strings.HasPrefix(addr, "127.0.0.1") { + httpRouter.ServeHTTP(w, r) + return + } + // todo Authentication manager authority + httpRouter.ServeHTTP(w, r) + return + } + f(w, r) +} + +func init() { + prometheus.MustRegister(uptime) + prometheus.MustRegister(gomaxprocs) + + expvar.NewString("GoVersion").Set(runtime.Version()) + expvar.NewInt("NumCPU").Set(int64(runtime.NumCPU())) + expvar.NewInt("Pid").Set(int64(os.Getpid())) + expvar.NewInt("StartTime").Set(startTime.Unix()) + + expvar.Publish("Uptime", expvar.Func(func() interface{} { return time.Since(startTime) / time.Second })) + expvar.Publish("NumGoroutine", expvar.Func(func() interface{} { return runtime.NumGoroutine() })) +} + +func NewProfileHandler(addr string) rpc.ProgressHandler { + // without profile setting + if without { + return nil + } + + ph := &profileHandler{} + + httpRouter.Router.HandlerFunc(http.MethodGet, "/", index) + httpRouter.Router.HandlerFunc(http.MethodGet, "/debug/pprof/", pprof.Index) + httpRouter.Router.HandlerFunc(http.MethodGet, "/debug/pprof/:key", secondIndex) + httpRouter.Router.HandlerFunc(http.MethodGet, "/debug/vars", expvar.Handler().ServeHTTP) + httpRouter.Router.HandlerFunc(http.MethodGet, "/metrics", promhttp.Handler().ServeHTTP) + httpRouter.Router.HandlerFunc(http.MethodGet, "/debug/var/", oneExpvar) + httpRouter.Router.HandlerFunc(http.MethodGet, "/debug/var/:key", oneExpvar) + + router.mu.Lock() + router.vars = []string{ + "/debug/vars", + "/debug/var/*", + } + + router.pprof = []string{ + "/debug/pprof/", + "/debug/pprof/cmdline", + "/debug/pprof/profile", + "/debug/pprof/symbol", + "/debug/pprof/trace", + } + router.metrics = []string{ + "/metrics", + } + router.mu.Unlock() + + if strings.HasPrefix(addr, ":") { + profileAddr = "http://127.0.0.1" + addr + } else { + profileAddr = "http://" + addr + } + + // do not gen files in `go test` + if !strings.HasSuffix(os.Args[0], ".test") { + genListenAddr() + genDumpScript() + genMetricExporter() + } + return ph +} + +// handle path /debug/ , show usage +func index(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, router.String()) +} + +// handler path /debug/var/, get one expvar +func oneExpvar(w http.ResponseWriter, r *http.Request) { + key := r.URL.Path[len("/debug/var/"):] + if key == "" { + w.WriteHeader(http.StatusBadRequest) + return + } + val := expvar.Get(key) + if val == nil { + w.WriteHeader(http.StatusNotFound) + return + } + w.Write([]byte(val.String())) +} + +func secondIndex(w http.ResponseWriter, r *http.Request) { + path := r.URL.Path + splits := strings.Split(path, "/") + switch splits[len(splits)-1] { + case "cmdline": + pprof.Cmdline(w, r) + case "profile": + pprof.Profile(w, r) + case "symbol": + pprof.Symbol(w, r) + case "trace": + pprof.Trace(w, r) + default: + pprof.Index(w, r) + } +} + +// HandleFunc register handler func to profile serve multiplexer. +func HandleFunc(method, pattern string, handler rpc.HandlerFunc, opts ...rpc.ServerOption) { + if without { + return + } + router.AddUserPath(pattern) + httpRouter.Handle(method, pattern, handler, opts...) +} diff --git a/blobstore/common/profile/profile_test.go b/blobstore/common/profile/profile_test.go new file mode 100644 index 000000000..1846de184 --- /dev/null +++ b/blobstore/common/profile/profile_test.go @@ -0,0 +1,120 @@ +package profile + +import ( + "expvar" + "io/ioutil" + "net" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +func TestProfileBase(t *testing.T) { + i := expvar.NewInt("int_key") + i.Set(100) + + HandleFunc(http.MethodGet, "/test/before", func(ctx *rpc.Context) { + ctx.RespondStatus(800) + ctx.Writer.Write([]byte("ok")) + }) + + var defaultHandleAddr string + initDone := make(chan bool) + go func() { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defaultHandleAddr = "http://" + ln.Addr().String() + close(initDone) + err = http.Serve(ln, nil) + require.NoError(t, err) + }() + <-initDone + + // default handle 404 + resp, err := http.Get(defaultHandleAddr + "/debug/var/") + require.NoError(t, err) + require.Equal(t, 404, resp.StatusCode) + resp.Body.Close() + resp, err = http.Get(defaultHandleAddr + "/metrics") + require.NoError(t, err) + require.Equal(t, 404, resp.StatusCode) + resp.Body.Close() + + defaultRouter := rpc.DefaultRouter + ph := NewProfileHandler("127.0.0.1:8888") + httpServer := &http.Server{ + Addr: "127.0.0.1:8888", + Handler: rpc.MiddlewareHandlerWith(defaultRouter, ph), + } + log.Info("Server is running at", "127.0.0.1:8888") + go func() { + err = httpServer.ListenAndServe() + require.NoError(t, err) + }() + time.Sleep(time.Millisecond * 10) + // profile handle 200 + resp, err = http.Get(profileAddr + "/") + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode) + buf := make([]byte, 1<<20) + n, _ := resp.Body.Read(buf) + t.Log(string(buf[:n])) + resp.Body.Close() + + resp, err = http.Get(profileAddr + "/debug/vars") + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode) + resp.Body.Close() + resp, err = http.Get(profileAddr + "/debug/pprof/") + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode) + resp.Body.Close() + + // get one expvar + resp, err = http.Get(profileAddr + "/debug/var/") + require.NoError(t, err) + require.Equal(t, 400, resp.StatusCode) + resp.Body.Close() + resp, err = http.Get(profileAddr + "/debug/var/not_exist_key") + require.NoError(t, err) + require.Equal(t, 404, resp.StatusCode) + resp.Body.Close() + resp, err = http.Get(profileAddr + "/debug/var/int_key") + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode) + data, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, "100", string(data)) + resp.Body.Close() + + resp, err = http.Get(profileAddr + "/test/before") + require.NoError(t, err) + require.Equal(t, 800, resp.StatusCode) + data, err = ioutil.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, "ok", string(data)) + resp.Body.Close() + + HandleFunc(http.MethodGet, "/test/handlfunc", func(ctx *rpc.Context) { + ctx.RespondStatus(900) + ctx.Writer.Write([]byte("ok")) + }) + resp, err = http.Get(profileAddr + "/test/handlfunc") + require.NoError(t, err) + require.Equal(t, 900, resp.StatusCode) + data, err = ioutil.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, "ok", string(data)) + resp.Body.Close() + + { + genListenAddr() + genDumpScript() + genMetricExporter() + } +} diff --git a/blobstore/common/profile/profile_with.go b/blobstore/common/profile/profile_with.go new file mode 100644 index 000000000..0630ddaa3 --- /dev/null +++ b/blobstore/common/profile/profile_with.go @@ -0,0 +1,20 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +//go:build !noprofile +// +build !noprofile + +package profile + +var without = false diff --git a/blobstore/common/profile/profile_without.go b/blobstore/common/profile/profile_without.go new file mode 100644 index 000000000..28e55acbd --- /dev/null +++ b/blobstore/common/profile/profile_without.go @@ -0,0 +1,20 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +//go:build noprofile +// +build noprofile + +package profile + +var without = true diff --git a/blobstore/common/redis/redis.go b/blobstore/common/redis/redis.go new file mode 100644 index 000000000..c9bd95296 --- /dev/null +++ b/blobstore/common/redis/redis.go @@ -0,0 +1,123 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package redis + +import ( + "context" + "encoding/json" + "errors" + "time" + + xredis "github.com/go-redis/redis/v8" +) + +var ( + // ErrNotFound redis key is not found + ErrNotFound = errors.New("key not found") + // ErrEncodeFailed encode failed + ErrEncodeFailed = errors.New("encode failed") + // ErrDecodeFailed decode failed + ErrDecodeFailed = errors.New("decode failed") +) + +// CodeC encode and decode value get/set redis +type CodeC interface { + Encode() ([]byte, error) + Decode(data []byte) error +} + +// ClusterConfig redis cluster config +// TODO: add more options like redis.ClusterOptions +type ClusterConfig struct { + Addrs []string `json:"addrs"` + Username string `json:"username"` + Password string `json:"password"` + + // time seconds + DialTimeout int `json:"dial_timeout"` + ReadTimeout int `json:"read_timeout"` + WriteTimeout int `json:"write_timeout"` +} + +// ClusterClient some as xredis.ClusterClient +// TODO: more redis commands to implement +type ClusterClient struct { + *xredis.ClusterClient +} + +// NewClusterClient new a redis cluster client +func NewClusterClient(config *ClusterConfig) *ClusterClient { + return &ClusterClient{ + ClusterClient: xredis.NewClusterClient(&xredis.ClusterOptions{ + Addrs: config.Addrs[:], + Username: config.Username, + Password: config.Password, + DialTimeout: time.Duration(config.DialTimeout) * time.Second, + ReadTimeout: time.Duration(config.ReadTimeout) * time.Second, + WriteTimeout: time.Duration(config.WriteTimeout) * time.Second, + }), + } +} + +// Get value of key in redis, return no such key if value is nil +func (cc *ClusterClient) Get(ctx context.Context, key string, value interface{}) error { + var ( + valBytes []byte + err error + ) + + valBytes, err = cc.ClusterClient.Get(ctx, key).Bytes() + if err == xredis.Nil { + return ErrNotFound + } + if err != nil { + return err + } + + if val, ok := value.(CodeC); ok { + err = val.Decode(valBytes) + } else { + err = json.Unmarshal(valBytes, value) + } + if err != nil { + return ErrDecodeFailed + } + + return nil +} + +// Set value of key to redis +func (cc *ClusterClient) Set(ctx context.Context, key string, value interface{}, expiration time.Duration) error { + var ( + valBytes []byte + err error + ) + + if val, ok := value.(CodeC); ok { + valBytes, err = val.Encode() + } else { + valBytes, err = json.Marshal(value) + } + if err != nil { + return ErrEncodeFailed + } + + err = cc.ClusterClient.Set(ctx, key, valBytes, expiration).Err() + if err != nil { + return err + } + + return nil +} diff --git a/blobstore/common/redis/redis_test.go b/blobstore/common/redis/redis_test.go new file mode 100644 index 000000000..86a85c746 --- /dev/null +++ b/blobstore/common/redis/redis_test.go @@ -0,0 +1,228 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package redis_test + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "errors" + "testing" + + "github.com/alicebob/miniredis/v2" + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/common/redis" +) + +const ( + testkey = "redis/test/key" +) + +var ( + mr *miniredis.Miniredis + cli *redis.ClusterClient + + errJSONGet = errors.New("json get errror") + errJSONSet = errors.New("json set errror") +) + +func init() { + mr, _ = miniredis.Run() + cli = redis.NewClusterClient(&redis.ClusterConfig{ + Addrs: []string{mr.Addr()}, + }) +} + +func TestRedis(t *testing.T) { + type Foo struct { + Foo string `json:"foo"` + } + val := Foo{ + Foo: "foo", + } + defer cli.Del(context.TODO(), testkey) + + err := cli.Get(context.TODO(), testkey, nil) + require.ErrorIs(t, redis.ErrNotFound, err) + err = cli.Set(context.TODO(), testkey, val, 0) + require.NoError(t, err) + + var v Foo + err = cli.Get(context.TODO(), testkey, &v) + require.NoError(t, err) + require.Equal(t, val, v) +} + +type valJSON struct { + Key []string `json:"key"` + Value map[string]struct{} `json:"value"` +} + +func (val *valJSON) Encode() ([]byte, error) { + return json.Marshal(val) +} + +func (val *valJSON) Decode(data []byte) error { + return json.Unmarshal(data, val) +} + +func TestCodecJOSN(t *testing.T) { + v := make(map[string]struct{}, 2) + v["key1"] = struct{}{} + var valueIn interface{} = &valJSON{ + Key: []string{"key1", "key2"}, + Value: v, + } + + val, ok := valueIn.(redis.CodeC) + require.True(t, ok) + data, err := val.Encode() + require.Nil(t, err) + + var valueOut interface{} = new(valJSON) + val, ok = valueOut.(redis.CodeC) + require.True(t, ok) + err = val.Decode(data) + require.Nil(t, err) + + require.Equal(t, valueIn, valueOut) +} + +func TestGetSetJOSN(t *testing.T) { + v := make(map[string]struct{}, 2) + v["key1"] = struct{}{} + valueIn := &valJSON{ + Key: []string{"key1", "key2"}, + Value: v, + } + defer cli.Del(context.TODO(), testkey) + + err := cli.Set(context.TODO(), testkey, valueIn, 0) + require.NoError(t, err) + + var valueOut valJSON + err = cli.Get(context.TODO(), testkey, &valueOut) + require.NoError(t, err) + require.Equal(t, *valueIn, valueOut) +} + +type valBinary struct { + Key uint32 + Value uint64 +} + +func (val *valBinary) Encode() ([]byte, error) { + buf := new(bytes.Buffer) + err := binary.Write(buf, binary.BigEndian, val) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (val *valBinary) Decode(data []byte) error { + reader := bytes.NewReader(data) + err := binary.Read(reader, binary.BigEndian, val) + if err != nil { + return err + } + return nil +} + +func TestCodecBinary(t *testing.T) { + var valueIn interface{} = &valBinary{uint32(100), uint64(200)} + + val, ok := valueIn.(redis.CodeC) + require.True(t, ok) + data, err := val.Encode() + require.Nil(t, err) + + var valueOut interface{} = &valBinary{} + val, ok = valueOut.(redis.CodeC) + require.True(t, ok) + err = val.Decode(data) + require.Nil(t, err) + + require.Equal(t, valueIn, valueOut) +} + +func TestGetSetBinary(t *testing.T) { + valueIn := &valBinary{uint32(100), uint64(200)} + defer cli.Del(context.TODO(), testkey) + + err := cli.Set(context.TODO(), testkey, valueIn, 0) + require.NoError(t, err) + + var valueOut valBinary + err = cli.Get(context.TODO(), testkey, &valueOut) + require.NoError(t, err) + require.Equal(t, *valueIn, valueOut) +} + +type valJSONSet struct { + Key []string `json:"key"` + Value map[string]struct{} `json:"value"` +} + +func (val *valJSONSet) Encode() ([]byte, error) { + return nil, errJSONSet +} + +func (val *valJSONSet) Decode(data []byte) error { + return json.Unmarshal(data, val) +} + +func TestJOSNErrorSet(t *testing.T) { + valueIn := &valJSONSet{ + Key: []string{"key1", "key2"}, + } + defer cli.Del(context.TODO(), testkey) + + err := cli.Set(context.TODO(), testkey, valueIn, 0) + require.ErrorIs(t, redis.ErrEncodeFailed, err) + + var valueOut valJSONSet + err = cli.Get(context.TODO(), testkey, &valueOut) + require.ErrorIs(t, redis.ErrNotFound, err) +} + +type valJSONGet struct { + Key []string `json:"key"` + Value map[string]struct{} `json:"value"` +} + +func (val *valJSONGet) Encode() ([]byte, error) { + return json.Marshal(val) +} + +func (val *valJSONGet) Decode(data []byte) error { + return errJSONGet +} + +func TestJOSNErrorGet(t *testing.T) { + valueIn := &valJSONGet{ + Key: []string{"key1", "key2"}, + } + defer cli.Del(context.TODO(), testkey) + + err := cli.Set(context.TODO(), testkey, valueIn, 0) + require.NoError(t, err) + + var valueOut valJSONGet + err = cli.Get(context.TODO(), testkey, &valueOut) + require.ErrorIs(t, redis.ErrDecodeFailed, err) +} diff --git a/blobstore/common/resourcepool/chan_pool.go b/blobstore/common/resourcepool/chan_pool.go new file mode 100644 index 000000000..ebe304c0d --- /dev/null +++ b/blobstore/common/resourcepool/chan_pool.go @@ -0,0 +1,178 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package resourcepool + +import ( + "runtime" + "sync" + "sync/atomic" + "time" +) + +// cache in chan pool will not be released by runtime.GC(). +// pool chan length dynamicly changes by EMA(Exponential Moving Average). +// redundant buffers will released by GC. + +const maxMemorySize = 1 << 32 // 4G + +// type SliceHeader is 24 bytes. +// buffered channel malloc memory in one call. +// `makechan` see more at: https://github.com/golang/go/blob/master/src/runtime/chan.go +// +// limit max channel memory to 96MB (24 * (1<<22)), +// can reduce to 32MB if using type *[]byte. +const maxChanSize = 1 << 22 // 4m + +var releaseInterval int64 = int64(time.Minute) * 2 + +// SetReleaseInterval set release interval duration +func SetReleaseInterval(duration time.Duration) { + if duration > time.Millisecond*100 { + atomic.StoreInt64(&releaseInterval, int64(duration)) + } +} + +type chPool struct { + chBuffer chan []byte + newBuffer func() []byte + capacity int + concurrence int32 + + closeCh chan struct{} + closeOnce sync.Once +} + +// NewChanPool return Pool with capacity, no limit if capacity is negative +func NewChanPool(newFunc func() []byte, capacity int) Pool { + chCap := capacity + if chCap < 0 { + buf := newFunc() + chCap = maxMemorySize / len(buf) + } + if chCap > maxChanSize { + chCap = maxChanSize + } + + pool := &chPool{ + chBuffer: make(chan []byte, chCap), + newBuffer: newFunc, + capacity: capacity, + closeCh: make(chan struct{}), + } + runtime.SetFinalizer(pool, func(p *chPool) { + p.closeOnce.Do(func() { + close(p.closeCh) + }) + }) + + go pool.loopRelease() + return pool +} + +// loopRelease release redundant buffers in chan. +// check EMA concurrence per round of release interval duration. +// release the redundant buffers per release interval duration. +// +// reserve 30% redundancy of capacity +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// | length of buffer chan | concurrence | +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// | redundant | capacity | reserved | +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// | to release | buffers keep in memory | +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +func (p *chPool) loopRelease() { + const emaRound = 5 + + ticker := time.NewTicker(time.Duration(atomic.LoadInt64(&releaseInterval)) / emaRound) + defer ticker.Stop() + + var ( + turn int + capacity int32 + ) + for { + select { + case <-p.closeCh: + for { + select { + case <-p.chBuffer: + default: + return + } + } + case <-ticker.C: + nowConc := atomic.LoadInt32(&p.concurrence) + capacity = ema(nowConc, capacity) + if turn = (turn + 1) % emaRound; turn != 0 { + continue + } + + capa := capacity * 13 / 10 + redundant := len(p.chBuffer) + int(nowConc-capa) + if redundant <= 0 { + continue + } + has := true + for ii := 0; has && ii < redundant; ii++ { + select { + case <-p.chBuffer: + default: + has = false + } + } + } + } +} + +func (p *chPool) Get() (interface{}, error) { + atomic.AddInt32(&p.concurrence, 1) + + select { + case buf := <-p.chBuffer: + return buf, nil + default: + return p.newBuffer(), nil + } +} + +func (p *chPool) Put(x interface{}) { + buf, ok := x.([]byte) + if !ok { + return + } + + select { + case p.chBuffer <- buf: + default: + } + atomic.AddInt32(&p.concurrence, -1) +} + +func (p *chPool) Cap() int { + return p.capacity +} + +func (p *chPool) Len() int { + return int(atomic.LoadInt32(&p.concurrence)) +} + +func (p *chPool) Idle() int { + return len(p.chBuffer) +} + +func ema(val, lastVal int32) int32 { + return (val*2 + lastVal*8) / 10 +} diff --git a/blobstore/common/resourcepool/chan_pool_test.go b/blobstore/common/resourcepool/chan_pool_test.go new file mode 100644 index 000000000..f736b2685 --- /dev/null +++ b/blobstore/common/resourcepool/chan_pool_test.go @@ -0,0 +1,125 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package resourcepool_test + +import ( + "math/rand" + "testing" + "time" + + rp "github.com/cubefs/cubefs/blobstore/common/resourcepool" + "github.com/stretchr/testify/require" +) + +func TestChanPoolSizeLimited(t *testing.T) { + p := rp.NewChanPool(func() []byte { + return make([]byte, 1) + }, -1) + require.Equal(t, -1, p.Cap()) + + b := make([]byte, 1) + for range [1 << 23]struct{}{} { + p.Put(b) + } + require.Equal(t, 1<<22, p.Idle()) +} + +func TestChanPoolRelease(t *testing.T) { + rp.SetReleaseInterval(200 * time.Millisecond) + defer rp.SetReleaseInterval(time.Minute * 2) + + p := rp.NewChanPool(func() []byte { + return make([]byte, 1024) + }, -1) + + for i := range [1000]struct{}{} { + go func(i int) { + buf, err := p.Get() + require.NoError(t, err) + time.Sleep(1 + time.Millisecond*time.Duration(rand.Intn(2000))) + p.Put(buf) + }(i) + } + time.Sleep(time.Second) +} + +func TestChanPoolBase(t *testing.T) { + makeN := 0 + p := rp.NewChanPool(func() []byte { + makeN++ + return make([]byte, 1024) + }, 2) + + require.Equal(t, 2, p.Cap()) + require.Equal(t, 0, p.Len()) + require.Equal(t, 0, p.Idle()) + + _, err := p.Get() + require.NoError(t, err) + _, err = p.Get() + require.NoError(t, err) + + require.Equal(t, 2, p.Cap()) + require.Equal(t, 2, p.Len()) + require.Equal(t, 0, p.Idle()) + + buf, err := p.Get() + require.NoError(t, err) + require.Equal(t, 3, makeN) + + p.Put(buf) + require.Equal(t, 1, p.Idle()) + _, err = p.Get() + require.NoError(t, err) + require.Equal(t, 3, makeN) + require.Equal(t, 0, p.Idle()) +} + +func TestChanPoolNoLimit(t *testing.T) { + { + makeN := 0 + p := rp.NewChanPool(func() []byte { + makeN++ + return make([]byte, 1024) + }, 0) + require.Equal(t, 0, p.Cap()) + + _, err := p.Get() + require.NoError(t, err) + require.Equal(t, 1, makeN) + } + { + p := rp.NewChanPool(func() []byte { + return make([]byte, 1024) + }, -1) + require.Equal(t, -1, p.Cap()) + require.Equal(t, 0, p.Idle()) + + for range [1000]struct{}{} { + buf, err := p.Get() + require.NoError(t, err) + p.Put(buf) + } + require.Equal(t, 1, p.Idle()) + } +} + +func BenchmarkChanPoolGetPut(b *testing.B) { + benchmarkPoolGetPut(b, func(size, capacity int) rp.Pool { + return rp.NewChanPool(func() []byte { + return make([]byte, size) + }, capacity) + }) +} diff --git a/blobstore/common/resourcepool/mempool.go b/blobstore/common/resourcepool/mempool.go new file mode 100644 index 000000000..5f4ad2d88 --- /dev/null +++ b/blobstore/common/resourcepool/mempool.go @@ -0,0 +1,147 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package resourcepool + +import ( + "errors" + "sort" +) + +// ErrNoSuitableSizeClass no suitable pool of size +var ErrNoSuitableSizeClass = errors.New("no suitable size class") + +// zero bytes, high performance at the 16KB, see more in the benchmark: +// BenchmarkZero/4MB-16KB-4 13338 88378 ns/op +// BenchmarkZero/8MB-16KB-4 6670 183987 ns/op +// BenchmarkZero/16MB-16KB-4 1926 590422 ns/op +const zeroLen = 1 << 14 + +var zero = make([]byte, zeroLen) + +// MemPool reused buffer pool +type MemPool struct { + pool []Pool + poolSize []int +} + +// Status memory pools status +type Status []PoolStatus + +// PoolStatus status of the pool +type PoolStatus struct { + Size int `json:"size"` + Capacity int `json:"capacity"` + Running int `json:"running"` + Idle int `json:"idle"` +} + +// NewMemPool returns a MemPool within chan pool +func NewMemPool(sizeClasses map[int]int) *MemPool { + return NewMemPoolWith(sizeClasses, func(size, capacity int) Pool { + return NewChanPool(func() []byte { + return make([]byte, size) + }, capacity) + }) +} + +// NewMemPoolWith new MemPool with size-class and self-defined pool +func NewMemPoolWith(sizeClasses map[int]int, newPool func(size, capacity int) Pool) *MemPool { + pool := make([]Pool, 0, len(sizeClasses)) + poolSize := make([]int, 0, len(sizeClasses)) + for sizeClass := range sizeClasses { + if sizeClass > 0 { + poolSize = append(poolSize, sizeClass) + } + } + + sort.Ints(poolSize) + for _, sizeClass := range poolSize { + pool = append(pool, newPool(sizeClass, sizeClasses[sizeClass])) + } + + return &MemPool{ + pool: pool, + poolSize: poolSize, + } +} + +// Get return a suitable buffer +func (p *MemPool) Get(size int) ([]byte, error) { + for idx, ps := range p.poolSize { + if size <= ps { + buf, err := p.pool[idx].Get() + if err != nil { + return nil, err + } + buff := buf.([]byte) + return buff[:size], nil + } + } + + return nil, ErrNoSuitableSizeClass +} + +// Alloc return a buffer, make a new if oversize +func (p *MemPool) Alloc(size int) ([]byte, error) { + buf, err := p.Get(size) + if err == ErrNoSuitableSizeClass { + return make([]byte, size), nil + } + + return buf, err +} + +// Put adds x to the pool, appropriately resize +func (p *MemPool) Put(b []byte) error { + sizeClass := cap(b) + b = b[0:sizeClass] + for ii := len(p.poolSize) - 1; ii >= 0; ii-- { + if sizeClass >= p.poolSize[ii] { + b = b[0:p.poolSize[ii]] + p.pool[ii].Put(b) + return nil + } + } + + return ErrNoSuitableSizeClass +} + +// Zero clean up the buffer b to zero bytes +func (p *MemPool) Zero(b []byte) { + Zero(b) +} + +// Status returns status of memory pool +func (p *MemPool) Status() Status { + st := make(Status, len(p.poolSize)) + for idx, size := range p.poolSize { + pool := p.pool[idx] + st[idx] = PoolStatus{ + Size: size, + Capacity: pool.Cap(), + Running: pool.Len(), + Idle: pool.Idle(), + } + } + return st +} + +// Zero clean up the buffer b to zero bytes +func Zero(b []byte) { + for len(b) > 0 { + n := copy(b, zero) + b = b[n:] + } +} diff --git a/blobstore/common/resourcepool/mempool_test.go b/blobstore/common/resourcepool/mempool_test.go new file mode 100644 index 000000000..f426e5c62 --- /dev/null +++ b/blobstore/common/resourcepool/mempool_test.go @@ -0,0 +1,368 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package resourcepool_test + +import ( + "crypto/rand" + "encoding/json" + "fmt" + "runtime" + "testing" + + "github.com/stretchr/testify/require" + + rp "github.com/cubefs/cubefs/blobstore/common/resourcepool" +) + +const ( + kb = 1 << 10 + mb = 1 << 20 + mb4 = 4 * mb +) + +func TestMemPoolBase(t *testing.T) { + classes := map[int]int{kb: 2, mb: 1} + + pool := rp.NewMemPool(classes) + require.NotNil(t, pool) + + bufm, err := pool.Get(mb) + require.NoError(t, err) + require.Equal(t, mb, len(bufm)) + require.Equal(t, mb, cap(bufm)) + + _, err = pool.Get(mb) + require.NoError(t, err) + // require.ErrorIs(t, err, rp.ErrPoolLimit) + + bufk, err := pool.Get(kb) + require.NoError(t, err) + require.Equal(t, kb, len(bufk)) + require.Equal(t, kb, cap(bufk)) + + bufk2, err := pool.Get(kb / 2) + require.NoError(t, err) + require.Equal(t, kb/2, len(bufk2)) + require.Equal(t, kb, cap(bufk2)) + + err = pool.Put(bufm) + require.NoError(t, err) + + bufmx, err := pool.Get(mb) + require.NoError(t, err) + require.Equal(t, bufm, bufmx) +} + +func TestMemPoolEmpty(t *testing.T) { + pool := rp.NewMemPool(nil) + require.NotNil(t, pool) + + _, err := pool.Get(kb) + require.ErrorIs(t, err, rp.ErrNoSuitableSizeClass) + + buf, err := pool.Alloc(kb) + require.NoError(t, err) + require.Equal(t, kb, len(buf)) + require.Equal(t, kb, cap(buf)) + + err = pool.Put(buf) + require.ErrorIs(t, err, rp.ErrNoSuitableSizeClass) +} + +func TestMemPoolChanAlloc(t *testing.T) { + classes := map[int]int{mb: 1} + pool := rp.NewMemPool(classes) + require.NotNil(t, pool) + + bufm, err := pool.Get(mb) + require.NoError(t, err) + require.Equal(t, mb, len(bufm)) + require.Equal(t, mb, cap(bufm)) + + _, err = pool.Get(mb) + require.NoError(t, err) + + // if matched size class, return ErrPoolLimit + _, err = pool.Alloc(mb) + require.NoError(t, err) + + // if oversize, make new buffer + bufm4, err := pool.Alloc(mb4) + require.NoError(t, err) + require.Equal(t, mb4, len(bufm4)) + require.Equal(t, mb4, cap(bufm4)) + + // put oversize buffer to top class + err = pool.Put(bufm4) + require.NoError(t, err) + + // maybe get the oversize buffer + bufm, err = pool.Get(mb) + require.NoError(t, err) + require.Equal(t, mb, len(bufm)) + require.True(t, mb4 == cap(bufm) || mb == cap(bufm)) + + err = pool.Put(bufm) + require.NoError(t, err) +} + +func TestMemPoolSyncAlloc(t *testing.T) { + classes := map[int]int{mb: 1} + pool := rp.NewMemPoolWith(classes, func(size, capacity int) rp.Pool { + return rp.NewPool(func() interface{} { + return make([]byte, size) + }, capacity) + }) + require.NotNil(t, pool) + + bufm, err := pool.Get(mb) + require.NoError(t, err) + require.Equal(t, mb, len(bufm)) + require.Equal(t, mb, cap(bufm)) + + _, err = pool.Get(mb) + require.ErrorIs(t, err, rp.ErrPoolLimit) + + // if matched size class, return ErrPoolLimit + _, err = pool.Alloc(mb) + require.ErrorIs(t, err, rp.ErrPoolLimit) + + // if oversize, make new buffer + bufm4, err := pool.Alloc(mb4) + require.NoError(t, err) + require.Equal(t, mb4, len(bufm4)) + require.Equal(t, mb4, cap(bufm4)) + + // put oversize buffer to top class + err = pool.Put(bufm4) + require.NoError(t, err) + + // maybe get the oversize buffer + bufm, err = pool.Get(mb) + require.NoError(t, err) + require.Equal(t, mb, len(bufm)) + require.True(t, mb4 == cap(bufm) || mb == cap(bufm)) + + err = pool.Put(bufm) + require.NoError(t, err) + + // bufm4 released by gc after twice runtime.GC() + // see sync/pool.go: runtime_registerPoolCleanup(poolCleanup) + runtime.GC() + runtime.GC() + bufm, err = pool.Get(mb) + require.NoError(t, err) + require.Equal(t, mb, len(bufm)) + require.Equal(t, mb, cap(bufm)) +} + +func TestMemPoolPutGet(t *testing.T) { + classes := map[int]int{mb: 1} + + pool := rp.NewMemPool(classes) + require.NotNil(t, pool) + + bufm, err := pool.Get(mb) + require.NoError(t, err) + require.Equal(t, mb, len(bufm)) + require.Equal(t, mb, cap(bufm)) + + zero := make([]byte, len(bufm)) + rand.Read(bufm) + require.NotEqual(t, zero, bufm) + + err = pool.Put(bufm) + require.NoError(t, err) + + bufmx, err := pool.Get(mb) + require.NoError(t, err) + require.Equal(t, bufm, bufmx) + + // if oversize, make new buffer + bufm4, err := pool.Alloc(mb4) + require.NoError(t, err) + require.Equal(t, mb4, len(bufm4)) + require.Equal(t, mb4, cap(bufm4)) + + rand.Read(bufm4) + require.NotEqual(t, zero, bufm4[:mb]) + + // put oversize buffer to top class + err = pool.Put(bufm4) + require.NoError(t, err) + + // maybe get the oversize buffer + bufm, err = pool.Get(mb) + require.NoError(t, err) + require.Equal(t, mb, len(bufm)) + if cap(bufm) == mb4 { + require.Equal(t, bufm4[:mb], bufm) + } else if cap(bufm) == mb { + require.Equal(t, zero, bufm) + } else { + require.Fail(t, "cant be there") + } +} + +func TestMemPoolZero(t *testing.T) { + buffer := make([]byte, 1<<24) + zero := make([]byte, 1<<24) + cases := []struct { + from, to int + }{ + {0, 1 << 20}, + {0, 1 << 24}, + {1, 1 << 20}, + {1 << 20, 1 << 20}, + {1 << 20, 1 << 22}, + {1 << 20, 1 << 24}, + {1029, 1029}, + {1029, 1 << 14}, + {1029, 1 << 24}, + } + + for _, cs := range cases { + pool := rp.NewMemPool(nil) + require.NotNil(t, pool) + + buf := buffer[cs.from:cs.to] + rand.Read(buf) + if cs.from != cs.to { + require.NotEqual(t, zero[cs.from:cs.to], buf) + } + pool.Zero(buf) + require.Equal(t, cs.to-cs.from, len(buf)) + require.Equal(t, zero[cs.from:cs.to], buf) + } +} + +func TestMemPoolStatus(t *testing.T) { + { + pool := rp.NewMemPool(nil) + require.NotNil(t, pool) + t.Logf("status empty: %+v", pool.Status()) + } + { + classes := map[int]int{kb: 2, mb: 1} + + pool := rp.NewMemPool(classes) + require.NotNil(t, pool) + t.Logf("status init: %+v", pool.Status()) + + _, err := pool.Get(mb) + require.NoError(t, err) + bufk, err := pool.Get(kb) + require.NoError(t, err) + t.Logf("status running: %+v", pool.Status()) + + pool.Put(bufk) + data, _ := json.Marshal(pool.Status()) + t.Logf("status json: %s", string(data)) + } +} + +func BenchmarkMempool(b *testing.B) { + mp := rp.NewMemPool(map[int]int{ + 1 << 11: -1, + 1 << 14: -1, + 1 << 16: -1, + 1 << 18: -1, + 1 << 20: -1, + 1 << 21: -1, + 1 << 22: -1, + }) + + for _, size := range []int{ + 1 << 10, + 1 << 16, + 1 << 18, + 1 << 20, + 1 << 21, + 1 << 22, + } { + b.ResetTimer() + b.Run(humman(size), func(b *testing.B) { + b.ResetTimer() + for ii := 0; ii <= b.N; ii++ { + if buf, err := mp.Get(size); err == nil { + _ = mp.Put(buf) + } + } + }) + } +} + +func BenchmarkZero(b *testing.B) { + funcZero := func(b, zero []byte) { + for len(b) > 0 { + n := copy(b, zero) + b = b[n:] + } + } + + cases := []struct { + size int + zero int + }{} + + for _, size := range []int{ + 1 << 0, + 1 << 5, + 1 << 10, + 1 << 15, + 1 << 20, + 1 << 21, + 1 << 22, + 1 << 23, + 1 << 24, + 1 << 25, + } { + // zoro buffer from 256B - 32M + for i := 8; i <= 24; i++ { + cases = append(cases, struct { + size int + zero int + }{size, 1 << i}) + } + } + + for _, cs := range cases { + b.ResetTimer() + b.Run(fmt.Sprintf("%s-%s", humman(cs.size), humman(cs.zero)), func(b *testing.B) { + zero := make([]byte, cs.zero) + buff := make([]byte, cs.size) + b.ResetTimer() + for ii := 0; ii <= b.N; ii++ { + funcZero(buff, zero) + } + }) + } +} + +var size2humman = []func(int) string{ + func(size int) string { return fmt.Sprintf("%dB", size) }, + func(size int) string { return fmt.Sprintf("%dKB", size/1024) }, + func(size int) string { return fmt.Sprintf("%dMB", size/1024/1024) }, + func(size int) string { return fmt.Sprintf("%dGB", size/1024/1024/1024) }, +} + +func humman(size int) string { + for ii := len(size2humman) - 1; ii >= 0; ii-- { + if size >= (1 << (10 * ii)) { + return size2humman[ii](size) + } + } + return "-" +} diff --git a/blobstore/common/resourcepool/pool.go b/blobstore/common/resourcepool/pool.go new file mode 100644 index 000000000..331716240 --- /dev/null +++ b/blobstore/common/resourcepool/pool.go @@ -0,0 +1,85 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package resourcepool + +// sync.Pool cache will be released by runtime.GC() +// see sync/pool.go: runtime_registerPoolCleanup(poolCleanup) + +import ( + "errors" + "sync" + "sync/atomic" +) + +// ErrPoolLimit pool elements exceed its capacity +var ErrPoolLimit = errors.New("resource pool limit") + +// Pool resource pool support for sync.pool and capacity limit +// release resource if no used anymore +// no limit if capacity is negative +type Pool interface { + // Get return nil and error if exceed pool's capacity + Get() (interface{}, error) + Put(x interface{}) + Cap() int + Len() int + // Idle return cached idle objects in pool. + Idle() int +} + +// sync pool Idle return -1 if no limit +type pool struct { + sp sync.Pool + capacity int32 + current int32 +} + +// NewPool return Pool with capacity, no limit if capacity is negative +func NewPool(newFunc func() interface{}, capacity int) Pool { + return &pool{ + sp: sync.Pool{New: newFunc}, + capacity: int32(capacity), + current: int32(0), + } +} + +func (p *pool) Get() (interface{}, error) { + current := atomic.AddInt32(&p.current, 1) + if p.capacity >= 0 && current > p.capacity { + atomic.AddInt32(&p.current, -1) + return nil, ErrPoolLimit + } + return p.sp.Get(), nil +} + +func (p *pool) Put(x interface{}) { + p.sp.Put(x) + atomic.AddInt32(&p.current, -1) +} + +func (p *pool) Cap() int { + return int(p.capacity) +} + +func (p *pool) Len() int { + return int(atomic.LoadInt32(&p.current)) +} + +func (p *pool) Idle() int { + if p.capacity < 0 { + return -1 + } + return p.Cap() - p.Len() +} diff --git a/blobstore/common/resourcepool/pool_test.go b/blobstore/common/resourcepool/pool_test.go new file mode 100644 index 000000000..1dcdd2c2d --- /dev/null +++ b/blobstore/common/resourcepool/pool_test.go @@ -0,0 +1,144 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package resourcepool_test + +import ( + "context" + "fmt" + "runtime" + "testing" + + "github.com/stretchr/testify/require" + + rp "github.com/cubefs/cubefs/blobstore/common/resourcepool" + "github.com/cubefs/cubefs/blobstore/util/task" +) + +func TestPoolBase(t *testing.T) { + p := rp.NewPool(func() interface{} { + return 1 + }, 2) + + require.Equal(t, 2, p.Cap()) + require.Equal(t, 0, p.Len()) + require.Equal(t, 2, p.Idle()) + + _, err := p.Get() + require.NoError(t, err) + _, err = p.Get() + require.NoError(t, err) + + require.Equal(t, 2, p.Len()) + require.Equal(t, 0, p.Idle()) + + _, err = p.Get() + require.Equal(t, rp.ErrPoolLimit, err) + _, err = p.Get() + require.Equal(t, rp.ErrPoolLimit, err) + + p.Put(1) + _, err = p.Get() + require.NoError(t, err) +} + +func TestPoolNoLimit(t *testing.T) { + { + p := rp.NewPool(func() interface{} { + return 1 + }, 0) + require.Equal(t, 0, p.Cap()) + require.Equal(t, 0, p.Len()) + require.Equal(t, 0, p.Idle()) + + _, err := p.Get() + require.ErrorIs(t, rp.ErrPoolLimit, err) + } + { + p := rp.NewPool(func() interface{} { + return 1 + }, -10) + require.Equal(t, -10, p.Cap()) + require.Equal(t, 0, p.Len()) + require.Equal(t, -1, p.Idle()) + + tasks := make([]func() error, 0, 10000) + for range [10000]struct{}{} { + tasks = append(tasks, func() error { + _, err := p.Get() + return err + }) + } + + err := task.Run(context.Background(), tasks...) + require.NoError(t, err) + } +} + +func BenchmarkPoolGetPut(b *testing.B) { + benchmarkPoolGetPut(b, func(size, capacity int) rp.Pool { + return rp.NewPool(func() interface{} { + return make([]byte, size) + }, capacity) + }) +} + +func benchmarkPoolGetPut(b *testing.B, newPool func(zize, capacity int) rp.Pool) { + const kb = 1 << 10 + const mb = 1 << 20 + + cases := []struct { + size int + }{ + {kb * 2}, + {kb * 64}, + {kb * 512}, + {mb}, + {mb * 2}, + {mb * 4}, + } + + for _, cs := range cases { + runtime.GC() + b.ResetTimer() + b.Run(fmt.Sprintf("%s-%s", humman(cs.size), "unlimited"), func(b *testing.B) { + pool := newPool(cs.size, -1) + b.ResetTimer() + for ii := 0; ii <= b.N; ii++ { + if val, err := pool.Get(); err == nil { + pool.Put(val) + } + } + }) + + b.ResetTimer() + b.Run(fmt.Sprintf("%s-%s", humman(cs.size), "with_1m"), func(b *testing.B) { + pool := newPool(cs.size, 1000000) + b.ResetTimer() + for ii := 0; ii <= b.N; ii++ { + if val, err := pool.Get(); err == nil { + pool.Put(val) + } + } + }) + + b.ResetTimer() + b.Run(fmt.Sprintf("%s-%s", humman(cs.size), "make"), func(b *testing.B) { + b.ResetTimer() + for ii := 0; ii <= b.N; ii++ { + _ = make([]byte, cs.size) + } + }) + } +} diff --git a/blobstore/common/rpc/argument.go b/blobstore/common/rpc/argument.go new file mode 100644 index 000000000..28a283e61 --- /dev/null +++ b/blobstore/common/rpc/argument.go @@ -0,0 +1,316 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package rpc + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "io" + "reflect" + "strconv" + "strings" + + "github.com/cubefs/cubefs/blobstore/util/bytespool" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +type ( + parserKey struct { + PkgPath string + Name string + FieldName string + } + parserVal struct { + Name string + Opt struct { + Ignore bool // "-" + Omitempty bool // ",omitempty" + Base64 bool // ",base64" + } + } +) + +var registeredParsers map[parserKey]parserVal + +func init() { + registeredParsers = make(map[parserKey]parserVal) +} + +// RegisterArgsParser regist your argument need parse in +// uri, query, form, or postform. +// the tags is sorted. +// NOTE: the function is thread-unsafe. +func RegisterArgsParser(args interface{}, tags ...string) { + if args == nil { + return + } + + if _, ok := args.(Parser); ok { + return + } + + typ := reflect.TypeOf(args) + val := reflect.ValueOf(args) + if typ.Kind() != reflect.Ptr { + log.Panicf("args(%s) must be pointer", typ.Name()) + } + + typ = typ.Elem() + if typ.Kind() != reflect.Struct { + log.Panicf("args(%s) reference must be struct", typ.Name()) + } + + val = val.Elem() + t := val.Type() + for i := 0; i < val.NumField(); i++ { + ft := t.Field(i) + + pVal := parserVal{ + Name: strings.ToLower(ft.Name), + } + for _, tag := range tags { + tagStr := ft.Tag.Get(tag) + if tagStr != "" { + ts := strings.Split(tagStr, ",") + if ts[0] == "-" { + pVal.Opt.Ignore = true + break + } + + if ts[0] != "" { + pVal.Name = ts[0] + } + for _, t := range ts[1:] { + switch t { + case "omitempty": + pVal.Opt.Omitempty = true + case "base64": + pVal.Opt.Base64 = true + } + } + break + } + } + + pKey := parserKey{ + PkgPath: typ.PkgPath(), + Name: typ.Name(), + FieldName: ft.Name, + } + registeredParsers[pKey] = pVal + + log.Infof("register args field:%+v val:%+v", pKey, pVal) + } +} + +func parseArgs(c *Context, args interface{}, opts ...ServerOption) error { + if args == nil { + return nil + } + opt := c.opts + if len(opts) > 0 { + opt = c.opts.copy() + for _, o := range opts { + o.apply(opt) + } + } + + if opt.argsBody { + size, err := c.RequestLength() + if err != nil { + return err + } + buf := bytespool.Alloc(size) + defer bytespool.Free(buf) + + if _, err = io.ReadFull(c.Request.Body, buf); err != nil { + return err + } + + if arg, ok := args.(Unmarshaler); ok { + return arg.Unmarshal(buf) + } + return json.Unmarshal(buf, args) + } + + if !opt.hasArgs() { + return nil + } + + getter := func(fKey string) string { + if opt.argsURI { + if val := c.Param.ByName(fKey); val != "" { + return val + } + } + if opt.argsQuery { + if val := c.Request.URL.Query().Get(fKey); val != "" { + return val + } + } + if opt.argsForm { + if val := c.Request.Form.Get(fKey); val != "" { + return val + } + } + if opt.argsPostForm { + if val := c.Request.PostForm.Get(fKey); val != "" { + return val + } + } + return "" + } + + if arg, ok := args.(Parser); ok { + return arg.Parse(getter) + } + + typ := reflect.TypeOf(args) + val := reflect.ValueOf(args) + + if typ.Kind() != reflect.Ptr { + return fmt.Errorf("args(%s) must be pointer", typ.Name()) + } + + typ = typ.Elem() + if typ.Kind() != reflect.Struct { + return fmt.Errorf("args(%s) reference must be struct", typ.Name()) + } + + val = val.Elem() + t := val.Type() + for i := 0; i < val.NumField(); i++ { + ft := t.Field(i) + + pVal, ok := registeredParsers[parserKey{ + PkgPath: typ.PkgPath(), + Name: typ.Name(), + FieldName: ft.Name, + }] + if !ok { + pVal = parserVal{Name: strings.ToLower(ft.Name)} + } + if pVal.Opt.Ignore { + continue + } + + fVal := getter(pVal.Name) + if fVal == "" { + if pVal.Opt.Omitempty { + continue + } + return fmt.Errorf("args(%s) field(%s) do not omit", typ.Name(), ft.Name) + } + + if pVal.Opt.Base64 { + switch len(fVal) & 3 { + case 2: + fVal += "==" + case 3: + fVal += "=" + } + b, err := base64.URLEncoding.DecodeString(fVal) + if err != nil { + return fmt.Errorf("args(%s) field(%s) invalid base64(%s)", typ.Name(), ft.Name, fVal) + } + fVal = string(b) + } + + fv := val.Field(i) + if err := parseValue(fv, fVal); err != nil { + return err + } + } + + return nil +} + +func parseValue(val reflect.Value, str string) (err error) { + var ( + bv bool + iv int64 + uv uint64 + fv float64 + ) + +RETRY: + switch val.Kind() { + case reflect.Bool: + bv, err = strconv.ParseBool(str) + val.SetBool(bv) + + case reflect.Int: + iv, err = strconv.ParseInt(str, 10, 0) + val.SetInt(iv) + case reflect.Int8: + iv, err = strconv.ParseInt(str, 10, 8) + val.SetInt(iv) + case reflect.Int16: + iv, err = strconv.ParseInt(str, 10, 16) + val.SetInt(iv) + case reflect.Int32: + iv, err = strconv.ParseInt(str, 10, 32) + val.SetInt(iv) + case reflect.Int64: + iv, err = strconv.ParseInt(str, 10, 64) + val.SetInt(iv) + + case reflect.Uint: + uv, err = strconv.ParseUint(str, 10, 0) + val.SetUint(uv) + case reflect.Uint8: + uv, err = strconv.ParseUint(str, 10, 8) + val.SetUint(uv) + case reflect.Uint16: + uv, err = strconv.ParseUint(str, 10, 16) + val.SetUint(uv) + case reflect.Uint32: + uv, err = strconv.ParseUint(str, 10, 32) + val.SetUint(uv) + case reflect.Uint64: + uv, err = strconv.ParseUint(str, 10, 64) + val.SetUint(uv) + + case reflect.Float32: + fv, err = strconv.ParseFloat(str, 32) + val.SetFloat(fv) + case reflect.Float64: + fv, err = strconv.ParseFloat(str, 64) + val.SetFloat(fv) + + case reflect.String: + val.SetString(str) + case reflect.Uintptr: + uv, err = strconv.ParseUint(str, 10, 64) + val.SetUint(uv) + case reflect.Ptr: + elem := reflect.New(val.Type().Elem()) + val.Set(elem) + val = elem.Elem() + goto RETRY + + case reflect.Slice: + if val.Type().Elem().Kind() == reflect.Uint8 { + val.SetBytes([]byte(str)) + } else { + return fmt.Errorf("unsupported type(%s) of slice", val.Type().Elem().Kind().String()) + } + + default: + return fmt.Errorf("unsupported type(%s)", val.Kind().String()) + } + return +} diff --git a/blobstore/common/rpc/argument_test.go b/blobstore/common/rpc/argument_test.go new file mode 100644 index 000000000..e8e08e28c --- /dev/null +++ b/blobstore/common/rpc/argument_test.go @@ -0,0 +1,282 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package rpc + +import ( + "encoding/base64" + "net/http" + "testing" + + "github.com/julienschmidt/httprouter" + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/util/log" +) + +func TestArgumentRegister(t *testing.T) { + log.SetOutputLevel(log.Linfo) + defer setLogLevel() + defer func() { + registeredParsers = make(map[parserKey]parserVal) + }() + { + RegisterArgsParser(nil) + RegisterArgsParser(struct{ Parser }{}) + } + { + type args struct{} + require.Panics(t, func() { + RegisterArgsParser(args{}) + }) + require.Panics(t, func() { + RegisterArgsParser(make(map[int]int)) + }) + require.Panics(t, func() { + var i int + RegisterArgsParser(&i) + }) + RegisterArgsParser(&args{}) + } + { + type Args struct { + F1 string `taglv3:"f1-3" taglv2:"f1-2" taglv1:"f1-1"` + F2 string `taglv2:"f2-2" taglv1:"f2-1"` + F3 string `taglv1:"f3-1,omitempty"` + F4 []byte `taglv1:"f4-1,base64"` + F5 string + F6 string `taglv3:"-"` + } + RegisterArgsParser(&Args{}, "taglv3", "taglv2", "taglv1") + + byteVal := base64.URLEncoding.EncodeToString([]byte("f4-1")) + args := new(Args) + c := new(Context) + c.opts = new(serverOptions) + c.Param = httprouter.Params{ + httprouter.Param{Key: "f1-3", Value: "f1-3"}, + httprouter.Param{Key: "f1-2", Value: "f1-2"}, + httprouter.Param{Key: "f1-1", Value: "f1-1"}, + httprouter.Param{Key: "f2-2", Value: "f2-2"}, + httprouter.Param{Key: "f2-1", Value: "f2-1"}, + httprouter.Param{Key: "f4-1", Value: byteVal}, + httprouter.Param{Key: "f5", Value: "f5"}, + httprouter.Param{Key: "f6", Value: "f6"}, + } + parseArgs(c, args, OptArgsURI()) + + require.Equal(t, "f1-3", args.F1) + require.Equal(t, "f2-2", args.F2) + require.Equal(t, "", args.F3) + require.Equal(t, []byte("f4-1"), args.F4) + require.Equal(t, "f5", args.F5) + require.Equal(t, "", args.F6) + } +} + +func TestArgumentBase64(t *testing.T) { + log.SetOutputLevel(log.Linfo) + defer setLogLevel() + defer func() { + registeredParsers = make(map[parserKey]parserVal) + }() + type Args struct { + Base64 []byte `tag:"base64,base64"` + } + RegisterArgsParser(&Args{}, "tag") + + // "ZjQtMQ==" == []byte("f4-1") + cases := []struct { + hasErr bool + val string + }{ + {false, "ZjQtMQ=="}, + {false, "ZjQtMQ="}, + {false, "ZjQtMQ"}, + {true, "ZjQtM==="}, + {true, "ZjQtM"}, + } + for _, cs := range cases { + args := new(Args) + c := new(Context) + c.opts = new(serverOptions) + c.Param = httprouter.Params{ + httprouter.Param{Key: "base64", Value: cs.val}, + } + err := parseArgs(c, args, OptArgsURI()) + if cs.hasErr { + require.Error(t, err) + } else { + require.Equal(t, []byte("f4-1"), args.Base64) + require.NoError(t, err) + } + } +} + +type unmarshalerArgs struct { + I int +} + +func (args *unmarshalerArgs) Unmarshal([]byte) error { + args.I = 111 + return nil +} + +type normalArgs struct { + Bool bool + + Int int + Int8 int8 + Int16 int16 + Int32 int32 + Int64 int64 + + Uint uint + Uint8 uint8 + Uint16 uint16 + Uint32 uint32 + Uint64 uint64 + + Float32 float32 + Float64 float64 + + String string + Uintptr uintptr + Ptr *string + + Slice []byte +} + +type parserArgs struct { + getter string + normalArgs +} + +func (args *parserArgs) Parse(getter ValueGetter) error { + args.getter = "getter" + return nil +} + +func TestArgumentParser(t *testing.T) { + { + err := parseArgs(nil, nil) + require.NoError(t, err) + + c := new(Context) + c.opts = new(serverOptions) + err = parseArgs(c, struct{}{}, OptArgsURI()) + require.Error(t, err) + var i int + err = parseArgs(c, &i, OptArgsURI()) + require.Error(t, err) + + args := new(unmarshalerArgs) + err = parseArgs(c, args, OptArgsURI()) + require.Error(t, err) + } + { + c := new(Context) + c.opts = new(serverOptions) + c.Param = httprouter.Params{ + httprouter.Param{Key: "slice", Value: "[]"}, + httprouter.Param{Key: "map", Value: "{}"}, + } + + args := struct { + Slice []string + }{} + err := parseArgs(c, &args, OptArgsURI()) + require.Error(t, err) + + argsx := struct { + Map map[int]string + }{} + err = parseArgs(c, &argsx, OptArgsURI()) + require.Error(t, err) + } + { + c := new(Context) + c.opts = new(serverOptions) + c.Request, _ = http.NewRequest("", "", nil) + + args := new(unmarshalerArgs) + err := parseArgs(c, args, OptArgsBody()) + require.NoError(t, err) + require.Equal(t, 111, args.I) + + args = new(unmarshalerArgs) + err = parseArgs(c, args) + require.NoError(t, err) + require.Equal(t, 0, args.I) + } + { + c := new(Context) + c.opts = new(serverOptions) + c.Param = httprouter.Params{ + httprouter.Param{Key: "bool", Value: "true"}, + httprouter.Param{Key: "int", Value: "-1111"}, + httprouter.Param{Key: "int8", Value: "-1"}, + httprouter.Param{Key: "int16", Value: "-1111"}, + httprouter.Param{Key: "int32", Value: "-1111"}, + httprouter.Param{Key: "int64", Value: "-1111"}, + httprouter.Param{Key: "uint", Value: "1111"}, + httprouter.Param{Key: "uint8", Value: "1"}, + httprouter.Param{Key: "uint16", Value: "1111"}, + httprouter.Param{Key: "uint32", Value: "1111"}, + httprouter.Param{Key: "uint64", Value: "1111"}, + httprouter.Param{Key: "float32", Value: "1e-3"}, + httprouter.Param{Key: "float64", Value: "1e-32"}, + httprouter.Param{Key: "string", Value: "string"}, + httprouter.Param{Key: "uintptr", Value: "1111"}, + httprouter.Param{Key: "ptr", Value: "string"}, + httprouter.Param{Key: "slice", Value: "slice"}, + } + + args := new(normalArgs) + err := parseArgs(c, args, OptArgsURI()) + require.NoError(t, err) + + require.True(t, args.Bool) + require.Equal(t, -1111, args.Int) + require.Equal(t, int8(-1), args.Int8) + require.Equal(t, int16(-1111), args.Int16) + require.Equal(t, int32(-1111), args.Int32) + require.Equal(t, int64(-1111), args.Int64) + require.Equal(t, uint(1111), args.Uint) + require.Equal(t, uint8(1), args.Uint8) + require.Equal(t, uint16(1111), args.Uint16) + require.Equal(t, uint32(1111), args.Uint32) + require.Equal(t, uint64(1111), args.Uint64) + require.Equal(t, float32(0.001), args.Float32) + require.Equal(t, float64(1e-32), args.Float64) + require.Equal(t, "string", args.String) + require.Equal(t, uintptr(1111), args.Uintptr) + require.Equal(t, "string", *args.Ptr) + require.Equal(t, []byte("slice"), args.Slice) + } + { + c := new(Context) + c.opts = new(serverOptions) + c.Param = httprouter.Params{ + httprouter.Param{Key: "bool", Value: "true"}, + } + + args := new(parserArgs) + err := parseArgs(c, args, OptArgsURI()) + require.NoError(t, err) + + require.False(t, args.Bool) + require.Equal(t, "getter", args.getter) + } +} diff --git a/blobstore/common/rpc/codec.go b/blobstore/common/rpc/codec.go new file mode 100644 index 000000000..90312f746 --- /dev/null +++ b/blobstore/common/rpc/codec.go @@ -0,0 +1,37 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package rpc + +import ( + "net/http" + + "github.com/cubefs/cubefs/blobstore/common/crc32block" +) + +type crcDecoder struct{} + +var _ ProgressHandler = (*crcDecoder)(nil) + +func (*crcDecoder) Handler(w http.ResponseWriter, req *http.Request, f func(http.ResponseWriter, *http.Request)) { + if req.Header.Get(HeaderCrcEncoded) != "" && w.Header().Get(HeaderAckCrcEncoded) == "" { + if size := req.ContentLength; size > 0 && req.Body != nil { + decoder := crc32block.NewBodyDecoder(req.Body) + req.ContentLength = decoder.CodeSize(size) + req.Body = decoder + } + w.Header().Set(HeaderAckCrcEncoded, "1") + } + f(w, req) +} diff --git a/blobstore/common/rpc/codec_test.go b/blobstore/common/rpc/codec_test.go new file mode 100644 index 000000000..735fa8594 --- /dev/null +++ b/blobstore/common/rpc/codec_test.go @@ -0,0 +1,114 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package rpc + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "net/http" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestServerCrcDecode(t *testing.T) { + router := New() + router.Handle(http.MethodPut, "/put/:size", func(c *Context) { + size, _ := strconv.Atoi(c.Param.ByName("size")) + + n, err := io.Copy(ioutil.Discard, c.Request.Body) + if err != nil && err != io.EOF { + c.RespondError(err) + return + } + + if int(n) != size { + c.RespondError(NewError(500, "size", fmt.Errorf("%d != %d", n, size))) + return + } + c.Respond() + }) + + { + w := new(mockResponseWriter) + body := bytes.NewBuffer(make([]byte, 1024)) + req, _ := http.NewRequest(http.MethodPut, "/put/1024", body) + router.ServeHTTP(w, req) + assert.Equal(t, http.StatusOK, w.status) + } + { + w := new(mockResponseWriter) + body := bytes.NewBuffer(make([]byte, 1024)) + req, _ := http.NewRequest(http.MethodPut, "/put/1024", body) + WithCrcEncode()(req) + router.ServeHTTP(w, req) + assert.Equal(t, http.StatusOK, w.status) + } + { + w := new(mockResponseWriter) + body := bytes.NewBuffer(make([]byte, 1024)) + req, _ := http.NewRequest(http.MethodPut, "/put/1025", body) + WithCrcEncode()(req) + router.ServeHTTP(w, req) + assert.Equal(t, http.StatusInternalServerError, w.status) + } +} + +func TestServerCrcDecodeWithMiddleware(t *testing.T) { + router := New() + router.Handle(http.MethodPut, "/put/:size", func(c *Context) { + size, _ := strconv.Atoi(c.Param.ByName("size")) + + n, err := io.Copy(ioutil.Discard, c.Request.Body) + if err != nil && err != io.EOF { + c.RespondError(err) + return + } + + if int(n) != size { + c.RespondError(NewError(500, "size", fmt.Errorf("%d != %d", n, size))) + return + } + c.Respond() + }) + + handler := MiddlewareHandlerFuncWith(router) + { + w := new(mockResponseWriter) + body := bytes.NewBuffer(make([]byte, 1024)) + req, _ := http.NewRequest(http.MethodPut, "/put/1024", body) + handler.ServeHTTP(w, req) + assert.Equal(t, http.StatusOK, w.status) + } + { + w := new(mockResponseWriter) + body := bytes.NewBuffer(make([]byte, 1024)) + req, _ := http.NewRequest(http.MethodPut, "/put/1024", body) + WithCrcEncode()(req) + handler.ServeHTTP(w, req) + assert.Equal(t, http.StatusOK, w.status) + } + { + w := new(mockResponseWriter) + body := bytes.NewBuffer(make([]byte, 1024)) + req, _ := http.NewRequest(http.MethodPut, "/put/1025", body) + WithCrcEncode()(req) + handler.ServeHTTP(w, req) + assert.Equal(t, http.StatusInternalServerError, w.status) + } +} diff --git a/blobstore/common/rpc/context.go b/blobstore/common/rpc/context.go new file mode 100644 index 000000000..122339841 --- /dev/null +++ b/blobstore/common/rpc/context.go @@ -0,0 +1,267 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package rpc + +import ( + "bufio" + "fmt" + "io" + "math" + "net" + "net/http" + "strconv" + "strings" + "sync" + + "github.com/julienschmidt/httprouter" +) + +const ( + abortIndex int8 = math.MaxInt8 >> 1 +) + +var jsonNull = [4]byte{'n', 'u', 'l', 'l'} + +// Context handler context with http variables +type Context struct { + opts *serverOptions + Param httprouter.Params + + Request *http.Request + Writer http.ResponseWriter + + // pass key/value in whole request + mu sync.RWMutex + Meta map[string]interface{} + + wroteHeader bool + + // interceptors control + index int8 + handlers []HandlerFunc +} + +// ArgsBody args in body +func (c *Context) ArgsBody(args interface{}) error { + return c.ParseArgs(args, OptArgsBody()) +} + +// ArgsURI args in uri +func (c *Context) ArgsURI(args interface{}) error { + return c.ParseArgs(args, OptArgsURI()) +} + +// ArgsQuery args in query +func (c *Context) ArgsQuery(args interface{}) error { + return c.ParseArgs(args, OptArgsQuery()) +} + +// ArgsForm args in form +func (c *Context) ArgsForm(args interface{}) error { + return c.ParseArgs(args, OptArgsForm()) +} + +// ArgsPostForm args in post form +func (c *Context) ArgsPostForm(args interface{}) error { + return c.ParseArgs(args, OptArgsPostForm()) +} + +// ParseArgs reflect param to args +func (c *Context) ParseArgs(args interface{}, opts ...ServerOption) error { + if err := parseArgs(c, args, opts...); err != nil { + return NewError(http.StatusBadRequest, "Argument", err) + } + return nil +} + +// RequestLength read request body length +func (c *Context) RequestLength() (int, error) { + cl := c.Request.ContentLength + if cl < 0 { + return 0, fmt.Errorf("Unknown content length in request") + } + return int(cl), nil +} + +// Next should be used only inside interceptor. +// It executes the pending handlers inside the calling handler. +func (c *Context) Next() { + c.index++ + for c.index < int8(len(c.handlers)) { + c.handlers[c.index](c) + c.index++ + } +} + +// IsAborted return aborted or not +func (c *Context) IsAborted() bool { + return c.index >= abortIndex +} + +// Abort the next handlers +func (c *Context) Abort() { + c.index = abortIndex +} + +// AbortWithStatus abort with status +func (c *Context) AbortWithStatus(statusCode int) { + c.RespondStatus(statusCode) + c.Abort() +} + +// AbortWithStatusJSON abort with status and response data +func (c *Context) AbortWithStatusJSON(statusCode int, obj interface{}) { + c.RespondStatusData(statusCode, obj) + c.Abort() +} + +// AbortWithError abort with error +func (c *Context) AbortWithError(err error) { + c.RespondError(err) + c.Abort() +} + +// Respond response 200, and Content-Length: 0 +func (c *Context) Respond() { + c.Writer.Header().Set(HeaderContentLength, "0") + c.RespondStatus(http.StatusOK) +} + +// RespondStatus response status code +func (c *Context) RespondStatus(statusCode int) { + c.Writer.WriteHeader(statusCode) + c.wroteHeader = true +} + +// RespondError response error +func (c *Context) RespondError(err error) { + httpErr := Error2HTTPError(err) + if httpErr == nil { + c.Respond() + return + } + c.RespondStatusData(httpErr.StatusCode(), errorResponse{ + Error: httpErr.Error(), + Code: httpErr.ErrorCode(), + }) +} + +// RespondJSON response json +func (c *Context) RespondJSON(obj interface{}) { + c.RespondStatusData(http.StatusOK, obj) +} + +// RespondStatusData response data with code +func (c *Context) RespondStatusData(statusCode int, obj interface{}) { + body, ct, err := marshalObj(obj) + if err != nil { + c.RespondError(err) + return + } + c.RespondWith(statusCode, ct, body) +} + +// RespondWith response with code, content-type, bytes +func (c *Context) RespondWith(statusCode int, contentType string, body []byte) { + c.Writer.Header().Set(HeaderContentType, contentType) + c.Writer.Header().Set(HeaderContentLength, strconv.Itoa(len(body))) + + c.Writer.WriteHeader(statusCode) + c.wroteHeader = true + c.Writer.Write(body) +} + +// RespondWithReader response with code, content-length, content-type, an io.Reader and extra headers +func (c *Context) RespondWithReader(statusCode int, contentLength int, contentType string, + body io.Reader, extraHeaders map[string]string) { + c.Writer.Header().Set(HeaderContentType, contentType) + c.Writer.Header().Set(HeaderContentLength, strconv.Itoa(contentLength)) + for key, val := range extraHeaders { + c.Writer.Header().Set(key, val) + } + + c.Writer.WriteHeader(statusCode) + c.wroteHeader = true + io.Copy(c.Writer, body) +} + +// Stream sends a streaming response and returns a boolean +// indicates "Is client disconnected in middle of stream" +func (c *Context) Stream(step func(w io.Writer) bool) bool { + w := c.Writer + clientGone := c.Request.Context().Done() + for { + select { + case <-clientGone: + return true + default: + keepOpen := step(w) + c.Flush() + if !keepOpen { + return false + } + } + } +} + +// Set is used to store a new key/value pair exclusively for this context. +func (c *Context) Set(key string, val interface{}) { + c.mu.Lock() + if c.Meta == nil { + c.Meta = make(map[string]interface{}) + } + c.Meta[key] = val + c.mu.Unlock() +} + +// Get returns the value for the given key, +// If the value does not exists it returns (nil, false). +func (c *Context) Get(key string) (val interface{}, exists bool) { + c.mu.RLock() + val, exists = c.Meta[key] + c.mu.RUnlock() + return +} + +// RemoteIP parses the IP from Request.RemoteAddr, returns the net.IP (without the port). +func (c *Context) RemoteIP() (net.IP, bool) { + ip, _, err := net.SplitHostPort(strings.TrimSpace(c.Request.RemoteAddr)) + if err != nil { + return nil, false + } + remoteIP := net.ParseIP(ip) + if remoteIP == nil { + return nil, false + } + return remoteIP, true +} + +// Hijack implements the http.Hijacker interface. +func (c *Context) Hijack() (net.Conn, *bufio.ReadWriter, error) { + return c.Writer.(http.Hijacker).Hijack() +} + +// Flush implements the http.Flush interface. +func (c *Context) Flush() { + c.Writer.(http.Flusher).Flush() +} + +// Pusher implements the http.Pusher interface. +func (c *Context) Pusher() http.Pusher { + if pusher, ok := c.Writer.(http.Pusher); ok { + return pusher + } + return nil +} diff --git a/blobstore/common/rpc/context_test.go b/blobstore/common/rpc/context_test.go new file mode 100644 index 000000000..35cd2dc6a --- /dev/null +++ b/blobstore/common/rpc/context_test.go @@ -0,0 +1,155 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package rpc + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/util/log" +) + +const ( + addr = "127.0.0.1:9999" +) + +var cli = http.Client{Transport: &http.Transport{}} + +func setLogLevel() { + log.SetOutputLevel(log.Lerror) +} + +func init() { + mux := New() + mux.Handle(http.MethodGet, "/", func(c *Context) { + remoteIP, ok := c.RemoteIP() + if ok { + b, _ := remoteIP.MarshalText() + c.RespondWith(200, "", b) + } else { + c.RespondError(NewError(400, "", fmt.Errorf("no remote ip"))) + } + }) + mux.Handle(http.MethodGet, "/hijack", func(c *Context) { + conn, _, err := c.Hijack() + if err != nil { + c.RespondError(err) + return + } + conn.Close() + }) + mux.Handle(http.MethodGet, "/flush", func(c *Context) { + c.RespondWithReader(200, 1<<10, "", bytes.NewBuffer(make([]byte, 1<<10)), map[string]string{ + "x-rpc-name": "flush", + }) + c.Flush() + }) + mux.Handle(http.MethodGet, "/stream", func(c *Context) { + n := 0 + clientGone := c.Stream(func(w io.Writer) bool { + if n < 1<<20 { + n += 1 << 10 + w.Write(make([]byte, 1<<10)) + time.Sleep(10 * time.Microsecond) + return true + } + return false + }) + + if clientGone { + log.Infof("client gone") + } + }) + + httpServer := &http.Server{ + Addr: addr, + Handler: mux.Router, + } + go func() { + if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatal("server exits:", err) + } + }() + time.Sleep(time.Second) + + setLogLevel() +} + +func TestServerResponse(t *testing.T) { + { + url := fmt.Sprintf("http://%s/", addr) + req, _ := http.NewRequest(http.MethodGet, url, nil) + resp, err := cli.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + buf, _ := ioutil.ReadAll(resp.Body) + require.Equal(t, 200, resp.StatusCode) + require.Equal(t, "127.0.0.1", string(buf)) + } + { + url := fmt.Sprintf("http://%s/hijack", addr) + req, _ := http.NewRequest(http.MethodGet, url, nil) + resp, err := cli.Do(req) + require.Error(t, io.EOF, err) + if resp != nil { + resp.Body.Close() + } + } + { + url := fmt.Sprintf("http://%s/flush", addr) + req, _ := http.NewRequest(http.MethodGet, url, nil) + resp, err := cli.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + require.Equal(t, 200, resp.StatusCode) + require.Equal(t, "flush", resp.Header.Get("x-rpc-name")) + buf := make([]byte, 1<<20) + n, err := resp.Body.Read(buf) + require.ErrorIs(t, io.EOF, err) + require.Equal(t, 1024, n) + } + { + url := fmt.Sprintf("http://%s/stream", addr) + req, _ := http.NewRequest(http.MethodGet, url, nil) + resp, err := cli.Do(req) + require.NoError(t, err) + resp.Body.Close() + + require.Equal(t, 200, resp.StatusCode) + buf := make([]byte, 1<<22) + n, _ := resp.Body.Read(buf) + require.Equal(t, 0, n) + } + { + url := fmt.Sprintf("http://%s/stream", addr) + req, _ := http.NewRequest(http.MethodGet, url, nil) + resp, err := cli.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + require.Equal(t, 200, resp.StatusCode) + buf := make([]byte, 1<<18) + _, err = io.ReadFull(resp.Body, buf) + require.NoError(t, err) + } +} diff --git a/blobstore/common/rpc/error.go b/blobstore/common/rpc/error.go new file mode 100644 index 000000000..9e5465473 --- /dev/null +++ b/blobstore/common/rpc/error.go @@ -0,0 +1,158 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package rpc + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "strconv" + "syscall" +) + +type ( + // Error implements HTTPError + Error struct { + Status int // http status code + Code string // error code + Err error // error + } + + // errorResponse response error with json + // internal type between server and client + errorResponse struct { + Error string `json:"error"` + Code string `json:"code,omitempty"` + } + + statusCoder interface { + StatusCode() int + } + errorCoder interface { + ErrorCode() string + } +) + +var _ HTTPError = &Error{} + +// NewError new error with +func NewError(statusCode int, errCode string, err error) *Error { + return &Error{ + Status: statusCode, + Code: errCode, + Err: err, + } +} + +// StatusCode returns http status code +func (e *Error) StatusCode() int { + return e.Status +} + +// ErrorCode returns special defined code +func (e *Error) ErrorCode() string { + return e.Code +} + +// Error implements error +func (e *Error) Error() string { + if e.Err == nil { + return "" + } + return e.Err.Error() +} + +// Unwrap errors.Is(), errors.As() and errors.Unwrap() +func (e *Error) Unwrap() error { + return e.Err +} + +// DetectStatusCode returns http status code +func DetectStatusCode(err error) int { + if err == nil { + return http.StatusOK + } + + var st statusCoder + if errors.As(err, &st) { + return st.StatusCode() + } + + switch err { + case syscall.EINVAL: + return http.StatusBadRequest + case context.Canceled: + return 499 + } + return http.StatusInternalServerError +} + +// DetectErrorCode returns error code +func DetectErrorCode(err error) string { + if err == nil { + return "" + } + + var ec errorCoder + if errors.As(err, &ec) { + return ec.ErrorCode() + } + + switch err { + case syscall.EINVAL: + return "BadRequest" + case context.Canceled: + return "Canceled" + } + return "InternalServerError" +} + +// DetectError returns status code, error code, error +func DetectError(err error) (int, string, error) { + return DetectStatusCode(err), DetectErrorCode(err), errors.Unwrap(err) +} + +// Error2HTTPError returns an interface HTTPError from an error +func Error2HTTPError(err error) HTTPError { + if err == nil { + return nil + } + if httpErr, ok := err.(HTTPError); ok { + return httpErr + } + + status, code, _ := DetectError(err) + return NewError(status, code, err) +} + +// ReplyErr directly reply error with response writer +func ReplyErr(w http.ResponseWriter, code int, err string) { + msg, _ := json.Marshal(NewError(code, "", errors.New(err))) + h := w.Header() + h.Set("Content-Length", strconv.Itoa(len(msg))) + h.Set("Content-Type", MIMEJSON) + w.WriteHeader(code) + w.Write(msg) +} + +// ReplyWith directly reply body with response writer +func ReplyWith(w http.ResponseWriter, code int, bodyType string, msg []byte) { + h := w.Header() + h.Set("Content-Length", strconv.Itoa(len(msg))) + h.Set("Content-Type", bodyType) + w.WriteHeader(code) + w.Write(msg) +} diff --git a/blobstore/common/rpc/error_test.go b/blobstore/common/rpc/error_test.go new file mode 100644 index 000000000..2c9d7ef19 --- /dev/null +++ b/blobstore/common/rpc/error_test.go @@ -0,0 +1,109 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package rpc + +import ( + "context" + "errors" + "syscall" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestErrorBase(t *testing.T) { + { + err := NewError(0, "", nil) + require.Equal(t, 0, err.StatusCode()) + require.Equal(t, "", err.ErrorCode()) + require.Equal(t, "", err.Error()) + } + { + err := NewError(499, "Unknown", errors.New("unknown")) + require.Equal(t, 499, err.StatusCode()) + require.Equal(t, "Unknown", err.ErrorCode()) + require.Equal(t, "unknown", err.Error()) + } + { + errBase := NewError(400, "", errors.New("")) + err := NewError(499, "Unknown", errBase) + require.ErrorIs(t, errBase, errors.Unwrap(err)) + } + + { + require.Equal(t, 200, DetectStatusCode(nil)) + require.Equal(t, 222, DetectStatusCode(&Error{ + Status: 222, + Code: "CODE", + Err: errors.New("error"), + })) + require.Equal(t, 400, DetectStatusCode(syscall.EINVAL)) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.Equal(t, 499, DetectStatusCode(ctx.Err())) + + require.Equal(t, 500, DetectStatusCode(errors.New("server error"))) + } + { + require.Equal(t, "", DetectErrorCode(nil)) + require.Equal(t, "CODE", DetectErrorCode(&Error{ + Status: 222, + Code: "CODE", + Err: errors.New("error"), + })) + require.Equal(t, "BadRequest", DetectErrorCode(syscall.EINVAL)) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.Equal(t, "Canceled", DetectErrorCode(ctx.Err())) + + require.Equal(t, "InternalServerError", DetectErrorCode(errors.New("server error"))) + } + { + errBase := errors.New("error") + status, code, err := DetectError(&Error{ + Status: 222, + Code: "CODE", + Err: errBase, + }) + require.Equal(t, 222, status) + require.Equal(t, "CODE", code) + require.ErrorIs(t, errBase, err) + } + + { + httpErr := Error2HTTPError(nil) + require.Nil(t, httpErr) + } + { + errBase := errors.New("error") + httpErr := Error2HTTPError(&Error{ + Status: 222, + Code: "CODE", + Err: errBase, + }) + require.Equal(t, 222, httpErr.StatusCode()) + require.Equal(t, "CODE", httpErr.ErrorCode()) + require.Equal(t, "error", httpErr.Error()) + } + { + errBase := errors.New("error") + httpErr := Error2HTTPError(errBase) + require.Equal(t, 500, httpErr.StatusCode()) + require.Equal(t, "InternalServerError", httpErr.ErrorCode()) + require.Equal(t, "error", httpErr.Error()) + } +} diff --git a/blobstore/common/rpc/example/app.go b/blobstore/common/rpc/example/app.go new file mode 100644 index 000000000..a1f940895 --- /dev/null +++ b/blobstore/common/rpc/example/app.go @@ -0,0 +1,479 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package example + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "sync" + "time" + + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/trace" +) + +var ( + errExceed = rpc.NewError(http.StatusNotAcceptable, "MaxFiles", errors.New("exceed max files")) + errBadRequst = rpc.NewError(http.StatusBadRequest, "BadRequest", errors.New("bad request")) + errForbidden = rpc.NewError(http.StatusForbidden, "Forbidden", errors.New("forbidden")) + errNotFound = rpc.NewError(http.StatusNotFound, "NotFound", errors.New("not found")) +) + +// Mode file mode +type Mode uint8 + +// file mode +const ( + _ Mode = iota + ModeRW + ModeW + ModeR + ModeNone +) + +// FileApp app handlers +type FileApp interface { + Upload(*rpc.Context) + Update(*rpc.Context) + Delete(*rpc.Context) + Download(*rpc.Context) + Stream(*rpc.Context) + Exist(*rpc.Context) + Stat(*rpc.Context) + List(*rpc.Context) + OptionalArgs(*rpc.Context) +} + +// AppConfig app configure +type AppConfig struct { + MaxFiles int `json:"max_files"` + + SimpleConfig SimpleConfig `json:"simple_config"` + LBConfig LBConfig `json:"lb_config"` +} + +type fileStat struct { + Name string + Size int + Mode Mode + Desc []byte + Ctime time.Time + Mtime time.Time + Meta map[string]string +} + +type app struct { + mu sync.RWMutex + files map[string]*fileStat + + fileCli Client + metaCli Client + config AppConfig +} + +// NewApp new app +func NewApp(cfg AppConfig) FileApp { + return &app{ + files: make(map[string]*fileStat, cfg.MaxFiles), + fileCli: NewFileClient(&cfg.SimpleConfig), + metaCli: NewMetaClient(&cfg.LBConfig, nil), + config: cfg, + } +} + +// ArgsUpload args upload +// you should register the args to rpc, cos field name in tag +type ArgsUpload struct { + Name string `flag:"name"` // required + Size int `flag:"size"` // required + Mode Mode `flag:"mode"` // required + Desc []byte `flag:"desc,base64"` // required, base64 urlencode string + Nothing interface{} `flag:"-"` // ignored +} + +func (a *app) Upload(c *rpc.Context) { + span := trace.SpanFromContextSafe(c.Request.Context()) + + args := new(ArgsUpload) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + span.Debugf("receive upload request, args: %#v", args) + + if args.Name == "" || args.Mode > ModeNone { + c.RespondError(errBadRequst) + return + } + + a.mu.RLock() + if len(a.files) >= a.config.MaxFiles { + a.mu.RUnlock() + c.RespondError(errExceed) + return + } + a.mu.RUnlock() + + // append module cost, u should add track log before c.Respond* + startBody := time.Now().Add(-100 * time.Millisecond) + if err := a.fileCli.Write(c.Request.Context(), args.Size, c.Request.Body); err != nil { + span.AppendTrackLog("body", startBody, err) + c.RespondError(rpc.NewError(http.StatusGone, "ReadBody", err)) + return + } + span.AppendTrackLog("body", startBody, nil) + + var metaData []byte = []byte("meta") + // construct meta data, write by rpc to server + if err := a.metaCli.Write(c.Request.Context(), len(metaData), bytes.NewReader(metaData)); err != nil { + c.RespondError(rpc.NewError(http.StatusGone, "ReadMeta", err)) + return + } + + a.mu.Lock() + now := time.Now() + a.files[args.Name] = &fileStat{ + Name: args.Name, + Size: args.Size, + Mode: args.Mode, + Desc: args.Desc[:], + Ctime: now, + Mtime: now, + Meta: make(map[string]string), + } + a.mu.Unlock() + + c.Respond() +} + +// ArgsUpdate args update +// args in body, you can implement rpc.Unmarshaler to unmarshal +type ArgsUpdate struct { + Name string + Desc []byte + Meta map[string]string +} + +func (a *app) Update(c *rpc.Context) { + span := trace.SpanFromContextSafe(c.Request.Context()) + + args := new(ArgsUpdate) + if err := c.ArgsBody(args); err != nil { + c.RespondError(err) + return + } + span.Infof("update: %+v", args) + + if args.Name == "" { + c.RespondError(errBadRequst) + return + } + + a.mu.Lock() + defer a.mu.Unlock() + file, ok := a.files[args.Name] + if !ok { + c.RespondStatus(http.StatusNotFound) + return + } + + for k, v := range args.Meta { + file.Meta[k] = v + } + file.Desc = args.Desc[:] + file.Mtime = time.Now() + + c.Respond() +} + +// ArgsDelete args delete +// args in query string, the key in getter is lowercase of field name +type ArgsDelete struct { + Name string // key == name + Mode Mode // key == mode +} + +func (a *app) Delete(c *rpc.Context) { + span := trace.SpanFromContextSafe(c.Request.Context()) + + args := new(ArgsDelete) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + span.Infof("delete: %+v", args) + + if args.Name == "" { + c.RespondError(errBadRequst) + return + } + + a.mu.Lock() + defer a.mu.Unlock() + + file, ok := a.files[args.Name] + if !ok { + c.RespondStatus(http.StatusNoContent) + return + } + + if args.Mode > file.Mode { + c.RespondError(errForbidden) + return + } + + delete(a.files, args.Name) + c.Respond() +} + +// ArgsDownload args download +// you can define mulits tag on fields, but need to specify +// tag name's order in RegisterArgsParser +type ArgsDownload struct { + Name string `form:"name"` + Mode Mode `form:"mode"` + Offset int `formx:"offset,omitempty" flag:"xxx"` + Read int `formx:"read,omitempty" flag:"vvv"` +} + +func (a *app) Download(c *rpc.Context) { + span := trace.SpanFromContextSafe(c.Request.Context()) + + args := new(ArgsDownload) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + span.Debugf("receive download request, args: %#v", args) + + a.mu.RLock() + file, ok := a.files[args.Name] + a.mu.RUnlock() + + if args.Name == "" || args.Offset < 0 || args.Read < 0 || + args.Offset+args.Read > file.Size { + c.RespondError(errBadRequst) + return + } + + if !ok { + c.RespondError(errNotFound) + return + } + + if args.Mode > file.Mode { + c.RespondError(errForbidden) + return + } + + if args.Read == 0 { + c.Respond() + return + } + + span.AppendTrackLog("body", time.Now().Add(-1*time.Second), nil) + + reader, err := a.fileCli.Read(c.Request.Context(), args.Read) + if err != nil { + span.Error(err) + c.RespondError(err) + return + } + defer reader.Close() + + // Note: rpc server do not handler response write-error + if args.Offset+args.Read < file.Size { + c.RespondWithReader(http.StatusPartialContent, args.Read, rpc.MIMEStream, + reader, map[string]string{ + rpc.HeaderContentRange: fmt.Sprintf("bytes %d-%d/%d", + args.Offset, args.Offset+args.Read-1, file.Size), + }) + } else { + c.RespondWithReader(http.StatusOK, args.Read, rpc.MIMEStream, reader, nil) + } +} + +func (a *app) Stream(c *rpc.Context) { + span := trace.SpanFromContextSafe(c.Request.Context()) + c.RespondStatus(http.StatusMultiStatus) + if disconnected := c.Stream(func(w io.Writer) bool { + reader, err := a.fileCli.Read(c.Request.Context(), 1<<30) + if err != nil { + span.Warn(err) + return false + } + defer reader.Close() + + _, err = io.CopyN(w, reader, 1<<30) + if err != nil { + span.Warn(err) + return false + } + return true + }); disconnected { + span.Warn("client gone") + } +} + +// ArgsExist args exist or not +type ArgsExist struct { + Name string +} + +// Parse implements rpc.Parser, parse args by yourself +func (args *ArgsExist) Parse(getter rpc.ValueGetter) error { + name := getter("queryname") + if name == "" { + name = getter("name") + } + if name == "" { + return errors.New("empty name") + } + args.Name = name + return nil +} + +var _ rpc.Parser = &ArgsExist{} + +func (a *app) Exist(c *rpc.Context) { + span := trace.SpanFromContextSafe(c.Request.Context()) + args := new(ArgsExist) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + span.Infof("exist: %+v", args) + + a.mu.RLock() + file, ok := a.files[args.Name] + a.mu.RUnlock() + if !ok { + c.RespondError(errNotFound) + return + } + + c.Writer.Header().Set("x-file-name", file.Name) + c.Writer.Header().Set("x-file-size", strconv.Itoa(file.Size)) + c.Writer.Header().Set("x-file-mode", strconv.Itoa(int(file.Mode))) + c.Writer.Header().Set("x-file-desc", base64.URLEncoding.EncodeToString(file.Desc)) + c.Writer.Header().Set("x-file-ctime", file.Ctime.Format(time.RFC1123)) + c.Writer.Header().Set("x-file-mtime", file.Mtime.Format(time.RFC1123)) + + for k, v := range file.Meta { + c.Writer.Header().Set("x-file-meta-"+k, v) + } + + c.Respond() +} + +// RespStat args stat, use json tag +type RespStat struct { + Name string `json:"name"` + Size int `json:"size"` + Mode Mode + Desc []byte `json:"desc"` + Ctime int64 + Mtime int64 + Meta map[string]string +} + +var _ rpc.Marshaler = &RespStat{} + +// Marshal implements rpc.Marshaler, define you own marshaler +func (stat *RespStat) Marshal() ([]byte, string, error) { + b, err := json.Marshal(stat) + return b, "application/x-json", err +} + +func (a *app) Stat(c *rpc.Context) { + span := trace.SpanFromContextSafe(c.Request.Context()) + args := new(ArgsExist) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + span.Infof("stat: %+v", args) + + a.mu.RLock() + defer a.mu.RUnlock() + file, ok := a.files[args.Name] + if !ok { + c.RespondError(errNotFound) + return + } + + resp := RespStat{ + Name: file.Name, + Size: file.Size, + Mode: file.Mode, + Desc: make([]byte, len(file.Desc)), + Ctime: file.Ctime.Unix(), + Mtime: file.Mtime.Unix(), + Meta: make(map[string]string, len(file.Meta)), + } + copy(resp.Desc, file.Desc) + for k, v := range file.Meta { + resp.Meta[k] = v + } + + c.RespondJSON(resp) +} + +func (a *app) List(c *rpc.Context) { + a.mu.RLock() + defer a.mu.RUnlock() + + files := make([]RespStat, 0, len(a.files)) + for _, file := range a.files { + resp := RespStat{ + Name: file.Name, + Size: file.Size, + Mode: file.Mode, + Desc: make([]byte, len(file.Desc)), + Ctime: file.Ctime.Unix(), + Mtime: file.Mtime.Unix(), + Meta: make(map[string]string, len(file.Meta)), + } + copy(resp.Desc, file.Desc) + for k, v := range file.Meta { + resp.Meta[k] = v + } + + files = append(files, resp) + } + + c.RespondStatusData(http.StatusOK, files) +} + +// ArgsURIOptional argument in uri with omitempty +type ArgsURIOptional struct { + Require string `json:"require"` + Option string `json:"option,omitempty"` +} + +func (a *app) OptionalArgs(c *rpc.Context) { + args := new(ArgsURIOptional) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + c.RespondJSON(args) +} diff --git a/blobstore/common/rpc/example/client.go b/blobstore/common/rpc/example/client.go new file mode 100644 index 000000000..2788ebe67 --- /dev/null +++ b/blobstore/common/rpc/example/client.go @@ -0,0 +1,103 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package example + +import ( + "context" + "fmt" + "io" + "net/http" + + "github.com/cubefs/cubefs/blobstore/common/rpc" +) + +// Client read and write client +type Client interface { + Write(context.Context, int, io.Reader) error + Read(context.Context, int) (io.ReadCloser, error) +} + +// SimpleConfig simple client config +type SimpleConfig struct { + Host string `json:"host"` + Config rpc.Config `json:"rpc_config"` +} + +// NewFileClient returns file client +func NewFileClient(conf *SimpleConfig) Client { + return &fileClient{conf.Host, rpc.NewClient(&conf.Config)} +} + +type fileClient struct { + host string + rpc.Client +} + +func (fc *fileClient) Write(ctx context.Context, size int, body io.Reader) (err error) { + request, err := http.NewRequest(http.MethodPut, fmt.Sprintf("%s/write/%d", fc.host, size), body) + if err != nil { + return + } + return fc.DoWith(ctx, request, nil, rpc.WithCrcEncode()) +} + +func (fc *fileClient) Read(ctx context.Context, size int) (io.ReadCloser, error) { + resp, err := fc.Get(ctx, fmt.Sprintf("%s/read/%d", fc.host, size)) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("invalid httpcode %d", resp.StatusCode) + } + return resp.Body, nil +} + +// LBConfig lb client config +type LBConfig struct { + Config rpc.LbConfig `json:"rpc_lb_config"` +} + +// NewMetaClient returns meta client +func NewMetaClient(conf *LBConfig, selector rpc.Selector) Client { + return &metaClient{rpc.NewLbClient(&conf.Config, selector)} +} + +type metaClient struct { + rpc.Client +} + +func (fc *metaClient) Write(ctx context.Context, size int, body io.Reader) (err error) { + request, err := http.NewRequest(http.MethodPut, fmt.Sprintf("/write/%d", size), body) + if err != nil { + return + } + resp, err := fc.Do(ctx, request) + if err != nil { + return + } + resp.Body.Close() + return +} + +func (fc *metaClient) Read(ctx context.Context, size int) (io.ReadCloser, error) { + resp, err := fc.Get(ctx, fmt.Sprintf("/read/%d", size)) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("invalid httpcode %d", resp.StatusCode) + } + return resp.Body, nil +} diff --git a/blobstore/common/rpc/example/file.go b/blobstore/common/rpc/example/file.go new file mode 100644 index 000000000..e98faa5fd --- /dev/null +++ b/blobstore/common/rpc/example/file.go @@ -0,0 +1,129 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package example + +import ( + "io" + "net/http" + "time" + + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/trace" +) + +// Filer file interface +type Filer interface { + Write(*rpc.Context) + Read(*rpc.Context) +} + +type file struct{} + +// NewFile new file +func NewFile() Filer { + return &file{} +} + +type noopReader struct { + size int + io.Reader +} + +func (r *noopReader) Read(p []byte) (int, error) { + if len(p) >= r.size { + size := r.size + r.size = 0 + return size, io.EOF + } + r.size -= len(p) + return len(p), nil +} + +type noopWriter struct { + size int + io.Writer +} + +func (w *noopWriter) Write(p []byte) (int, error) { + if len(p) >= w.size { + size := w.size + w.size = 0 + return size, io.EOF + } + w.size -= len(p) + return len(p), nil +} + +// ArgsWrite args upload +type ArgsWrite struct { + Size int `flag:"size"` +} + +func (f *file) Write(c *rpc.Context) { + span := trace.SpanFromContextSafe(c.Request.Context()) + + args := new(ArgsWrite) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + span.Debugf("receive file write request, args: %#v", args) + + if args.Size <= 0 { + c.RespondError(errBadRequst) + return + } + + // do some trace tags + span.SetTag("limit-key", "limit-write") + + start := time.Now() + w := &noopWriter{size: args.Size} + if _, err := io.CopyN(w, c.Request.Body, int64(args.Size)); err != nil { + span.AppendTrackLog("copy", start, err) + c.RespondError(rpc.NewError(http.StatusGone, "ReadBody", err)) + return + } + span.AppendTrackLog("copy", start, nil) + + c.Respond() +} + +// ArgsRead args read +type ArgsRead struct { + Size int +} + +func (*file) Read(c *rpc.Context) { + span := trace.SpanFromContextSafe(c.Request.Context()) + + args := new(ArgsRead) + if err := c.ParseArgs(args); err != nil { + c.RespondError(err) + return + } + span.Debugf("receive file read request, args: %#v", args) + + if args.Size < 0 { + c.RespondError(errBadRequst) + return + } + + // do something + span.SetTag("test-for-nil", nil) + + c.RespondWithReader(http.StatusOK, args.Size, rpc.MIMEStream, + &noopReader{size: args.Size}, nil) +} diff --git a/blobstore/common/rpc/example/main/main.conf b/blobstore/common/rpc/example/main/main.conf new file mode 100644 index 000000000..e09051487 --- /dev/null +++ b/blobstore/common/rpc/example/main/main.conf @@ -0,0 +1,40 @@ +{ + "log_level": "debug", + "auditlog": { + "logdir": "/tmp/examples/", + "rotate_new": false + }, + "app": { + "bind_addr": ":9999", + "app": { + "max_files": 32, + "lb_config": { + "rpc_lb_config": { + "hosts": ["http://127.0.0.1:9997"], + "try_times": 2 + } + }, + "simple_config": { + "host": "http://127.0.0.1:9998", + "rpc_config": { + "client_timeout_ms": 10000, + "transport_config": { + "dial_timeout_ms": 1000, + "disable_compression": true, + "idle_conn_timeout_ms": 60000, + "max_conns_per_host": 100, + "max_idle_conns": 100, + "max_idle_conns_per_host": 10, + "response_header_timeout_ms": 3000 + } + } + } + } + }, + "file": { + "bind_addr": ":9998" + }, + "meta": { + "bind_addr": ":9997" + } +} diff --git a/blobstore/common/rpc/example/main/main.go b/blobstore/common/rpc/example/main/main.go new file mode 100644 index 000000000..a086647ec --- /dev/null +++ b/blobstore/common/rpc/example/main/main.go @@ -0,0 +1,103 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package main + +import ( + "net/http" + "os" + + "github.com/cubefs/cubefs/blobstore/common/config" + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/rpc/auditlog" + "github.com/cubefs/cubefs/blobstore/common/rpc/example" + "github.com/cubefs/cubefs/blobstore/util/log" +) + +// Config service config +type Config struct { + LogLevel log.Level `json:"log_level"` + AuditLog auditlog.Config `json:"auditlog"` + + App struct { + BindAddr string `json:"bind_addr"` + App example.AppConfig `json:"app"` + } `json:"app"` + File struct { + BindAddr string `json:"bind_addr"` + } `json:"file"` + Meta struct { + BindAddr string `json:"bind_addr"` + } `json:"meta"` +} + +func main() { + config.Init("f", "", "main.conf") + + var conf Config + err := config.Load(&conf) + if err != nil { + log.Fatal(err) + } + log.SetOutputLevel(conf.LogLevel) + + os.MkdirAll(conf.AuditLog.LogDir, 0o644) + lh, lf, _ := auditlog.Open("RPC", &conf.AuditLog) + defer lf.Close() + + { + c := conf.Meta + httpServer := &http.Server{ + Addr: c.BindAddr, + Handler: rpc.MiddlewareHandlerWith(example.NewMetaHandler(), lh), + } + + log.Info("meta server is running at:", c.BindAddr) + go func() { + if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Panic("server exits:", err) + } + }() + } + { + c := conf.File + httpServer := &http.Server{ + Addr: c.BindAddr, + Handler: rpc.MiddlewareHandlerWith(example.NewFileHandler(), lh), + } + + log.Info("file server is running at:", c.BindAddr) + go func() { + if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Panic("server exits:", err) + } + }() + } + { + c := conf.App + httpServer := &http.Server{ + Addr: c.BindAddr, + Handler: rpc.MiddlewareHandlerWith(example.NewAppHandler(c.App), lh), + } + + log.Info("app server is running at:", c.BindAddr) + go func() { + if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Panic("server exits:", err) + } + }() + } + + <-make(chan struct{}) +} diff --git a/blobstore/common/rpc/example/service.go b/blobstore/common/rpc/example/service.go new file mode 100644 index 000000000..66d0764fe --- /dev/null +++ b/blobstore/common/rpc/example/service.go @@ -0,0 +1,112 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package example + +import ( + "net/http" + "time" + + "github.com/cubefs/cubefs/blobstore/common/rpc" + "github.com/cubefs/cubefs/blobstore/common/trace" +) + +type metaValue struct { + val int +} + +func interceptorWholeRequest(c *rpc.Context) { + span := trace.SpanFromContextSafe(c.Request.Context()) + + c.Set("var", &metaValue{0}) + start := time.Now() + c.Next() + metaVal, _ := c.Get("var") + val := metaVal.(*metaValue) + span.Infof("Request %s: done Val=%d, spent %s", c.Request.URL.Path, val.val, time.Since(start).String()) +} + +func interceptorNothing1(c *rpc.Context) { + span := trace.SpanFromContextSafe(c.Request.Context()) + + metaVal, _ := c.Get("var") + val := metaVal.(*metaValue) + val.val += 10 + span.Infof("Request %s: interceptor nothing-1 Val=%d", c.Request.URL.Path, val.val) +} + +func interceptorNothing2(c *rpc.Context) { + span := trace.SpanFromContextSafe(c.Request.Context()) + + metaVal, _ := c.Get("var") + val := metaVal.(*metaValue) + val.val++ + span.Debugf("Request %s: interceptor nothing-2 Val=%d", c.Request.URL.Path, val.val) +} + +// NewAppHandler returns app server handler, use rpc.DefaultRouter +func NewAppHandler(cfg AppConfig) *rpc.Router { + // must be register args with tag + rpc.RegisterArgsParser(&ArgsUpload{}, "flag") + rpc.RegisterArgsParser(&ArgsDelete{}) + rpc.RegisterArgsParser(&ArgsDownload{}, "form", "formx") + rpc.RegisterArgsParser(&ArgsExist{}) + rpc.RegisterArgsParser(&ArgsURIOptional{}, "json") + + app := NewApp(cfg) + + rpc.Use(interceptorWholeRequest) // first interceptor + rpc.Use(interceptorNothing1, interceptorNothing2) // second, then third + + // args key/val in c.Param + rpc.PUT("/upload/:name/:size/:mode/:desc", app.Upload, rpc.OptArgsURI()) + // parse args in handler by c.ArgsBody + rpc.POST("/update", app.Update) + rpc.DELETE("/delete", app.Delete, rpc.OptArgsQuery()) + rpc.POST("/download", app.Download, rpc.OptArgsForm()) + rpc.GET("/stream", app.Stream) + // param priority is uri > query > form > postfrom by default + // but you can define you own rpc.Parser on args struct + rpc.HEAD("/exist/:name", app.Exist, rpc.OptArgsURI(), rpc.OptArgsQuery()) + rpc.GET("/stat/:name", app.Stat, rpc.OptArgsURI(), rpc.OptArgsQuery()) + // initialized 8 room for key/val map of c.Meta + rpc.GET("/list", app.List, rpc.OptMetaCapacity(8)) + + // argument in uri with option + rpc.GET("/args/:require/:option", app.OptionalArgs, rpc.OptArgsURI()) + // argument in uri without option + rpc.GET("/args/:require", app.OptionalArgs, rpc.OptArgsURI()) + + return rpc.DefaultRouter +} + +// NewFileHandler returns file server handler +func NewFileHandler() *rpc.Router { + rpc.RegisterArgsParser(&ArgsWrite{}, "flag") + + router := rpc.New() + filer := NewFile() + router.Handle(http.MethodPut, "/write/:size", filer.Write, rpc.OptArgsURI()) + router.Handle(http.MethodGet, "/read/:size", filer.Read, rpc.OptArgsURI()) + return router +} + +// NewMetaHandler returns meta server handler +func NewMetaHandler() *rpc.Router { + router := rpc.New() + filer := NewFile() + router.Handle(http.MethodPut, "/write/:size", filer.Write, rpc.OptArgsURI()) + router.Handle(http.MethodGet, "/read/:size", filer.Read, rpc.OptArgsURI()) + return router +} diff --git a/blobstore/common/rpc/middleware.go b/blobstore/common/rpc/middleware.go new file mode 100644 index 000000000..69ba159d2 --- /dev/null +++ b/blobstore/common/rpc/middleware.go @@ -0,0 +1,60 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package rpc + +import "net/http" + +// MiddlewareHandler middleware above rpc server default router. +// Run sorted by progress handler order. +func MiddlewareHandler(phs ...ProgressHandler) http.Handler { + DefaultRouter.hasMiddleware = true + phs = append(DefaultRouter.headMiddlewares, phs...) + return buildHTTPHandler(DefaultRouter.ServeHTTP, phs...) +} + +// MiddlewareHandlerFunc middleware func above rpc server default router. +// Run sorted by progress handler order. +func MiddlewareHandlerFunc(phs ...ProgressHandler) http.HandlerFunc { + DefaultRouter.hasMiddleware = true + phs = append(DefaultRouter.headMiddlewares, phs...) + return buildHTTPHandler(DefaultRouter.ServeHTTP, phs...) +} + +// MiddlewareHandlerWith middleware above rpc server router +// Run sorted by progress handler order. +func MiddlewareHandlerWith(r *Router, phs ...ProgressHandler) http.Handler { + r.hasMiddleware = true + phs = append(r.headMiddlewares, phs...) + return buildHTTPHandler(r.ServeHTTP, phs...) +} + +// MiddlewareHandlerFuncWith middleware func above rpc server router +// Run sorted by progress handler order. +func MiddlewareHandlerFuncWith(r *Router, phs ...ProgressHandler) http.HandlerFunc { + r.hasMiddleware = true + phs = append(r.headMiddlewares, phs...) + return buildHTTPHandler(r.ServeHTTP, phs...) +} + +func buildHTTPHandler(h http.HandlerFunc, phs ...ProgressHandler) http.HandlerFunc { + if len(phs) == 0 { + return h + } + + last := len(phs) - 1 + return buildHTTPHandler(func(w http.ResponseWriter, req *http.Request) { + phs[last].Handler(w, req, h) + }, phs[:last]...) +} diff --git a/blobstore/common/rpc/middleware_test.go b/blobstore/common/rpc/middleware_test.go new file mode 100644 index 000000000..299931261 --- /dev/null +++ b/blobstore/common/rpc/middleware_test.go @@ -0,0 +1,198 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package rpc + +import ( + "errors" + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/require" +) + +type ( + testMiddleware struct { + Name string + RunIn func() error + RunOut func() error + ProgressHandler + } +) + +func (mw testMiddleware) Handler(w http.ResponseWriter, req *http.Request, f func(http.ResponseWriter, *http.Request)) { + fmt.Println("In Middleware", mw.Name) + if err := mw.RunIn(); err != nil { + return + } + f(w, req) + if err := mw.RunOut(); err != nil { + return + } + fmt.Println("Out Middleware", mw.Name) +} + +func TestMiddlewareBase(t *testing.T) { + { + var routed string + router := New() + router.Handle(http.MethodGet, "/", func(c *Context) { routed += "app" }) + handler := MiddlewareHandlerWith(router) + + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodGet, "/", nil) + handler.ServeHTTP(w, req) + require.Equal(t, "app", routed) + } + { + var routed string + router := New() + router.Handle(http.MethodGet, "/", func(c *Context) { routed += "app" }) + handler := MiddlewareHandlerFuncWith(router, + testMiddleware{ + Name: "middleware1", + RunIn: func() error { + routed += "1-in" + return nil + }, + RunOut: func() error { + routed += "1-out" + return nil + }, + }, + ) + + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodGet, "/", nil) + handler.ServeHTTP(w, req) + require.Equal(t, "1-inapp1-out", routed) + } + { + var routed string + router := New() + router.Handle(http.MethodGet, "/", func(c *Context) { routed += "app" }) + handler := MiddlewareHandlerWith(router, + testMiddleware{ + Name: "middleware1", + RunIn: func() error { + routed += "1-in" + return errors.New("") + }, + RunOut: func() error { + routed += "1-out" + return nil + }, + }, + ) + + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodGet, "/", nil) + handler.ServeHTTP(w, req) + require.Equal(t, "1-in", routed) + } + { + var routed string + router := New() + router.Handle(http.MethodGet, "/", func(c *Context) { routed += "app" }) + + handler := MiddlewareHandlerWith(router, + testMiddleware{ + Name: "middleware1", + RunIn: func() error { + routed += "1-in" + return nil + }, + RunOut: func() error { + routed += "1-out" + return nil + }, + }, + testMiddleware{ + Name: "middleware2", + RunIn: func() error { + routed += "2-in" + return nil + }, + RunOut: func() error { + routed += "2-out" + return errors.New("") + }, + }, + testMiddleware{ + Name: "middleware3", + RunIn: func() error { + routed += "3-in" + return errors.New("") + }, + RunOut: func() error { + routed += "3-out" + return errors.New("") + }, + }, + testMiddleware{ + Name: "middleware4", + RunIn: func() error { + routed += "4-in" + return errors.New("") + }, + RunOut: func() error { + routed += "4-out" + return nil + }, + }, + ) + + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodGet, "/", nil) + handler.ServeHTTP(w, req) + require.Equal(t, "1-in2-in3-in2-out1-out", routed) + } +} + +func TestMiddlewareDefault(t *testing.T) { + defer initDefaultRouter() + { + var routed string + GET("/", func(c *Context) { routed += "app" }) + handler := MiddlewareHandler() + + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodGet, "/", nil) + handler.ServeHTTP(w, req) + require.Equal(t, "app", routed) + } + { + var routed string + GET("/get", func(c *Context) { routed += "app" }) + handler := MiddlewareHandlerFunc( + testMiddleware{ + Name: "middleware1", + RunIn: func() error { + routed += "1-in" + return nil + }, + RunOut: func() error { + routed += "1-out" + return nil + }, + }, + ) + + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodGet, "/get", nil) + handler.ServeHTTP(w, req) + require.Equal(t, "1-inapp1-out", routed) + } +} diff --git a/blobstore/common/rpc/proto.go b/blobstore/common/rpc/proto.go new file mode 100644 index 000000000..49a85d9ef --- /dev/null +++ b/blobstore/common/rpc/proto.go @@ -0,0 +1,148 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package rpc + +import ( + "crypto/md5" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "path" + "runtime" + "strings" + + "github.com/cubefs/cubefs/blobstore/util/version" +) + +// headers +const ( + HeaderContentType = "Content-Type" + HeaderContentLength = "Content-Length" + HeaderContentRange = "Content-Range" + HeaderContentMD5 = "Content-MD5" + HeaderUA = "User-Agent" + + // trace + HeaderTraceLog = "Trace-Log" + HeaderTraceTags = "Trace-Tags" + + // crc checker + HeaderCrcEncoded = "X-Crc-Encoded" + HeaderAckCrcEncoded = "X-Ack-Crc-Encoded" +) + +// mime +const ( + MIMEStream = "application/octet-stream" + MIMEJSON = "application/json" + MIMEXML = "application/xml" + MIMEPlain = "text/plain" + MIMEPOSTForm = "application/x-www-form-urlencoded" + MIMEMultipartPOSTForm = "multipart/form-data" + MIMEYAML = "application/x-yaml" +) + +// encoding +const ( + GzipEncodingType = "gzip" +) + +// UserAgent user agent +var UserAgent = "Golang blobstore/rpc package" + +type ( + // ValueGetter get value from url values or http params + ValueGetter func(string) string + // Parser is the interface implemented by types + // that can parse themselves from url.Values. + Parser interface { + Parse(ValueGetter) error + } + // Marshaler is the interface implemented by types that + // can marshal themselves into bytes, second parameter + // is content type. + Marshaler interface { + Marshal() ([]byte, string, error) + } + // Unmarshaler is the interface implemented by types + // that can unmarshal themselves from bytes. + Unmarshaler interface { + Unmarshal([]byte) error + } + + // HTTPError interface of error with http status code + HTTPError interface { + // StatusCode http status code + StatusCode() int + // ErrorCode special defined code + ErrorCode() string + // Error detail message + Error() string + } +) + +// ProgressHandler http progress handler +type ProgressHandler interface { + Handler(http.ResponseWriter, *http.Request, func(http.ResponseWriter, *http.Request)) +} + +func marshalObj(obj interface{}) ([]byte, string, error) { + var ( + body []byte + ct string = MIMEJSON + err error + ) + if obj == nil { + body = jsonNull[:] + } else if o, ok := obj.(Marshaler); ok { + body, ct, err = o.Marshal() + } else { + body, err = json.Marshal(obj) + } + return body, ct, err +} + +func programVersion() string { + sp := strings.Fields(strings.TrimSpace(version.Version())) + if len(sp) == 0 || sp[0] == "develop" { + data, err := ioutil.ReadFile(os.Args[0]) + if err != nil { + return "_" + } + return fmt.Sprintf("%x", md5.Sum(data))[:10] + } + if len(sp) > 10 { + return sp[0][:10] + } + return sp[0] +} + +func init() { + hostname, _ := os.Hostname() + ua := fmt.Sprintf("%s/%s (%s/%s; %s) %s/%s", + path.Base(os.Args[0]), + programVersion(), + runtime.GOOS, + runtime.GOARCH, + runtime.Version(), + hostname, + fmt.Sprint(os.Getpid()), + ) + if UserAgent != ua { + UserAgent = ua + } +} diff --git a/blobstore/common/rpc/recovery.go b/blobstore/common/rpc/recovery.go new file mode 100644 index 000000000..46fe85c7b --- /dev/null +++ b/blobstore/common/rpc/recovery.go @@ -0,0 +1,77 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package rpc + +import ( + "bytes" + "fmt" + "net" + "net/http" + "os" + "runtime" + "strings" + + "github.com/cubefs/cubefs/blobstore/util/log" +) + +// defaultRecovery logging panic info, then panic to next handler +func defaultRecovery(w http.ResponseWriter, req *http.Request, err interface{}) { + var brokenPipe bool + if ne, ok := err.(*net.OpError); ok { + if se, ok := ne.Err.(*os.SyscallError); ok { + if strings.Contains(strings.ToLower(se.Error()), "broken pipe") || + strings.Contains(strings.ToLower(se.Error()), "connection reset by peer") { + brokenPipe = true + } + } + } + + stack := stack(3) + if brokenPipe { + log.Warnf("handle panic: %s on broken pipe\n%s", err, stack) + } else { + log.Errorf("handle panic: %s\n%s", err, stack) + panic(err) + } +} + +func stack(skip int) []byte { + buf := new(bytes.Buffer) + for i := skip; ; i++ { + pc, file, line, ok := runtime.Caller(i) + if !ok { + break + } + fmt.Fprintf(buf, "%s:%d (0x%x:%s)\n", file, line, pc, funcname(pc)) + } + return buf.Bytes() +} + +// funcname returns the name of the function +func funcname(pc uintptr) []byte { + fn := runtime.FuncForPC(pc) + if fn == nil { + return []byte("???") + } + name := []byte(fn.Name()) + + if last := bytes.LastIndex(name, []byte("/")); last >= 0 { + name = name[last+1:] + } + if first := bytes.Index(name, []byte(".")); first >= 0 { + name = name[first+1:] + } + return name +} diff --git a/blobstore/common/rpc/recovery_test.go b/blobstore/common/rpc/recovery_test.go new file mode 100644 index 000000000..05d644f81 --- /dev/null +++ b/blobstore/common/rpc/recovery_test.go @@ -0,0 +1,94 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package rpc + +import ( + "errors" + "net" + "net/http" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestServerRecovery(t *testing.T) { + { + router := New() + router.Router.PanicHandler = defaultRecovery + router.Use(func(c *Context) { + panic("in interceptor") + }) + router.Handle(http.MethodGet, "/", func(c *Context) {}) + + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodGet, "/", nil) + require.Panics(t, func() { + router.ServeHTTP(w, req) + }) + } + { + router := New() + router.Router.PanicHandler = defaultRecovery + router.Use(func(c *Context) {}) + router.Handle(http.MethodGet, "/", func(c *Context) { + panic("in app") + }) + + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodGet, "/", nil) + require.Panics(t, func() { + router.ServeHTTP(w, req) + }) + } + { + router := New() + router.Router.PanicHandler = defaultRecovery + router.Use(func(c *Context) {}) + router.Handle(http.MethodGet, "/", func(c *Context) { + panic(&net.OpError{ + Err: &os.SyscallError{ + Syscall: "App", + Err: errors.New("broken pipe"), + }, + }) + }) + + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodGet, "/", nil) + require.NotPanics(t, func() { + router.ServeHTTP(w, req) + }) + } + { + router := New() + router.Router.PanicHandler = defaultRecovery + router.Use(func(c *Context) { + panic(&net.OpError{ + Err: &os.SyscallError{ + Syscall: "Middleware", + Err: errors.New("connection reset by peer"), + }, + }) + }) + router.Handle(http.MethodGet, "/", func(c *Context) {}) + + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodGet, "/", nil) + require.NotPanics(t, func() { + router.ServeHTTP(w, req) + }) + } +} diff --git a/blobstore/common/rpc/route.go b/blobstore/common/rpc/route.go new file mode 100644 index 000000000..b98902052 --- /dev/null +++ b/blobstore/common/rpc/route.go @@ -0,0 +1,170 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package rpc + +import ( + "net/http" + "reflect" + "runtime" + + "github.com/julienschmidt/httprouter" + + "github.com/cubefs/cubefs/blobstore/util/log" +) + +type ( + // Router router with interceptors + // Interceptor is middleware for http serve but named `interceptor`. + // Middleware within this Router called `interceptor`. + // + // headMiddlewares is middlewares run firstly. + // Running order is: + // headMiddlewares --> middlewares --> interceptors --> handler + // + // example: + // router := New() + // router.Use(interceptor1, interceptor2) + // router.Handle(http.MethodGet, "/get/:name", handlerGet) + // router.Handle(http.MethodPut, "/put/:name", handlerPut) + Router struct { + Router *httprouter.Router // router + hasMiddleware bool // true if the router with Middleware* + headMiddlewares []ProgressHandler // middlewares run firstly of all + headHandler http.HandlerFunc // run thie handler if has no middlewares + interceptors []HandlerFunc // interceptors after middlewares + } +) + +// ServeHTTP makes the router implement the http.Handler interface. +func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if !r.hasMiddleware { + r.headHandler(w, req) + return + } + r.Router.ServeHTTP(w, req) +} + +// DefaultRouter default router for server +var DefaultRouter *Router + +func init() { + initDefaultRouter() +} + +func initDefaultRouter() { + DefaultRouter = New() + DefaultRouter.Router.PanicHandler = defaultRecovery +} + +// New alias of httprouter.New +// Return a Router, control by yourself +func New() *Router { + r := &Router{ + Router: httprouter.New(), + hasMiddleware: false, + headMiddlewares: []ProgressHandler{&crcDecoder{}}, + } + r.headHandler = buildHTTPHandler(r.Router.ServeHTTP, r.headMiddlewares...) + return r +} + +// Use attaches a global interceptor to the router. +// You should Use interceptor before register handler. +// It is sorted by registered order. +func (r *Router) Use(interceptors ...HandlerFunc) { + if len(r.interceptors)+len(interceptors) >= int(abortIndex) { + panic("too many regiter handlers") + } + r.interceptors = append(r.interceptors, interceptors...) +} + +// Handle registers a new request handle with the given path and method. +// +// For HEAD, GET, POST, PUT, PATCH and DELETE requests the respective shortcut +// functions can be used. +func (r *Router) Handle(method, path string, handler HandlerFunc, opts ...ServerOption) { + // Notice: in golang, sentence [ sliceA := append(sliceA, item) ] + // the pointer of sliceA is the pointer of sliceB, if sliceB has enough capacity. + // so we need make a new slice. + handlers := make([]HandlerFunc, 0, len(r.interceptors)+1) + handlers = append(handlers, r.interceptors...) + handlers = append(handlers, handler) + if len(handlers) >= int(abortIndex) { + panic("too many regiter handlers") + } + r.Router.Handle(method, path, makeHandler(handlers, opts...)) + + opt := new(serverOptions) + for _, o := range opts { + o.apply(opt) + } + icnames := make([]string, 0, len(r.interceptors)) + for _, ic := range r.interceptors { + icnames = append(icnames, runtime.FuncForPC(reflect.ValueOf(ic).Pointer()).Name()) + } + name := runtime.FuncForPC(reflect.ValueOf(handler).Pointer()).Name() + log.Infof("register handler method:%s, path:%s, interceptors:%s, handler:%s, opts:%+v", + method, path, icnames, name, opt) +} + +// Use attaches a global interceptor to the default router. +// You should Use interceptor before register handler. +// It is sorted by registered order. +func Use(interceptors ...HandlerFunc) { + DefaultRouter.interceptors = append(DefaultRouter.interceptors, interceptors...) +} + +// HEAD is a shortcut for Handle(http.MethodHead, path, handle) +func HEAD(path string, handler HandlerFunc, opts ...ServerOption) { + Handle(http.MethodHead, path, handler, opts...) +} + +// GET is a shortcut for Handle(http.MethodGet, path, handle) +func GET(path string, handler HandlerFunc, opts ...ServerOption) { + Handle(http.MethodGet, path, handler, opts...) +} + +// POST is a shortcut for Handle(http.MethodPost, path, handle) +func POST(path string, handler HandlerFunc, opts ...ServerOption) { + Handle(http.MethodPost, path, handler, opts...) +} + +// PUT is a shortcut for Handle(http.MethodPut, path, handle) +func PUT(path string, handler HandlerFunc, opts ...ServerOption) { + Handle(http.MethodPut, path, handler, opts...) +} + +// DELETE is a shortcut for Handle(http.MethodDelete, path, handle) +func DELETE(path string, handler HandlerFunc, opts ...ServerOption) { + Handle(http.MethodDelete, path, handler, opts...) +} + +// OPTIONS is a shortcut for Handle(http.MethodOptions, path, handle) +func OPTIONS(path string, handler HandlerFunc, opts ...ServerOption) { + Handle(http.MethodOptions, path, handler, opts...) +} + +// PATCH is a shortcut for Handle(http.MethodPatch, path, handle) +func PATCH(path string, handler HandlerFunc, opts ...ServerOption) { + Handle(http.MethodPatch, path, handler, opts...) +} + +// Handle registers a new request handle with the given path and method. +// +// For HEAD, GET, POST, PUT, PATCH and DELETE requests the respective shortcut +// functions can be used. +func Handle(method, path string, handler HandlerFunc, opts ...ServerOption) { + DefaultRouter.Handle(method, path, handler, opts...) +} diff --git a/blobstore/common/rpc/route_test.go b/blobstore/common/rpc/route_test.go new file mode 100644 index 000000000..f4a0cc59b --- /dev/null +++ b/blobstore/common/rpc/route_test.go @@ -0,0 +1,82 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package rpc + +import ( + "net/http" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/util/log" +) + +func TestServerRouterDefault(t *testing.T) { + log.SetOutputLevel(log.Linfo) + defer setLogLevel() + defer initDefaultRouter() + + str := "" + Use(func(c *Context) { + str += "m1" + }) + Use(func(c *Context) { + c.Next() + str += "m2" + }) + + HEAD("/head", func(c *Context) { str += "head" }) + GET("/get", func(c *Context) { str += "get" }) + POST("/post", func(c *Context) { str += "post" }) + PUT("/put", func(c *Context) { str += "put" }) + DELETE("/delete", func(c *Context) { str += "delete" }) + OPTIONS("/options", func(c *Context) { str += "options" }) + PATCH("/patch", func(c *Context) { str += "patch" }) + Handle(http.MethodTrace, "/trace", func(c *Context) { str += "trace" }) + + for _, method := range []string{ + "head", "get", "post", "put", + "delete", "options", "patch", "trace", + } { + str = "" + w := new(mockResponseWriter) + req, _ := http.NewRequest(strings.ToUpper(method), "/"+method, nil) + DefaultRouter.Router.ServeHTTP(w, req) + + require.Equal(t, "m1"+method+"m2", str) + } +} + +func TestServerRouterMaxHandlers(t *testing.T) { + { + router := New() + router.Use(make([]HandlerFunc, abortIndex-2)...) + router.Handle(http.MethodGet, "/", func(c *Context) {}) + } + { + router := New() + router.Use(make([]HandlerFunc, abortIndex-1)...) + require.Panics(t, func() { + router.Handle(http.MethodGet, "/", func(c *Context) {}) + }) + } + { + router := New() + require.Panics(t, func() { + router.Use(make([]HandlerFunc, abortIndex)...) + }) + } +} diff --git a/blobstore/common/rpc/server.go b/blobstore/common/rpc/server.go new file mode 100644 index 000000000..e177d3ed6 --- /dev/null +++ b/blobstore/common/rpc/server.go @@ -0,0 +1,142 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package rpc + +import ( + "net/http" + + "github.com/julienschmidt/httprouter" +) + +type ( + // HandlerFunc defines the handler of app function + HandlerFunc func(*Context) + + // ServerOption server option applier + // Order: if args in body ignore others options, + // else uri > query > form > postfrom + ServerOption interface { + apply(*serverOptions) + } + serverOptions struct { + argsBody bool + argsURI bool + argsQuery bool + argsForm bool + argsPostForm bool + + metaCapacity int + } + funcServerOption struct { + f func(*serverOptions) + } +) + +func (so *serverOptions) copy() *serverOptions { + return &serverOptions{ + argsBody: so.argsBody, + argsURI: so.argsURI, + argsQuery: so.argsQuery, + argsForm: so.argsForm, + argsPostForm: so.argsPostForm, + + metaCapacity: so.metaCapacity, + } +} + +func (so *serverOptions) hasArgs() bool { + return so.argsBody || so.argsURI || so.argsQuery || so.argsForm || so.argsPostForm +} + +func (fo *funcServerOption) apply(f *serverOptions) { + fo.f(f) +} + +func newFuncServerOption(f func(*serverOptions)) *funcServerOption { + return &funcServerOption{ + f: f, + } +} + +// OptArgsBody argument in request body +func OptArgsBody() ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.argsBody = true + }) +} + +// OptArgsURI argument in uri +func OptArgsURI() ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.argsURI = true + }) +} + +// OptArgsQuery argument in query string +func OptArgsQuery() ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.argsQuery = true + }) +} + +// OptArgsForm argument in form +func OptArgsForm() ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.argsForm = true + }) +} + +// OptArgsPostForm argument in post form +func OptArgsPostForm() ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.argsPostForm = true + }) +} + +// OptMetaCapacity initial meta capacity +func OptMetaCapacity(capacity int) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + if capacity >= 0 { + o.metaCapacity = capacity + } + }) +} + +// makeHandler make handle of httprouter +func makeHandler(handlers []HandlerFunc, opts ...ServerOption) httprouter.Handle { + opt := new(serverOptions) + for _, o := range opts { + o.apply(opt) + } + + return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + c := &Context{ + opts: opt, + Param: ps, + + Request: r, + Writer: w, + + Meta: make(map[string]interface{}, opt.metaCapacity), + + index: -1, + handlers: handlers, + } + c.Next() + if !c.wroteHeader { + c.RespondStatus(http.StatusOK) + } + } +} diff --git a/blobstore/common/rpc/server_test.go b/blobstore/common/rpc/server_test.go new file mode 100644 index 000000000..b4f3e8e08 --- /dev/null +++ b/blobstore/common/rpc/server_test.go @@ -0,0 +1,393 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package rpc + +import ( + "bytes" + "encoding/json" + "errors" + "net/http" + "net/url" + "strconv" + "testing" + + "github.com/stretchr/testify/require" +) + +type mockResponseWriter struct { + headers http.Header + status int + body []byte +} + +var _ http.ResponseWriter = (*mockResponseWriter)(nil) + +func (m *mockResponseWriter) Header() (h http.Header) { + if m.headers == nil { + m.headers = http.Header{} + } + return m.headers +} + +func (m *mockResponseWriter) Write(p []byte) (n int, err error) { + m.body = p + return len(p), nil +} + +func (m *mockResponseWriter) WriteHeader(status int) { + m.status = status +} + +func TestServerRouterBase(t *testing.T) { + router := New() + + type A struct { + Bar string + } + args := new(A) + str := "" + router.Use(func(c *Context) { + str += "1" + c.Next() + str += "2" + }) + router.Use(func(c *Context) { + str += "3" + c.Next() + str += "4" + }) + router.Handle(http.MethodGet, "/foo/:bar", func(c *Context) { + if err := c.ParseArgs(args); err != nil { + panic(err) + } + str += args.Bar + }, OptArgsURI()) + + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodGet, "/foo/you", nil) + router.Router.ServeHTTP(w, req) + + require.Equal(t, "you", args.Bar) + require.Equal(t, "13you42", str) +} + +func TestServerAbort(t *testing.T) { + router := New() + str := "" + router.Use(func(c *Context) { + str += "1" + c.Next() + + require.True(t, c.IsAborted()) + val, ok := c.Get("Abort") + require.True(t, ok) + require.Equal(t, int(1), val.(int)) + + str += "2" + }) + router.Use(func(c *Context) { + str += "3" + c.Set("Abort", 1) + c.Abort() + c.Next() // aborted, + str += "4" + }) + router.Handle(http.MethodGet, "/foo/:bar", func(c *Context) { + str += "5" + }, OptArgsURI()) + + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodGet, "/foo/you", nil) + router.Router.ServeHTTP(w, req) + + require.Equal(t, "1342", str) +} + +func TestServerAbortStatus(t *testing.T) { + resp := func(r *Router) *mockResponseWriter { + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodGet, "/", nil) + r.Router.ServeHTTP(w, req) + return w + } + + { + router := New() + router.Use(func(c *Context) {}) + router.Handle(http.MethodGet, "/", func(c *Context) { c.Respond() }) + w := resp(router) + require.Equal(t, 200, w.status) + require.Equal(t, "0", w.Header().Get(HeaderContentLength)) + } + { + router := New() + router.Use(func(c *Context) {}) + router.Handle(http.MethodGet, "/", func(c *Context) { c.RespondStatus(200) }) + w := resp(router) + require.Equal(t, 200, w.status) + require.Equal(t, "", w.Header().Get(HeaderContentLength)) + } + { + router := New() + router.Use(func(c *Context) {}) + router.Handle(http.MethodGet, "/", func(c *Context) {}) + w := resp(router) + require.Equal(t, 200, w.status) + require.Equal(t, "", w.Header().Get(HeaderContentLength)) + } + { + router := New() + router.Use(func(c *Context) { c.Abort() }) + router.Handle(http.MethodGet, "/", func(c *Context) { c.RespondStatus(204) }) + w := resp(router) + require.Equal(t, 200, w.status) + } + { + router := New() + router.Use(func(c *Context) { c.AbortWithStatus(400) }) + router.Handle(http.MethodGet, "/", func(c *Context) { c.RespondStatus(204) }) + w := resp(router) + require.Equal(t, 400, w.status) + } + { + router := New() + router.Use(func(c *Context) { c.AbortWithStatusJSON(399, struct{ I int }{10}) }) + router.Handle(http.MethodGet, "/", func(c *Context) { c.RespondStatus(204) }) + w := resp(router) + require.Equal(t, 399, w.status) + } + { + router := New() + router.Use(func(c *Context) { c.AbortWithError(&Error{Status: 412}) }) + router.Handle(http.MethodGet, "/", func(c *Context) { c.RespondStatus(204) }) + w := resp(router) + require.Equal(t, 412, w.status) + } +} + +func TestServerOptArgs(t *testing.T) { + type A struct { + Bar string + } + body := "{\"bar\": \"bodybar\"}" + { + router := New() + args := new(A) + router.Handle(http.MethodGet, "/:bar", func(c *Context) { + if err := c.ArgsBody(args); err != nil { + panic(err) + } + }) + + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodGet, "/uribar?bar=querybar", bytes.NewBufferString(body)) + req.Header.Set(HeaderContentLength, strconv.Itoa(len(body))) + router.Router.ServeHTTP(w, req) + + require.Equal(t, "bodybar", args.Bar) + } + { + router := New() + args := new(A) + router.Handle(http.MethodGet, "/:bar", func(c *Context) { + if err := c.ArgsURI(args); err != nil { + panic(err) + } + }) + + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodGet, "/uribar?bar=querybar", bytes.NewBufferString(body)) + req.Header.Set(HeaderContentLength, strconv.Itoa(len(body))) + router.Router.ServeHTTP(w, req) + + require.Equal(t, "uribar", args.Bar) + } + { + router := New() + args := new(A) + router.Handle(http.MethodGet, "/:bar", func(c *Context) { + if err := c.ArgsQuery(args); err != nil { + panic(err) + } + }) + + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodGet, "/uribar?bar=querybar", bytes.NewBufferString(body)) + req.Header.Set(HeaderContentLength, strconv.Itoa(len(body))) + router.Router.ServeHTTP(w, req) + + require.Equal(t, "querybar", args.Bar) + } + { + router := New() + args := new(A) + router.Handle(http.MethodPost, "/:bar", func(c *Context) { + if err := c.ArgsForm(args); err != nil { + panic(err) + } + }) + + form := url.Values{} + form.Set("bar", "formbar") + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodPost, "/uribar?bar=querybar", nil) + req.Form = form + router.Router.ServeHTTP(w, req) + + require.Equal(t, "formbar", args.Bar) + } + { + router := New() + args := new(A) + router.Handle(http.MethodPost, "/:bar", func(c *Context) { + if err := c.ArgsPostForm(args); err != nil { + panic(err) + } + }) + + form := url.Values{} + form.Set("bar", "formbar") + postForm := url.Values{} + postForm.Set("bar", "postformbar") + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodPost, "/uribar?bar=querybar", nil) + req.Form = form + req.PostForm = postForm + router.Router.ServeHTTP(w, req) + + require.Equal(t, "postformbar", args.Bar) + } +} + +func TestServerOptMetaCapacity(t *testing.T) { + router := New() + router.Handle(http.MethodGet, "/none", func(c *Context) { + if c.opts.metaCapacity != 0 { + panic(c.opts.metaCapacity) + } + }, OptMetaCapacity(-199)) + router.Handle(http.MethodGet, "/1024", func(c *Context) { + if c.opts.metaCapacity != 1024 { + panic(c.opts.metaCapacity) + } + }, OptMetaCapacity(1024)) + + { + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodGet, "/none", nil) + router.ServeHTTP(w, req) + } + { + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodGet, "/1024", nil) + router.ServeHTTP(w, req) + } +} + +type marshalData struct { + I int + S string +} + +func (d *marshalData) Marshal() ([]byte, string, error) { + if d.I > 0 { + return []byte{0x00, 0xff}, MIMEStream, nil + } + return nil, "", &Error{Status: 400, Err: errors.New("fake error")} +} + +func TestServerResponseJSON(t *testing.T) { + resp := func(r *Router) *mockResponseWriter { + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodGet, "/", nil) + r.Router.ServeHTTP(w, req) + return w + } + + { + router := New() + router.Use(func(c *Context) {}) + router.Handle(http.MethodGet, "/", func(c *Context) { c.Respond() }) + w := resp(router) + require.Equal(t, 0, len(w.body)) + } + { + router := New() + router.Use(func(c *Context) { c.AbortWithStatusJSON(400, "bad") }) + router.Handle(http.MethodGet, "/", func(c *Context) { c.RespondStatus(204) }) + w := resp(router) + require.Equal(t, "\"bad\"", string(w.body)) + } + { + router := New() + router.Use(func(c *Context) {}) + router.Handle(http.MethodGet, "/", func(c *Context) { c.RespondStatusData(255, nil) }) + w := resp(router) + require.Equal(t, 255, w.status) + require.Equal(t, "null", string(w.body)) + } + { + router := New() + router.Handle(http.MethodGet, "/", func(c *Context) { + c.RespondJSON(&marshalData{I: 11, S: "foo bar"}) + }) + w := resp(router) + require.Equal(t, 200, w.status) + require.Equal(t, []byte{0x0, 0xff}, w.body) + } + { + router := New() + router.Handle(http.MethodGet, "/", func(c *Context) { + c.RespondJSON(&marshalData{I: -11, S: "foo bar"}) + }) + w := resp(router) + require.Equal(t, 400, w.status) + + ret := new(errorResponse) + err := json.Unmarshal(w.body, ret) + require.NoError(t, err) + require.Equal(t, "", ret.Code) + require.Equal(t, "fake error", ret.Error) + } + { + router := New() + router.Handle(http.MethodGet, "/", func(c *Context) { + c.RespondJSON(marshalData{I: 11, S: "foo bar"}) + }) + w := resp(router) + require.Equal(t, 200, w.status) + + ret := new(marshalData) + err := json.Unmarshal(w.body, ret) + require.NoError(t, err) + require.Equal(t, 11, ret.I) + require.Equal(t, "foo bar", ret.S) + } +} + +func BenchmarkServerBase(b *testing.B) { + router := New() + router.Use(func(c *Context) { c.Next() }) + router.Use(func(c *Context) { c.Next() }) + router.Handle(http.MethodGet, "/", func(c *Context) { c.Respond() }) + + b.Helper() + b.ResetTimer() + for ii := 0; ii < b.N; ii++ { + w := new(mockResponseWriter) + req, _ := http.NewRequest(http.MethodGet, "/", nil) + router.ServeHTTP(w, req) + } +} diff --git a/blobstore/common/uptoken/token.go b/blobstore/common/uptoken/token.go new file mode 100644 index 000000000..fd3730bcb --- /dev/null +++ b/blobstore/common/uptoken/token.go @@ -0,0 +1,143 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package uptoken + +import ( + "bytes" + "crypto/hmac" + "crypto/sha1" + "encoding/binary" + "encoding/hex" + "time" + + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +const ( + // TokenSize max size of token array + // clusterID + vid + bid + count + time + size + TokenSize = 4 + 4 + 10 + 5 + 5 + 5 +) + +// UploadToken token between alloc and putat +// [0:8] hmac (8) first 8 bytes of sha1 summary +// [8:16] minBid (8) in the SliceInfo +// [16:20] count (4) in the SliceInfo +// [20:24] time (4) expired unix utc time, 0 means not expired +type UploadToken struct { + Data [TokenSize]byte + Offset uint8 +} + +// IsValidBid returns the bid is valid or not +func (t *UploadToken) IsValidBid(bid proto.BlobID) bool { + minBid, n := binary.Uvarint(t.Data[8:]) + if n <= 0 { + return false + } + count, n := binary.Uvarint(t.Data[8+n:]) + if n <= 0 { + return false + } + return minBid <= uint64(bid) && uint64(bid) < minBid+count +} + +// IsValid returns the token is valid or not +func (t *UploadToken) IsValid(clusterID proto.ClusterID, vid proto.Vid, bid proto.BlobID, + size uint32, secretKey []byte) bool { + var ( + minBid, count, expiredTime uint64 + ok bool + ) + + data := t.Data[:] + offset := 8 + next := func() (uint64, bool) { + val, n := binary.Uvarint(data[offset:]) + if n <= 0 { + return 0, false + } + offset += n + return val, true + } + + if minBid, ok = next(); !ok { + return false + } + if count, ok = next(); !ok { + return false + } + if !(minBid <= uint64(bid) && uint64(bid) < minBid+count) { + return false + } + + if expiredTime, ok = next(); !ok { + return false + } + if expiredTime != 0 && time.Now().UTC().Unix() > int64(expiredTime) { + return false + } + + token := newUploadToken(clusterID, vid, proto.BlobID(minBid), + uint32(count), size, uint32(expiredTime), secretKey) + return bytes.Equal(token.Data[0:8], t.Data[0:8]) +} + +// NewUploadToken returns a token corresponding one SliceInfo in Location with expiration +// expiration = 0 means not expired forever +func NewUploadToken(clusterID proto.ClusterID, vid proto.Vid, bid proto.BlobID, + count, size uint32, expiration time.Duration, secretKey []byte) UploadToken { + expiredTime := uint32(0) + if expiration != time.Duration(0) { + expiredTime = uint32(time.Now().Add(expiration).UTC().Unix()) + } + return newUploadToken(clusterID, vid, bid, count, size, expiredTime, secretKey) +} + +func newUploadToken(clusterID proto.ClusterID, vid proto.Vid, bid proto.BlobID, + count, size uint32, expiredTime uint32, secretKey []byte) UploadToken { + var token UploadToken + data := token.Data[:] + binary.BigEndian.PutUint32(data[0:4], uint32(clusterID)) + binary.BigEndian.PutUint32(data[4:8], uint32(vid)) + + offset := 8 + offset += binary.PutUvarint(data[offset:], uint64(bid)) + offset += binary.PutUvarint(data[offset:], uint64(count)) + offset += binary.PutUvarint(data[offset:], uint64(expiredTime)) + token.Offset = uint8(offset) + offset += binary.PutUvarint(data[offset:], uint64(size)) + + h := hmac.New(sha1.New, secretKey) + h.Write(data[:offset]) + sum := h.Sum(nil) + + // replace clusterID and vid + copy(data[0:8], sum[0:8]) + return token +} + +// EncodeToken encode token to string +func EncodeToken(token UploadToken) string { + return hex.EncodeToString(token.Data[:token.Offset]) +} + +// DecodeToken decode token from string +func DecodeToken(s string) UploadToken { + var token UploadToken + src, _ := hex.DecodeString(s) + token.Offset = uint8(copy(token.Data[:], src)) + return token +} diff --git a/blobstore/common/uptoken/token_test.go b/blobstore/common/uptoken/token_test.go new file mode 100644 index 000000000..602c54328 --- /dev/null +++ b/blobstore/common/uptoken/token_test.go @@ -0,0 +1,166 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package uptoken_test + +import ( + "crypto/rand" + mrand "math/rand" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/uptoken" +) + +func TestAccessServerTokenBase(t *testing.T) { + token := uptoken.NewUploadToken(1, 1, 1, 1, 1, 0, []byte{}) + require.Equal(t, "d29f19731941e44a010100", uptoken.EncodeToken(token)) +} + +func TestAccessServerTokenValid(t *testing.T) { + for ii := 0; ii < 1000; ii++ { + cid := proto.ClusterID(mrand.Uint32()) + vid := proto.Vid(mrand.Uint32()) + bid := proto.BlobID(mrand.Uint64()) + count := mrand.Uint32() + size := mrand.Uint32() + secretKey := make([]byte, mrand.Intn(40)+1) + rand.Read(secretKey) + secretKey[len(secretKey)-1] = 0xff + + if cid == 0 || vid == 0 || bid == 0 || count == 0 { + continue + } + + token := uptoken.NewUploadToken(cid, vid, bid, count, size, time.Minute, secretKey) + require.True(t, token.IsValidBid(bid)) + require.True(t, token.IsValid(cid, vid, bid, size, secretKey)) + + { // invalid bid + for i := 0; i < 100; i++ { + bidx := proto.BlobID(mrand.Int63n(int64(bid >> 1))) + require.False(t, token.IsValidBid(bidx)) + require.False(t, token.IsValid(cid, vid, bidx, size, secretKey)) + } + } + { // valid bid + for i := 0; i < 100; i++ { + bidx := bid + proto.BlobID(mrand.Int63n(int64(count))) + require.True(t, token.IsValidBid(bidx)) + require.True(t, token.IsValid(cid, vid, bidx, size, secretKey)) + } + } + { // invalid bid + for i := uint32(1); i < 100; i++ { + bidx := bid + proto.BlobID(count+i) + require.False(t, token.IsValidBid(bidx)) + require.False(t, token.IsValid(cid, vid, bidx, size, secretKey)) + } + } + { // invalid clusterID + for i := 0; i < 100; i++ { + cidx := proto.ClusterID(mrand.Uint32()) + if cid != cidx { + require.False(t, token.IsValid(cidx, vid, bid, size, secretKey)) + } + } + } + { // invalid vid + for i := 0; i < 100; i++ { + vidx := proto.Vid(mrand.Uint32()) + if vid != vidx { + require.False(t, token.IsValid(cid, vidx, bid, size, secretKey)) + } + } + } + { // valid size + for i := 0; i < 100; i++ { + sizex := mrand.Uint32() + if size != sizex { + require.False(t, token.IsValid(cid, vid, bid, sizex, secretKey)) + } + } + } + { // invalid secretKey + for i := 0; i < 100; i++ { + secretKeyx := secretKey[:len(secretKey)/2] + require.False(t, token.IsValid(cid, vid, bid, size, secretKeyx)) + } + } + { // invalid token + for i := 0; i < 9; i++ { + var tokenx uptoken.UploadToken + tokenx.Offset = uint8(copy(tokenx.Data[:], token.Data[:token.Offset])) + char := mrand.Int31n(0xff) + if char != int32(token.Data[i]) { + tokenx.Data[i] = byte(char) + require.False(t, tokenx.IsValid(cid, vid, bid, size, secretKey)) + } + } + } + { // encode decode + str := uptoken.EncodeToken(token) + tokenx := uptoken.DecodeToken(str) + require.Equal(t, token.Offset, tokenx.Offset) + require.Equal(t, token.Data[:token.Offset], tokenx.Data[:tokenx.Offset]) + } + } +} + +func TestAccessServerTokenExpired(t *testing.T) { + secretKey := []byte{0x1f, 0xff} + for ii := 0; ii < 1000; ii++ { + expired := mrand.Intn(40) - 20 + token := uptoken.NewUploadToken(1, 1, 1, 1, 1, time.Duration(expired)*time.Second, secretKey) + if expired >= 0 { + require.True(t, token.IsValid(1, 1, 1, 1, secretKey)) + } else { + require.False(t, token.IsValid(1, 1, 1, 1, secretKey)) + } + } +} + +func BenchmarkAccessServerTokenNew(b *testing.B) { + secretKey := []byte{} + for ii := 0; ii <= b.N; ii++ { + uptoken.NewUploadToken(1, 1, 1, 1, 1, 1, secretKey) + } +} + +func BenchmarkAccessServerTokenValid(b *testing.B) { + secretKey := []byte{} + token := uptoken.NewUploadToken(1, 1, 1, 1, 1, 1, secretKey) + b.ResetTimer() + for ii := 0; ii <= b.N; ii++ { + token.IsValid(1, 1, 1, 1, secretKey) + } +} + +func BenchmarkAccessServerTokenEncode(b *testing.B) { + secretKey := []byte{} + token := uptoken.NewUploadToken(1, 1, 1, 1, 1, 1, secretKey) + b.ResetTimer() + for ii := 0; ii <= b.N; ii++ { + uptoken.EncodeToken(token) + } +} + +func BenchmarkAccessServerTokenDecode(b *testing.B) { + for ii := 0; ii <= b.N; ii++ { + uptoken.DecodeToken("d29f19731941e44a010100") + } +} diff --git a/blobstore/go.mod b/blobstore/go.mod new file mode 100644 index 000000000..803068fa5 --- /dev/null +++ b/blobstore/go.mod @@ -0,0 +1,52 @@ +module github.com/cubefs/cubefs/blobstore + +go 1.15 + +replace github.com/desertbit/grumble v1.1.1 => github.com/sejust/grumble v1.1.2-0.20210930091007-2c8622e565d3 + +require ( + github.com/Shopify/sarama v1.22.1 + github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5 + github.com/alicebob/miniredis/v2 v2.16.1 + github.com/benbjohnson/clock v1.3.0 + github.com/deniswernert/go-fstab v0.0.0-20141204152952-eb4090f26517 + github.com/desertbit/grumble v1.1.1 + github.com/dustin/go-humanize v1.0.0 + github.com/fatih/color v1.13.0 + github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 + github.com/go-redis/redis/v8 v8.11.4 + github.com/golang/mock v1.6.0 + github.com/google/uuid v1.3.0 + github.com/gorilla/mux v1.8.0 + github.com/hashicorp/consul/api v1.11.0 + github.com/hashicorp/golang-lru v0.5.4 + github.com/julienschmidt/httprouter v1.3.0 + github.com/klauspost/reedsolomon v1.9.2 + github.com/opentracing/opentracing-go v1.2.0 + github.com/prometheus/client_golang v1.11.0 + github.com/stretchr/testify v1.7.0 + github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c + go.etcd.io/etcd/raft/v3 v3.5.1 + go.mongodb.org/mongo-driver v1.1.0 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c + golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 + gopkg.in/go-playground/validator.v9 v9.31.0 + gopkg.in/natefinch/lumberjack.v2 v2.0.0 + gopkg.in/yaml.v2 v2.4.0 +) + +require ( + github.com/BurntSushi/toml v0.4.1 // indirect + github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect + github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect + github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect + github.com/go-playground/universal-translator v0.18.0 // indirect + github.com/golang/protobuf v1.5.2 + github.com/klauspost/cpuid v1.3.1 // indirect + github.com/leodido/go-urn v1.2.1 // indirect + github.com/onsi/ginkgo v1.16.5 // indirect + github.com/onsi/gomega v1.17.0 // indirect + github.com/smartystreets/goconvey v1.7.2 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + gopkg.in/go-playground/assert.v1 v1.2.1 // indirect +) diff --git a/blobstore/go.sum b/blobstore/go.sum new file mode 100644 index 000000000..5f7caa1e2 --- /dev/null +++ b/blobstore/go.sum @@ -0,0 +1,479 @@ +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= +github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 h1:2T/jmrHeTezcCM58lvEQXs0UpQJCo5SoGAcg+mbSTIg= +github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8/go.mod h1:oX5x61PbNXchhh0oikYAH+4Pcfw5LKv21+Jnpr6r6Pc= +github.com/Netflix/go-expect v0.0.0-20190729225929-0e00d9168667/go.mod h1:oX5x61PbNXchhh0oikYAH+4Pcfw5LKv21+Jnpr6r6Pc= +github.com/Shopify/sarama v1.22.1 h1:exyEsKLGyCsDiqpV5Lr4slFi8ev2KiM3cP1KZ6vnCQ0= +github.com/Shopify/sarama v1.22.1/go.mod h1:FRzlvRpMFO/639zY1SDxUxkqH97Y0ndM5CbGj6oG3As= +github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5 h1:rFw4nCn9iMW+Vajsk51NtYIcwSTkXr+JGrMd36kTDJw= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= +github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= +github.com/alicebob/miniredis/v2 v2.16.1 h1:ikfCfUHWlfiVCVVaaDO60SBgPWS4UNIi1A7p7QmUVyw= +github.com/alicebob/miniredis/v2 v2.16.1/go.mod h1:gquAfGbzn92jvtrSC69+6zZnwSODVXVpYDRaGhWaL6I= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 h1:uH66TXeswKn5PW5zdZ39xEwfS9an067BirqA+P4QaLI= +github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 h1:xD/lrqdvwsc+O2bjSSi3YqY73Ke3LAiSCx49aCesA0E= +github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= +github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKDqs= +github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deniswernert/go-fstab v0.0.0-20141204152952-eb4090f26517 h1:YMvaGdOIUowdD6ZybqLsUamGvWONZViUeW6T22U7fP0= +github.com/deniswernert/go-fstab v0.0.0-20141204152952-eb4090f26517/go.mod h1:ixLGX4GUQg44igA/iJawr+KYZLyWOoAzAgTCQcJ/K9Y= +github.com/desertbit/closer/v3 v3.1.2 h1:a6+2DmwIcNygW04XXWYq+Qp2X9uIk9QbZCP9//qEkb0= +github.com/desertbit/closer/v3 v3.1.2/go.mod h1:AAC4KRd8DC40nwvV967J/kDFhujMEiuwIKQfN0IDxXw= +github.com/desertbit/columnize v2.1.0+incompatible h1:h55rYmdrWoTj7w9aAnCkxzM3C2Eb8zuFa2W41t0o5j0= +github.com/desertbit/columnize v2.1.0+incompatible/go.mod h1:5kPrzQwKbQ8E5D28nvTVPqIBJyj+8jvJzwt6HXZvXgI= +github.com/desertbit/go-shlex v0.1.1 h1:c65HnbgX1QyC6kPL1dMzUpZ4puNUE6ai/eVucWNLNsk= +github.com/desertbit/go-shlex v0.1.1/go.mod h1:Qbb+mJNud5AypgHZ81EL8syOGaWlwvAOTqS7XmWI4pQ= +github.com/desertbit/readline v1.5.1 h1:/wOIZkWYl1s+IvJm/5bOknfUgs6MhS9svRNZpFM53Os= +github.com/desertbit/readline v1.5.1/go.mod h1:pHQgTsCFs9Cpfh5mlSUFi9Xa5kkL4d8L1Jo4UVWzPw0= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.1.0 h1:1NtRmCAqadE2FN4ZcN6g90TP3uk8cg9rn9eNK2197aU= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is= +github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg= +github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/hashicorp/consul/api v1.11.0 h1:Hw/G8TtRvOElqxVIhBzXciiSTbapq8hZ2XKZsXk5ZCE= +github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= +github.com/hashicorp/consul/sdk v0.8.0 h1:OJtKBtEjboEZvG6AOUdh4Z1Zbyu0WcxQ0qatRrZHTVU= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.12.0 h1:d4QkX8FRTYaKaCZBoXYY8zJX2BXjWxurN/GA2tkrmZM= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= +github.com/hashicorp/memberlist v0.2.2 h1:5+RffWKwqJ71YPu9mWsF7ZOscZmwfasdA8kbdC7AO2g= +github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/serf v0.9.5 h1:EBWvyu9tcRszt3Bxp3KNssBMP1KuHWyO51lz9+786iM= +github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= +github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174/go.mod h1:DqJ97dSdRW1W22yXSB90986pcOyQ7r45iio1KN2ez1A= +github.com/hinshun/vt10x v0.0.0-20180809195222-d55458df857c/go.mod h1:DqJ97dSdRW1W22yXSB90986pcOyQ7r45iio1KN2ez1A= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= +github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= +github.com/klauspost/reedsolomon v1.9.2 h1:E9CMS2Pqbv+C7tsrYad4YC9MfhnMVWhMRsTi7U0UB18= +github.com/klauspost/reedsolomon v1.9.2/go.mod h1:CwCi+NUr9pqSVktrkN+Ondf06rkhYZ/pcNv7fu+8Un4= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.26 h1:gPxPSwALAeHJSjarOs00QjVdV9QoBvc1D2ujQUr5BzU= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41 h1:GeinFsrjWz97fAxVUEd748aV0cYL+I6k44gFJTCVvpU= +github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sejust/grumble v1.1.2-0.20210930091007-2c8622e565d3 h1:LfzzBYdvO9z7tC5K343ORCNGrTi2VNbQ7ubCa1cKg98= +github.com/sejust/grumble v1.1.2-0.20210930091007-2c8622e565d3/go.mod h1:r7j3ShNy5EmOsegRD2DzTutIaGiLiA3M5yBTXXeLwcs= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= +github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= +github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da h1:NimzV1aGyq29m5ukMK0AMWEhFaL/lrEOaephfuoiARg= +github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da/go.mod h1:E1AXubJBdNmFERAOucpDIxNzeGfLzg0mYh+UfMWdChA= +go.etcd.io/etcd/client/pkg/v3 v3.5.1 h1:XIQcHCFSG53bJETYeRJtIxdLv2EWRGxcfzR8lSnTH4E= +go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/raft/v3 v3.5.1 h1:nthXrxmATKB9OZ9C64sk3QZaJ1WZ8tmlI0VwzpJSZwg= +go.etcd.io/etcd/raft/v3 v3.5.1/go.mod h1:WIlKzH/rjc54LDZ8SOa7GObrrdX3z96MkP1WDfODBeA= +go.mongodb.org/mongo-driver v1.1.0 h1:aeOqSrhl9eDRAap/3T5pCfMBEBxZ0vuXBP+RMtp2KX8= +go.mongodb.org/mongo-driver v1.1.0/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180606202747-9527bec2660b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 h1:GZokNIeuVkl3aZHJchRrr13WCsols02MLUcz1U9is6M= +golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/AlecAivazis/survey.v1 v1.8.5/go.mod h1:iBNOmqKz/NUbZx3bA+4hAGLRC7fSK7tgtVDT4tB22XA= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M= +gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/blobstore/testing/mocks.go b/blobstore/testing/mocks.go new file mode 100644 index 000000000..5609f7273 --- /dev/null +++ b/blobstore/testing/mocks.go @@ -0,0 +1,37 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +// Package testing for mocking interfaces with `go generate` +package testing + +// github.com/cubefs/cubefs/blobstore/util/... util interfaces +//go:generate mockgen -destination=./mocks/util_selector.go -package=mocks -mock_names Selector=MockSelector github.com/cubefs/cubefs/blobstore/util/selector Selector + +// github.com/cubefs/cubefs/blobstore/common/... common interfaces +//go:generate mockgen -destination=./mocks/raft_server.go -package=mocks -mock_names RaftServer=MockRaftServer github.com/cubefs/cubefs/blobstore/common/raftserver RaftServer +//go:generate mockgen -destination=./mocks/rpc_client.go -package=mocks -mock_names Client=MockRPCClient github.com/cubefs/cubefs/blobstore/common/rpc Client +//go:generate mockgen -destination=./mocks/encoder.go -package=mocks -mock_names Encoder=MockEncoder github.com/cubefs/cubefs/blobstore/common/recordlog Encoder +//go:generate mockgen -destination=./mocks/task_switch.go -package=mocks -mock_names ISwitcher=MockSwitcher github.com/cubefs/cubefs/blobstore/common/taskswitch ISwitcher + +// github.com/cubefs/cubefs/blobstore/api/... api interfaces +//go:generate mockgen -destination=./mocks/api_access.go -package=mocks -mock_names API=MockAccessAPI github.com/cubefs/cubefs/blobstore/api/access API +//go:generate mockgen -destination=./mocks/api_clustermgr.go -package=mocks -mock_names ClientAPI=MockClientAPI github.com/cubefs/cubefs/blobstore/api/clustermgr ClientAPI +//go:generate mockgen -destination=./mocks/api_blobnode.go -package=mocks -mock_names StorageAPI=MockStorageAPI github.com/cubefs/cubefs/blobstore/api/blobnode StorageAPI +//go:generate mockgen -destination=./mocks/api_scheduler.go -package=mocks -mock_names IScheduler=MockIScheduler github.com/cubefs/cubefs/blobstore/api/scheduler IScheduler +//go:generate mockgen -destination=./mocks/api_proxy.go -package=mocks -mock_names Client=MockProxyClient,LbMsgSender=MockProxyLbRpcClient github.com/cubefs/cubefs/blobstore/api/proxy Client,LbMsgSender + +import ( + // add package to go.mod for `go generate` + _ "github.com/golang/mock/mockgen/model" +) diff --git a/blobstore/testing/mocks/api_access.go b/blobstore/testing/mocks/api_access.go new file mode 100644 index 000000000..81f9bb3ba --- /dev/null +++ b/blobstore/testing/mocks/api_access.go @@ -0,0 +1,83 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cubefs/cubefs/blobstore/api/access (interfaces: API) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + io "io" + reflect "reflect" + + access "github.com/cubefs/cubefs/blobstore/api/access" + gomock "github.com/golang/mock/gomock" +) + +// MockAccessAPI is a mock of API interface. +type MockAccessAPI struct { + ctrl *gomock.Controller + recorder *MockAccessAPIMockRecorder +} + +// MockAccessAPIMockRecorder is the mock recorder for MockAccessAPI. +type MockAccessAPIMockRecorder struct { + mock *MockAccessAPI +} + +// NewMockAccessAPI creates a new mock instance. +func NewMockAccessAPI(ctrl *gomock.Controller) *MockAccessAPI { + mock := &MockAccessAPI{ctrl: ctrl} + mock.recorder = &MockAccessAPIMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAccessAPI) EXPECT() *MockAccessAPIMockRecorder { + return m.recorder +} + +// Delete mocks base method. +func (m *MockAccessAPI) Delete(arg0 context.Context, arg1 *access.DeleteArgs) ([]access.Location, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", arg0, arg1) + ret0, _ := ret[0].([]access.Location) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Delete indicates an expected call of Delete. +func (mr *MockAccessAPIMockRecorder) Delete(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockAccessAPI)(nil).Delete), arg0, arg1) +} + +// Get mocks base method. +func (m *MockAccessAPI) Get(arg0 context.Context, arg1 *access.GetArgs) (io.ReadCloser, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", arg0, arg1) + ret0, _ := ret[0].(io.ReadCloser) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockAccessAPIMockRecorder) Get(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockAccessAPI)(nil).Get), arg0, arg1) +} + +// Put mocks base method. +func (m *MockAccessAPI) Put(arg0 context.Context, arg1 *access.PutArgs) (access.Location, access.HashSumMap, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Put", arg0, arg1) + ret0, _ := ret[0].(access.Location) + ret1, _ := ret[1].(access.HashSumMap) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// Put indicates an expected call of Put. +func (mr *MockAccessAPIMockRecorder) Put(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockAccessAPI)(nil).Put), arg0, arg1) +} diff --git a/blobstore/testing/mocks/api_blobnode.go b/blobstore/testing/mocks/api_blobnode.go new file mode 100644 index 000000000..a55eb7c12 --- /dev/null +++ b/blobstore/testing/mocks/api_blobnode.go @@ -0,0 +1,346 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cubefs/cubefs/blobstore/api/blobnode (interfaces: StorageAPI) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + io "io" + reflect "reflect" + + blobnode "github.com/cubefs/cubefs/blobstore/api/blobnode" + proto "github.com/cubefs/cubefs/blobstore/common/proto" + gomock "github.com/golang/mock/gomock" +) + +// MockStorageAPI is a mock of StorageAPI interface. +type MockStorageAPI struct { + ctrl *gomock.Controller + recorder *MockStorageAPIMockRecorder +} + +// MockStorageAPIMockRecorder is the mock recorder for MockStorageAPI. +type MockStorageAPIMockRecorder struct { + mock *MockStorageAPI +} + +// NewMockStorageAPI creates a new mock instance. +func NewMockStorageAPI(ctrl *gomock.Controller) *MockStorageAPI { + mock := &MockStorageAPI{ctrl: ctrl} + mock.recorder = &MockStorageAPIMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockStorageAPI) EXPECT() *MockStorageAPIMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockStorageAPI) Close(arg0 context.Context, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockStorageAPIMockRecorder) Close(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockStorageAPI)(nil).Close), arg0, arg1) +} + +// CreateChunk mocks base method. +func (m *MockStorageAPI) CreateChunk(arg0 context.Context, arg1 string, arg2 *blobnode.CreateChunkArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateChunk", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateChunk indicates an expected call of CreateChunk. +func (mr *MockStorageAPIMockRecorder) CreateChunk(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateChunk", reflect.TypeOf((*MockStorageAPI)(nil).CreateChunk), arg0, arg1, arg2) +} + +// DeleteShard mocks base method. +func (m *MockStorageAPI) DeleteShard(arg0 context.Context, arg1 string, arg2 *blobnode.DeleteShardArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteShard", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteShard indicates an expected call of DeleteShard. +func (mr *MockStorageAPIMockRecorder) DeleteShard(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteShard", reflect.TypeOf((*MockStorageAPI)(nil).DeleteShard), arg0, arg1, arg2) +} + +// DiskInfo mocks base method. +func (m *MockStorageAPI) DiskInfo(arg0 context.Context, arg1 string, arg2 *blobnode.DiskStatArgs) (*blobnode.DiskInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DiskInfo", arg0, arg1, arg2) + ret0, _ := ret[0].(*blobnode.DiskInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DiskInfo indicates an expected call of DiskInfo. +func (mr *MockStorageAPIMockRecorder) DiskInfo(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiskInfo", reflect.TypeOf((*MockStorageAPI)(nil).DiskInfo), arg0, arg1, arg2) +} + +// GetShard mocks base method. +func (m *MockStorageAPI) GetShard(arg0 context.Context, arg1 string, arg2 *blobnode.GetShardArgs) (io.ReadCloser, uint32, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetShard", arg0, arg1, arg2) + ret0, _ := ret[0].(io.ReadCloser) + ret1, _ := ret[1].(uint32) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetShard indicates an expected call of GetShard. +func (mr *MockStorageAPIMockRecorder) GetShard(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetShard", reflect.TypeOf((*MockStorageAPI)(nil).GetShard), arg0, arg1, arg2) +} + +// GetShards mocks base method. +func (m *MockStorageAPI) GetShards(arg0 context.Context, arg1 string, arg2 *blobnode.GetShardsArgs) (io.ReadCloser, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetShards", arg0, arg1, arg2) + ret0, _ := ret[0].(io.ReadCloser) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetShards indicates an expected call of GetShards. +func (mr *MockStorageAPIMockRecorder) GetShards(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetShards", reflect.TypeOf((*MockStorageAPI)(nil).GetShards), arg0, arg1, arg2) +} + +// IsOnline mocks base method. +func (m *MockStorageAPI) IsOnline(arg0 context.Context, arg1 string) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsOnline", arg0, arg1) + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsOnline indicates an expected call of IsOnline. +func (mr *MockStorageAPIMockRecorder) IsOnline(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsOnline", reflect.TypeOf((*MockStorageAPI)(nil).IsOnline), arg0, arg1) +} + +// ListChunks mocks base method. +func (m *MockStorageAPI) ListChunks(arg0 context.Context, arg1 string, arg2 *blobnode.ListChunkArgs) ([]*blobnode.ChunkInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListChunks", arg0, arg1, arg2) + ret0, _ := ret[0].([]*blobnode.ChunkInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListChunks indicates an expected call of ListChunks. +func (mr *MockStorageAPIMockRecorder) ListChunks(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListChunks", reflect.TypeOf((*MockStorageAPI)(nil).ListChunks), arg0, arg1, arg2) +} + +// ListShards mocks base method. +func (m *MockStorageAPI) ListShards(arg0 context.Context, arg1 string, arg2 *blobnode.ListShardsArgs) ([]*blobnode.ShardInfo, proto.BlobID, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListShards", arg0, arg1, arg2) + ret0, _ := ret[0].([]*blobnode.ShardInfo) + ret1, _ := ret[1].(proto.BlobID) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// ListShards indicates an expected call of ListShards. +func (mr *MockStorageAPIMockRecorder) ListShards(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListShards", reflect.TypeOf((*MockStorageAPI)(nil).ListShards), arg0, arg1, arg2) +} + +// MarkDeleteShard mocks base method. +func (m *MockStorageAPI) MarkDeleteShard(arg0 context.Context, arg1 string, arg2 *blobnode.DeleteShardArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MarkDeleteShard", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// MarkDeleteShard indicates an expected call of MarkDeleteShard. +func (mr *MockStorageAPIMockRecorder) MarkDeleteShard(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkDeleteShard", reflect.TypeOf((*MockStorageAPI)(nil).MarkDeleteShard), arg0, arg1, arg2) +} + +// PutShard mocks base method. +func (m *MockStorageAPI) PutShard(arg0 context.Context, arg1 string, arg2 *blobnode.PutShardArgs) (uint32, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutShard", arg0, arg1, arg2) + ret0, _ := ret[0].(uint32) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutShard indicates an expected call of PutShard. +func (mr *MockStorageAPIMockRecorder) PutShard(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutShard", reflect.TypeOf((*MockStorageAPI)(nil).PutShard), arg0, arg1, arg2) +} + +// RangeGetShard mocks base method. +func (m *MockStorageAPI) RangeGetShard(arg0 context.Context, arg1 string, arg2 *blobnode.RangeGetShardArgs) (io.ReadCloser, uint32, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RangeGetShard", arg0, arg1, arg2) + ret0, _ := ret[0].(io.ReadCloser) + ret1, _ := ret[1].(uint32) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// RangeGetShard indicates an expected call of RangeGetShard. +func (mr *MockStorageAPIMockRecorder) RangeGetShard(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RangeGetShard", reflect.TypeOf((*MockStorageAPI)(nil).RangeGetShard), arg0, arg1, arg2) +} + +// ReleaseChunk mocks base method. +func (m *MockStorageAPI) ReleaseChunk(arg0 context.Context, arg1 string, arg2 *blobnode.ChangeChunkStatusArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReleaseChunk", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// ReleaseChunk indicates an expected call of ReleaseChunk. +func (mr *MockStorageAPIMockRecorder) ReleaseChunk(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReleaseChunk", reflect.TypeOf((*MockStorageAPI)(nil).ReleaseChunk), arg0, arg1, arg2) +} + +// RepairShard mocks base method. +func (m *MockStorageAPI) RepairShard(arg0 context.Context, arg1 string, arg2 *blobnode.ShardRepairArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RepairShard", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// RepairShard indicates an expected call of RepairShard. +func (mr *MockStorageAPIMockRecorder) RepairShard(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RepairShard", reflect.TypeOf((*MockStorageAPI)(nil).RepairShard), arg0, arg1, arg2) +} + +// SetChunkReadonly mocks base method. +func (m *MockStorageAPI) SetChunkReadonly(arg0 context.Context, arg1 string, arg2 *blobnode.ChangeChunkStatusArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetChunkReadonly", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetChunkReadonly indicates an expected call of SetChunkReadonly. +func (mr *MockStorageAPIMockRecorder) SetChunkReadonly(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetChunkReadonly", reflect.TypeOf((*MockStorageAPI)(nil).SetChunkReadonly), arg0, arg1, arg2) +} + +// SetChunkReadwrite mocks base method. +func (m *MockStorageAPI) SetChunkReadwrite(arg0 context.Context, arg1 string, arg2 *blobnode.ChangeChunkStatusArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetChunkReadwrite", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetChunkReadwrite indicates an expected call of SetChunkReadwrite. +func (mr *MockStorageAPIMockRecorder) SetChunkReadwrite(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetChunkReadwrite", reflect.TypeOf((*MockStorageAPI)(nil).SetChunkReadwrite), arg0, arg1, arg2) +} + +// Stat mocks base method. +func (m *MockStorageAPI) Stat(arg0 context.Context, arg1 string) ([]*blobnode.DiskInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Stat", arg0, arg1) + ret0, _ := ret[0].([]*blobnode.DiskInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Stat indicates an expected call of Stat. +func (mr *MockStorageAPIMockRecorder) Stat(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stat", reflect.TypeOf((*MockStorageAPI)(nil).Stat), arg0, arg1) +} + +// StatChunk mocks base method. +func (m *MockStorageAPI) StatChunk(arg0 context.Context, arg1 string, arg2 *blobnode.StatChunkArgs) (*blobnode.ChunkInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StatChunk", arg0, arg1, arg2) + ret0, _ := ret[0].(*blobnode.ChunkInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// StatChunk indicates an expected call of StatChunk. +func (mr *MockStorageAPIMockRecorder) StatChunk(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StatChunk", reflect.TypeOf((*MockStorageAPI)(nil).StatChunk), arg0, arg1, arg2) +} + +// StatShard mocks base method. +func (m *MockStorageAPI) StatShard(arg0 context.Context, arg1 string, arg2 *blobnode.StatShardArgs) (*blobnode.ShardInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StatShard", arg0, arg1, arg2) + ret0, _ := ret[0].(*blobnode.ShardInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// StatShard indicates an expected call of StatShard. +func (mr *MockStorageAPIMockRecorder) StatShard(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StatShard", reflect.TypeOf((*MockStorageAPI)(nil).StatShard), arg0, arg1, arg2) +} + +// String mocks base method. +func (m *MockStorageAPI) String(arg0 context.Context, arg1 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "String", arg0, arg1) + ret0, _ := ret[0].(string) + return ret0 +} + +// String indicates an expected call of String. +func (mr *MockStorageAPIMockRecorder) String(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "String", reflect.TypeOf((*MockStorageAPI)(nil).String), arg0, arg1) +} + +// WorkerStats mocks base method. +func (m *MockStorageAPI) WorkerStats(arg0 context.Context, arg1 string) (blobnode.WorkerStats, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WorkerStats", arg0, arg1) + ret0, _ := ret[0].(blobnode.WorkerStats) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// WorkerStats indicates an expected call of WorkerStats. +func (mr *MockStorageAPIMockRecorder) WorkerStats(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WorkerStats", reflect.TypeOf((*MockStorageAPI)(nil).WorkerStats), arg0, arg1) +} diff --git a/blobstore/testing/mocks/api_clustermgr.go b/blobstore/testing/mocks/api_clustermgr.go new file mode 100644 index 000000000..8aea3ce45 --- /dev/null +++ b/blobstore/testing/mocks/api_clustermgr.go @@ -0,0 +1,157 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cubefs/cubefs/blobstore/api/clustermgr (interfaces: ClientAPI) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + blobnode "github.com/cubefs/cubefs/blobstore/api/blobnode" + clustermgr "github.com/cubefs/cubefs/blobstore/api/clustermgr" + proto "github.com/cubefs/cubefs/blobstore/common/proto" + gomock "github.com/golang/mock/gomock" +) + +// MockClientAPI is a mock of ClientAPI interface. +type MockClientAPI struct { + ctrl *gomock.Controller + recorder *MockClientAPIMockRecorder +} + +// MockClientAPIMockRecorder is the mock recorder for MockClientAPI. +type MockClientAPIMockRecorder struct { + mock *MockClientAPI +} + +// NewMockClientAPI creates a new mock instance. +func NewMockClientAPI(ctrl *gomock.Controller) *MockClientAPI { + mock := &MockClientAPI{ctrl: ctrl} + mock.recorder = &MockClientAPIMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockClientAPI) EXPECT() *MockClientAPIMockRecorder { + return m.recorder +} + +// AllocBid mocks base method. +func (m *MockClientAPI) AllocBid(arg0 context.Context, arg1 *clustermgr.BidScopeArgs) (*clustermgr.BidScopeRet, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AllocBid", arg0, arg1) + ret0, _ := ret[0].(*clustermgr.BidScopeRet) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AllocBid indicates an expected call of AllocBid. +func (mr *MockClientAPIMockRecorder) AllocBid(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocBid", reflect.TypeOf((*MockClientAPI)(nil).AllocBid), arg0, arg1) +} + +// AllocVolume mocks base method. +func (m *MockClientAPI) AllocVolume(arg0 context.Context, arg1 *clustermgr.AllocVolumeArgs) (clustermgr.AllocatedVolumeInfos, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AllocVolume", arg0, arg1) + ret0, _ := ret[0].(clustermgr.AllocatedVolumeInfos) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AllocVolume indicates an expected call of AllocVolume. +func (mr *MockClientAPIMockRecorder) AllocVolume(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocVolume", reflect.TypeOf((*MockClientAPI)(nil).AllocVolume), arg0, arg1) +} + +// DiskInfo mocks base method. +func (m *MockClientAPI) DiskInfo(arg0 context.Context, arg1 proto.DiskID) (*blobnode.DiskInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DiskInfo", arg0, arg1) + ret0, _ := ret[0].(*blobnode.DiskInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DiskInfo indicates an expected call of DiskInfo. +func (mr *MockClientAPIMockRecorder) DiskInfo(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiskInfo", reflect.TypeOf((*MockClientAPI)(nil).DiskInfo), arg0, arg1) +} + +// GetConfig mocks base method. +func (m *MockClientAPI) GetConfig(arg0 context.Context, arg1 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetConfig", arg0, arg1) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetConfig indicates an expected call of GetConfig. +func (mr *MockClientAPIMockRecorder) GetConfig(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfig", reflect.TypeOf((*MockClientAPI)(nil).GetConfig), arg0, arg1) +} + +// GetService mocks base method. +func (m *MockClientAPI) GetService(arg0 context.Context, arg1 clustermgr.GetServiceArgs) (clustermgr.ServiceInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetService", arg0, arg1) + ret0, _ := ret[0].(clustermgr.ServiceInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetService indicates an expected call of GetService. +func (mr *MockClientAPIMockRecorder) GetService(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetService", reflect.TypeOf((*MockClientAPI)(nil).GetService), arg0, arg1) +} + +// GetVolumeInfo mocks base method. +func (m *MockClientAPI) GetVolumeInfo(arg0 context.Context, arg1 *clustermgr.GetVolumeArgs) (*clustermgr.VolumeInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVolumeInfo", arg0, arg1) + ret0, _ := ret[0].(*clustermgr.VolumeInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetVolumeInfo indicates an expected call of GetVolumeInfo. +func (mr *MockClientAPIMockRecorder) GetVolumeInfo(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVolumeInfo", reflect.TypeOf((*MockClientAPI)(nil).GetVolumeInfo), arg0, arg1) +} + +// RegisterService mocks base method. +func (m *MockClientAPI) RegisterService(arg0 context.Context, arg1 clustermgr.ServiceNode, arg2, arg3, arg4 uint32) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RegisterService", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].(error) + return ret0 +} + +// RegisterService indicates an expected call of RegisterService. +func (mr *MockClientAPIMockRecorder) RegisterService(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterService", reflect.TypeOf((*MockClientAPI)(nil).RegisterService), arg0, arg1, arg2, arg3, arg4) +} + +// RetainVolume mocks base method. +func (m *MockClientAPI) RetainVolume(arg0 context.Context, arg1 *clustermgr.RetainVolumeArgs) (clustermgr.RetainVolumes, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RetainVolume", arg0, arg1) + ret0, _ := ret[0].(clustermgr.RetainVolumes) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RetainVolume indicates an expected call of RetainVolume. +func (mr *MockClientAPIMockRecorder) RetainVolume(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RetainVolume", reflect.TypeOf((*MockClientAPI)(nil).RetainVolume), arg0, arg1) +} diff --git a/blobstore/testing/mocks/api_proxy.go b/blobstore/testing/mocks/api_proxy.go new file mode 100644 index 000000000..19ff807e5 --- /dev/null +++ b/blobstore/testing/mocks/api_proxy.go @@ -0,0 +1,130 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cubefs/cubefs/blobstore/api/proxy (interfaces: Client,LbMsgSender) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + proxy "github.com/cubefs/cubefs/blobstore/api/proxy" + gomock "github.com/golang/mock/gomock" +) + +// MockProxyClient is a mock of Client interface. +type MockProxyClient struct { + ctrl *gomock.Controller + recorder *MockProxyClientMockRecorder +} + +// MockProxyClientMockRecorder is the mock recorder for MockProxyClient. +type MockProxyClientMockRecorder struct { + mock *MockProxyClient +} + +// NewMockProxyClient creates a new mock instance. +func NewMockProxyClient(ctrl *gomock.Controller) *MockProxyClient { + mock := &MockProxyClient{ctrl: ctrl} + mock.recorder = &MockProxyClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockProxyClient) EXPECT() *MockProxyClientMockRecorder { + return m.recorder +} + +// SendDeleteMsg mocks base method. +func (m *MockProxyClient) SendDeleteMsg(arg0 context.Context, arg1 string, arg2 *proxy.DeleteArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendDeleteMsg", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendDeleteMsg indicates an expected call of SendDeleteMsg. +func (mr *MockProxyClientMockRecorder) SendDeleteMsg(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendDeleteMsg", reflect.TypeOf((*MockProxyClient)(nil).SendDeleteMsg), arg0, arg1, arg2) +} + +// SendShardRepairMsg mocks base method. +func (m *MockProxyClient) SendShardRepairMsg(arg0 context.Context, arg1 string, arg2 *proxy.ShardRepairArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendShardRepairMsg", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendShardRepairMsg indicates an expected call of SendShardRepairMsg. +func (mr *MockProxyClientMockRecorder) SendShardRepairMsg(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendShardRepairMsg", reflect.TypeOf((*MockProxyClient)(nil).SendShardRepairMsg), arg0, arg1, arg2) +} + +// VolumeAlloc mocks base method. +func (m *MockProxyClient) VolumeAlloc(arg0 context.Context, arg1 string, arg2 *proxy.AllocVolsArgs) ([]proxy.AllocRet, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VolumeAlloc", arg0, arg1, arg2) + ret0, _ := ret[0].([]proxy.AllocRet) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// VolumeAlloc indicates an expected call of VolumeAlloc. +func (mr *MockProxyClientMockRecorder) VolumeAlloc(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VolumeAlloc", reflect.TypeOf((*MockProxyClient)(nil).VolumeAlloc), arg0, arg1, arg2) +} + +// MockProxyLbRpcClient is a mock of LbMsgSender interface. +type MockProxyLbRpcClient struct { + ctrl *gomock.Controller + recorder *MockProxyLbRpcClientMockRecorder +} + +// MockProxyLbRpcClientMockRecorder is the mock recorder for MockProxyLbRpcClient. +type MockProxyLbRpcClientMockRecorder struct { + mock *MockProxyLbRpcClient +} + +// NewMockProxyLbRpcClient creates a new mock instance. +func NewMockProxyLbRpcClient(ctrl *gomock.Controller) *MockProxyLbRpcClient { + mock := &MockProxyLbRpcClient{ctrl: ctrl} + mock.recorder = &MockProxyLbRpcClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockProxyLbRpcClient) EXPECT() *MockProxyLbRpcClientMockRecorder { + return m.recorder +} + +// SendDeleteMsg mocks base method. +func (m *MockProxyLbRpcClient) SendDeleteMsg(arg0 context.Context, arg1 *proxy.DeleteArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendDeleteMsg", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendDeleteMsg indicates an expected call of SendDeleteMsg. +func (mr *MockProxyLbRpcClientMockRecorder) SendDeleteMsg(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendDeleteMsg", reflect.TypeOf((*MockProxyLbRpcClient)(nil).SendDeleteMsg), arg0, arg1) +} + +// SendShardRepairMsg mocks base method. +func (m *MockProxyLbRpcClient) SendShardRepairMsg(arg0 context.Context, arg1 *proxy.ShardRepairArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendShardRepairMsg", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendShardRepairMsg indicates an expected call of SendShardRepairMsg. +func (mr *MockProxyLbRpcClientMockRecorder) SendShardRepairMsg(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendShardRepairMsg", reflect.TypeOf((*MockProxyLbRpcClient)(nil).SendShardRepairMsg), arg0, arg1) +} diff --git a/blobstore/testing/mocks/api_scheduler.go b/blobstore/testing/mocks/api_scheduler.go new file mode 100644 index 000000000..f460743b5 --- /dev/null +++ b/blobstore/testing/mocks/api_scheduler.go @@ -0,0 +1,270 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cubefs/cubefs/blobstore/api/scheduler (interfaces: IScheduler) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + scheduler "github.com/cubefs/cubefs/blobstore/api/scheduler" + proto "github.com/cubefs/cubefs/blobstore/common/proto" + gomock "github.com/golang/mock/gomock" +) + +// MockIScheduler is a mock of IScheduler interface. +type MockIScheduler struct { + ctrl *gomock.Controller + recorder *MockISchedulerMockRecorder +} + +// MockISchedulerMockRecorder is the mock recorder for MockIScheduler. +type MockISchedulerMockRecorder struct { + mock *MockIScheduler +} + +// NewMockIScheduler creates a new mock instance. +func NewMockIScheduler(ctrl *gomock.Controller) *MockIScheduler { + mock := &MockIScheduler{ctrl: ctrl} + mock.recorder = &MockISchedulerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockIScheduler) EXPECT() *MockISchedulerMockRecorder { + return m.recorder +} + +// AcquireInspectTask mocks base method. +func (m *MockIScheduler) AcquireInspectTask(arg0 context.Context) (*scheduler.WorkerInspectTask, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AcquireInspectTask", arg0) + ret0, _ := ret[0].(*scheduler.WorkerInspectTask) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AcquireInspectTask indicates an expected call of AcquireInspectTask. +func (mr *MockISchedulerMockRecorder) AcquireInspectTask(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AcquireInspectTask", reflect.TypeOf((*MockIScheduler)(nil).AcquireInspectTask), arg0) +} + +// AcquireTask mocks base method. +func (m *MockIScheduler) AcquireTask(arg0 context.Context, arg1 *scheduler.AcquireArgs) (*scheduler.WorkerTask, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AcquireTask", arg0, arg1) + ret0, _ := ret[0].(*scheduler.WorkerTask) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AcquireTask indicates an expected call of AcquireTask. +func (mr *MockISchedulerMockRecorder) AcquireTask(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AcquireTask", reflect.TypeOf((*MockIScheduler)(nil).AcquireTask), arg0, arg1) +} + +// AddManualMigrateTask mocks base method. +func (m *MockIScheduler) AddManualMigrateTask(arg0 context.Context, arg1 *scheduler.AddManualMigrateArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddManualMigrateTask", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddManualMigrateTask indicates an expected call of AddManualMigrateTask. +func (mr *MockISchedulerMockRecorder) AddManualMigrateTask(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddManualMigrateTask", reflect.TypeOf((*MockIScheduler)(nil).AddManualMigrateTask), arg0, arg1) +} + +// BalanceTaskDetail mocks base method. +func (m *MockIScheduler) BalanceTaskDetail(arg0 context.Context, arg1 *scheduler.TaskStatArgs) (scheduler.MigrateTaskDetail, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BalanceTaskDetail", arg0, arg1) + ret0, _ := ret[0].(scheduler.MigrateTaskDetail) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// BalanceTaskDetail indicates an expected call of BalanceTaskDetail. +func (mr *MockISchedulerMockRecorder) BalanceTaskDetail(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BalanceTaskDetail", reflect.TypeOf((*MockIScheduler)(nil).BalanceTaskDetail), arg0, arg1) +} + +// CancelTask mocks base method. +func (m *MockIScheduler) CancelTask(arg0 context.Context, arg1 *scheduler.CancelTaskArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CancelTask", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// CancelTask indicates an expected call of CancelTask. +func (mr *MockISchedulerMockRecorder) CancelTask(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CancelTask", reflect.TypeOf((*MockIScheduler)(nil).CancelTask), arg0, arg1) +} + +// CompleteInspect mocks base method. +func (m *MockIScheduler) CompleteInspect(arg0 context.Context, arg1 *scheduler.CompleteInspectArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CompleteInspect", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// CompleteInspect indicates an expected call of CompleteInspect. +func (mr *MockISchedulerMockRecorder) CompleteInspect(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompleteInspect", reflect.TypeOf((*MockIScheduler)(nil).CompleteInspect), arg0, arg1) +} + +// CompleteTask mocks base method. +func (m *MockIScheduler) CompleteTask(arg0 context.Context, arg1 *scheduler.CompleteTaskArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CompleteTask", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// CompleteTask indicates an expected call of CompleteTask. +func (mr *MockISchedulerMockRecorder) CompleteTask(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompleteTask", reflect.TypeOf((*MockIScheduler)(nil).CompleteTask), arg0, arg1) +} + +// DiskDropTaskDetail mocks base method. +func (m *MockIScheduler) DiskDropTaskDetail(arg0 context.Context, arg1 *scheduler.TaskStatArgs) (scheduler.MigrateTaskDetail, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DiskDropTaskDetail", arg0, arg1) + ret0, _ := ret[0].(scheduler.MigrateTaskDetail) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DiskDropTaskDetail indicates an expected call of DiskDropTaskDetail. +func (mr *MockISchedulerMockRecorder) DiskDropTaskDetail(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiskDropTaskDetail", reflect.TypeOf((*MockIScheduler)(nil).DiskDropTaskDetail), arg0, arg1) +} + +// DiskRepairTaskDetail mocks base method. +func (m *MockIScheduler) DiskRepairTaskDetail(arg0 context.Context, arg1 *scheduler.TaskStatArgs) (scheduler.RepairTaskDetail, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DiskRepairTaskDetail", arg0, arg1) + ret0, _ := ret[0].(scheduler.RepairTaskDetail) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DiskRepairTaskDetail indicates an expected call of DiskRepairTaskDetail. +func (mr *MockISchedulerMockRecorder) DiskRepairTaskDetail(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiskRepairTaskDetail", reflect.TypeOf((*MockIScheduler)(nil).DiskRepairTaskDetail), arg0, arg1) +} + +// LeaderStats mocks base method. +func (m *MockIScheduler) LeaderStats(arg0 context.Context) (scheduler.TasksStat, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LeaderStats", arg0) + ret0, _ := ret[0].(scheduler.TasksStat) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// LeaderStats indicates an expected call of LeaderStats. +func (mr *MockISchedulerMockRecorder) LeaderStats(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LeaderStats", reflect.TypeOf((*MockIScheduler)(nil).LeaderStats), arg0) +} + +// ManualMigrateTaskDetail mocks base method. +func (m *MockIScheduler) ManualMigrateTaskDetail(arg0 context.Context, arg1 *scheduler.TaskStatArgs) (scheduler.MigrateTaskDetail, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ManualMigrateTaskDetail", arg0, arg1) + ret0, _ := ret[0].(scheduler.MigrateTaskDetail) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ManualMigrateTaskDetail indicates an expected call of ManualMigrateTaskDetail. +func (mr *MockISchedulerMockRecorder) ManualMigrateTaskDetail(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ManualMigrateTaskDetail", reflect.TypeOf((*MockIScheduler)(nil).ManualMigrateTaskDetail), arg0, arg1) +} + +// ReclaimTask mocks base method. +func (m *MockIScheduler) ReclaimTask(arg0 context.Context, arg1 *scheduler.ReclaimTaskArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReclaimTask", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// ReclaimTask indicates an expected call of ReclaimTask. +func (mr *MockISchedulerMockRecorder) ReclaimTask(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReclaimTask", reflect.TypeOf((*MockIScheduler)(nil).ReclaimTask), arg0, arg1) +} + +// RenewalTask mocks base method. +func (m *MockIScheduler) RenewalTask(arg0 context.Context, arg1 *scheduler.TaskRenewalArgs) (*scheduler.TaskRenewalRet, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RenewalTask", arg0, arg1) + ret0, _ := ret[0].(*scheduler.TaskRenewalRet) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RenewalTask indicates an expected call of RenewalTask. +func (mr *MockISchedulerMockRecorder) RenewalTask(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RenewalTask", reflect.TypeOf((*MockIScheduler)(nil).RenewalTask), arg0, arg1) +} + +// ReportTask mocks base method. +func (m *MockIScheduler) ReportTask(arg0 context.Context, arg1 *scheduler.TaskReportArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReportTask", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// ReportTask indicates an expected call of ReportTask. +func (mr *MockISchedulerMockRecorder) ReportTask(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportTask", reflect.TypeOf((*MockIScheduler)(nil).ReportTask), arg0, arg1) +} + +// Stats mocks base method. +func (m *MockIScheduler) Stats(arg0 context.Context, arg1 string) (scheduler.TasksStat, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Stats", arg0, arg1) + ret0, _ := ret[0].(scheduler.TasksStat) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Stats indicates an expected call of Stats. +func (mr *MockISchedulerMockRecorder) Stats(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stats", reflect.TypeOf((*MockIScheduler)(nil).Stats), arg0, arg1) +} + +// UpdateVol mocks base method. +func (m *MockIScheduler) UpdateVol(arg0 context.Context, arg1 string, arg2 proto.Vid) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateVol", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateVol indicates an expected call of UpdateVol. +func (mr *MockISchedulerMockRecorder) UpdateVol(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateVol", reflect.TypeOf((*MockIScheduler)(nil).UpdateVol), arg0, arg1, arg2) +} diff --git a/blobstore/testing/mocks/encoder.go b/blobstore/testing/mocks/encoder.go new file mode 100644 index 000000000..0f2f79a01 --- /dev/null +++ b/blobstore/testing/mocks/encoder.go @@ -0,0 +1,62 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cubefs/cubefs/blobstore/common/recordlog (interfaces: Encoder) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockEncoder is a mock of Encoder interface. +type MockEncoder struct { + ctrl *gomock.Controller + recorder *MockEncoderMockRecorder +} + +// MockEncoderMockRecorder is the mock recorder for MockEncoder. +type MockEncoderMockRecorder struct { + mock *MockEncoder +} + +// NewMockEncoder creates a new mock instance. +func NewMockEncoder(ctrl *gomock.Controller) *MockEncoder { + mock := &MockEncoder{ctrl: ctrl} + mock.recorder = &MockEncoderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockEncoder) EXPECT() *MockEncoderMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockEncoder) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockEncoderMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockEncoder)(nil).Close)) +} + +// Encode mocks base method. +func (m *MockEncoder) Encode(arg0 interface{}) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Encode", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Encode indicates an expected call of Encode. +func (mr *MockEncoderMockRecorder) Encode(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Encode", reflect.TypeOf((*MockEncoder)(nil).Encode), arg0) +} diff --git a/blobstore/testing/mocks/raft_server.go b/blobstore/testing/mocks/raft_server.go new file mode 100644 index 000000000..1954cb5b0 --- /dev/null +++ b/blobstore/testing/mocks/raft_server.go @@ -0,0 +1,158 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cubefs/cubefs/blobstore/common/raftserver (interfaces: RaftServer) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + raftserver "github.com/cubefs/cubefs/blobstore/common/raftserver" + gomock "github.com/golang/mock/gomock" +) + +// MockRaftServer is a mock of RaftServer interface. +type MockRaftServer struct { + ctrl *gomock.Controller + recorder *MockRaftServerMockRecorder +} + +// MockRaftServerMockRecorder is the mock recorder for MockRaftServer. +type MockRaftServerMockRecorder struct { + mock *MockRaftServer +} + +// NewMockRaftServer creates a new mock instance. +func NewMockRaftServer(ctrl *gomock.Controller) *MockRaftServer { + mock := &MockRaftServer{ctrl: ctrl} + mock.recorder = &MockRaftServerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRaftServer) EXPECT() *MockRaftServerMockRecorder { + return m.recorder +} + +// AddMember mocks base method. +func (m *MockRaftServer) AddMember(arg0 context.Context, arg1 raftserver.Member) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddMember", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddMember indicates an expected call of AddMember. +func (mr *MockRaftServerMockRecorder) AddMember(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddMember", reflect.TypeOf((*MockRaftServer)(nil).AddMember), arg0, arg1) +} + +// IsLeader mocks base method. +func (m *MockRaftServer) IsLeader() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsLeader") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsLeader indicates an expected call of IsLeader. +func (mr *MockRaftServerMockRecorder) IsLeader() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsLeader", reflect.TypeOf((*MockRaftServer)(nil).IsLeader)) +} + +// Propose mocks base method. +func (m *MockRaftServer) Propose(arg0 context.Context, arg1 []byte) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Propose", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Propose indicates an expected call of Propose. +func (mr *MockRaftServerMockRecorder) Propose(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Propose", reflect.TypeOf((*MockRaftServer)(nil).Propose), arg0, arg1) +} + +// ReadIndex mocks base method. +func (m *MockRaftServer) ReadIndex(arg0 context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadIndex", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// ReadIndex indicates an expected call of ReadIndex. +func (mr *MockRaftServerMockRecorder) ReadIndex(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadIndex", reflect.TypeOf((*MockRaftServer)(nil).ReadIndex), arg0) +} + +// RemoveMember mocks base method. +func (m *MockRaftServer) RemoveMember(arg0 context.Context, arg1 uint64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveMember", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// RemoveMember indicates an expected call of RemoveMember. +func (mr *MockRaftServerMockRecorder) RemoveMember(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveMember", reflect.TypeOf((*MockRaftServer)(nil).RemoveMember), arg0, arg1) +} + +// Status mocks base method. +func (m *MockRaftServer) Status() raftserver.Status { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Status") + ret0, _ := ret[0].(raftserver.Status) + return ret0 +} + +// Status indicates an expected call of Status. +func (mr *MockRaftServerMockRecorder) Status() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Status", reflect.TypeOf((*MockRaftServer)(nil).Status)) +} + +// Stop mocks base method. +func (m *MockRaftServer) Stop() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Stop") +} + +// Stop indicates an expected call of Stop. +func (mr *MockRaftServerMockRecorder) Stop() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*MockRaftServer)(nil).Stop)) +} + +// TransferLeadership mocks base method. +func (m *MockRaftServer) TransferLeadership(arg0 context.Context, arg1, arg2 uint64) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "TransferLeadership", arg0, arg1, arg2) +} + +// TransferLeadership indicates an expected call of TransferLeadership. +func (mr *MockRaftServerMockRecorder) TransferLeadership(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TransferLeadership", reflect.TypeOf((*MockRaftServer)(nil).TransferLeadership), arg0, arg1, arg2) +} + +// Truncate mocks base method. +func (m *MockRaftServer) Truncate(arg0 uint64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Truncate", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Truncate indicates an expected call of Truncate. +func (mr *MockRaftServerMockRecorder) Truncate(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Truncate", reflect.TypeOf((*MockRaftServer)(nil).Truncate), arg0) +} diff --git a/blobstore/testing/mocks/rpc_client.go b/blobstore/testing/mocks/rpc_client.go new file mode 100644 index 000000000..a61cc5203 --- /dev/null +++ b/blobstore/testing/mocks/rpc_client.go @@ -0,0 +1,225 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cubefs/cubefs/blobstore/common/rpc (interfaces: Client) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + http "net/http" + reflect "reflect" + + rpc "github.com/cubefs/cubefs/blobstore/common/rpc" + gomock "github.com/golang/mock/gomock" +) + +// MockRPCClient is a mock of Client interface. +type MockRPCClient struct { + ctrl *gomock.Controller + recorder *MockRPCClientMockRecorder +} + +// MockRPCClientMockRecorder is the mock recorder for MockRPCClient. +type MockRPCClientMockRecorder struct { + mock *MockRPCClient +} + +// NewMockRPCClient creates a new mock instance. +func NewMockRPCClient(ctrl *gomock.Controller) *MockRPCClient { + mock := &MockRPCClient{ctrl: ctrl} + mock.recorder = &MockRPCClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRPCClient) EXPECT() *MockRPCClientMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockRPCClient) Close() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Close") +} + +// Close indicates an expected call of Close. +func (mr *MockRPCClientMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockRPCClient)(nil).Close)) +} + +// Delete mocks base method. +func (m *MockRPCClient) Delete(arg0 context.Context, arg1 string) (*http.Response, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", arg0, arg1) + ret0, _ := ret[0].(*http.Response) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Delete indicates an expected call of Delete. +func (mr *MockRPCClientMockRecorder) Delete(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockRPCClient)(nil).Delete), arg0, arg1) +} + +// Do mocks base method. +func (m *MockRPCClient) Do(arg0 context.Context, arg1 *http.Request) (*http.Response, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Do", arg0, arg1) + ret0, _ := ret[0].(*http.Response) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Do indicates an expected call of Do. +func (mr *MockRPCClientMockRecorder) Do(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Do", reflect.TypeOf((*MockRPCClient)(nil).Do), arg0, arg1) +} + +// DoWith mocks base method. +func (m *MockRPCClient) DoWith(arg0 context.Context, arg1 *http.Request, arg2 interface{}, arg3 ...rpc.Option) error { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1, arg2} + for _, a := range arg3 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DoWith", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// DoWith indicates an expected call of DoWith. +func (mr *MockRPCClientMockRecorder) DoWith(arg0, arg1, arg2 interface{}, arg3 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1, arg2}, arg3...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DoWith", reflect.TypeOf((*MockRPCClient)(nil).DoWith), varargs...) +} + +// Form mocks base method. +func (m *MockRPCClient) Form(arg0 context.Context, arg1, arg2 string, arg3 map[string][]string) (*http.Response, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Form", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(*http.Response) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Form indicates an expected call of Form. +func (mr *MockRPCClientMockRecorder) Form(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Form", reflect.TypeOf((*MockRPCClient)(nil).Form), arg0, arg1, arg2, arg3) +} + +// Get mocks base method. +func (m *MockRPCClient) Get(arg0 context.Context, arg1 string) (*http.Response, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", arg0, arg1) + ret0, _ := ret[0].(*http.Response) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockRPCClientMockRecorder) Get(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockRPCClient)(nil).Get), arg0, arg1) +} + +// GetWith mocks base method. +func (m *MockRPCClient) GetWith(arg0 context.Context, arg1 string, arg2 interface{}) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetWith", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// GetWith indicates an expected call of GetWith. +func (mr *MockRPCClientMockRecorder) GetWith(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWith", reflect.TypeOf((*MockRPCClient)(nil).GetWith), arg0, arg1, arg2) +} + +// Head mocks base method. +func (m *MockRPCClient) Head(arg0 context.Context, arg1 string) (*http.Response, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Head", arg0, arg1) + ret0, _ := ret[0].(*http.Response) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Head indicates an expected call of Head. +func (mr *MockRPCClientMockRecorder) Head(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Head", reflect.TypeOf((*MockRPCClient)(nil).Head), arg0, arg1) +} + +// Post mocks base method. +func (m *MockRPCClient) Post(arg0 context.Context, arg1 string, arg2 interface{}) (*http.Response, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Post", arg0, arg1, arg2) + ret0, _ := ret[0].(*http.Response) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Post indicates an expected call of Post. +func (mr *MockRPCClientMockRecorder) Post(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Post", reflect.TypeOf((*MockRPCClient)(nil).Post), arg0, arg1, arg2) +} + +// PostWith mocks base method. +func (m *MockRPCClient) PostWith(arg0 context.Context, arg1 string, arg2, arg3 interface{}, arg4 ...rpc.Option) error { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1, arg2, arg3} + for _, a := range arg4 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "PostWith", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// PostWith indicates an expected call of PostWith. +func (mr *MockRPCClientMockRecorder) PostWith(arg0, arg1, arg2, arg3 interface{}, arg4 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1, arg2, arg3}, arg4...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PostWith", reflect.TypeOf((*MockRPCClient)(nil).PostWith), varargs...) +} + +// Put mocks base method. +func (m *MockRPCClient) Put(arg0 context.Context, arg1 string, arg2 interface{}) (*http.Response, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Put", arg0, arg1, arg2) + ret0, _ := ret[0].(*http.Response) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Put indicates an expected call of Put. +func (mr *MockRPCClientMockRecorder) Put(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockRPCClient)(nil).Put), arg0, arg1, arg2) +} + +// PutWith mocks base method. +func (m *MockRPCClient) PutWith(arg0 context.Context, arg1 string, arg2, arg3 interface{}, arg4 ...rpc.Option) error { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1, arg2, arg3} + for _, a := range arg4 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "PutWith", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// PutWith indicates an expected call of PutWith. +func (mr *MockRPCClientMockRecorder) PutWith(arg0, arg1, arg2, arg3 interface{}, arg4 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1, arg2, arg3}, arg4...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutWith", reflect.TypeOf((*MockRPCClient)(nil).PutWith), varargs...) +} diff --git a/blobstore/testing/mocks/task_switch.go b/blobstore/testing/mocks/task_switch.go new file mode 100644 index 000000000..26cd3ce7b --- /dev/null +++ b/blobstore/testing/mocks/task_switch.go @@ -0,0 +1,60 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cubefs/cubefs/blobstore/common/taskswitch (interfaces: ISwitcher) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockSwitcher is a mock of ISwitcher interface. +type MockSwitcher struct { + ctrl *gomock.Controller + recorder *MockSwitcherMockRecorder +} + +// MockSwitcherMockRecorder is the mock recorder for MockSwitcher. +type MockSwitcherMockRecorder struct { + mock *MockSwitcher +} + +// NewMockSwitcher creates a new mock instance. +func NewMockSwitcher(ctrl *gomock.Controller) *MockSwitcher { + mock := &MockSwitcher{ctrl: ctrl} + mock.recorder = &MockSwitcherMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSwitcher) EXPECT() *MockSwitcherMockRecorder { + return m.recorder +} + +// Enabled mocks base method. +func (m *MockSwitcher) Enabled() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Enabled") + ret0, _ := ret[0].(bool) + return ret0 +} + +// Enabled indicates an expected call of Enabled. +func (mr *MockSwitcherMockRecorder) Enabled() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Enabled", reflect.TypeOf((*MockSwitcher)(nil).Enabled)) +} + +// WaitEnable mocks base method. +func (m *MockSwitcher) WaitEnable() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "WaitEnable") +} + +// WaitEnable indicates an expected call of WaitEnable. +func (mr *MockSwitcherMockRecorder) WaitEnable() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitEnable", reflect.TypeOf((*MockSwitcher)(nil).WaitEnable)) +} diff --git a/blobstore/testing/mocks/util_selector.go b/blobstore/testing/mocks/util_selector.go new file mode 100644 index 000000000..5c13696fe --- /dev/null +++ b/blobstore/testing/mocks/util_selector.go @@ -0,0 +1,76 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cubefs/cubefs/blobstore/util/selector (interfaces: Selector) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockSelector is a mock of Selector interface. +type MockSelector struct { + ctrl *gomock.Controller + recorder *MockSelectorMockRecorder +} + +// MockSelectorMockRecorder is the mock recorder for MockSelector. +type MockSelectorMockRecorder struct { + mock *MockSelector +} + +// NewMockSelector creates a new mock instance. +func NewMockSelector(ctrl *gomock.Controller) *MockSelector { + mock := &MockSelector{ctrl: ctrl} + mock.recorder = &MockSelectorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSelector) EXPECT() *MockSelectorMockRecorder { + return m.recorder +} + +// GetHashN mocks base method. +func (m *MockSelector) GetHashN(arg0 int, arg1 []byte) []string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetHashN", arg0, arg1) + ret0, _ := ret[0].([]string) + return ret0 +} + +// GetHashN indicates an expected call of GetHashN. +func (mr *MockSelectorMockRecorder) GetHashN(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHashN", reflect.TypeOf((*MockSelector)(nil).GetHashN), arg0, arg1) +} + +// GetRandomN mocks base method. +func (m *MockSelector) GetRandomN(arg0 int) []string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRandomN", arg0) + ret0, _ := ret[0].([]string) + return ret0 +} + +// GetRandomN indicates an expected call of GetRandomN. +func (mr *MockSelectorMockRecorder) GetRandomN(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRandomN", reflect.TypeOf((*MockSelector)(nil).GetRandomN), arg0) +} + +// GetRoundRobinN mocks base method. +func (m *MockSelector) GetRoundRobinN(arg0 int) []string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRoundRobinN", arg0) + ret0, _ := ret[0].([]string) + return ret0 +} + +// GetRoundRobinN indicates an expected call of GetRoundRobinN. +func (mr *MockSelectorMockRecorder) GetRoundRobinN(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRoundRobinN", reflect.TypeOf((*MockSelector)(nil).GetRoundRobinN), arg0) +} diff --git a/blobstore/util/bytespool/pool.go b/blobstore/util/bytespool/pool.go new file mode 100644 index 000000000..8d459fffe --- /dev/null +++ b/blobstore/util/bytespool/pool.go @@ -0,0 +1,97 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package bytespool + +import "sync" + +func newBytes(size int) func() interface{} { + return func() interface{} { + return make([]byte, size) + } +} + +const ( + zeroSize int = 1 << 14 // 16K + + // 1K - 2K - 4K - 8K - 16K - 32K - 64K + numPools = 7 + sizeStep = 2 + startSize int = 1 << 10 // 1K + maxSize int = 1 << 16 // 64K +) + +var ( + zero = make([]byte, zeroSize) + + pools [numPools]sync.Pool + poolSize [numPools]int +) + +func init() { + size := startSize + for ii := 0; ii < numPools; ii++ { + pools[ii] = sync.Pool{ + New: newBytes(size), + } + poolSize[ii] = size + size *= sizeStep + } +} + +// GetPool returns a sync.Pool that generates bytes slice with the size. +// Return nil if no such pool exists. +func GetPool(size int) *sync.Pool { + for idx, psize := range poolSize { + if size <= psize { + return &pools[idx] + } + } + return nil +} + +// Alloc returns a bytes slice with the size. +// Make a new bytes slice if oversize. +func Alloc(size int) []byte { + if pool := GetPool(size); pool != nil { + b := pool.Get().([]byte) + return b[:size] + } + return make([]byte, size) +} + +// Free puts the bytes slice into suitable pool. +// Discard the bytes slice if oversize. +func Free(b []byte) { + size := cap(b) + if size > maxSize { + return + } + + b = b[0:size] + for ii := numPools - 1; ii >= 0; ii-- { + if size >= poolSize[ii] { + pools[ii].Put(b) // nolint: staticcheck + return + } + } +} + +// Zero clean up the bytes slice b to zero. +func Zero(b []byte) { + for len(b) > 0 { + n := copy(b, zero) + b = b[n:] + } +} diff --git a/blobstore/util/closer/closer.go b/blobstore/util/closer/closer.go new file mode 100644 index 000000000..b06f9b348 --- /dev/null +++ b/blobstore/util/closer/closer.go @@ -0,0 +1,57 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package closer + +import ( + "sync" +) + +// Closer is the interface for object that can release its resources. +type Closer interface { + // Close release all resources holded by the object. + Close() + // Done returns a channel that's closed when object was closed. + Done() <-chan struct{} +} + +// Close release all resources holded by the object. +func Close(obj interface{}) { + if obj == nil { + return + } + if c, ok := obj.(Closer); ok { + c.Close() + } +} + +// New returns a closer. +func New() Closer { + return &closer{ch: make(chan struct{})} +} + +type closer struct { + once sync.Once + ch chan struct{} +} + +func (c *closer) Close() { + c.once.Do(func() { + close(c.ch) + }) +} + +func (c *closer) Done() <-chan struct{} { + return c.ch +} diff --git a/blobstore/util/closer/closer_test.go b/blobstore/util/closer/closer_test.go new file mode 100644 index 000000000..b3dc51aaa --- /dev/null +++ b/blobstore/util/closer/closer_test.go @@ -0,0 +1,73 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package closer_test + +import ( + "sync" + "testing" + "time" + + "github.com/cubefs/cubefs/blobstore/util/closer" +) + +func TestCloserClose(t *testing.T) { + { + closer.Close(nil) + closer.Close(closer.New()) + closer.Close(0b10) + closer.Close("0x19") + } + { + c := closer.New() + for range [1 << 10]struct{}{} { + c.Close() + } + } + { + c := closer.New() + wg := sync.WaitGroup{} + wg.Add(1 << 10) + for range [1 << 10]struct{}{} { + go func() { + c.Close() + wg.Done() + }() + } + wg.Wait() + } +} + +func TestCloserDone(t *testing.T) { + { + c := closer.New() + c.Close() + for range [1 << 10]struct{}{} { + c.Done() + } + } + { + c := closer.New() + time.AfterFunc(200*time.Millisecond, c.Close) + wg := sync.WaitGroup{} + wg.Add(1 << 10) + for range [1 << 10]struct{}{} { + go func() { + <-c.Done() + wg.Done() + }() + } + wg.Wait() + } +} diff --git a/blobstore/util/defaulter/defaulter.go b/blobstore/util/defaulter/defaulter.go new file mode 100644 index 000000000..2ba87c347 --- /dev/null +++ b/blobstore/util/defaulter/defaulter.go @@ -0,0 +1,114 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package defaulter + +// Set basic type's value to default. +// Epsilon of float is 1e-9. + +import ( + "fmt" + "math" + "reflect" +) + +// Empty sets string value to default if it's empty. +func Empty(valPointer *string, defaultVal string) { + if *valPointer == "" { + *valPointer = defaultVal + } +} + +// Equal sets basic value to default if it equal zero. +func Equal(valPointer interface{}, defaultVal interface{}) { + setDefault(valPointer, defaultVal, equalZero) +} + +// Less sets basic value to default if it is less than zero. +func Less(valPointer interface{}, defaultVal interface{}) { + setDefault(valPointer, defaultVal, lessZero) +} + +// LessOrEqual sets basic value to default if it is not greater than zero. +func LessOrEqual(valPointer interface{}, defaultVal interface{}) { + setDefault(valPointer, defaultVal, lessOrEqualZero) +} + +func setDefault(valPointer interface{}, defaultVal interface{}, + cmp func(reflect.Value, reflect.Kind) bool) { + typ := reflect.TypeOf(valPointer) + if typ.Kind() != reflect.Ptr { + panic(typ.Name() + " must be pointer") + } + typ = typ.Elem() + val := reflect.ValueOf(valPointer).Elem() + + dTyp, dVal := parseDefault(defaultVal) + if typ.Kind() != dTyp.Kind() { + panic(fmt.Sprintf("not the same type %s != %s", typ.Kind().String(), dTyp.Kind().String())) + } + + if cmp(val, typ.Kind()) { + val.Set(dVal) + } +} + +func parseDefault(defaultVal interface{}) (reflect.Type, reflect.Value) { + typ, val := reflect.TypeOf(defaultVal), reflect.ValueOf(defaultVal) + if typ.Kind() == reflect.Ptr { + typ, val = typ.Elem(), val.Elem() + } + return typ, val +} + +func equalZero(val reflect.Value, typ reflect.Kind) bool { + switch typ { + case reflect.Bool: + return !val.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return val.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return val.Uint() == 0 + case reflect.Float32, reflect.Float64: + return math.Float64bits(val.Float()) == 0 + default: + panic("unsupported type " + typ.String()) + } +} + +func lessZero(val reflect.Value, typ reflect.Kind) bool { + switch typ { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return val.Int() < 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return false + case reflect.Float32, reflect.Float64: + return val.Float() < -1e-9 + default: + panic("unsupported type " + typ.String()) + } +} + +func lessOrEqualZero(val reflect.Value, typ reflect.Kind) bool { + switch typ { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return val.Int() <= 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return val.Uint() == 0 + case reflect.Float32, reflect.Float64: + return val.Float() < 1e-9 + default: + panic("unsupported type " + typ.String()) + } +} diff --git a/blobstore/util/defaulter/defaulter_test.go b/blobstore/util/defaulter/defaulter_test.go new file mode 100644 index 000000000..a3dc54d8f --- /dev/null +++ b/blobstore/util/defaulter/defaulter_test.go @@ -0,0 +1,241 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package defaulter_test + +import ( + "testing" + + "github.com/cubefs/cubefs/blobstore/util/defaulter" + "github.com/stretchr/testify/require" +) + +func TestDefaulterString(t *testing.T) { + for idx, cs := range []struct { + val, def, exp string + }{ + {"foo", "bar", "foo"}, + {"foo", "", "foo"}, + {"", "bar", "bar"}, + {"", "", ""}, + } { + defaulter.Empty(&cs.val, cs.def) + require.Equal(t, cs.exp, cs.val, idx) + } + + pfoo := func() *string { + foo := "foo" + return &foo + } + pempty := func() *string { + empty := "" + return &empty + } + for idx, cs := range []struct { + val *string + def, exp string + }{ + {pfoo(), "bar", "foo"}, + {pfoo(), "", "foo"}, + {pempty(), "bar", "bar"}, + {pempty(), "", ""}, + } { + defaulter.Empty(cs.val, cs.def) + require.Equal(t, cs.exp, *cs.val, idx) + } +} + +func TestDefaulterBasicNotType(t *testing.T) { + require.Panics(t, func() { + val := int(0) + defaulter.Equal(val, 10) + }) + require.Panics(t, func() { + val := int(0) + defaulter.Equal(&val, int64(10)) + }) + require.Panics(t, func() { + val := "" + defaulter.Equal(&val, "def") + }) + require.Panics(t, func() { + type none struct{} + defaulter.Less(&none{}, none{}) + }) + require.Panics(t, func() { + type none struct{} + defaulter.LessOrEqual(&none{}, none{}) + }) +} + +func TestDefaulterBasicEqual(t *testing.T) { + for idx, cs := range []struct { + val bool + def, exp interface{} + }{ + {true, true, true}, + {false, true, true}, + {false, false, false}, + } { + defaulter.Equal(&cs.val, cs.def) + require.Equal(t, cs.exp, cs.val, idx) + } + for idx, cs := range []struct { + val, def, exp int64 + }{ + {-1, 1, -1}, + {-1, 0, -1}, + {0, 1, 1}, + {0, 0, 0}, + {1, 2, 1}, + {1, 0, 1}, + } { + defaulter.Equal(&cs.val, cs.def) + require.Equal(t, cs.exp, cs.val, idx) + } + for idx, cs := range []struct { + val, def, exp uint64 + }{ + {0, 1, 1}, + {0, 0, 0}, + {1, 2, 1}, + {1, 0, 1}, + } { + defaulter.Equal(&cs.val, cs.def) + require.Equal(t, cs.exp, cs.val, idx) + } + for idx, cs := range []struct { + val, def, exp float64 + }{ + {-1.1, 0.1, -1.1}, + {-1e-10, 0.1, -1e-10}, + {0, 0.1, 0.1}, + {0, 0, 0}, + {1e-10, 0.1, 1e-10}, + {1.1, 2.1, 1.1}, + {1, 0, 1}, + } { + defaulter.Equal(&cs.val, cs.def) + require.Equal(t, cs.exp, cs.val, idx) + } +} + +func TestDefaulterBasicLess(t *testing.T) { + for idx, cs := range []struct { + val, def, exp int64 + }{ + {-1, 1, 1}, + {-1, 0, 0}, + {0, 1, 0}, + {0, 0, 0}, + {1, 2, 1}, + {1, 0, 1}, + } { + defaulter.Less(&cs.val, cs.def) + require.Equal(t, cs.exp, cs.val, idx) + } + for idx, cs := range []struct { + val, def, exp uint64 + }{ + {0, 1, 0}, + {0, 0, 0}, + {1, 2, 1}, + } { + defaulter.Less(&cs.val, cs.def) + require.Equal(t, cs.exp, cs.val, idx) + } + for idx, cs := range []struct { + val, def, exp float64 + }{ + {-1.1, 0.1, 0.1}, + {-1e-10, 0.1, -1e-10}, + {0, 0.1, 0}, + {0, 0, 0}, + {1e-10, 0.1, 1e-10}, + {1.1, 2.1, 1.1}, + {1, 0, 1}, + } { + defaulter.Less(&cs.val, cs.def) + require.Equal(t, cs.exp, cs.val, idx) + } +} + +func TestDefaulterBasicLessOrEqual(t *testing.T) { + for idx, cs := range []struct { + val, def, exp int64 + }{ + {-1, 1, 1}, + {-1, 0, 0}, + {0, 1, 1}, + {0, 0, 0}, + {1, 2, 1}, + {1, 0, 1}, + } { + defaulter.LessOrEqual(&cs.val, &cs.def) + require.Equal(t, cs.exp, cs.val, idx) + } + for idx, cs := range []struct { + val, def, exp uint64 + }{ + {0, 1, 1}, + {0, 0, 0}, + {1, 2, 1}, + } { + defaulter.LessOrEqual(&cs.val, &cs.def) + require.Equal(t, cs.exp, cs.val, idx) + } + for idx, cs := range []struct { + val, def, exp float64 + }{ + {-1.1, 0.1, 0.1}, + {-1e-10, 0.1, 0.1}, + {0, 0.1, 0.1}, + {0, 0, 0}, + {1e-10, 0.1, 0.1}, + {1e-8, 0.1, 1e-8}, + {1.1, 2.1, 1.1}, + {1, 0, 1}, + } { + defaulter.LessOrEqual(&cs.val, &cs.def) + require.Equal(t, cs.exp, cs.val, idx) + } +} + +func BenchmarkDefaulterString(b *testing.B) { + val, def := "", "foo" + for ii := 0; ii < b.N; ii++ { + defaulter.Empty(&val, def) + } +} + +func BenchmarkDefaulterInt(b *testing.B) { + val, def := int(0), int(1) + for ii := 0; ii < b.N; ii++ { + defaulter.LessOrEqual(&val, def) + } +} + +func BenchmarkDefaulterUint(b *testing.B) { + val, def := uint(0), uint(1) + for ii := 0; ii < b.N; ii++ { + defaulter.LessOrEqual(&val, def) + } +} + +func BenchmarkDefaulterFloat(b *testing.B) { + val, def := float32(0), float32(1) + for ii := 0; ii < b.N; ii++ { + defaulter.LessOrEqual(&val, def) + } +} diff --git a/blobstore/util/limit/count/count.go b/blobstore/util/limit/count/count.go new file mode 100644 index 000000000..cd479c0eb --- /dev/null +++ b/blobstore/util/limit/count/count.go @@ -0,0 +1,76 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package count + +import ( + "sync/atomic" + + "github.com/cubefs/cubefs/blobstore/util/limit" +) + +const minusOne = ^uint32(0) + +type countLimit struct { + limit uint32 + current uint32 +} + +// New returns limiter with concurrent n +func New(n int) limit.Limiter { + return &countLimit{limit: uint32(n)} +} + +func (l *countLimit) Running() int { + return int(atomic.LoadUint32(&l.current)) +} + +func (l *countLimit) Acquire(keys ...interface{}) error { + if atomic.AddUint32(&l.current, 1) > l.limit { + atomic.AddUint32(&l.current, minusOne) + return limit.ErrLimited + } + return nil +} + +func (l *countLimit) Release(keys ...interface{}) { + atomic.AddUint32(&l.current, minusOne) +} + +type blockingCountLimit struct { + ch chan struct{} +} + +// NewBlockingCount returns limiter with concurrent n +// Blocking acquire if no available concurrence +func NewBlockingCount(n int) limit.Limiter { + ch := make(chan struct{}, n) + for i := 0; i < n; i++ { + ch <- struct{}{} + } + return &blockingCountLimit{ch: ch} +} + +func (l *blockingCountLimit) Running() int { + return cap(l.ch) - len(l.ch) +} + +func (l *blockingCountLimit) Acquire(keys ...interface{}) error { + <-l.ch + return nil +} + +func (l *blockingCountLimit) Release(keys ...interface{}) { + l.ch <- struct{}{} +} diff --git a/blobstore/util/limit/count/count_test.go b/blobstore/util/limit/count/count_test.go new file mode 100644 index 000000000..d8b0d7656 --- /dev/null +++ b/blobstore/util/limit/count/count_test.go @@ -0,0 +1,106 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package count + +import ( + "runtime" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/util/limit" +) + +func init() { + runtime.GOMAXPROCS(4) +} + +func TestCountLimit(t *testing.T) { + l := New(2) + key := "a" + require.Equal(t, 0, l.Running()) + + require.NoError(t, l.Acquire(key)) + require.NoError(t, l.Acquire(key)) + require.ErrorIs(t, limit.ErrLimited, l.Acquire(key)) + require.Equal(t, 2, l.Running()) + + l.Release(key) + require.Equal(t, 1, l.Running()) + require.NoError(t, l.Acquire(key)) + require.ErrorIs(t, limit.ErrLimited, l.Acquire(key)) + require.Equal(t, 2, l.Running()) + + l.Release(key) + l.Release(key) + require.Equal(t, 0, l.Running()) + require.NoError(t, l.Acquire(key)) + require.NoError(t, l.Acquire(key)) + require.ErrorIs(t, limit.ErrLimited, l.Acquire(key)) + require.Equal(t, 2, l.Running()) +} + +func TestBlockingCountLimit(t *testing.T) { + l := NewBlockingCount(2) + var key interface{} = nil + require.Equal(t, 0, l.Running()) + + require.NoError(t, l.Acquire(key)) + require.NoError(t, l.Acquire(key)) + require.Equal(t, 2, l.Running()) + + l.Release(key) + require.Equal(t, 1, l.Running()) + require.NoError(t, l.Acquire(key)) + require.Equal(t, 2, l.Running()) + + l.Release(key) + l.Release(key) + require.Equal(t, 0, l.Running()) + require.NoError(t, l.Acquire(key)) + require.NoError(t, l.Acquire(key)) + require.Equal(t, 2, l.Running()) + + done := &atomicBool{} + go func() { + require.NoError(t, l.Acquire(key)) // blocking + done.Set(true) + }() + time.Sleep(.5e9) + require.Equal(t, 2, l.Running()) + require.False(t, done.Get()) + l.Release(key) + time.Sleep(.5e9) + require.Equal(t, 2, l.Running()) + require.True(t, done.Get()) +} + +type atomicBool struct { + ret int32 +} + +func (a *atomicBool) Set(v bool) { + if v { + atomic.StoreInt32(&a.ret, 1) + } else { + atomic.StoreInt32(&a.ret, 0) + } +} + +func (a *atomicBool) Get() (v bool) { + return atomic.LoadInt32(&a.ret) == 1 +} diff --git a/blobstore/util/limit/keycount/keycount.go b/blobstore/util/limit/keycount/keycount.go new file mode 100644 index 000000000..3b5b8e018 --- /dev/null +++ b/blobstore/util/limit/keycount/keycount.go @@ -0,0 +1,189 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package keycount + +import ( + "sync" + + "github.com/cubefs/cubefs/blobstore/util/limit" +) + +type keyCountLimit struct { + mutex sync.Mutex + limit uint32 + current map[interface{}]uint32 +} + +// New returns limiter with concurrent n by everyone key +func New(n int) limit.ResettableLimiter { + return &keyCountLimit{ + limit: uint32(n), + current: make(map[interface{}]uint32), + } +} + +func (l *keyCountLimit) Running() int { + l.mutex.Lock() + defer l.mutex.Unlock() + all := uint32(0) + for _, v := range l.current { + all += v + } + return int(all) +} + +func (l *keyCountLimit) Acquire(keys ...interface{}) error { + l.mutex.Lock() + defer l.mutex.Unlock() + + for _, key := range keys { + n := l.current[key] + if n >= l.limit { + return limit.ErrLimited + } + } + for _, key := range keys { + l.current[key]++ + } + + return nil +} + +func (l *keyCountLimit) Release(keys ...interface{}) { + l.mutex.Lock() + defer l.mutex.Unlock() + for _, key := range keys { + n, ok := l.current[key] + if !ok || n == 0 { + panic("released by 0") + } + if n == 1 { + delete(l.current, key) + } else { + l.current[key]-- + } + } +} + +func (l *keyCountLimit) Reset(n int) { + l.mutex.Lock() + l.limit = uint32(n) + l.mutex.Unlock() +} + +type blocker struct { + ref int32 + ready chan struct{} +} + +func newBlocker(n int) *blocker { + s := &blocker{ + ref: 0, + ready: make(chan struct{}, n), + } + for i := 0; i < n; i++ { + s.ready <- struct{}{} + } + return s +} + +func (s *blocker) acquire() { + <-s.ready +} + +func (s *blocker) release() { + s.ready <- struct{}{} +} + +func (s *blocker) addRef() { + s.ref++ +} + +func (s *blocker) subRef() int32 { + s.ref-- + return s.ref +} + +type blockingKeyCountLimit struct { + lock sync.RWMutex + limit int + keyMap map[interface{}]*blocker +} + +// NewBlockingKeyCountLimit returns blocking limiter +// with concurrent n by everyone key +func NewBlockingKeyCountLimit(n int) limit.Limiter { + return &blockingKeyCountLimit{ + limit: n, + keyMap: make(map[interface{}]*blocker), + } +} + +func (l *blockingKeyCountLimit) Running() int { + l.lock.RLock() + defer l.lock.RUnlock() + all := 0 + for _, v := range l.keyMap { + all += l.limit - len(v.ready) + } + return all +} + +func (l *blockingKeyCountLimit) Acquire(keys ...interface{}) error { + if len(keys) == 0 { + return limit.ErrLimited + } + kls := make([]*blocker, 0, len(keys)) + l.lock.Lock() + for _, key := range keys { + kl, ok := l.keyMap[key] + if !ok { + kl = newBlocker(l.limit) + l.keyMap[key] = kl + } + kl.addRef() + kls = append(kls, kl) + } + l.lock.Unlock() + for _, kl := range kls { + kl.acquire() + } + return nil +} + +func (l *blockingKeyCountLimit) Release(keys ...interface{}) { + kls := make([]*blocker, 0, len(keys)) + l.lock.Lock() + for _, key := range keys { + kl, ok := l.keyMap[key] + if !ok { + l.lock.Unlock() + panic("key not in map. Possible reason: Release without Acquire.") + } + ref := kl.subRef() + if ref < 0 { + l.lock.Unlock() + panic("internal error: refs < 0") + } + if ref == 0 { + delete(l.keyMap, key) + } + kls = append(kls, kl) + } + l.lock.Unlock() + for _, kl := range kls { + kl.release() + } +} diff --git a/blobstore/util/limit/keycount/keycount_test.go b/blobstore/util/limit/keycount/keycount_test.go new file mode 100644 index 000000000..82d799cad --- /dev/null +++ b/blobstore/util/limit/keycount/keycount_test.go @@ -0,0 +1,241 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package keycount + +import ( + "runtime" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/util/limit" +) + +func TestKeyLimitPanic(t *testing.T) { + l := New(2) + require.Panics(t, func() { l.Release("not acquired") }) + require.Panics(t, func() { l.Acquire([]byte("unhashable type as map key")) }) +} + +func TestKeyLimitBase(t *testing.T) { + l := New(2) + key1, key2, key3 := 1, "2", [1]byte{'3'} + require.Equal(t, 0, l.Running()) + + require.NoError(t, l.Acquire(key1)) + require.NoError(t, l.Acquire(key2)) + require.NoError(t, l.Acquire(key1)) + require.NoError(t, l.Acquire(key2)) + require.NoError(t, l.Acquire(key3)) + require.Equal(t, limit.ErrLimited, l.Acquire(key2)) + + require.Equal(t, 5, l.Running()) + + require.Equal(t, limit.ErrLimited, l.Acquire(key1)) + require.Equal(t, limit.ErrLimited, l.Acquire(key2)) + require.NoError(t, l.Acquire(key3)) + require.Equal(t, 6, l.Running()) + + l.Release(key1) + l.Release(key2) + require.Equal(t, 4, l.Running()) + require.NoError(t, l.Acquire(key1)) + require.NoError(t, l.Acquire(key2)) + require.Equal(t, limit.ErrLimited, l.Acquire(key3)) + require.Equal(t, 6, l.Running()) + + l.Reset(3) + require.NoError(t, l.Acquire(key1)) + require.Equal(t, limit.ErrLimited, l.Acquire(key1)) + l.Reset(2) + l.Release(key1) + require.Equal(t, limit.ErrLimited, l.Acquire(key1)) + l.Release(key1) + l.Release(key1) + require.NoError(t, l.Acquire(key1)) + require.NoError(t, l.Acquire(key1)) + require.Equal(t, limit.ErrLimited, l.Acquire(key1)) +} + +func TestBlockingKeyLimitPanic(t *testing.T) { + l := NewBlockingKeyCountLimit(2) + require.Panics(t, func() { l.Release("not acquired") }) + require.Panics(t, func() { l.Acquire([]byte("unhashable type as map key")) }) +} + +func TestBlockingKeyLimitBase(t *testing.T) { + l := NewBlockingKeyCountLimit(2) + key1, key2, key3 := 1, "2", [1]byte{'3'} + var done1, done2, done3 atomicBool + require.Equal(t, 0, l.Running()) + + require.NoError(t, l.Acquire(key1)) + require.NoError(t, l.Acquire(key2)) + require.NoError(t, l.Acquire(key1)) + require.NoError(t, l.Acquire(key2)) + require.NoError(t, l.Acquire(key3)) + go func() { + require.NoError(t, l.Acquire(key2)) // blocking + done2.Set(true) + }() + + time.Sleep(0.5e9) + require.Equal(t, 5, l.Running()) + + go func() { + require.NoError(t, l.Acquire(key1)) // blocking + done1.Set(true) + }() + time.Sleep(0.5e9) + require.NoError(t, l.Acquire(key3)) + require.Equal(t, 6, l.Running()) + require.False(t, done1.Get()) + require.False(t, done2.Get()) + + l.Release(key1) + l.Release(key2) + time.Sleep(0.5e9) + require.True(t, done1.Get()) + require.True(t, done2.Get()) + require.Equal(t, 6, l.Running()) + + done1, done2 = atomicBool{}, atomicBool{} + go func() { + require.NoError(t, l.Acquire(key1)) // blocking + done1.Set(true) + }() + go func() { + require.NoError(t, l.Acquire(key2)) // blocking + done2.Set(true) + }() + go func() { + require.NoError(t, l.Acquire(key3)) // blocking + done3.Set(true) + }() + time.Sleep(0.5e9) + require.False(t, done1.Get()) + require.False(t, done2.Get()) + require.False(t, done3.Get()) + + // for deadlock test + key := "1" + var wg sync.WaitGroup + runtime.GOMAXPROCS(10) + limiter := NewBlockingKeyCountLimit(1) + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { + limiter.Acquire(key) + time.Sleep(time.Millisecond * 10) + limiter.Release(key) + wg.Done() + }() + } + wg.Wait() +} + +type atomicBool struct { + ret int32 +} + +func (a *atomicBool) Set(v bool) { + if v { + atomic.StoreInt32(&a.ret, 1) + } else { + atomic.StoreInt32(&a.ret, 0) + } +} + +func (a *atomicBool) Get() (v bool) { + return atomic.LoadInt32(&a.ret) == 1 +} + +func testRelease(limiter limit.Limiter, number int, key []interface{}) { + for i := 0; i < number; i++ { + limiter.Acquire(key...) + } + var wg sync.WaitGroup + wg.Add(number) + for i := 0; i < number; i++ { + go func() { + limiter.Release(key...) + wg.Done() + }() + } + wg.Wait() +} + +func testAcquire(limiter limit.Limiter, number int, key []interface{}) { + var wg sync.WaitGroup + wg.Add(number) + for i := 0; i < number; i++ { + go func() { + limiter.Acquire(key...) + wg.Done() + }() + } + wg.Wait() +} + +func BenchmarkKeyCountLimit_Release(b *testing.B) { + for i := 0; i < b.N; i++ { + for _, thread := range []int{500} { + l := New(thread) + for _, value := range []int{1} { + testRelease(l, thread, getKeys(value)) + } + } + } +} + +func getKeys(num int) []interface{} { + ints := make([]interface{}, num) + for i := 0; i < num; i++ { + ints[i] = i + } + return ints +} + +func BenchmarkKeyCountLimit_Acquire(b *testing.B) { + // n := b.N + for i := 0; i < b.N; i++ { + for _, thread := range []int{500} { + l := New(thread) + for _, value := range []int{1} { + testAcquire(l, thread, getKeys(value)) + } + } + } +} + +func BenchmarkBlockingKeyCountLimit_Acquire(b *testing.B) { + l := NewBlockingKeyCountLimit(b.N) + key1 := 1 + for i := 0; i < b.N; i++ { + l.Acquire(key1) + } +} + +func BenchmarkBlockingKeyCountLimit_Release(b *testing.B) { + l := NewBlockingKeyCountLimit(b.N) + key1 := 1 + for i := 0; i < b.N; i++ { + l.Acquire(key1) + l.Release(key1) + } +} diff --git a/blobstore/util/limit/limit.go b/blobstore/util/limit/limit.go new file mode 100644 index 000000000..1a5efb2e5 --- /dev/null +++ b/blobstore/util/limit/limit.go @@ -0,0 +1,44 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package limit + +import "errors" + +// ErrLimited limited error for non-blocking +var ErrLimited = errors.New("limit exceeded") + +// Limiter to limit all by key +type Limiter interface { + // returns how many holder are running + // return -1 if u donot want to implement this + Running() int + + // Acquire by this keys, returns error if no available resource + // Panic if key is unhashable type necessarily + Acquire(keys ...interface{}) error + + // Release this keys holder + // Panic if not acquire yet necessarily + // Panic if key is unhashable type necessarily + Release(keys ...interface{}) +} + +// ResettableLimiter resetable limiter +type ResettableLimiter interface { + Limiter + + // Reset the available resource + Reset(n int) +} diff --git a/blobstore/util/limit/null/null.go b/blobstore/util/limit/null/null.go new file mode 100644 index 000000000..a3bca688f --- /dev/null +++ b/blobstore/util/limit/null/null.go @@ -0,0 +1,28 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package null + +import "github.com/cubefs/cubefs/blobstore/util/limit" + +type nullLimit struct{} + +// New returns limiter with nothing +func New() limit.Limiter { + return nullLimit{} +} + +func (l nullLimit) Running() int { return -1 } +func (l nullLimit) Acquire(keys ...interface{}) error { return nil } +func (l nullLimit) Release(keys ...interface{}) {} diff --git a/blobstore/util/limit/null/null_test.go b/blobstore/util/limit/null/null_test.go new file mode 100644 index 000000000..a4c319d06 --- /dev/null +++ b/blobstore/util/limit/null/null_test.go @@ -0,0 +1,38 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package null + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNullLimit(t *testing.T) { + l := New() + require.Equal(t, -1, l.Running()) + + for i := 0; i < 1000; i++ { + keys := make([]interface{}, rand.Intn(10)) + err := l.Acquire(keys...) + require.NoError(t, err) + } + + for i := 0; i < 100; i++ { + keys := make([]interface{}, rand.Intn(10)) + l.Release(keys...) + } +} diff --git a/blobstore/util/retry/insist.go b/blobstore/util/retry/insist.go new file mode 100644 index 000000000..110a90c9c --- /dev/null +++ b/blobstore/util/retry/insist.go @@ -0,0 +1,81 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package retry + +import ( + "context" + "time" +) + +// Insist successfully on f. +// On error every duration if has error. +// Sleep duration time after f run. +func Insist(duration time.Duration, f func() error, onError func(error)) { + err := f() + if err == nil { + return + } + onError(err) + + timer := time.NewTimer(duration) + defer timer.Stop() + <-timer.C + + for { + err = f() + if err == nil { + return + } + onError(err) + + timer.Reset(duration) + <-timer.C + } +} + +// InsistContext successfully on f or done with context. +// On error every duration if has error. +// Sleep duration time after f run. +func InsistContext(ctx context.Context, duration time.Duration, f func() error, onError func(error)) { + err := f() + if err == nil { + return + } + onError(err) + + timer := time.NewTimer(duration) + defer timer.Stop() + + select { + case <-ctx.Done(): + return + case <-timer.C: + } + + for { + err = f() + if err == nil { + return + } + onError(err) + + timer.Reset(duration) + select { + case <-ctx.Done(): + return + case <-timer.C: + } + } +} diff --git a/blobstore/util/retry/insist_test.go b/blobstore/util/retry/insist_test.go new file mode 100644 index 000000000..fb89138e4 --- /dev/null +++ b/blobstore/util/retry/insist_test.go @@ -0,0 +1,183 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package retry_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/util/retry" +) + +var ms100 = 100 * time.Millisecond + +func TestRetryInsistBase(t *testing.T) { + var i int + cases := []struct { + exp int + f func() error + on func(error) + d time.Duration + mind time.Duration + maxd time.Duration + }{ + { + exp: 0, + f: func() error { return nil }, + on: func(error) {}, + d: ms100, + mind: 0, + maxd: ms100, + }, + { + exp: 3, + f: func() error { + i++ + if i < 3 { + return fmt.Errorf("fake") + } + return nil + }, + on: func(error) {}, + d: 5 * ms100, + mind: 10 * ms100, + maxd: 12 * ms100, + }, + { + exp: 4, + f: func() error { + i += 2 + if i < 3 { + return fmt.Errorf("fake") + } + return nil + }, + on: func(error) {}, + d: 5 * ms100, + mind: 5 * ms100, + maxd: 7 * ms100, + }, + { + exp: 3, + f: func() error { + i++ + if i < 3 { + return fmt.Errorf("fake") + } + return nil + }, + on: func(error) { i++ }, + d: 5 * ms100, + mind: 5 * ms100, + maxd: 7 * ms100, + }, + } + + for _, cs := range cases { + i = 0 + + startTime := time.Now() + retry.Insist(cs.d, cs.f, cs.on) + require.Equal(t, cs.exp, i) + + duration := time.Since(startTime) + require.LessOrEqual(t, cs.mind, duration, "less duration: ", duration) + require.GreaterOrEqual(t, cs.maxd, duration, "greater duration: ", duration) + } +} + +func TestRetryInsistContext(t *testing.T) { + var i int + ctxDuration := 3 * ms100 + cases := []struct { + exp int + f func() error + on func(error) + d time.Duration + mind time.Duration + maxd time.Duration + }{ + { + exp: 0, + f: func() error { return nil }, + on: func(error) {}, + d: ms100, + mind: 0, + maxd: ms100, + }, + { + exp: 2, + f: func() error { + i++ + if i < 3 { + return fmt.Errorf("fake") + } + return nil + }, + on: func(error) {}, + d: 2 * ms100, + mind: 2 * ms100, + maxd: ctxDuration + ms100, + }, + { + exp: 2, + f: func() error { + i += 2 + if i < 3 { + return fmt.Errorf("fake") + } + return nil + }, + on: func(error) {}, + d: 5 * ms100, + mind: ctxDuration, + maxd: ctxDuration + ms100, + }, + { + exp: 3, + f: func() error { + i++ + if i < 3 { + return fmt.Errorf("fake") + } + return nil + }, + on: func(error) { i++ }, + d: ms100, + mind: ms100, + maxd: 2 * ms100, + }, + } + + for _, cs := range cases { + i = 0 + + startTime := time.Now() + ctx, cancel := context.WithDeadline(context.TODO(), startTime.Add(ctxDuration)) + + retry.InsistContext(ctx, cs.d, cs.f, cs.on) + require.Equal(t, cs.exp, i) + + duration := time.Since(startTime) + require.LessOrEqual(t, cs.mind, duration, "less duration: ", duration) + require.GreaterOrEqual(t, cs.maxd, duration, "greater duration: ", duration) + + cancel() + } +} diff --git a/blobstore/util/retry/retry.go b/blobstore/util/retry/retry.go new file mode 100644 index 000000000..6a6eac9bb --- /dev/null +++ b/blobstore/util/retry/retry.go @@ -0,0 +1,109 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package retry + +import ( + "errors" + "time" +) + +var ( + // ErrRetryFailed all retry attempts failed. + ErrRetryFailed = errors.New("retry: all retry attempts failed") + // ErrRetryNext retry next on interrupt. + ErrRetryNext = errors.New("retry: retry next on interrupt") +) + +// Retryer is an interface retry on a specific function. +type Retryer interface { + // On performs a retry on function, until it doesn't return any error. + On(func() error) error + // RuptOn performs a retry on function, until it doesn't return any error or interrupt. + RuptOn(func() (bool, error)) error +} + +type retry struct { + attempts int + nextDelay func() uint32 +} + +// On implements Retryer.On. +func (r *retry) On(caller func() error) error { + var lastErr error + attempt := 1 + for attempt <= r.attempts { + if lastErr = caller(); lastErr == nil { + return nil + } + + // do not wait on last useless delay + if attempt >= r.attempts { + break + } + time.Sleep(time.Duration(r.nextDelay()) * time.Millisecond) + attempt++ + } + return lastErr +} + +// RuptOn implements Retryer.RuptOn. +func (r *retry) RuptOn(caller func() (bool, error)) error { + var lastErr error + attempt := 1 + for attempt <= r.attempts { + interrupted, err := caller() + if err == nil { + return nil + } + // return last error of method, if interrupted + if err != ErrRetryNext { + lastErr = err + } + if interrupted { + break + } + + // do not wait on last useless delay + if attempt >= r.attempts { + break + } + time.Sleep(time.Duration(r.nextDelay()) * time.Millisecond) + attempt++ + } + return lastErr +} + +// Timed returns a retry with fixed interval delay. +func Timed(attempts int, delay uint32) Retryer { + return &retry{ + attempts: attempts, + nextDelay: func() uint32 { + return delay + }, + } +} + +// ExponentialBackoff returns a retry with exponential delay. +func ExponentialBackoff(attempts int, expDelay uint32) Retryer { + next := expDelay + return &retry{ + attempts: attempts, + nextDelay: func() uint32 { + r := next + next += expDelay + return r + }, + } +} diff --git a/blobstore/util/retry/retry_test.go b/blobstore/util/retry/retry_test.go new file mode 100644 index 000000000..7fd0c5e9a --- /dev/null +++ b/blobstore/util/retry/retry_test.go @@ -0,0 +1,171 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package retry_test + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/util/retry" +) + +var ( + errTestOnly = errors.New("test: this is a fake error") + errTestInterrupt = errors.New("test: this is a interruptable error") +) + +func TestRetryNoRetry(t *testing.T) { + st := time.Now().Unix() + err := retry.Timed(10, 1000000).On(func() error { + return nil + }) + et := time.Now().Unix() + + require.NoError(t, err) + require.LessOrEqual(t, et, st) +} + +func TestRetryOnce(t *testing.T) { + st := time.Now() + called := 0 + err := retry.Timed(10, 1000).On(func() error { + if called == 0 { + called++ + return errTestOnly + } + return nil + }) + duration := time.Since(st) + + require.NoError(t, err) + require.Equal(t, 1, called) + v := int64(duration / time.Millisecond) + require.Less(t, int64(900), v, "duration: ", v) +} + +func TestRetryMultiple(t *testing.T) { + st := time.Now() + called := 0 + err := retry.Timed(10, 1000).On(func() error { + if called < 5 { + called++ + return errTestOnly + } + return nil + }) + duration := time.Since(st) + + require.NoError(t, err) + require.Equal(t, 5, called) + v := int64(duration / time.Millisecond) + require.Less(t, int64(4900), v, "duration: ", v) +} + +func TestRetryExhausted(t *testing.T) { + st := time.Now() + called := 0 + err := retry.Timed(2, 1000).On(func() error { + called++ + if called == 2 { + return retry.ErrRetryFailed + } + return errTestOnly + }) + duration := time.Since(st) + + require.ErrorIs(t, err, retry.ErrRetryFailed) + v := int64(duration / time.Millisecond) + require.Less(t, int64(900), v, "duration: ", v) +} + +func TestRetryExponentialBackoff(t *testing.T) { + st := time.Now() + called := 0 + err := retry.ExponentialBackoff(10, 100).On(func() error { + called++ + return errTestOnly + }) + duration := time.Since(st) + + require.ErrorIs(t, err, errTestOnly) + v := int64(duration / time.Millisecond) + require.Less(t, int64(4300), v, "duration: ", v) + require.Greater(t, int64(4700), v, "duration: ", v) +} + +func TestRetryInterrupted(t *testing.T) { + st := time.Now() + called := 0 + err := retry.Timed(10, 1000).RuptOn(func() (bool, error) { + if called < 2 { + called++ + return false, errTestOnly + } + if called == 2 { + return true, errTestInterrupt + } + return false, nil + }) + duration := time.Since(st) + + require.ErrorIs(t, errTestInterrupt, err) + require.Equal(t, 2, called) + v := int64(duration / time.Millisecond) + require.Less(t, int64(1900), v, "duration: ", v) +} + +func TestRetryInterruptedError(t *testing.T) { + st := time.Now() + called := 0 + err := retry.Timed(10, 1000).RuptOn(func() (bool, error) { + if called < 1 { + called++ + return false, errTestOnly + } + if called == 1 { + return true, retry.ErrRetryNext + } + return false, nil + }) + duration := time.Since(st) + + // get last error if interrupt with ErrRetryNext + require.ErrorIs(t, errTestOnly, err) + require.Equal(t, 1, called) + v := int64(duration / time.Millisecond) + require.Less(t, int64(900), v, "duration: ", v) + + called = 0 + err = retry.Timed(10, 1000).RuptOn(func() (bool, error) { + if called < 1 { + called++ + return false, retry.ErrRetryNext + } + if called == 1 { + return true, retry.ErrRetryNext + } + return false, nil + }) + duration = time.Since(st) + + // ignored the ErrRetryNext + require.ErrorIs(t, nil, err) + require.Equal(t, 1, called) + v = int64(duration / time.Millisecond) + require.Less(t, int64(900), v, "duration: ", v) +} diff --git a/blobstore/util/selector/selector.go b/blobstore/util/selector/selector.go new file mode 100644 index 000000000..15f93bc91 --- /dev/null +++ b/blobstore/util/selector/selector.go @@ -0,0 +1,179 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package selector + +import ( + "errors" + "hash/fnv" + "math/rand" + "sort" + "sync" + "time" +) + +// Selector select hosts or something from somewhere. +type Selector interface { + GetHashN(n int, key []byte) []string + GetRandomN(n int) []string + GetRoundRobinN(n int) []string +} + +type selector struct { + sync.RWMutex + + interval int64 + lastUpdate int64 + nextIndex int + cachedValues []string + getter func() ([]string, error) +} + +// NewSelector implements Selector +func NewSelector(intervalMs int64, getter func() ([]string, error)) (Selector, error) { + values, err := getter() + if err != nil { + return nil, err + } + if len(values) == 0 { + return nil, errors.New("not found initial values from getter") + } + + s := &selector{ + interval: int64(time.Millisecond) * intervalMs, + lastUpdate: time.Now().UnixNano(), + nextIndex: 0, + cachedValues: values, + getter: getter, + } + return s, nil +} + +// NewSelectorWithGetter always return a selector +func NewSelectorWithGetter(intervalMs int64, getter func() ([]string, error)) Selector { + values, _ := getter() + return &selector{ + interval: int64(time.Millisecond) * intervalMs, + lastUpdate: time.Now().UnixNano(), + nextIndex: 0, + cachedValues: values, + getter: getter, + } +} + +func (s *selector) GetHashN(n int, key []byte) []string { + if n <= 0 { + return nil + } + values := s.getValues(n, false) + if len(values) <= 1 { + return values + } + + idx := int(keyHash(key) % uint64(len(values))) + values[0], values[idx] = values[idx], values[0] + + hashed := values[1:] + rand.Shuffle(len(hashed), func(i, j int) { + hashed[i], hashed[j] = hashed[j], hashed[i] + }) + + if n <= len(values) { + values = values[:n] + } + return values +} + +func (s *selector) GetRandomN(n int) []string { + if n <= 0 { + return nil + } + values := s.getValues(n, false) + rand.Shuffle(len(values), func(i, j int) { + values[i], values[j] = values[j], values[i] + }) + + if n <= len(values) { + values = values[:n] + } + return values +} + +func (s *selector) GetRoundRobinN(n int) []string { + if n <= 0 { + return nil + } + values := s.getValues(n, true) + if n <= len(values) { + values = values[:n] + } + return values +} + +func (s *selector) sync() { + values, err := s.getter() + if err != nil { + return + } + if len(values) == 0 { + return + } + sort.Strings(values) + + s.Lock() + if s.nextIndex >= len(values) { + s.nextIndex = 0 + } + s.cachedValues = values + s.lastUpdate = time.Now().UnixNano() + s.Unlock() +} + +func (s *selector) getValues(n int, rr bool) []string { + s.RLock() + interval := s.interval + lastUpdate := s.lastUpdate + idx := s.nextIndex + values := make([]string, len(s.cachedValues)) + copy(values, s.cachedValues) + s.RUnlock() + + if rr { + if n > len(values) { + n = len(values) + } + + s.Lock() + s.nextIndex = (s.nextIndex + n) % len(s.cachedValues) + s.Unlock() + + values = append(values[idx:], values[0:idx]...) + } + + if time.Now().UnixNano() >= lastUpdate+interval { + go s.sync() + } + + return values +} + +func keyHash(key []byte) uint64 { + f := fnv.New64() + f.Write(key) + return f.Sum64() +} + +func init() { + rand.Seed(time.Now().UnixNano()) +} diff --git a/blobstore/util/selector/selector_test.go b/blobstore/util/selector/selector_test.go new file mode 100644 index 000000000..3a7a62de5 --- /dev/null +++ b/blobstore/util/selector/selector_test.go @@ -0,0 +1,185 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package selector + +import ( + "errors" + "sort" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestSelector_NewSelector(t *testing.T) { + { + _, err := NewSelector(100, func() (strings []string, e error) { + return nil, errors.New("no hosts") + }) + require.Error(t, err) + } + { + _, err := NewSelector(100, func() (strings []string, e error) { + return nil, nil + }) + require.Error(t, err) + } + { + _, err := NewSelector(100, func() (strings []string, e error) { + return []string{}, nil + }) + require.Error(t, err) + } +} + +func TestSelector_GetRandomN(t *testing.T) { + type fields struct { + hosts []string + } + type args struct { + n int + } + tests := []struct { + name string + fields fields + args args + expectHosts []string + }{ + {"1", fields{hosts: []string{"A"}}, args{0}, nil}, + {"2", fields{hosts: []string{"A"}}, args{-1}, nil}, + {"3", fields{hosts: []string{"A"}}, args{1}, []string{"A"}}, + {"4", fields{hosts: []string{"A"}}, args{2}, []string{"A"}}, + {"4", fields{hosts: []string{"A", "B"}}, args{2}, []string{"A", "B"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + selector, err := NewSelector(100, func() (strings []string, e error) { + return tt.fields.hosts, nil + }) + require.NoError(t, err) + + gotHosts := selector.GetRandomN(tt.args.n) + sort.Strings(gotHosts) + require.Equal(t, tt.expectHosts, gotHosts) + }) + } +} + +func TestSelector_GetHashN(t *testing.T) { + type fields struct { + hosts []string + } + type args struct { + n int + key string + } + h := []string{"A", "B", "C", "D", "E"} + + key1 := []byte("1234") + key2 := []byte("4321") + key3 := []byte("") + + idx1 := keyHash(key1) % uint64(len(h)) + idx2 := keyHash(key2) % uint64(len(h)) + idx3 := keyHash(key3) % uint64(len(h)) + tests := []struct { + name string + fields fields + args args + expectHosts []string + expectErr bool + }{ + {"1", fields{hosts: []string{}}, args{n: 0}, nil, true}, + {"2", fields{hosts: []string{}}, args{n: -1}, nil, true}, + {"3", fields{hosts: []string{"A"}}, args{n: 1}, []string{"A"}, false}, + {"4", fields{hosts: []string{"A"}}, args{n: 2}, []string{"A"}, false}, + {"5", fields{hosts: []string{}}, args{n: 2}, nil, true}, + {"6", fields{hosts: h}, args{n: 1, key: "1234"}, []string{h[idx1]}, false}, + {"7", fields{hosts: h}, args{n: 1, key: "4321"}, []string{h[idx2]}, false}, + {"8", fields{hosts: h}, args{n: 1, key: ""}, []string{h[idx3]}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + selector, err := NewSelector(100, func() (strings []string, e error) { + return tt.fields.hosts, nil + }) + if tt.expectErr { + require.Error(t, err) + return + } + require.NoError(t, err) + + gotHosts := selector.GetHashN(tt.args.n, []byte(tt.args.key)) + require.Equal(t, tt.expectHosts, gotHosts) + }) + } +} + +func TestSelector_GetRoundRobinN(t *testing.T) { + first := true + selector, err := NewSelector(500, func() (strings []string, e error) { + if first { + first = false + return []string{"1", "2", "3"}, nil + } + + return []string{"4", "5"}, nil + }) + require.NoError(t, err) + + require.Nil(t, selector.GetRoundRobinN(-1)) + require.Nil(t, selector.GetRoundRobinN(0)) + + require.Equal(t, []string{"1"}, selector.GetRoundRobinN(1)) + require.Equal(t, []string{"2"}, selector.GetRoundRobinN(1)) + require.Equal(t, []string{"3"}, selector.GetRoundRobinN(1)) + require.Equal(t, []string{"1", "2"}, selector.GetRoundRobinN(2)) + require.Equal(t, []string{"3", "1"}, selector.GetRoundRobinN(2)) + + time.Sleep(time.Millisecond * 500) + require.Equal(t, []string{"2"}, selector.GetRoundRobinN(1)) // cached + time.Sleep(time.Millisecond * 100) + require.Equal(t, []string{"4"}, selector.GetRoundRobinN(1)) + require.Equal(t, []string{"5", "4"}, selector.GetRoundRobinN(2)) + require.Equal(t, []string{"5", "4"}, selector.GetRoundRobinN(100)) +} + +func Benchmark_GetHashN(b *testing.B) { + selector, _ := NewSelector(10000, func() (strings []string, e error) { + return []string{"", "1", "2", "3"}, nil + }) + key := []byte("key") + for ii := 0; ii < b.N; ii++ { + selector.GetHashN(1, key) + } +} + +func Benchmark_GetRandomN(b *testing.B) { + selector, _ := NewSelector(10000, func() (strings []string, e error) { + return []string{"", "1", "2", "3"}, nil + }) + for ii := 0; ii < b.N; ii++ { + selector.GetRandomN(1) + } +} + +func Benchmark_GetRoundRobinN(b *testing.B) { + selector, _ := NewSelector(10000, func() (strings []string, e error) { + return []string{"", "1", "2", "3"}, nil + }) + for ii := 0; ii < b.N; ii++ { + selector.GetRoundRobinN(1) + } +} diff --git a/blobstore/util/task/concurrent.go b/blobstore/util/task/concurrent.go new file mode 100644 index 000000000..e4f97a266 --- /dev/null +++ b/blobstore/util/task/concurrent.go @@ -0,0 +1,43 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package task + +import "context" + +var ( + // C alias of Concurrent + C = Concurrent + // CC alias of ConcurrentContext + CC = ConcurrentContext +) + +// Concurrent is tasks run concurrently. +func Concurrent(f func(index int, arg interface{}), args []interface{}) { + ConcurrentContext(context.Background(), f, args) +} + +// ConcurrentContext is tasks run concurrently with context. +// How to make []interface{} see: https://golang.org/doc/faq#convert_slice_of_interface +func ConcurrentContext(ctx context.Context, f func(index int, arg interface{}), args []interface{}) { + tasks := make([]func() error, len(args)) + for ii := 0; ii < len(args); ii++ { + index, arg := ii, args[ii] + tasks[ii] = func() error { + f(index, arg) + return nil + } + } + Run(ctx, tasks...) +} diff --git a/blobstore/util/task/concurrent_test.go b/blobstore/util/task/concurrent_test.go new file mode 100644 index 000000000..39267730a --- /dev/null +++ b/blobstore/util/task/concurrent_test.go @@ -0,0 +1,57 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package task_test + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/util/task" +) + +func TestTaskExecuteConcurrent(t *testing.T) { + unique := uint32(0) + args := []interface{}{uint32(1), uint32(2), uint32(4)} + task.C(func(index int, arg interface{}) { + time.Sleep(time.Millisecond * 100) + atomic.AddUint32(&unique, arg.(uint32)) + }, args) + require.Equal(t, uint32(7), atomic.LoadUint32(&unique)) +} + +func TestTaskExecuteContextCancel(t *testing.T) { + unique := uint32(0) + ctx, cancel := context.WithCancel(context.Background()) + err := task.Run(ctx, func() error { + task.C(func(index int, arg interface{}) { + time.Sleep(time.Millisecond * 2000) + atomic.AddUint32(&unique, 1) + }, []interface{}{1, 2, 4}) + return nil + }, func() error { + time.Sleep(time.Millisecond * 5000) + return errors.New("test") + }, func() error { + cancel() + return nil + }) + require.Contains(t, err.Error(), "canceled") + require.Equal(t, uint32(0), atomic.LoadUint32(&unique)) +} diff --git a/blobstore/util/task/task.go b/blobstore/util/task/task.go new file mode 100644 index 000000000..0803e3224 --- /dev/null +++ b/blobstore/util/task/task.go @@ -0,0 +1,77 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package task + +import ( + "context" +) + +type semaphore struct { + ready chan struct{} +} + +func newSemaphore(n int) *semaphore { + s := &semaphore{ + ready: make(chan struct{}, n), + } + for ii := 0; ii < n; ii++ { + s.ready <- struct{}{} + } + return s +} + +func (s *semaphore) Wait() <-chan struct{} { + return s.ready +} + +func (s *semaphore) Signal() { + s.ready <- struct{}{} +} + +// Run executes list of tasks in parallel, +// returns the first error or nil if all tasks done. +func Run(ctx context.Context, tasks ...func() error) error { + n := len(tasks) + semaphore := newSemaphore(n) + errorCh := make(chan error, 1) + + for _, task := range tasks { + <-semaphore.Wait() + go func(task func() error) { + err := task() + if err == nil { + semaphore.Signal() + return + } + + select { + case errorCh <- err: + default: + } + }(task) + } + + for ii := 0; ii < n; ii++ { + select { + case err := <-errorCh: + return err + case <-ctx.Done(): + return ctx.Err() + case <-semaphore.Wait(): + } + } + + return nil +} diff --git a/blobstore/util/task/task_test.go b/blobstore/util/task/task_test.go new file mode 100644 index 000000000..8795b5641 --- /dev/null +++ b/blobstore/util/task/task_test.go @@ -0,0 +1,75 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package task_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/util/task" +) + +func TestTaskRunParallel(t *testing.T) { + err := task.Run(context.Background(), + func() error { + time.Sleep(time.Millisecond * 100) + return errors.New("test") + }, + func() error { + time.Sleep(time.Millisecond * 500) + return errors.New("test2") + }, + ) + require.Error(t, err) + require.Equal(t, "test", err.Error()) +} + +func TestTaskRunParallelCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + err := task.Run(ctx, + func() error { + time.Sleep(time.Second * 1) + return errors.New("test") + }, + func() error { + time.Sleep(time.Second * 5) + return errors.New("test2") + }, + func() error { + cancel() + return nil + }, + ) + require.Error(t, err) + require.Contains(t, err.Error(), "canceled") +} + +func BenchmarkTaskRunOne(b *testing.B) { + none := func() error { return nil } + for ii := 0; ii < b.N; ii++ { + task.Run(context.Background(), none) + } +} + +func BenchmarkTaskRunTwo(b *testing.B) { + none := func() error { return nil } + for ii := 0; ii < b.N; ii++ { + task.Run(context.Background(), none, none) + } +} diff --git a/blobstore/util/taskpool/taskpool.go b/blobstore/util/taskpool/taskpool.go new file mode 100644 index 000000000..902488ba7 --- /dev/null +++ b/blobstore/util/taskpool/taskpool.go @@ -0,0 +1,58 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +// Package taskpool provides limited pool running task +package taskpool + +// TaskPool limited pool +type TaskPool struct { + pool chan func() +} + +// New returns task pool with workerCount and poolSize +func New(workerCount, poolSize int) TaskPool { + pool := make(chan func(), poolSize) + for i := 0; i < workerCount; i++ { + go func() { + for { + task, ok := <-pool + if !ok { + break + } + task() + } + }() + } + return TaskPool{pool: pool} +} + +// Run add task to pool, block if pool is full +func (tp TaskPool) Run(task func()) { + tp.pool <- task +} + +// TryRun try to add task to pool, return immediately +func (tp TaskPool) TryRun(task func()) bool { + select { + case tp.pool <- task: + return true + default: + return false + } +} + +// Close the pool, the function is concurrent unsafe +func (tp TaskPool) Close() { + close(tp.pool) +} diff --git a/blobstore/util/taskpool/taskpool_test.go b/blobstore/util/taskpool/taskpool_test.go new file mode 100644 index 000000000..c46f1c82d --- /dev/null +++ b/blobstore/util/taskpool/taskpool_test.go @@ -0,0 +1,69 @@ +// Copyright 2022 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package taskpool_test + +import ( + "math" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cubefs/cubefs/blobstore/util/taskpool" +) + +func TestTaskpoolBase(t *testing.T) { + runner := taskpool.New(0, 0) + ok := runner.TryRun(func() { + t.Fatal("can not be here") + }) + require.False(t, ok) + runner.Close() + + runner = taskpool.New(1, 0) + time.Sleep(time.Millisecond * 100) + runner.Run(func() { + time.Sleep(time.Second) + }) + require.False(t, runner.TryRun(func() {})) + runner.Close() +} + +func BenchmarkGoroutine(b *testing.B) { + var wg sync.WaitGroup + for i := 0; i < b.N; i++ { + wg.Add(1) + go func(i int) { + math.Pow(math.Pi, float64(i%10)) + wg.Done() + }(i) + } + wg.Wait() +} + +func BenchmarkTaskPool(b *testing.B) { + var wg sync.WaitGroup + runner := taskpool.New(1024, 0) + for i := 0; i < b.N; i++ { + wg.Add(1) + idx := i + runner.Run(func() { + math.Pow(math.Pi, float64(idx%10)) + wg.Done() + }) + } + wg.Wait() +}