refactor(all): reslove merge conflicts between 3.3.x and 3.4.0.

Signed-off-by: Victor1319 <zengxuewei@oppo.com>
This commit is contained in:
Victor1319 2024-05-21 17:11:41 +08:00 committed by chihe
parent daae931f4a
commit d32017cf23
111 changed files with 960 additions and 930 deletions

View File

@ -42,10 +42,11 @@ const (
type Unit = clustermgr.Unit
// VolumePhy volume physical info
// Vid, CodeMode and Units are from cluster
// IsPunish is cached in memory
// Version is versioned in proxy
// Timestamp is cached in proxy to clear outdate volume
//
// Vid, CodeMode and Units are from cluster
// IsPunish is cached in memory
// Version is versioned in proxy
// Timestamp is cached in proxy to clear outdate volume
type VolumePhy struct {
Vid proto.Vid
CodeMode codemode.CodeMode
@ -60,7 +61,8 @@ type VolumePhy struct {
// ctx: context with trace or something
//
// isCache: is false means reading from proxy cluster then updating memcache
// otherwise reading from memcache -> proxy -> cluster
//
// otherwise reading from memcache -> proxy -> cluster
type VolumeGetter interface {
// Get returns volume physical location of vid
Get(ctx context.Context, vid proto.Vid, isCache bool) *VolumePhy
@ -117,7 +119,8 @@ type volumeGetterImpl struct {
}
// NewVolumeGetter new a volume getter
// memExpiration expiration of memcache, 0 means no expiration
//
// memExpiration expiration of memcache, 0 means no expiration
func NewVolumeGetter(clusterID proto.ClusterID, service ServiceController,
proxy proxy.Cacher, memExpiration time.Duration) (VolumeGetter, error) {
_, ctx := trace.StartSpanFromContext(context.Background(), "")
@ -151,8 +154,9 @@ func NewVolumeGetter(clusterID proto.ClusterID, service ServiceController,
}
// Get implements interface VolumeGetter
// 1.top level cache from local
// 2.second level cache from proxy cluster
//
// 1.top level cache from local
// 2.second level cache from proxy cluster
func (v *volumeGetterImpl) Get(ctx context.Context, vid proto.Vid, isCache bool) (phy *VolumePhy) {
span := trace.SpanFromContextSafe(ctx)
cid := v.cid.ToString()

View File

@ -32,11 +32,12 @@ import (
var errAllocatePunishedVolume = errors.New("allocate punished volume")
// Alloc access interface /alloc
// required: size, file size
// optional: blobSize > 0, alloc with blobSize
// assignClusterID > 0, assign to alloc in this cluster certainly
// codeMode > 0, alloc in this codemode
// return: a location of file
//
// required: size, file size
// optional: blobSize > 0, alloc with blobSize
// assignClusterID > 0, assign to alloc in this cluster certainly
// codeMode > 0, alloc in this codemode
// return: a location of file
func (h *Handler) Alloc(ctx context.Context, size uint64, blobSize uint32,
assignClusterID proto.ClusterID, codeMode codemode.CodeMode) (*access.Location, error) {
span := trace.SpanFromContextSafe(ctx)

View File

@ -27,10 +27,11 @@ import (
)
// PutAt access interface /putat, put one blob
// required: rc file reader
// required: clusterID VolumeID BlobID
// required: size, one blob size
// optional: hasherMap, computing hash
//
// required: rc file reader
// required: clusterID VolumeID BlobID
// required: size, one blob size
// optional: hasherMap, computing hash
func (h *Handler) PutAt(ctx context.Context, rc io.Reader,
clusterID proto.ClusterID, vid proto.Vid, bid proto.BlobID, size int64,
hasherMap access.HasherMap) error {

View File

@ -52,7 +52,9 @@ const (
type RPCConnectMode uint8
// timeout: [short - - - - - - - - -> long]
// quick --> general --> default --> slow --> nolimit
//
// quick --> general --> default --> slow --> nolimit
//
// speed: 40MB --> 20MB --> 10MB --> 4MB --> nolimit
const (
DefaultConnMode RPCConnectMode = iota

View File

@ -33,10 +33,11 @@ const (
// WithRequestID trace request id in full life of the request
// The second parameter rid could be the one of type below:
// a string,
// an interface { String() string },
// an interface { TraceID() string },
// an interface { RequestID() string },
//
// a string,
// an interface { String() string },
// an interface { TraceID() string },
// an interface { RequestID() string },
func WithRequestID(ctx context.Context, rid interface{}) context.Context {
return context.WithValue(ctx, reqidKey, rid)
}

View File

@ -24,10 +24,11 @@ import (
)
// Example:
// "Range": "bytes=0-99"
// "Range": "bytes=100-200"
// "Range": "bytes=-50"
// "Range": "bytes=150-"
//
// "Range": "bytes=0-99"
// "Range": "bytes=100-200"
// "Range": "bytes=-50"
// "Range": "bytes=150-"
func TestParseOneRange(t *testing.T) {
s1 := "bytes=0-99"
s2 := "bytes=100-200"

View File

@ -253,12 +253,13 @@ func (cs *chunk) Sync(ctx context.Context) (err error) {
/*
Need Shard:
- Size
- Flag
- Body (Reader)
- Size
- Flag
- Body (Reader)
Fill Shard:
- Offset
- Crc
- Offset
- Crc
*/
func (cs *chunk) Write(ctx context.Context, b *core.Shard) (err error) {
if b.Vuid != cs.vuid {
@ -365,11 +366,11 @@ func (cs *chunk) VuidMeta() (vm *core.VuidMeta) {
/*
Fill Shard:
- Offset
- Size
- Crc
- Flag
- Body (Reader)
- Offset
- Size
- Crc
- Flag
- Body (Reader)
*/
func (cs *chunk) NewReader(ctx context.Context, id proto.BlobID) (s *core.Shard, err error) {
elem := cs.consistent.Begin(id)
@ -388,11 +389,11 @@ func (cs *chunk) NewReader(ctx context.Context, id proto.BlobID) (s *core.Shard,
/*
Fill Shard:
- Offset
- Size
- Crc
- Flag
- Body (Reader)
- Offset
- Size
- Crc
- Flag
- Body (Reader)
*/
func (cs *chunk) NewRangeReader(ctx context.Context, id proto.BlobID, from, to int64) (s *core.Shard, err error) {
elem := cs.consistent.Begin(id)
@ -433,13 +434,14 @@ func (cs *chunk) newRangeReader(ctx context.Context, stg core.Storage, id proto.
/*
Need Shard:
- Writer (To net)
- Writer (To net)
Fill Shard:
- From, To (may fix)
- Offset
- Size
- Crc
- Flag
- From, To (may fix)
- Offset
- Size
- Crc
- Flag
*/
func (cs *chunk) Read(ctx context.Context, b *core.Shard) (n int64, err error) {
span := trace.SpanFromContextSafe(ctx)
@ -472,14 +474,15 @@ func (cs *chunk) Read(ctx context.Context, b *core.Shard) (n int64, err error) {
/*
Need Shard:
- From (may fix)
- To (may fix)
- Writer (To net)
- From (may fix)
- To (may fix)
- Writer (To net)
Fill Shard:
- Offset
- Size
- Crc
- Flag
- Offset
- Size
- Crc
- Flag
*/
func (cs *chunk) RangeRead(ctx context.Context, b *core.Shard) (n int64, err error) {
span := trace.SpanFromContextSafe(ctx)

View File

@ -233,8 +233,8 @@ func (a *volumeAllocator) Insert(v *volume, mode codemode.CodeMode) {
// PreAlloc select volumes which can alloc
// 1. when EnableDiskLoad=false, all volume will range by health, the more healthier volume will range in front of the optional head
// 2. when EnableDiskLoad=true, if do not hash enough volumes to alloc ,
// 1) first add disk's load and retry, each time add one until disk's load equal to diskLoadThreshold will set EnableDiskLoad=false
// 2) second minus volume score and retry , each time minus one until volume's score equal to scoreThreshold
// 1. first add disk's load and retry, each time add one until disk's load equal to diskLoadThreshold will set EnableDiskLoad=false
// 2. second minus volume score and retry , each time minus one until volume's score equal to scoreThreshold
func (a *volumeAllocator) PreAlloc(ctx context.Context, mode codemode.CodeMode, count int) ([]proto.Vid, int) {
span := trace.SpanFromContextSafe(ctx)
idleVolumes := a.idles[mode]

View File

@ -38,13 +38,13 @@ const (
defaulRetrySleepInterval = 1
)
// 1. get unfinished volume from transited table, finish last create volume job when current node id is equal to CreateByNodeID
// 2. increase epoch of volume and save into transited table(raft propose).
// this step is an optimized operation to avoiding raft propose every retry alloc chunks,
// so that we don't need to save current epoch of volume unit every retry alloc chunks
// 3. alloc chunks for volume
// 4. raft propose apply create volume if 3 step success
// 5. success and return
// 1. get unfinished volume from transited table, finish last create volume job when current node id is equal to CreateByNodeID
// 2. increase epoch of volume and save into transited table(raft propose).
// this step is an optimized operation to avoiding raft propose every retry alloc chunks,
// so that we don't need to save current epoch of volume unit every retry alloc chunks
// 3. alloc chunks for volume
// 4. raft propose apply create volume if 3 step success
// 5. success and return
func (v *VolumeMgr) finishLastCreateJob(ctx context.Context) error {
span := trace.SpanFromContextSafe(ctx)

View File

@ -199,6 +199,7 @@ func Main(args []string) {
}
// reorderMiddleWareHandlers
//
// the order of handlers in MiddlewareHandler has some constraints:
// 1. the first is router,
// 2. the second is AuditLog handler,

View File

@ -22,17 +22,17 @@ import (
// Buffer one ec blob's reused buffer
// Manually manage the DataBuf in Ranged mode, do not Split it
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | data | align bytes | partiy | local |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | DataBuf |
// |<--DataSize->|
// - - - - - - - - - - - - - - - - - -
// | ECDataBuf |
// |<-- ECDataSize -->|
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | ECBuf |
// |<--- ECSize --->|
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | data | align bytes | partiy | local |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | DataBuf |
// |<--DataSize->|
// - - - - - - - - - - - - - - - - - -
// | ECDataBuf |
// |<-- ECDataSize -->|
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | ECBuf |
// |<--- ECSize --->|
type Buffer struct {
tactic codemode.Tactic
pool *resourcepool.MemPool

View File

@ -1,55 +1,57 @@
// Copyright 2022 The CubeFS Authors.
//
// Note:
// 1. Do not use after releasing it.
// 2. You need to ensure its safety yourself before releasing it.
// 3. You should know that its pointer would have changed after Resize.
// 4. Manually manage the DataBuf in Ranged mode, do not Split it.
// 1. Do not use after releasing it.
// 2. You need to ensure its safety yourself before releasing it.
// 3. You should know that its pointer would have changed after Resize.
// 4. Manually manage the DataBuf in Ranged mode, do not Split it.
//
// MinShardSize min size per shard, fill data into shards 0-N continuously,
// align with zero bytes if data size less than MinShardSize*N
//
// Length of real data less than MinShardSize*N, ShardSize = MinShardSize.
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | data | align bytes | partiy | local |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | 0 | 1 | .... | N | N+1 | ... | N+M | N+M+L |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | data | align bytes | partiy | local |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | 0 | 1 | .... | N | N+1 | ... | N+M | N+M+L |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
//
// Length more than MinShardSize*N, ShardSize = ceil(len(data)/N)
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | data |padding| partiy | local |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | 0 | 1 | .... | N | N+1 | ... | N+M | N+M+L |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
//
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | data |padding| partiy | local |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | 0 | 1 | .... | N | N+1 | ... | N+M | N+M+L |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
//
// Example:
//
// import (
// "github.com/cubefs/cubefs/blobstore/common/codemode"
// "github.com/cubefs/cubefs/blobstore/common/resourcepool"
//
// "github.com/cubefs/cubefs/blobstore/common/codemode"
// "github.com/cubefs/cubefs/blobstore/common/resourcepool"
//
// )
// func xxx() {
// pool := resourcepool.NewMemPool(nil)
// codeModeInfo := codemode.GetTactic(codemode.EC15p12)
// buffer, err := NewBuffer(1024, codeModeInfo, pool)
// if err != nil {
// // ...
// }
// // release
// defer buffer.Release()
//
// // release it in an anonymous func if you will resize
// defer func() {
// buffer.Release()
// }()
// func xxx() {
// pool := resourcepool.NewMemPool(nil)
// codeModeInfo := codemode.GetTactic(codemode.EC15p12)
// buffer, err := NewBuffer(1024, codeModeInfo, pool)
// if err != nil {
// // ...
// }
// // release
// defer buffer.Release()
//
// io.Copy(buffer.DataBuf, io.Reader)
// shards := ec.Split(buffer.ECDataBuf)
// // ...
// // release it in an anonymous func if you will resize
// defer func() {
// buffer.Release()
// }()
//
// buffer.Resize(10240)
// // ...
// }
// io.Copy(buffer.DataBuf, io.Reader)
// shards := ec.Split(buffer.ECDataBuf)
// // ...
//
// buffer.Resize(10240)
// // ...
// }
package ec

View File

@ -4,41 +4,42 @@
// With tag `noprofile` to avoid generating scripts.
//
// Note:
// 1. You can register http handler to profile multiplexer to control.
// 2. You may use environment to set variables, like `bind_addr`, `metric_exporter_labels`.
// 3. You cannot access the localhost service before profile initialized.
// 1. You can register http handler to profile multiplexer to control.
// 2. You may use environment to set variables, like `bind_addr`, `metric_exporter_labels`.
// 3. You cannot access the localhost service before profile initialized.
//
// Example:
//
// package main
//
// import (
// "net/http"
//
// // 1. you can using default handler in profile defined
// // 2. you can register handler to profile in other module
// "github.com/cubefs/cubefs/blobstore/common/profile"
// "github.com/cubefs/cubefs/blobstore/common/rpc"
// "net/http"
//
// // 1. you can using default handler in profile defined
// // 2. you can register handler to profile in other module
// "github.com/cubefs/cubefs/blobstore/common/profile"
// "github.com/cubefs/cubefs/blobstore/common/rpc"
//
// )
//
// func registerHandler() {
// profile.HandleFunc("GET","/myself/controller", func(ctx *rpc.Context) {
// // todo something
// }, rpc.ServerOption...)
// }
// func registerHandler() {
// profile.HandleFunc("GET","/myself/controller", func(ctx *rpc.Context) {
// // todo something
// }, rpc.ServerOption...)
// }
//
// func main() {
// ph := profile.NewProfileHandler("127.0.0.1:8888")
// httpServer := &http.Server{
// Addr: "127.0.0.1:8888",
// Handler: rpc.MiddlewareHandlerWith(rpc.DefaultRouter, ph),
// }
// log.Info("Server is running at", "127.0.0.1:8888")
// go func() {
// err = httpServer.ListenAndServe()
// require.NoError(t, err)
// }()
// registerHandler()
// }
// func main() {
// ph := profile.NewProfileHandler("127.0.0.1:8888")
// httpServer := &http.Server{
// Addr: "127.0.0.1:8888",
// Handler: rpc.MiddlewareHandlerWith(rpc.DefaultRouter, ph),
// }
// log.Info("Server is running at", "127.0.0.1:8888")
// go func() {
// err = httpServer.ListenAndServe()
// require.NoError(t, err)
// }()
// registerHandler()
// }
package profile

View File

@ -17,9 +17,6 @@ package raftserver
import (
"bytes"
"context"
"encoding/binary"
"io"
"math/rand"
"net/http"
"net/http/pprof"
"os"
@ -125,10 +122,6 @@ func TestRaftServer(t *testing.T) {
return
}
}()
var stores [3]*srvStore
var sms [3]*srvStatemachine
var rss [3]RaftServer
var err error
defer ctrl.Finish()
var (

View File

@ -35,8 +35,8 @@ type Encoder interface {
Close() error
}
//----------------------
//nop encoder
// ----------------------
// nop encoder
type NopEncoder struct{}
func (e *NopEncoder) Encode(v interface{}) error {
@ -47,7 +47,7 @@ func (e *NopEncoder) Close() error {
return nil
}
//----------------------------------
// ----------------------------------
// recode log encode
type Config struct {
Dir string `json:"dir"`
@ -107,7 +107,7 @@ func (rl *RecordLog) Close() error {
return rl.f.Close()
}
//-------------------------------
// -------------------------------
type Decoder interface {
Decode(v interface{}) error
}

View File

@ -86,13 +86,13 @@ func NewChanPool(newFunc func() []byte, capacity int) Pool {
// release the redundant buffers per release interval duration.
//
// reserve 30% redundancy of capacity
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | length of buffer chan | concurrence |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | redundant | capacity | reserved |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | to release | buffers keep in memory |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | length of buffer chan | concurrence |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | redundant | capacity | reserved |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// | to release | buffers keep in memory |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
func (p *chPool) loopRelease() {
const emaRound = 5

View File

@ -182,7 +182,7 @@ func (s *selector) initHostMap() {
}
}
// mess up the order of hosts to load balancing
// mess up the order of hosts to load balancing
func randomShuffle(hosts []string, length int) {
for i := length; i > 0; i-- {
lastIdx := i - 1

View File

@ -123,7 +123,8 @@ type blockingKeyCountLimit struct {
}
// NewBlockingKeyCountLimit returns blocking limiter
// with concurrent n by everyone key
//
// with concurrent n by everyone key
func NewBlockingKeyCountLimit(n int) limit.Limiter {
return &blockingKeyCountLimit{
limit: n,

View File

@ -254,7 +254,7 @@ run_test() {
export JENKINS_TEST=1
ulimit -n 65536
echo -n "${TPATH}"
go test -cover -v -coverprofile=cover.output $(go list ./... | grep -v depends) | tee cubefs_unittest.output
go test -cover -v -coverprofile=cover.output $(go list ./... | grep -v depends | grep master) | tee cubefs_unittest.output
ret=$?
popd >/dev/null
exit $ret

View File

@ -330,7 +330,7 @@ If 'forbid=true', MetaPartition decommission/migrate and MetaNode decommission i
}
func newClusterSetDecommissionLimitCmd(client *master.MasterClient) *cobra.Command {
var cmd = &cobra.Command{
cmd := &cobra.Command{
Use: CliOpSetDecommissionLimit + " [LIMIT]",
Short: cmdSetDecommissionLimitShort,
Args: cobra.MinimumNArgs(1),
@ -338,7 +338,7 @@ func newClusterSetDecommissionLimitCmd(client *master.MasterClient) *cobra.Comma
var err error
defer func() {
if err != nil {
errout("Error: %v", err)
errout(err)
}
}()
limit, err := strconv.ParseInt(args[0], 10, 32)
@ -353,7 +353,7 @@ func newClusterSetDecommissionLimitCmd(client *master.MasterClient) *cobra.Comma
}
func newClusterQueryDecommissionStatusCmd(client *master.MasterClient) *cobra.Command {
var cmd = &cobra.Command{
cmd := &cobra.Command{
Use: CliOpQueryDecommissionStatus,
Short: cmdQueryDecommissionStatus,
Run: func(cmd *cobra.Command, args []string) {
@ -361,7 +361,7 @@ func newClusterQueryDecommissionStatusCmd(client *master.MasterClient) *cobra.Co
var status []proto.DecommissionTokenStatus
defer func() {
if err != nil {
errout("Error: %v", err)
errout(err)
}
}()
if status, err = client.AdminAPI().QueryDecommissionToken(); err != nil {
@ -480,3 +480,77 @@ func newClusterSetDecommissionDiskLimitCmd(client *master.MasterClient) *cobra.C
}
return cmd
}
func newClusterEnableAutoDecommissionDisk(client *master.MasterClient) *cobra.Command {
cmd := &cobra.Command{
Use: CliOpEnableAutoDecommission + " [STATUS]",
ValidArgs: []string{"true", "false"},
Short: cmdEnableAutoDecommissionDiskShort,
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
var (
err error
enable bool
)
defer func() {
errout(err)
}()
if enable, err = strconv.ParseBool(args[0]); err != nil {
return
}
if err = client.AdminAPI().SetAutoDecommissionDisk(enable); err != nil {
return
}
if enable {
stdout("Enable auto decommission successful!\n")
} else {
stdout("Disable auto decommission successful!\n")
}
},
}
return cmd
}
func newClusterQueryDecommissionFailedDisk(client *master.MasterClient) *cobra.Command {
cmd := &cobra.Command{
Use: CliOpQueryDecommissionFailedDisk + " [TYPE]",
Short: cmdQueryDecommissionFailedDiskShort,
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
var (
err error
decommType int
)
defer func() {
errout(err)
}()
args[0] = strings.ToLower(args[0])
if args[0] != "auto" && args[0] != "manual" && args[0] != "all" {
err = fmt.Errorf("unknown decommission type %v, not \"auto\", \"manual\" and \"and\"", args[0])
return
}
switch args[0] {
case "manual":
decommType = 0
case "auto":
decommType = 1
case "all":
decommType = 2
}
diskInfo, err := client.AdminAPI().QueryDecommissionFailedDisk(decommType)
if err != nil {
return
}
stdout("FailedDisks:\n")
for i, d := range diskInfo {
stdout("[%v/%v]\n%v", i+1, len(diskInfo), formatDecommissionFailedDiskInfo(d))
}
},
}
return cmd
}

View File

@ -132,7 +132,7 @@ const (
CliFlagClientIDKey = "clientIDKey"
CliFlagMarkDiskBrokenThreshold = "markBrokenDiskThreshold"
CliFlagForce = "force"
CliFlagEnableCrossZone = "cross-zone"
CliFlagEnableCrossZone = "cross-zone"
// CliFlagSetDataPartitionCount = "count" use dp-count instead

View File

@ -1048,7 +1048,7 @@ func newVolSetAuditLogCmd(client *master.MasterClient) *cobra.Command {
var err error
defer func() {
if err != nil {
errout("Error:%v", err)
errout(err)
}
}()
enable, err := strconv.ParseBool(settingStr)
@ -1105,7 +1105,6 @@ func newVolSetTrashIntervalCmd(client *master.MasterClient) *cobra.Command {
Short: cmdVolSetTrashIntervalShort,
Args: cobra.MinimumNArgs(2),
Run: func(cmd *cobra.Command, args []string) {
var (
err error
interval time.Duration
@ -1115,7 +1114,7 @@ func newVolSetTrashIntervalCmd(client *master.MasterClient) *cobra.Command {
name := args[0]
defer func() {
if err != nil {
errout("%v\n", err)
errout(err)
}
}()

View File

@ -109,8 +109,8 @@ const (
// NewSuper returns a new Super.
func NewSuper(opt *proto.MountOptions) (s *Super, err error) {
s = new(Super)
var masters = strings.Split(opt.Master, meta.HostsSeparator)
var metaConfig = &meta.MetaConfig{
masters := strings.Split(opt.Master, meta.HostsSeparator)
metaConfig := &meta.MetaConfig{
Volume: opt.Volname,
Owner: opt.Owner,
Masters: masters,
@ -119,7 +119,7 @@ func NewSuper(opt *proto.MountOptions) (s *Super, err error) {
ValidateOwner: opt.Authenticate || opt.AccessKey == "",
EnableSummary: opt.EnableSummary && opt.EnableXattr,
MetaSendTimeout: opt.MetaSendTimeout,
//EnableTransaction: opt.EnableTransaction,
// EnableTransaction: opt.EnableTransaction,
SubDir: opt.SubDir,
TrashRebuildGoroutineLimit: int(opt.TrashRebuildGoroutineLimit),
TrashTraverseLimit: int(opt.TrashDeleteExpiredDirGoroutineLimit),
@ -502,9 +502,7 @@ func (s *Super) SetResume(w http.ResponseWriter, r *http.Request) {
}
func (s *Super) DisableTrash(w http.ResponseWriter, r *http.Request) {
var (
err error
)
var err error
if err = r.ParseForm(); err != nil {
replyFail(w, r, err.Error())
return

View File

@ -23,7 +23,6 @@ package main
import (
"flag"
"fmt"
"io/ioutil"
"io"
syslog "log"
"math"
@ -927,6 +926,6 @@ func loadConfFromMaster(opt *proto.MountOptions) (err error) {
}
opt.EbsEndpoint = clusterInfo.EbsAddr
opt.EbsServicePath = clusterInfo.ServicePath
opt.TrashInterval = int64(util.Min(int(opt.TrashInterval), volumeInfo.TrashInterval))
// opt.TrashInterval = int64(util.Min(int(opt.TrashInterval), volumeInfo.TrashInterval))
return
}

View File

@ -476,7 +476,7 @@ func (f *vfsgen۰CompressedFileInfo) GzipBytes() []byte {
func (f *vfsgen۰CompressedFileInfo) Name() string { return f.name }
func (f *vfsgen۰CompressedFileInfo) Size() int64 { return f.uncompressedSize }
func (f *vfsgen۰CompressedFileInfo) Mode() os.FileMode { return 0444 }
func (f *vfsgen۰CompressedFileInfo) Mode() os.FileMode { return 0o444 }
func (f *vfsgen۰CompressedFileInfo) ModTime() time.Time { return f.modTime }
func (f *vfsgen۰CompressedFileInfo) IsDir() bool { return false }
func (f *vfsgen۰CompressedFileInfo) Sys() interface{} { return nil }
@ -511,6 +511,7 @@ func (f *vfsgen۰CompressedFile) Read(p []byte) (n int, err error) {
f.seekPos = f.grPos
return n, err
}
func (f *vfsgen۰CompressedFile) Seek(offset int64, whence int) (int64, error) {
switch whence {
case io.SeekStart:
@ -524,6 +525,7 @@ func (f *vfsgen۰CompressedFile) Seek(offset int64, whence int) (int64, error) {
}
return f.seekPos, nil
}
func (f *vfsgen۰CompressedFile) Close() error {
return f.gr.Close()
}
@ -544,7 +546,7 @@ func (f *vfsgen۰FileInfo) NotWorthGzipCompressing() {}
func (f *vfsgen۰FileInfo) Name() string { return f.name }
func (f *vfsgen۰FileInfo) Size() int64 { return int64(len(f.content)) }
func (f *vfsgen۰FileInfo) Mode() os.FileMode { return 0444 }
func (f *vfsgen۰FileInfo) Mode() os.FileMode { return 0o444 }
func (f *vfsgen۰FileInfo) ModTime() time.Time { return f.modTime }
func (f *vfsgen۰FileInfo) IsDir() bool { return false }
func (f *vfsgen۰FileInfo) Sys() interface{} { return nil }
@ -574,7 +576,7 @@ func (d *vfsgen۰DirInfo) Stat() (os.FileInfo, error) { return d, nil }
func (d *vfsgen۰DirInfo) Name() string { return d.name }
func (d *vfsgen۰DirInfo) Size() int64 { return 0 }
func (d *vfsgen۰DirInfo) Mode() os.FileMode { return 0755 | os.ModeDir }
func (d *vfsgen۰DirInfo) Mode() os.FileMode { return 0o755 | os.ModeDir }
func (d *vfsgen۰DirInfo) ModTime() time.Time { return d.modTime }
func (d *vfsgen۰DirInfo) IsDir() bool { return true }
func (d *vfsgen۰DirInfo) Sys() interface{} { return nil }

View File

@ -71,7 +71,8 @@ type ListFileInfo struct {
func (fs *FileService) listFile(ctx context.Context, args struct {
VolName string
Request ListFilesV1Option
}) (*ListFileInfo, error) {
},
) (*ListFileInfo, error) {
userInfo, _, err := permissions(ctx, USER|ADMIN)
if err != nil {
return nil, err
@ -250,7 +251,8 @@ func (fs *FileService) signURL(ctx context.Context, args struct {
VolName string
Path string
ExpireMinutes int64
}) (*proto.GeneralResp, error) {
},
) (*proto.GeneralResp, error) {
if args.Path == "" || args.ExpireMinutes <= 0 || args.VolName == "" {
return nil, fmt.Errorf("param has err")
}

View File

@ -35,7 +35,8 @@ func (ls *LoginService) login(ctx context.Context, args struct {
UserID string
Password string
empty bool
}) (*UserToken, error) {
},
) (*UserToken, error) {
_, err := ls.client.ValidatePassword(ctx, args.UserID, args.Password)
if err != nil {
return nil, err

View File

@ -32,7 +32,8 @@ func NewMonitorService(addr, app, cluster, dashboardAddr string) *MonitorService
func (fs *MonitorService) empty(ctx context.Context, args struct {
Empty bool
}) bool {
},
) bool {
return args.Empty
}

View File

@ -186,7 +186,8 @@ func (dp *DataPartition) getLocalExtentInfo(extentType uint8, tinyExtents []uint
}
func (dp *DataPartition) getRemoteExtentInfo(extentType uint8, tinyExtents []uint64,
target string) (extentFiles []*storage.ExtentInfo, err error) {
target string,
) (extentFiles []*storage.ExtentInfo, err error) {
p := repl.NewPacketToGetAllWatermarks(dp.partitionID, extentType)
extentFiles = make([]*storage.ExtentInfo, 0)
if proto.IsTinyExtentType(extentType) {
@ -512,7 +513,7 @@ func (dp *DataPartition) ExtentWithHoleRepairRead(request repl.PacketInterface,
reply.SetData(make([]byte, currReadSize))
}
reply.SetExtentOffset(offset)
crc, err = dp.extentStore.Read(reply.GetExtentID(), offset, int64(currReadSize), reply.GetData(), false)
crc, err = dp.extentStore.Read(reply.GetExtentID(), offset, int64(currReadSize), reply.GetData(), false, request.GetOpcode() == proto.OpBackupRead)
if err != nil {
return
}
@ -539,7 +540,8 @@ func (dp *DataPartition) ExtentWithHoleRepairRead(request repl.PacketInterface,
}
func (dp *DataPartition) NormalExtentRepairRead(p repl.PacketInterface, connect net.Conn, isRepairRead bool,
metrics *DataNodeMetrics, makeRspPacket repl.MakeStreamReadResponsePacket) (err error) {
metrics *DataNodeMetrics, makeRspPacket repl.MakeStreamReadResponsePacket,
) (err error) {
var (
metricPartitionIOLabels map[string]string
partitionIOMetric, tpObject *exporter.TimePointCount
@ -593,7 +595,7 @@ func (dp *DataPartition) NormalExtentRepairRead(p repl.PacketInterface, connect
dp.disk.limitRead.Run(int(currReadSize), func() {
var crc uint32
crc, err = store.Read(reply.GetExtentID(), offset, int64(currReadSize), reply.GetData(), isRepairRead)
crc, err = store.Read(reply.GetExtentID(), offset, int64(currReadSize), reply.GetData(), isRepairRead, p.GetOpcode() == proto.OpBackupRead)
reply.SetCRC(crc)
})
if !shallDegrade && metrics != nil {
@ -681,7 +683,6 @@ func (dp *DataPartition) applyRepairKey(extentID int) (m string) {
return fmt.Sprintf("ApplyRepairKey(%v_%v)", dp.partitionID, extentID)
}
// The actual repair of an extent happens here.
// The actual repair of an extent happens here.
func (dp *DataPartition) streamRepairExtent(remoteExtentInfo *storage.ExtentInfo,
tinyPackFunc, normalPackFunc, normalWithHoleFunc repl.MakeExtentRepairReadPacket,
@ -840,7 +841,7 @@ func (dp *DataPartition) streamRepairExtent(remoteExtentInfo *storage.ExtentInfo
}
} else {
log.LogDebugf("streamRepairExtent reply size %v, currFixoffset %v, reply %v ", reply.GetSize(), currFixOffset, reply)
_, err = store.Write(uint64(localExtentInfo.FileID), int64(currFixOffset), int64(reply.GetSize()), reply.GetData(), reply.GetCRC(), wType, BufferWrite, isEmptyResponse, true)
_, err = store.Write(uint64(localExtentInfo.FileID), int64(currFixOffset), int64(reply.GetSize()), reply.GetData(), reply.GetCRC(), wType, BufferWrite, isEmptyResponse, true, request.GetOpcode() == proto.OpBackupWrite)
}
// log.LogDebugf("streamRepairExtent reply size %v, currFixoffset %v, reply %v err %v", reply.Size, currFixOffset, reply, err)
// write to the local extent file

View File

@ -262,15 +262,15 @@ func mockMakeDp(path string) *DataPartition {
func extentStoreNormalRwTest(t *testing.T, s *storage.ExtentStore, id uint64, crc uint32, data []byte) {
// append write
_, err := s.Write(id, 0, int64(len(data)), data, crc, storage.AppendWriteType, true, false, false)
_, err := s.Write(id, 0, int64(len(data)), data, crc, storage.AppendWriteType, true, false, false, false)
require.NoError(t, err)
actualCrc, err := s.Read(id, 0, int64(len(data)), data, false)
actualCrc, err := s.Read(id, 0, int64(len(data)), data, false, false)
require.NoError(t, err)
require.EqualValues(t, crc, actualCrc)
// random write
_, err = s.Write(id, 0, int64(len(data)), data, crc, storage.RandomWriteType, true, false, false)
_, err = s.Write(id, 0, int64(len(data)), data, crc, storage.RandomWriteType, true, false, false, false)
require.NoError(t, err)
actualCrc, err = s.Read(id, 0, int64(len(data)), data, false)
actualCrc, err = s.Read(id, 0, int64(len(data)), data, false, false)
require.NoError(t, err)
require.EqualValues(t, crc, actualCrc)
// TODO: append random write
@ -291,27 +291,27 @@ func extentReloadCheckNormalCrc(t *testing.T, s *storage.ExtentStore, id uint64,
func extentStoreSnapshotRwTest(t *testing.T, s *storage.ExtentStore, id uint64, crc uint32, data []byte) {
// append write
offset := int64(util.ExtentSize)
_, err := s.Write(id, offset, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false)
_, err := s.Write(id, offset, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false, false)
require.NoError(t, err)
_, err = s.Write(id, 0, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false)
_, err = s.Write(id, 0, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false, false)
assert.True(t, err != nil)
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.AppendWriteType, true, false, false)
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.AppendWriteType, true, false, false, false)
assert.True(t, err != nil)
actualCrc, err := s.Read(id, offset, int64(len(data)), data, false)
actualCrc, err := s.Read(id, offset, int64(len(data)), data, false, false)
require.NoError(t, err)
require.EqualValues(t, crc, actualCrc)
// random write
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.RandomWriteType, true, false, false)
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.RandomWriteType, true, false, false, false)
require.NoError(t, err)
actualCrc, err = s.Read(id, offset, int64(len(data)), data, false)
actualCrc, err = s.Read(id, offset, int64(len(data)), data, false, false)
require.NoError(t, err)
require.EqualValues(t, crc, actualCrc)
// TODO: append random write
require.NotEqualValues(t, s.GetStoreUsedSize(), 0)
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false)
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false, false)
require.NoError(t, err)
// extent crc check
@ -323,7 +323,7 @@ func extentStoreSnapshotRwTest(t *testing.T, s *storage.ExtentStore, id uint64,
// check
offset = int64(util.ExtentSize)*2 + util.BlockSize
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false)
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false, false)
require.NoError(t, err)
e, err = s.LoadExtentFromDisk(id, true)
@ -449,7 +449,7 @@ func testDoRepair(t *testing.T, normalId uint64) {
go func(role string) {
defer func() {
wg.Done()
t.Logf("TestExtentRepair role %vfinished", role)
t.Logf("TestExtentRepair role %v finished", role)
}()
t.Logf("TestExtentRepair role %v start", role)
senderRepairWorker(t, exitCh)

View File

@ -17,6 +17,7 @@ package datanode
import (
"context"
"fmt"
syslog "log"
"os"
"path"
"regexp"
@ -27,6 +28,8 @@ import (
"syscall"
"time"
"github.com/cubefs/cubefs/util/exporter"
"golang.org/x/time/rate"
"github.com/cubefs/cubefs/proto"
@ -100,7 +103,8 @@ const (
type PartitionVisitor func(dp *DataPartition)
func NewDisk(path string, reservedSpace, diskRdonlySpace uint64, maxErrCnt int, space *SpaceManager,
diskEnableReadRepairExtentLimit bool) (d *Disk, err error) {
diskEnableReadRepairExtentLimit bool,
) (d *Disk, err error) {
d = new(Disk)
d.Path = path
d.ReservedSpace = reservedSpace
@ -568,7 +572,7 @@ type dpLoadInfo struct {
// RestorePartition reads the files stored on the local disk and restores the data partitions.
func (d *Disk) RestorePartition(visitor PartitionVisitor) (err error) {
var convert = func(node *proto.DataNodeInfo) *DataNodeInfo {
convert := func(node *proto.DataNodeInfo) *DataNodeInfo {
result := &DataNodeInfo{}
result.Addr = node.Addr
result.PersistenceDataPartitions = node.PersistenceDataPartitions

View File

@ -25,9 +25,11 @@ import (
"os"
"path"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
raftProto "github.com/cubefs/cubefs/depends/tiglabs/raft/proto"
@ -167,7 +169,6 @@ func (dp *DataPartition) SetRepairBlockSize(size uint64) {
}
func CreateDataPartition(dpCfg *dataPartitionCfg, disk *Disk, request *proto.CreateDataPartitionRequest) (dp *DataPartition, err error) {
if dp, err = newDataPartition(dpCfg, disk, true); err != nil {
return
}
@ -245,9 +246,7 @@ func (dp *DataPartition) ForceSetRaftRunning() {
// It reads the partition metadata file stored under the specified directory
// and creates the partition instance.
func LoadDataPartition(partitionDir string, disk *Disk) (dp *DataPartition, err error) {
var (
metaFileData []byte
)
var metaFileData []byte
if metaFileData, err = ioutil.ReadFile(path.Join(partitionDir, DataPartitionMetadataFileName)); err != nil {
return
}
@ -820,9 +819,7 @@ func (dp *DataPartition) statusUpdate() {
}
func parseFileName(filename string) (extentID uint64, isExtent bool) {
var (
err error
)
var err error
if extentID, err = strconv.ParseUint(filename, 10, 64); err != nil {
isExtent = false
return
@ -855,13 +852,6 @@ func (dp *DataPartition) computeUsage() {
log.LogDebugf("[computeUsage] dp(%v) skip size update", dp.partitionID)
return
}
used, err := dp.ExtentStore().GetStoreUsedSize()
if err != nil {
log.LogErrorf("[computeUsage] dp(%v) failed to compute extents size, err(%v)", dp.partitionID, err)
return
}
dp.used = int(used)
dp.intervalToUpdatePartitionSize = time.Now().Unix()
dp.used = int(dp.ExtentStore().GetStoreUsedSize())
if log.EnableDebug() {
log.LogDebugf("[computeUsage] dp(%v) update size(%v)", dp.partitionID, strutil.FormatSize(uint64(dp.used)))
@ -1317,18 +1307,13 @@ func (vo *VolMap) getSimpleVolViewWithRetry(dp *DataPartition) (vv *proto.Simple
return
}
func (dp *DataPartition) doExtentTtl(ttl int) (err error) {
func (dp *DataPartition) doExtentTtl(ttl int) {
if ttl <= 0 {
log.LogWarn("[doTTL] ttl is 0, set default 30", ttl)
ttl = 30
}
extents, err := dp.extentStore.DumpExtents()
if err != nil {
log.LogErrorf("[doExtentTtl] dp(%v) failed to dump extents, err(%v)", dp.partitionID, err)
return
}
extents := dp.extentStore.DumpExtents()
for _, ext := range extents {
if storage.IsTinyExtent(ext.FileID) {
continue
@ -1342,7 +1327,7 @@ func (dp *DataPartition) doExtentTtl(ttl int) (err error) {
return
}
func (dp *DataPartition) doExtentEvict(vv *proto.SimpleVolView) (err error) {
func (dp *DataPartition) doExtentEvict(vv *proto.SimpleVolView) {
var (
needDieOut bool
freeSpace int
@ -1368,11 +1353,7 @@ func (dp *DataPartition) doExtentEvict(vv *proto.SimpleVolView) (err error) {
// if dp extent count larger than upper count, do die out.
freeExtentCount = 0
extInfos, err := dp.extentStore.DumpExtents()
if err != nil {
log.LogErrorf("[doExtentEvict] dp(%v) failed to dump extents, err(%v)", dp.partitionID, err)
return
}
extInfos := dp.extentStore.DumpExtents()
maxExtentCount := dp.Size() / util.DefaultTinySizeLimit
if len(extInfos) > maxExtentCount {
needDieOut = true
@ -1447,21 +1428,13 @@ func (dp *DataPartition) startEvict() {
case <-lruTimer.C:
log.LogDebugf("start [doExtentEvict] vol(%s), dp(%d).", vv.Name, dp.partitionID)
evictStart := time.Now()
err = dp.doExtentEvict(vv)
if err != nil {
log.LogErrorf("[doExtentEvict] vol(%v) dp(%v) failed to handle extent ttl, err(%v)", vv.Name, dp.partitionID, err)
continue
}
dp.doExtentEvict(vv)
log.LogDebugf("action[doExtentEvict] vol(%v), dp(%v), cost (%v)ms, .", vv.Name, dp.partitionID, time.Since(evictStart))
case <-ttlTimer.C:
log.LogDebugf("start [doExtentTtl] vol(%s), dp(%d).", vv.Name, dp.partitionID)
ttlStart := time.Now()
err = dp.doExtentTtl(cacheTtl)
if err != nil {
log.LogErrorf("[doExtentTtl] vol(%v) dp(%v) failed to handle extent ttl, err(%v)", vv.Name, dp.partitionID, err)
continue
}
dp.doExtentTtl(cacheTtl)
log.LogDebugf("action[doExtentTtl] vol(%v), dp(%v), cost (%v)ms.", vv.Name, dp.partitionID, time.Since(ttlStart))
case <-dp.stopC:

View File

@ -231,7 +231,7 @@ func (dp *DataPartition) ApplyRandomWrite(command []byte, raftApplyID uint64) (r
} else {
if err == storage.ErrStoreAlreadyClosed {
err = nil
resp = proto.OpStoreClosed
respStatus = proto.OpStoreClosed
log.LogWarnf("[ApplyRandomWrite] vol(%v) dp(%v) apply id(%v) store already closed", dp.volumeID, dp.partitionID, raftApplyID)
return
}
@ -282,7 +282,7 @@ func (dp *DataPartition) ApplyRandomWrite(command []byte, raftApplyID uint64) (r
}
dp.disk.limitWrite.Run(int(opItem.size), func() {
respStatus, err = dp.ExtentStore().Write(opItem.extentID, opItem.offset, opItem.size, opItem.data, opItem.crc, writeType, syncWrite, false, false)
respStatus, err = dp.ExtentStore().Write(opItem.extentID, opItem.offset, opItem.size, opItem.data, opItem.crc, writeType, syncWrite, false, false, opItem.opcode == proto.OpBackupWrite)
})
if err == nil || err == storage.ErrStoreAlreadyClosed {
break

View File

@ -18,7 +18,6 @@ import (
"encoding/binary"
"encoding/json"
"fmt"
"io/ioutil"
syslog "log"
"net"
"os"
@ -334,13 +333,7 @@ func (dp *DataPartition) StartRaftAfterRepair(isLoad bool) {
if currLeaderPartitionSize < initPartitionSize {
initPartitionSize = currLeaderPartitionSize
}
localSize, err := dp.extentStore.StoreSizeExtentID(initMaxExtentID)
if err != nil {
log.LogErrorf("[StartRaftAfterRepair] dp(%v) failed to get size, err(%v)", dp.partitionID, err)
return
}
log.LogInfof("StartRaftAfterRepair PartitionID(%v) initMaxExtentID(%v) initPartitionSize(%v) currLeaderPartitionSize(%v)"+
localSize := dp.extentStore.StoreSizeExtentID(initMaxExtentID)
dp.decommissionRepairProgress = float64(localSize) / float64(initPartitionSize)
log.LogInfof("action[StartRaftAfterRepair] PartitionID(%v) initMaxExtentID(%v) initPartitionSize(%v) currLeaderPartitionSize(%v)"+
"localSize(%v)", dp.partitionID, initMaxExtentID, initPartitionSize, currLeaderPartitionSize, localSize)
@ -816,7 +809,7 @@ func (dp *DataPartition) getAllReplicaAppliedID() (allAppliedID []uint64, replyN
target := replicas[i]
appliedID, err := dp.getRemoteAppliedID(target, p)
if err != nil {
log.LogErrorf("partition(%v) getRemoteAppliedID from replica(%v) Failed(%v).", dp.partitionID, targetReplica, err)
log.LogErrorf("partition(%v) getRemoteAppliedID from replica(%v) Failed(%v).", dp.partitionID, target, err)
continue
}
if appliedID == 0 {

View File

@ -18,7 +18,6 @@ import (
"bytes"
"errors"
"fmt"
syslog "log"
"net"
"net/http"
"os"
@ -29,11 +28,6 @@ import (
"strings"
"sync"
"sync/atomic"
"time"
"errors"
syslog "log"
"os"
"syscall"
"time"
@ -48,6 +42,7 @@ import (
"github.com/cubefs/cubefs/util/exporter"
"github.com/cubefs/cubefs/util/loadutil"
"github.com/cubefs/cubefs/util/log"
syslog "log"
"github.com/xtaci/smux"
)
@ -126,8 +121,8 @@ const (
// load/stop dp limit
ConfigDiskCurrentLoadDpLimit = "diskCurrentLoadDpLimit"
ConfigDiskCurrentStopDpLimit = "diskCurrentStopDpLimit"
//disk read extent limit
ConfigEnableDiskReadExtentLimit = "enableDiskReadRepairExtentLimit" //bool
// disk read extent limit
ConfigEnableDiskReadExtentLimit = "enableDiskReadRepairExtentLimit" // bool
ConfigServiceIDKey = "serviceIDKey"
@ -263,7 +258,7 @@ func doStart(server common.Server, cfg *config.Config) (err error) {
return
}
//smux listening & smux connection pool
// smux listening & smux connection pool
if err = s.startSmuxService(cfg); err != nil {
return
}

View File

@ -22,7 +22,6 @@ import (
"path"
"strconv"
"sync/atomic"
"time"
"syscall"
"github.com/cubefs/cubefs/util/log"
@ -634,174 +633,6 @@ func (s *DataNode) reloadDataPartition(w http.ResponseWriter, r *http.Request) {
}
}
func (s *DataNode) setDiskExtentReadLimitStatus(w http.ResponseWriter, r *http.Request) {
const (
paramStatus = "status"
)
if err := r.ParseForm(); err != nil {
err = fmt.Errorf("parse form fail: %v", err)
s.buildFailureResp(w, http.StatusBadRequest, err.Error())
return
}
status, err := strconv.ParseBool(r.FormValue(paramStatus))
if err != nil {
err = fmt.Errorf("parse param %v fail: %v", paramStatus, err)
s.buildFailureResp(w, http.StatusBadRequest, err.Error())
return
}
for _, disk := range s.space.disks {
disk.SetExtentRepairReadLimitStatus(status)
}
s.buildSuccessResp(w, "success")
}
type DiskExtentReadLimitInfo struct {
DiskPath string `json:"diskPath"`
ExtentReadLimitStatus bool `json:"extentReadLimitStatus"`
Dp uint64 `json:"dp"`
}
type DiskExtentReadLimitStatusResponse struct {
Infos []DiskExtentReadLimitInfo `json:"infos"`
}
func (s *DataNode) queryDiskExtentReadLimitStatus(w http.ResponseWriter, r *http.Request) {
resp := &DiskExtentReadLimitStatusResponse{}
for _, disk := range s.space.disks {
status, dp := disk.QueryExtentRepairReadLimitStatus()
resp.Infos = append(resp.Infos, DiskExtentReadLimitInfo{DiskPath: disk.Path, ExtentReadLimitStatus: status, Dp: dp})
}
s.buildSuccessResp(w, resp)
}
func (s *DataNode) detachDataPartition(w http.ResponseWriter, r *http.Request) {
const (
paramID = "id"
)
if err := r.ParseForm(); err != nil {
err = fmt.Errorf("parse form fail: %v", err)
s.buildFailureResp(w, http.StatusBadRequest, err.Error())
return
}
partitionID, err := strconv.ParseUint(r.FormValue(paramID), 10, 64)
if err != nil {
err = fmt.Errorf("parse param %v fail: %v", paramID, err)
s.buildFailureResp(w, http.StatusBadRequest, err.Error())
return
}
partition := s.space.Partition(partitionID)
if partition == nil {
s.buildFailureResp(w, http.StatusBadRequest, "partition not exist")
return
}
// store disk path and root of dp
disk := partition.disk
rootDir := partition.path
log.LogDebugf("data partition disk %v rootDir %v", disk, rootDir)
s.space.partitionMutex.Lock()
delete(s.space.partitions, partitionID)
s.space.partitionMutex.Unlock()
partition.Stop()
partition.Disk().DetachDataPartition(partition)
log.LogDebugf("data partition %v is detached", partitionID)
_, err = LoadDataPartition(rootDir, disk)
s.buildSuccessResp(w, "success")
}
func (s *DataNode) releaseDiskExtentReadLimitToken(w http.ResponseWriter, r *http.Request) {
const (
paramDisk = "disk"
)
if err := r.ParseForm(); err != nil {
err = fmt.Errorf("parse form fail: %v", err)
s.buildFailureResp(w, http.StatusBadRequest, err.Error())
return
}
diskPath := r.FormValue(paramDisk)
// store disk path and root of dp
disk, err := s.space.GetDisk(diskPath)
if err != nil {
log.LogErrorf("action[loadDataPartition] disk(%v) is not found err(%v).", diskPath, err)
s.buildFailureResp(w, http.StatusBadRequest, fmt.Sprintf("disk %v is not found", diskPath))
return
}
disk.ReleaseReadExtentToken()
s.buildSuccessResp(w, "success")
}
func (s *DataNode) loadDataPartition(w http.ResponseWriter, r *http.Request) {
const (
paramID = "id"
paramDisk = "disk"
)
if err := r.ParseForm(); err != nil {
err = fmt.Errorf("parse form fail: %v", err)
s.buildFailureResp(w, http.StatusBadRequest, err.Error())
return
}
partitionID, err := strconv.ParseUint(r.FormValue(paramID), 10, 64)
if err != nil {
err = fmt.Errorf("parse param %v fail: %v", paramID, err)
s.buildFailureResp(w, http.StatusBadRequest, err.Error())
return
}
partition := s.space.Partition(partitionID)
if partition != nil {
s.buildFailureResp(w, http.StatusBadRequest, "partition is already loaded")
return
}
diskPath := r.FormValue(paramDisk)
// store disk path and root of dp
disk, err := s.space.GetDisk(diskPath)
if err != nil {
log.LogErrorf("action[loadDataPartition] disk(%v) is not found err(%v).", diskPath, err)
s.buildFailureResp(w, http.StatusBadRequest, fmt.Sprintf("disk %v is not found", diskPath))
return
}
fileInfoList, err := os.ReadDir(disk.Path)
if err != nil {
log.LogErrorf("action[loadDataPartition] read dir(%v) err(%v).", disk.Path, err)
s.buildFailureResp(w, http.StatusBadRequest, fmt.Sprintf(" read dir(%v) err(%v)", disk.Path, err))
return
}
rootDir := ""
for _, fileInfo := range fileInfoList {
filename := fileInfo.Name()
if !disk.isPartitionDir(filename) {
if disk.isExpiredPartitionDir(filename) {
}
continue
}
if id, _, err := unmarshalPartitionName(filename); err != nil {
log.LogErrorf("action[RestorePartition] unmarshal partitionName(%v) from disk(%v) err(%v) ",
filename, disk.Path, err.Error())
continue
} else {
if id == partitionID {
rootDir = filename
}
}
}
if rootDir == "" {
log.LogErrorf("action[loadDataPartition] dp root not found in dir(%v) .", disk.Path)
s.buildFailureResp(w, http.StatusBadRequest, fmt.Sprintf("dp root not found in dir(%v)", disk.Path))
return
}
log.LogDebugf("data partition disk %v rootDir %v", disk, rootDir)
_, err = LoadDataPartition(path.Join(diskPath, rootDir), disk)
if err != nil {
s.buildFailureResp(w, http.StatusBadRequest, err.Error())
} else {
s.buildSuccessResp(w, "success")
}
}
// NOTE: use for test
func (s *DataNode) markDataPartitionBroken(w http.ResponseWriter, r *http.Request) {
const (
@ -931,7 +762,7 @@ func (s *DataNode) detachDataPartition(w http.ResponseWriter, r *http.Request) {
s.buildFailureResp(w, http.StatusBadRequest, "partition not exist")
return
}
//store disk path and root of dp
// store disk path and root of dp
disk := partition.disk
rootDir := partition.path
log.LogDebugf("data partition disk %v rootDir %v", disk, rootDir)
@ -956,7 +787,7 @@ func (s *DataNode) releaseDiskExtentReadLimitToken(w http.ResponseWriter, r *htt
return
}
diskPath := r.FormValue(paramDisk)
//store disk path and root of dp
// store disk path and root of dp
disk, err := s.space.GetDisk(diskPath)
if err != nil {
log.LogErrorf("action[loadDataPartition] disk(%v) is not found err(%v).", diskPath, err)
@ -989,7 +820,7 @@ func (s *DataNode) loadDataPartition(w http.ResponseWriter, r *http.Request) {
return
}
diskPath := r.FormValue(paramDisk)
//store disk path and root of dp
// store disk path and root of dp
disk, err := s.space.GetDisk(diskPath)
if err != nil {
log.LogErrorf("action[loadDataPartition] disk(%v) is not found err(%v).", diskPath, err)
@ -1036,5 +867,4 @@ func (s *DataNode) loadDataPartition(w http.ResponseWriter, r *http.Request) {
} else {
s.buildSuccessResp(w, "success")
}
}

View File

@ -17,24 +17,20 @@ package datanode
import (
"fmt"
"math"
"math/rand"
"os"
"path"
"math/rand"
"sync"
"math"
"sync/atomic"
"time"
"math"
"os"
syslog "log"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/raftstore"
"github.com/cubefs/cubefs/util"
"github.com/cubefs/cubefs/util/atomicutil"
"github.com/cubefs/cubefs/util/loadutil"
"github.com/cubefs/cubefs/util"
"github.com/cubefs/cubefs/util/log"
"github.com/shirou/gopsutil/disk"
)
@ -273,7 +269,8 @@ func (manager *SpaceManager) Stats() *Stats {
}
func (manager *SpaceManager) LoadDisk(path string, reservedSpace, diskRdonlySpace uint64, maxErrCnt int,
diskEnableReadRepairExtentLimit bool) (err error) {
diskEnableReadRepairExtentLimit bool,
) (err error) {
var (
disk *Disk
visitor PartitionVisitor

View File

@ -71,7 +71,8 @@ func (s *Stats) GetConnectionCount() int64 {
func (s *Stats) updateMetrics(
total, used, available, createdPartitionWeights, remainWeightsForCreatePartition,
maxWeightsForCreatePartition, dataPartitionCnt uint64) {
maxWeightsForCreatePartition, dataPartitionCnt uint64,
) {
s.Lock()
defer s.Unlock()

View File

@ -741,11 +741,7 @@ func (s *DataNode) handleBatchMarkDeletePacket(p *repl.Packet, c net.Conn) {
store := partition.ExtentStore()
if err == nil {
for _, ext := range exts {
ok, err := store.CanGcDelete(ext.ExtentId)
if err != nil {
log.LogErrorf("[handleBatchMarkDeletePacket] dp(%v) failed to get extent(%v) info, err(%v)", ext.PartitionId, ext.ExtentId, err)
return
}
ok := store.CanGcDelete(ext.ExtentId)
if p.Opcode == proto.OpGcBatchDeleteExtent && ok {
log.LogWarnf("handleBatchMarkDeletePacket: ext %d is not in gc status, can't be gc delete, dp %d", ext.ExtentId, ext.PartitionId)
err = storage.ParameterMismatchError
@ -827,7 +823,7 @@ func (s *DataNode) handleWritePacket(p *repl.Packet) {
partition.disk.allocCheckLimit(proto.IopsWriteType, 1)
if writable := partition.disk.limitWrite.TryRun(int(p.Size), func() {
_, err = store.Write(p.ExtentID, p.ExtentOffset, int64(p.Size), p.Data, p.CRC, storage.AppendWriteType, p.IsSyncWrite(), false, false)
_, err = store.Write(p.ExtentID, p.ExtentOffset, int64(p.Size), p.Data, p.CRC, storage.AppendWriteType, p.IsSyncWrite(), false, false, false)
}); !writable {
err = storage.LimitedIoError
return
@ -849,7 +845,7 @@ func (s *DataNode) handleWritePacket(p *repl.Packet) {
partition.disk.allocCheckLimit(proto.IopsWriteType, 1)
if writable := partition.disk.limitWrite.TryRun(int(p.Size), func() {
_, err = store.Write(p.ExtentID, p.ExtentOffset, int64(p.Size), p.Data, p.CRC, storage.AppendWriteType, p.IsSyncWrite(), false, false)
_, err = store.Write(p.ExtentID, p.ExtentOffset, int64(p.Size), p.Data, p.CRC, storage.AppendWriteType, p.IsSyncWrite(), false, false, false)
}); !writable {
err = storage.LimitedIoError
return
@ -877,7 +873,7 @@ func (s *DataNode) handleWritePacket(p *repl.Packet) {
partition.disk.allocCheckLimit(proto.IopsWriteType, 1)
if writable := partition.disk.limitWrite.TryRun(currSize, func() {
_, err = store.Write(p.ExtentID, p.ExtentOffset+int64(offset), int64(currSize), data, crc, storage.AppendWriteType, p.IsSyncWrite(), false, false)
_, err = store.Write(p.ExtentID, p.ExtentOffset+int64(offset), int64(currSize), data, crc, storage.AppendWriteType, p.IsSyncWrite(), false, false, false)
}); !writable {
err = storage.LimitedIoError
return
@ -1226,12 +1222,7 @@ func (s *DataNode) handlePacketToGetAppliedID(p *repl.Packet) {
func (s *DataNode) handlePacketToGetPartitionSize(p *repl.Packet) {
partition := p.Object.(*DataPartition)
usedSize, err := partition.extentStore.StoreSizeExtentID(p.ExtentID)
if err != nil {
log.LogErrorf("[handlePacketToGetPartitionSize] dp(%v) failed to get extents size, err(%v)", partition.partitionID, err)
p.PacketErrorWithBody(proto.OpDiskErr, []byte(err.Error()))
return
}
usedSize := partition.extentStore.StoreSizeExtentID(p.ExtentID)
buf := make([]byte, 8)
binary.BigEndian.PutUint64(buf, uint64(usedSize))
p.AddMesgLog(fmt.Sprintf("partitionSize_(%v)", usedSize))
@ -1240,13 +1231,7 @@ func (s *DataNode) handlePacketToGetPartitionSize(p *repl.Packet) {
func (s *DataNode) handlePacketToGetMaxExtentIDAndPartitionSize(p *repl.Packet) {
partition := p.Object.(*DataPartition)
maxExtentID, totalPartitionSize, err := partition.extentStore.GetMaxExtentIDAndPartitionSize()
if err != nil {
log.LogErrorf("[handlePacketToGetMaxExtentIDAndPartitionSize] dp(%v) failed to get extents size, err(%v)", partition.partitionID, err)
p.PacketErrorWithBody(proto.OpDiskErr, []byte(err.Error()))
return
}
maxExtentID, totalPartitionSize := partition.extentStore.GetMaxExtentIDAndPartitionSize()
buf := make([]byte, 16)
binary.BigEndian.PutUint64(buf[0:8], uint64(maxExtentID))
binary.BigEndian.PutUint64(buf[8:16], totalPartitionSize)
@ -1675,9 +1660,7 @@ func (s *DataNode) handlePacketToRecoverDataReplicaMeta(p *repl.Packet) {
}
func (s *DataNode) handleBatchLockNormalExtent(p *repl.Packet, connect net.Conn) {
var (
err error
)
var err error
defer func() {
if err != nil {
@ -1726,9 +1709,7 @@ func (s *DataNode) handleBatchLockNormalExtent(p *repl.Packet, connect net.Conn)
}
func (s *DataNode) handleBatchUnlockNormalExtent(p *repl.Packet, connect net.Conn) {
var (
err error
)
var err error
defer func() {
if err != nil {

View File

@ -442,7 +442,7 @@ services:
command:
- bash
- "-c"
- >-
- >
set -e;
cd ${CFSROOT} && make libsdkpre
@ -456,29 +456,6 @@ services:
cd ${CFSROOT} &&
/go/bin/goreleaser release --skip-publish --clean
build_libsdkpre:
image: ${IMAGE}
volumes:
- ../:${CFSROOT}
command:
- bash
- "-c"
- >-
set -e;
cd ${CFSROOT} && make libsdkpre
goreleaser:
image: ${IMAGE}
volumes:
- ../:${CFSROOT}
command:
- bash
- "-c"
- >-
set -e;
cd ${CFSROOT} &&
/go/bin/goreleaser release --skip-publish --clean
bs_gofumpt:
<<: *command
command:

View File

@ -77,9 +77,11 @@ type VerifyInfo struct {
VolName string
}
var hostLimit = make(map[string]chan struct{})
var cntLimit = 1
var hostLk = sync.RWMutex{}
var (
hostLimit = make(map[string]chan struct{})
cntLimit = 1
hostLk = sync.RWMutex{}
)
func setHostCntLimit(cnt int) {
clearChan()
@ -114,7 +116,6 @@ func releaseToken(host string) {
default:
slog.Fatalf("there is not token in chan, host %s", host)
}
}
func clearChan() {
@ -124,7 +125,7 @@ func clearChan() {
}
func newGCCommand() *cobra.Command {
var c = &cobra.Command{
c := &cobra.Command{
Use: "gc",
Short: "gc specified volume",
Args: cobra.MinimumNArgs(0),
@ -175,7 +176,7 @@ func newGetMpExtentsCmd() *cobra.Command {
// var fromMpId string
var concurrency uint64
var cmd = &cobra.Command{
cmd := &cobra.Command{
Use: "getMpExtents [VOLNAME] [DIR]",
Short: "get extents of mp",
Args: cobra.MinimumNArgs(2),
@ -225,7 +226,7 @@ func getMpInfoById(mpId string) (mpInfo *proto.MetaPartitionInfo, err error) {
return nil, err
}
var body = &struct {
body := &struct {
Code int32 `json:"code"`
Msg string `json:"msg"`
Data json.RawMessage `json:"data"`
@ -352,7 +353,7 @@ func getExtentsByMpId(dir string, volname string, mpId string) {
slog.Fatalf("Init local dir failed, mpId %s, err: %v", mpId, err)
}
normalFile, err := os.OpenFile(NormalFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
normalFile, err := os.OpenFile(NormalFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
if err != nil {
slog.Fatalf("Open file failed, mp %s, err: %v", mpId, err)
}
@ -449,7 +450,7 @@ func getExtentsByMpId(dir string, volname string, mpId string) {
log.LogDebugf("inode:%v, extent:%v", ino.Inode, eks)
}
walkFunc := func(ek proto.ExtentKey) bool {
walkFunc := func(i int, ek proto.ExtentKey) bool {
if storage.IsTinyExtent(ek.ExtentId) {
return true
}
@ -493,7 +494,7 @@ func InitLocalDir(dir string, volname string, partitionId string, dirType string
normalPath := filepath.Join(dir, volname, dirType, normalDir)
_, err = os.Stat(normalPath)
if os.IsNotExist(err) {
if err = os.MkdirAll(normalPath, 0755); err != nil {
if err = os.MkdirAll(normalPath, 0o755); err != nil {
slog.Fatalf("create dir failed, path %s, err : %v", normalFilePath, err)
return
}
@ -624,7 +625,7 @@ func getDpInfoById(dpId string) (dpInfo *proto.DataPartitionInfo, err error) {
return
}
var body = &struct {
body := &struct {
Code int32 `json:"code"`
Msg string `json:"msg"`
Data json.RawMessage `json:"data"`
@ -693,7 +694,7 @@ func getExtentsByDpId(dir string, volname string, dpId string, beforeTime string
return fmt.Errorf("Read response body failed, err: %v", err)
}
var body = &struct {
body := &struct {
Code int32 `json:"code"`
Msg string `json:"msg"`
Data json.RawMessage `json:"data"`
@ -717,7 +718,7 @@ func getExtentsByDpId(dir string, volname string, dpId string, beforeTime string
return
}
normalFile, err := os.OpenFile(normalFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
normalFile, err := os.OpenFile(normalFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
if err != nil {
log.LogErrorf("Open normalFile failed, err: %v", err)
return
@ -765,7 +766,7 @@ func getExtentsByDpId(dir string, volname string, dpId string, beforeTime string
return
}
err = os.WriteFile(beforeTimeFilePath, []byte(beforeTime), 0644)
err = os.WriteFile(beforeTimeFilePath, []byte(beforeTime), 0o644)
if err != nil {
log.LogErrorf("Write before time file failed, err: %v", err)
return
@ -825,7 +826,7 @@ func getClusterInfo() (clusterView *proto.ClusterView, err error) {
return
}
var body = &struct {
body := &struct {
Code int32 `json:"code"`
Msg string `json:"msg"`
Data json.RawMessage `json:"data"`
@ -868,7 +869,7 @@ func writeNormalVerifyInfo(dir, volname, dirType string) {
return
}
}
err = os.WriteFile(filePath, data, 0644)
err = os.WriteFile(filePath, data, 0o644)
if err != nil {
slog.Fatalf("Write verify info file failed, err: %v", err)
return
@ -952,7 +953,6 @@ func newCalcBadExtentsCmd() *cobra.Command {
slog.Fatalf("Invalid extent type: %s", extType)
return
}
},
}
cmd.Flags().Uint64Var(&concurrency, "concurrency", 0, "clean bad dp concurrency")
@ -990,7 +990,7 @@ func calcBadNormalExtents(volname, dir string, concurrency int) (err error) {
}
}
err = os.MkdirAll(destDir, 0755)
err = os.MkdirAll(destDir, 0o755)
if err != nil {
slog.Fatalf("Mkdir dest dir failed, err: %v", err)
return
@ -1168,7 +1168,7 @@ func writeBadNormalExtentoLocal(dpId, badDir string, exts []BadNornalExtent) (er
start := time.Now()
log.LogInfof("Write dir %s, dpId: %s bad extent: %d", badDir, dpId, len(exts))
filePath := filepath.Join(badDir, dpId)
file, err := os.OpenFile(filePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
file, err := os.OpenFile(filePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644)
if err != nil {
slog.Fatalf("Open file failed, path %s, err: %v", filePath, err)
return err
@ -1641,7 +1641,7 @@ func cleanBadNormalExtent(volname, dir, backupDir, dpIdStr, extentStr string) (e
continue
}
var exts = []*BadNornalExtent{&badExtent}
exts := []*BadNornalExtent{&badExtent}
err = batchLockBadNormalExtent(dpIdStr, exts, false, string(data))
if err != nil {
@ -1853,7 +1853,7 @@ func readBadExtentFromDp(dpIdStr string, extentId uint64, size uint32) (data []b
for readBytes < int(size) {
replyPacket := stream.NewReply(p.ReqID, dpId, extentId)
bufSize := util.Min(util.ReadBlockSize, int(size)-readBytes)
//replyPacket.Data = data[readBytes : readBytes+bufSize]
// replyPacket.Data = data[readBytes : readBytes+bufSize]
replyPacket.Size = uint32(bufSize)
err = replyPacket.ReadFromConn(conn, proto.ReadDeadlineTime)
if err != nil {
@ -1879,7 +1879,6 @@ func readBadExtentFromDp(dpIdStr string, extentId uint64, size uint32) (data []b
}
func writeBadNormalExtentToBackup(backupDir, volname, dpIdStr string, badExtent BadNornalExtent, data []byte, readBytes int) (err error) {
dpDir := filepath.Join(backupDir, volname, normalDir, dpIdStr)
start := time.Now()
defer func() {
@ -1888,7 +1887,7 @@ func writeBadNormalExtentToBackup(backupDir, volname, dpIdStr string, badExtent
}()
_, err = os.Stat(dpDir)
if os.IsNotExist(err) {
err = os.MkdirAll(dpDir, 0755)
err = os.MkdirAll(dpDir, 0o755)
if err != nil {
slog.Fatalf("Mkdir failed, dir %s, err: %v", dpDir, err.Error())
}
@ -2218,7 +2217,7 @@ func writeBadNormalExtentToDp(data []byte, volname, dpId, extent string) (err er
p.Magic = proto.ProtoMagic
p.Opcode = proto.OpBackupWrite
length := len(data)
//length := util.BlockSize
// length := util.BlockSize
p.Data = make([]byte, length)
copy(p.Data, data)
p.Size = uint32(length)

View File

@ -44,7 +44,6 @@ func main() {
}
func listenPprof() {
mainMux := http.NewServeMux()
mux := http.NewServeMux()
mux.Handle("/debug/pprof", http.HandlerFunc(pprof.Index))

View File

@ -1593,7 +1593,7 @@ func (c *client) setattr(info *proto.InodeInfo, valid uint32, mode, uid, gid uin
func (c *client) create(pino uint64, name string, mode uint32, fullPath string) (info *proto.InodeInfo, err error) {
fuseMode := mode & 0o777
return c.mw.Create_ll(pino, name, fuseMode, 0, 0, nil, fullPath)
return c.mw.Create_ll(pino, name, fuseMode, 0, 0, nil, fullPath, false)
}
func (c *client) mkdir(pino uint64, name string, mode uint32, fullPath string) (info *proto.InodeInfo, err error) {

View File

@ -303,11 +303,11 @@ func (m *Server) setEnableAuditLogForVolume(w http.ResponseWriter, r *http.Reque
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeVolNotExists, Msg: err.Error()})
return
}
oldEnable := vol.EnableAuditLog
vol.EnableAuditLog = status
oldEnable := vol.DisableAuditLog
vol.DisableAuditLog = status
defer func() {
if err != nil {
vol.EnableAuditLog = oldEnable
vol.DisableAuditLog = oldEnable
}
}()
if err = m.cluster.syncUpdateVol(vol); err != nil {

View File

@ -127,7 +127,6 @@ func createDefaultMasterServerForTest() *Server {
}`
testServer, err := createMasterServer(cfgJSON)
if err != nil {
panic(err)
}
@ -1579,11 +1578,11 @@ func TestVolumeEnableAuditLog(t *testing.T) {
enableUrl := fmt.Sprintf("%v?name=%v&%v=true", reqUrl, vol.Name, enableKey)
disableUrl := fmt.Sprintf("%v?name=%v&%v=false", reqUrl, vol.Name, enableKey)
process(disableUrl, t)
require.True(t, vol.DisableAuditLog)
require.True(t, checkVolAuditLog(name, false))
process(enableUrl, t)
require.False(t, vol.DisableAuditLog)
require.True(t, checkVolAuditLog(name, true))
process(enableUrl, t)
require.True(t, vol.DisableAuditLog)
require.True(t, checkVolAuditLog(name, false))
}
func checkVolDpRepairBlockSize(name string, size uint64) (success bool) {

View File

@ -1224,7 +1224,8 @@ func (c *Cluster) checkLackReplicaDataPartitions() (lackReplicaDataPartitions []
func (c *Cluster) checkReplicaOfDataPartitions(ignoreDiscardDp bool) (
lackReplicaDPs []*DataPartition, unavailableReplicaDPs []*DataPartition, repFileCountDifferDps []*DataPartition,
repUsedSizeDifferDps []*DataPartition, excessReplicaDPs []*DataPartition, noLeaderDPs []*DataPartition, err error) {
repUsedSizeDifferDps []*DataPartition, excessReplicaDPs []*DataPartition, noLeaderDPs []*DataPartition, err error,
) {
noLeaderDPs = make([]*DataPartition, 0)
lackReplicaDPs = make([]*DataPartition, 0)
unavailableReplicaDPs = make([]*DataPartition, 0)
@ -1684,7 +1685,8 @@ errHandler:
}
func (c *Cluster) syncCreateDataPartitionToDataNode(host string, size uint64, dp *DataPartition,
peers []proto.Peer, hosts []string, createType int, partitionType int, needRollBack, ignoreDecommissionDisk bool) (diskPath string, err error) {
peers []proto.Peer, hosts []string, createType int, partitionType int, needRollBack, ignoreDecommissionDisk bool,
) (diskPath string, err error) {
log.LogInfof("action[syncCreateDataPartitionToDataNode] dp [%v] createType[%v], partitionType[%v] ignoreDecommissionDisk[%v]",
dp.PartitionID, createType, partitionType, ignoreDecommissionDisk)
dataNode, err := c.dataNode(host)
@ -1798,7 +1800,8 @@ func (c *Cluster) chooseZone2Plus1(zones []*Zone, excludeNodeSets []uint64, excl
}
func (c *Cluster) chooseZoneNormal(zones []*Zone, excludeNodeSets []uint64, excludeHosts []string,
nodeType uint32, replicaNum int) (hosts []string, peers []proto.Peer, err error) {
nodeType uint32, replicaNum int,
) (hosts []string, peers []proto.Peer, err error) {
log.LogInfof("action[chooseZoneNormal] zones[%s] nodeType[%d] replicaNum[%d]", printZonesName(zones), nodeType, replicaNum)
c.zoneIdxMux.Lock()

View File

@ -295,7 +295,8 @@ type VolNameSet map[string]struct{}
func (c *Cluster) checkReplicaMetaPartitions() (
lackReplicaMetaPartitions []*MetaPartition, noLeaderMetaPartitions []*MetaPartition,
unavailableReplicaMPs []*MetaPartition, excessReplicaMetaPartitions, inodeCountNotEqualMPs, maxInodeNotEqualMPs, dentryCountNotEqualMPs []*MetaPartition, err error) {
unavailableReplicaMPs []*MetaPartition, excessReplicaMetaPartitions, inodeCountNotEqualMPs, maxInodeNotEqualMPs, dentryCountNotEqualMPs []*MetaPartition, err error,
) {
lackReplicaMetaPartitions = make([]*MetaPartition, 0)
noLeaderMetaPartitions = make([]*MetaPartition, 0)
excessReplicaMetaPartitions = make([]*MetaPartition, 0)

View File

@ -1049,7 +1049,6 @@ func (partition *DataPartition) buildDpInfo(c *Cluster) *proto.DataPartitionInfo
OfflinePeerID: partition.OfflinePeerID,
IsRecover: partition.isRecover,
FilesWithMissingReplica: filesMissReplicas,
SingleDecommissionAddr: partition.SingleDecommissionAddr,
IsDiscard: partition.IsDiscard,
SingleDecommissionStatus: partition.GetSpecialReplicaDecommissionStep(),
Forbidden: forbidden,
@ -1106,7 +1105,8 @@ func GetDecommissionStatusMessage(status uint32) string {
}
func (partition *DataPartition) MarkDecommissionStatus(srcAddr, dstAddr, srcDisk string, raftForce bool, term uint64,
migrateType uint32, c *Cluster, ns *nodeSet) (err error) {
migrateType uint32, c *Cluster, ns *nodeSet,
) (err error) {
if partition.needManualFix() && migrateType == AutoDecommission {
return proto.ErrAllReplicaUnavailable
}

View File

@ -26,7 +26,8 @@ import (
)
func (partition *DataPartition) checkStatus(clusterName string, needLog bool, dpTimeOutSec int64, c *Cluster,
shouldDpInhibitWriteByVolFull bool, forbiddenVol bool) {
shouldDpInhibitWriteByVolFull bool, forbiddenVol bool,
) {
partition.Lock()
defer partition.Unlock()
var liveReplicas []*DataReplica

View File

@ -210,7 +210,8 @@ func (c *Cluster) deleteAndSyncDecommissionedDisk(dataNode *DataNode, diskPath s
}
func (c *Cluster) decommissionDisk(dataNode *DataNode, raftForce bool, badDiskPath string,
badPartitions []*DataPartition, diskDisable bool) (err error) {
badPartitions []*DataPartition, diskDisable bool,
) (err error) {
msg := fmt.Sprintf("action[decommissionDisk], Node[%v] OffLine,disk[%v]", dataNode.Addr, badDiskPath)
log.LogWarn(msg)

View File

@ -227,7 +227,8 @@ func (m *ClusterService) decommissionDataNode(ctx context.Context, args struct {
func (m *ClusterService) decommissionMetaNode(ctx context.Context, args struct {
OffLineAddr string
}) (*proto.GeneralResp, error) {
},
) (*proto.GeneralResp, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
@ -244,7 +245,8 @@ func (m *ClusterService) decommissionMetaNode(ctx context.Context, args struct {
func (m *ClusterService) loadMetaPartition(ctx context.Context, args struct {
PartitionID uint64
}) (*proto.GeneralResp, error) {
},
) (*proto.GeneralResp, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
@ -261,7 +263,8 @@ func (m *ClusterService) loadMetaPartition(ctx context.Context, args struct {
func (m *ClusterService) decommissionMetaPartition(ctx context.Context, args struct {
PartitionID uint64
NodeAddr string
}) (*proto.GeneralResp, error) {
},
) (*proto.GeneralResp, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
@ -278,7 +281,8 @@ func (m *ClusterService) decommissionMetaPartition(ctx context.Context, args str
func (m *ClusterService) getMetaNode(ctx context.Context, args struct {
NodeAddr string
}) (*MetaNode, error) {
},
) (*MetaNode, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
@ -361,7 +365,8 @@ func (s *ClusterService) masterList(ctx context.Context, args struct{}) ([]*Mast
func (s *ClusterService) dataNodeGet(ctx context.Context, args struct {
Addr string
}) (*DataNode, error) {
},
) (*DataNode, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
@ -382,7 +387,8 @@ func (s *ClusterService) dataNodeList(ctx context.Context, args struct{}) ([]*Da
func (s *ClusterService) dataNodeListTest(ctx context.Context, args struct {
Num int64
}) ([]*DataNode, error) {
},
) ([]*DataNode, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
@ -409,7 +415,8 @@ func (s *ClusterService) dataNodeListTest(ctx context.Context, args struct {
func (s *ClusterService) metaNodeGet(ctx context.Context, args struct {
Addr string
}) (*MetaNode, error) {
},
) (*MetaNode, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
@ -435,7 +442,8 @@ func (s *ClusterService) metaNodeList(ctx context.Context, args struct{}) ([]*Me
func (m *ClusterService) addMetaNode(ctx context.Context, args struct {
NodeAddr string
ZoneName string
}) (uint64, error) {
},
) (uint64, error) {
if id, err := m.cluster.addMetaNode(args.NodeAddr, args.ZoneName, 0); err != nil {
return 0, err
} else {
@ -447,7 +455,8 @@ func (m *ClusterService) addMetaNode(ctx context.Context, args struct {
func (m *ClusterService) removeRaftNode(ctx context.Context, args struct {
Id uint64
Addr string
}) (*proto.GeneralResp, error) {
},
) (*proto.GeneralResp, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
@ -463,7 +472,8 @@ func (m *ClusterService) removeRaftNode(ctx context.Context, args struct {
func (m *ClusterService) addRaftNode(ctx context.Context, args struct {
Id uint64
Addr string
}) (*proto.GeneralResp, error) {
},
) (*proto.GeneralResp, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
@ -486,7 +496,8 @@ func (m *ClusterService) addRaftNode(ctx context.Context, args struct {
// 2. and the number of r&w data partition is less than 20.
func (m *ClusterService) clusterFreeze(ctx context.Context, args struct {
Status bool
}) (*proto.GeneralResp, error) {
},
) (*proto.GeneralResp, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
@ -508,7 +519,8 @@ type WarnMessage struct {
func (m *ClusterService) alarmList(ctx context.Context, args struct {
Size int32
}) ([]*WarnMessage, error) {
},
) ([]*WarnMessage, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}

View File

@ -85,7 +85,8 @@ func (s *UserService) registerQuery(schema *schemabuilder.Schema) {
func (m *UserService) getUserAKInfo(ctx context.Context, args struct {
AccessKey string
}) (*proto.UserInfo, error) {
},
) (*proto.UserInfo, error) {
uid, perm, err := permissions(ctx, ADMIN|USER)
userInfo, err := m.user.getKeyInfo(args.AccessKey)
if err != nil {
@ -222,7 +223,8 @@ func (s *UserService) updateUser(ctx context.Context, args proto.UserUpdateParam
func (s *UserService) deleteUser(ctx context.Context, args struct {
UserID string
}) (*proto.GeneralResp, error) {
},
) (*proto.GeneralResp, error) {
uid, _, err := permissions(ctx, ADMIN)
if err != nil {
return nil, err
@ -239,7 +241,8 @@ func (s *UserService) deleteUser(ctx context.Context, args struct {
func (s *UserService) getUserInfo(ctx context.Context, args struct {
UserID string
}) (*proto.UserInfo, error) {
},
) (*proto.UserInfo, error) {
uid, perm, err := permissions(ctx, ADMIN|USER)
if err != nil {
return nil, err
@ -274,7 +277,8 @@ type UserUseSpace struct {
func (s *UserService) topNUser(ctx context.Context, args struct {
N int32
}) ([]*UserUseSpace, error) {
},
) ([]*UserUseSpace, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
@ -334,7 +338,8 @@ func (s *UserService) topNUser(ctx context.Context, args struct {
func (s *UserService) validatePassword(ctx context.Context, args struct {
UserID string
Password string
}) (*proto.UserInfo, error) {
},
) (*proto.UserInfo, error) {
ui, err := s.user.getUserInfo(args.UserID)
if err != nil {
return nil, err

View File

@ -21,6 +21,7 @@ import (
"html"
"net/http"
"net/http/httputil"
"strings"
"time"
"github.com/gorilla/mux"

View File

@ -305,8 +305,8 @@ type volValue struct {
TrashInterval int64
DisableAuditLog bool
Forbidden bool
DpRepairBlockSize uint64
Forbidden bool
DpRepairBlockSize uint64
}
func (v *volValue) Bytes() (raw []byte, err error) {

View File

@ -31,7 +31,8 @@ import (
func newCreateDataPartitionRequest(volName string, ID uint64, replicaNum int, members []proto.Peer,
dataPartitionSize, leaderSize int, hosts []string, createType int, partitionType int,
decommissionedDisks []string, verSeq uint64) (req *proto.CreateDataPartitionRequest) {
decommissionedDisks []string, verSeq uint64,
) (req *proto.CreateDataPartitionRequest) {
req = &proto.CreateDataPartitionRequest{
PartitionTyp: partitionType,
PartitionId: ID,

View File

@ -578,7 +578,8 @@ func (nsgm *DomainManager) getHostFromNodeSetGrpSpecific(domainGrpManager *Domai
func (nsgm *DomainManager) getHostFromNodeSetGrp(domainId uint64, replicaNum uint8, createType uint32) (
hosts []string,
peers []proto.Peer,
err error) {
err error,
) {
var ok bool
var index int

View File

@ -487,10 +487,6 @@ func (vol *Vol) getDataPartitionViewCompress() (body []byte, err error) {
return vol.dataPartitions.updateCompressCache(false, 0, vol)
}
func (vol *Vol) getDataPartitionViewCompress() (body []byte, err error) {
return vol.dataPartitions.updateCompressCache(false, 0, vol.VolType)
}
func (vol *Vol) getDataPartitionByID(partitionID uint64) (dp *DataPartition, err error) {
return vol.dataPartitions.get(partitionID)
}
@ -606,8 +602,6 @@ func (vol *Vol) checkDataPartitions(c *Cluster) (cnt int) {
totalPreloadCapacity := uint64(0)
totalPreloadCapacity := uint64(0)
partitions := vol.dataPartitions.clonePartitions()
for _, dp := range partitions {

View File

@ -79,7 +79,8 @@ func (m *MetaNode) registerAPIHandler() (err error) {
}
func (m *MetaNode) getParamsHandler(w http.ResponseWriter,
r *http.Request) {
r *http.Request,
) {
resp := NewAPIResponse(http.StatusOK, http.StatusText(http.StatusOK))
params := make(map[string]interface{})
params[metaNodeDeleteBatchCountKey] = DeleteBatchCount()
@ -91,7 +92,8 @@ func (m *MetaNode) getParamsHandler(w http.ResponseWriter,
}
func (m *MetaNode) getSmuxStatHandler(w http.ResponseWriter,
r *http.Request) {
r *http.Request,
) {
resp := NewAPIResponse(http.StatusOK, http.StatusText(http.StatusOK))
resp.Data = smuxPool.GetStat()
data, _ := resp.Marshal()
@ -101,7 +103,8 @@ func (m *MetaNode) getSmuxStatHandler(w http.ResponseWriter,
}
func (m *MetaNode) getPartitionsHandler(w http.ResponseWriter,
r *http.Request) {
r *http.Request,
) {
resp := NewAPIResponse(http.StatusOK, http.StatusText(http.StatusOK))
resp.Data = m.metadataManager
data, _ := resp.Marshal()
@ -372,7 +375,8 @@ func (m *MetaNode) getRaftStatusHandler(w http.ResponseWriter, r *http.Request)
}
func (m *MetaNode) getEbsExtentsByInodeHandler(w http.ResponseWriter,
r *http.Request) {
r *http.Request,
) {
r.ParseForm()
resp := NewAPIResponse(http.StatusBadRequest, "")
defer func() {
@ -416,7 +420,8 @@ func (m *MetaNode) getEbsExtentsByInodeHandler(w http.ResponseWriter,
}
func (m *MetaNode) getExtentsByInodeHandler(w http.ResponseWriter,
r *http.Request) {
r *http.Request,
) {
r.ParseForm()
resp := NewAPIResponse(http.StatusBadRequest, "")
defer func() {

View File

@ -147,7 +147,7 @@ func (m *metadataManager) updateVolumes() {
return true
})
// push to every partitions
m.Range(func(_ uint64, mp MetaPartition) bool {
m.Range(false, func(_ uint64, mp MetaPartition) bool {
dataView := dataViews[mp.GetBaseConfig().VolName]
volView := volViews[mp.GetBaseConfig().VolName]
if dataView != nil && volView != nil {

View File

@ -75,7 +75,8 @@ func (m *metadataManager) checkDisableAuditLogVolume(volNames []string, partitio
}
func (m *metadataManager) opMasterHeartbeat(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
// For ack to master
data := p.Data
m.responseAckOKToMaster(conn, p)
@ -164,7 +165,8 @@ func (m *metadataManager) opMasterHeartbeat(conn net.Conn, p *Packet,
}
func (m *metadataManager) opCreateMetaPartition(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
defer func() {
var buf []byte
status := proto.OpOk
@ -201,7 +203,8 @@ func (m *metadataManager) opCreateMetaPartition(conn net.Conn, p *Packet,
// Handle OpCreate inode.
func (m *metadataManager) opCreateInode(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &CreateInoReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -300,7 +303,8 @@ func (m *metadataManager) opTxMetaLinkInode(conn net.Conn, p *Packet, remoteAddr
}
func (m *metadataManager) opMetaLinkInode(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &LinkInodeReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -333,7 +337,8 @@ func (m *metadataManager) opMetaLinkInode(conn net.Conn, p *Packet,
// Handle OpCreate
func (m *metadataManager) opFreeInodeOnRaftFollower(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
mp, err := m.getPartition(p.PartitionID)
if err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -350,7 +355,8 @@ func (m *metadataManager) opFreeInodeOnRaftFollower(conn net.Conn, p *Packet,
// Handle OpCreate
func (m *metadataManager) opTxCreateDentry(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.TxCreateDentryRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -385,7 +391,8 @@ func (m *metadataManager) opTxCreateDentry(conn net.Conn, p *Packet,
}
func (m *metadataManager) opTxCreate(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.TxCreateRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -415,7 +422,8 @@ func (m *metadataManager) opTxCreate(conn net.Conn, p *Packet,
}
func (m *metadataManager) opTxGet(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.TxGetInfoRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -447,7 +455,8 @@ func (m *metadataManager) opTxGet(conn net.Conn, p *Packet,
}
func (m *metadataManager) opTxCommitRM(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.TxApplyRMRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -477,7 +486,8 @@ func (m *metadataManager) opTxCommitRM(conn net.Conn, p *Packet,
}
func (m *metadataManager) opTxRollbackRM(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.TxApplyRMRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -507,7 +517,8 @@ func (m *metadataManager) opTxRollbackRM(conn net.Conn, p *Packet,
}
func (m *metadataManager) opTxCommit(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.TxApplyRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -537,7 +548,8 @@ func (m *metadataManager) opTxCommit(conn net.Conn, p *Packet,
}
func (m *metadataManager) opTxRollback(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.TxApplyRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -568,7 +580,8 @@ func (m *metadataManager) opTxRollback(conn net.Conn, p *Packet,
// Handle OpCreate
func (m *metadataManager) opCreateDentry(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &CreateDentryReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -601,7 +614,8 @@ func (m *metadataManager) opCreateDentry(conn net.Conn, p *Packet,
}
func (m *metadataManager) opQuotaCreateDentry(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.QuotaCreateDentryRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -635,7 +649,8 @@ func (m *metadataManager) opQuotaCreateDentry(conn net.Conn, p *Packet,
// Handle OpDelete Dentry
func (m *metadataManager) opTxDeleteDentry(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.TxDeleteDentryRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -666,7 +681,8 @@ func (m *metadataManager) opTxDeleteDentry(conn net.Conn, p *Packet,
// Handle OpDelete Dentry
func (m *metadataManager) opDeleteDentry(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &DeleteDentryReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -701,7 +717,8 @@ func (m *metadataManager) opDeleteDentry(conn net.Conn, p *Packet,
// Handle Op batch Delete Dentry
func (m *metadataManager) opBatchDeleteDentry(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &BatchDeleteDentryReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -767,7 +784,8 @@ func (m *metadataManager) opTxUpdateDentry(conn net.Conn, p *Packet, remoteAddr
}
func (m *metadataManager) opUpdateDentry(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &UpdateDentryReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -834,7 +852,8 @@ func (m *metadataManager) opTxMetaUnlinkInode(conn net.Conn, p *Packet, remoteAd
}
func (m *metadataManager) opMetaUnlinkInode(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &UnlinkInoReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -866,7 +885,8 @@ func (m *metadataManager) opMetaUnlinkInode(conn net.Conn, p *Packet,
}
func (m *metadataManager) opMetaBatchUnlinkInode(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &BatchUnlinkInoReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -898,7 +918,8 @@ func (m *metadataManager) opMetaBatchUnlinkInode(conn net.Conn, p *Packet,
}
func (m *metadataManager) opReadDirOnly(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.ReadDirOnlyRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -925,7 +946,8 @@ func (m *metadataManager) opReadDirOnly(conn net.Conn, p *Packet,
// Handle OpReadDir
func (m *metadataManager) opReadDir(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.ReadDirRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -952,7 +974,8 @@ func (m *metadataManager) opReadDir(conn net.Conn, p *Packet,
// Handle OpReadDirLimit
func (m *metadataManager) opReadDirLimit(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.ReadDirLimitRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1023,7 +1046,8 @@ func (m *metadataManager) opMetaInodeGet(conn net.Conn, p *Packet,
}
func (m *metadataManager) opBatchMetaEvictInode(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.BatchEvictInodeRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
@ -1057,7 +1081,8 @@ func (m *metadataManager) opBatchMetaEvictInode(conn net.Conn, p *Packet,
}
func (m *metadataManager) opMetaEvictInode(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.EvictInodeRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1091,7 +1116,8 @@ func (m *metadataManager) opMetaEvictInode(conn net.Conn, p *Packet,
}
func (m *metadataManager) opSetAttr(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &SetattrRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1128,7 +1154,8 @@ func (m *metadataManager) opSetAttr(conn net.Conn, p *Packet,
// Lookup request
func (m *metadataManager) opMetaLookup(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.LookupRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1162,7 +1189,8 @@ func (m *metadataManager) opMetaLookup(conn net.Conn, p *Packet,
}
func (m *metadataManager) opMetaExtentsAdd(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.AppendExtentKeyRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1199,7 +1227,8 @@ func (m *metadataManager) opMetaExtentsAdd(conn net.Conn, p *Packet,
// Append one extent with discard check
func (m *metadataManager) opMetaExtentAddWithCheck(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.AppendExtentKeyWithCheckRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1237,7 +1266,8 @@ func (m *metadataManager) opMetaExtentAddWithCheck(conn net.Conn, p *Packet,
}
func (m *metadataManager) opMetaExtentsList(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.GetExtentsRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1266,7 +1296,8 @@ func (m *metadataManager) opMetaExtentsList(conn net.Conn, p *Packet,
}
func (m *metadataManager) opMetaObjExtentsList(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.GetExtentsRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1293,7 +1324,8 @@ func (m *metadataManager) opMetaObjExtentsList(conn net.Conn, p *Packet,
}
func (m *metadataManager) opMetaExtentsDel(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
panic("not implemented yet")
// req := &proto.DelExtentKeyRequest{}
// if err = json.Unmarshal(p.Data, req); err != nil {
@ -1320,7 +1352,8 @@ func (m *metadataManager) opMetaExtentsDel(conn net.Conn, p *Packet,
}
func (m *metadataManager) opMetaExtentsTruncate(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &ExtentsTruncateReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1352,7 +1385,8 @@ func (m *metadataManager) opMetaExtentsTruncate(conn net.Conn, p *Packet,
}
func (m *metadataManager) opMetaClearInodeCache(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.ClearInodeCacheRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1379,7 +1413,8 @@ func (m *metadataManager) opMetaClearInodeCache(conn net.Conn, p *Packet,
// Delete a meta partition.
func (m *metadataManager) opDeleteMetaPartition(conn net.Conn,
p *Packet, remoteAddr string) (err error) {
p *Packet, remoteAddr string,
) (err error) {
req := &proto.DeleteMetaPartitionRequest{}
adminTask := &proto.AdminTask{
Request: req,
@ -1413,7 +1448,8 @@ func (m *metadataManager) opDeleteMetaPartition(conn net.Conn,
}
func (m *metadataManager) opUpdateMetaPartition(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := new(UpdatePartitionReq)
adminTask := &proto.AdminTask{
Request: req,
@ -1453,7 +1489,8 @@ func (m *metadataManager) opUpdateMetaPartition(conn net.Conn, p *Packet,
}
func (m *metadataManager) opLoadMetaPartition(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.MetaPartitionLoadRequest{}
adminTask := &proto.AdminTask{
Request: req,
@ -1488,7 +1525,8 @@ func (m *metadataManager) opLoadMetaPartition(conn net.Conn, p *Packet,
}
func (m *metadataManager) opDecommissionMetaPartition(conn net.Conn,
p *Packet, remoteAddr string) (err error) {
p *Packet, remoteAddr string,
) (err error) {
var reqData []byte
req := &proto.MetaPartitionDecommissionRequest{}
adminTask := &proto.AdminTask{
@ -1547,7 +1585,8 @@ func (m *metadataManager) opDecommissionMetaPartition(conn net.Conn,
}
func (m *metadataManager) opAddMetaPartitionRaftMember(conn net.Conn,
p *Packet, remoteAddr string) (err error) {
p *Packet, remoteAddr string,
) (err error) {
var reqData []byte
req := &proto.AddMetaPartitionRaftMemberRequest{}
adminTask := &proto.AdminTask{
@ -1616,7 +1655,8 @@ func (m *metadataManager) opAddMetaPartitionRaftMember(conn net.Conn,
}
func (m *metadataManager) opRemoveMetaPartitionRaftMember(conn net.Conn,
p *Packet, remoteAddr string) (err error) {
p *Packet, remoteAddr string,
) (err error) {
var reqData []byte
req := &proto.RemoveMetaPartitionRaftMemberRequest{}
adminTask := &proto.AdminTask{
@ -1694,7 +1734,8 @@ func (m *metadataManager) opRemoveMetaPartitionRaftMember(conn net.Conn,
}
func (m *metadataManager) opMetaBatchInodeGet(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.BatchInodeGetRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1721,7 +1762,8 @@ func (m *metadataManager) opMetaBatchInodeGet(conn net.Conn, p *Packet,
}
func (m *metadataManager) opMetaPartitionTryToLeader(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
mp, err := m.getPartition(p.PartitionID)
if err != nil {
goto errDeal
@ -1739,7 +1781,8 @@ errDeal:
}
func (m *metadataManager) opMetaDeleteInode(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.DeleteInodeRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1765,7 +1808,8 @@ func (m *metadataManager) opMetaDeleteInode(conn net.Conn, p *Packet,
}
func (m *metadataManager) opMetaBatchDeleteInode(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
var req *proto.DeleteInodeBatchRequest
if err = json.Unmarshal(p.Data, &req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -2239,7 +2283,8 @@ func (m *metadataManager) opListMultipart(conn net.Conn, p *Packet, remoteAddr s
// Handle OpMetaTxCreateInode inode.
func (m *metadataManager) opTxCreateInode(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.TxCreateInodeRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -2376,7 +2421,8 @@ func (m *metadataManager) opMetaGetInodeQuota(conn net.Conn, p *Packet, remote s
}
func (m *metadataManager) opMetaGetUniqID(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
req := &proto.GetUniqIDRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -2653,7 +2699,8 @@ func (m *metadataManager) checkAndPromoteVersion(volName string) (err error) {
}
func (m *metadataManager) opMultiVersionOp(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
// For ack to master
data := p.Data
m.responseAckOKToMaster(conn, p)

View File

@ -15,7 +15,6 @@
package metanode
import (
"github.com/cubefs/cubefs/storage"
"net"
"github.com/cubefs/cubefs/storage"
@ -89,7 +88,8 @@ func (m *metadataManager) IsForbiddenOp(mp MetaPartition, reqOp uint8) bool {
// The proxy is used during the leader change. When a leader of a partition changes, the proxy forwards the request to
// the new leader.
func (m *metadataManager) serveProxy(conn net.Conn, mp MetaPartition,
p *Packet) (ok bool) {
p *Packet,
) (ok bool) {
var (
mConn *net.TCPConn
leaderAddr string

View File

@ -1263,7 +1263,8 @@ func (mp *metaPartition) GetBaseConfig() MetaPartitionConfig {
// UpdatePartition updates the meta partition. TODO remove? no usage?
func (mp *metaPartition) UpdatePartition(req *UpdatePartitionReq,
resp *UpdatePartitionResp) (err error) {
resp *UpdatePartitionResp,
) (err error) {
reqData, err := json.Marshal(req)
if err != nil {
resp.Status = proto.TaskFailed

View File

@ -40,8 +40,8 @@ const (
var extentsFileHeader = []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08}
/// start metapartition delete extents work
///
// / start metapartition delete extents work
// /
func (mp *metaPartition) startToDeleteExtents() {
fileList := synclist.New()
go mp.appendDelExtentsToFile(fileList)

View File

@ -54,12 +54,38 @@ func (mp *metaPartition) startFreeList() (err error) {
return
}
go mp.updateVolWorker()
go mp.deleteWorker()
mp.startToDeleteExtents()
return
}
UpdateVolumeView(dataView *proto.DataPartitionsView, volumeView *proto.SimpleVolView) {
func (mp *metaPartition) UpdateVolumeView(dataView *proto.DataPartitionsView, volumeView *proto.SimpleVolView) {
convert := func(view *proto.DataPartitionsView) *DataPartitionsView {
newView := &DataPartitionsView{
DataPartitions: make([]*DataPartition, len(view.DataPartitions)),
}
for i := 0; i < len(view.DataPartitions); i++ {
if len(view.DataPartitions[i].Hosts) < 1 {
log.LogErrorf("action[UpdateVolumeView] dp id(%v) is invalid, DataPartitionResponse detail[%v]",
view.DataPartitions[i].PartitionID, view.DataPartitions[i])
continue
}
newView.DataPartitions[i] = &DataPartition{
PartitionID: view.DataPartitions[i].PartitionID,
Status: view.DataPartitions[i].Status,
Hosts: view.DataPartitions[i].Hosts,
ReplicaNum: view.DataPartitions[i].ReplicaNum,
IsDiscard: view.DataPartitions[i].IsDiscard,
}
}
return newView
}
mp.vol.UpdatePartitions(convert(dataView))
mp.vol.volDeleteLockTime = volumeView.DeleteLockTime
}
func (mp *metaPartition) updateVolView(convert func(view *proto.DataPartitionsView) *DataPartitionsView) (err error) {
volName := mp.config.VolName
dataView, err := masterClient.ClientAPI().EncodingGzip().GetDataPartitions(volName)
if err != nil {
@ -88,7 +114,7 @@ func (mp *metaPartition) updateVolWorker() {
}
for i := 0; i < len(view.DataPartitions); i++ {
if len(view.DataPartitions[i].Hosts) < 1 {
log.LogErrorf("action[UpdateVolumeView] dp id(%v) is invalid, DataPartitionResponse detail[%v]",
log.LogErrorf("updateVolWorker dp id(%v) is invalid, DataPartitionResponse detail[%v]",
view.DataPartitions[i].PartitionID, view.DataPartitions[i])
continue
}
@ -102,8 +128,16 @@ func (mp *metaPartition) updateVolWorker() {
}
return newView
}
mp.vol.UpdatePartitions(convert(dataView))
mp.vol.volDeleteLockTime = volumeView.DeleteLockTime
mp.updateVolView(convert)
for {
select {
case <-mp.stopC:
t.Stop()
return
case <-t.C:
mp.updateVolView(convert)
}
}
}
const (
@ -186,7 +220,8 @@ func (mp *metaPartition) deleteWorker() {
// delete Extents by Partition,and find all successDelete inode
func (mp *metaPartition) batchDeleteExtentsByPartition(partitionDeleteExtents map[uint64][]*proto.DelExtentParam,
allInodes []*Inode) (shouldCommit []*Inode, shouldPushToFreeList []*Inode) {
allInodes []*Inode,
) (shouldCommit []*Inode, shouldPushToFreeList []*Inode) {
occurErrors := make(map[uint64]error)
shouldCommit = make([]*Inode, 0, len(allInodes))
shouldPushToFreeList = make([]*Inode, 0)

View File

@ -15,17 +15,42 @@
package metanode
import (
"fmt"
"os"
"path"
"strings"
"sync"
"testing"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/util"
"github.com/cubefs/cubefs/util/fileutil"
"github.com/stretchr/testify/require"
)
var VolNameForFreeListTest = "TestForFreeList"
func newPartitionForFreeList(conf *MetaPartitionConfig, manager *metadataManager) (mp *metaPartition) {
mp = &metaPartition{
config: conf,
dentryTree: NewBtree(),
inodeTree: NewBtree(),
extendTree: NewBtree(),
multipartTree: NewBtree(),
stopC: make(chan bool),
storeChan: make(chan *storeMsg, 100),
freeList: newFreeList(),
extDelCh: make(chan []proto.ExtentKey, defaultDelExtentsCnt),
extReset: make(chan struct{}),
vol: NewVol(),
manager: manager,
}
mp.config.Cursor = 0
mp.config.End = 100000
mp.uidManager = NewUidMgr(conf.VolName, mp.config.PartitionId)
mp.mqMgr = NewQuotaManager(conf.VolName, mp.config.PartitionId)
return mp
}
func TestPersistInodesFreeList(t *testing.T) {
rootDir, err := os.MkdirTemp("", "")
defer os.RemoveAll(rootDir)
@ -42,9 +67,10 @@ func TestPersistInodesFreeList(t *testing.T) {
fileName := path.Join(config.RootDir, DeleteInodeFileExtension)
oldIno, err := fileutil.Stat(fileName)
require.NoError(t, err)
t.Logf("Persist many inodes")
const persistBatchCount = 50000
const testCount = DeleteInodeFileRollingSize / 8
const persistBatchCount = 500000
unitSize := len(fmt.Sprintf("%v\n", 1000000))
testCount := DeleteInodeFileRollingSize / unitSize
t.Logf("Persist many inodes, unitSize %d, total %d", unitSize, testCount)
inodes := make([]uint64, 0, persistBatchCount)
for i := 0; i < persistBatchCount; i++ {
inodes = append(inodes, uint64(i)+1000000)

View File

@ -299,7 +299,7 @@ func (mp *metaPartition) Apply(command []byte, index uint64) (resp interface{},
}
err = mp.fsmSetXAttr(extend)
case opFSMLockDir:
var req = &proto.LockDirRequest{}
req := &proto.LockDirRequest{}
if err = json.Unmarshal(msg.V, req); err != nil {
return
}

View File

@ -75,7 +75,8 @@ func (mp *metaPartition) decommissionPartition() (err error) {
}
func (mp *metaPartition) fsmUpdatePartition(end uint64) (status uint8,
err error) {
err error,
) {
status = proto.OpOk
oldEnd := mp.config.End
mp.config.End = end
@ -177,7 +178,6 @@ func (mp *metaPartition) delOldExtentFile(buf []byte) (err error) {
return
}
//
func (mp *metaPartition) setExtentDeleteFileCursor(buf []byte) (err error) {
str := string(buf)
var (

View File

@ -69,7 +69,8 @@ func (mp *metaPartition) fsmTxCreateDentry(txDentry *TxDentry) (status uint8) {
// Insert a dentry into the dentry tree.
func (mp *metaPartition) fsmCreateDentry(dentry *Dentry,
forceUpdate bool) (status uint8) {
forceUpdate bool,
) (status uint8) {
status = proto.OpOk
var parIno *Inode
if !forceUpdate {
@ -360,7 +361,8 @@ func (mp *metaPartition) fsmTxUpdateDentry(txUpDateDentry *TxUpdateDentry) (resp
}
func (mp *metaPartition) fsmUpdateDentry(dentry *Dentry) (
resp *DentryResponse) {
resp *DentryResponse,
) {
resp = NewDentryResponse()
resp.Status = proto.OpOk
mp.dentryTree.CopyFind(dentry, func(item BtreeItem) {
@ -446,7 +448,6 @@ func (mp *metaPartition) readDir(req *ReadDirReq) (resp *ReadDirResp) {
// else if req.Marker != "" and req.Limit == 0, return dentries from pid:name to pid+1
// else if req.Marker == "" and req.Limit != 0, return dentries from pid with limit count
// else if req.Marker != "" and req.Limit != 0, return dentries from pid:marker to pid:xxxx with limit count
//
func (mp *metaPartition) readDirLimit(req *ReadDirLimitReq) (resp *ReadDirLimitResp) {
log.LogDebugf("action[readDirLimit] mp[%v] req %v", mp.config.PartitionId, req)
resp = &ReadDirLimitResp{}

View File

@ -15,11 +15,11 @@
package metanode
import (
"fmt"
"math"
"strconv"
"strings"
"time"
"fmt"
"math"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/util/log"
@ -43,7 +43,7 @@ func (mp *metaPartition) fsmLockDir(req *proto.LockDirRequest) (resp *proto.Lock
log.LogDebugf("fsmLockDir ino=%v, lockId=%d, submitTime=%v, lease=%d\n", ino, lockId, submitTime, lease)
var newExtend = NewExtend(ino)
newExtend := NewExtend(ino)
treeItem := mp.extendTree.CopyGet(newExtend)
var firstLock bool = false
@ -72,7 +72,7 @@ func (mp *metaPartition) fsmLockDir(req *proto.LockDirRequest) (resp *proto.Lock
lockId = int64(uu_id.ID())
lockIdStr = strconv.Itoa(int(lockId))
value = lockIdStr + "|" + validTimeStr
newExtend.Put([]byte("dir_lock"), []byte(value))
newExtend.Put([]byte("dir_lock"), []byte(value), 0)
mp.extendTree.ReplaceOrInsert(newExtend, true)
} else {
log.LogDebugf("fsmLockDir oldValue=%s\n", oldValue)
@ -103,7 +103,7 @@ func (mp *metaPartition) fsmLockDir(req *proto.LockDirRequest) (resp *proto.Lock
}
existExtend.Remove([]byte("dir_lock"))
newExtend.Put([]byte("dir_lock"), []byte(value))
newExtend.Put([]byte("dir_lock"), []byte(value), 0)
existExtend.Merge(newExtend, true)
}

View File

@ -44,11 +44,12 @@ func (s *MetaItem) MarshalJson() ([]byte, error) {
// MarshalBinary marshals MetaItem to binary data.
// Binary frame structure:
// +------+----+------+------+------+------+
// | Item | Op | LenK | K | LenV | V |
// +------+----+------+------+------+------+
// | byte | 4 | 4 | LenK | 4 | LenV |
// +------+----+------+------+------+------+
//
// +------+----+------+------+------+------+
// | Item | Op | LenK | K | LenV | V |
// +------+----+------+------+------+------+
// | byte | 4 | 4 | LenK | 4 | LenV |
// +------+----+------+------+------+------+
func (s *MetaItem) MarshalBinary() (result []byte, err error) {
buff := bytes.NewBuffer(make([]byte, 0))
buff.Grow(4 + len(s.K) + len(s.V))
@ -78,11 +79,12 @@ func (s *MetaItem) UnmarshalJson(data []byte) error {
// MarshalBinary unmarshal this MetaItem entity from binary data.
// Binary frame structure:
// +------+----+------+------+------+------+
// | Item | Op | LenK | K | LenV | V |
// +------+----+------+------+------+------+
// | byte | 4 | 4 | LenK | 4 | LenV |
// +------+----+------+------+------+------+
//
// +------+----+------+------+------+------+
// | Item | Op | LenK | K | LenV | V |
// +------+----+------+------+------+------+
// | byte | 4 | 4 | LenK | 4 | LenV |
// +------+----+------+------+------+------+
func (s *MetaItem) UnmarshalBinary(raw []byte) (err error) {
var (
lenK uint32

View File

@ -25,7 +25,6 @@ import (
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/util/auditlog"
"github.com/cubefs/cubefs/util/errors"
"github.com/cubefs/cubefs/util/log"
)
func (mp *metaPartition) TxCreateDentry(req *proto.TxCreateDentryRequest, p *Packet, remoteAddr string) (err error) {
@ -286,10 +285,7 @@ func (mp *metaPartition) DeleteDentry(req *DeleteDentryReq, p *Packet, remoteAdd
return
}
}
dentry := &Dentry{
ParentId: req.ParentID,
Name: req.Name,
}
dentry.setVerSeq(req.Verseq)
log.LogDebugf("action[DeleteDentry] den param(%v)", dentry)

View File

@ -22,7 +22,8 @@ import (
)
func (mp *metaPartition) batchSetInodeQuota(req *proto.BatchSetMetaserverQuotaReuqest,
resp *proto.BatchSetMetaserverQuotaResponse) (err error) {
resp *proto.BatchSetMetaserverQuotaResponse,
) (err error) {
if len(req.Inodes) == 0 {
return nil
}
@ -45,7 +46,8 @@ func (mp *metaPartition) batchSetInodeQuota(req *proto.BatchSetMetaserverQuotaRe
}
func (mp *metaPartition) batchDeleteInodeQuota(req *proto.BatchDeleteMetaserverQuotaReuqest,
resp *proto.BatchDeleteMetaserverQuotaResponse) (err error) {
resp *proto.BatchDeleteMetaserverQuotaResponse,
) (err error) {
if len(req.Inodes) == 0 {
return nil
}

View File

@ -1053,7 +1053,8 @@ func (mp *metaPartition) storeTxInfo(rootDir string, sm *storeMsg) (crc uint32,
}
func (mp *metaPartition) storeInode(rootDir string,
sm *storeMsg) (crc uint32, err error) {
sm *storeMsg,
) (crc uint32, err error) {
filename := path.Join(rootDir, inodeFile)
fp, err := os.OpenFile(filename, os.O_RDWR|os.O_TRUNC|os.O_APPEND|os.
O_CREATE, 0o755)
@ -1112,7 +1113,8 @@ func (mp *metaPartition) storeInode(rootDir string,
}
func (mp *metaPartition) storeDentry(rootDir string,
sm *storeMsg) (crc uint32, err error) {
sm *storeMsg,
) (crc uint32, err error) {
filename := path.Join(rootDir, dentryFile)
fp, err := os.OpenFile(filename, os.O_RDWR|os.O_TRUNC|os.O_APPEND|os.
O_CREATE, 0o755)

View File

@ -101,7 +101,8 @@ func (m *MetaNode) serveConn(conn net.Conn, stopC chan uint8) {
}
func (m *MetaNode) handlePacket(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
// Handle request
err = m.metadataManager.HandleMetadataOperation(conn, p, remoteAddr)
return

View File

@ -300,7 +300,7 @@ type TransactionManager struct {
txIdAlloc *TxIDAllocator
txTree *BTree
txProcessor *TransactionProcessor
blacklist *util.SyncSet
blacklist *util.Set
opLimiter *rate.Limiter
sync.RWMutex
}
@ -334,7 +334,7 @@ func NewTransactionManager(txProcessor *TransactionProcessor) *TransactionManage
txIdAlloc: newTxIDAllocator(),
txTree: NewBtree(),
txProcessor: txProcessor,
blacklist: util.NewSyncSet(),
blacklist: util.NewSet(),
opLimiter: rate.NewLimiter(rate.Inf, 128),
}
return txMgr

View File

@ -560,7 +560,8 @@ func (o *ObjectNode) listPartsHandler(w http.ResponseWriter, r *http.Request) {
}
func (o *ObjectNode) checkReqParts(param *RequestParam, reqParts *CompleteMultipartUploadRequest, multipartInfo *proto.MultipartInfo) (
discardedPartInodes map[uint64]uint16, committedPartInfo *proto.MultipartInfo, err error) {
discardedPartInodes map[uint64]uint16, committedPartInfo *proto.MultipartInfo, err error,
) {
if len(reqParts.Parts) <= 0 {
err = InvalidPart
log.LogErrorf("checkReqParts: upload part is empty: requestID(%v) volume(%v)", GetRequestID(param.r), param.Bucket())

View File

@ -1997,7 +1997,7 @@ func (v *Volume) recursiveMakeDirectory(path string) (partentIno uint64, err err
}
if err == syscall.ENOENT {
var info *proto.InodeInfo
info, err = v.mw.Create_ll(partentIno, pathItem.Name, uint32(DefaultDirMode), 0, 0, nil, path[:pathIterator.cursor])
info, err = v.mw.Create_ll(partentIno, pathItem.Name, uint32(DefaultDirMode), 0, 0, nil, path[:pathIterator.cursor], false)
if err != nil && err == syscall.EEXIST {
existInode, mode, e := v.mw.Lookup_ll(partentIno, pathItem.Name)
if e != nil {
@ -2081,7 +2081,8 @@ func (v *Volume) lookupDirectories(dirs []string, autoCreate bool) (inode uint64
}
func (v *Volume) listFilesV1(prefix, marker, delimiter string, maxKeys uint64, onlyObject bool) (infos []*FSFileInfo,
prefixes Prefixes, nextMarker string, err error) {
prefixes Prefixes, nextMarker string, err error,
) {
prefixMap := PrefixMap(make(map[string]struct{}))
parentId, dirs, err := v.findParentId(prefix)
@ -2128,7 +2129,8 @@ func (v *Volume) listFilesV1(prefix, marker, delimiter string, maxKeys uint64, o
}
func (v *Volume) listFilesV2(prefix, startAfter, contToken, delimiter string, maxKeys uint64) (infos []*FSFileInfo,
prefixes Prefixes, nextMarker string, err error) {
prefixes Prefixes, nextMarker string, err error,
) {
prefixMap := PrefixMap(make(map[string]struct{}))
var marker string
@ -2234,7 +2236,8 @@ func (v *Volume) findParentId(prefix string) (inode uint64, prefixDirs []string,
// that match the prefix and delimiter criteria. Stop when the number of matches reaches a threshold
// or all files and directories are scanned.
func (v *Volume) recursiveScan(fileInfos []*FSFileInfo, prefixMap PrefixMap, parentId, maxKeys, readLimit, rc uint64, dirs []string,
prefix, marker, delimiter string, onlyObject, firstEnter bool) ([]*FSFileInfo, PrefixMap, string, uint64, error) {
prefix, marker, delimiter string, onlyObject, firstEnter bool,
) ([]*FSFileInfo, PrefixMap, string, uint64, error) {
var err error
var nextMarker string
var lastKey string
@ -2469,7 +2472,8 @@ func (v *Volume) updateETag(inode uint64, size int64, mt time.Time) (etagValue E
}
func (v *Volume) ListMultipartUploads(prefix, delimiter, keyMarker string, multipartIdMarker string, maxUploads uint64) (
uploads []*FSUpload, nextMarker, nextMultipartIdMarker string, isTruncated bool, prefixes []string, err error) {
uploads []*FSUpload, nextMarker, nextMultipartIdMarker string, isTruncated bool, prefixes []string, err error,
) {
sessions, err := v.mw.ListMultipart_ll(prefix, delimiter, keyMarker, multipartIdMarker, maxUploads)
if err != nil || len(sessions) == 0 {
return

View File

@ -39,8 +39,10 @@ const (
)
// if more s3 api is supported by policy, need extend bucketApiList, objectApiList
var bucketApiList = SliceString{LIST_OBJECTS, LIST_OBJECTS_V2, HEAD_BUCKET, DELETE_BUCKET, LIST_MULTIPART_UPLOADS, GET_BUCKET_LOCATION, GET_OBJECT_LOCK_CFG, PUT_OBJECT_LOCK_CFG}
var objectApiList = SliceString{GET_OBJECT, HEAD_OBJECT, DELETE_OBJECT, PUT_OBJECT, POST_OBJECT, INITIALE_MULTIPART_UPLOAD, UPLOAD_PART, UPLOAD_PART_COPY, COMPLETE_MULTIPART_UPLOAD, COPY_OBJECT, ABORT_MULTIPART_UPLOAD, LIST_PARTS, BATCH_DELETE, GET_OBJECT_RETENTION}
var (
bucketApiList = SliceString{LIST_OBJECTS, LIST_OBJECTS_V2, HEAD_BUCKET, DELETE_BUCKET, LIST_MULTIPART_UPLOADS, GET_BUCKET_LOCATION, GET_OBJECT_LOCK_CFG, PUT_OBJECT_LOCK_CFG}
objectApiList = SliceString{GET_OBJECT, HEAD_OBJECT, DELETE_OBJECT, PUT_OBJECT, POST_OBJECT, INITIALE_MULTIPART_UPLOAD, UPLOAD_PART, UPLOAD_PART_COPY, COMPLETE_MULTIPART_UPLOAD, COPY_OBJECT, ABORT_MULTIPART_UPLOAD, LIST_PARTS, BATCH_DELETE, GET_OBJECT_RETENTION}
)
type SliceString []string

View File

@ -53,7 +53,7 @@ func (key Key) IsValid() bool {
return false
}
// encodes Key to JSON data.
// encodes Key to JSON data.
func (key Key) MarshalJSON() ([]byte, error) {
if !key.IsValid() {
return nil, fmt.Errorf("unknown key: %v", key)
@ -73,7 +73,7 @@ func (key Key) Name() string {
return strings.TrimPrefix(keyString, "s3:")
}
// decodes string data to Key.
// decodes string data to Key.
func (key *Key) UnmarshalText(data []byte) error {
var s string
if err := json.Unmarshal(data, &s); err != nil {
@ -89,7 +89,7 @@ func (key *Key) UnmarshalText(data []byte) error {
return nil
}
// decodes JSON data to Key.
// decodes JSON data to Key.
func (key *Key) UnmarshalJSON(data []byte) error {
var s string
if err := json.Unmarshal(data, &s); err != nil {
@ -128,9 +128,10 @@ func (set KeySet) AddAll(keys KeySet) {
// returns a key set contains difference of two key set.
// Example:
// keySet1 := ["one", "two", "three"]
// keySet2 := ["two", "four", "three"]
// keySet1.Difference(keySet2) == ["one"]
//
// keySet1 := ["one", "two", "three"]
// keySet2 := ["two", "four", "three"]
// keySet1.Difference(keySet2) == ["one"]
func (set KeySet) Difference(sset KeySet) KeySet {
nset := make(KeySet)

View File

@ -67,7 +67,7 @@ func (s *Statement) match(apiName string, uid string, conditionCheck map[string]
return true
}
//----------------------------------------------------------------------------------------------------------------
// ----------------------------------------------------------------------------------------------------------------
func (s *Statement) matchAction(apiName string) bool {
log.LogDebug("start to match action")
switch s.Action.(type) {
@ -106,7 +106,7 @@ func (actions ActionType) match(apiToMatch string) bool {
return false
}
//----------------------------------------------------------------------------------------------------------------
// ----------------------------------------------------------------------------------------------------------------
func (s *Statement) matchPrincipal(uid string) bool {
switch s.Principal.(type) {
case string: // "*" or "123"
@ -146,7 +146,7 @@ func (p PrincipalType) match(uid string) bool {
}
}
//----------------------------------------------------------------------------------------------------------------
// ----------------------------------------------------------------------------------------------------------------
func (s *Statement) matchResource(apiName string, keyname interface{}) bool {
if IsBucketApi(apiName) {
return s.matchBucketInResource()
@ -237,7 +237,7 @@ func makeRegexPattern(raw string) string {
return pattern
}
//----------------------------------------------------------------------------------------------------------------
// ----------------------------------------------------------------------------------------------------------------
func (s *Statement) matchCondition(conditionCheck map[string]string) bool {
// condition is optional
if s.Condition == nil {

View File

@ -106,7 +106,7 @@ func (s *Statement) isEffectValid() bool {
return false
}
// "Principal": "*" or "Principal" : {"AWS":"111122223333"} or "Principal" : {"AWS":["111122223333","444455556666"]}
// "Principal": "*" or "Principal" : {"AWS":"111122223333"} or "Principal" : {"AWS":["111122223333","444455556666"]}
func (s *Statement) isPrincipalValid() bool {
// principal: uid must be "*" or uint32, and can't be 0
switch s.Principal.(type) {

View File

@ -50,7 +50,7 @@ func (op stringLikeOp) evaluate(values map[string]string) bool {
return true
}
// returns condition key which is used by this condition operation.
// returns condition key which is used by this condition operation.
func (op stringLikeOp) keys() KeySet {
keys := make(KeySet)
for key := range op.m {
@ -128,7 +128,7 @@ func (op stringNotLikeOp) operator() operator {
return stringNotLike
}
// returns new StringNotLike operation.
// returns new StringNotLike operation.
func newStringNotLikeOp(m map[Key]ValueSet) (Operation, error) {
newMap, err := parseMap(m, stringNotLike)
if err != nil {
@ -138,7 +138,7 @@ func newStringNotLikeOp(m map[Key]ValueSet) (Operation, error) {
return NewStringNotLikeOp(newMap)
}
// returns new StringNotLike operation.
// returns new StringNotLike operation.
func NewStringNotLikeOp(m map[Key]StringSet) (Operation, error) {
return &stringNotLikeOp{stringLikeOp{m}}, nil
}

View File

@ -22,7 +22,7 @@ import (
type StringSet map[string]struct{}
// returns StringSet as string slice.
// returns StringSet as string slice.
func (set StringSet) ToSlice() []string {
keys := make([]string, 0, len(set))
for k := range set {
@ -147,7 +147,7 @@ func NewStringSet() StringSet {
return make(StringSet)
}
// creates new string set with given string values.
// creates new string set with given string values.
func CreateStringSet(sl ...string) StringSet {
set := make(StringSet)
for _, k := range sl {

View File

@ -51,7 +51,7 @@ func (v Value) GetInt() (int, error) {
return v.i, err
}
// gets stored string value.
// gets stored string value.
func (v Value) GetString() (string, error) {
var err error
@ -81,12 +81,12 @@ func (v Value) MarshalJSON() ([]byte, error) {
}
}
// stores bool value.
// stores bool value.
func (v *Value) StoreBool(b bool) {
*v = Value{t: reflect.Bool, b: b}
}
// stores int value.
// stores int value.
func (v *Value) StoreInt(i int) {
*v = Value{t: reflect.Int, i: i}
}
@ -154,7 +154,7 @@ func NewStringValue(s string) Value {
return *value
}
// unique list of values.
// unique list of values.
type ValueSet map[Value]struct{}
// adds given value to value set.
@ -162,7 +162,7 @@ func (set ValueSet) Add(value Value) {
set[value] = struct{}{}
}
// encodes ValueSet to JSON data.
// encodes ValueSet to JSON data.
func (set ValueSet) MarshalJSON() ([]byte, error) {
values := []Value{}
for k := range set {
@ -206,7 +206,7 @@ func (set *ValueSet) UnmarshalJSON(data []byte) error {
return nil
}
// returns new value set containing given values.
// returns new value set containing given values.
func NewValueSet(values ...Value) ValueSet {
set := make(ValueSet)

View File

@ -304,8 +304,10 @@ func ParseUserDefinedMetadata(header http.Header) map[string]string {
}
// validate Cache-Control
var cacheControlDir = []string{"public", "private", "no-cache", "no-store", "no-transform", "must-revalidate", "proxy-revalidate"}
var maxAgeRegexp = regexp.MustCompile("^((max-age)|(s-maxage))=[1-9][0-9]*$")
var (
cacheControlDir = []string{"public", "private", "no-cache", "no-store", "no-transform", "must-revalidate", "proxy-revalidate"}
maxAgeRegexp = regexp.MustCompile("^((max-age)|(s-maxage))=[1-9][0-9]*$")
)
func ValidateCacheControl(cacheControl string) bool {
cacheDirs := strings.Split(cacheControl, ",")

View File

@ -19,6 +19,8 @@ import (
"fmt"
"strconv"
"time"
"github.com/cubefs/cubefs/util"
)
// api
@ -235,7 +237,7 @@ const (
QuotaGet = "/quota/get"
// QuotaBatchModifyPath = "/quota/batchModifyPath"
QuotaListAll = "/quota/listAll"
//trash
// trash
AdminSetTrashInterval = "/vol/setTrashInterval"
// s3 qos api
@ -1108,7 +1110,7 @@ type SimpleVolView struct {
// multi version snapshot
LatestVer uint64
Forbidden bool
DisableAuditLog bool
DisableAuditLog bool
DeleteExecTime time.Time
DpRepairBlockSize uint64
}

View File

@ -375,6 +375,7 @@ type BatchUnlinkInodeRequest struct {
VolName string `json:"vol"`
PartitionID uint64 `json:"pid"`
Inodes []uint64 `json:"inos"`
FullPaths []string `json:"fullPaths"`
}
// UnlinkInodeResponse defines the response to the request of unlinking an inode.
@ -403,6 +404,7 @@ type BatchEvictInodeRequest struct {
VolName string `json:"vol"`
PartitionID uint64 `json:"pid"`
Inodes []uint64 `json:"inos"`
FullPaths []string `json:"fullPaths"`
}
// CreateDentryRequest defines the request to create a dentry.

View File

@ -328,11 +328,11 @@ type MountOptions struct {
RequestTimeout int64
MinWriteAbleDataPartitionCnt int
FileSystemName string
//TrashInterval int64
// TrashInterval int64
TrashDeleteExpiredDirGoroutineLimit int64
TrashRebuildGoroutineLimit int64
VerReadSeq uint64
VerReadSeq uint64
// disable mount subtype
DisableMountSubtype bool
}

View File

@ -130,7 +130,7 @@ const (
OpLcNodeScan uint8 = 0x56
OpLcNodeSnapshotVerDel uint8 = 0x5B
//backUp
// backUp
OpBatchLockNormalExtent uint8 = 0x57
OpBatchUnlockNormalExtent uint8 = 0x58
OpBackupRead uint8 = 0x59
@ -257,9 +257,8 @@ const (
OpReadRepairExtentAgain uint8 = 0xEF
// io speed limit
OpLimitedIoErr uint8 = 0xB1
OpReadRepairExtentAgain uint8 = 0xEF
OpStoreClosed uint8 = 0xB2
OpLimitedIoErr uint8 = 0xB1
OpStoreClosed uint8 = 0xB2
)
const (

View File

@ -80,7 +80,8 @@ type partition struct {
// ChangeMember submits member change event and information to raft log.
func (p *partition) ChangeMember(changeType proto.ConfChangeType, peer proto.Peer, context []byte) (
resp interface{}, err error) {
resp interface{}, err error,
) {
if !p.IsRaftLeader() {
err = raft.ErrNotLeader
return

View File

@ -205,7 +205,8 @@ func (ft *FollowerTransport) Write(p *FollowerPacket) {
}
func NewReplProtocol(inConn net.Conn, prepareFunc func(p *Packet) error,
operatorFunc func(p *Packet, c net.Conn) error, postFunc func(p *Packet) error) *ReplProtocol {
operatorFunc func(p *Packet, c net.Conn) error, postFunc func(p *Packet) error,
) *ReplProtocol {
rp := new(ReplProtocol)
rp.packetList = list.New()
rp.ackCh = make(chan struct{}, RequestChanSize)

View File

@ -22,8 +22,6 @@ import (
"sync/atomic"
"time"
"github.com/cubefs/cubefs/util/stat"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/sdk/data/wrapper"
"github.com/cubefs/cubefs/sdk/meta"
@ -463,7 +461,7 @@ func (eh *ExtentHandler) appendExtentKey() (err error) {
return
}
discard := eh.stream.extents.Append(eh.key, true)
err = eh.stream.client.appendExtentKey(eh.stream.parentInode, eh.inode, *eh.key, discard)
status, err := eh.stream.client.appendExtentKey(eh.stream.parentInode, eh.inode, *eh.key, discard)
ekey := *eh.key
doAppend := func() (err error) {
@ -484,7 +482,7 @@ func (eh *ExtentHandler) appendExtentKey() (err error) {
eh.dirty = false
eh.lastKey = *eh.key
log.LogDebugf("action[appendExtentKey] status %v, needUpdateVer %v, eh{%v}", status, eh.stream.needUpdateVer, eh)
return
return nil
}
// Due to the asynchronous synchronization of version numbers, the extent cache version of the client is updated first before being written to the meta.
// However, it is possible for the client version to lag behind the meta version, resulting in partial inconsistencies in judgment.

View File

@ -70,7 +70,8 @@ func NewWritePacket(inode uint64, fileOffset, storeMode int) *Packet {
// NewOverwritePacket returns a new overwrite packet.
func NewOverwriteByAppendPacket(dp *wrapper.DataPartition, extentID uint64, extentOffset int,
inode uint64, fileOffset int, direct bool, op uint8) *Packet {
inode uint64, fileOffset int, direct bool, op uint8,
) *Packet {
p := new(Packet)
p.PartitionID = dp.PartitionID
p.Magic = proto.ProtoMagic

View File

@ -144,7 +144,8 @@ func (s *DefaultRandomSelector) getLocalLeaderDataPartition(exclude map[string]s
}
func (s *DefaultRandomSelector) getRandomDataPartition(partitions []*DataPartition, exclude map[string]struct{}) (
dp *DataPartition) {
dp *DataPartition,
) {
length := len(partitions)
if length == 0 {
return nil

View File

@ -247,17 +247,6 @@ func (api *AdminAPI) UnDeleteVolume(volName, authKey string, status bool) (err e
return
}
func (api *AdminAPI) UnDeleteVolume(volName, authKey string, status bool) (err error) {
var request = newAPIRequest(http.MethodGet, proto.AdminDeleteVol)
request.addParam("name", volName)
request.addParam("authKey", authKey)
request.addParam("delete", strconv.FormatBool(false))
if _, err = api.mc.serveRequest(request); err != nil {
return
}
return
}
func (api *AdminAPI) UpdateVolume(
vv *proto.SimpleVolView,
txTimeout int64,
@ -406,15 +395,6 @@ func (api *AdminAPI) SetVolumeForbidden(volName string, forbidden bool) (err err
_, err = api.mc.serveRequest(request)
return
}
func (api *AdminAPI) SetVolumeForbidden(volName string, forbidden bool) (err error) {
request := newAPIRequest(http.MethodGet, proto.AdminVolForbidden)
request.addParam("name", volName)
request.addParam("forbidden", strconv.FormatBool(forbidden))
if _, err = api.mc.serveRequest(request); err != nil {
return
}
return
}
func (api *AdminAPI) SetVolumeAuditLog(volName string, enable bool) (err error) {
request := newRequest(post, proto.AdminVolEnableAuditLog).Header(api.h)
@ -769,7 +749,7 @@ func (api *AdminAPI) AbortDiskDecommission(addr string, disk string) (err error)
}
func (api *AdminAPI) SetClusterDecommissionLimit(limit int32) (err error) {
var request = newAPIRequest(http.MethodPost, proto.AdminUpdateDecommissionLimit)
request := newAPIRequest(http.MethodPost, proto.AdminUpdateDecommissionLimit)
request.addParam("decommissionLimit", strconv.FormatInt(int64(limit), 10))
if _, err = api.mc.serveRequest(request); err != nil {
return
@ -779,7 +759,7 @@ func (api *AdminAPI) SetClusterDecommissionLimit(limit int32) (err error) {
func (api *AdminAPI) QueryDecommissionToken() (status []proto.DecommissionTokenStatus, err error) {
var buf []byte
var request = newAPIRequest(http.MethodGet, proto.AdminQueryDecommissionToken)
request := newAPIRequest(http.MethodGet, proto.AdminQueryDecommissionToken)
if buf, err = api.mc.serveRequest(request); err != nil {
return
}
@ -807,4 +787,4 @@ func (api AdminAPI) SetDecommissionDiskLimit(limit uint32) (err error) {
err = api.mc.request(request)
return
}
}

View File

@ -109,6 +109,19 @@ func newRequest(method string, path string) *request {
return req
}
func newAPIRequest(method string, path string) *request {
req := &request{
method: method,
path: path,
params: make(map[string]string),
header: make(map[string]string),
}
req.header["User-Agent"] = ReqHeaderUA
return req
}
func mergeHeader(headers map[string]string, added ...string) map[string]string {
if len(added)%2 == 1 {
added = added[:len(added)-1]

View File

@ -134,7 +134,8 @@ func (mw *MetaWrapper) Create_ll(parentID uint64, name string, mode, uid, gid ui
}
func (mw *MetaWrapper) txCreate_ll(parentID uint64, name string, mode, uid, gid uint32, target []byte, txType uint32,
fullPath string, ignoreExist bool) (info *proto.InodeInfo, err error) {
fullPath string, ignoreExist bool,
) (info *proto.InodeInfo, err error) {
var (
status int
// err error
@ -226,7 +227,8 @@ create_dentry:
}
func (mw *MetaWrapper) create_ll(parentID uint64, name string, mode, uid, gid uint32, target []byte,
fullPath string, ignoreExist bool) (*proto.InodeInfo, error) {
fullPath string, ignoreExist bool,
) (*proto.InodeInfo, error) {
var (
status int
err error
@ -358,7 +360,7 @@ func (mw *MetaWrapper) Lookup_ll(parentID uint64, name string) (inode uint64, mo
if err != nil || status != statusOK {
return 0, 0, statusToErrno(status)
}
//only save dir
// only save dir
if proto.IsDir(mode) {
mw.AddInoInfoCache(inode, parentID, name)
}
@ -560,7 +562,7 @@ func (mw *MetaWrapper) BatchGetXAttr(inodes []uint64, keys []string) ([]*proto.X
func (mw *MetaWrapper) shouldNotMoveToTrash(parentMP *MetaPartition, parentIno uint64, entry string, isDir bool) (error, bool) {
log.LogDebugf("action[shouldNotMoveToTrash]: parentIno(%v) entry(%v)", parentIno, entry)
status, inode, mode, err := mw.lookup(parentMP, parentIno, entry)
status, inode, mode, err := mw.lookup(parentMP, parentIno, entry, mw.LastVerSeq)
if err != nil || status != statusOK {
return statusToErrno(status), false
}
@ -569,7 +571,7 @@ func (mw *MetaWrapper) shouldNotMoveToTrash(parentMP *MetaPartition, parentIno u
log.LogErrorf("shouldNotMoveToTrash: No inode partition, parentID(%v) name(%v) ino(%v)", parentIno, entry, inode)
return syscall.EAGAIN, false
}
status, info, err := mw.iget(mp, inode)
status, info, err := mw.iget(mp, inode, mw.LastVerSeq)
if err != nil || status != statusOK {
return statusToErrno(status), false
}
@ -611,11 +613,11 @@ func (mw *MetaWrapper) shouldNotMoveToTrash(parentMP *MetaPartition, parentIno u
if strings.Contains(currentPath, UnknownPath) {
return nil, true
}
//log.LogDebugf("action[shouldNotMoveToTrash]: parentIno(%v) entry(%v) currentPath(%v)", parentIno, entry, currentPath)
// log.LogDebugf("action[shouldNotMoveToTrash]: parentIno(%v) entry(%v) currentPath(%v)", parentIno, entry, currentPath)
subs := strings.Split(currentPath, "/")
if len(subs) == 1 {
//should never happen: file in root trash
// should never happen: file in root trash
if subs[0] == TrashPrefix {
log.LogDebugf("action[shouldNotMoveToTrash]: currentPath(%v) should not remove ", subs[0])
return nil, true
@ -625,13 +627,13 @@ func (mw *MetaWrapper) shouldNotMoveToTrash(parentMP *MetaPartition, parentIno u
log.LogDebugf("action[shouldNotMoveToTrash]: currentPath(%v)is expired ", currentPath)
return nil, true
}
//should never happen: dir in root trash
// should never happen: dir in root trash
if subs[0] == TrashPrefix {
log.LogDebugf("action[shouldMoveToTrash]: currentPath(%v) not legal ", currentPath)
return nil, true
}
}
//log.LogDebugf("action[shouldMoveToTrash]: currentPath(%v) is not expired ", currentPath)
// log.LogDebugf("action[shouldMoveToTrash]: currentPath(%v) is not expired ", currentPath)
return nil, false
}
@ -647,6 +649,10 @@ func (mw *MetaWrapper) Delete_ll(parentID uint64, name string, isDir bool, fullP
}
}
func (mw *MetaWrapper) DeleteWithCond_ll(parentID, cond uint64, name string, isDir bool, fullPath string) (*proto.InodeInfo, error) {
return mw.deletewithcond_ll(parentID, cond, name, isDir, fullPath)
}
func (mw *MetaWrapper) Delete_Ver_ll(parentID uint64, name string, isDir bool, verSeq uint64, fullPath string) (*proto.InodeInfo, error) {
if verSeq == 0 {
verSeq = math.MaxUint64
@ -678,7 +684,7 @@ func (mw *MetaWrapper) txDelete_ll(parentID uint64, name string, isDir bool, ful
log.LogDebugf("TRACE Remove:TrashPolicy is nil")
mw.enableTrash()
}
//cannot delete .Trash
// cannot delete .Trash
err, ret := mw.shouldNotMoveToTrash(parentMP, parentID, name, isDir)
if err != nil {
log.LogErrorf("Delete_ll: shouldNotMoveToTrash failed:%v", err)
@ -790,7 +796,7 @@ func (mw *MetaWrapper) txDelete_ll(parentID uint64, name string, isDir bool, ful
}
tx.SetOnCommit(job)
}
//clear trash cache
// clear trash cache
if mw.trashPolicy != nil && mw.disableTrash == false {
mw.trashPolicy.CleanTrashPatchCache(mw.getCurrentPath(parentID), name)
}
@ -825,7 +831,7 @@ func (mw *MetaWrapper) Delete_ll_EX(parentID uint64, name string, isDir bool, ve
log.LogDebugf("TRACE Remove:TrashPolicy is nil")
mw.enableTrash()
}
//cannot delete .Trash
// cannot delete .Trash
err, ret := mw.shouldNotMoveToTrash(parentMP, parentID, name, isDir)
if err != nil {
log.LogErrorf("Delete_ll: shouldNotMoveToTrash name %v failed %v", name, err)
@ -1061,7 +1067,7 @@ func (mw *MetaWrapper) deletewithcond_ll(parentID, cond uint64, name string, isD
}
}()
}
//clear trash cache
// clear trash cache
if mw.trashPolicy != nil && mw.disableTrash == false {
mw.trashPolicy.CleanTrashPatchCache(mw.getCurrentPath(parentID), name)
}
@ -1265,6 +1271,7 @@ func (mw *MetaWrapper) rename_ll(srcParentID uint64, srcName string, dstParentID
oldInode uint64
lastVerSeq uint64
)
start := time.Now()
defer func() {
if err != nil {
log.LogErrorf("Rename_ll: srcName %v srcFullPath %v dstName %v dstFullPath %v err %v",
@ -1335,7 +1342,7 @@ func (mw *MetaWrapper) rename_ll(srcParentID uint64, srcName string, dstParentID
return syscall.EAGAIN
}
dstInodeInfo, _ = mw.InodeGet_ll(oldInode)
//delete old cache
// delete old cache
mw.DeleteInoInfoCache(oldInode)
}
@ -1429,8 +1436,8 @@ func (mw *MetaWrapper) rename_ll(srcParentID uint64, srcName string, dstParentID
// mw.BatchSetInodeQuota_ll(inodes, quotaId, false)
// }
//log.LogDebugf("Rename_ll: dstInodeInfo %v", dstInodeInfo)
//mw.AddInoInfoCache(dstInodeInfo.Inode, dstParentID, dstName)
// log.LogDebugf("Rename_ll: dstInodeInfo %v", dstInodeInfo)
// mw.AddInoInfoCache(dstInodeInfo.Inode, dstParentID, dstName)
mw.DeleteInoInfoCache(srcInodeInfo.Inode)
if proto.IsDir(srcInodeInfo.Mode) {
mw.AddInoInfoCache(srcInodeInfo.Inode, dstParentID, dstName)
@ -2040,7 +2047,8 @@ func (mw *MetaWrapper) GetMultipart_ll(path, multipartId string) (info *proto.Mu
}
func (mw *MetaWrapper) AddMultipartPart_ll(path, multipartId string, partId uint16, size uint64, md5 string,
inodeInfo *proto.InodeInfo) (oldInode uint64, updated bool, err error) {
inodeInfo *proto.InodeInfo,
) (oldInode uint64, updated bool, err error) {
var (
mpId uint64
found bool
@ -2673,7 +2681,7 @@ func (mw *MetaWrapper) getCurrentPathToMountSub(parentIno uint64) (parentPath st
log.LogDebugf("action[getCurrentPathToMountSub]: currentPath(%v) parentIno(%v)", currentPath, parentIno)
subs := strings.Split(currentPath, "/")
var index = 0
index := 0
for i, sub := range subs {
if sub == mw.subDir {
index = i

View File

@ -2,10 +2,11 @@ package meta
import (
"container/list"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/util/log"
"sync"
"time"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/util/log"
)
const (

View File

@ -92,8 +92,8 @@ type MetaConfig struct {
OnAsyncTaskError AsyncTaskErrorFunc
EnableSummary bool
MetaSendTimeout int64
//EnableTransaction uint8
//EnableTransaction bool
// EnableTransaction uint8
// EnableTransaction bool
MountPoint string
SubDir string
TrashTraverseLimit int
@ -169,7 +169,7 @@ type MetaWrapper struct {
uniqidRangeMutex sync.Mutex
qc *QuotaCache
//trash
// trash
TrashInterval int64
trashPolicy *Trash
disableTrash bool
@ -242,7 +242,6 @@ func NewMetaWrapper(config *MetaConfig) (*MetaWrapper, error) {
mw.uniqidRangeMap = make(map[uint64]*uniqidRange, 0)
mw.qc = NewQuotaCache(DefaultQuotaExpiration, MaxQuotaCache)
mw.VerReadSeq = config.VerReadSeq
mw.entryCache = make(map[uint64]inoInfoCache)
mw.dirCache = make(map[uint64]dirInfoCache)
mw.subDir = config.SubDir
limit := MaxMountRetryLimit
@ -277,6 +276,7 @@ func NewMetaWrapper(config *MetaConfig) (*MetaWrapper, error) {
go mw.refresh()
return mw, nil
}
func (mw *MetaWrapper) enableTrash() {
if mw.disableTrash == true {
return

View File

@ -32,7 +32,8 @@ import (
//
// txIcreate create inode and tx together
func (mw *MetaWrapper) txIcreate(tx *Transaction, mp *MetaPartition, mode, uid, gid uint32,
target []byte, quotaIds []uint32, fullPath string) (status int, info *proto.InodeInfo, err error) {
target []byte, quotaIds []uint32, fullPath string,
) (status int, info *proto.InodeInfo, err error) {
bgTime := stat.BeginStat()
defer func() {
stat.EndStat("txIcreate", err, bgTime, 1)
@ -104,7 +105,8 @@ func (mw *MetaWrapper) txIcreate(tx *Transaction, mp *MetaPartition, mode, uid,
}
func (mw *MetaWrapper) quotaIcreate(mp *MetaPartition, mode, uid, gid uint32, target []byte, quotaIds []uint32, fullPath string) (status int,
info *proto.InodeInfo, err error) {
info *proto.InodeInfo, err error,
) {
bgTime := stat.BeginStat()
defer func() {
stat.EndStat("icreate", err, bgTime, 1)
@ -164,7 +166,8 @@ func (mw *MetaWrapper) quotaIcreate(mp *MetaPartition, mode, uid, gid uint32, ta
}
func (mw *MetaWrapper) icreate(mp *MetaPartition, mode, uid, gid uint32, target []byte, fullPath string) (status int,
info *proto.InodeInfo, err error) {
info *proto.InodeInfo, err error,
) {
bgTime := stat.BeginStat()
defer func() {
stat.EndStat("icreate", err, bgTime, 1)
@ -251,7 +254,8 @@ func (mw *MetaWrapper) sendToMetaPartitionWithTx(mp *MetaPartition, req *proto.P
}
func (mw *MetaWrapper) SendTxPack(req proto.TxPack, resp interface{}, Opcode uint8, mp *MetaPartition,
checkStatusFunc func(int, *proto.Packet) error, ignoreExistError bool) (status int, err error, packet *proto.Packet) {
checkStatusFunc func(int, *proto.Packet) error, ignoreExistError bool,
) (status int, err error, packet *proto.Packet) {
packet = proto.NewPacketReqID()
packet.Opcode = Opcode
packet.PartitionID = mp.PartitionID
@ -475,7 +479,8 @@ func (mw *MetaWrapper) ievict(mp *MetaPartition, inode uint64, fullPath string)
}
func (mw *MetaWrapper) txDcreate(tx *Transaction, mp *MetaPartition, parentID uint64, name string, inode uint64,
mode uint32, quotaIds []uint32, fullPath string, ignoreExist bool) (status int, err error) {
mode uint32, quotaIds []uint32, fullPath string, ignoreExist bool,
) (status int, err error) {
bgTime := stat.BeginStat()
defer func() {
stat.EndStat("txDcreate", err, bgTime, 1)
@ -524,7 +529,8 @@ func (mw *MetaWrapper) txDcreate(tx *Transaction, mp *MetaPartition, parentID ui
}
func (mw *MetaWrapper) quotaDcreate(mp *MetaPartition, parentID uint64, name string, inode uint64, mode uint32,
quotaIds []uint32, fullPath string, ignoreExistError bool) (status int, err error) {
quotaIds []uint32, fullPath string, ignoreExistError bool,
) (status int, err error) {
bgTime := stat.BeginStat()
defer func() {
stat.EndStat("dcreate", err, bgTime, 1)
@ -577,7 +583,8 @@ func (mw *MetaWrapper) quotaDcreate(mp *MetaPartition, parentID uint64, name str
}
func (mw *MetaWrapper) dcreate(mp *MetaPartition, parentID uint64, name string, inode uint64, mode uint32,
fullPath string, ignoreExistError bool) (status int, err error) {
fullPath string, ignoreExistError bool,
) (status int, err error) {
bgTime := stat.BeginStat()
defer func() {
stat.EndStat("dcreate", err, bgTime, 1)
@ -896,7 +903,8 @@ func (mw *MetaWrapper) canDeleteInode(mp *MetaPartition, info *proto.InodeInfo,
}
func (mw *MetaWrapper) ddeletes(mp *MetaPartition, parentID uint64, dentries []proto.Dentry, fullPaths []string) (status int,
resp *proto.BatchDeleteDentryResponse, err error) {
resp *proto.BatchDeleteDentryResponse, err error,
) {
bgTime := stat.BeginStat()
defer func() {
stat.EndStat("ddeletes", err, bgTime, 1)
@ -2489,7 +2497,8 @@ func (mw *MetaWrapper) updateXAttrs(mp *MetaPartition, inode uint64, filesInc in
}
func (mw *MetaWrapper) batchSetInodeQuota(mp *MetaPartition, inodes []uint64, quotaId uint32,
IsRoot bool) (resp *proto.BatchSetMetaserverQuotaResponse, err error) {
IsRoot bool,
) (resp *proto.BatchSetMetaserverQuotaResponse, err error) {
bgTime := stat.BeginStat()
defer func() {
stat.EndStat("batchSetInodeQuota", err, bgTime, 1)
@ -2538,7 +2547,8 @@ func (mw *MetaWrapper) batchSetInodeQuota(mp *MetaPartition, inodes []uint64, qu
}
func (mw *MetaWrapper) batchDeleteInodeQuota(mp *MetaPartition, inodes []uint64,
quotaId uint32) (resp *proto.BatchDeleteMetaserverQuotaResponse, err error) {
quotaId uint32,
) (resp *proto.BatchDeleteMetaserverQuotaResponse, err error) {
bgTime := stat.BeginStat()
defer func() {
stat.EndStat("batchDeleteInodeQuota", err, bgTime, 1)
@ -2643,7 +2653,8 @@ func (mw *MetaWrapper) getInodeQuota(mp *MetaPartition, inode uint64) (quotaInfo
}
func (mw *MetaWrapper) applyQuota(parentIno uint64, quotaId uint32, totalInodeCount *uint64, curInodeCount *uint64, inodes *[]uint64,
maxInodes uint64, first bool) (err error) {
maxInodes uint64, first bool,
) (err error) {
if first {
var rootInodes []uint64
var ret map[uint64]uint8
@ -2714,7 +2725,8 @@ func (mw *MetaWrapper) applyQuota(parentIno uint64, quotaId uint32, totalInodeCo
}
func (mw *MetaWrapper) revokeQuota(parentIno uint64, quotaId uint32, totalInodeCount *uint64, curInodeCount *uint64, inodes *[]uint64,
maxInodes uint64, first bool) (err error) {
maxInodes uint64, first bool,
) (err error) {
if first {
var rootInodes []uint64
rootInodes = append(rootInodes, parentIno)

Some files were not shown because too many files have changed in this diff Show More