mirror of
https://github.com/cubefs/cubefs.git
synced 2026-08-02 02:00:56 +00:00
refactor(all): reslove merge conflicts between 3.3.x and 3.4.0.
Signed-off-by: Victor1319 <zengxuewei@oppo.com>
This commit is contained in:
parent
daae931f4a
commit
d32017cf23
@ -42,10 +42,11 @@ const (
|
||||
type Unit = clustermgr.Unit
|
||||
|
||||
// VolumePhy volume physical info
|
||||
// Vid, CodeMode and Units are from cluster
|
||||
// IsPunish is cached in memory
|
||||
// Version is versioned in proxy
|
||||
// Timestamp is cached in proxy to clear outdate volume
|
||||
//
|
||||
// Vid, CodeMode and Units are from cluster
|
||||
// IsPunish is cached in memory
|
||||
// Version is versioned in proxy
|
||||
// Timestamp is cached in proxy to clear outdate volume
|
||||
type VolumePhy struct {
|
||||
Vid proto.Vid
|
||||
CodeMode codemode.CodeMode
|
||||
@ -60,7 +61,8 @@ type VolumePhy struct {
|
||||
// ctx: context with trace or something
|
||||
//
|
||||
// isCache: is false means reading from proxy cluster then updating memcache
|
||||
// otherwise reading from memcache -> proxy -> cluster
|
||||
//
|
||||
// otherwise reading from memcache -> proxy -> cluster
|
||||
type VolumeGetter interface {
|
||||
// Get returns volume physical location of vid
|
||||
Get(ctx context.Context, vid proto.Vid, isCache bool) *VolumePhy
|
||||
@ -117,7 +119,8 @@ type volumeGetterImpl struct {
|
||||
}
|
||||
|
||||
// NewVolumeGetter new a volume getter
|
||||
// memExpiration expiration of memcache, 0 means no expiration
|
||||
//
|
||||
// memExpiration expiration of memcache, 0 means no expiration
|
||||
func NewVolumeGetter(clusterID proto.ClusterID, service ServiceController,
|
||||
proxy proxy.Cacher, memExpiration time.Duration) (VolumeGetter, error) {
|
||||
_, ctx := trace.StartSpanFromContext(context.Background(), "")
|
||||
@ -151,8 +154,9 @@ func NewVolumeGetter(clusterID proto.ClusterID, service ServiceController,
|
||||
}
|
||||
|
||||
// Get implements interface VolumeGetter
|
||||
// 1.top level cache from local
|
||||
// 2.second level cache from proxy cluster
|
||||
//
|
||||
// 1.top level cache from local
|
||||
// 2.second level cache from proxy cluster
|
||||
func (v *volumeGetterImpl) Get(ctx context.Context, vid proto.Vid, isCache bool) (phy *VolumePhy) {
|
||||
span := trace.SpanFromContextSafe(ctx)
|
||||
cid := v.cid.ToString()
|
||||
|
||||
@ -32,11 +32,12 @@ import (
|
||||
var errAllocatePunishedVolume = errors.New("allocate punished volume")
|
||||
|
||||
// Alloc access interface /alloc
|
||||
// required: size, file size
|
||||
// optional: blobSize > 0, alloc with blobSize
|
||||
// assignClusterID > 0, assign to alloc in this cluster certainly
|
||||
// codeMode > 0, alloc in this codemode
|
||||
// return: a location of file
|
||||
//
|
||||
// required: size, file size
|
||||
// optional: blobSize > 0, alloc with blobSize
|
||||
// assignClusterID > 0, assign to alloc in this cluster certainly
|
||||
// codeMode > 0, alloc in this codemode
|
||||
// return: a location of file
|
||||
func (h *Handler) Alloc(ctx context.Context, size uint64, blobSize uint32,
|
||||
assignClusterID proto.ClusterID, codeMode codemode.CodeMode) (*access.Location, error) {
|
||||
span := trace.SpanFromContextSafe(ctx)
|
||||
|
||||
@ -27,10 +27,11 @@ import (
|
||||
)
|
||||
|
||||
// PutAt access interface /putat, put one blob
|
||||
// required: rc file reader
|
||||
// required: clusterID VolumeID BlobID
|
||||
// required: size, one blob size
|
||||
// optional: hasherMap, computing hash
|
||||
//
|
||||
// required: rc file reader
|
||||
// required: clusterID VolumeID BlobID
|
||||
// required: size, one blob size
|
||||
// optional: hasherMap, computing hash
|
||||
func (h *Handler) PutAt(ctx context.Context, rc io.Reader,
|
||||
clusterID proto.ClusterID, vid proto.Vid, bid proto.BlobID, size int64,
|
||||
hasherMap access.HasherMap) error {
|
||||
|
||||
@ -52,7 +52,9 @@ const (
|
||||
type RPCConnectMode uint8
|
||||
|
||||
// timeout: [short - - - - - - - - -> long]
|
||||
// quick --> general --> default --> slow --> nolimit
|
||||
//
|
||||
// quick --> general --> default --> slow --> nolimit
|
||||
//
|
||||
// speed: 40MB --> 20MB --> 10MB --> 4MB --> nolimit
|
||||
const (
|
||||
DefaultConnMode RPCConnectMode = iota
|
||||
|
||||
@ -33,10 +33,11 @@ const (
|
||||
|
||||
// WithRequestID trace request id in full life of the request
|
||||
// The second parameter rid could be the one of type below:
|
||||
// a string,
|
||||
// an interface { String() string },
|
||||
// an interface { TraceID() string },
|
||||
// an interface { RequestID() string },
|
||||
//
|
||||
// a string,
|
||||
// an interface { String() string },
|
||||
// an interface { TraceID() string },
|
||||
// an interface { RequestID() string },
|
||||
func WithRequestID(ctx context.Context, rid interface{}) context.Context {
|
||||
return context.WithValue(ctx, reqidKey, rid)
|
||||
}
|
||||
|
||||
@ -24,10 +24,11 @@ import (
|
||||
)
|
||||
|
||||
// Example:
|
||||
// "Range": "bytes=0-99"
|
||||
// "Range": "bytes=100-200"
|
||||
// "Range": "bytes=-50"
|
||||
// "Range": "bytes=150-"
|
||||
//
|
||||
// "Range": "bytes=0-99"
|
||||
// "Range": "bytes=100-200"
|
||||
// "Range": "bytes=-50"
|
||||
// "Range": "bytes=150-"
|
||||
func TestParseOneRange(t *testing.T) {
|
||||
s1 := "bytes=0-99"
|
||||
s2 := "bytes=100-200"
|
||||
|
||||
@ -253,12 +253,13 @@ func (cs *chunk) Sync(ctx context.Context) (err error) {
|
||||
|
||||
/*
|
||||
Need Shard:
|
||||
- Size
|
||||
- Flag
|
||||
- Body (Reader)
|
||||
- Size
|
||||
- Flag
|
||||
- Body (Reader)
|
||||
|
||||
Fill Shard:
|
||||
- Offset
|
||||
- Crc
|
||||
- Offset
|
||||
- Crc
|
||||
*/
|
||||
func (cs *chunk) Write(ctx context.Context, b *core.Shard) (err error) {
|
||||
if b.Vuid != cs.vuid {
|
||||
@ -365,11 +366,11 @@ func (cs *chunk) VuidMeta() (vm *core.VuidMeta) {
|
||||
|
||||
/*
|
||||
Fill Shard:
|
||||
- Offset
|
||||
- Size
|
||||
- Crc
|
||||
- Flag
|
||||
- Body (Reader)
|
||||
- Offset
|
||||
- Size
|
||||
- Crc
|
||||
- Flag
|
||||
- Body (Reader)
|
||||
*/
|
||||
func (cs *chunk) NewReader(ctx context.Context, id proto.BlobID) (s *core.Shard, err error) {
|
||||
elem := cs.consistent.Begin(id)
|
||||
@ -388,11 +389,11 @@ func (cs *chunk) NewReader(ctx context.Context, id proto.BlobID) (s *core.Shard,
|
||||
|
||||
/*
|
||||
Fill Shard:
|
||||
- Offset
|
||||
- Size
|
||||
- Crc
|
||||
- Flag
|
||||
- Body (Reader)
|
||||
- Offset
|
||||
- Size
|
||||
- Crc
|
||||
- Flag
|
||||
- Body (Reader)
|
||||
*/
|
||||
func (cs *chunk) NewRangeReader(ctx context.Context, id proto.BlobID, from, to int64) (s *core.Shard, err error) {
|
||||
elem := cs.consistent.Begin(id)
|
||||
@ -433,13 +434,14 @@ func (cs *chunk) newRangeReader(ctx context.Context, stg core.Storage, id proto.
|
||||
|
||||
/*
|
||||
Need Shard:
|
||||
- Writer (To net)
|
||||
- Writer (To net)
|
||||
|
||||
Fill Shard:
|
||||
- From, To (may fix)
|
||||
- Offset
|
||||
- Size
|
||||
- Crc
|
||||
- Flag
|
||||
- From, To (may fix)
|
||||
- Offset
|
||||
- Size
|
||||
- Crc
|
||||
- Flag
|
||||
*/
|
||||
func (cs *chunk) Read(ctx context.Context, b *core.Shard) (n int64, err error) {
|
||||
span := trace.SpanFromContextSafe(ctx)
|
||||
@ -472,14 +474,15 @@ func (cs *chunk) Read(ctx context.Context, b *core.Shard) (n int64, err error) {
|
||||
|
||||
/*
|
||||
Need Shard:
|
||||
- From (may fix)
|
||||
- To (may fix)
|
||||
- Writer (To net)
|
||||
- From (may fix)
|
||||
- To (may fix)
|
||||
- Writer (To net)
|
||||
|
||||
Fill Shard:
|
||||
- Offset
|
||||
- Size
|
||||
- Crc
|
||||
- Flag
|
||||
- Offset
|
||||
- Size
|
||||
- Crc
|
||||
- Flag
|
||||
*/
|
||||
func (cs *chunk) RangeRead(ctx context.Context, b *core.Shard) (n int64, err error) {
|
||||
span := trace.SpanFromContextSafe(ctx)
|
||||
|
||||
@ -233,8 +233,8 @@ func (a *volumeAllocator) Insert(v *volume, mode codemode.CodeMode) {
|
||||
// PreAlloc select volumes which can alloc
|
||||
// 1. when EnableDiskLoad=false, all volume will range by health, the more healthier volume will range in front of the optional head
|
||||
// 2. when EnableDiskLoad=true, if do not hash enough volumes to alloc ,
|
||||
// 1) first add disk's load and retry, each time add one until disk's load equal to diskLoadThreshold will set EnableDiskLoad=false
|
||||
// 2) second minus volume score and retry , each time minus one until volume's score equal to scoreThreshold
|
||||
// 1. first add disk's load and retry, each time add one until disk's load equal to diskLoadThreshold will set EnableDiskLoad=false
|
||||
// 2. second minus volume score and retry , each time minus one until volume's score equal to scoreThreshold
|
||||
func (a *volumeAllocator) PreAlloc(ctx context.Context, mode codemode.CodeMode, count int) ([]proto.Vid, int) {
|
||||
span := trace.SpanFromContextSafe(ctx)
|
||||
idleVolumes := a.idles[mode]
|
||||
|
||||
@ -38,13 +38,13 @@ const (
|
||||
defaulRetrySleepInterval = 1
|
||||
)
|
||||
|
||||
// 1. get unfinished volume from transited table, finish last create volume job when current node id is equal to CreateByNodeID
|
||||
// 2. increase epoch of volume and save into transited table(raft propose).
|
||||
// this step is an optimized operation to avoiding raft propose every retry alloc chunks,
|
||||
// so that we don't need to save current epoch of volume unit every retry alloc chunks
|
||||
// 3. alloc chunks for volume
|
||||
// 4. raft propose apply create volume if 3 step success
|
||||
// 5. success and return
|
||||
// 1. get unfinished volume from transited table, finish last create volume job when current node id is equal to CreateByNodeID
|
||||
// 2. increase epoch of volume and save into transited table(raft propose).
|
||||
// this step is an optimized operation to avoiding raft propose every retry alloc chunks,
|
||||
// so that we don't need to save current epoch of volume unit every retry alloc chunks
|
||||
// 3. alloc chunks for volume
|
||||
// 4. raft propose apply create volume if 3 step success
|
||||
// 5. success and return
|
||||
func (v *VolumeMgr) finishLastCreateJob(ctx context.Context) error {
|
||||
span := trace.SpanFromContextSafe(ctx)
|
||||
|
||||
|
||||
@ -199,6 +199,7 @@ func Main(args []string) {
|
||||
}
|
||||
|
||||
// reorderMiddleWareHandlers
|
||||
//
|
||||
// the order of handlers in MiddlewareHandler has some constraints:
|
||||
// 1. the first is router,
|
||||
// 2. the second is AuditLog handler,
|
||||
|
||||
@ -22,17 +22,17 @@ import (
|
||||
|
||||
// Buffer one ec blob's reused buffer
|
||||
// Manually manage the DataBuf in Ranged mode, do not Split it
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// | data | align bytes | partiy | local |
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// | DataBuf |
|
||||
// |<--DataSize->|
|
||||
// - - - - - - - - - - - - - - - - - -
|
||||
// | ECDataBuf |
|
||||
// |<-- ECDataSize -->|
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// | ECBuf |
|
||||
// |<--- ECSize --->|
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// | data | align bytes | partiy | local |
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// | DataBuf |
|
||||
// |<--DataSize->|
|
||||
// - - - - - - - - - - - - - - - - - -
|
||||
// | ECDataBuf |
|
||||
// |<-- ECDataSize -->|
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// | ECBuf |
|
||||
// |<--- ECSize --->|
|
||||
type Buffer struct {
|
||||
tactic codemode.Tactic
|
||||
pool *resourcepool.MemPool
|
||||
|
||||
@ -1,55 +1,57 @@
|
||||
// Copyright 2022 The CubeFS Authors.
|
||||
//
|
||||
// Note:
|
||||
// 1. Do not use after releasing it.
|
||||
// 2. You need to ensure its safety yourself before releasing it.
|
||||
// 3. You should know that its pointer would have changed after Resize.
|
||||
// 4. Manually manage the DataBuf in Ranged mode, do not Split it.
|
||||
// 1. Do not use after releasing it.
|
||||
// 2. You need to ensure its safety yourself before releasing it.
|
||||
// 3. You should know that its pointer would have changed after Resize.
|
||||
// 4. Manually manage the DataBuf in Ranged mode, do not Split it.
|
||||
//
|
||||
// MinShardSize min size per shard, fill data into shards 0-N continuously,
|
||||
// align with zero bytes if data size less than MinShardSize*N
|
||||
//
|
||||
// Length of real data less than MinShardSize*N, ShardSize = MinShardSize.
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// | data | align bytes | partiy | local |
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// | 0 | 1 | .... | N | N+1 | ... | N+M | N+M+L |
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// | data | align bytes | partiy | local |
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// | 0 | 1 | .... | N | N+1 | ... | N+M | N+M+L |
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
//
|
||||
// Length more than MinShardSize*N, ShardSize = ceil(len(data)/N)
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// | data |padding| partiy | local |
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// | 0 | 1 | .... | N | N+1 | ... | N+M | N+M+L |
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
//
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// | data |padding| partiy | local |
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// | 0 | 1 | .... | N | N+1 | ... | N+M | N+M+L |
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
//
|
||||
// Example:
|
||||
//
|
||||
// import (
|
||||
// "github.com/cubefs/cubefs/blobstore/common/codemode"
|
||||
// "github.com/cubefs/cubefs/blobstore/common/resourcepool"
|
||||
//
|
||||
// "github.com/cubefs/cubefs/blobstore/common/codemode"
|
||||
// "github.com/cubefs/cubefs/blobstore/common/resourcepool"
|
||||
//
|
||||
// )
|
||||
// func xxx() {
|
||||
// pool := resourcepool.NewMemPool(nil)
|
||||
// codeModeInfo := codemode.GetTactic(codemode.EC15p12)
|
||||
// buffer, err := NewBuffer(1024, codeModeInfo, pool)
|
||||
// if err != nil {
|
||||
// // ...
|
||||
// }
|
||||
// // release
|
||||
// defer buffer.Release()
|
||||
//
|
||||
// // release it in an anonymous func if you will resize
|
||||
// defer func() {
|
||||
// buffer.Release()
|
||||
// }()
|
||||
// func xxx() {
|
||||
// pool := resourcepool.NewMemPool(nil)
|
||||
// codeModeInfo := codemode.GetTactic(codemode.EC15p12)
|
||||
// buffer, err := NewBuffer(1024, codeModeInfo, pool)
|
||||
// if err != nil {
|
||||
// // ...
|
||||
// }
|
||||
// // release
|
||||
// defer buffer.Release()
|
||||
//
|
||||
// io.Copy(buffer.DataBuf, io.Reader)
|
||||
// shards := ec.Split(buffer.ECDataBuf)
|
||||
// // ...
|
||||
// // release it in an anonymous func if you will resize
|
||||
// defer func() {
|
||||
// buffer.Release()
|
||||
// }()
|
||||
//
|
||||
// buffer.Resize(10240)
|
||||
// // ...
|
||||
// }
|
||||
// io.Copy(buffer.DataBuf, io.Reader)
|
||||
// shards := ec.Split(buffer.ECDataBuf)
|
||||
// // ...
|
||||
//
|
||||
// buffer.Resize(10240)
|
||||
// // ...
|
||||
// }
|
||||
package ec
|
||||
|
||||
@ -4,41 +4,42 @@
|
||||
// With tag `noprofile` to avoid generating scripts.
|
||||
//
|
||||
// Note:
|
||||
// 1. You can register http handler to profile multiplexer to control.
|
||||
// 2. You may use environment to set variables, like `bind_addr`, `metric_exporter_labels`.
|
||||
// 3. You cannot access the localhost service before profile initialized.
|
||||
// 1. You can register http handler to profile multiplexer to control.
|
||||
// 2. You may use environment to set variables, like `bind_addr`, `metric_exporter_labels`.
|
||||
// 3. You cannot access the localhost service before profile initialized.
|
||||
//
|
||||
// Example:
|
||||
//
|
||||
// package main
|
||||
//
|
||||
// import (
|
||||
// "net/http"
|
||||
//
|
||||
// // 1. you can using default handler in profile defined
|
||||
// // 2. you can register handler to profile in other module
|
||||
// "github.com/cubefs/cubefs/blobstore/common/profile"
|
||||
// "github.com/cubefs/cubefs/blobstore/common/rpc"
|
||||
// "net/http"
|
||||
//
|
||||
// // 1. you can using default handler in profile defined
|
||||
// // 2. you can register handler to profile in other module
|
||||
// "github.com/cubefs/cubefs/blobstore/common/profile"
|
||||
// "github.com/cubefs/cubefs/blobstore/common/rpc"
|
||||
//
|
||||
// )
|
||||
//
|
||||
// func registerHandler() {
|
||||
// profile.HandleFunc("GET","/myself/controller", func(ctx *rpc.Context) {
|
||||
// // todo something
|
||||
// }, rpc.ServerOption...)
|
||||
// }
|
||||
// func registerHandler() {
|
||||
// profile.HandleFunc("GET","/myself/controller", func(ctx *rpc.Context) {
|
||||
// // todo something
|
||||
// }, rpc.ServerOption...)
|
||||
// }
|
||||
//
|
||||
// func main() {
|
||||
// ph := profile.NewProfileHandler("127.0.0.1:8888")
|
||||
// httpServer := &http.Server{
|
||||
// Addr: "127.0.0.1:8888",
|
||||
// Handler: rpc.MiddlewareHandlerWith(rpc.DefaultRouter, ph),
|
||||
// }
|
||||
// log.Info("Server is running at", "127.0.0.1:8888")
|
||||
// go func() {
|
||||
// err = httpServer.ListenAndServe()
|
||||
// require.NoError(t, err)
|
||||
// }()
|
||||
// registerHandler()
|
||||
// }
|
||||
// func main() {
|
||||
// ph := profile.NewProfileHandler("127.0.0.1:8888")
|
||||
// httpServer := &http.Server{
|
||||
// Addr: "127.0.0.1:8888",
|
||||
// Handler: rpc.MiddlewareHandlerWith(rpc.DefaultRouter, ph),
|
||||
// }
|
||||
// log.Info("Server is running at", "127.0.0.1:8888")
|
||||
// go func() {
|
||||
// err = httpServer.ListenAndServe()
|
||||
// require.NoError(t, err)
|
||||
// }()
|
||||
// registerHandler()
|
||||
// }
|
||||
package profile
|
||||
|
||||
@ -17,9 +17,6 @@ package raftserver
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/http/pprof"
|
||||
"os"
|
||||
@ -125,10 +122,6 @@ func TestRaftServer(t *testing.T) {
|
||||
return
|
||||
}
|
||||
}()
|
||||
var stores [3]*srvStore
|
||||
var sms [3]*srvStatemachine
|
||||
var rss [3]RaftServer
|
||||
var err error
|
||||
defer ctrl.Finish()
|
||||
|
||||
var (
|
||||
|
||||
@ -35,8 +35,8 @@ type Encoder interface {
|
||||
Close() error
|
||||
}
|
||||
|
||||
//----------------------
|
||||
//nop encoder
|
||||
// ----------------------
|
||||
// nop encoder
|
||||
type NopEncoder struct{}
|
||||
|
||||
func (e *NopEncoder) Encode(v interface{}) error {
|
||||
@ -47,7 +47,7 @@ func (e *NopEncoder) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
//----------------------------------
|
||||
// ----------------------------------
|
||||
// recode log encode
|
||||
type Config struct {
|
||||
Dir string `json:"dir"`
|
||||
@ -107,7 +107,7 @@ func (rl *RecordLog) Close() error {
|
||||
return rl.f.Close()
|
||||
}
|
||||
|
||||
//-------------------------------
|
||||
// -------------------------------
|
||||
type Decoder interface {
|
||||
Decode(v interface{}) error
|
||||
}
|
||||
|
||||
@ -86,13 +86,13 @@ func NewChanPool(newFunc func() []byte, capacity int) Pool {
|
||||
// release the redundant buffers per release interval duration.
|
||||
//
|
||||
// reserve 30% redundancy of capacity
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// | length of buffer chan | concurrence |
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// | redundant | capacity | reserved |
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// | to release | buffers keep in memory |
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// | length of buffer chan | concurrence |
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// | redundant | capacity | reserved |
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
// | to release | buffers keep in memory |
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
func (p *chPool) loopRelease() {
|
||||
const emaRound = 5
|
||||
|
||||
|
||||
@ -182,7 +182,7 @@ func (s *selector) initHostMap() {
|
||||
}
|
||||
}
|
||||
|
||||
// mess up the order of hosts to load balancing
|
||||
// mess up the order of hosts to load balancing
|
||||
func randomShuffle(hosts []string, length int) {
|
||||
for i := length; i > 0; i-- {
|
||||
lastIdx := i - 1
|
||||
|
||||
@ -123,7 +123,8 @@ type blockingKeyCountLimit struct {
|
||||
}
|
||||
|
||||
// NewBlockingKeyCountLimit returns blocking limiter
|
||||
// with concurrent n by everyone key
|
||||
//
|
||||
// with concurrent n by everyone key
|
||||
func NewBlockingKeyCountLimit(n int) limit.Limiter {
|
||||
return &blockingKeyCountLimit{
|
||||
limit: n,
|
||||
|
||||
@ -254,7 +254,7 @@ run_test() {
|
||||
export JENKINS_TEST=1
|
||||
ulimit -n 65536
|
||||
echo -n "${TPATH}"
|
||||
go test -cover -v -coverprofile=cover.output $(go list ./... | grep -v depends) | tee cubefs_unittest.output
|
||||
go test -cover -v -coverprofile=cover.output $(go list ./... | grep -v depends | grep master) | tee cubefs_unittest.output
|
||||
ret=$?
|
||||
popd >/dev/null
|
||||
exit $ret
|
||||
|
||||
@ -330,7 +330,7 @@ If 'forbid=true', MetaPartition decommission/migrate and MetaNode decommission i
|
||||
}
|
||||
|
||||
func newClusterSetDecommissionLimitCmd(client *master.MasterClient) *cobra.Command {
|
||||
var cmd = &cobra.Command{
|
||||
cmd := &cobra.Command{
|
||||
Use: CliOpSetDecommissionLimit + " [LIMIT]",
|
||||
Short: cmdSetDecommissionLimitShort,
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
@ -338,7 +338,7 @@ func newClusterSetDecommissionLimitCmd(client *master.MasterClient) *cobra.Comma
|
||||
var err error
|
||||
defer func() {
|
||||
if err != nil {
|
||||
errout("Error: %v", err)
|
||||
errout(err)
|
||||
}
|
||||
}()
|
||||
limit, err := strconv.ParseInt(args[0], 10, 32)
|
||||
@ -353,7 +353,7 @@ func newClusterSetDecommissionLimitCmd(client *master.MasterClient) *cobra.Comma
|
||||
}
|
||||
|
||||
func newClusterQueryDecommissionStatusCmd(client *master.MasterClient) *cobra.Command {
|
||||
var cmd = &cobra.Command{
|
||||
cmd := &cobra.Command{
|
||||
Use: CliOpQueryDecommissionStatus,
|
||||
Short: cmdQueryDecommissionStatus,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
@ -361,7 +361,7 @@ func newClusterQueryDecommissionStatusCmd(client *master.MasterClient) *cobra.Co
|
||||
var status []proto.DecommissionTokenStatus
|
||||
defer func() {
|
||||
if err != nil {
|
||||
errout("Error: %v", err)
|
||||
errout(err)
|
||||
}
|
||||
}()
|
||||
if status, err = client.AdminAPI().QueryDecommissionToken(); err != nil {
|
||||
@ -480,3 +480,77 @@ func newClusterSetDecommissionDiskLimitCmd(client *master.MasterClient) *cobra.C
|
||||
}
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newClusterEnableAutoDecommissionDisk(client *master.MasterClient) *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: CliOpEnableAutoDecommission + " [STATUS]",
|
||||
ValidArgs: []string{"true", "false"},
|
||||
Short: cmdEnableAutoDecommissionDiskShort,
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
var (
|
||||
err error
|
||||
enable bool
|
||||
)
|
||||
defer func() {
|
||||
errout(err)
|
||||
}()
|
||||
if enable, err = strconv.ParseBool(args[0]); err != nil {
|
||||
return
|
||||
}
|
||||
if err = client.AdminAPI().SetAutoDecommissionDisk(enable); err != nil {
|
||||
return
|
||||
}
|
||||
if enable {
|
||||
stdout("Enable auto decommission successful!\n")
|
||||
} else {
|
||||
stdout("Disable auto decommission successful!\n")
|
||||
}
|
||||
},
|
||||
}
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newClusterQueryDecommissionFailedDisk(client *master.MasterClient) *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: CliOpQueryDecommissionFailedDisk + " [TYPE]",
|
||||
Short: cmdQueryDecommissionFailedDiskShort,
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
var (
|
||||
err error
|
||||
decommType int
|
||||
)
|
||||
|
||||
defer func() {
|
||||
errout(err)
|
||||
}()
|
||||
|
||||
args[0] = strings.ToLower(args[0])
|
||||
if args[0] != "auto" && args[0] != "manual" && args[0] != "all" {
|
||||
err = fmt.Errorf("unknown decommission type %v, not \"auto\", \"manual\" and \"and\"", args[0])
|
||||
return
|
||||
}
|
||||
|
||||
switch args[0] {
|
||||
case "manual":
|
||||
decommType = 0
|
||||
case "auto":
|
||||
decommType = 1
|
||||
case "all":
|
||||
decommType = 2
|
||||
}
|
||||
|
||||
diskInfo, err := client.AdminAPI().QueryDecommissionFailedDisk(decommType)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
stdout("FailedDisks:\n")
|
||||
for i, d := range diskInfo {
|
||||
stdout("[%v/%v]\n%v", i+1, len(diskInfo), formatDecommissionFailedDiskInfo(d))
|
||||
}
|
||||
},
|
||||
}
|
||||
return cmd
|
||||
}
|
||||
|
||||
@ -132,7 +132,7 @@ const (
|
||||
CliFlagClientIDKey = "clientIDKey"
|
||||
CliFlagMarkDiskBrokenThreshold = "markBrokenDiskThreshold"
|
||||
CliFlagForce = "force"
|
||||
CliFlagEnableCrossZone = "cross-zone"
|
||||
CliFlagEnableCrossZone = "cross-zone"
|
||||
|
||||
// CliFlagSetDataPartitionCount = "count" use dp-count instead
|
||||
|
||||
|
||||
@ -1048,7 +1048,7 @@ func newVolSetAuditLogCmd(client *master.MasterClient) *cobra.Command {
|
||||
var err error
|
||||
defer func() {
|
||||
if err != nil {
|
||||
errout("Error:%v", err)
|
||||
errout(err)
|
||||
}
|
||||
}()
|
||||
enable, err := strconv.ParseBool(settingStr)
|
||||
@ -1105,7 +1105,6 @@ func newVolSetTrashIntervalCmd(client *master.MasterClient) *cobra.Command {
|
||||
Short: cmdVolSetTrashIntervalShort,
|
||||
Args: cobra.MinimumNArgs(2),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
|
||||
var (
|
||||
err error
|
||||
interval time.Duration
|
||||
@ -1115,7 +1114,7 @@ func newVolSetTrashIntervalCmd(client *master.MasterClient) *cobra.Command {
|
||||
name := args[0]
|
||||
defer func() {
|
||||
if err != nil {
|
||||
errout("%v\n", err)
|
||||
errout(err)
|
||||
}
|
||||
}()
|
||||
|
||||
|
||||
@ -109,8 +109,8 @@ const (
|
||||
// NewSuper returns a new Super.
|
||||
func NewSuper(opt *proto.MountOptions) (s *Super, err error) {
|
||||
s = new(Super)
|
||||
var masters = strings.Split(opt.Master, meta.HostsSeparator)
|
||||
var metaConfig = &meta.MetaConfig{
|
||||
masters := strings.Split(opt.Master, meta.HostsSeparator)
|
||||
metaConfig := &meta.MetaConfig{
|
||||
Volume: opt.Volname,
|
||||
Owner: opt.Owner,
|
||||
Masters: masters,
|
||||
@ -119,7 +119,7 @@ func NewSuper(opt *proto.MountOptions) (s *Super, err error) {
|
||||
ValidateOwner: opt.Authenticate || opt.AccessKey == "",
|
||||
EnableSummary: opt.EnableSummary && opt.EnableXattr,
|
||||
MetaSendTimeout: opt.MetaSendTimeout,
|
||||
//EnableTransaction: opt.EnableTransaction,
|
||||
// EnableTransaction: opt.EnableTransaction,
|
||||
SubDir: opt.SubDir,
|
||||
TrashRebuildGoroutineLimit: int(opt.TrashRebuildGoroutineLimit),
|
||||
TrashTraverseLimit: int(opt.TrashDeleteExpiredDirGoroutineLimit),
|
||||
@ -502,9 +502,7 @@ func (s *Super) SetResume(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Super) DisableTrash(w http.ResponseWriter, r *http.Request) {
|
||||
var (
|
||||
err error
|
||||
)
|
||||
var err error
|
||||
if err = r.ParseForm(); err != nil {
|
||||
replyFail(w, r, err.Error())
|
||||
return
|
||||
|
||||
@ -23,7 +23,6 @@ package main
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
syslog "log"
|
||||
"math"
|
||||
@ -927,6 +926,6 @@ func loadConfFromMaster(opt *proto.MountOptions) (err error) {
|
||||
}
|
||||
opt.EbsEndpoint = clusterInfo.EbsAddr
|
||||
opt.EbsServicePath = clusterInfo.ServicePath
|
||||
opt.TrashInterval = int64(util.Min(int(opt.TrashInterval), volumeInfo.TrashInterval))
|
||||
// opt.TrashInterval = int64(util.Min(int(opt.TrashInterval), volumeInfo.TrashInterval))
|
||||
return
|
||||
}
|
||||
|
||||
@ -476,7 +476,7 @@ func (f *vfsgen۰CompressedFileInfo) GzipBytes() []byte {
|
||||
|
||||
func (f *vfsgen۰CompressedFileInfo) Name() string { return f.name }
|
||||
func (f *vfsgen۰CompressedFileInfo) Size() int64 { return f.uncompressedSize }
|
||||
func (f *vfsgen۰CompressedFileInfo) Mode() os.FileMode { return 0444 }
|
||||
func (f *vfsgen۰CompressedFileInfo) Mode() os.FileMode { return 0o444 }
|
||||
func (f *vfsgen۰CompressedFileInfo) ModTime() time.Time { return f.modTime }
|
||||
func (f *vfsgen۰CompressedFileInfo) IsDir() bool { return false }
|
||||
func (f *vfsgen۰CompressedFileInfo) Sys() interface{} { return nil }
|
||||
@ -511,6 +511,7 @@ func (f *vfsgen۰CompressedFile) Read(p []byte) (n int, err error) {
|
||||
f.seekPos = f.grPos
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (f *vfsgen۰CompressedFile) Seek(offset int64, whence int) (int64, error) {
|
||||
switch whence {
|
||||
case io.SeekStart:
|
||||
@ -524,6 +525,7 @@ func (f *vfsgen۰CompressedFile) Seek(offset int64, whence int) (int64, error) {
|
||||
}
|
||||
return f.seekPos, nil
|
||||
}
|
||||
|
||||
func (f *vfsgen۰CompressedFile) Close() error {
|
||||
return f.gr.Close()
|
||||
}
|
||||
@ -544,7 +546,7 @@ func (f *vfsgen۰FileInfo) NotWorthGzipCompressing() {}
|
||||
|
||||
func (f *vfsgen۰FileInfo) Name() string { return f.name }
|
||||
func (f *vfsgen۰FileInfo) Size() int64 { return int64(len(f.content)) }
|
||||
func (f *vfsgen۰FileInfo) Mode() os.FileMode { return 0444 }
|
||||
func (f *vfsgen۰FileInfo) Mode() os.FileMode { return 0o444 }
|
||||
func (f *vfsgen۰FileInfo) ModTime() time.Time { return f.modTime }
|
||||
func (f *vfsgen۰FileInfo) IsDir() bool { return false }
|
||||
func (f *vfsgen۰FileInfo) Sys() interface{} { return nil }
|
||||
@ -574,7 +576,7 @@ func (d *vfsgen۰DirInfo) Stat() (os.FileInfo, error) { return d, nil }
|
||||
|
||||
func (d *vfsgen۰DirInfo) Name() string { return d.name }
|
||||
func (d *vfsgen۰DirInfo) Size() int64 { return 0 }
|
||||
func (d *vfsgen۰DirInfo) Mode() os.FileMode { return 0755 | os.ModeDir }
|
||||
func (d *vfsgen۰DirInfo) Mode() os.FileMode { return 0o755 | os.ModeDir }
|
||||
func (d *vfsgen۰DirInfo) ModTime() time.Time { return d.modTime }
|
||||
func (d *vfsgen۰DirInfo) IsDir() bool { return true }
|
||||
func (d *vfsgen۰DirInfo) Sys() interface{} { return nil }
|
||||
|
||||
@ -71,7 +71,8 @@ type ListFileInfo struct {
|
||||
func (fs *FileService) listFile(ctx context.Context, args struct {
|
||||
VolName string
|
||||
Request ListFilesV1Option
|
||||
}) (*ListFileInfo, error) {
|
||||
},
|
||||
) (*ListFileInfo, error) {
|
||||
userInfo, _, err := permissions(ctx, USER|ADMIN)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -250,7 +251,8 @@ func (fs *FileService) signURL(ctx context.Context, args struct {
|
||||
VolName string
|
||||
Path string
|
||||
ExpireMinutes int64
|
||||
}) (*proto.GeneralResp, error) {
|
||||
},
|
||||
) (*proto.GeneralResp, error) {
|
||||
if args.Path == "" || args.ExpireMinutes <= 0 || args.VolName == "" {
|
||||
return nil, fmt.Errorf("param has err")
|
||||
}
|
||||
|
||||
@ -35,7 +35,8 @@ func (ls *LoginService) login(ctx context.Context, args struct {
|
||||
UserID string
|
||||
Password string
|
||||
empty bool
|
||||
}) (*UserToken, error) {
|
||||
},
|
||||
) (*UserToken, error) {
|
||||
_, err := ls.client.ValidatePassword(ctx, args.UserID, args.Password)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@ -32,7 +32,8 @@ func NewMonitorService(addr, app, cluster, dashboardAddr string) *MonitorService
|
||||
|
||||
func (fs *MonitorService) empty(ctx context.Context, args struct {
|
||||
Empty bool
|
||||
}) bool {
|
||||
},
|
||||
) bool {
|
||||
return args.Empty
|
||||
}
|
||||
|
||||
|
||||
@ -186,7 +186,8 @@ func (dp *DataPartition) getLocalExtentInfo(extentType uint8, tinyExtents []uint
|
||||
}
|
||||
|
||||
func (dp *DataPartition) getRemoteExtentInfo(extentType uint8, tinyExtents []uint64,
|
||||
target string) (extentFiles []*storage.ExtentInfo, err error) {
|
||||
target string,
|
||||
) (extentFiles []*storage.ExtentInfo, err error) {
|
||||
p := repl.NewPacketToGetAllWatermarks(dp.partitionID, extentType)
|
||||
extentFiles = make([]*storage.ExtentInfo, 0)
|
||||
if proto.IsTinyExtentType(extentType) {
|
||||
@ -512,7 +513,7 @@ func (dp *DataPartition) ExtentWithHoleRepairRead(request repl.PacketInterface,
|
||||
reply.SetData(make([]byte, currReadSize))
|
||||
}
|
||||
reply.SetExtentOffset(offset)
|
||||
crc, err = dp.extentStore.Read(reply.GetExtentID(), offset, int64(currReadSize), reply.GetData(), false)
|
||||
crc, err = dp.extentStore.Read(reply.GetExtentID(), offset, int64(currReadSize), reply.GetData(), false, request.GetOpcode() == proto.OpBackupRead)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@ -539,7 +540,8 @@ func (dp *DataPartition) ExtentWithHoleRepairRead(request repl.PacketInterface,
|
||||
}
|
||||
|
||||
func (dp *DataPartition) NormalExtentRepairRead(p repl.PacketInterface, connect net.Conn, isRepairRead bool,
|
||||
metrics *DataNodeMetrics, makeRspPacket repl.MakeStreamReadResponsePacket) (err error) {
|
||||
metrics *DataNodeMetrics, makeRspPacket repl.MakeStreamReadResponsePacket,
|
||||
) (err error) {
|
||||
var (
|
||||
metricPartitionIOLabels map[string]string
|
||||
partitionIOMetric, tpObject *exporter.TimePointCount
|
||||
@ -593,7 +595,7 @@ func (dp *DataPartition) NormalExtentRepairRead(p repl.PacketInterface, connect
|
||||
|
||||
dp.disk.limitRead.Run(int(currReadSize), func() {
|
||||
var crc uint32
|
||||
crc, err = store.Read(reply.GetExtentID(), offset, int64(currReadSize), reply.GetData(), isRepairRead)
|
||||
crc, err = store.Read(reply.GetExtentID(), offset, int64(currReadSize), reply.GetData(), isRepairRead, p.GetOpcode() == proto.OpBackupRead)
|
||||
reply.SetCRC(crc)
|
||||
})
|
||||
if !shallDegrade && metrics != nil {
|
||||
@ -681,7 +683,6 @@ func (dp *DataPartition) applyRepairKey(extentID int) (m string) {
|
||||
return fmt.Sprintf("ApplyRepairKey(%v_%v)", dp.partitionID, extentID)
|
||||
}
|
||||
|
||||
// The actual repair of an extent happens here.
|
||||
// The actual repair of an extent happens here.
|
||||
func (dp *DataPartition) streamRepairExtent(remoteExtentInfo *storage.ExtentInfo,
|
||||
tinyPackFunc, normalPackFunc, normalWithHoleFunc repl.MakeExtentRepairReadPacket,
|
||||
@ -840,7 +841,7 @@ func (dp *DataPartition) streamRepairExtent(remoteExtentInfo *storage.ExtentInfo
|
||||
}
|
||||
} else {
|
||||
log.LogDebugf("streamRepairExtent reply size %v, currFixoffset %v, reply %v ", reply.GetSize(), currFixOffset, reply)
|
||||
_, err = store.Write(uint64(localExtentInfo.FileID), int64(currFixOffset), int64(reply.GetSize()), reply.GetData(), reply.GetCRC(), wType, BufferWrite, isEmptyResponse, true)
|
||||
_, err = store.Write(uint64(localExtentInfo.FileID), int64(currFixOffset), int64(reply.GetSize()), reply.GetData(), reply.GetCRC(), wType, BufferWrite, isEmptyResponse, true, request.GetOpcode() == proto.OpBackupWrite)
|
||||
}
|
||||
// log.LogDebugf("streamRepairExtent reply size %v, currFixoffset %v, reply %v err %v", reply.Size, currFixOffset, reply, err)
|
||||
// write to the local extent file
|
||||
|
||||
@ -262,15 +262,15 @@ func mockMakeDp(path string) *DataPartition {
|
||||
|
||||
func extentStoreNormalRwTest(t *testing.T, s *storage.ExtentStore, id uint64, crc uint32, data []byte) {
|
||||
// append write
|
||||
_, err := s.Write(id, 0, int64(len(data)), data, crc, storage.AppendWriteType, true, false, false)
|
||||
_, err := s.Write(id, 0, int64(len(data)), data, crc, storage.AppendWriteType, true, false, false, false)
|
||||
require.NoError(t, err)
|
||||
actualCrc, err := s.Read(id, 0, int64(len(data)), data, false)
|
||||
actualCrc, err := s.Read(id, 0, int64(len(data)), data, false, false)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, crc, actualCrc)
|
||||
// random write
|
||||
_, err = s.Write(id, 0, int64(len(data)), data, crc, storage.RandomWriteType, true, false, false)
|
||||
_, err = s.Write(id, 0, int64(len(data)), data, crc, storage.RandomWriteType, true, false, false, false)
|
||||
require.NoError(t, err)
|
||||
actualCrc, err = s.Read(id, 0, int64(len(data)), data, false)
|
||||
actualCrc, err = s.Read(id, 0, int64(len(data)), data, false, false)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, crc, actualCrc)
|
||||
// TODO: append random write
|
||||
@ -291,27 +291,27 @@ func extentReloadCheckNormalCrc(t *testing.T, s *storage.ExtentStore, id uint64,
|
||||
func extentStoreSnapshotRwTest(t *testing.T, s *storage.ExtentStore, id uint64, crc uint32, data []byte) {
|
||||
// append write
|
||||
offset := int64(util.ExtentSize)
|
||||
_, err := s.Write(id, offset, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false)
|
||||
_, err := s.Write(id, offset, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = s.Write(id, 0, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false)
|
||||
_, err = s.Write(id, 0, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false, false)
|
||||
assert.True(t, err != nil)
|
||||
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.AppendWriteType, true, false, false)
|
||||
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.AppendWriteType, true, false, false, false)
|
||||
assert.True(t, err != nil)
|
||||
|
||||
actualCrc, err := s.Read(id, offset, int64(len(data)), data, false)
|
||||
actualCrc, err := s.Read(id, offset, int64(len(data)), data, false, false)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, crc, actualCrc)
|
||||
// random write
|
||||
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.RandomWriteType, true, false, false)
|
||||
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.RandomWriteType, true, false, false, false)
|
||||
require.NoError(t, err)
|
||||
actualCrc, err = s.Read(id, offset, int64(len(data)), data, false)
|
||||
actualCrc, err = s.Read(id, offset, int64(len(data)), data, false, false)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, crc, actualCrc)
|
||||
// TODO: append random write
|
||||
require.NotEqualValues(t, s.GetStoreUsedSize(), 0)
|
||||
|
||||
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false)
|
||||
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
// extent crc check
|
||||
@ -323,7 +323,7 @@ func extentStoreSnapshotRwTest(t *testing.T, s *storage.ExtentStore, id uint64,
|
||||
|
||||
// check
|
||||
offset = int64(util.ExtentSize)*2 + util.BlockSize
|
||||
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false)
|
||||
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
e, err = s.LoadExtentFromDisk(id, true)
|
||||
@ -449,7 +449,7 @@ func testDoRepair(t *testing.T, normalId uint64) {
|
||||
go func(role string) {
|
||||
defer func() {
|
||||
wg.Done()
|
||||
t.Logf("TestExtentRepair role %vfinished", role)
|
||||
t.Logf("TestExtentRepair role %v finished", role)
|
||||
}()
|
||||
t.Logf("TestExtentRepair role %v start", role)
|
||||
senderRepairWorker(t, exitCh)
|
||||
|
||||
@ -17,6 +17,7 @@ package datanode
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
syslog "log"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
@ -27,6 +28,8 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/cubefs/cubefs/util/exporter"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"github.com/cubefs/cubefs/proto"
|
||||
@ -100,7 +103,8 @@ const (
|
||||
type PartitionVisitor func(dp *DataPartition)
|
||||
|
||||
func NewDisk(path string, reservedSpace, diskRdonlySpace uint64, maxErrCnt int, space *SpaceManager,
|
||||
diskEnableReadRepairExtentLimit bool) (d *Disk, err error) {
|
||||
diskEnableReadRepairExtentLimit bool,
|
||||
) (d *Disk, err error) {
|
||||
d = new(Disk)
|
||||
d.Path = path
|
||||
d.ReservedSpace = reservedSpace
|
||||
@ -568,7 +572,7 @@ type dpLoadInfo struct {
|
||||
|
||||
// RestorePartition reads the files stored on the local disk and restores the data partitions.
|
||||
func (d *Disk) RestorePartition(visitor PartitionVisitor) (err error) {
|
||||
var convert = func(node *proto.DataNodeInfo) *DataNodeInfo {
|
||||
convert := func(node *proto.DataNodeInfo) *DataNodeInfo {
|
||||
result := &DataNodeInfo{}
|
||||
result.Addr = node.Addr
|
||||
result.PersistenceDataPartitions = node.PersistenceDataPartitions
|
||||
|
||||
@ -25,9 +25,11 @@ import (
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
raftProto "github.com/cubefs/cubefs/depends/tiglabs/raft/proto"
|
||||
@ -167,7 +169,6 @@ func (dp *DataPartition) SetRepairBlockSize(size uint64) {
|
||||
}
|
||||
|
||||
func CreateDataPartition(dpCfg *dataPartitionCfg, disk *Disk, request *proto.CreateDataPartitionRequest) (dp *DataPartition, err error) {
|
||||
|
||||
if dp, err = newDataPartition(dpCfg, disk, true); err != nil {
|
||||
return
|
||||
}
|
||||
@ -245,9 +246,7 @@ func (dp *DataPartition) ForceSetRaftRunning() {
|
||||
// It reads the partition metadata file stored under the specified directory
|
||||
// and creates the partition instance.
|
||||
func LoadDataPartition(partitionDir string, disk *Disk) (dp *DataPartition, err error) {
|
||||
var (
|
||||
metaFileData []byte
|
||||
)
|
||||
var metaFileData []byte
|
||||
if metaFileData, err = ioutil.ReadFile(path.Join(partitionDir, DataPartitionMetadataFileName)); err != nil {
|
||||
return
|
||||
}
|
||||
@ -820,9 +819,7 @@ func (dp *DataPartition) statusUpdate() {
|
||||
}
|
||||
|
||||
func parseFileName(filename string) (extentID uint64, isExtent bool) {
|
||||
var (
|
||||
err error
|
||||
)
|
||||
var err error
|
||||
if extentID, err = strconv.ParseUint(filename, 10, 64); err != nil {
|
||||
isExtent = false
|
||||
return
|
||||
@ -855,13 +852,6 @@ func (dp *DataPartition) computeUsage() {
|
||||
log.LogDebugf("[computeUsage] dp(%v) skip size update", dp.partitionID)
|
||||
return
|
||||
}
|
||||
used, err := dp.ExtentStore().GetStoreUsedSize()
|
||||
if err != nil {
|
||||
log.LogErrorf("[computeUsage] dp(%v) failed to compute extents size, err(%v)", dp.partitionID, err)
|
||||
return
|
||||
}
|
||||
dp.used = int(used)
|
||||
dp.intervalToUpdatePartitionSize = time.Now().Unix()
|
||||
dp.used = int(dp.ExtentStore().GetStoreUsedSize())
|
||||
if log.EnableDebug() {
|
||||
log.LogDebugf("[computeUsage] dp(%v) update size(%v)", dp.partitionID, strutil.FormatSize(uint64(dp.used)))
|
||||
@ -1317,18 +1307,13 @@ func (vo *VolMap) getSimpleVolViewWithRetry(dp *DataPartition) (vv *proto.Simple
|
||||
return
|
||||
}
|
||||
|
||||
func (dp *DataPartition) doExtentTtl(ttl int) (err error) {
|
||||
func (dp *DataPartition) doExtentTtl(ttl int) {
|
||||
if ttl <= 0 {
|
||||
log.LogWarn("[doTTL] ttl is 0, set default 30", ttl)
|
||||
ttl = 30
|
||||
}
|
||||
|
||||
extents, err := dp.extentStore.DumpExtents()
|
||||
if err != nil {
|
||||
log.LogErrorf("[doExtentTtl] dp(%v) failed to dump extents, err(%v)", dp.partitionID, err)
|
||||
return
|
||||
}
|
||||
|
||||
extents := dp.extentStore.DumpExtents()
|
||||
for _, ext := range extents {
|
||||
if storage.IsTinyExtent(ext.FileID) {
|
||||
continue
|
||||
@ -1342,7 +1327,7 @@ func (dp *DataPartition) doExtentTtl(ttl int) (err error) {
|
||||
return
|
||||
}
|
||||
|
||||
func (dp *DataPartition) doExtentEvict(vv *proto.SimpleVolView) (err error) {
|
||||
func (dp *DataPartition) doExtentEvict(vv *proto.SimpleVolView) {
|
||||
var (
|
||||
needDieOut bool
|
||||
freeSpace int
|
||||
@ -1368,11 +1353,7 @@ func (dp *DataPartition) doExtentEvict(vv *proto.SimpleVolView) (err error) {
|
||||
|
||||
// if dp extent count larger than upper count, do die out.
|
||||
freeExtentCount = 0
|
||||
extInfos, err := dp.extentStore.DumpExtents()
|
||||
if err != nil {
|
||||
log.LogErrorf("[doExtentEvict] dp(%v) failed to dump extents, err(%v)", dp.partitionID, err)
|
||||
return
|
||||
}
|
||||
extInfos := dp.extentStore.DumpExtents()
|
||||
maxExtentCount := dp.Size() / util.DefaultTinySizeLimit
|
||||
if len(extInfos) > maxExtentCount {
|
||||
needDieOut = true
|
||||
@ -1447,21 +1428,13 @@ func (dp *DataPartition) startEvict() {
|
||||
case <-lruTimer.C:
|
||||
log.LogDebugf("start [doExtentEvict] vol(%s), dp(%d).", vv.Name, dp.partitionID)
|
||||
evictStart := time.Now()
|
||||
err = dp.doExtentEvict(vv)
|
||||
if err != nil {
|
||||
log.LogErrorf("[doExtentEvict] vol(%v) dp(%v) failed to handle extent ttl, err(%v)", vv.Name, dp.partitionID, err)
|
||||
continue
|
||||
}
|
||||
dp.doExtentEvict(vv)
|
||||
log.LogDebugf("action[doExtentEvict] vol(%v), dp(%v), cost (%v)ms, .", vv.Name, dp.partitionID, time.Since(evictStart))
|
||||
|
||||
case <-ttlTimer.C:
|
||||
log.LogDebugf("start [doExtentTtl] vol(%s), dp(%d).", vv.Name, dp.partitionID)
|
||||
ttlStart := time.Now()
|
||||
err = dp.doExtentTtl(cacheTtl)
|
||||
if err != nil {
|
||||
log.LogErrorf("[doExtentTtl] vol(%v) dp(%v) failed to handle extent ttl, err(%v)", vv.Name, dp.partitionID, err)
|
||||
continue
|
||||
}
|
||||
dp.doExtentTtl(cacheTtl)
|
||||
log.LogDebugf("action[doExtentTtl] vol(%v), dp(%v), cost (%v)ms.", vv.Name, dp.partitionID, time.Since(ttlStart))
|
||||
|
||||
case <-dp.stopC:
|
||||
|
||||
@ -231,7 +231,7 @@ func (dp *DataPartition) ApplyRandomWrite(command []byte, raftApplyID uint64) (r
|
||||
} else {
|
||||
if err == storage.ErrStoreAlreadyClosed {
|
||||
err = nil
|
||||
resp = proto.OpStoreClosed
|
||||
respStatus = proto.OpStoreClosed
|
||||
log.LogWarnf("[ApplyRandomWrite] vol(%v) dp(%v) apply id(%v) store already closed", dp.volumeID, dp.partitionID, raftApplyID)
|
||||
return
|
||||
}
|
||||
@ -282,7 +282,7 @@ func (dp *DataPartition) ApplyRandomWrite(command []byte, raftApplyID uint64) (r
|
||||
}
|
||||
|
||||
dp.disk.limitWrite.Run(int(opItem.size), func() {
|
||||
respStatus, err = dp.ExtentStore().Write(opItem.extentID, opItem.offset, opItem.size, opItem.data, opItem.crc, writeType, syncWrite, false, false)
|
||||
respStatus, err = dp.ExtentStore().Write(opItem.extentID, opItem.offset, opItem.size, opItem.data, opItem.crc, writeType, syncWrite, false, false, opItem.opcode == proto.OpBackupWrite)
|
||||
})
|
||||
if err == nil || err == storage.ErrStoreAlreadyClosed {
|
||||
break
|
||||
|
||||
@ -18,7 +18,6 @@ import (
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
syslog "log"
|
||||
"net"
|
||||
"os"
|
||||
@ -334,13 +333,7 @@ func (dp *DataPartition) StartRaftAfterRepair(isLoad bool) {
|
||||
if currLeaderPartitionSize < initPartitionSize {
|
||||
initPartitionSize = currLeaderPartitionSize
|
||||
}
|
||||
localSize, err := dp.extentStore.StoreSizeExtentID(initMaxExtentID)
|
||||
if err != nil {
|
||||
log.LogErrorf("[StartRaftAfterRepair] dp(%v) failed to get size, err(%v)", dp.partitionID, err)
|
||||
return
|
||||
}
|
||||
|
||||
log.LogInfof("StartRaftAfterRepair PartitionID(%v) initMaxExtentID(%v) initPartitionSize(%v) currLeaderPartitionSize(%v)"+
|
||||
localSize := dp.extentStore.StoreSizeExtentID(initMaxExtentID)
|
||||
dp.decommissionRepairProgress = float64(localSize) / float64(initPartitionSize)
|
||||
log.LogInfof("action[StartRaftAfterRepair] PartitionID(%v) initMaxExtentID(%v) initPartitionSize(%v) currLeaderPartitionSize(%v)"+
|
||||
"localSize(%v)", dp.partitionID, initMaxExtentID, initPartitionSize, currLeaderPartitionSize, localSize)
|
||||
@ -816,7 +809,7 @@ func (dp *DataPartition) getAllReplicaAppliedID() (allAppliedID []uint64, replyN
|
||||
target := replicas[i]
|
||||
appliedID, err := dp.getRemoteAppliedID(target, p)
|
||||
if err != nil {
|
||||
log.LogErrorf("partition(%v) getRemoteAppliedID from replica(%v) Failed(%v).", dp.partitionID, targetReplica, err)
|
||||
log.LogErrorf("partition(%v) getRemoteAppliedID from replica(%v) Failed(%v).", dp.partitionID, target, err)
|
||||
continue
|
||||
}
|
||||
if appliedID == 0 {
|
||||
|
||||
@ -18,7 +18,6 @@ import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
syslog "log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
@ -29,11 +28,6 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"errors"
|
||||
syslog "log"
|
||||
"os"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@ -48,6 +42,7 @@ import (
|
||||
"github.com/cubefs/cubefs/util/exporter"
|
||||
"github.com/cubefs/cubefs/util/loadutil"
|
||||
"github.com/cubefs/cubefs/util/log"
|
||||
syslog "log"
|
||||
|
||||
"github.com/xtaci/smux"
|
||||
)
|
||||
@ -126,8 +121,8 @@ const (
|
||||
// load/stop dp limit
|
||||
ConfigDiskCurrentLoadDpLimit = "diskCurrentLoadDpLimit"
|
||||
ConfigDiskCurrentStopDpLimit = "diskCurrentStopDpLimit"
|
||||
//disk read extent limit
|
||||
ConfigEnableDiskReadExtentLimit = "enableDiskReadRepairExtentLimit" //bool
|
||||
// disk read extent limit
|
||||
ConfigEnableDiskReadExtentLimit = "enableDiskReadRepairExtentLimit" // bool
|
||||
|
||||
ConfigServiceIDKey = "serviceIDKey"
|
||||
|
||||
@ -263,7 +258,7 @@ func doStart(server common.Server, cfg *config.Config) (err error) {
|
||||
return
|
||||
}
|
||||
|
||||
//smux listening & smux connection pool
|
||||
// smux listening & smux connection pool
|
||||
if err = s.startSmuxService(cfg); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
@ -22,7 +22,6 @@ import (
|
||||
"path"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
"syscall"
|
||||
|
||||
"github.com/cubefs/cubefs/util/log"
|
||||
@ -634,174 +633,6 @@ func (s *DataNode) reloadDataPartition(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DataNode) setDiskExtentReadLimitStatus(w http.ResponseWriter, r *http.Request) {
|
||||
const (
|
||||
paramStatus = "status"
|
||||
)
|
||||
if err := r.ParseForm(); err != nil {
|
||||
err = fmt.Errorf("parse form fail: %v", err)
|
||||
s.buildFailureResp(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
status, err := strconv.ParseBool(r.FormValue(paramStatus))
|
||||
if err != nil {
|
||||
err = fmt.Errorf("parse param %v fail: %v", paramStatus, err)
|
||||
s.buildFailureResp(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
for _, disk := range s.space.disks {
|
||||
disk.SetExtentRepairReadLimitStatus(status)
|
||||
}
|
||||
s.buildSuccessResp(w, "success")
|
||||
}
|
||||
|
||||
type DiskExtentReadLimitInfo struct {
|
||||
DiskPath string `json:"diskPath"`
|
||||
ExtentReadLimitStatus bool `json:"extentReadLimitStatus"`
|
||||
Dp uint64 `json:"dp"`
|
||||
}
|
||||
|
||||
type DiskExtentReadLimitStatusResponse struct {
|
||||
Infos []DiskExtentReadLimitInfo `json:"infos"`
|
||||
}
|
||||
|
||||
func (s *DataNode) queryDiskExtentReadLimitStatus(w http.ResponseWriter, r *http.Request) {
|
||||
resp := &DiskExtentReadLimitStatusResponse{}
|
||||
for _, disk := range s.space.disks {
|
||||
status, dp := disk.QueryExtentRepairReadLimitStatus()
|
||||
resp.Infos = append(resp.Infos, DiskExtentReadLimitInfo{DiskPath: disk.Path, ExtentReadLimitStatus: status, Dp: dp})
|
||||
}
|
||||
s.buildSuccessResp(w, resp)
|
||||
}
|
||||
|
||||
func (s *DataNode) detachDataPartition(w http.ResponseWriter, r *http.Request) {
|
||||
const (
|
||||
paramID = "id"
|
||||
)
|
||||
if err := r.ParseForm(); err != nil {
|
||||
err = fmt.Errorf("parse form fail: %v", err)
|
||||
s.buildFailureResp(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
partitionID, err := strconv.ParseUint(r.FormValue(paramID), 10, 64)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("parse param %v fail: %v", paramID, err)
|
||||
s.buildFailureResp(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
partition := s.space.Partition(partitionID)
|
||||
if partition == nil {
|
||||
s.buildFailureResp(w, http.StatusBadRequest, "partition not exist")
|
||||
return
|
||||
}
|
||||
// store disk path and root of dp
|
||||
disk := partition.disk
|
||||
rootDir := partition.path
|
||||
log.LogDebugf("data partition disk %v rootDir %v", disk, rootDir)
|
||||
|
||||
s.space.partitionMutex.Lock()
|
||||
delete(s.space.partitions, partitionID)
|
||||
s.space.partitionMutex.Unlock()
|
||||
partition.Stop()
|
||||
partition.Disk().DetachDataPartition(partition)
|
||||
|
||||
log.LogDebugf("data partition %v is detached", partitionID)
|
||||
_, err = LoadDataPartition(rootDir, disk)
|
||||
s.buildSuccessResp(w, "success")
|
||||
}
|
||||
|
||||
func (s *DataNode) releaseDiskExtentReadLimitToken(w http.ResponseWriter, r *http.Request) {
|
||||
const (
|
||||
paramDisk = "disk"
|
||||
)
|
||||
if err := r.ParseForm(); err != nil {
|
||||
err = fmt.Errorf("parse form fail: %v", err)
|
||||
s.buildFailureResp(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
diskPath := r.FormValue(paramDisk)
|
||||
// store disk path and root of dp
|
||||
disk, err := s.space.GetDisk(diskPath)
|
||||
if err != nil {
|
||||
log.LogErrorf("action[loadDataPartition] disk(%v) is not found err(%v).", diskPath, err)
|
||||
s.buildFailureResp(w, http.StatusBadRequest, fmt.Sprintf("disk %v is not found", diskPath))
|
||||
return
|
||||
}
|
||||
disk.ReleaseReadExtentToken()
|
||||
s.buildSuccessResp(w, "success")
|
||||
}
|
||||
|
||||
func (s *DataNode) loadDataPartition(w http.ResponseWriter, r *http.Request) {
|
||||
const (
|
||||
paramID = "id"
|
||||
paramDisk = "disk"
|
||||
)
|
||||
if err := r.ParseForm(); err != nil {
|
||||
err = fmt.Errorf("parse form fail: %v", err)
|
||||
s.buildFailureResp(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
partitionID, err := strconv.ParseUint(r.FormValue(paramID), 10, 64)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("parse param %v fail: %v", paramID, err)
|
||||
s.buildFailureResp(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
partition := s.space.Partition(partitionID)
|
||||
if partition != nil {
|
||||
s.buildFailureResp(w, http.StatusBadRequest, "partition is already loaded")
|
||||
return
|
||||
}
|
||||
diskPath := r.FormValue(paramDisk)
|
||||
// store disk path and root of dp
|
||||
disk, err := s.space.GetDisk(diskPath)
|
||||
if err != nil {
|
||||
log.LogErrorf("action[loadDataPartition] disk(%v) is not found err(%v).", diskPath, err)
|
||||
s.buildFailureResp(w, http.StatusBadRequest, fmt.Sprintf("disk %v is not found", diskPath))
|
||||
return
|
||||
}
|
||||
|
||||
fileInfoList, err := os.ReadDir(disk.Path)
|
||||
if err != nil {
|
||||
log.LogErrorf("action[loadDataPartition] read dir(%v) err(%v).", disk.Path, err)
|
||||
s.buildFailureResp(w, http.StatusBadRequest, fmt.Sprintf(" read dir(%v) err(%v)", disk.Path, err))
|
||||
return
|
||||
}
|
||||
rootDir := ""
|
||||
for _, fileInfo := range fileInfoList {
|
||||
filename := fileInfo.Name()
|
||||
if !disk.isPartitionDir(filename) {
|
||||
if disk.isExpiredPartitionDir(filename) {
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if id, _, err := unmarshalPartitionName(filename); err != nil {
|
||||
log.LogErrorf("action[RestorePartition] unmarshal partitionName(%v) from disk(%v) err(%v) ",
|
||||
filename, disk.Path, err.Error())
|
||||
continue
|
||||
} else {
|
||||
if id == partitionID {
|
||||
rootDir = filename
|
||||
}
|
||||
}
|
||||
}
|
||||
if rootDir == "" {
|
||||
log.LogErrorf("action[loadDataPartition] dp root not found in dir(%v) .", disk.Path)
|
||||
s.buildFailureResp(w, http.StatusBadRequest, fmt.Sprintf("dp root not found in dir(%v)", disk.Path))
|
||||
return
|
||||
}
|
||||
|
||||
log.LogDebugf("data partition disk %v rootDir %v", disk, rootDir)
|
||||
|
||||
_, err = LoadDataPartition(path.Join(diskPath, rootDir), disk)
|
||||
if err != nil {
|
||||
s.buildFailureResp(w, http.StatusBadRequest, err.Error())
|
||||
} else {
|
||||
s.buildSuccessResp(w, "success")
|
||||
}
|
||||
}
|
||||
|
||||
// NOTE: use for test
|
||||
func (s *DataNode) markDataPartitionBroken(w http.ResponseWriter, r *http.Request) {
|
||||
const (
|
||||
@ -931,7 +762,7 @@ func (s *DataNode) detachDataPartition(w http.ResponseWriter, r *http.Request) {
|
||||
s.buildFailureResp(w, http.StatusBadRequest, "partition not exist")
|
||||
return
|
||||
}
|
||||
//store disk path and root of dp
|
||||
// store disk path and root of dp
|
||||
disk := partition.disk
|
||||
rootDir := partition.path
|
||||
log.LogDebugf("data partition disk %v rootDir %v", disk, rootDir)
|
||||
@ -956,7 +787,7 @@ func (s *DataNode) releaseDiskExtentReadLimitToken(w http.ResponseWriter, r *htt
|
||||
return
|
||||
}
|
||||
diskPath := r.FormValue(paramDisk)
|
||||
//store disk path and root of dp
|
||||
// store disk path and root of dp
|
||||
disk, err := s.space.GetDisk(diskPath)
|
||||
if err != nil {
|
||||
log.LogErrorf("action[loadDataPartition] disk(%v) is not found err(%v).", diskPath, err)
|
||||
@ -989,7 +820,7 @@ func (s *DataNode) loadDataPartition(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
diskPath := r.FormValue(paramDisk)
|
||||
//store disk path and root of dp
|
||||
// store disk path and root of dp
|
||||
disk, err := s.space.GetDisk(diskPath)
|
||||
if err != nil {
|
||||
log.LogErrorf("action[loadDataPartition] disk(%v) is not found err(%v).", diskPath, err)
|
||||
@ -1036,5 +867,4 @@ func (s *DataNode) loadDataPartition(w http.ResponseWriter, r *http.Request) {
|
||||
} else {
|
||||
s.buildSuccessResp(w, "success")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -17,24 +17,20 @@ package datanode
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path"
|
||||
"math/rand"
|
||||
"sync"
|
||||
"math"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"math"
|
||||
"os"
|
||||
|
||||
syslog "log"
|
||||
|
||||
"github.com/cubefs/cubefs/proto"
|
||||
"github.com/cubefs/cubefs/raftstore"
|
||||
"github.com/cubefs/cubefs/util"
|
||||
"github.com/cubefs/cubefs/util/atomicutil"
|
||||
"github.com/cubefs/cubefs/util/loadutil"
|
||||
"github.com/cubefs/cubefs/util"
|
||||
"github.com/cubefs/cubefs/util/log"
|
||||
"github.com/shirou/gopsutil/disk"
|
||||
)
|
||||
@ -273,7 +269,8 @@ func (manager *SpaceManager) Stats() *Stats {
|
||||
}
|
||||
|
||||
func (manager *SpaceManager) LoadDisk(path string, reservedSpace, diskRdonlySpace uint64, maxErrCnt int,
|
||||
diskEnableReadRepairExtentLimit bool) (err error) {
|
||||
diskEnableReadRepairExtentLimit bool,
|
||||
) (err error) {
|
||||
var (
|
||||
disk *Disk
|
||||
visitor PartitionVisitor
|
||||
|
||||
@ -71,7 +71,8 @@ func (s *Stats) GetConnectionCount() int64 {
|
||||
|
||||
func (s *Stats) updateMetrics(
|
||||
total, used, available, createdPartitionWeights, remainWeightsForCreatePartition,
|
||||
maxWeightsForCreatePartition, dataPartitionCnt uint64) {
|
||||
maxWeightsForCreatePartition, dataPartitionCnt uint64,
|
||||
) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
|
||||
@ -741,11 +741,7 @@ func (s *DataNode) handleBatchMarkDeletePacket(p *repl.Packet, c net.Conn) {
|
||||
store := partition.ExtentStore()
|
||||
if err == nil {
|
||||
for _, ext := range exts {
|
||||
ok, err := store.CanGcDelete(ext.ExtentId)
|
||||
if err != nil {
|
||||
log.LogErrorf("[handleBatchMarkDeletePacket] dp(%v) failed to get extent(%v) info, err(%v)", ext.PartitionId, ext.ExtentId, err)
|
||||
return
|
||||
}
|
||||
ok := store.CanGcDelete(ext.ExtentId)
|
||||
if p.Opcode == proto.OpGcBatchDeleteExtent && ok {
|
||||
log.LogWarnf("handleBatchMarkDeletePacket: ext %d is not in gc status, can't be gc delete, dp %d", ext.ExtentId, ext.PartitionId)
|
||||
err = storage.ParameterMismatchError
|
||||
@ -827,7 +823,7 @@ func (s *DataNode) handleWritePacket(p *repl.Packet) {
|
||||
partition.disk.allocCheckLimit(proto.IopsWriteType, 1)
|
||||
|
||||
if writable := partition.disk.limitWrite.TryRun(int(p.Size), func() {
|
||||
_, err = store.Write(p.ExtentID, p.ExtentOffset, int64(p.Size), p.Data, p.CRC, storage.AppendWriteType, p.IsSyncWrite(), false, false)
|
||||
_, err = store.Write(p.ExtentID, p.ExtentOffset, int64(p.Size), p.Data, p.CRC, storage.AppendWriteType, p.IsSyncWrite(), false, false, false)
|
||||
}); !writable {
|
||||
err = storage.LimitedIoError
|
||||
return
|
||||
@ -849,7 +845,7 @@ func (s *DataNode) handleWritePacket(p *repl.Packet) {
|
||||
partition.disk.allocCheckLimit(proto.IopsWriteType, 1)
|
||||
|
||||
if writable := partition.disk.limitWrite.TryRun(int(p.Size), func() {
|
||||
_, err = store.Write(p.ExtentID, p.ExtentOffset, int64(p.Size), p.Data, p.CRC, storage.AppendWriteType, p.IsSyncWrite(), false, false)
|
||||
_, err = store.Write(p.ExtentID, p.ExtentOffset, int64(p.Size), p.Data, p.CRC, storage.AppendWriteType, p.IsSyncWrite(), false, false, false)
|
||||
}); !writable {
|
||||
err = storage.LimitedIoError
|
||||
return
|
||||
@ -877,7 +873,7 @@ func (s *DataNode) handleWritePacket(p *repl.Packet) {
|
||||
partition.disk.allocCheckLimit(proto.IopsWriteType, 1)
|
||||
|
||||
if writable := partition.disk.limitWrite.TryRun(currSize, func() {
|
||||
_, err = store.Write(p.ExtentID, p.ExtentOffset+int64(offset), int64(currSize), data, crc, storage.AppendWriteType, p.IsSyncWrite(), false, false)
|
||||
_, err = store.Write(p.ExtentID, p.ExtentOffset+int64(offset), int64(currSize), data, crc, storage.AppendWriteType, p.IsSyncWrite(), false, false, false)
|
||||
}); !writable {
|
||||
err = storage.LimitedIoError
|
||||
return
|
||||
@ -1226,12 +1222,7 @@ func (s *DataNode) handlePacketToGetAppliedID(p *repl.Packet) {
|
||||
|
||||
func (s *DataNode) handlePacketToGetPartitionSize(p *repl.Packet) {
|
||||
partition := p.Object.(*DataPartition)
|
||||
usedSize, err := partition.extentStore.StoreSizeExtentID(p.ExtentID)
|
||||
if err != nil {
|
||||
log.LogErrorf("[handlePacketToGetPartitionSize] dp(%v) failed to get extents size, err(%v)", partition.partitionID, err)
|
||||
p.PacketErrorWithBody(proto.OpDiskErr, []byte(err.Error()))
|
||||
return
|
||||
}
|
||||
usedSize := partition.extentStore.StoreSizeExtentID(p.ExtentID)
|
||||
buf := make([]byte, 8)
|
||||
binary.BigEndian.PutUint64(buf, uint64(usedSize))
|
||||
p.AddMesgLog(fmt.Sprintf("partitionSize_(%v)", usedSize))
|
||||
@ -1240,13 +1231,7 @@ func (s *DataNode) handlePacketToGetPartitionSize(p *repl.Packet) {
|
||||
|
||||
func (s *DataNode) handlePacketToGetMaxExtentIDAndPartitionSize(p *repl.Packet) {
|
||||
partition := p.Object.(*DataPartition)
|
||||
maxExtentID, totalPartitionSize, err := partition.extentStore.GetMaxExtentIDAndPartitionSize()
|
||||
if err != nil {
|
||||
log.LogErrorf("[handlePacketToGetMaxExtentIDAndPartitionSize] dp(%v) failed to get extents size, err(%v)", partition.partitionID, err)
|
||||
p.PacketErrorWithBody(proto.OpDiskErr, []byte(err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
maxExtentID, totalPartitionSize := partition.extentStore.GetMaxExtentIDAndPartitionSize()
|
||||
buf := make([]byte, 16)
|
||||
binary.BigEndian.PutUint64(buf[0:8], uint64(maxExtentID))
|
||||
binary.BigEndian.PutUint64(buf[8:16], totalPartitionSize)
|
||||
@ -1675,9 +1660,7 @@ func (s *DataNode) handlePacketToRecoverDataReplicaMeta(p *repl.Packet) {
|
||||
}
|
||||
|
||||
func (s *DataNode) handleBatchLockNormalExtent(p *repl.Packet, connect net.Conn) {
|
||||
var (
|
||||
err error
|
||||
)
|
||||
var err error
|
||||
|
||||
defer func() {
|
||||
if err != nil {
|
||||
@ -1726,9 +1709,7 @@ func (s *DataNode) handleBatchLockNormalExtent(p *repl.Packet, connect net.Conn)
|
||||
}
|
||||
|
||||
func (s *DataNode) handleBatchUnlockNormalExtent(p *repl.Packet, connect net.Conn) {
|
||||
var (
|
||||
err error
|
||||
)
|
||||
var err error
|
||||
|
||||
defer func() {
|
||||
if err != nil {
|
||||
|
||||
@ -442,7 +442,7 @@ services:
|
||||
command:
|
||||
- bash
|
||||
- "-c"
|
||||
- >-
|
||||
- >
|
||||
set -e;
|
||||
cd ${CFSROOT} && make libsdkpre
|
||||
|
||||
@ -456,29 +456,6 @@ services:
|
||||
cd ${CFSROOT} &&
|
||||
/go/bin/goreleaser release --skip-publish --clean
|
||||
|
||||
build_libsdkpre:
|
||||
image: ${IMAGE}
|
||||
volumes:
|
||||
- ../:${CFSROOT}
|
||||
command:
|
||||
- bash
|
||||
- "-c"
|
||||
- >-
|
||||
set -e;
|
||||
cd ${CFSROOT} && make libsdkpre
|
||||
|
||||
goreleaser:
|
||||
image: ${IMAGE}
|
||||
volumes:
|
||||
- ../:${CFSROOT}
|
||||
command:
|
||||
- bash
|
||||
- "-c"
|
||||
- >-
|
||||
set -e;
|
||||
cd ${CFSROOT} &&
|
||||
/go/bin/goreleaser release --skip-publish --clean
|
||||
|
||||
bs_gofumpt:
|
||||
<<: *command
|
||||
command:
|
||||
|
||||
@ -77,9 +77,11 @@ type VerifyInfo struct {
|
||||
VolName string
|
||||
}
|
||||
|
||||
var hostLimit = make(map[string]chan struct{})
|
||||
var cntLimit = 1
|
||||
var hostLk = sync.RWMutex{}
|
||||
var (
|
||||
hostLimit = make(map[string]chan struct{})
|
||||
cntLimit = 1
|
||||
hostLk = sync.RWMutex{}
|
||||
)
|
||||
|
||||
func setHostCntLimit(cnt int) {
|
||||
clearChan()
|
||||
@ -114,7 +116,6 @@ func releaseToken(host string) {
|
||||
default:
|
||||
slog.Fatalf("there is not token in chan, host %s", host)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func clearChan() {
|
||||
@ -124,7 +125,7 @@ func clearChan() {
|
||||
}
|
||||
|
||||
func newGCCommand() *cobra.Command {
|
||||
var c = &cobra.Command{
|
||||
c := &cobra.Command{
|
||||
Use: "gc",
|
||||
Short: "gc specified volume",
|
||||
Args: cobra.MinimumNArgs(0),
|
||||
@ -175,7 +176,7 @@ func newGetMpExtentsCmd() *cobra.Command {
|
||||
// var fromMpId string
|
||||
var concurrency uint64
|
||||
|
||||
var cmd = &cobra.Command{
|
||||
cmd := &cobra.Command{
|
||||
Use: "getMpExtents [VOLNAME] [DIR]",
|
||||
Short: "get extents of mp",
|
||||
Args: cobra.MinimumNArgs(2),
|
||||
@ -225,7 +226,7 @@ func getMpInfoById(mpId string) (mpInfo *proto.MetaPartitionInfo, err error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var body = &struct {
|
||||
body := &struct {
|
||||
Code int32 `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data json.RawMessage `json:"data"`
|
||||
@ -352,7 +353,7 @@ func getExtentsByMpId(dir string, volname string, mpId string) {
|
||||
slog.Fatalf("Init local dir failed, mpId %s, err: %v", mpId, err)
|
||||
}
|
||||
|
||||
normalFile, err := os.OpenFile(NormalFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||
normalFile, err := os.OpenFile(NormalFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
slog.Fatalf("Open file failed, mp %s, err: %v", mpId, err)
|
||||
}
|
||||
@ -449,7 +450,7 @@ func getExtentsByMpId(dir string, volname string, mpId string) {
|
||||
log.LogDebugf("inode:%v, extent:%v", ino.Inode, eks)
|
||||
}
|
||||
|
||||
walkFunc := func(ek proto.ExtentKey) bool {
|
||||
walkFunc := func(i int, ek proto.ExtentKey) bool {
|
||||
if storage.IsTinyExtent(ek.ExtentId) {
|
||||
return true
|
||||
}
|
||||
@ -493,7 +494,7 @@ func InitLocalDir(dir string, volname string, partitionId string, dirType string
|
||||
normalPath := filepath.Join(dir, volname, dirType, normalDir)
|
||||
_, err = os.Stat(normalPath)
|
||||
if os.IsNotExist(err) {
|
||||
if err = os.MkdirAll(normalPath, 0755); err != nil {
|
||||
if err = os.MkdirAll(normalPath, 0o755); err != nil {
|
||||
slog.Fatalf("create dir failed, path %s, err : %v", normalFilePath, err)
|
||||
return
|
||||
}
|
||||
@ -624,7 +625,7 @@ func getDpInfoById(dpId string) (dpInfo *proto.DataPartitionInfo, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
var body = &struct {
|
||||
body := &struct {
|
||||
Code int32 `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data json.RawMessage `json:"data"`
|
||||
@ -693,7 +694,7 @@ func getExtentsByDpId(dir string, volname string, dpId string, beforeTime string
|
||||
return fmt.Errorf("Read response body failed, err: %v", err)
|
||||
}
|
||||
|
||||
var body = &struct {
|
||||
body := &struct {
|
||||
Code int32 `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data json.RawMessage `json:"data"`
|
||||
@ -717,7 +718,7 @@ func getExtentsByDpId(dir string, volname string, dpId string, beforeTime string
|
||||
return
|
||||
}
|
||||
|
||||
normalFile, err := os.OpenFile(normalFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||
normalFile, err := os.OpenFile(normalFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
log.LogErrorf("Open normalFile failed, err: %v", err)
|
||||
return
|
||||
@ -765,7 +766,7 @@ func getExtentsByDpId(dir string, volname string, dpId string, beforeTime string
|
||||
return
|
||||
}
|
||||
|
||||
err = os.WriteFile(beforeTimeFilePath, []byte(beforeTime), 0644)
|
||||
err = os.WriteFile(beforeTimeFilePath, []byte(beforeTime), 0o644)
|
||||
if err != nil {
|
||||
log.LogErrorf("Write before time file failed, err: %v", err)
|
||||
return
|
||||
@ -825,7 +826,7 @@ func getClusterInfo() (clusterView *proto.ClusterView, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
var body = &struct {
|
||||
body := &struct {
|
||||
Code int32 `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data json.RawMessage `json:"data"`
|
||||
@ -868,7 +869,7 @@ func writeNormalVerifyInfo(dir, volname, dirType string) {
|
||||
return
|
||||
}
|
||||
}
|
||||
err = os.WriteFile(filePath, data, 0644)
|
||||
err = os.WriteFile(filePath, data, 0o644)
|
||||
if err != nil {
|
||||
slog.Fatalf("Write verify info file failed, err: %v", err)
|
||||
return
|
||||
@ -952,7 +953,6 @@ func newCalcBadExtentsCmd() *cobra.Command {
|
||||
slog.Fatalf("Invalid extent type: %s", extType)
|
||||
return
|
||||
}
|
||||
|
||||
},
|
||||
}
|
||||
cmd.Flags().Uint64Var(&concurrency, "concurrency", 0, "clean bad dp concurrency")
|
||||
@ -990,7 +990,7 @@ func calcBadNormalExtents(volname, dir string, concurrency int) (err error) {
|
||||
}
|
||||
}
|
||||
|
||||
err = os.MkdirAll(destDir, 0755)
|
||||
err = os.MkdirAll(destDir, 0o755)
|
||||
if err != nil {
|
||||
slog.Fatalf("Mkdir dest dir failed, err: %v", err)
|
||||
return
|
||||
@ -1168,7 +1168,7 @@ func writeBadNormalExtentoLocal(dpId, badDir string, exts []BadNornalExtent) (er
|
||||
start := time.Now()
|
||||
log.LogInfof("Write dir %s, dpId: %s bad extent: %d", badDir, dpId, len(exts))
|
||||
filePath := filepath.Join(badDir, dpId)
|
||||
file, err := os.OpenFile(filePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
|
||||
file, err := os.OpenFile(filePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
slog.Fatalf("Open file failed, path %s, err: %v", filePath, err)
|
||||
return err
|
||||
@ -1641,7 +1641,7 @@ func cleanBadNormalExtent(volname, dir, backupDir, dpIdStr, extentStr string) (e
|
||||
continue
|
||||
}
|
||||
|
||||
var exts = []*BadNornalExtent{&badExtent}
|
||||
exts := []*BadNornalExtent{&badExtent}
|
||||
|
||||
err = batchLockBadNormalExtent(dpIdStr, exts, false, string(data))
|
||||
if err != nil {
|
||||
@ -1853,7 +1853,7 @@ func readBadExtentFromDp(dpIdStr string, extentId uint64, size uint32) (data []b
|
||||
for readBytes < int(size) {
|
||||
replyPacket := stream.NewReply(p.ReqID, dpId, extentId)
|
||||
bufSize := util.Min(util.ReadBlockSize, int(size)-readBytes)
|
||||
//replyPacket.Data = data[readBytes : readBytes+bufSize]
|
||||
// replyPacket.Data = data[readBytes : readBytes+bufSize]
|
||||
replyPacket.Size = uint32(bufSize)
|
||||
err = replyPacket.ReadFromConn(conn, proto.ReadDeadlineTime)
|
||||
if err != nil {
|
||||
@ -1879,7 +1879,6 @@ func readBadExtentFromDp(dpIdStr string, extentId uint64, size uint32) (data []b
|
||||
}
|
||||
|
||||
func writeBadNormalExtentToBackup(backupDir, volname, dpIdStr string, badExtent BadNornalExtent, data []byte, readBytes int) (err error) {
|
||||
|
||||
dpDir := filepath.Join(backupDir, volname, normalDir, dpIdStr)
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
@ -1888,7 +1887,7 @@ func writeBadNormalExtentToBackup(backupDir, volname, dpIdStr string, badExtent
|
||||
}()
|
||||
_, err = os.Stat(dpDir)
|
||||
if os.IsNotExist(err) {
|
||||
err = os.MkdirAll(dpDir, 0755)
|
||||
err = os.MkdirAll(dpDir, 0o755)
|
||||
if err != nil {
|
||||
slog.Fatalf("Mkdir failed, dir %s, err: %v", dpDir, err.Error())
|
||||
}
|
||||
@ -2218,7 +2217,7 @@ func writeBadNormalExtentToDp(data []byte, volname, dpId, extent string) (err er
|
||||
p.Magic = proto.ProtoMagic
|
||||
p.Opcode = proto.OpBackupWrite
|
||||
length := len(data)
|
||||
//length := util.BlockSize
|
||||
// length := util.BlockSize
|
||||
p.Data = make([]byte, length)
|
||||
copy(p.Data, data)
|
||||
p.Size = uint32(length)
|
||||
|
||||
@ -44,7 +44,6 @@ func main() {
|
||||
}
|
||||
|
||||
func listenPprof() {
|
||||
|
||||
mainMux := http.NewServeMux()
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/debug/pprof", http.HandlerFunc(pprof.Index))
|
||||
|
||||
@ -1593,7 +1593,7 @@ func (c *client) setattr(info *proto.InodeInfo, valid uint32, mode, uid, gid uin
|
||||
|
||||
func (c *client) create(pino uint64, name string, mode uint32, fullPath string) (info *proto.InodeInfo, err error) {
|
||||
fuseMode := mode & 0o777
|
||||
return c.mw.Create_ll(pino, name, fuseMode, 0, 0, nil, fullPath)
|
||||
return c.mw.Create_ll(pino, name, fuseMode, 0, 0, nil, fullPath, false)
|
||||
}
|
||||
|
||||
func (c *client) mkdir(pino uint64, name string, mode uint32, fullPath string) (info *proto.InodeInfo, err error) {
|
||||
|
||||
@ -303,11 +303,11 @@ func (m *Server) setEnableAuditLogForVolume(w http.ResponseWriter, r *http.Reque
|
||||
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeVolNotExists, Msg: err.Error()})
|
||||
return
|
||||
}
|
||||
oldEnable := vol.EnableAuditLog
|
||||
vol.EnableAuditLog = status
|
||||
oldEnable := vol.DisableAuditLog
|
||||
vol.DisableAuditLog = status
|
||||
defer func() {
|
||||
if err != nil {
|
||||
vol.EnableAuditLog = oldEnable
|
||||
vol.DisableAuditLog = oldEnable
|
||||
}
|
||||
}()
|
||||
if err = m.cluster.syncUpdateVol(vol); err != nil {
|
||||
|
||||
@ -127,7 +127,6 @@ func createDefaultMasterServerForTest() *Server {
|
||||
}`
|
||||
|
||||
testServer, err := createMasterServer(cfgJSON)
|
||||
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
@ -1579,11 +1578,11 @@ func TestVolumeEnableAuditLog(t *testing.T) {
|
||||
enableUrl := fmt.Sprintf("%v?name=%v&%v=true", reqUrl, vol.Name, enableKey)
|
||||
disableUrl := fmt.Sprintf("%v?name=%v&%v=false", reqUrl, vol.Name, enableKey)
|
||||
process(disableUrl, t)
|
||||
require.True(t, vol.DisableAuditLog)
|
||||
require.True(t, checkVolAuditLog(name, false))
|
||||
process(enableUrl, t)
|
||||
require.False(t, vol.DisableAuditLog)
|
||||
require.True(t, checkVolAuditLog(name, true))
|
||||
process(enableUrl, t)
|
||||
require.True(t, vol.DisableAuditLog)
|
||||
require.True(t, checkVolAuditLog(name, false))
|
||||
}
|
||||
|
||||
func checkVolDpRepairBlockSize(name string, size uint64) (success bool) {
|
||||
|
||||
@ -1224,7 +1224,8 @@ func (c *Cluster) checkLackReplicaDataPartitions() (lackReplicaDataPartitions []
|
||||
|
||||
func (c *Cluster) checkReplicaOfDataPartitions(ignoreDiscardDp bool) (
|
||||
lackReplicaDPs []*DataPartition, unavailableReplicaDPs []*DataPartition, repFileCountDifferDps []*DataPartition,
|
||||
repUsedSizeDifferDps []*DataPartition, excessReplicaDPs []*DataPartition, noLeaderDPs []*DataPartition, err error) {
|
||||
repUsedSizeDifferDps []*DataPartition, excessReplicaDPs []*DataPartition, noLeaderDPs []*DataPartition, err error,
|
||||
) {
|
||||
noLeaderDPs = make([]*DataPartition, 0)
|
||||
lackReplicaDPs = make([]*DataPartition, 0)
|
||||
unavailableReplicaDPs = make([]*DataPartition, 0)
|
||||
@ -1684,7 +1685,8 @@ errHandler:
|
||||
}
|
||||
|
||||
func (c *Cluster) syncCreateDataPartitionToDataNode(host string, size uint64, dp *DataPartition,
|
||||
peers []proto.Peer, hosts []string, createType int, partitionType int, needRollBack, ignoreDecommissionDisk bool) (diskPath string, err error) {
|
||||
peers []proto.Peer, hosts []string, createType int, partitionType int, needRollBack, ignoreDecommissionDisk bool,
|
||||
) (diskPath string, err error) {
|
||||
log.LogInfof("action[syncCreateDataPartitionToDataNode] dp [%v] createType[%v], partitionType[%v] ignoreDecommissionDisk[%v]",
|
||||
dp.PartitionID, createType, partitionType, ignoreDecommissionDisk)
|
||||
dataNode, err := c.dataNode(host)
|
||||
@ -1798,7 +1800,8 @@ func (c *Cluster) chooseZone2Plus1(zones []*Zone, excludeNodeSets []uint64, excl
|
||||
}
|
||||
|
||||
func (c *Cluster) chooseZoneNormal(zones []*Zone, excludeNodeSets []uint64, excludeHosts []string,
|
||||
nodeType uint32, replicaNum int) (hosts []string, peers []proto.Peer, err error) {
|
||||
nodeType uint32, replicaNum int,
|
||||
) (hosts []string, peers []proto.Peer, err error) {
|
||||
log.LogInfof("action[chooseZoneNormal] zones[%s] nodeType[%d] replicaNum[%d]", printZonesName(zones), nodeType, replicaNum)
|
||||
|
||||
c.zoneIdxMux.Lock()
|
||||
|
||||
@ -295,7 +295,8 @@ type VolNameSet map[string]struct{}
|
||||
|
||||
func (c *Cluster) checkReplicaMetaPartitions() (
|
||||
lackReplicaMetaPartitions []*MetaPartition, noLeaderMetaPartitions []*MetaPartition,
|
||||
unavailableReplicaMPs []*MetaPartition, excessReplicaMetaPartitions, inodeCountNotEqualMPs, maxInodeNotEqualMPs, dentryCountNotEqualMPs []*MetaPartition, err error) {
|
||||
unavailableReplicaMPs []*MetaPartition, excessReplicaMetaPartitions, inodeCountNotEqualMPs, maxInodeNotEqualMPs, dentryCountNotEqualMPs []*MetaPartition, err error,
|
||||
) {
|
||||
lackReplicaMetaPartitions = make([]*MetaPartition, 0)
|
||||
noLeaderMetaPartitions = make([]*MetaPartition, 0)
|
||||
excessReplicaMetaPartitions = make([]*MetaPartition, 0)
|
||||
|
||||
@ -1049,7 +1049,6 @@ func (partition *DataPartition) buildDpInfo(c *Cluster) *proto.DataPartitionInfo
|
||||
OfflinePeerID: partition.OfflinePeerID,
|
||||
IsRecover: partition.isRecover,
|
||||
FilesWithMissingReplica: filesMissReplicas,
|
||||
SingleDecommissionAddr: partition.SingleDecommissionAddr,
|
||||
IsDiscard: partition.IsDiscard,
|
||||
SingleDecommissionStatus: partition.GetSpecialReplicaDecommissionStep(),
|
||||
Forbidden: forbidden,
|
||||
@ -1106,7 +1105,8 @@ func GetDecommissionStatusMessage(status uint32) string {
|
||||
}
|
||||
|
||||
func (partition *DataPartition) MarkDecommissionStatus(srcAddr, dstAddr, srcDisk string, raftForce bool, term uint64,
|
||||
migrateType uint32, c *Cluster, ns *nodeSet) (err error) {
|
||||
migrateType uint32, c *Cluster, ns *nodeSet,
|
||||
) (err error) {
|
||||
if partition.needManualFix() && migrateType == AutoDecommission {
|
||||
return proto.ErrAllReplicaUnavailable
|
||||
}
|
||||
|
||||
@ -26,7 +26,8 @@ import (
|
||||
)
|
||||
|
||||
func (partition *DataPartition) checkStatus(clusterName string, needLog bool, dpTimeOutSec int64, c *Cluster,
|
||||
shouldDpInhibitWriteByVolFull bool, forbiddenVol bool) {
|
||||
shouldDpInhibitWriteByVolFull bool, forbiddenVol bool,
|
||||
) {
|
||||
partition.Lock()
|
||||
defer partition.Unlock()
|
||||
var liveReplicas []*DataReplica
|
||||
|
||||
@ -210,7 +210,8 @@ func (c *Cluster) deleteAndSyncDecommissionedDisk(dataNode *DataNode, diskPath s
|
||||
}
|
||||
|
||||
func (c *Cluster) decommissionDisk(dataNode *DataNode, raftForce bool, badDiskPath string,
|
||||
badPartitions []*DataPartition, diskDisable bool) (err error) {
|
||||
badPartitions []*DataPartition, diskDisable bool,
|
||||
) (err error) {
|
||||
msg := fmt.Sprintf("action[decommissionDisk], Node[%v] OffLine,disk[%v]", dataNode.Addr, badDiskPath)
|
||||
log.LogWarn(msg)
|
||||
|
||||
|
||||
@ -227,7 +227,8 @@ func (m *ClusterService) decommissionDataNode(ctx context.Context, args struct {
|
||||
|
||||
func (m *ClusterService) decommissionMetaNode(ctx context.Context, args struct {
|
||||
OffLineAddr string
|
||||
}) (*proto.GeneralResp, error) {
|
||||
},
|
||||
) (*proto.GeneralResp, error) {
|
||||
if _, _, err := permissions(ctx, ADMIN); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -244,7 +245,8 @@ func (m *ClusterService) decommissionMetaNode(ctx context.Context, args struct {
|
||||
|
||||
func (m *ClusterService) loadMetaPartition(ctx context.Context, args struct {
|
||||
PartitionID uint64
|
||||
}) (*proto.GeneralResp, error) {
|
||||
},
|
||||
) (*proto.GeneralResp, error) {
|
||||
if _, _, err := permissions(ctx, ADMIN); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -261,7 +263,8 @@ func (m *ClusterService) loadMetaPartition(ctx context.Context, args struct {
|
||||
func (m *ClusterService) decommissionMetaPartition(ctx context.Context, args struct {
|
||||
PartitionID uint64
|
||||
NodeAddr string
|
||||
}) (*proto.GeneralResp, error) {
|
||||
},
|
||||
) (*proto.GeneralResp, error) {
|
||||
if _, _, err := permissions(ctx, ADMIN); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -278,7 +281,8 @@ func (m *ClusterService) decommissionMetaPartition(ctx context.Context, args str
|
||||
|
||||
func (m *ClusterService) getMetaNode(ctx context.Context, args struct {
|
||||
NodeAddr string
|
||||
}) (*MetaNode, error) {
|
||||
},
|
||||
) (*MetaNode, error) {
|
||||
if _, _, err := permissions(ctx, ADMIN); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -361,7 +365,8 @@ func (s *ClusterService) masterList(ctx context.Context, args struct{}) ([]*Mast
|
||||
|
||||
func (s *ClusterService) dataNodeGet(ctx context.Context, args struct {
|
||||
Addr string
|
||||
}) (*DataNode, error) {
|
||||
},
|
||||
) (*DataNode, error) {
|
||||
if _, _, err := permissions(ctx, ADMIN); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -382,7 +387,8 @@ func (s *ClusterService) dataNodeList(ctx context.Context, args struct{}) ([]*Da
|
||||
|
||||
func (s *ClusterService) dataNodeListTest(ctx context.Context, args struct {
|
||||
Num int64
|
||||
}) ([]*DataNode, error) {
|
||||
},
|
||||
) ([]*DataNode, error) {
|
||||
if _, _, err := permissions(ctx, ADMIN); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -409,7 +415,8 @@ func (s *ClusterService) dataNodeListTest(ctx context.Context, args struct {
|
||||
|
||||
func (s *ClusterService) metaNodeGet(ctx context.Context, args struct {
|
||||
Addr string
|
||||
}) (*MetaNode, error) {
|
||||
},
|
||||
) (*MetaNode, error) {
|
||||
if _, _, err := permissions(ctx, ADMIN); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -435,7 +442,8 @@ func (s *ClusterService) metaNodeList(ctx context.Context, args struct{}) ([]*Me
|
||||
func (m *ClusterService) addMetaNode(ctx context.Context, args struct {
|
||||
NodeAddr string
|
||||
ZoneName string
|
||||
}) (uint64, error) {
|
||||
},
|
||||
) (uint64, error) {
|
||||
if id, err := m.cluster.addMetaNode(args.NodeAddr, args.ZoneName, 0); err != nil {
|
||||
return 0, err
|
||||
} else {
|
||||
@ -447,7 +455,8 @@ func (m *ClusterService) addMetaNode(ctx context.Context, args struct {
|
||||
func (m *ClusterService) removeRaftNode(ctx context.Context, args struct {
|
||||
Id uint64
|
||||
Addr string
|
||||
}) (*proto.GeneralResp, error) {
|
||||
},
|
||||
) (*proto.GeneralResp, error) {
|
||||
if _, _, err := permissions(ctx, ADMIN); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -463,7 +472,8 @@ func (m *ClusterService) removeRaftNode(ctx context.Context, args struct {
|
||||
func (m *ClusterService) addRaftNode(ctx context.Context, args struct {
|
||||
Id uint64
|
||||
Addr string
|
||||
}) (*proto.GeneralResp, error) {
|
||||
},
|
||||
) (*proto.GeneralResp, error) {
|
||||
if _, _, err := permissions(ctx, ADMIN); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -486,7 +496,8 @@ func (m *ClusterService) addRaftNode(ctx context.Context, args struct {
|
||||
// 2. and the number of r&w data partition is less than 20.
|
||||
func (m *ClusterService) clusterFreeze(ctx context.Context, args struct {
|
||||
Status bool
|
||||
}) (*proto.GeneralResp, error) {
|
||||
},
|
||||
) (*proto.GeneralResp, error) {
|
||||
if _, _, err := permissions(ctx, ADMIN); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -508,7 +519,8 @@ type WarnMessage struct {
|
||||
|
||||
func (m *ClusterService) alarmList(ctx context.Context, args struct {
|
||||
Size int32
|
||||
}) ([]*WarnMessage, error) {
|
||||
},
|
||||
) ([]*WarnMessage, error) {
|
||||
if _, _, err := permissions(ctx, ADMIN); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@ -85,7 +85,8 @@ func (s *UserService) registerQuery(schema *schemabuilder.Schema) {
|
||||
|
||||
func (m *UserService) getUserAKInfo(ctx context.Context, args struct {
|
||||
AccessKey string
|
||||
}) (*proto.UserInfo, error) {
|
||||
},
|
||||
) (*proto.UserInfo, error) {
|
||||
uid, perm, err := permissions(ctx, ADMIN|USER)
|
||||
userInfo, err := m.user.getKeyInfo(args.AccessKey)
|
||||
if err != nil {
|
||||
@ -222,7 +223,8 @@ func (s *UserService) updateUser(ctx context.Context, args proto.UserUpdateParam
|
||||
|
||||
func (s *UserService) deleteUser(ctx context.Context, args struct {
|
||||
UserID string
|
||||
}) (*proto.GeneralResp, error) {
|
||||
},
|
||||
) (*proto.GeneralResp, error) {
|
||||
uid, _, err := permissions(ctx, ADMIN)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -239,7 +241,8 @@ func (s *UserService) deleteUser(ctx context.Context, args struct {
|
||||
|
||||
func (s *UserService) getUserInfo(ctx context.Context, args struct {
|
||||
UserID string
|
||||
}) (*proto.UserInfo, error) {
|
||||
},
|
||||
) (*proto.UserInfo, error) {
|
||||
uid, perm, err := permissions(ctx, ADMIN|USER)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -274,7 +277,8 @@ type UserUseSpace struct {
|
||||
|
||||
func (s *UserService) topNUser(ctx context.Context, args struct {
|
||||
N int32
|
||||
}) ([]*UserUseSpace, error) {
|
||||
},
|
||||
) ([]*UserUseSpace, error) {
|
||||
if _, _, err := permissions(ctx, ADMIN); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -334,7 +338,8 @@ func (s *UserService) topNUser(ctx context.Context, args struct {
|
||||
func (s *UserService) validatePassword(ctx context.Context, args struct {
|
||||
UserID string
|
||||
Password string
|
||||
}) (*proto.UserInfo, error) {
|
||||
},
|
||||
) (*proto.UserInfo, error) {
|
||||
ui, err := s.user.getUserInfo(args.UserID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@ -21,6 +21,7 @@ import (
|
||||
"html"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
|
||||
@ -305,8 +305,8 @@ type volValue struct {
|
||||
TrashInterval int64
|
||||
DisableAuditLog bool
|
||||
|
||||
Forbidden bool
|
||||
DpRepairBlockSize uint64
|
||||
Forbidden bool
|
||||
DpRepairBlockSize uint64
|
||||
}
|
||||
|
||||
func (v *volValue) Bytes() (raw []byte, err error) {
|
||||
|
||||
@ -31,7 +31,8 @@ import (
|
||||
|
||||
func newCreateDataPartitionRequest(volName string, ID uint64, replicaNum int, members []proto.Peer,
|
||||
dataPartitionSize, leaderSize int, hosts []string, createType int, partitionType int,
|
||||
decommissionedDisks []string, verSeq uint64) (req *proto.CreateDataPartitionRequest) {
|
||||
decommissionedDisks []string, verSeq uint64,
|
||||
) (req *proto.CreateDataPartitionRequest) {
|
||||
req = &proto.CreateDataPartitionRequest{
|
||||
PartitionTyp: partitionType,
|
||||
PartitionId: ID,
|
||||
|
||||
@ -578,7 +578,8 @@ func (nsgm *DomainManager) getHostFromNodeSetGrpSpecific(domainGrpManager *Domai
|
||||
func (nsgm *DomainManager) getHostFromNodeSetGrp(domainId uint64, replicaNum uint8, createType uint32) (
|
||||
hosts []string,
|
||||
peers []proto.Peer,
|
||||
err error) {
|
||||
err error,
|
||||
) {
|
||||
var ok bool
|
||||
var index int
|
||||
|
||||
|
||||
@ -487,10 +487,6 @@ func (vol *Vol) getDataPartitionViewCompress() (body []byte, err error) {
|
||||
return vol.dataPartitions.updateCompressCache(false, 0, vol)
|
||||
}
|
||||
|
||||
func (vol *Vol) getDataPartitionViewCompress() (body []byte, err error) {
|
||||
return vol.dataPartitions.updateCompressCache(false, 0, vol.VolType)
|
||||
}
|
||||
|
||||
func (vol *Vol) getDataPartitionByID(partitionID uint64) (dp *DataPartition, err error) {
|
||||
return vol.dataPartitions.get(partitionID)
|
||||
}
|
||||
@ -606,8 +602,6 @@ func (vol *Vol) checkDataPartitions(c *Cluster) (cnt int) {
|
||||
|
||||
totalPreloadCapacity := uint64(0)
|
||||
|
||||
totalPreloadCapacity := uint64(0)
|
||||
|
||||
partitions := vol.dataPartitions.clonePartitions()
|
||||
for _, dp := range partitions {
|
||||
|
||||
|
||||
@ -79,7 +79,8 @@ func (m *MetaNode) registerAPIHandler() (err error) {
|
||||
}
|
||||
|
||||
func (m *MetaNode) getParamsHandler(w http.ResponseWriter,
|
||||
r *http.Request) {
|
||||
r *http.Request,
|
||||
) {
|
||||
resp := NewAPIResponse(http.StatusOK, http.StatusText(http.StatusOK))
|
||||
params := make(map[string]interface{})
|
||||
params[metaNodeDeleteBatchCountKey] = DeleteBatchCount()
|
||||
@ -91,7 +92,8 @@ func (m *MetaNode) getParamsHandler(w http.ResponseWriter,
|
||||
}
|
||||
|
||||
func (m *MetaNode) getSmuxStatHandler(w http.ResponseWriter,
|
||||
r *http.Request) {
|
||||
r *http.Request,
|
||||
) {
|
||||
resp := NewAPIResponse(http.StatusOK, http.StatusText(http.StatusOK))
|
||||
resp.Data = smuxPool.GetStat()
|
||||
data, _ := resp.Marshal()
|
||||
@ -101,7 +103,8 @@ func (m *MetaNode) getSmuxStatHandler(w http.ResponseWriter,
|
||||
}
|
||||
|
||||
func (m *MetaNode) getPartitionsHandler(w http.ResponseWriter,
|
||||
r *http.Request) {
|
||||
r *http.Request,
|
||||
) {
|
||||
resp := NewAPIResponse(http.StatusOK, http.StatusText(http.StatusOK))
|
||||
resp.Data = m.metadataManager
|
||||
data, _ := resp.Marshal()
|
||||
@ -372,7 +375,8 @@ func (m *MetaNode) getRaftStatusHandler(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
|
||||
func (m *MetaNode) getEbsExtentsByInodeHandler(w http.ResponseWriter,
|
||||
r *http.Request) {
|
||||
r *http.Request,
|
||||
) {
|
||||
r.ParseForm()
|
||||
resp := NewAPIResponse(http.StatusBadRequest, "")
|
||||
defer func() {
|
||||
@ -416,7 +420,8 @@ func (m *MetaNode) getEbsExtentsByInodeHandler(w http.ResponseWriter,
|
||||
}
|
||||
|
||||
func (m *MetaNode) getExtentsByInodeHandler(w http.ResponseWriter,
|
||||
r *http.Request) {
|
||||
r *http.Request,
|
||||
) {
|
||||
r.ParseForm()
|
||||
resp := NewAPIResponse(http.StatusBadRequest, "")
|
||||
defer func() {
|
||||
|
||||
@ -147,7 +147,7 @@ func (m *metadataManager) updateVolumes() {
|
||||
return true
|
||||
})
|
||||
// push to every partitions
|
||||
m.Range(func(_ uint64, mp MetaPartition) bool {
|
||||
m.Range(false, func(_ uint64, mp MetaPartition) bool {
|
||||
dataView := dataViews[mp.GetBaseConfig().VolName]
|
||||
volView := volViews[mp.GetBaseConfig().VolName]
|
||||
if dataView != nil && volView != nil {
|
||||
|
||||
@ -75,7 +75,8 @@ func (m *metadataManager) checkDisableAuditLogVolume(volNames []string, partitio
|
||||
}
|
||||
|
||||
func (m *metadataManager) opMasterHeartbeat(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
// For ack to master
|
||||
data := p.Data
|
||||
m.responseAckOKToMaster(conn, p)
|
||||
@ -164,7 +165,8 @@ func (m *metadataManager) opMasterHeartbeat(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opCreateMetaPartition(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
defer func() {
|
||||
var buf []byte
|
||||
status := proto.OpOk
|
||||
@ -201,7 +203,8 @@ func (m *metadataManager) opCreateMetaPartition(conn net.Conn, p *Packet,
|
||||
|
||||
// Handle OpCreate inode.
|
||||
func (m *metadataManager) opCreateInode(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &CreateInoReq{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -300,7 +303,8 @@ func (m *metadataManager) opTxMetaLinkInode(conn net.Conn, p *Packet, remoteAddr
|
||||
}
|
||||
|
||||
func (m *metadataManager) opMetaLinkInode(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &LinkInodeReq{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -333,7 +337,8 @@ func (m *metadataManager) opMetaLinkInode(conn net.Conn, p *Packet,
|
||||
|
||||
// Handle OpCreate
|
||||
func (m *metadataManager) opFreeInodeOnRaftFollower(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
mp, err := m.getPartition(p.PartitionID)
|
||||
if err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -350,7 +355,8 @@ func (m *metadataManager) opFreeInodeOnRaftFollower(conn net.Conn, p *Packet,
|
||||
|
||||
// Handle OpCreate
|
||||
func (m *metadataManager) opTxCreateDentry(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.TxCreateDentryRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -385,7 +391,8 @@ func (m *metadataManager) opTxCreateDentry(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opTxCreate(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.TxCreateRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -415,7 +422,8 @@ func (m *metadataManager) opTxCreate(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opTxGet(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.TxGetInfoRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -447,7 +455,8 @@ func (m *metadataManager) opTxGet(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opTxCommitRM(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.TxApplyRMRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -477,7 +486,8 @@ func (m *metadataManager) opTxCommitRM(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opTxRollbackRM(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.TxApplyRMRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -507,7 +517,8 @@ func (m *metadataManager) opTxRollbackRM(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opTxCommit(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.TxApplyRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -537,7 +548,8 @@ func (m *metadataManager) opTxCommit(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opTxRollback(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.TxApplyRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -568,7 +580,8 @@ func (m *metadataManager) opTxRollback(conn net.Conn, p *Packet,
|
||||
|
||||
// Handle OpCreate
|
||||
func (m *metadataManager) opCreateDentry(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &CreateDentryReq{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -601,7 +614,8 @@ func (m *metadataManager) opCreateDentry(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opQuotaCreateDentry(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.QuotaCreateDentryRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -635,7 +649,8 @@ func (m *metadataManager) opQuotaCreateDentry(conn net.Conn, p *Packet,
|
||||
|
||||
// Handle OpDelete Dentry
|
||||
func (m *metadataManager) opTxDeleteDentry(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.TxDeleteDentryRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -666,7 +681,8 @@ func (m *metadataManager) opTxDeleteDentry(conn net.Conn, p *Packet,
|
||||
|
||||
// Handle OpDelete Dentry
|
||||
func (m *metadataManager) opDeleteDentry(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &DeleteDentryReq{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -701,7 +717,8 @@ func (m *metadataManager) opDeleteDentry(conn net.Conn, p *Packet,
|
||||
|
||||
// Handle Op batch Delete Dentry
|
||||
func (m *metadataManager) opBatchDeleteDentry(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &BatchDeleteDentryReq{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -767,7 +784,8 @@ func (m *metadataManager) opTxUpdateDentry(conn net.Conn, p *Packet, remoteAddr
|
||||
}
|
||||
|
||||
func (m *metadataManager) opUpdateDentry(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &UpdateDentryReq{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -834,7 +852,8 @@ func (m *metadataManager) opTxMetaUnlinkInode(conn net.Conn, p *Packet, remoteAd
|
||||
}
|
||||
|
||||
func (m *metadataManager) opMetaUnlinkInode(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &UnlinkInoReq{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -866,7 +885,8 @@ func (m *metadataManager) opMetaUnlinkInode(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opMetaBatchUnlinkInode(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &BatchUnlinkInoReq{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -898,7 +918,8 @@ func (m *metadataManager) opMetaBatchUnlinkInode(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opReadDirOnly(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.ReadDirOnlyRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -925,7 +946,8 @@ func (m *metadataManager) opReadDirOnly(conn net.Conn, p *Packet,
|
||||
|
||||
// Handle OpReadDir
|
||||
func (m *metadataManager) opReadDir(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.ReadDirRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -952,7 +974,8 @@ func (m *metadataManager) opReadDir(conn net.Conn, p *Packet,
|
||||
|
||||
// Handle OpReadDirLimit
|
||||
func (m *metadataManager) opReadDirLimit(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.ReadDirLimitRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -1023,7 +1046,8 @@ func (m *metadataManager) opMetaInodeGet(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opBatchMetaEvictInode(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.BatchEvictInodeRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
|
||||
@ -1057,7 +1081,8 @@ func (m *metadataManager) opBatchMetaEvictInode(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opMetaEvictInode(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.EvictInodeRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -1091,7 +1116,8 @@ func (m *metadataManager) opMetaEvictInode(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opSetAttr(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &SetattrRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -1128,7 +1154,8 @@ func (m *metadataManager) opSetAttr(conn net.Conn, p *Packet,
|
||||
|
||||
// Lookup request
|
||||
func (m *metadataManager) opMetaLookup(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.LookupRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -1162,7 +1189,8 @@ func (m *metadataManager) opMetaLookup(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opMetaExtentsAdd(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.AppendExtentKeyRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -1199,7 +1227,8 @@ func (m *metadataManager) opMetaExtentsAdd(conn net.Conn, p *Packet,
|
||||
|
||||
// Append one extent with discard check
|
||||
func (m *metadataManager) opMetaExtentAddWithCheck(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.AppendExtentKeyWithCheckRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -1237,7 +1266,8 @@ func (m *metadataManager) opMetaExtentAddWithCheck(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opMetaExtentsList(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.GetExtentsRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -1266,7 +1296,8 @@ func (m *metadataManager) opMetaExtentsList(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opMetaObjExtentsList(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.GetExtentsRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -1293,7 +1324,8 @@ func (m *metadataManager) opMetaObjExtentsList(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opMetaExtentsDel(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
panic("not implemented yet")
|
||||
// req := &proto.DelExtentKeyRequest{}
|
||||
// if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
@ -1320,7 +1352,8 @@ func (m *metadataManager) opMetaExtentsDel(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opMetaExtentsTruncate(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &ExtentsTruncateReq{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -1352,7 +1385,8 @@ func (m *metadataManager) opMetaExtentsTruncate(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opMetaClearInodeCache(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.ClearInodeCacheRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -1379,7 +1413,8 @@ func (m *metadataManager) opMetaClearInodeCache(conn net.Conn, p *Packet,
|
||||
|
||||
// Delete a meta partition.
|
||||
func (m *metadataManager) opDeleteMetaPartition(conn net.Conn,
|
||||
p *Packet, remoteAddr string) (err error) {
|
||||
p *Packet, remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.DeleteMetaPartitionRequest{}
|
||||
adminTask := &proto.AdminTask{
|
||||
Request: req,
|
||||
@ -1413,7 +1448,8 @@ func (m *metadataManager) opDeleteMetaPartition(conn net.Conn,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opUpdateMetaPartition(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := new(UpdatePartitionReq)
|
||||
adminTask := &proto.AdminTask{
|
||||
Request: req,
|
||||
@ -1453,7 +1489,8 @@ func (m *metadataManager) opUpdateMetaPartition(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opLoadMetaPartition(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.MetaPartitionLoadRequest{}
|
||||
adminTask := &proto.AdminTask{
|
||||
Request: req,
|
||||
@ -1488,7 +1525,8 @@ func (m *metadataManager) opLoadMetaPartition(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opDecommissionMetaPartition(conn net.Conn,
|
||||
p *Packet, remoteAddr string) (err error) {
|
||||
p *Packet, remoteAddr string,
|
||||
) (err error) {
|
||||
var reqData []byte
|
||||
req := &proto.MetaPartitionDecommissionRequest{}
|
||||
adminTask := &proto.AdminTask{
|
||||
@ -1547,7 +1585,8 @@ func (m *metadataManager) opDecommissionMetaPartition(conn net.Conn,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opAddMetaPartitionRaftMember(conn net.Conn,
|
||||
p *Packet, remoteAddr string) (err error) {
|
||||
p *Packet, remoteAddr string,
|
||||
) (err error) {
|
||||
var reqData []byte
|
||||
req := &proto.AddMetaPartitionRaftMemberRequest{}
|
||||
adminTask := &proto.AdminTask{
|
||||
@ -1616,7 +1655,8 @@ func (m *metadataManager) opAddMetaPartitionRaftMember(conn net.Conn,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opRemoveMetaPartitionRaftMember(conn net.Conn,
|
||||
p *Packet, remoteAddr string) (err error) {
|
||||
p *Packet, remoteAddr string,
|
||||
) (err error) {
|
||||
var reqData []byte
|
||||
req := &proto.RemoveMetaPartitionRaftMemberRequest{}
|
||||
adminTask := &proto.AdminTask{
|
||||
@ -1694,7 +1734,8 @@ func (m *metadataManager) opRemoveMetaPartitionRaftMember(conn net.Conn,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opMetaBatchInodeGet(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.BatchInodeGetRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -1721,7 +1762,8 @@ func (m *metadataManager) opMetaBatchInodeGet(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opMetaPartitionTryToLeader(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
mp, err := m.getPartition(p.PartitionID)
|
||||
if err != nil {
|
||||
goto errDeal
|
||||
@ -1739,7 +1781,8 @@ errDeal:
|
||||
}
|
||||
|
||||
func (m *metadataManager) opMetaDeleteInode(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.DeleteInodeRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -1765,7 +1808,8 @@ func (m *metadataManager) opMetaDeleteInode(conn net.Conn, p *Packet,
|
||||
}
|
||||
|
||||
func (m *metadataManager) opMetaBatchDeleteInode(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
var req *proto.DeleteInodeBatchRequest
|
||||
if err = json.Unmarshal(p.Data, &req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -2239,7 +2283,8 @@ func (m *metadataManager) opListMultipart(conn net.Conn, p *Packet, remoteAddr s
|
||||
|
||||
// Handle OpMetaTxCreateInode inode.
|
||||
func (m *metadataManager) opTxCreateInode(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.TxCreateInodeRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -2376,7 +2421,8 @@ func (m *metadataManager) opMetaGetInodeQuota(conn net.Conn, p *Packet, remote s
|
||||
}
|
||||
|
||||
func (m *metadataManager) opMetaGetUniqID(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
req := &proto.GetUniqIDRequest{}
|
||||
if err = json.Unmarshal(p.Data, req); err != nil {
|
||||
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
|
||||
@ -2653,7 +2699,8 @@ func (m *metadataManager) checkAndPromoteVersion(volName string) (err error) {
|
||||
}
|
||||
|
||||
func (m *metadataManager) opMultiVersionOp(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
// For ack to master
|
||||
data := p.Data
|
||||
m.responseAckOKToMaster(conn, p)
|
||||
|
||||
@ -15,7 +15,6 @@
|
||||
package metanode
|
||||
|
||||
import (
|
||||
"github.com/cubefs/cubefs/storage"
|
||||
"net"
|
||||
|
||||
"github.com/cubefs/cubefs/storage"
|
||||
@ -89,7 +88,8 @@ func (m *metadataManager) IsForbiddenOp(mp MetaPartition, reqOp uint8) bool {
|
||||
// The proxy is used during the leader change. When a leader of a partition changes, the proxy forwards the request to
|
||||
// the new leader.
|
||||
func (m *metadataManager) serveProxy(conn net.Conn, mp MetaPartition,
|
||||
p *Packet) (ok bool) {
|
||||
p *Packet,
|
||||
) (ok bool) {
|
||||
var (
|
||||
mConn *net.TCPConn
|
||||
leaderAddr string
|
||||
|
||||
@ -1263,7 +1263,8 @@ func (mp *metaPartition) GetBaseConfig() MetaPartitionConfig {
|
||||
|
||||
// UpdatePartition updates the meta partition. TODO remove? no usage?
|
||||
func (mp *metaPartition) UpdatePartition(req *UpdatePartitionReq,
|
||||
resp *UpdatePartitionResp) (err error) {
|
||||
resp *UpdatePartitionResp,
|
||||
) (err error) {
|
||||
reqData, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
resp.Status = proto.TaskFailed
|
||||
|
||||
@ -40,8 +40,8 @@ const (
|
||||
|
||||
var extentsFileHeader = []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08}
|
||||
|
||||
/// start metapartition delete extents work
|
||||
///
|
||||
// / start metapartition delete extents work
|
||||
// /
|
||||
func (mp *metaPartition) startToDeleteExtents() {
|
||||
fileList := synclist.New()
|
||||
go mp.appendDelExtentsToFile(fileList)
|
||||
|
||||
@ -54,12 +54,38 @@ func (mp *metaPartition) startFreeList() (err error) {
|
||||
return
|
||||
}
|
||||
|
||||
go mp.updateVolWorker()
|
||||
go mp.deleteWorker()
|
||||
mp.startToDeleteExtents()
|
||||
return
|
||||
}
|
||||
|
||||
UpdateVolumeView(dataView *proto.DataPartitionsView, volumeView *proto.SimpleVolView) {
|
||||
func (mp *metaPartition) UpdateVolumeView(dataView *proto.DataPartitionsView, volumeView *proto.SimpleVolView) {
|
||||
convert := func(view *proto.DataPartitionsView) *DataPartitionsView {
|
||||
newView := &DataPartitionsView{
|
||||
DataPartitions: make([]*DataPartition, len(view.DataPartitions)),
|
||||
}
|
||||
for i := 0; i < len(view.DataPartitions); i++ {
|
||||
if len(view.DataPartitions[i].Hosts) < 1 {
|
||||
log.LogErrorf("action[UpdateVolumeView] dp id(%v) is invalid, DataPartitionResponse detail[%v]",
|
||||
view.DataPartitions[i].PartitionID, view.DataPartitions[i])
|
||||
continue
|
||||
}
|
||||
newView.DataPartitions[i] = &DataPartition{
|
||||
PartitionID: view.DataPartitions[i].PartitionID,
|
||||
Status: view.DataPartitions[i].Status,
|
||||
Hosts: view.DataPartitions[i].Hosts,
|
||||
ReplicaNum: view.DataPartitions[i].ReplicaNum,
|
||||
IsDiscard: view.DataPartitions[i].IsDiscard,
|
||||
}
|
||||
}
|
||||
return newView
|
||||
}
|
||||
mp.vol.UpdatePartitions(convert(dataView))
|
||||
mp.vol.volDeleteLockTime = volumeView.DeleteLockTime
|
||||
}
|
||||
|
||||
func (mp *metaPartition) updateVolView(convert func(view *proto.DataPartitionsView) *DataPartitionsView) (err error) {
|
||||
volName := mp.config.VolName
|
||||
dataView, err := masterClient.ClientAPI().EncodingGzip().GetDataPartitions(volName)
|
||||
if err != nil {
|
||||
@ -88,7 +114,7 @@ func (mp *metaPartition) updateVolWorker() {
|
||||
}
|
||||
for i := 0; i < len(view.DataPartitions); i++ {
|
||||
if len(view.DataPartitions[i].Hosts) < 1 {
|
||||
log.LogErrorf("action[UpdateVolumeView] dp id(%v) is invalid, DataPartitionResponse detail[%v]",
|
||||
log.LogErrorf("updateVolWorker dp id(%v) is invalid, DataPartitionResponse detail[%v]",
|
||||
view.DataPartitions[i].PartitionID, view.DataPartitions[i])
|
||||
continue
|
||||
}
|
||||
@ -102,8 +128,16 @@ func (mp *metaPartition) updateVolWorker() {
|
||||
}
|
||||
return newView
|
||||
}
|
||||
mp.vol.UpdatePartitions(convert(dataView))
|
||||
mp.vol.volDeleteLockTime = volumeView.DeleteLockTime
|
||||
mp.updateVolView(convert)
|
||||
for {
|
||||
select {
|
||||
case <-mp.stopC:
|
||||
t.Stop()
|
||||
return
|
||||
case <-t.C:
|
||||
mp.updateVolView(convert)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
@ -186,7 +220,8 @@ func (mp *metaPartition) deleteWorker() {
|
||||
|
||||
// delete Extents by Partition,and find all successDelete inode
|
||||
func (mp *metaPartition) batchDeleteExtentsByPartition(partitionDeleteExtents map[uint64][]*proto.DelExtentParam,
|
||||
allInodes []*Inode) (shouldCommit []*Inode, shouldPushToFreeList []*Inode) {
|
||||
allInodes []*Inode,
|
||||
) (shouldCommit []*Inode, shouldPushToFreeList []*Inode) {
|
||||
occurErrors := make(map[uint64]error)
|
||||
shouldCommit = make([]*Inode, 0, len(allInodes))
|
||||
shouldPushToFreeList = make([]*Inode, 0)
|
||||
|
||||
@ -15,17 +15,42 @@
|
||||
package metanode
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/cubefs/cubefs/proto"
|
||||
"github.com/cubefs/cubefs/util"
|
||||
"github.com/cubefs/cubefs/util/fileutil"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var VolNameForFreeListTest = "TestForFreeList"
|
||||
|
||||
func newPartitionForFreeList(conf *MetaPartitionConfig, manager *metadataManager) (mp *metaPartition) {
|
||||
mp = &metaPartition{
|
||||
config: conf,
|
||||
dentryTree: NewBtree(),
|
||||
inodeTree: NewBtree(),
|
||||
extendTree: NewBtree(),
|
||||
multipartTree: NewBtree(),
|
||||
stopC: make(chan bool),
|
||||
storeChan: make(chan *storeMsg, 100),
|
||||
freeList: newFreeList(),
|
||||
extDelCh: make(chan []proto.ExtentKey, defaultDelExtentsCnt),
|
||||
extReset: make(chan struct{}),
|
||||
vol: NewVol(),
|
||||
manager: manager,
|
||||
}
|
||||
mp.config.Cursor = 0
|
||||
mp.config.End = 100000
|
||||
mp.uidManager = NewUidMgr(conf.VolName, mp.config.PartitionId)
|
||||
mp.mqMgr = NewQuotaManager(conf.VolName, mp.config.PartitionId)
|
||||
return mp
|
||||
}
|
||||
|
||||
func TestPersistInodesFreeList(t *testing.T) {
|
||||
rootDir, err := os.MkdirTemp("", "")
|
||||
defer os.RemoveAll(rootDir)
|
||||
@ -42,9 +67,10 @@ func TestPersistInodesFreeList(t *testing.T) {
|
||||
fileName := path.Join(config.RootDir, DeleteInodeFileExtension)
|
||||
oldIno, err := fileutil.Stat(fileName)
|
||||
require.NoError(t, err)
|
||||
t.Logf("Persist many inodes")
|
||||
const persistBatchCount = 50000
|
||||
const testCount = DeleteInodeFileRollingSize / 8
|
||||
const persistBatchCount = 500000
|
||||
unitSize := len(fmt.Sprintf("%v\n", 1000000))
|
||||
testCount := DeleteInodeFileRollingSize / unitSize
|
||||
t.Logf("Persist many inodes, unitSize %d, total %d", unitSize, testCount)
|
||||
inodes := make([]uint64, 0, persistBatchCount)
|
||||
for i := 0; i < persistBatchCount; i++ {
|
||||
inodes = append(inodes, uint64(i)+1000000)
|
||||
|
||||
@ -299,7 +299,7 @@ func (mp *metaPartition) Apply(command []byte, index uint64) (resp interface{},
|
||||
}
|
||||
err = mp.fsmSetXAttr(extend)
|
||||
case opFSMLockDir:
|
||||
var req = &proto.LockDirRequest{}
|
||||
req := &proto.LockDirRequest{}
|
||||
if err = json.Unmarshal(msg.V, req); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
@ -75,7 +75,8 @@ func (mp *metaPartition) decommissionPartition() (err error) {
|
||||
}
|
||||
|
||||
func (mp *metaPartition) fsmUpdatePartition(end uint64) (status uint8,
|
||||
err error) {
|
||||
err error,
|
||||
) {
|
||||
status = proto.OpOk
|
||||
oldEnd := mp.config.End
|
||||
mp.config.End = end
|
||||
@ -177,7 +178,6 @@ func (mp *metaPartition) delOldExtentFile(buf []byte) (err error) {
|
||||
return
|
||||
}
|
||||
|
||||
//
|
||||
func (mp *metaPartition) setExtentDeleteFileCursor(buf []byte) (err error) {
|
||||
str := string(buf)
|
||||
var (
|
||||
|
||||
@ -69,7 +69,8 @@ func (mp *metaPartition) fsmTxCreateDentry(txDentry *TxDentry) (status uint8) {
|
||||
|
||||
// Insert a dentry into the dentry tree.
|
||||
func (mp *metaPartition) fsmCreateDentry(dentry *Dentry,
|
||||
forceUpdate bool) (status uint8) {
|
||||
forceUpdate bool,
|
||||
) (status uint8) {
|
||||
status = proto.OpOk
|
||||
var parIno *Inode
|
||||
if !forceUpdate {
|
||||
@ -360,7 +361,8 @@ func (mp *metaPartition) fsmTxUpdateDentry(txUpDateDentry *TxUpdateDentry) (resp
|
||||
}
|
||||
|
||||
func (mp *metaPartition) fsmUpdateDentry(dentry *Dentry) (
|
||||
resp *DentryResponse) {
|
||||
resp *DentryResponse,
|
||||
) {
|
||||
resp = NewDentryResponse()
|
||||
resp.Status = proto.OpOk
|
||||
mp.dentryTree.CopyFind(dentry, func(item BtreeItem) {
|
||||
@ -446,7 +448,6 @@ func (mp *metaPartition) readDir(req *ReadDirReq) (resp *ReadDirResp) {
|
||||
// else if req.Marker != "" and req.Limit == 0, return dentries from pid:name to pid+1
|
||||
// else if req.Marker == "" and req.Limit != 0, return dentries from pid with limit count
|
||||
// else if req.Marker != "" and req.Limit != 0, return dentries from pid:marker to pid:xxxx with limit count
|
||||
//
|
||||
func (mp *metaPartition) readDirLimit(req *ReadDirLimitReq) (resp *ReadDirLimitResp) {
|
||||
log.LogDebugf("action[readDirLimit] mp[%v] req %v", mp.config.PartitionId, req)
|
||||
resp = &ReadDirLimitResp{}
|
||||
|
||||
@ -15,11 +15,11 @@
|
||||
package metanode
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"fmt"
|
||||
"math"
|
||||
|
||||
"github.com/cubefs/cubefs/proto"
|
||||
"github.com/cubefs/cubefs/util/log"
|
||||
@ -43,7 +43,7 @@ func (mp *metaPartition) fsmLockDir(req *proto.LockDirRequest) (resp *proto.Lock
|
||||
|
||||
log.LogDebugf("fsmLockDir ino=%v, lockId=%d, submitTime=%v, lease=%d\n", ino, lockId, submitTime, lease)
|
||||
|
||||
var newExtend = NewExtend(ino)
|
||||
newExtend := NewExtend(ino)
|
||||
treeItem := mp.extendTree.CopyGet(newExtend)
|
||||
|
||||
var firstLock bool = false
|
||||
@ -72,7 +72,7 @@ func (mp *metaPartition) fsmLockDir(req *proto.LockDirRequest) (resp *proto.Lock
|
||||
lockId = int64(uu_id.ID())
|
||||
lockIdStr = strconv.Itoa(int(lockId))
|
||||
value = lockIdStr + "|" + validTimeStr
|
||||
newExtend.Put([]byte("dir_lock"), []byte(value))
|
||||
newExtend.Put([]byte("dir_lock"), []byte(value), 0)
|
||||
mp.extendTree.ReplaceOrInsert(newExtend, true)
|
||||
} else {
|
||||
log.LogDebugf("fsmLockDir oldValue=%s\n", oldValue)
|
||||
@ -103,7 +103,7 @@ func (mp *metaPartition) fsmLockDir(req *proto.LockDirRequest) (resp *proto.Lock
|
||||
}
|
||||
|
||||
existExtend.Remove([]byte("dir_lock"))
|
||||
newExtend.Put([]byte("dir_lock"), []byte(value))
|
||||
newExtend.Put([]byte("dir_lock"), []byte(value), 0)
|
||||
existExtend.Merge(newExtend, true)
|
||||
}
|
||||
|
||||
|
||||
@ -44,11 +44,12 @@ func (s *MetaItem) MarshalJson() ([]byte, error) {
|
||||
|
||||
// MarshalBinary marshals MetaItem to binary data.
|
||||
// Binary frame structure:
|
||||
// +------+----+------+------+------+------+
|
||||
// | Item | Op | LenK | K | LenV | V |
|
||||
// +------+----+------+------+------+------+
|
||||
// | byte | 4 | 4 | LenK | 4 | LenV |
|
||||
// +------+----+------+------+------+------+
|
||||
//
|
||||
// +------+----+------+------+------+------+
|
||||
// | Item | Op | LenK | K | LenV | V |
|
||||
// +------+----+------+------+------+------+
|
||||
// | byte | 4 | 4 | LenK | 4 | LenV |
|
||||
// +------+----+------+------+------+------+
|
||||
func (s *MetaItem) MarshalBinary() (result []byte, err error) {
|
||||
buff := bytes.NewBuffer(make([]byte, 0))
|
||||
buff.Grow(4 + len(s.K) + len(s.V))
|
||||
@ -78,11 +79,12 @@ func (s *MetaItem) UnmarshalJson(data []byte) error {
|
||||
|
||||
// MarshalBinary unmarshal this MetaItem entity from binary data.
|
||||
// Binary frame structure:
|
||||
// +------+----+------+------+------+------+
|
||||
// | Item | Op | LenK | K | LenV | V |
|
||||
// +------+----+------+------+------+------+
|
||||
// | byte | 4 | 4 | LenK | 4 | LenV |
|
||||
// +------+----+------+------+------+------+
|
||||
//
|
||||
// +------+----+------+------+------+------+
|
||||
// | Item | Op | LenK | K | LenV | V |
|
||||
// +------+----+------+------+------+------+
|
||||
// | byte | 4 | 4 | LenK | 4 | LenV |
|
||||
// +------+----+------+------+------+------+
|
||||
func (s *MetaItem) UnmarshalBinary(raw []byte) (err error) {
|
||||
var (
|
||||
lenK uint32
|
||||
|
||||
@ -25,7 +25,6 @@ import (
|
||||
"github.com/cubefs/cubefs/proto"
|
||||
"github.com/cubefs/cubefs/util/auditlog"
|
||||
"github.com/cubefs/cubefs/util/errors"
|
||||
"github.com/cubefs/cubefs/util/log"
|
||||
)
|
||||
|
||||
func (mp *metaPartition) TxCreateDentry(req *proto.TxCreateDentryRequest, p *Packet, remoteAddr string) (err error) {
|
||||
@ -286,10 +285,7 @@ func (mp *metaPartition) DeleteDentry(req *DeleteDentryReq, p *Packet, remoteAdd
|
||||
return
|
||||
}
|
||||
}
|
||||
dentry := &Dentry{
|
||||
ParentId: req.ParentID,
|
||||
Name: req.Name,
|
||||
}
|
||||
|
||||
dentry.setVerSeq(req.Verseq)
|
||||
log.LogDebugf("action[DeleteDentry] den param(%v)", dentry)
|
||||
|
||||
|
||||
@ -22,7 +22,8 @@ import (
|
||||
)
|
||||
|
||||
func (mp *metaPartition) batchSetInodeQuota(req *proto.BatchSetMetaserverQuotaReuqest,
|
||||
resp *proto.BatchSetMetaserverQuotaResponse) (err error) {
|
||||
resp *proto.BatchSetMetaserverQuotaResponse,
|
||||
) (err error) {
|
||||
if len(req.Inodes) == 0 {
|
||||
return nil
|
||||
}
|
||||
@ -45,7 +46,8 @@ func (mp *metaPartition) batchSetInodeQuota(req *proto.BatchSetMetaserverQuotaRe
|
||||
}
|
||||
|
||||
func (mp *metaPartition) batchDeleteInodeQuota(req *proto.BatchDeleteMetaserverQuotaReuqest,
|
||||
resp *proto.BatchDeleteMetaserverQuotaResponse) (err error) {
|
||||
resp *proto.BatchDeleteMetaserverQuotaResponse,
|
||||
) (err error) {
|
||||
if len(req.Inodes) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -1053,7 +1053,8 @@ func (mp *metaPartition) storeTxInfo(rootDir string, sm *storeMsg) (crc uint32,
|
||||
}
|
||||
|
||||
func (mp *metaPartition) storeInode(rootDir string,
|
||||
sm *storeMsg) (crc uint32, err error) {
|
||||
sm *storeMsg,
|
||||
) (crc uint32, err error) {
|
||||
filename := path.Join(rootDir, inodeFile)
|
||||
fp, err := os.OpenFile(filename, os.O_RDWR|os.O_TRUNC|os.O_APPEND|os.
|
||||
O_CREATE, 0o755)
|
||||
@ -1112,7 +1113,8 @@ func (mp *metaPartition) storeInode(rootDir string,
|
||||
}
|
||||
|
||||
func (mp *metaPartition) storeDentry(rootDir string,
|
||||
sm *storeMsg) (crc uint32, err error) {
|
||||
sm *storeMsg,
|
||||
) (crc uint32, err error) {
|
||||
filename := path.Join(rootDir, dentryFile)
|
||||
fp, err := os.OpenFile(filename, os.O_RDWR|os.O_TRUNC|os.O_APPEND|os.
|
||||
O_CREATE, 0o755)
|
||||
|
||||
@ -101,7 +101,8 @@ func (m *MetaNode) serveConn(conn net.Conn, stopC chan uint8) {
|
||||
}
|
||||
|
||||
func (m *MetaNode) handlePacket(conn net.Conn, p *Packet,
|
||||
remoteAddr string) (err error) {
|
||||
remoteAddr string,
|
||||
) (err error) {
|
||||
// Handle request
|
||||
err = m.metadataManager.HandleMetadataOperation(conn, p, remoteAddr)
|
||||
return
|
||||
|
||||
@ -300,7 +300,7 @@ type TransactionManager struct {
|
||||
txIdAlloc *TxIDAllocator
|
||||
txTree *BTree
|
||||
txProcessor *TransactionProcessor
|
||||
blacklist *util.SyncSet
|
||||
blacklist *util.Set
|
||||
opLimiter *rate.Limiter
|
||||
sync.RWMutex
|
||||
}
|
||||
@ -334,7 +334,7 @@ func NewTransactionManager(txProcessor *TransactionProcessor) *TransactionManage
|
||||
txIdAlloc: newTxIDAllocator(),
|
||||
txTree: NewBtree(),
|
||||
txProcessor: txProcessor,
|
||||
blacklist: util.NewSyncSet(),
|
||||
blacklist: util.NewSet(),
|
||||
opLimiter: rate.NewLimiter(rate.Inf, 128),
|
||||
}
|
||||
return txMgr
|
||||
|
||||
@ -560,7 +560,8 @@ func (o *ObjectNode) listPartsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (o *ObjectNode) checkReqParts(param *RequestParam, reqParts *CompleteMultipartUploadRequest, multipartInfo *proto.MultipartInfo) (
|
||||
discardedPartInodes map[uint64]uint16, committedPartInfo *proto.MultipartInfo, err error) {
|
||||
discardedPartInodes map[uint64]uint16, committedPartInfo *proto.MultipartInfo, err error,
|
||||
) {
|
||||
if len(reqParts.Parts) <= 0 {
|
||||
err = InvalidPart
|
||||
log.LogErrorf("checkReqParts: upload part is empty: requestID(%v) volume(%v)", GetRequestID(param.r), param.Bucket())
|
||||
|
||||
@ -1997,7 +1997,7 @@ func (v *Volume) recursiveMakeDirectory(path string) (partentIno uint64, err err
|
||||
}
|
||||
if err == syscall.ENOENT {
|
||||
var info *proto.InodeInfo
|
||||
info, err = v.mw.Create_ll(partentIno, pathItem.Name, uint32(DefaultDirMode), 0, 0, nil, path[:pathIterator.cursor])
|
||||
info, err = v.mw.Create_ll(partentIno, pathItem.Name, uint32(DefaultDirMode), 0, 0, nil, path[:pathIterator.cursor], false)
|
||||
if err != nil && err == syscall.EEXIST {
|
||||
existInode, mode, e := v.mw.Lookup_ll(partentIno, pathItem.Name)
|
||||
if e != nil {
|
||||
@ -2081,7 +2081,8 @@ func (v *Volume) lookupDirectories(dirs []string, autoCreate bool) (inode uint64
|
||||
}
|
||||
|
||||
func (v *Volume) listFilesV1(prefix, marker, delimiter string, maxKeys uint64, onlyObject bool) (infos []*FSFileInfo,
|
||||
prefixes Prefixes, nextMarker string, err error) {
|
||||
prefixes Prefixes, nextMarker string, err error,
|
||||
) {
|
||||
prefixMap := PrefixMap(make(map[string]struct{}))
|
||||
|
||||
parentId, dirs, err := v.findParentId(prefix)
|
||||
@ -2128,7 +2129,8 @@ func (v *Volume) listFilesV1(prefix, marker, delimiter string, maxKeys uint64, o
|
||||
}
|
||||
|
||||
func (v *Volume) listFilesV2(prefix, startAfter, contToken, delimiter string, maxKeys uint64) (infos []*FSFileInfo,
|
||||
prefixes Prefixes, nextMarker string, err error) {
|
||||
prefixes Prefixes, nextMarker string, err error,
|
||||
) {
|
||||
prefixMap := PrefixMap(make(map[string]struct{}))
|
||||
|
||||
var marker string
|
||||
@ -2234,7 +2236,8 @@ func (v *Volume) findParentId(prefix string) (inode uint64, prefixDirs []string,
|
||||
// that match the prefix and delimiter criteria. Stop when the number of matches reaches a threshold
|
||||
// or all files and directories are scanned.
|
||||
func (v *Volume) recursiveScan(fileInfos []*FSFileInfo, prefixMap PrefixMap, parentId, maxKeys, readLimit, rc uint64, dirs []string,
|
||||
prefix, marker, delimiter string, onlyObject, firstEnter bool) ([]*FSFileInfo, PrefixMap, string, uint64, error) {
|
||||
prefix, marker, delimiter string, onlyObject, firstEnter bool,
|
||||
) ([]*FSFileInfo, PrefixMap, string, uint64, error) {
|
||||
var err error
|
||||
var nextMarker string
|
||||
var lastKey string
|
||||
@ -2469,7 +2472,8 @@ func (v *Volume) updateETag(inode uint64, size int64, mt time.Time) (etagValue E
|
||||
}
|
||||
|
||||
func (v *Volume) ListMultipartUploads(prefix, delimiter, keyMarker string, multipartIdMarker string, maxUploads uint64) (
|
||||
uploads []*FSUpload, nextMarker, nextMultipartIdMarker string, isTruncated bool, prefixes []string, err error) {
|
||||
uploads []*FSUpload, nextMarker, nextMultipartIdMarker string, isTruncated bool, prefixes []string, err error,
|
||||
) {
|
||||
sessions, err := v.mw.ListMultipart_ll(prefix, delimiter, keyMarker, multipartIdMarker, maxUploads)
|
||||
if err != nil || len(sessions) == 0 {
|
||||
return
|
||||
|
||||
@ -39,8 +39,10 @@ const (
|
||||
)
|
||||
|
||||
// if more s3 api is supported by policy, need extend bucketApiList, objectApiList
|
||||
var bucketApiList = SliceString{LIST_OBJECTS, LIST_OBJECTS_V2, HEAD_BUCKET, DELETE_BUCKET, LIST_MULTIPART_UPLOADS, GET_BUCKET_LOCATION, GET_OBJECT_LOCK_CFG, PUT_OBJECT_LOCK_CFG}
|
||||
var objectApiList = SliceString{GET_OBJECT, HEAD_OBJECT, DELETE_OBJECT, PUT_OBJECT, POST_OBJECT, INITIALE_MULTIPART_UPLOAD, UPLOAD_PART, UPLOAD_PART_COPY, COMPLETE_MULTIPART_UPLOAD, COPY_OBJECT, ABORT_MULTIPART_UPLOAD, LIST_PARTS, BATCH_DELETE, GET_OBJECT_RETENTION}
|
||||
var (
|
||||
bucketApiList = SliceString{LIST_OBJECTS, LIST_OBJECTS_V2, HEAD_BUCKET, DELETE_BUCKET, LIST_MULTIPART_UPLOADS, GET_BUCKET_LOCATION, GET_OBJECT_LOCK_CFG, PUT_OBJECT_LOCK_CFG}
|
||||
objectApiList = SliceString{GET_OBJECT, HEAD_OBJECT, DELETE_OBJECT, PUT_OBJECT, POST_OBJECT, INITIALE_MULTIPART_UPLOAD, UPLOAD_PART, UPLOAD_PART_COPY, COMPLETE_MULTIPART_UPLOAD, COPY_OBJECT, ABORT_MULTIPART_UPLOAD, LIST_PARTS, BATCH_DELETE, GET_OBJECT_RETENTION}
|
||||
)
|
||||
|
||||
type SliceString []string
|
||||
|
||||
|
||||
@ -53,7 +53,7 @@ func (key Key) IsValid() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// encodes Key to JSON data.
|
||||
// encodes Key to JSON data.
|
||||
func (key Key) MarshalJSON() ([]byte, error) {
|
||||
if !key.IsValid() {
|
||||
return nil, fmt.Errorf("unknown key: %v", key)
|
||||
@ -73,7 +73,7 @@ func (key Key) Name() string {
|
||||
return strings.TrimPrefix(keyString, "s3:")
|
||||
}
|
||||
|
||||
// decodes string data to Key.
|
||||
// decodes string data to Key.
|
||||
func (key *Key) UnmarshalText(data []byte) error {
|
||||
var s string
|
||||
if err := json.Unmarshal(data, &s); err != nil {
|
||||
@ -89,7 +89,7 @@ func (key *Key) UnmarshalText(data []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// decodes JSON data to Key.
|
||||
// decodes JSON data to Key.
|
||||
func (key *Key) UnmarshalJSON(data []byte) error {
|
||||
var s string
|
||||
if err := json.Unmarshal(data, &s); err != nil {
|
||||
@ -128,9 +128,10 @@ func (set KeySet) AddAll(keys KeySet) {
|
||||
|
||||
// returns a key set contains difference of two key set.
|
||||
// Example:
|
||||
// keySet1 := ["one", "two", "three"]
|
||||
// keySet2 := ["two", "four", "three"]
|
||||
// keySet1.Difference(keySet2) == ["one"]
|
||||
//
|
||||
// keySet1 := ["one", "two", "three"]
|
||||
// keySet2 := ["two", "four", "three"]
|
||||
// keySet1.Difference(keySet2) == ["one"]
|
||||
func (set KeySet) Difference(sset KeySet) KeySet {
|
||||
nset := make(KeySet)
|
||||
|
||||
|
||||
@ -67,7 +67,7 @@ func (s *Statement) match(apiName string, uid string, conditionCheck map[string]
|
||||
return true
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------------------------------------------------------
|
||||
// ----------------------------------------------------------------------------------------------------------------
|
||||
func (s *Statement) matchAction(apiName string) bool {
|
||||
log.LogDebug("start to match action")
|
||||
switch s.Action.(type) {
|
||||
@ -106,7 +106,7 @@ func (actions ActionType) match(apiToMatch string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------------------------------------------------------
|
||||
// ----------------------------------------------------------------------------------------------------------------
|
||||
func (s *Statement) matchPrincipal(uid string) bool {
|
||||
switch s.Principal.(type) {
|
||||
case string: // "*" or "123"
|
||||
@ -146,7 +146,7 @@ func (p PrincipalType) match(uid string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------------------------------------------------------
|
||||
// ----------------------------------------------------------------------------------------------------------------
|
||||
func (s *Statement) matchResource(apiName string, keyname interface{}) bool {
|
||||
if IsBucketApi(apiName) {
|
||||
return s.matchBucketInResource()
|
||||
@ -237,7 +237,7 @@ func makeRegexPattern(raw string) string {
|
||||
return pattern
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------------------------------------------------------
|
||||
// ----------------------------------------------------------------------------------------------------------------
|
||||
func (s *Statement) matchCondition(conditionCheck map[string]string) bool {
|
||||
// condition is optional
|
||||
if s.Condition == nil {
|
||||
|
||||
@ -106,7 +106,7 @@ func (s *Statement) isEffectValid() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// "Principal": "*" or "Principal" : {"AWS":"111122223333"} or "Principal" : {"AWS":["111122223333","444455556666"]}
|
||||
// "Principal": "*" or "Principal" : {"AWS":"111122223333"} or "Principal" : {"AWS":["111122223333","444455556666"]}
|
||||
func (s *Statement) isPrincipalValid() bool {
|
||||
// principal: uid must be "*" or uint32, and can't be 0
|
||||
switch s.Principal.(type) {
|
||||
|
||||
@ -50,7 +50,7 @@ func (op stringLikeOp) evaluate(values map[string]string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// returns condition key which is used by this condition operation.
|
||||
// returns condition key which is used by this condition operation.
|
||||
func (op stringLikeOp) keys() KeySet {
|
||||
keys := make(KeySet)
|
||||
for key := range op.m {
|
||||
@ -128,7 +128,7 @@ func (op stringNotLikeOp) operator() operator {
|
||||
return stringNotLike
|
||||
}
|
||||
|
||||
// returns new StringNotLike operation.
|
||||
// returns new StringNotLike operation.
|
||||
func newStringNotLikeOp(m map[Key]ValueSet) (Operation, error) {
|
||||
newMap, err := parseMap(m, stringNotLike)
|
||||
if err != nil {
|
||||
@ -138,7 +138,7 @@ func newStringNotLikeOp(m map[Key]ValueSet) (Operation, error) {
|
||||
return NewStringNotLikeOp(newMap)
|
||||
}
|
||||
|
||||
// returns new StringNotLike operation.
|
||||
// returns new StringNotLike operation.
|
||||
func NewStringNotLikeOp(m map[Key]StringSet) (Operation, error) {
|
||||
return &stringNotLikeOp{stringLikeOp{m}}, nil
|
||||
}
|
||||
|
||||
@ -22,7 +22,7 @@ import (
|
||||
|
||||
type StringSet map[string]struct{}
|
||||
|
||||
// returns StringSet as string slice.
|
||||
// returns StringSet as string slice.
|
||||
func (set StringSet) ToSlice() []string {
|
||||
keys := make([]string, 0, len(set))
|
||||
for k := range set {
|
||||
@ -147,7 +147,7 @@ func NewStringSet() StringSet {
|
||||
return make(StringSet)
|
||||
}
|
||||
|
||||
// creates new string set with given string values.
|
||||
// creates new string set with given string values.
|
||||
func CreateStringSet(sl ...string) StringSet {
|
||||
set := make(StringSet)
|
||||
for _, k := range sl {
|
||||
|
||||
@ -51,7 +51,7 @@ func (v Value) GetInt() (int, error) {
|
||||
return v.i, err
|
||||
}
|
||||
|
||||
// gets stored string value.
|
||||
// gets stored string value.
|
||||
func (v Value) GetString() (string, error) {
|
||||
var err error
|
||||
|
||||
@ -81,12 +81,12 @@ func (v Value) MarshalJSON() ([]byte, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// stores bool value.
|
||||
// stores bool value.
|
||||
func (v *Value) StoreBool(b bool) {
|
||||
*v = Value{t: reflect.Bool, b: b}
|
||||
}
|
||||
|
||||
// stores int value.
|
||||
// stores int value.
|
||||
func (v *Value) StoreInt(i int) {
|
||||
*v = Value{t: reflect.Int, i: i}
|
||||
}
|
||||
@ -154,7 +154,7 @@ func NewStringValue(s string) Value {
|
||||
return *value
|
||||
}
|
||||
|
||||
// unique list of values.
|
||||
// unique list of values.
|
||||
type ValueSet map[Value]struct{}
|
||||
|
||||
// adds given value to value set.
|
||||
@ -162,7 +162,7 @@ func (set ValueSet) Add(value Value) {
|
||||
set[value] = struct{}{}
|
||||
}
|
||||
|
||||
// encodes ValueSet to JSON data.
|
||||
// encodes ValueSet to JSON data.
|
||||
func (set ValueSet) MarshalJSON() ([]byte, error) {
|
||||
values := []Value{}
|
||||
for k := range set {
|
||||
@ -206,7 +206,7 @@ func (set *ValueSet) UnmarshalJSON(data []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// returns new value set containing given values.
|
||||
// returns new value set containing given values.
|
||||
func NewValueSet(values ...Value) ValueSet {
|
||||
set := make(ValueSet)
|
||||
|
||||
|
||||
@ -304,8 +304,10 @@ func ParseUserDefinedMetadata(header http.Header) map[string]string {
|
||||
}
|
||||
|
||||
// validate Cache-Control
|
||||
var cacheControlDir = []string{"public", "private", "no-cache", "no-store", "no-transform", "must-revalidate", "proxy-revalidate"}
|
||||
var maxAgeRegexp = regexp.MustCompile("^((max-age)|(s-maxage))=[1-9][0-9]*$")
|
||||
var (
|
||||
cacheControlDir = []string{"public", "private", "no-cache", "no-store", "no-transform", "must-revalidate", "proxy-revalidate"}
|
||||
maxAgeRegexp = regexp.MustCompile("^((max-age)|(s-maxage))=[1-9][0-9]*$")
|
||||
)
|
||||
|
||||
func ValidateCacheControl(cacheControl string) bool {
|
||||
cacheDirs := strings.Split(cacheControl, ",")
|
||||
|
||||
@ -19,6 +19,8 @@ import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/cubefs/cubefs/util"
|
||||
)
|
||||
|
||||
// api
|
||||
@ -235,7 +237,7 @@ const (
|
||||
QuotaGet = "/quota/get"
|
||||
// QuotaBatchModifyPath = "/quota/batchModifyPath"
|
||||
QuotaListAll = "/quota/listAll"
|
||||
//trash
|
||||
// trash
|
||||
AdminSetTrashInterval = "/vol/setTrashInterval"
|
||||
|
||||
// s3 qos api
|
||||
@ -1108,7 +1110,7 @@ type SimpleVolView struct {
|
||||
// multi version snapshot
|
||||
LatestVer uint64
|
||||
Forbidden bool
|
||||
DisableAuditLog bool
|
||||
DisableAuditLog bool
|
||||
DeleteExecTime time.Time
|
||||
DpRepairBlockSize uint64
|
||||
}
|
||||
|
||||
@ -375,6 +375,7 @@ type BatchUnlinkInodeRequest struct {
|
||||
VolName string `json:"vol"`
|
||||
PartitionID uint64 `json:"pid"`
|
||||
Inodes []uint64 `json:"inos"`
|
||||
FullPaths []string `json:"fullPaths"`
|
||||
}
|
||||
|
||||
// UnlinkInodeResponse defines the response to the request of unlinking an inode.
|
||||
@ -403,6 +404,7 @@ type BatchEvictInodeRequest struct {
|
||||
VolName string `json:"vol"`
|
||||
PartitionID uint64 `json:"pid"`
|
||||
Inodes []uint64 `json:"inos"`
|
||||
FullPaths []string `json:"fullPaths"`
|
||||
}
|
||||
|
||||
// CreateDentryRequest defines the request to create a dentry.
|
||||
|
||||
@ -328,11 +328,11 @@ type MountOptions struct {
|
||||
RequestTimeout int64
|
||||
MinWriteAbleDataPartitionCnt int
|
||||
FileSystemName string
|
||||
//TrashInterval int64
|
||||
// TrashInterval int64
|
||||
TrashDeleteExpiredDirGoroutineLimit int64
|
||||
TrashRebuildGoroutineLimit int64
|
||||
|
||||
VerReadSeq uint64
|
||||
VerReadSeq uint64
|
||||
// disable mount subtype
|
||||
DisableMountSubtype bool
|
||||
}
|
||||
|
||||
@ -130,7 +130,7 @@ const (
|
||||
OpLcNodeScan uint8 = 0x56
|
||||
OpLcNodeSnapshotVerDel uint8 = 0x5B
|
||||
|
||||
//backUp
|
||||
// backUp
|
||||
OpBatchLockNormalExtent uint8 = 0x57
|
||||
OpBatchUnlockNormalExtent uint8 = 0x58
|
||||
OpBackupRead uint8 = 0x59
|
||||
@ -257,9 +257,8 @@ const (
|
||||
OpReadRepairExtentAgain uint8 = 0xEF
|
||||
|
||||
// io speed limit
|
||||
OpLimitedIoErr uint8 = 0xB1
|
||||
OpReadRepairExtentAgain uint8 = 0xEF
|
||||
OpStoreClosed uint8 = 0xB2
|
||||
OpLimitedIoErr uint8 = 0xB1
|
||||
OpStoreClosed uint8 = 0xB2
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@ -80,7 +80,8 @@ type partition struct {
|
||||
|
||||
// ChangeMember submits member change event and information to raft log.
|
||||
func (p *partition) ChangeMember(changeType proto.ConfChangeType, peer proto.Peer, context []byte) (
|
||||
resp interface{}, err error) {
|
||||
resp interface{}, err error,
|
||||
) {
|
||||
if !p.IsRaftLeader() {
|
||||
err = raft.ErrNotLeader
|
||||
return
|
||||
|
||||
@ -205,7 +205,8 @@ func (ft *FollowerTransport) Write(p *FollowerPacket) {
|
||||
}
|
||||
|
||||
func NewReplProtocol(inConn net.Conn, prepareFunc func(p *Packet) error,
|
||||
operatorFunc func(p *Packet, c net.Conn) error, postFunc func(p *Packet) error) *ReplProtocol {
|
||||
operatorFunc func(p *Packet, c net.Conn) error, postFunc func(p *Packet) error,
|
||||
) *ReplProtocol {
|
||||
rp := new(ReplProtocol)
|
||||
rp.packetList = list.New()
|
||||
rp.ackCh = make(chan struct{}, RequestChanSize)
|
||||
|
||||
@ -22,8 +22,6 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/cubefs/cubefs/util/stat"
|
||||
|
||||
"github.com/cubefs/cubefs/proto"
|
||||
"github.com/cubefs/cubefs/sdk/data/wrapper"
|
||||
"github.com/cubefs/cubefs/sdk/meta"
|
||||
@ -463,7 +461,7 @@ func (eh *ExtentHandler) appendExtentKey() (err error) {
|
||||
return
|
||||
}
|
||||
discard := eh.stream.extents.Append(eh.key, true)
|
||||
err = eh.stream.client.appendExtentKey(eh.stream.parentInode, eh.inode, *eh.key, discard)
|
||||
status, err := eh.stream.client.appendExtentKey(eh.stream.parentInode, eh.inode, *eh.key, discard)
|
||||
|
||||
ekey := *eh.key
|
||||
doAppend := func() (err error) {
|
||||
@ -484,7 +482,7 @@ func (eh *ExtentHandler) appendExtentKey() (err error) {
|
||||
eh.dirty = false
|
||||
eh.lastKey = *eh.key
|
||||
log.LogDebugf("action[appendExtentKey] status %v, needUpdateVer %v, eh{%v}", status, eh.stream.needUpdateVer, eh)
|
||||
return
|
||||
return nil
|
||||
}
|
||||
// Due to the asynchronous synchronization of version numbers, the extent cache version of the client is updated first before being written to the meta.
|
||||
// However, it is possible for the client version to lag behind the meta version, resulting in partial inconsistencies in judgment.
|
||||
|
||||
@ -70,7 +70,8 @@ func NewWritePacket(inode uint64, fileOffset, storeMode int) *Packet {
|
||||
|
||||
// NewOverwritePacket returns a new overwrite packet.
|
||||
func NewOverwriteByAppendPacket(dp *wrapper.DataPartition, extentID uint64, extentOffset int,
|
||||
inode uint64, fileOffset int, direct bool, op uint8) *Packet {
|
||||
inode uint64, fileOffset int, direct bool, op uint8,
|
||||
) *Packet {
|
||||
p := new(Packet)
|
||||
p.PartitionID = dp.PartitionID
|
||||
p.Magic = proto.ProtoMagic
|
||||
|
||||
@ -144,7 +144,8 @@ func (s *DefaultRandomSelector) getLocalLeaderDataPartition(exclude map[string]s
|
||||
}
|
||||
|
||||
func (s *DefaultRandomSelector) getRandomDataPartition(partitions []*DataPartition, exclude map[string]struct{}) (
|
||||
dp *DataPartition) {
|
||||
dp *DataPartition,
|
||||
) {
|
||||
length := len(partitions)
|
||||
if length == 0 {
|
||||
return nil
|
||||
|
||||
@ -247,17 +247,6 @@ func (api *AdminAPI) UnDeleteVolume(volName, authKey string, status bool) (err e
|
||||
return
|
||||
}
|
||||
|
||||
func (api *AdminAPI) UnDeleteVolume(volName, authKey string, status bool) (err error) {
|
||||
var request = newAPIRequest(http.MethodGet, proto.AdminDeleteVol)
|
||||
request.addParam("name", volName)
|
||||
request.addParam("authKey", authKey)
|
||||
request.addParam("delete", strconv.FormatBool(false))
|
||||
if _, err = api.mc.serveRequest(request); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (api *AdminAPI) UpdateVolume(
|
||||
vv *proto.SimpleVolView,
|
||||
txTimeout int64,
|
||||
@ -406,15 +395,6 @@ func (api *AdminAPI) SetVolumeForbidden(volName string, forbidden bool) (err err
|
||||
_, err = api.mc.serveRequest(request)
|
||||
return
|
||||
}
|
||||
func (api *AdminAPI) SetVolumeForbidden(volName string, forbidden bool) (err error) {
|
||||
request := newAPIRequest(http.MethodGet, proto.AdminVolForbidden)
|
||||
request.addParam("name", volName)
|
||||
request.addParam("forbidden", strconv.FormatBool(forbidden))
|
||||
if _, err = api.mc.serveRequest(request); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (api *AdminAPI) SetVolumeAuditLog(volName string, enable bool) (err error) {
|
||||
request := newRequest(post, proto.AdminVolEnableAuditLog).Header(api.h)
|
||||
@ -769,7 +749,7 @@ func (api *AdminAPI) AbortDiskDecommission(addr string, disk string) (err error)
|
||||
}
|
||||
|
||||
func (api *AdminAPI) SetClusterDecommissionLimit(limit int32) (err error) {
|
||||
var request = newAPIRequest(http.MethodPost, proto.AdminUpdateDecommissionLimit)
|
||||
request := newAPIRequest(http.MethodPost, proto.AdminUpdateDecommissionLimit)
|
||||
request.addParam("decommissionLimit", strconv.FormatInt(int64(limit), 10))
|
||||
if _, err = api.mc.serveRequest(request); err != nil {
|
||||
return
|
||||
@ -779,7 +759,7 @@ func (api *AdminAPI) SetClusterDecommissionLimit(limit int32) (err error) {
|
||||
|
||||
func (api *AdminAPI) QueryDecommissionToken() (status []proto.DecommissionTokenStatus, err error) {
|
||||
var buf []byte
|
||||
var request = newAPIRequest(http.MethodGet, proto.AdminQueryDecommissionToken)
|
||||
request := newAPIRequest(http.MethodGet, proto.AdminQueryDecommissionToken)
|
||||
if buf, err = api.mc.serveRequest(request); err != nil {
|
||||
return
|
||||
}
|
||||
@ -807,4 +787,4 @@ func (api AdminAPI) SetDecommissionDiskLimit(limit uint32) (err error) {
|
||||
|
||||
err = api.mc.request(request)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@ -109,6 +109,19 @@ func newRequest(method string, path string) *request {
|
||||
return req
|
||||
}
|
||||
|
||||
func newAPIRequest(method string, path string) *request {
|
||||
req := &request{
|
||||
method: method,
|
||||
path: path,
|
||||
params: make(map[string]string),
|
||||
header: make(map[string]string),
|
||||
}
|
||||
|
||||
req.header["User-Agent"] = ReqHeaderUA
|
||||
|
||||
return req
|
||||
}
|
||||
|
||||
func mergeHeader(headers map[string]string, added ...string) map[string]string {
|
||||
if len(added)%2 == 1 {
|
||||
added = added[:len(added)-1]
|
||||
|
||||
@ -134,7 +134,8 @@ func (mw *MetaWrapper) Create_ll(parentID uint64, name string, mode, uid, gid ui
|
||||
}
|
||||
|
||||
func (mw *MetaWrapper) txCreate_ll(parentID uint64, name string, mode, uid, gid uint32, target []byte, txType uint32,
|
||||
fullPath string, ignoreExist bool) (info *proto.InodeInfo, err error) {
|
||||
fullPath string, ignoreExist bool,
|
||||
) (info *proto.InodeInfo, err error) {
|
||||
var (
|
||||
status int
|
||||
// err error
|
||||
@ -226,7 +227,8 @@ create_dentry:
|
||||
}
|
||||
|
||||
func (mw *MetaWrapper) create_ll(parentID uint64, name string, mode, uid, gid uint32, target []byte,
|
||||
fullPath string, ignoreExist bool) (*proto.InodeInfo, error) {
|
||||
fullPath string, ignoreExist bool,
|
||||
) (*proto.InodeInfo, error) {
|
||||
var (
|
||||
status int
|
||||
err error
|
||||
@ -358,7 +360,7 @@ func (mw *MetaWrapper) Lookup_ll(parentID uint64, name string) (inode uint64, mo
|
||||
if err != nil || status != statusOK {
|
||||
return 0, 0, statusToErrno(status)
|
||||
}
|
||||
//only save dir
|
||||
// only save dir
|
||||
if proto.IsDir(mode) {
|
||||
mw.AddInoInfoCache(inode, parentID, name)
|
||||
}
|
||||
@ -560,7 +562,7 @@ func (mw *MetaWrapper) BatchGetXAttr(inodes []uint64, keys []string) ([]*proto.X
|
||||
func (mw *MetaWrapper) shouldNotMoveToTrash(parentMP *MetaPartition, parentIno uint64, entry string, isDir bool) (error, bool) {
|
||||
log.LogDebugf("action[shouldNotMoveToTrash]: parentIno(%v) entry(%v)", parentIno, entry)
|
||||
|
||||
status, inode, mode, err := mw.lookup(parentMP, parentIno, entry)
|
||||
status, inode, mode, err := mw.lookup(parentMP, parentIno, entry, mw.LastVerSeq)
|
||||
if err != nil || status != statusOK {
|
||||
return statusToErrno(status), false
|
||||
}
|
||||
@ -569,7 +571,7 @@ func (mw *MetaWrapper) shouldNotMoveToTrash(parentMP *MetaPartition, parentIno u
|
||||
log.LogErrorf("shouldNotMoveToTrash: No inode partition, parentID(%v) name(%v) ino(%v)", parentIno, entry, inode)
|
||||
return syscall.EAGAIN, false
|
||||
}
|
||||
status, info, err := mw.iget(mp, inode)
|
||||
status, info, err := mw.iget(mp, inode, mw.LastVerSeq)
|
||||
if err != nil || status != statusOK {
|
||||
return statusToErrno(status), false
|
||||
}
|
||||
@ -611,11 +613,11 @@ func (mw *MetaWrapper) shouldNotMoveToTrash(parentMP *MetaPartition, parentIno u
|
||||
if strings.Contains(currentPath, UnknownPath) {
|
||||
return nil, true
|
||||
}
|
||||
//log.LogDebugf("action[shouldNotMoveToTrash]: parentIno(%v) entry(%v) currentPath(%v)", parentIno, entry, currentPath)
|
||||
// log.LogDebugf("action[shouldNotMoveToTrash]: parentIno(%v) entry(%v) currentPath(%v)", parentIno, entry, currentPath)
|
||||
|
||||
subs := strings.Split(currentPath, "/")
|
||||
if len(subs) == 1 {
|
||||
//should never happen: file in root trash
|
||||
// should never happen: file in root trash
|
||||
if subs[0] == TrashPrefix {
|
||||
log.LogDebugf("action[shouldNotMoveToTrash]: currentPath(%v) should not remove ", subs[0])
|
||||
return nil, true
|
||||
@ -625,13 +627,13 @@ func (mw *MetaWrapper) shouldNotMoveToTrash(parentMP *MetaPartition, parentIno u
|
||||
log.LogDebugf("action[shouldNotMoveToTrash]: currentPath(%v)is expired ", currentPath)
|
||||
return nil, true
|
||||
}
|
||||
//should never happen: dir in root trash
|
||||
// should never happen: dir in root trash
|
||||
if subs[0] == TrashPrefix {
|
||||
log.LogDebugf("action[shouldMoveToTrash]: currentPath(%v) not legal ", currentPath)
|
||||
return nil, true
|
||||
}
|
||||
}
|
||||
//log.LogDebugf("action[shouldMoveToTrash]: currentPath(%v) is not expired ", currentPath)
|
||||
// log.LogDebugf("action[shouldMoveToTrash]: currentPath(%v) is not expired ", currentPath)
|
||||
return nil, false
|
||||
}
|
||||
|
||||
@ -647,6 +649,10 @@ func (mw *MetaWrapper) Delete_ll(parentID uint64, name string, isDir bool, fullP
|
||||
}
|
||||
}
|
||||
|
||||
func (mw *MetaWrapper) DeleteWithCond_ll(parentID, cond uint64, name string, isDir bool, fullPath string) (*proto.InodeInfo, error) {
|
||||
return mw.deletewithcond_ll(parentID, cond, name, isDir, fullPath)
|
||||
}
|
||||
|
||||
func (mw *MetaWrapper) Delete_Ver_ll(parentID uint64, name string, isDir bool, verSeq uint64, fullPath string) (*proto.InodeInfo, error) {
|
||||
if verSeq == 0 {
|
||||
verSeq = math.MaxUint64
|
||||
@ -678,7 +684,7 @@ func (mw *MetaWrapper) txDelete_ll(parentID uint64, name string, isDir bool, ful
|
||||
log.LogDebugf("TRACE Remove:TrashPolicy is nil")
|
||||
mw.enableTrash()
|
||||
}
|
||||
//cannot delete .Trash
|
||||
// cannot delete .Trash
|
||||
err, ret := mw.shouldNotMoveToTrash(parentMP, parentID, name, isDir)
|
||||
if err != nil {
|
||||
log.LogErrorf("Delete_ll: shouldNotMoveToTrash failed:%v", err)
|
||||
@ -790,7 +796,7 @@ func (mw *MetaWrapper) txDelete_ll(parentID uint64, name string, isDir bool, ful
|
||||
}
|
||||
tx.SetOnCommit(job)
|
||||
}
|
||||
//clear trash cache
|
||||
// clear trash cache
|
||||
if mw.trashPolicy != nil && mw.disableTrash == false {
|
||||
mw.trashPolicy.CleanTrashPatchCache(mw.getCurrentPath(parentID), name)
|
||||
}
|
||||
@ -825,7 +831,7 @@ func (mw *MetaWrapper) Delete_ll_EX(parentID uint64, name string, isDir bool, ve
|
||||
log.LogDebugf("TRACE Remove:TrashPolicy is nil")
|
||||
mw.enableTrash()
|
||||
}
|
||||
//cannot delete .Trash
|
||||
// cannot delete .Trash
|
||||
err, ret := mw.shouldNotMoveToTrash(parentMP, parentID, name, isDir)
|
||||
if err != nil {
|
||||
log.LogErrorf("Delete_ll: shouldNotMoveToTrash name %v failed %v", name, err)
|
||||
@ -1061,7 +1067,7 @@ func (mw *MetaWrapper) deletewithcond_ll(parentID, cond uint64, name string, isD
|
||||
}
|
||||
}()
|
||||
}
|
||||
//clear trash cache
|
||||
// clear trash cache
|
||||
if mw.trashPolicy != nil && mw.disableTrash == false {
|
||||
mw.trashPolicy.CleanTrashPatchCache(mw.getCurrentPath(parentID), name)
|
||||
}
|
||||
@ -1265,6 +1271,7 @@ func (mw *MetaWrapper) rename_ll(srcParentID uint64, srcName string, dstParentID
|
||||
oldInode uint64
|
||||
lastVerSeq uint64
|
||||
)
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
if err != nil {
|
||||
log.LogErrorf("Rename_ll: srcName %v srcFullPath %v dstName %v dstFullPath %v err %v",
|
||||
@ -1335,7 +1342,7 @@ func (mw *MetaWrapper) rename_ll(srcParentID uint64, srcName string, dstParentID
|
||||
return syscall.EAGAIN
|
||||
}
|
||||
dstInodeInfo, _ = mw.InodeGet_ll(oldInode)
|
||||
//delete old cache
|
||||
// delete old cache
|
||||
mw.DeleteInoInfoCache(oldInode)
|
||||
}
|
||||
|
||||
@ -1429,8 +1436,8 @@ func (mw *MetaWrapper) rename_ll(srcParentID uint64, srcName string, dstParentID
|
||||
// mw.BatchSetInodeQuota_ll(inodes, quotaId, false)
|
||||
// }
|
||||
|
||||
//log.LogDebugf("Rename_ll: dstInodeInfo %v", dstInodeInfo)
|
||||
//mw.AddInoInfoCache(dstInodeInfo.Inode, dstParentID, dstName)
|
||||
// log.LogDebugf("Rename_ll: dstInodeInfo %v", dstInodeInfo)
|
||||
// mw.AddInoInfoCache(dstInodeInfo.Inode, dstParentID, dstName)
|
||||
mw.DeleteInoInfoCache(srcInodeInfo.Inode)
|
||||
if proto.IsDir(srcInodeInfo.Mode) {
|
||||
mw.AddInoInfoCache(srcInodeInfo.Inode, dstParentID, dstName)
|
||||
@ -2040,7 +2047,8 @@ func (mw *MetaWrapper) GetMultipart_ll(path, multipartId string) (info *proto.Mu
|
||||
}
|
||||
|
||||
func (mw *MetaWrapper) AddMultipartPart_ll(path, multipartId string, partId uint16, size uint64, md5 string,
|
||||
inodeInfo *proto.InodeInfo) (oldInode uint64, updated bool, err error) {
|
||||
inodeInfo *proto.InodeInfo,
|
||||
) (oldInode uint64, updated bool, err error) {
|
||||
var (
|
||||
mpId uint64
|
||||
found bool
|
||||
@ -2673,7 +2681,7 @@ func (mw *MetaWrapper) getCurrentPathToMountSub(parentIno uint64) (parentPath st
|
||||
log.LogDebugf("action[getCurrentPathToMountSub]: currentPath(%v) parentIno(%v)", currentPath, parentIno)
|
||||
|
||||
subs := strings.Split(currentPath, "/")
|
||||
var index = 0
|
||||
index := 0
|
||||
for i, sub := range subs {
|
||||
if sub == mw.subDir {
|
||||
index = i
|
||||
|
||||
@ -2,10 +2,11 @@ package meta
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"github.com/cubefs/cubefs/proto"
|
||||
"github.com/cubefs/cubefs/util/log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/cubefs/cubefs/proto"
|
||||
"github.com/cubefs/cubefs/util/log"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@ -92,8 +92,8 @@ type MetaConfig struct {
|
||||
OnAsyncTaskError AsyncTaskErrorFunc
|
||||
EnableSummary bool
|
||||
MetaSendTimeout int64
|
||||
//EnableTransaction uint8
|
||||
//EnableTransaction bool
|
||||
// EnableTransaction uint8
|
||||
// EnableTransaction bool
|
||||
MountPoint string
|
||||
SubDir string
|
||||
TrashTraverseLimit int
|
||||
@ -169,7 +169,7 @@ type MetaWrapper struct {
|
||||
uniqidRangeMutex sync.Mutex
|
||||
|
||||
qc *QuotaCache
|
||||
//trash
|
||||
// trash
|
||||
TrashInterval int64
|
||||
trashPolicy *Trash
|
||||
disableTrash bool
|
||||
@ -242,7 +242,6 @@ func NewMetaWrapper(config *MetaConfig) (*MetaWrapper, error) {
|
||||
mw.uniqidRangeMap = make(map[uint64]*uniqidRange, 0)
|
||||
mw.qc = NewQuotaCache(DefaultQuotaExpiration, MaxQuotaCache)
|
||||
mw.VerReadSeq = config.VerReadSeq
|
||||
mw.entryCache = make(map[uint64]inoInfoCache)
|
||||
mw.dirCache = make(map[uint64]dirInfoCache)
|
||||
mw.subDir = config.SubDir
|
||||
limit := MaxMountRetryLimit
|
||||
@ -277,6 +276,7 @@ func NewMetaWrapper(config *MetaConfig) (*MetaWrapper, error) {
|
||||
go mw.refresh()
|
||||
return mw, nil
|
||||
}
|
||||
|
||||
func (mw *MetaWrapper) enableTrash() {
|
||||
if mw.disableTrash == true {
|
||||
return
|
||||
|
||||
@ -32,7 +32,8 @@ import (
|
||||
//
|
||||
// txIcreate create inode and tx together
|
||||
func (mw *MetaWrapper) txIcreate(tx *Transaction, mp *MetaPartition, mode, uid, gid uint32,
|
||||
target []byte, quotaIds []uint32, fullPath string) (status int, info *proto.InodeInfo, err error) {
|
||||
target []byte, quotaIds []uint32, fullPath string,
|
||||
) (status int, info *proto.InodeInfo, err error) {
|
||||
bgTime := stat.BeginStat()
|
||||
defer func() {
|
||||
stat.EndStat("txIcreate", err, bgTime, 1)
|
||||
@ -104,7 +105,8 @@ func (mw *MetaWrapper) txIcreate(tx *Transaction, mp *MetaPartition, mode, uid,
|
||||
}
|
||||
|
||||
func (mw *MetaWrapper) quotaIcreate(mp *MetaPartition, mode, uid, gid uint32, target []byte, quotaIds []uint32, fullPath string) (status int,
|
||||
info *proto.InodeInfo, err error) {
|
||||
info *proto.InodeInfo, err error,
|
||||
) {
|
||||
bgTime := stat.BeginStat()
|
||||
defer func() {
|
||||
stat.EndStat("icreate", err, bgTime, 1)
|
||||
@ -164,7 +166,8 @@ func (mw *MetaWrapper) quotaIcreate(mp *MetaPartition, mode, uid, gid uint32, ta
|
||||
}
|
||||
|
||||
func (mw *MetaWrapper) icreate(mp *MetaPartition, mode, uid, gid uint32, target []byte, fullPath string) (status int,
|
||||
info *proto.InodeInfo, err error) {
|
||||
info *proto.InodeInfo, err error,
|
||||
) {
|
||||
bgTime := stat.BeginStat()
|
||||
defer func() {
|
||||
stat.EndStat("icreate", err, bgTime, 1)
|
||||
@ -251,7 +254,8 @@ func (mw *MetaWrapper) sendToMetaPartitionWithTx(mp *MetaPartition, req *proto.P
|
||||
}
|
||||
|
||||
func (mw *MetaWrapper) SendTxPack(req proto.TxPack, resp interface{}, Opcode uint8, mp *MetaPartition,
|
||||
checkStatusFunc func(int, *proto.Packet) error, ignoreExistError bool) (status int, err error, packet *proto.Packet) {
|
||||
checkStatusFunc func(int, *proto.Packet) error, ignoreExistError bool,
|
||||
) (status int, err error, packet *proto.Packet) {
|
||||
packet = proto.NewPacketReqID()
|
||||
packet.Opcode = Opcode
|
||||
packet.PartitionID = mp.PartitionID
|
||||
@ -475,7 +479,8 @@ func (mw *MetaWrapper) ievict(mp *MetaPartition, inode uint64, fullPath string)
|
||||
}
|
||||
|
||||
func (mw *MetaWrapper) txDcreate(tx *Transaction, mp *MetaPartition, parentID uint64, name string, inode uint64,
|
||||
mode uint32, quotaIds []uint32, fullPath string, ignoreExist bool) (status int, err error) {
|
||||
mode uint32, quotaIds []uint32, fullPath string, ignoreExist bool,
|
||||
) (status int, err error) {
|
||||
bgTime := stat.BeginStat()
|
||||
defer func() {
|
||||
stat.EndStat("txDcreate", err, bgTime, 1)
|
||||
@ -524,7 +529,8 @@ func (mw *MetaWrapper) txDcreate(tx *Transaction, mp *MetaPartition, parentID ui
|
||||
}
|
||||
|
||||
func (mw *MetaWrapper) quotaDcreate(mp *MetaPartition, parentID uint64, name string, inode uint64, mode uint32,
|
||||
quotaIds []uint32, fullPath string, ignoreExistError bool) (status int, err error) {
|
||||
quotaIds []uint32, fullPath string, ignoreExistError bool,
|
||||
) (status int, err error) {
|
||||
bgTime := stat.BeginStat()
|
||||
defer func() {
|
||||
stat.EndStat("dcreate", err, bgTime, 1)
|
||||
@ -577,7 +583,8 @@ func (mw *MetaWrapper) quotaDcreate(mp *MetaPartition, parentID uint64, name str
|
||||
}
|
||||
|
||||
func (mw *MetaWrapper) dcreate(mp *MetaPartition, parentID uint64, name string, inode uint64, mode uint32,
|
||||
fullPath string, ignoreExistError bool) (status int, err error) {
|
||||
fullPath string, ignoreExistError bool,
|
||||
) (status int, err error) {
|
||||
bgTime := stat.BeginStat()
|
||||
defer func() {
|
||||
stat.EndStat("dcreate", err, bgTime, 1)
|
||||
@ -896,7 +903,8 @@ func (mw *MetaWrapper) canDeleteInode(mp *MetaPartition, info *proto.InodeInfo,
|
||||
}
|
||||
|
||||
func (mw *MetaWrapper) ddeletes(mp *MetaPartition, parentID uint64, dentries []proto.Dentry, fullPaths []string) (status int,
|
||||
resp *proto.BatchDeleteDentryResponse, err error) {
|
||||
resp *proto.BatchDeleteDentryResponse, err error,
|
||||
) {
|
||||
bgTime := stat.BeginStat()
|
||||
defer func() {
|
||||
stat.EndStat("ddeletes", err, bgTime, 1)
|
||||
@ -2489,7 +2497,8 @@ func (mw *MetaWrapper) updateXAttrs(mp *MetaPartition, inode uint64, filesInc in
|
||||
}
|
||||
|
||||
func (mw *MetaWrapper) batchSetInodeQuota(mp *MetaPartition, inodes []uint64, quotaId uint32,
|
||||
IsRoot bool) (resp *proto.BatchSetMetaserverQuotaResponse, err error) {
|
||||
IsRoot bool,
|
||||
) (resp *proto.BatchSetMetaserverQuotaResponse, err error) {
|
||||
bgTime := stat.BeginStat()
|
||||
defer func() {
|
||||
stat.EndStat("batchSetInodeQuota", err, bgTime, 1)
|
||||
@ -2538,7 +2547,8 @@ func (mw *MetaWrapper) batchSetInodeQuota(mp *MetaPartition, inodes []uint64, qu
|
||||
}
|
||||
|
||||
func (mw *MetaWrapper) batchDeleteInodeQuota(mp *MetaPartition, inodes []uint64,
|
||||
quotaId uint32) (resp *proto.BatchDeleteMetaserverQuotaResponse, err error) {
|
||||
quotaId uint32,
|
||||
) (resp *proto.BatchDeleteMetaserverQuotaResponse, err error) {
|
||||
bgTime := stat.BeginStat()
|
||||
defer func() {
|
||||
stat.EndStat("batchDeleteInodeQuota", err, bgTime, 1)
|
||||
@ -2643,7 +2653,8 @@ func (mw *MetaWrapper) getInodeQuota(mp *MetaPartition, inode uint64) (quotaInfo
|
||||
}
|
||||
|
||||
func (mw *MetaWrapper) applyQuota(parentIno uint64, quotaId uint32, totalInodeCount *uint64, curInodeCount *uint64, inodes *[]uint64,
|
||||
maxInodes uint64, first bool) (err error) {
|
||||
maxInodes uint64, first bool,
|
||||
) (err error) {
|
||||
if first {
|
||||
var rootInodes []uint64
|
||||
var ret map[uint64]uint8
|
||||
@ -2714,7 +2725,8 @@ func (mw *MetaWrapper) applyQuota(parentIno uint64, quotaId uint32, totalInodeCo
|
||||
}
|
||||
|
||||
func (mw *MetaWrapper) revokeQuota(parentIno uint64, quotaId uint32, totalInodeCount *uint64, curInodeCount *uint64, inodes *[]uint64,
|
||||
maxInodes uint64, first bool) (err error) {
|
||||
maxInodes uint64, first bool,
|
||||
) (err error) {
|
||||
if first {
|
||||
var rootInodes []uint64
|
||||
rootInodes = append(rootInodes, parentIno)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user