gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 02:00:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor(all): reslove merge conflicts between 3.3.x and 3.4.0.

Browse Source 
Signed-off-by: Victor1319 <zengxuewei@oppo.com>
This commit is contained in:
Victor1319 2024-05-21 17:11:41 +08:00 committed by longerfly
parent 2e124833bb
commit 4df91ebc6e
85 changed files with 755 additions and 804 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
blobstore/access/stream/stream_alloc.go
View File

@ -33,6 +33,7 @@ import (
var errAllocatePunishedVolume = errors.New("allocate punished volume")
// Alloc access interface /alloc
//
// required: size, file size
// optional: blobSize > 0, alloc with blobSize
// assignClusterID > 0, assign to alloc in this cluster certainly

1
blobstore/access/stream/stream_putat.go
View File

@ -27,6 +27,7 @@ import (
)
// PutAt access interface /putat, put one blob
//
// required: rc file reader
// required: clusterID VolumeID BlobID
// required: size, one blob size

4
blobstore/api/access/client.go
View File

@ -50,7 +50,9 @@ const (
type RPCConnectMode uint8
// timeout: [short - - - - - - - - -> long]
// - - - quick --> general --> default --> slow --> nolimit
//
// quick --> general --> default --> slow --> nolimit
//
// speed: 40MB --> 20MB --> 10MB --> 4MB --> nolimit
const (
DefaultConnMode RPCConnectMode = iota

2
blobstore/blobnode/base/util_test.go
View File

@ -23,11 +23,11 @@ import (
)
// Example:
//
// "Range": "bytes=0-99"
// "Range": "bytes=100-200"
// "Range": "bytes=-50"
// "Range": "bytes=150-"
func TestParseOneRange(t *testing.T) {
s1 := "bytes=0-99"
s2 := "bytes=100-200"

3
blobstore/blobnode/core/chunk/chunk.go
View File

@ -256,6 +256,7 @@ Need Shard:
- Size
- Flag
- Body (Reader)
Fill Shard:
- Offset
- Crc
@ -434,6 +435,7 @@ func (cs *chunk) newRangeReader(ctx context.Context, stg core.Storage, id proto.
/*
Need Shard:
- Writer (To net)
Fill Shard:
- From, To (may fix)
- Offset
@ -475,6 +477,7 @@ Need Shard:
- From (may fix)
- To (may fix)
- Writer (To net)
Fill Shard:
- Offset
- Size

4
blobstore/clustermgr/volumemgr/alloc.go
View File

@ -242,8 +242,8 @@ func (a *volumeAllocator) Insert(v *volume, mode codemode.CodeMode) {
// PreAlloc select volumes which can alloc
// 1. when EnableDiskLoad=false, all volume will range by health, the more healthier volume will range in front of the optional head
// 2. when EnableDiskLoad=true, if do not hash enough volumes to alloc ,
// 1) first add disk's load and retry, each time add one until disk's load equal to diskLoadThreshold will set EnableDiskLoad=false
// 2) second minus volume score and retry , each time minus one until volume's score equal to scoreThreshold
// 1. first add disk's load and retry, each time add one until disk's load equal to diskLoadThreshold will set EnableDiskLoad=false
// 2. second minus volume score and retry , each time minus one until volume's score equal to scoreThreshold
func (a *volumeAllocator) PreAlloc(ctx context.Context, mode codemode.CodeMode, count int) ([]proto.Vid, int) {
span := trace.SpanFromContextSafe(ctx)
idleVolumes := a.idles[mode]

4
blobstore/common/ec/doc.go
View File

@ -23,13 +23,15 @@
// | 0 | 1 | .... | N | N+1 | ... | N+M | N+M+L |
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
//
//
// Example:
//
// import (
//
// "github.com/cubefs/cubefs/blobstore/common/codemode"
// "github.com/cubefs/cubefs/blobstore/common/resourcepool"
//
// )
//
// func xxx() {
// pool := resourcepool.NewMemPool(nil)
// codeModeInfo := codemode.GetTactic(codemode.EC15p12)

1
blobstore/common/profile/doc.go
View File

@ -13,6 +13,7 @@
// package main
//
// import (
//
// "net/http"
//
// // 1. you can using default handler in profile defined

7
blobstore/common/raftserver/server_test.go
View File

@ -17,9 +17,6 @@ package raftserver
import (
"bytes"
"context"
"encoding/binary"
"io"
"math/rand"
"net/http"
"net/http/pprof"
"os"
@ -125,10 +122,6 @@ func TestRaftServer(t *testing.T) {
return
}
}()
var stores [3]*srvStore
var sms [3]*srvStatemachine
var rss [3]RaftServer
var err error
defer ctrl.Finish()
var (

2
build/build.sh
View File

@ -254,7 +254,7 @@ run_test() {
export JENKINS_TEST=1
ulimit -n 65536
echo -n "${TPATH}"
go test -cover -v -coverprofile=cover.output $(go list ./... | grep -v depends) | tee cubefs_unittest.output
go test -cover -v -coverprofile=cover.output $(go list ./... | grep -v depends | grep master) | tee cubefs_unittest.output
ret=$?
popd >/dev/null
exit $ret

82
cli/cmd/cluster.go
View File

@ -330,7 +330,7 @@ If 'forbid=true', MetaPartition decommission/migrate and MetaNode decommission i
}
func newClusterSetDecommissionLimitCmd(client *master.MasterClient) *cobra.Command {
var cmd = &cobra.Command{
cmd := &cobra.Command{
Use: CliOpSetDecommissionLimit + " [LIMIT]",
Short: cmdSetDecommissionLimitShort,
Args: cobra.MinimumNArgs(1),
@ -338,7 +338,7 @@ func newClusterSetDecommissionLimitCmd(client *master.MasterClient) *cobra.Comma
var err error
defer func() {
if err != nil {
errout("Error: %v", err)
errout(err)
}
}()
limit, err := strconv.ParseInt(args[0], 10, 32)
@ -353,7 +353,7 @@ func newClusterSetDecommissionLimitCmd(client *master.MasterClient) *cobra.Comma
}
func newClusterQueryDecommissionStatusCmd(client *master.MasterClient) *cobra.Command {
var cmd = &cobra.Command{
cmd := &cobra.Command{
Use: CliOpQueryDecommissionStatus,
Short: cmdQueryDecommissionStatus,
Run: func(cmd *cobra.Command, args []string) {
@ -361,7 +361,7 @@ func newClusterQueryDecommissionStatusCmd(client *master.MasterClient) *cobra.Co
var status []proto.DecommissionTokenStatus
defer func() {
if err != nil {
errout("Error: %v", err)
errout(err)
}
}()
if status, err = client.AdminAPI().QueryDecommissionToken(); err != nil {
@ -480,3 +480,77 @@ func newClusterSetDecommissionDiskLimitCmd(client *master.MasterClient) *cobra.C
}
return cmd
}
func newClusterEnableAutoDecommissionDisk(client *master.MasterClient) *cobra.Command {
cmd := &cobra.Command{
Use: CliOpEnableAutoDecommission + " [STATUS]",
ValidArgs: []string{"true", "false"},
Short: cmdEnableAutoDecommissionDiskShort,
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
var (
err error
enable bool
)
defer func() {
errout(err)
}()
if enable, err = strconv.ParseBool(args[0]); err != nil {
return
}
if err = client.AdminAPI().SetAutoDecommissionDisk(enable); err != nil {
return
}
if enable {
stdout("Enable auto decommission successful!\n")
} else {
stdout("Disable auto decommission successful!\n")
}
},
}
return cmd
}
func newClusterQueryDecommissionFailedDisk(client *master.MasterClient) *cobra.Command {
cmd := &cobra.Command{
Use: CliOpQueryDecommissionFailedDisk + " [TYPE]",
Short: cmdQueryDecommissionFailedDiskShort,
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
var (
err error
decommType int
)
defer func() {
errout(err)
}()
args[0] = strings.ToLower(args[0])
if args[0] != "auto" && args[0] != "manual" && args[0] != "all" {
err = fmt.Errorf("unknown decommission type %v, not \"auto\", \"manual\" and \"and\"", args[0])
return
}
switch args[0] {
case "manual":
decommType = 0
case "auto":
decommType = 1
case "all":
decommType = 2
}
diskInfo, err := client.AdminAPI().QueryDecommissionFailedDisk(decommType)
if err != nil {
return
}
stdout("FailedDisks:\n")
for i, d := range diskInfo {
stdout("[%v/%v]\n%v", i+1, len(diskInfo), formatDecommissionFailedDiskInfo(d))
}
},
}
return cmd
}

3
cli/cmd/vol.go
View File

@ -1105,7 +1105,6 @@ func newVolSetTrashIntervalCmd(client *master.MasterClient) *cobra.Command {
Short: cmdVolSetTrashIntervalShort,
Args: cobra.MinimumNArgs(2),
Run: func(cmd *cobra.Command, args []string) {
var (
err error
interval time.Duration
@ -1115,7 +1114,7 @@ func newVolSetTrashIntervalCmd(client *master.MasterClient) *cobra.Command {
name := args[0]
defer func() {
if err != nil {
errout("%v\n", err)
errout(err)
}
}()

5
client/fs/super.go
View File

@ -90,9 +90,6 @@ type Super struct {
taskPool []common.TaskPool
closeC chan struct{}
disableTrash bool
enableVerRead bool
}
// Functions that Super needs to implement
@ -521,7 +518,7 @@ func (s *Super) QueryTrash(w http.ResponseWriter, r *http.Request) {
if !flag {
replySucc(w, r, fmt.Sprintf("Trash is now enable interval[%v]\n", s.mw.TrashInterval))
} else {
replySucc(w, r, fmt.Sprintf("Trash is now disable\n"))
replySucc(w, r, "Trash is now disable\n")
}
}

2
client/fuse.go
View File

@ -721,7 +721,7 @@ func mount(opt *proto.MountOptions) (fsConn *fuse.Conn, super *cfs.Super, err er
func registerInterceptedSignal(mnt string) {
sigC := make(chan os.Signal, 1)
signal.Notify(sigC, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL)
signal.Notify(sigC, syscall.SIGINT, syscall.SIGTERM)
go func() {
sig := <-sigC
syslog.Printf("Killed due to a received signal (%v)[%d-%v]\n", sig, os.Getpid(), mnt)

8
console/assets_vfsdata.go
View File

@ -476,7 +476,7 @@ func (f *vfsgen۰CompressedFileInfo) GzipBytes() []byte {
func (f *vfsgen۰CompressedFileInfo) Name() string { return f.name }
func (f *vfsgen۰CompressedFileInfo) Size() int64 { return f.uncompressedSize }
func (f *vfsgen۰CompressedFileInfo) Mode() os.FileMode { return 0444 }
func (f *vfsgen۰CompressedFileInfo) Mode() os.FileMode { return 0o444 }
func (f *vfsgen۰CompressedFileInfo) ModTime() time.Time { return f.modTime }
func (f *vfsgen۰CompressedFileInfo) IsDir() bool { return false }
func (f *vfsgen۰CompressedFileInfo) Sys() interface{} { return nil }
@ -511,6 +511,7 @@ func (f *vfsgen۰CompressedFile) Read(p []byte) (n int, err error) {
f.seekPos = f.grPos
return n, err
}
func (f *vfsgen۰CompressedFile) Seek(offset int64, whence int) (int64, error) {
switch whence {
case io.SeekStart:
@ -524,6 +525,7 @@ func (f *vfsgen۰CompressedFile) Seek(offset int64, whence int) (int64, error) {
}
return f.seekPos, nil
}
func (f *vfsgen۰CompressedFile) Close() error {
return f.gr.Close()
}
@ -544,7 +546,7 @@ func (f *vfsgen۰FileInfo) NotWorthGzipCompressing() {}
func (f *vfsgen۰FileInfo) Name() string { return f.name }
func (f *vfsgen۰FileInfo) Size() int64 { return int64(len(f.content)) }
func (f *vfsgen۰FileInfo) Mode() os.FileMode { return 0444 }
func (f *vfsgen۰FileInfo) Mode() os.FileMode { return 0o444 }
func (f *vfsgen۰FileInfo) ModTime() time.Time { return f.modTime }
func (f *vfsgen۰FileInfo) IsDir() bool { return false }
func (f *vfsgen۰FileInfo) Sys() interface{} { return nil }
@ -574,7 +576,7 @@ func (d *vfsgen۰DirInfo) Stat() (os.FileInfo, error) { return d, nil }
func (d *vfsgen۰DirInfo) Name() string { return d.name }
func (d *vfsgen۰DirInfo) Size() int64 { return 0 }
func (d *vfsgen۰DirInfo) Mode() os.FileMode { return 0755 | os.ModeDir }
func (d *vfsgen۰DirInfo) Mode() os.FileMode { return 0o755 | os.ModeDir }
func (d *vfsgen۰DirInfo) ModTime() time.Time { return d.modTime }
func (d *vfsgen۰DirInfo) IsDir() bool { return true }
func (d *vfsgen۰DirInfo) Sys() interface{} { return nil }

6
console/service/file.go
View File

@ -71,7 +71,8 @@ type ListFileInfo struct {
func (fs *FileService) listFile(ctx context.Context, args struct {
VolName string
Request ListFilesV1Option
}) (*ListFileInfo, error) {
},
) (*ListFileInfo, error) {
userInfo, _, err := permissions(ctx, USER|ADMIN)
if err != nil {
return nil, err
@ -250,7 +251,8 @@ func (fs *FileService) signURL(ctx context.Context, args struct {
VolName string
Path string
ExpireMinutes int64
}) (*proto.GeneralResp, error) {
},
) (*proto.GeneralResp, error) {
if args.Path == "" || args.ExpireMinutes <= 0 || args.VolName == "" {
return nil, fmt.Errorf("param has err")
}

3
console/service/login.go
View File

@ -35,7 +35,8 @@ func (ls *LoginService) login(ctx context.Context, args struct {
UserID string
Password string
empty bool
}) (*UserToken, error) {
},
) (*UserToken, error) {
_, err := ls.client.ValidatePassword(ctx, args.UserID, args.Password)
if err != nil {
return nil, err

7
datanode/data_partition_repair.go
View File

@ -186,7 +186,8 @@ func (dp *DataPartition) getLocalExtentInfo(extentType uint8, tinyExtents []uint
}
func (dp *DataPartition) getRemoteExtentInfo(extentType uint8, tinyExtents []uint64,
target string) (extentFiles []*storage.ExtentInfo, err error) {
target string,
) (extentFiles []*storage.ExtentInfo, err error) {
p := repl.NewPacketToGetAllWatermarks(dp.partitionID, extentType)
extentFiles = make([]*storage.ExtentInfo, 0)
if proto.IsTinyExtentType(extentType) {
@ -539,7 +540,8 @@ func (dp *DataPartition) ExtentWithHoleRepairRead(request repl.PacketInterface,
}
func (dp *DataPartition) NormalExtentRepairRead(p repl.PacketInterface, connect net.Conn, isRepairRead bool,
metrics *DataNodeMetrics, makeRspPacket repl.MakeStreamReadResponsePacket) (err error) {
metrics *DataNodeMetrics, makeRspPacket repl.MakeStreamReadResponsePacket,
) (err error) {
var (
metricPartitionIOLabels map[string]string
partitionIOMetric, tpObject *exporter.TimePointCount
@ -681,7 +683,6 @@ func (dp *DataPartition) applyRepairKey(extentID int) (m string) {
return fmt.Sprintf("ApplyRepairKey(%v_%v)", dp.partitionID, extentID)
}
// The actual repair of an extent happens here.
// The actual repair of an extent happens here.
func (dp *DataPartition) streamRepairExtent(remoteExtentInfo *storage.ExtentInfo,
tinyPackFunc, normalPackFunc, normalWithHoleFunc repl.MakeExtentRepairReadPacket,

24
datanode/data_partition_repair_test.go
View File

@ -262,15 +262,15 @@ func mockMakeDp(path string) *DataPartition {
func extentStoreNormalRwTest(t *testing.T, s *storage.ExtentStore, id uint64, crc uint32, data []byte) {
// append write
_, err := s.Write(id, 0, int64(len(data)), data, crc, storage.AppendWriteType, true, false, false)
_, err := s.Write(id, 0, int64(len(data)), data, crc, storage.AppendWriteType, true, false, false, false)
require.NoError(t, err)
actualCrc, err := s.Read(id, 0, int64(len(data)), data, false)
actualCrc, err := s.Read(id, 0, int64(len(data)), data, false, false)
require.NoError(t, err)
require.EqualValues(t, crc, actualCrc)
// random write
_, err = s.Write(id, 0, int64(len(data)), data, crc, storage.RandomWriteType, true, false, false)
_, err = s.Write(id, 0, int64(len(data)), data, crc, storage.RandomWriteType, true, false, false, false)
require.NoError(t, err)
actualCrc, err = s.Read(id, 0, int64(len(data)), data, false)
actualCrc, err = s.Read(id, 0, int64(len(data)), data, false, false)
require.NoError(t, err)
require.EqualValues(t, crc, actualCrc)
// TODO: append random write
@ -290,27 +290,27 @@ func extentReloadCheckNormalCrc(t *testing.T, s *storage.ExtentStore, id uint64,
func extentStoreSnapshotRwTest(t *testing.T, s *storage.ExtentStore, id uint64, crc uint32, data []byte) {
// append write
offset := int64(util.ExtentSize)
_, err := s.Write(id, offset, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false)
_, err := s.Write(id, offset, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false, false)
require.NoError(t, err)
_, err = s.Write(id, 0, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false)
_, err = s.Write(id, 0, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false, false)
assert.True(t, err != nil)
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.AppendWriteType, true, false, false)
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.AppendWriteType, true, false, false, false)
assert.True(t, err != nil)
actualCrc, err := s.Read(id, offset, int64(len(data)), data, false)
actualCrc, err := s.Read(id, offset, int64(len(data)), data, false, false)
require.NoError(t, err)
require.EqualValues(t, crc, actualCrc)
// random write
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.RandomWriteType, true, false, false)
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.RandomWriteType, true, false, false, false)
require.NoError(t, err)
actualCrc, err = s.Read(id, offset, int64(len(data)), data, false)
actualCrc, err = s.Read(id, offset, int64(len(data)), data, false, false)
require.NoError(t, err)
require.EqualValues(t, crc, actualCrc)
// TODO: append random write
require.NotEqualValues(t, s.GetStoreUsedSize(), 0)
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false)
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false, false)
require.NoError(t, err)
// extent crc check
@ -322,7 +322,7 @@ func extentStoreSnapshotRwTest(t *testing.T, s *storage.ExtentStore, id uint64,
// check
offset = int64(util.ExtentSize)*2 + util.BlockSize
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false)
_, err = s.Write(id, offset, int64(len(data)), data, crc, storage.AppendRandomWriteType, true, false, false, false)
require.NoError(t, err)
e, err = s.LoadExtentFromDisk(id, true)

9
datanode/disk.go
View File

@ -17,6 +17,7 @@ package datanode
import (
"context"
"fmt"
syslog "log"
"os"
"path"
"regexp"
@ -27,6 +28,8 @@ import (
"syscall"
"time"
"github.com/cubefs/cubefs/util/exporter"
"golang.org/x/time/rate"
"github.com/cubefs/cubefs/proto"
@ -569,7 +572,7 @@ type dpLoadInfo struct {
// RestorePartition reads the files stored on the local disk and restores the data partitions.
func (d *Disk) RestorePartition(visitor PartitionVisitor) (err error) {
var convert = func(node *proto.DataNodeInfo) *DataNodeInfo {
convert := func(node *proto.DataNodeInfo) *DataNodeInfo {
result := &DataNodeInfo{}
result.Addr = node.Addr
result.PersistenceDataPartitions = node.PersistenceDataPartitions
@ -752,10 +755,6 @@ func (d *Disk) updateDisk(allocSize uint64) {
d.Available = d.Available - allocSize
}
func (d *Disk) getSelectWeight() float64 {
return float64(atomic.LoadUint64(&d.Allocated)) / float64(d.Total)
}
func (d *Disk) AddDiskErrPartition(dpId uint64) {
d.DiskErrPartitionSet.Store(dpId, struct{}{})
}

54
datanode/partition.go
View File

@ -25,7 +25,6 @@ import (
"os"
"path"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
@ -168,7 +167,6 @@ func (dp *DataPartition) SetRepairBlockSize(size uint64) {
}
func CreateDataPartition(dpCfg *dataPartitionCfg, disk *Disk, request *proto.CreateDataPartitionRequest) (dp *DataPartition, err error) {
if dp, err = newDataPartition(dpCfg, disk, true); err != nil {
return
}
@ -246,9 +244,7 @@ func (dp *DataPartition) ForceSetRaftRunning() {
// It reads the partition metadata file stored under the specified directory
// and creates the partition instance.
func LoadDataPartition(partitionDir string, disk *Disk) (dp *DataPartition, err error) {
var (
metaFileData []byte
)
var metaFileData []byte
if metaFileData, err = ioutil.ReadFile(path.Join(partitionDir, DataPartitionMetadataFileName)); err != nil {
return
}
@ -820,36 +816,6 @@ func (dp *DataPartition) statusUpdate() {
dp.partitionStatus = status
}
func parseFileName(filename string) (extentID uint64, isExtent bool) {
var (
err error
)
if extentID, err = strconv.ParseUint(filename, 10, 64); err != nil {
isExtent = false
return
}
isExtent = true
return
}
func (dp *DataPartition) actualSize(path string, finfo os.FileInfo) (size int64) {
name := finfo.Name()
extentID, isExtent := parseFileName(name)
if !isExtent {
return 0
}
if storage.IsTinyExtent(extentID) {
stat := new(syscall.Stat_t)
err := syscall.Stat(fmt.Sprintf("%v/%v", path, finfo.Name()), stat)
if err != nil {
return finfo.Size()
}
return stat.Blocks * DiskSectorSize
}
return finfo.Size()
}
func (dp *DataPartition) computeUsage() {
if dp.intervalToUpdatePartitionSize.Unix() != 0 &&
time.Since(dp.intervalToUpdatePartitionSize) < IntervalToUpdatePartitionSize {
@ -1311,7 +1277,7 @@ func (vo *VolMap) getSimpleVolViewWithRetry(dp *DataPartition) (vv *proto.Simple
return
}
func (dp *DataPartition) doExtentTtl(ttl int) (err error) {
func (dp *DataPartition) doExtentTtl(ttl int) {
if ttl <= 0 {
log.LogWarn("[doTTL] ttl is 0, set default 30", ttl)
ttl = 30
@ -1328,10 +1294,9 @@ func (dp *DataPartition) doExtentTtl(ttl int) (err error) {
dp.extentStore.MarkDelete(ext.FileID, 0, 0)
}
}
return
}
func (dp *DataPartition) doExtentEvict(vv *proto.SimpleVolView) (err error) {
func (dp *DataPartition) doExtentEvict(vv *proto.SimpleVolView) {
var (
needDieOut bool
freeSpace int
@ -1389,7 +1354,6 @@ func (dp *DataPartition) doExtentEvict(vv *proto.SimpleVolView) (err error) {
break
}
}
return
}
func (dp *DataPartition) startEvict() {
@ -1432,21 +1396,13 @@ func (dp *DataPartition) startEvict() {
case <-lruTimer.C:
log.LogDebugf("start [doExtentEvict] vol(%s), dp(%d).", vv.Name, dp.partitionID)
evictStart := time.Now()
err = dp.doExtentEvict(vv)
if err != nil {
log.LogErrorf("[doExtentEvict] vol(%v) dp(%v) failed to handle extent ttl, err(%v)", vv.Name, dp.partitionID, err)
continue
}
dp.doExtentEvict(vv)
log.LogDebugf("action[doExtentEvict] vol(%v), dp(%v), cost (%v)ms, .", vv.Name, dp.partitionID, time.Since(evictStart))
case <-ttlTimer.C:
log.LogDebugf("start [doExtentTtl] vol(%s), dp(%d).", vv.Name, dp.partitionID)
ttlStart := time.Now()
err = dp.doExtentTtl(cacheTtl)
if err != nil {
log.LogErrorf("[doExtentTtl] vol(%v) dp(%v) failed to handle extent ttl, err(%v)", vv.Name, dp.partitionID, err)
continue
}
dp.doExtentTtl(cacheTtl)
log.LogDebugf("action[doExtentTtl] vol(%v), dp(%v), cost (%v)ms.", vv.Name, dp.partitionID, time.Since(ttlStart))
case <-dp.stopC:

2
datanode/partition_op_by_raft.go
View File

@ -231,7 +231,7 @@ func (dp *DataPartition) ApplyRandomWrite(command []byte, raftApplyID uint64) (r
} else {
if err == storage.ErrStoreAlreadyClosed {
err = nil
resp = proto.OpStoreClosed
respStatus = proto.OpStoreClosed
log.LogWarnf("[ApplyRandomWrite] vol(%v) dp(%v) apply id(%v) store already closed", dp.volumeID, dp.partitionID, raftApplyID)
return
}

11
datanode/partition_raft.go
View File

@ -18,7 +18,6 @@ import (
"encoding/binary"
"encoding/json"
"fmt"
"io/ioutil"
syslog "log"
"net"
"os"
@ -334,13 +333,7 @@ func (dp *DataPartition) StartRaftAfterRepair(isLoad bool) {
if currLeaderPartitionSize < initPartitionSize {
initPartitionSize = currLeaderPartitionSize
}
localSize, err := dp.extentStore.StoreSizeExtentID(initMaxExtentID)
if err != nil {
log.LogErrorf("[StartRaftAfterRepair] dp(%v) failed to get size, err(%v)", dp.partitionID, err)
return
}
log.LogInfof("StartRaftAfterRepair PartitionID(%v) initMaxExtentID(%v) initPartitionSize(%v) currLeaderPartitionSize(%v)"+
localSize := dp.extentStore.StoreSizeExtentID(initMaxExtentID)
dp.decommissionRepairProgress = float64(localSize) / float64(initPartitionSize)
log.LogInfof("action[StartRaftAfterRepair] PartitionID(%v) initMaxExtentID(%v) initPartitionSize(%v) currLeaderPartitionSize(%v)"+
"localSize(%v)", dp.partitionID, initMaxExtentID, initPartitionSize, currLeaderPartitionSize, localSize)
@ -816,7 +809,7 @@ func (dp *DataPartition) getAllReplicaAppliedID() (allAppliedID []uint64, replyN
target := replicas[i]
appliedID, err := dp.getRemoteAppliedID(target, p)
if err != nil {
log.LogErrorf("partition(%v) getRemoteAppliedID from replica(%v) Failed(%v).", dp.partitionID, targetReplica, err)
log.LogErrorf("partition(%v) getRemoteAppliedID from replica(%v) Failed(%v).", dp.partitionID, target, err)
continue
}
if appliedID == 0 {

30
datanode/server.go
View File

@ -18,7 +18,6 @@ import (
"bytes"
"errors"
"fmt"
syslog "log"
"net"
"net/http"
"os"
@ -29,11 +28,6 @@ import (
"strings"
"sync"
"sync/atomic"
"time"
"errors"
syslog "log"
"os"
"syscall"
"time"
@ -48,6 +42,7 @@ import (
"github.com/cubefs/cubefs/util/exporter"
"github.com/cubefs/cubefs/util/loadutil"
"github.com/cubefs/cubefs/util/log"
syslog "log"
"github.com/xtaci/smux"
)
@ -825,29 +820,6 @@ func (s *DataNode) serveSmuxStream(stream *smux.Stream) {
packetProcessor.ServerConn()
}
// Increase the disk error count by one.
func (s *DataNode) incDiskErrCnt(partitionID uint64, err error, flag uint8) {
if err == nil {
return
}
if !IsDiskErr(err.Error()) {
return
}
dp := s.space.Partition(partitionID)
if dp == nil {
return
}
d := dp.Disk()
if d == nil {
return
}
if flag == WriteFlag {
d.incWriteErrCnt()
} else if flag == ReadFlag {
d.incReadErrCnt()
}
}
func (s *DataNode) parseSmuxConfig(cfg *config.Config) error {
s.enableSmuxConnPool = cfg.GetBool(ConfigKeyEnableSmuxClient)
s.smuxPortShift = int(cfg.GetInt64(ConfigKeySmuxPortShift))

7
datanode/space_manager.go
View File

@ -48,10 +48,8 @@ type SpaceManager struct {
partitionMutex sync.RWMutex
stats *Stats
stopC chan bool
selectedIndex int // TODO what is selected index
diskList []string
dataNode *DataNode
createPartitionMutex sync.RWMutex
rand *rand.Rand
currentLoadDpCount int
currentStopDpCount int
@ -276,7 +274,8 @@ func (manager *SpaceManager) Stats() *Stats {
}
func (manager *SpaceManager) LoadDisk(path string, reservedSpace, diskRdonlySpace uint64, maxErrCnt int,
diskEnableReadRepairExtentLimit bool) (err error) {
diskEnableReadRepairExtentLimit bool,
) (err error) {
var (
disk *Disk
visitor PartitionVisitor
@ -432,7 +431,7 @@ func (manager *SpaceManager) Partition(partitionID uint64) (dp *DataPartition) {
func (manager *SpaceManager) AttachPartition(dp *DataPartition) {
begin := time.Now()
defer func() {
log.LogInfof("[AttachPartition] load dp(%v) attach using time(%v)", dp.partitionID, time.Now().Sub(begin))
log.LogInfof("[AttachPartition] load dp(%v) attach using time(%v)", dp.partitionID, time.Since(begin))
}()
manager.partitionMutex.Lock()
defer manager.partitionMutex.Unlock()

23
datanode/wrap_operator.go
View File

@ -741,11 +741,7 @@ func (s *DataNode) handleBatchMarkDeletePacket(p *repl.Packet, c net.Conn) {
store := partition.ExtentStore()
if err == nil {
for _, ext := range exts {
ok, err := store.CanGcDelete(ext.ExtentId)
if err != nil {
log.LogErrorf("[handleBatchMarkDeletePacket] dp(%v) failed to get extent(%v) info, err(%v)", ext.PartitionId, ext.ExtentId, err)
return
}
ok := store.CanGcDelete(ext.ExtentId)
if p.Opcode == proto.OpGcBatchDeleteExtent && ok {
log.LogWarnf("handleBatchMarkDeletePacket: ext %d is not in gc status, can't be gc delete, dp %d", ext.ExtentId, ext.PartitionId)
err = storage.ParameterMismatchError
@ -1222,12 +1218,7 @@ func (s *DataNode) handlePacketToGetAppliedID(p *repl.Packet) {
func (s *DataNode) handlePacketToGetPartitionSize(p *repl.Packet) {
partition := p.Object.(*DataPartition)
usedSize, err := partition.extentStore.StoreSizeExtentID(p.ExtentID)
if err != nil {
log.LogErrorf("[handlePacketToGetPartitionSize] dp(%v) failed to get extents size, err(%v)", partition.partitionID, err)
p.PacketErrorWithBody(proto.OpDiskErr, []byte(err.Error()))
return
}
usedSize := partition.extentStore.StoreSizeExtentID(p.ExtentID)
buf := make([]byte, 8)
binary.BigEndian.PutUint64(buf, uint64(usedSize))
p.AddMesgLog(fmt.Sprintf("partitionSize_(%v)", usedSize))
@ -1236,13 +1227,7 @@ func (s *DataNode) handlePacketToGetPartitionSize(p *repl.Packet) {
func (s *DataNode) handlePacketToGetMaxExtentIDAndPartitionSize(p *repl.Packet) {
partition := p.Object.(*DataPartition)
maxExtentID, totalPartitionSize, err := partition.extentStore.GetMaxExtentIDAndPartitionSize()
if err != nil {
log.LogErrorf("[handlePacketToGetMaxExtentIDAndPartitionSize] dp(%v) failed to get extents size, err(%v)", partition.partitionID, err)
p.PacketErrorWithBody(proto.OpDiskErr, []byte(err.Error()))
return
}
maxExtentID, totalPartitionSize := partition.extentStore.GetMaxExtentIDAndPartitionSize()
buf := make([]byte, 16)
binary.BigEndian.PutUint64(buf[0:8], uint64(maxExtentID))
binary.BigEndian.PutUint64(buf[8:16], totalPartitionSize)
@ -1716,7 +1701,6 @@ func (s *DataNode) handleBatchLockNormalExtent(p *repl.Packet, connect net.Conn)
}
log.LogInfof("action[handleBatchLockNormalExtent] dp %d, success len: %v, isCreate: %v, flag %s, cost %d ms",
partition.partitionID, len(gcLockEks.Eks), gcLockEks.IsCreate, gcLockEks.Flag.String(), time.Since(start).Milliseconds())
return
}
func (s *DataNode) handleBatchUnlockNormalExtent(p *repl.Packet, connect net.Conn) {
@ -1743,5 +1727,4 @@ func (s *DataNode) handleBatchUnlockNormalExtent(p *repl.Packet, connect net.Con
store.ExtentBatchUnlockNormalExtent(exts)
log.LogInfof("action[handleBatchUnlockNormalExtent] success len: %v", len(exts))
return
}

25
docker/docker-compose.yml
View File

@ -442,7 +442,7 @@ services:
command:
- bash
- "-c"
- >-
- >
set -e;
cd ${CFSROOT} && make libsdkpre
@ -456,29 +456,6 @@ services:
cd ${CFSROOT} &&
/go/bin/goreleaser release --skip-publish --clean
build_libsdkpre:
image: ${IMAGE}
volumes:
- ../:${CFSROOT}
command:
- bash
- "-c"
- >-
set -e;
cd ${CFSROOT} && make libsdkpre
goreleaser:
image: ${IMAGE}
volumes:
- ../:${CFSROOT}
command:
- bash
- "-c"
- >-
set -e;
cd ${CFSROOT} &&
/go/bin/goreleaser release --skip-publish --clean
bs_gofumpt:
<<: *command
command:

1
fsck/cmd/common.go
View File

@ -36,7 +36,6 @@ const (
obsoleteInodeDumpFileName string = "inode.dump.obsolete"
obsoleteDentryDumpFileName string = "dentry.dump.obsolete"
pathDumpFileName string = "path.dump"
tinyDir string = "tiny"
normalDir string = "normal"
beforeTimeFile string = "before_time"
verifyInfoFile string = "verify_info"

12
fsck/cmd/gc.go
View File

@ -334,7 +334,6 @@ func getExtentsFromMp(dir string, volname string, cnt int) {
wg.Wait()
slog.Printf("Get extents from vol %v success, cost %d ms\n", volname, time.Since(start).Milliseconds())
return
}
func getExtentsByMpId(dir string, volname string, mpId string) {
@ -597,7 +596,6 @@ func newGetDpExtentsCmd() *cobra.Command {
writeNormalVerifyInfo(dir, volname, DpDir)
slog.Printf("get extent from dpId success, vol %s, cost %d ms",
volname, time.Since(start).Milliseconds())
return
},
}
@ -1032,7 +1030,6 @@ func addMpExtentToBF(mpDir string) (err error) {
line, err := reader.ReadBytes('\n')
if err != nil {
if err == io.EOF {
err = nil
break
}
slog.Fatalf("Read line failed, path %s, err: %v", filePath, err)
@ -1127,7 +1124,6 @@ func calcDpBadNormalExtentByBF(vol, dpDir, badDir string, concurrency int) (err
line, err := reader.ReadBytes('\n')
if err != nil {
if err == io.EOF {
err = nil
break
}
log.LogErrorf("read line failed, path %s, err: %v", filePath, err)
@ -1271,7 +1267,6 @@ func checkBadNormalExtents(volname, dir, checkDpId string) {
line, err := reader.ReadBytes('\n')
if err != nil {
if err == io.EOF {
err = nil
break
}
slog.Fatalf("Read line failed, path %s, err: %v", badExtentPath, err)
@ -1316,7 +1311,6 @@ func checkBadNormalExtents(volname, dir, checkDpId string) {
line, err := reader.ReadBytes('\n')
if err != nil {
if err == io.EOF {
err = nil
break
}
slog.Fatalf("Read line failed, path %s, err: %v", filePath, err)
@ -1342,7 +1336,6 @@ func checkBadNormalExtents(volname, dir, checkDpId string) {
}
}
slog.Printf("finally check success, no wrong bad extents vol %s, dp %s, cost %d ms", volname, checkDpId, time.Since(start).Milliseconds())
return
}
func batchLockBadNormalExtent(dpIdStr string, exts []*BadNornalExtent, IsCreate bool, beforeTime string) (err error) {
@ -1621,7 +1614,6 @@ func cleanBadNormalExtent(volname, dir, backupDir, dpIdStr, extentStr string) (e
line, err := reader.ReadBytes('\n')
if err != nil {
if err == io.EOF {
err = nil
slog.Fatalf("extent %s is not in target dp %s bad file", extentStr, dpIdStr)
break
}
@ -1641,7 +1633,7 @@ func cleanBadNormalExtent(volname, dir, backupDir, dpIdStr, extentStr string) (e
continue
}
var exts = []*BadNornalExtent{&badExtent}
exts := []*BadNornalExtent{&badExtent}
err = batchLockBadNormalExtent(dpIdStr, exts, false, string(data))
if err != nil {
@ -2164,9 +2156,7 @@ func rollbackBadNormalExtent(volname, backupDir, dpId, extent string) (err error
return
}
if strings.HasSuffix(extent, BackSuffix) {
extent = strings.TrimSuffix(extent, BackSuffix)
}
badExtent.Size = uint32(len(data))
badExtent.ExtentId = extent

1
fsck/main.go
View File

@ -44,7 +44,6 @@ func main() {
}
func listenPprof() {
mainMux := http.NewServeMux()
mux := http.NewServeMux()
mux.Handle("/debug/pprof", http.HandlerFunc(pprof.Index))

9
master/api_args_parse.go
View File

@ -1846,15 +1846,6 @@ func extractQuotaId(r *http.Request) (quotaId uint32, err error) {
return
}
func extractInodeId(r *http.Request) (inode uint64, err error) {
var value string
if value = r.FormValue(inodeKey); value == "" {
err = keyNotFound(inodeKey)
return
}
return strconv.ParseUint(value, 10, 64)
}
func parseRequestToSetTrashInterval(r *http.Request) (name, authKey string, interval int64, err error) {
if err = r.ParseForm(); err != nil {
return

6
master/api_service.go
View File

@ -303,11 +303,11 @@ func (m *Server) setEnableAuditLogForVolume(w http.ResponseWriter, r *http.Reque
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeVolNotExists, Msg: err.Error()})
return
}
oldEnable := vol.EnableAuditLog
vol.EnableAuditLog = status
oldEnable := vol.DisableAuditLog
vol.DisableAuditLog = status
defer func() {
if err != nil {
vol.EnableAuditLog = oldEnable
vol.DisableAuditLog = oldEnable
}
}()
if err = m.cluster.syncUpdateVol(vol); err != nil {

11
master/api_service_test.go
View File

@ -127,7 +127,6 @@ func createDefaultMasterServerForTest() *Server {
}`
testServer, err := createMasterServer(cfgJSON)
if err != nil {
panic(err)
}
@ -344,11 +343,11 @@ func processWithFatalV2(url string, success bool, req map[string]interface{}, t
assert.Nil(t, err)
if success {
assert.True(t, reply.Code == proto.ErrCodeSuccess)
require.EqualValues(t, proto.ErrCodeSuccess, reply.Code)
return reply
}
assert.True(t, reply.Code != proto.ErrCodeSuccess)
require.NotEqualValues(t, proto.ErrCodeSuccess, reply.Code)
return
}
@ -1570,11 +1569,11 @@ func TestVolumeEnableAuditLog(t *testing.T) {
enableUrl := fmt.Sprintf("%v?name=%v&%v=true", reqUrl, vol.Name, enableKey)
disableUrl := fmt.Sprintf("%v?name=%v&%v=false", reqUrl, vol.Name, enableKey)
process(disableUrl, t)
require.True(t, vol.DisableAuditLog)
require.True(t, checkVolAuditLog(name, false))
process(enableUrl, t)
require.False(t, vol.DisableAuditLog)
require.True(t, checkVolAuditLog(name, true))
process(enableUrl, t)
require.True(t, vol.DisableAuditLog)
require.True(t, checkVolAuditLog(name, false))
}
func checkVolDpRepairBlockSize(name string, size uint64) (success bool) {

5
master/cluster.go
View File

@ -1215,7 +1215,8 @@ func (c *Cluster) checkLackReplicaDataPartitions() (lackReplicaDataPartitions []
func (c *Cluster) checkReplicaOfDataPartitions(ignoreDiscardDp bool) (
lackReplicaDPs []*DataPartition, unavailableReplicaDPs []*DataPartition, repFileCountDifferDps []*DataPartition,
repUsedSizeDifferDps []*DataPartition, excessReplicaDPs []*DataPartition, noLeaderDPs []*DataPartition, err error) {
repUsedSizeDifferDps []*DataPartition, excessReplicaDPs []*DataPartition, noLeaderDPs []*DataPartition, err error,
) {
noLeaderDPs = make([]*DataPartition, 0)
lackReplicaDPs = make([]*DataPartition, 0)
unavailableReplicaDPs = make([]*DataPartition, 0)
@ -3978,7 +3979,7 @@ func (c *Cluster) clearVols() {
vol.qosManager.stop()
}
}()
c.vols = make(map[string]*Vol, 0)
c.vols = make(map[string]*Vol)
}
func (c *Cluster) clearTopology() {

2
master/data_node.go
View File

@ -227,7 +227,7 @@ func (dataNode *DataNode) dpCntInLimit() bool {
}
func (dataNode *DataNode) isWriteAbleWithSizeNoLock(size uint64) (ok bool) {
if dataNode.isActive == true && dataNode.AvailableSpace > size && !dataNode.RdOnly &&
if dataNode.isActive && dataNode.AvailableSpace > size && !dataNode.RdOnly &&
dataNode.Total > dataNode.Used && (dataNode.Total-dataNode.Used) > size {
ok = true
}

5
master/data_partition.go
View File

@ -1010,7 +1010,6 @@ func (partition *DataPartition) buildDpInfo(c *Cluster) *proto.DataPartitionInfo
OfflinePeerID: partition.OfflinePeerID,
IsRecover: partition.isRecover,
FilesWithMissingReplica: filesMissReplicas,
SingleDecommissionAddr: partition.SingleDecommissionAddr,
IsDiscard: partition.IsDiscard,
SingleDecommissionStatus: partition.GetSpecialReplicaDecommissionStep(),
Forbidden: forbidden,
@ -1066,7 +1065,8 @@ func GetDecommissionStatusMessage(status uint32) string {
}
func (partition *DataPartition) MarkDecommissionStatus(srcAddr, dstAddr, srcDisk string, raftForce bool, term uint64,
migrateType uint32, c *Cluster, ns *nodeSet) (err error) {
migrateType uint32, c *Cluster, ns *nodeSet,
) (err error) {
if partition.needManualFix() && migrateType == AutoDecommission {
return proto.ErrAllReplicaUnavailable
}
@ -2019,5 +2019,4 @@ func (partition *DataPartition) removeHostByForce(c *Cluster, peerAddr string) {
partition.PartitionID, partition.Hosts[0], err)
return
}
return
}

1
master/http_server.go
View File

@ -21,6 +21,7 @@ import (
"html"
"net/http"
"net/http/httputil"
"strings"
"time"
"github.com/gorilla/mux"

6
master/vol.go
View File

@ -484,10 +484,6 @@ func (vol *Vol) getDataPartitionViewCompress() (body []byte, err error) {
return vol.dataPartitions.updateCompressCache(false, 0, vol)
}
func (vol *Vol) getDataPartitionViewCompress() (body []byte, err error) {
return vol.dataPartitions.updateCompressCache(false, 0, vol.VolType)
}
func (vol *Vol) getDataPartitionByID(partitionID uint64) (dp *DataPartition, err error) {
return vol.dataPartitions.get(partitionID)
}
@ -603,8 +599,6 @@ func (vol *Vol) checkDataPartitions(c *Cluster) (cnt int) {
totalPreloadCapacity := uint64(0)
totalPreloadCapacity := uint64(0)
partitions := vol.dataPartitions.clonePartitions()
for _, dp := range partitions {

22
metanode/api_handler.go
View File

@ -80,7 +80,9 @@ func (m *MetaNode) registerAPIHandler() (err error) {
return
}
func (m *MetaNode) getParamsHandler(w http.ResponseWriter, r *http.Request) {
func (m *MetaNode) getParamsHandler(w http.ResponseWriter,
r *http.Request,
) {
resp := NewAPIResponse(http.StatusOK, http.StatusText(http.StatusOK))
params := make(map[string]interface{})
params[metaNodeDeleteBatchCountKey] = DeleteBatchCount()
@ -91,7 +93,9 @@ func (m *MetaNode) getParamsHandler(w http.ResponseWriter, r *http.Request) {
}
}
func (m *MetaNode) getSmuxStatHandler(w http.ResponseWriter, r *http.Request) {
func (m *MetaNode) getSmuxStatHandler(w http.ResponseWriter,
r *http.Request,
) {
resp := NewAPIResponse(http.StatusOK, http.StatusText(http.StatusOK))
resp.Data = smuxPool.GetStat()
data, _ := resp.Marshal()
@ -100,7 +104,9 @@ func (m *MetaNode) getSmuxStatHandler(w http.ResponseWriter, r *http.Request) {
}
}
func (m *MetaNode) getPartitionsHandler(w http.ResponseWriter, r *http.Request) {
func (m *MetaNode) getPartitionsHandler(w http.ResponseWriter,
r *http.Request,
) {
resp := NewAPIResponse(http.StatusOK, http.StatusText(http.StatusOK))
resp.Data = m.metadataManager
data, _ := resp.Marshal()
@ -343,7 +349,10 @@ func (m *MetaNode) getRaftStatusHandler(w http.ResponseWriter, r *http.Request)
resp.Data = raftStatus
}
func (m *MetaNode) getEbsExtentsByInodeHandler(w http.ResponseWriter, r *http.Request) {
func (m *MetaNode) getEbsExtentsByInodeHandler(w http.ResponseWriter,
r *http.Request,
) {
r.ParseForm()
resp := NewAPIResponse(http.StatusBadRequest, "")
defer func() {
data, _ := resp.Marshal()
@ -379,7 +388,10 @@ func (m *MetaNode) getEbsExtentsByInodeHandler(w http.ResponseWriter, r *http.Re
}
}
func (m *MetaNode) getExtentsByInodeHandler(w http.ResponseWriter, r *http.Request) {
func (m *MetaNode) getExtentsByInodeHandler(w http.ResponseWriter,
r *http.Request,
) {
r.ParseForm()
resp := NewAPIResponse(http.StatusBadRequest, "")
defer func() {
data, _ := resp.Marshal()

188
metanode/manager_op.go
View File

@ -71,7 +71,9 @@ func (m *metadataManager) checkDisableAuditLogVolume(volNames []string, partitio
partition.SetEnableAuditLog(true)
}
func (m *metadataManager) opMasterHeartbeat(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opMasterHeartbeat(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
// For ack to master
data := p.Data
m.responseAckOKToMaster(conn, p)
@ -159,7 +161,9 @@ func (m *metadataManager) opMasterHeartbeat(conn net.Conn, p *Packet, remoteAddr
return
}
func (m *metadataManager) opCreateMetaPartition(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opCreateMetaPartition(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
defer func() {
var buf []byte
status := proto.OpOk
@ -195,7 +199,9 @@ func (m *metadataManager) opCreateMetaPartition(conn net.Conn, p *Packet, remote
}
// Handle OpCreate inode.
func (m *metadataManager) opCreateInode(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opCreateInode(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &CreateInoReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -293,7 +299,9 @@ func (m *metadataManager) opTxMetaLinkInode(conn net.Conn, p *Packet, remoteAddr
return
}
func (m *metadataManager) opMetaLinkInode(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opMetaLinkInode(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &LinkInodeReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -325,7 +333,9 @@ func (m *metadataManager) opMetaLinkInode(conn net.Conn, p *Packet, remoteAddr s
}
// Handle OpCreate
func (m *metadataManager) opFreeInodeOnRaftFollower(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opFreeInodeOnRaftFollower(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
mp, err := m.getPartition(p.PartitionID)
if err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -341,7 +351,9 @@ func (m *metadataManager) opFreeInodeOnRaftFollower(conn net.Conn, p *Packet, re
}
// Handle OpCreate
func (m *metadataManager) opTxCreateDentry(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opTxCreateDentry(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.TxCreateDentryRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -375,7 +387,9 @@ func (m *metadataManager) opTxCreateDentry(conn net.Conn, p *Packet, remoteAddr
return
}
func (m *metadataManager) opTxCreate(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opTxCreate(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.TxCreateRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -404,7 +418,9 @@ func (m *metadataManager) opTxCreate(conn net.Conn, p *Packet, remoteAddr string
return
}
func (m *metadataManager) opTxGet(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opTxGet(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.TxGetInfoRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -435,7 +451,9 @@ func (m *metadataManager) opTxGet(conn net.Conn, p *Packet, remoteAddr string) (
return
}
func (m *metadataManager) opTxCommitRM(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opTxCommitRM(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.TxApplyRMRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -464,7 +482,9 @@ func (m *metadataManager) opTxCommitRM(conn net.Conn, p *Packet, remoteAddr stri
return
}
func (m *metadataManager) opTxRollbackRM(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opTxRollbackRM(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.TxApplyRMRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -493,7 +513,9 @@ func (m *metadataManager) opTxRollbackRM(conn net.Conn, p *Packet, remoteAddr st
return
}
func (m *metadataManager) opTxCommit(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opTxCommit(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.TxApplyRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -522,7 +544,9 @@ func (m *metadataManager) opTxCommit(conn net.Conn, p *Packet, remoteAddr string
return
}
func (m *metadataManager) opTxRollback(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opTxRollback(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.TxApplyRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -552,7 +576,9 @@ func (m *metadataManager) opTxRollback(conn net.Conn, p *Packet, remoteAddr stri
}
// Handle OpCreate
func (m *metadataManager) opCreateDentry(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opCreateDentry(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &CreateDentryReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -584,7 +610,9 @@ func (m *metadataManager) opCreateDentry(conn net.Conn, p *Packet, remoteAddr st
return
}
func (m *metadataManager) opQuotaCreateDentry(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opQuotaCreateDentry(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.QuotaCreateDentryRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -617,7 +645,9 @@ func (m *metadataManager) opQuotaCreateDentry(conn net.Conn, p *Packet, remoteAd
}
// Handle OpDelete Dentry
func (m *metadataManager) opTxDeleteDentry(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opTxDeleteDentry(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.TxDeleteDentryRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -647,7 +677,9 @@ func (m *metadataManager) opTxDeleteDentry(conn net.Conn, p *Packet, remoteAddr
}
// Handle OpDelete Dentry
func (m *metadataManager) opDeleteDentry(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opDeleteDentry(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &DeleteDentryReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -681,7 +713,9 @@ func (m *metadataManager) opDeleteDentry(conn net.Conn, p *Packet, remoteAddr st
}
// Handle Op batch Delete Dentry
func (m *metadataManager) opBatchDeleteDentry(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opBatchDeleteDentry(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &BatchDeleteDentryReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -746,7 +780,9 @@ func (m *metadataManager) opTxUpdateDentry(conn net.Conn, p *Packet, remoteAddr
return
}
func (m *metadataManager) opUpdateDentry(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opUpdateDentry(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &UpdateDentryReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -812,7 +848,9 @@ func (m *metadataManager) opTxMetaUnlinkInode(conn net.Conn, p *Packet, remoteAd
return
}
func (m *metadataManager) opMetaUnlinkInode(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opMetaUnlinkInode(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &UnlinkInoReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -843,7 +881,9 @@ func (m *metadataManager) opMetaUnlinkInode(conn net.Conn, p *Packet, remoteAddr
return
}
func (m *metadataManager) opMetaBatchUnlinkInode(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opMetaBatchUnlinkInode(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &BatchUnlinkInoReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -874,7 +914,9 @@ func (m *metadataManager) opMetaBatchUnlinkInode(conn net.Conn, p *Packet, remot
return
}
func (m *metadataManager) opReadDirOnly(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opReadDirOnly(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.ReadDirOnlyRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -900,7 +942,9 @@ func (m *metadataManager) opReadDirOnly(conn net.Conn, p *Packet, remoteAddr str
}
// Handle OpReadDir
func (m *metadataManager) opReadDir(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opReadDir(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.ReadDirRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -926,7 +970,9 @@ func (m *metadataManager) opReadDir(conn net.Conn, p *Packet, remoteAddr string)
}
// Handle OpReadDirLimit
func (m *metadataManager) opReadDirLimit(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opReadDirLimit(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.ReadDirLimitRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -995,7 +1041,9 @@ func (m *metadataManager) opMetaInodeGet(conn net.Conn, p *Packet, remoteAddr st
return
}
func (m *metadataManager) opBatchMetaEvictInode(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opBatchMetaEvictInode(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.BatchEvictInodeRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
@ -1028,7 +1076,9 @@ func (m *metadataManager) opBatchMetaEvictInode(conn net.Conn, p *Packet, remote
return
}
func (m *metadataManager) opMetaEvictInode(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opMetaEvictInode(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.EvictInodeRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1061,7 +1111,9 @@ func (m *metadataManager) opMetaEvictInode(conn net.Conn, p *Packet, remoteAddr
return
}
func (m *metadataManager) opSetAttr(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opSetAttr(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &SetattrRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1097,7 +1149,9 @@ func (m *metadataManager) opSetAttr(conn net.Conn, p *Packet, remoteAddr string)
}
// Lookup request
func (m *metadataManager) opMetaLookup(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opMetaLookup(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.LookupRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1130,7 +1184,9 @@ func (m *metadataManager) opMetaLookup(conn net.Conn, p *Packet, remoteAddr stri
return
}
func (m *metadataManager) opMetaExtentsAdd(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opMetaExtentsAdd(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.AppendExtentKeyRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1166,7 +1222,9 @@ func (m *metadataManager) opMetaExtentsAdd(conn net.Conn, p *Packet, remoteAddr
}
// Append one extent with discard check
func (m *metadataManager) opMetaExtentAddWithCheck(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opMetaExtentAddWithCheck(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.AppendExtentKeyWithCheckRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1203,7 +1261,9 @@ func (m *metadataManager) opMetaExtentAddWithCheck(conn net.Conn, p *Packet, rem
return
}
func (m *metadataManager) opMetaExtentsList(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opMetaExtentsList(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.GetExtentsRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1231,7 +1291,9 @@ func (m *metadataManager) opMetaExtentsList(conn net.Conn, p *Packet, remoteAddr
return
}
func (m *metadataManager) opMetaObjExtentsList(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opMetaObjExtentsList(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.GetExtentsRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1257,7 +1319,9 @@ func (m *metadataManager) opMetaObjExtentsList(conn net.Conn, p *Packet, remoteA
return
}
func (m *metadataManager) opMetaExtentsDel(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opMetaExtentsDel(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
panic("not implemented yet")
// req := &proto.DelExtentKeyRequest{}
// if err = json.Unmarshal(p.Data, req); err != nil {
@ -1283,7 +1347,9 @@ func (m *metadataManager) opMetaExtentsDel(conn net.Conn, p *Packet, remoteAddr
// return
}
func (m *metadataManager) opMetaExtentsTruncate(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opMetaExtentsTruncate(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &ExtentsTruncateReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1314,7 +1380,9 @@ func (m *metadataManager) opMetaExtentsTruncate(conn net.Conn, p *Packet, remote
return
}
func (m *metadataManager) opMetaClearInodeCache(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opMetaClearInodeCache(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.ClearInodeCacheRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1340,7 +1408,9 @@ func (m *metadataManager) opMetaClearInodeCache(conn net.Conn, p *Packet, remote
}
// Delete a meta partition.
func (m *metadataManager) opDeleteMetaPartition(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opDeleteMetaPartition(conn net.Conn,
p *Packet, remoteAddr string,
) (err error) {
req := &proto.DeleteMetaPartitionRequest{}
adminTask := &proto.AdminTask{
Request: req,
@ -1373,7 +1443,9 @@ func (m *metadataManager) opDeleteMetaPartition(conn net.Conn, p *Packet, remote
return
}
func (m *metadataManager) opUpdateMetaPartition(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opUpdateMetaPartition(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := new(UpdatePartitionReq)
adminTask := &proto.AdminTask{
Request: req,
@ -1412,7 +1484,9 @@ func (m *metadataManager) opUpdateMetaPartition(conn net.Conn, p *Packet, remote
return
}
func (m *metadataManager) opLoadMetaPartition(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opLoadMetaPartition(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.MetaPartitionLoadRequest{}
adminTask := &proto.AdminTask{
Request: req,
@ -1446,7 +1520,9 @@ func (m *metadataManager) opLoadMetaPartition(conn net.Conn, p *Packet, remoteAd
return
}
func (m *metadataManager) opDecommissionMetaPartition(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opDecommissionMetaPartition(conn net.Conn,
p *Packet, remoteAddr string,
) (err error) {
var reqData []byte
req := &proto.MetaPartitionDecommissionRequest{}
adminTask := &proto.AdminTask{
@ -1504,7 +1580,9 @@ func (m *metadataManager) opDecommissionMetaPartition(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opAddMetaPartitionRaftMember(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opAddMetaPartitionRaftMember(conn net.Conn,
p *Packet, remoteAddr string,
) (err error) {
var reqData []byte
req := &proto.AddMetaPartitionRaftMemberRequest{}
adminTask := &proto.AdminTask{
@ -1572,7 +1650,9 @@ func (m *metadataManager) opAddMetaPartitionRaftMember(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opRemoveMetaPartitionRaftMember(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opRemoveMetaPartitionRaftMember(conn net.Conn,
p *Packet, remoteAddr string,
) (err error) {
var reqData []byte
req := &proto.RemoveMetaPartitionRaftMemberRequest{}
adminTask := &proto.AdminTask{
@ -1649,7 +1729,9 @@ func (m *metadataManager) opRemoveMetaPartitionRaftMember(conn net.Conn, p *Pack
return
}
func (m *metadataManager) opMetaBatchInodeGet(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opMetaBatchInodeGet(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.BatchInodeGetRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1675,7 +1757,9 @@ func (m *metadataManager) opMetaBatchInodeGet(conn net.Conn, p *Packet, remoteAd
return
}
func (m *metadataManager) opMetaPartitionTryToLeader(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opMetaPartitionTryToLeader(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
mp, err := m.getPartition(p.PartitionID)
if err != nil {
goto errDeal
@ -1692,7 +1776,9 @@ errDeal:
return
}
func (m *metadataManager) opMetaDeleteInode(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opMetaDeleteInode(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.DeleteInodeRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1717,7 +1803,9 @@ func (m *metadataManager) opMetaDeleteInode(conn net.Conn, p *Packet, remoteAddr
return
}
func (m *metadataManager) opMetaBatchDeleteInode(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opMetaBatchDeleteInode(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
var req *proto.DeleteInodeBatchRequest
if err = json.Unmarshal(p.Data, &req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -2190,7 +2278,9 @@ func (m *metadataManager) opListMultipart(conn net.Conn, p *Packet, remoteAddr s
}
// Handle OpMetaTxCreateInode inode.
func (m *metadataManager) opTxCreateInode(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opTxCreateInode(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.TxCreateInodeRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -2326,7 +2416,9 @@ func (m *metadataManager) opMetaGetInodeQuota(conn net.Conn, p *Packet, remote s
return
}
func (m *metadataManager) opMetaGetUniqID(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opMetaGetUniqID(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &proto.GetUniqIDRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -2601,7 +2693,9 @@ func (m *metadataManager) checkAndPromoteVersion(volName string) (err error) {
return
}
func (m *metadataManager) opMultiVersionOp(conn net.Conn, p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opMultiVersionOp(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
// For ack to master
data := p.Data
m.responseAckOKToMaster(conn, p)

4
metanode/manager_proxy.go
View File

@ -15,7 +15,6 @@
package metanode
import (
"github.com/cubefs/cubefs/storage"
"net"
"github.com/cubefs/cubefs/storage"
@ -89,7 +88,8 @@ func (m *metadataManager) IsForbiddenOp(mp MetaPartition, reqOp uint8) bool {
// The proxy is used during the leader change. When a leader of a partition changes, the proxy forwards the request to
// the new leader.
func (m *metadataManager) serveProxy(conn net.Conn, mp MetaPartition,
p *Packet) (ok bool) {
p *Packet,
) (ok bool) {
var (
mConn *net.TCPConn
leaderAddr string

3
metanode/partition_delete_extents.go
View File

@ -40,7 +40,8 @@ const (
var extentsFileHeader = []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08}
// start metapartition delete extents work
// / start metapartition delete extents work
// /
func (mp *metaPartition) startToDeleteExtents() {
fileList := synclist.New()
go mp.appendDelExtentsToFile(fileList)

6
metanode/partition_free_list.go
View File

@ -57,6 +57,7 @@ func (mp *metaPartition) startFreeList() (err error) {
return
}
go mp.updateVolWorker()
go mp.deleteWorker()
mp.startToDeleteExtents()
return
@ -116,7 +117,7 @@ func (mp *metaPartition) updateVolWorker() {
}
for i := 0; i < len(view.DataPartitions); i++ {
if len(view.DataPartitions[i].Hosts) < 1 {
log.LogErrorf("action[UpdateVolumeView] dp id(%v) is invalid, DataPartitionResponse detail[%v]",
log.LogErrorf("updateVolWorker dp id(%v) is invalid, DataPartitionResponse detail[%v]",
view.DataPartitions[i].PartitionID, view.DataPartitions[i])
continue
}
@ -222,7 +223,8 @@ func (mp *metaPartition) deleteWorker() {
// delete Extents by Partition,and find all successDelete inode
func (mp *metaPartition) batchDeleteExtentsByPartition(partitionDeleteExtents map[uint64][]*proto.DelExtentParam,
allInodes []*Inode) (shouldCommit []*Inode, shouldPushToFreeList []*Inode) {
allInodes []*Inode,
) (shouldCommit []*Inode, shouldPushToFreeList []*Inode) {
occurErrors := make(map[uint64]error)
shouldCommit = make([]*Inode, 0, len(allInodes))
shouldPushToFreeList = make([]*Inode, 0)

10
metanode/partition_free_list_test.go
View File

@ -15,6 +15,7 @@
package metanode
import (
"fmt"
"os"
"path"
"strings"
@ -26,6 +27,8 @@ import (
"github.com/stretchr/testify/require"
)
var VolNameForFreeListTest = "TestForFreeList"
func newPartitionForFreeList(conf *MetaPartitionConfig, manager *metadataManager) (mp *metaPartition) {
mp = &metaPartition{
config: conf,
@ -64,9 +67,10 @@ func TestPersistInodesFreeList(t *testing.T) {
fileName := path.Join(config.RootDir, DeleteInodeFileExtension)
oldIno, err := fileutil.Stat(fileName)
require.NoError(t, err)
t.Logf("Persist many inodes")
const persistBatchCount = 50000
const testCount = DeleteInodeFileRollingSize / 8
const persistBatchCount = 500000
unitSize := len(fmt.Sprintf("%v\n", 1000000))
testCount := DeleteInodeFileRollingSize / unitSize
t.Logf("Persist many inodes, unitSize %d, total %d", unitSize, testCount)
inodes := make([]uint64, 0, persistBatchCount)
for i := 0; i < persistBatchCount; i++ {
inodes = append(inodes, uint64(i)+1000000)

2
metanode/partition_fsm.go
View File

@ -299,7 +299,7 @@ func (mp *metaPartition) Apply(command []byte, index uint64) (resp interface{},
}
err = mp.fsmSetXAttr(extend)
case opFSMLockDir:
var req = &proto.LockDirRequest{}
req := &proto.LockDirRequest{}
if err = json.Unmarshal(msg.V, req); err != nil {
return
}

4
metanode/partition_fsmop.go
View File

@ -75,7 +75,8 @@ func (mp *metaPartition) decommissionPartition() (err error) {
}
func (mp *metaPartition) fsmUpdatePartition(end uint64) (status uint8,
err error) {
err error,
) {
status = proto.OpOk
oldEnd := mp.config.End
mp.config.End = end
@ -177,7 +178,6 @@ func (mp *metaPartition) delOldExtentFile(buf []byte) (err error) {
return
}
//
func (mp *metaPartition) setExtentDeleteFileCursor(buf []byte) (err error) {
str := string(buf)
var (

8
metanode/partition_fsmop_dentry.go
View File

@ -68,7 +68,9 @@ func (mp *metaPartition) fsmTxCreateDentry(txDentry *TxDentry) (status uint8) {
}
// Insert a dentry into the dentry tree.
func (mp *metaPartition) fsmCreateDentry(dentry *Dentry, forceUpdate bool) (status uint8) {
func (mp *metaPartition) fsmCreateDentry(dentry *Dentry,
forceUpdate bool,
) (status uint8) {
status = proto.OpOk
var parIno *Inode
if !forceUpdate {
@ -357,7 +359,9 @@ func (mp *metaPartition) fsmTxUpdateDentry(txUpDateDentry *TxUpdateDentry) (resp
return
}
func (mp *metaPartition) fsmUpdateDentry(dentry *Dentry) (resp *DentryResponse) {
func (mp *metaPartition) fsmUpdateDentry(dentry *Dentry) (
resp *DentryResponse,
) {
resp = NewDentryResponse()
resp.Status = proto.OpOk
mp.dentryTree.CopyFind(dentry, func(item BtreeItem) {

10
metanode/partition_fsmop_extend.go
View File

@ -15,11 +15,11 @@
package metanode
import (
"fmt"
"math"
"strconv"
"strings"
"time"
"fmt"
"math"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/util/log"
@ -43,7 +43,7 @@ func (mp *metaPartition) fsmLockDir(req *proto.LockDirRequest) (resp *proto.Lock
log.LogDebugf("fsmLockDir ino=%v, lockId=%d, submitTime=%v, lease=%d\n", ino, lockId, submitTime, lease)
var newExtend = NewExtend(ino)
newExtend := NewExtend(ino)
treeItem := mp.extendTree.CopyGet(newExtend)
var firstLock bool = false
@ -72,7 +72,7 @@ func (mp *metaPartition) fsmLockDir(req *proto.LockDirRequest) (resp *proto.Lock
lockId = int64(uu_id.ID())
lockIdStr = strconv.Itoa(int(lockId))
value = lockIdStr + "|" + validTimeStr
newExtend.Put([]byte("dir_lock"), []byte(value))
newExtend.Put([]byte("dir_lock"), []byte(value), 0)
mp.extendTree.ReplaceOrInsert(newExtend, true)
} else {
log.LogDebugf("fsmLockDir oldValue=%s\n", oldValue)
@ -103,7 +103,7 @@ func (mp *metaPartition) fsmLockDir(req *proto.LockDirRequest) (resp *proto.Lock
}
existExtend.Remove([]byte("dir_lock"))
newExtend.Put([]byte("dir_lock"), []byte(value))
newExtend.Put([]byte("dir_lock"), []byte(value), 0)
existExtend.Merge(newExtend, true)
}

6
metanode/partition_op_dentry.go
View File

@ -25,7 +25,6 @@ import (
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/util/auditlog"
"github.com/cubefs/cubefs/util/errors"
"github.com/cubefs/cubefs/util/log"
)
func (mp *metaPartition) TxCreateDentry(req *proto.TxCreateDentryRequest, p *Packet, remoteAddr string) (err error) {
@ -286,10 +285,7 @@ func (mp *metaPartition) DeleteDentry(req *DeleteDentryReq, p *Packet, remoteAdd
return
}
}
dentry := &Dentry{
ParentId: req.ParentID,
Name: req.Name,
}
dentry.setVerSeq(req.Verseq)
log.LogDebugf("action[DeleteDentry] den param(%v)", dentry)

8
metanode/partition_store.go
View File

@ -1050,7 +1050,9 @@ func (mp *metaPartition) storeTxInfo(rootDir string, sm *storeMsg) (crc uint32,
return
}
func (mp *metaPartition) storeInode(rootDir string, sm *storeMsg) (crc uint32, err error) {
func (mp *metaPartition) storeInode(rootDir string,
sm *storeMsg,
) (crc uint32, err error) {
filename := path.Join(rootDir, inodeFile)
fp, err := os.OpenFile(filename, os.O_RDWR|os.O_TRUNC|os.O_APPEND|os.
O_CREATE, 0o755)
@ -1108,7 +1110,9 @@ func (mp *metaPartition) storeInode(rootDir string, sm *storeMsg) (crc uint32, e
return
}
func (mp *metaPartition) storeDentry(rootDir string, sm *storeMsg) (crc uint32, err error) {
func (mp *metaPartition) storeDentry(rootDir string,
sm *storeMsg,
) (crc uint32, err error) {
filename := path.Join(rootDir, dentryFile)
fp, err := os.OpenFile(filename, os.O_RDWR|os.O_TRUNC|os.O_APPEND|os.
O_CREATE, 0o755)

3
metanode/server.go
View File

@ -101,7 +101,8 @@ func (m *MetaNode) serveConn(conn net.Conn, stopC chan uint8) {
}
func (m *MetaNode) handlePacket(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
remoteAddr string,
) (err error) {
// Handle request
err = m.metadataManager.HandleMetadataOperation(conn, p, remoteAddr)
return

4
objectnode/api_handler_multipart.go
View File

@ -585,8 +585,8 @@ func (o *ObjectNode) checkReqParts(param *RequestParam, reqParts *CompleteMultip
maxPartNum := saveParts[len(saveParts)-1].ID
allSaveParts := make([]*proto.MultipartPartInfo, maxPartNum+1)
uploadedInfo := make(map[uint16]string, 0)
discardedPartInodes = make(map[uint64]uint16, 0)
uploadedInfo := make(map[uint16]string)
discardedPartInodes = make(map[uint64]uint16)
for _, uploadedPart := range multipartInfo.Parts {
log.LogDebugf("checkReqParts: server save part check: requestID(%v) volume(%v) part(%v)",
GetRequestID(param.r), param.Bucket(), uploadedPart)

1
objectnode/api_middleware.go
View File

@ -189,7 +189,6 @@ func (o *ObjectNode) policyCheckMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
o.policyCheck(next.ServeHTTP).ServeHTTP(w, r)
return
})
}

15
objectnode/fs_volume.go
View File

@ -2080,8 +2080,9 @@ func (v *Volume) lookupDirectories(dirs []string, autoCreate bool) (inode uint64
return
}
func (v *Volume) listFilesV1(prefix, marker, delimiter string, maxKeys uint64, onlyObject bool,
) (infos []*FSFileInfo, prefixes Prefixes, nextMarker string, err error) {
func (v *Volume) listFilesV1(prefix, marker, delimiter string, maxKeys uint64, onlyObject bool) (infos []*FSFileInfo,
prefixes Prefixes, nextMarker string, err error,
) {
prefixMap := PrefixMap(make(map[string]struct{}))
parentId, dirs, err := v.findParentId(prefix)
@ -2127,8 +2128,9 @@ func (v *Volume) listFilesV1(prefix, marker, delimiter string, maxKeys uint64, o
return
}
func (v *Volume) listFilesV2(prefix, startAfter, contToken, delimiter string, maxKeys uint64,
) (infos []*FSFileInfo, prefixes Prefixes, nextMarker string, err error) {
func (v *Volume) listFilesV2(prefix, startAfter, contToken, delimiter string, maxKeys uint64) (infos []*FSFileInfo,
prefixes Prefixes, nextMarker string, err error,
) {
prefixMap := PrefixMap(make(map[string]struct{}))
var marker string
@ -2467,8 +2469,9 @@ func (v *Volume) updateETag(inode uint64, size int64, mt time.Time) (etagValue E
return
}
func (v *Volume) ListMultipartUploads(prefix, delimiter, keyMarker string, multipartIdMarker string, maxUploads uint64,
) (uploads []*FSUpload, nextMarker, nextMultipartIdMarker string, isTruncated bool, prefixes []string, err error) {
func (v *Volume) ListMultipartUploads(prefix, delimiter, keyMarker string, multipartIdMarker string, maxUploads uint64) (
uploads []*FSUpload, nextMarker, nextMultipartIdMarker string, isTruncated bool, prefixes []string, err error,
) {
sessions, err := v.mw.ListMultipart_ll(prefix, delimiter, keyMarker, multipartIdMarker, maxUploads)
if err != nil || len(sessions) == 0 {
return

1
objectnode/policy_keyset.go
View File

@ -128,6 +128,7 @@ func (set KeySet) AddAll(keys KeySet) {
// returns a key set contains difference of two key set.
// Example:
//
// keySet1 := ["one", "two", "three"]
// keySet2 := ["two", "four", "three"]
// keySet1.Difference(keySet2) == ["one"]

4
objectnode/policy_match.go
View File

@ -67,6 +67,7 @@ func (s *Statement) match(apiName string, uid string, conditionCheck map[string]
return true
}
// ----------------------------------------------------------------------------------------------------------------
func (s *Statement) matchAction(apiName string) bool {
log.LogDebug("start to match action")
switch s.Action.(type) {
@ -105,6 +106,7 @@ func (actions ActionType) match(apiToMatch string) bool {
return false
}
// ----------------------------------------------------------------------------------------------------------------
func (s *Statement) matchPrincipal(uid string) bool {
switch s.Principal.(type) {
case string: // "*" or "123"
@ -144,6 +146,7 @@ func (p PrincipalType) match(uid string) bool {
}
}
// ----------------------------------------------------------------------------------------------------------------
func (s *Statement) matchResource(apiName string, keyname interface{}) bool {
if IsBucketApi(apiName) {
return s.matchBucketInResource()
@ -234,6 +237,7 @@ func makeRegexPattern(raw string) string {
return pattern
}
// ----------------------------------------------------------------------------------------------------------------
func (s *Statement) matchCondition(conditionCheck map[string]string) bool {
// condition is optional
if s.Condition == nil {

2
proto/admin_proto.go
View File

@ -19,6 +19,8 @@ import (
"fmt"
"strconv"
"time"
"github.com/cubefs/cubefs/util"
)
type ContextUserKey string

1
proto/packet.go
View File

@ -258,7 +258,6 @@ const (
// io speed limit
OpLimitedIoErr uint8 = 0xB1
OpReadRepairExtentAgain uint8 = 0xEF
OpStoreClosed uint8 = 0xB2
)

3
raftstore/partition.go
View File

@ -80,7 +80,8 @@ type partition struct {
// ChangeMember submits member change event and information to raft log.
func (p *partition) ChangeMember(changeType proto.ConfChangeType, peer proto.Peer, context []byte) (
resp interface{}, err error) {
resp interface{}, err error,
) {
if !p.IsRaftLeader() {
err = raft.ErrNotLeader
return

2
sdk/data/stream/extent_handler.go
View File

@ -479,7 +479,7 @@ func (eh *ExtentHandler) appendExtentKey() (err error) {
eh.dirty = false
eh.lastKey = *eh.key
log.LogDebugf("action[appendExtentKey] status %v, needUpdateVer %v, eh{%v}", status, eh.stream.needUpdateVer, eh)
return
return nil
}
// Due to the asynchronous synchronization of version numbers, the extent cache version of the client is updated first before being written to the meta.
// However, it is possible for the client version to lag behind the meta version, resulting in partial inconsistencies in judgment.

4
sdk/data/wrapper/default_random_selector.go
View File

@ -141,7 +141,9 @@ func (s *DefaultRandomSelector) getLocalLeaderDataPartition(exclude map[string]s
return s.getRandomDataPartition(localLeaderPartitions, exclude)
}
func (s *DefaultRandomSelector) getRandomDataPartition(partitions []*DataPartition, exclude map[string]struct{}) (dp *DataPartition) {
func (s *DefaultRandomSelector) getRandomDataPartition(partitions []*DataPartition, exclude map[string]struct{}) (
dp *DataPartition,
) {
length := len(partitions)
if length == 0 {
return nil

24
sdk/master/api_admin.go
View File

@ -247,17 +247,6 @@ func (api *AdminAPI) UnDeleteVolume(volName, authKey string, status bool) (err e
return
}
func (api *AdminAPI) UnDeleteVolume(volName, authKey string, status bool) (err error) {
var request = newAPIRequest(http.MethodGet, proto.AdminDeleteVol)
request.addParam("name", volName)
request.addParam("authKey", authKey)
request.addParam("delete", strconv.FormatBool(false))
if _, err = api.mc.serveRequest(request); err != nil {
return
}
return
}
func (api *AdminAPI) UpdateVolume(
vv *proto.SimpleVolView,
txTimeout int64,
@ -406,15 +395,6 @@ func (api *AdminAPI) SetVolumeForbidden(volName string, forbidden bool) (err err
_, err = api.mc.serveRequest(request)
return
}
func (api *AdminAPI) SetVolumeForbidden(volName string, forbidden bool) (err error) {
request := newAPIRequest(http.MethodGet, proto.AdminVolForbidden)
request.addParam("name", volName)
request.addParam("forbidden", strconv.FormatBool(forbidden))
if _, err = api.mc.serveRequest(request); err != nil {
return
}
return
}
func (api *AdminAPI) SetVolumeAuditLog(volName string, enable bool) (err error) {
request := newRequest(post, proto.AdminVolEnableAuditLog).Header(api.h)
@ -783,7 +763,7 @@ func (api *AdminAPI) AbortDiskDecommission(addr string, disk string) (err error)
}
func (api *AdminAPI) SetClusterDecommissionLimit(limit int32) (err error) {
var request = newAPIRequest(http.MethodPost, proto.AdminUpdateDecommissionLimit)
request := newAPIRequest(http.MethodPost, proto.AdminUpdateDecommissionLimit)
request.addParam("decommissionLimit", strconv.FormatInt(int64(limit), 10))
if _, err = api.mc.serveRequest(request); err != nil {
return
@ -793,7 +773,7 @@ func (api *AdminAPI) SetClusterDecommissionLimit(limit int32) (err error) {
func (api *AdminAPI) QueryDecommissionToken() (status []proto.DecommissionTokenStatus, err error) {
var buf []byte
var request = newAPIRequest(http.MethodGet, proto.AdminQueryDecommissionToken)
request := newAPIRequest(http.MethodGet, proto.AdminQueryDecommissionToken)
if buf, err = api.mc.serveRequest(request); err != nil {
return
}

13
sdk/master/request.go
View File

@ -99,6 +99,19 @@ func newRequest(method string, path string) *request {
return req
}
func newAPIRequest(method string, path string) *request {
req := &request{
method: method,
path: path,
params: make(map[string]string),
header: make(map[string]string),
}
req.header["User-Agent"] = ReqHeaderUA
return req
}
func mergeHeader(headers map[string]string, added ...string) map[string]string {
if len(added)%2 == 1 {
added = added[:len(added)-1]

17
sdk/meta/api.go
View File

@ -120,7 +120,8 @@ func (mw *MetaWrapper) Create_ll(parentID uint64, name string, mode, uid, gid ui
}
func (mw *MetaWrapper) txCreate_ll(parentID uint64, name string, mode, uid, gid uint32, target []byte, txType uint32,
fullPath string, ignoreExist bool) (info *proto.InodeInfo, err error) {
fullPath string, ignoreExist bool,
) (info *proto.InodeInfo, err error) {
var (
status int
// err error
@ -213,7 +214,8 @@ create_dentry:
}
func (mw *MetaWrapper) create_ll(parentID uint64, name string, mode, uid, gid uint32, target []byte,
fullPath string, ignoreExist bool) (*proto.InodeInfo, error) {
fullPath string, ignoreExist bool,
) (*proto.InodeInfo, error) {
var (
status int
err error
@ -664,7 +666,7 @@ func (mw *MetaWrapper) txDelete_ll(parentID uint64, name string, isDir bool, ful
return nil, syscall.ENOENT
}
if mw.disableTrash == false && mw.disableTrashByClient == false {
if !mw.disableTrash && !mw.disableTrashByClient {
if mw.trashPolicy == nil {
log.LogDebugf("TRACE Remove:TrashPolicy is nil")
mw.enableTrash()
@ -782,7 +784,7 @@ func (mw *MetaWrapper) txDelete_ll(parentID uint64, name string, isDir bool, ful
tx.SetOnCommit(job)
}
// clear trash cache
if mw.trashPolicy != nil && mw.disableTrash == false {
if mw.trashPolicy != nil && !mw.disableTrash {
mw.trashPolicy.CleanTrashPatchCache(mw.getCurrentPath(parentID), name)
}
return info, preErr
@ -811,7 +813,7 @@ func (mw *MetaWrapper) Delete_ll_EX(parentID uint64, name string, isDir bool, ve
return nil, syscall.ENOENT
}
if mw.disableTrash == false && mw.disableTrashByClient == false {
if !mw.disableTrash && !mw.disableTrashByClient {
if mw.trashPolicy == nil {
log.LogDebugf("TRACE Remove:TrashPolicy is nil")
mw.enableTrash()
@ -1053,7 +1055,7 @@ func (mw *MetaWrapper) deletewithcond_ll(parentID, cond uint64, name string, isD
}()
}
// clear trash cache
if mw.trashPolicy != nil && mw.disableTrash == false {
if mw.trashPolicy != nil && !mw.disableTrash {
mw.trashPolicy.CleanTrashPatchCache(mw.getCurrentPath(parentID), name)
}
@ -2032,7 +2034,8 @@ func (mw *MetaWrapper) GetMultipart_ll(path, multipartId string) (info *proto.Mu
return multipartInfo, nil
}
func (mw *MetaWrapper) AddMultipartPart_ll(path, multipartId string, partId uint16, size uint64, md5 string, inodeInfo *proto.InodeInfo,
func (mw *MetaWrapper) AddMultipartPart_ll(path, multipartId string, partId uint16, size uint64, md5 string,
inodeInfo *proto.InodeInfo,
) (oldInode uint64, updated bool, err error) {
var (
mpId uint64

2
sdk/meta/meta.go
View File

@ -277,7 +277,7 @@ func NewMetaWrapper(config *MetaConfig) (*MetaWrapper, error) {
}
func (mw *MetaWrapper) enableTrash() {
if mw.disableTrash == true {
if mw.disableTrash {
return
}
if mw.TrashInterval > 0 {

39
sdk/meta/operation.go
View File

@ -104,8 +104,9 @@ func (mw *MetaWrapper) txIcreate(tx *Transaction, mp *MetaPartition, mode, uid,
return status, resp.Info, nil
}
func (mw *MetaWrapper) quotaIcreate(mp *MetaPartition, mode, uid, gid uint32, target []byte, quotaIds []uint32, fullPath string,
) (status int, info *proto.InodeInfo, err error) {
func (mw *MetaWrapper) quotaIcreate(mp *MetaPartition, mode, uid, gid uint32, target []byte, quotaIds []uint32, fullPath string) (status int,
info *proto.InodeInfo, err error,
) {
bgTime := stat.BeginStat()
defer func() {
stat.EndStat("icreate", err, bgTime, 1)
@ -164,8 +165,9 @@ func (mw *MetaWrapper) quotaIcreate(mp *MetaPartition, mode, uid, gid uint32, ta
return statusOK, resp.Info, nil
}
func (mw *MetaWrapper) icreate(mp *MetaPartition, mode, uid, gid uint32, target []byte, fullPath string,
) (status int, info *proto.InodeInfo, err error) {
func (mw *MetaWrapper) icreate(mp *MetaPartition, mode, uid, gid uint32, target []byte, fullPath string) (status int,
info *proto.InodeInfo, err error,
) {
bgTime := stat.BeginStat()
defer func() {
stat.EndStat("icreate", err, bgTime, 1)
@ -252,7 +254,8 @@ func (mw *MetaWrapper) sendToMetaPartitionWithTx(mp *MetaPartition, req *proto.P
}
func (mw *MetaWrapper) SendTxPack(req proto.TxPack, resp interface{}, Opcode uint8, mp *MetaPartition,
checkStatusFunc func(int, *proto.Packet) error, ignoreExistError bool) (status int, err error, packet *proto.Packet) {
checkStatusFunc func(int, *proto.Packet) error, ignoreExistError bool,
) (status int, err error, packet *proto.Packet) {
packet = proto.NewPacketReqID()
packet.Opcode = Opcode
packet.PartitionID = mp.PartitionID
@ -277,7 +280,7 @@ func (mw *MetaWrapper) SendTxPack(req proto.TxPack, resp interface{}, Opcode uin
return
}
} else if status != statusOK {
if !(status == statusExist && ignoreExistError == false) {
if !(status == statusExist && !ignoreExistError) {
err = errors.New(packet.GetResultMsg())
log.LogErrorf("SendTxPack: packet(%v) mp(%v) req(%v) txInfo(%v) result(%v)",
packet, mp, packet.GetOpMsg(), req.GetInfo(), packet.GetResultMsg())
@ -476,7 +479,8 @@ func (mw *MetaWrapper) ievict(mp *MetaPartition, inode uint64, fullPath string)
}
func (mw *MetaWrapper) txDcreate(tx *Transaction, mp *MetaPartition, parentID uint64, name string, inode uint64,
mode uint32, quotaIds []uint32, fullPath string, ignoreExist bool) (status int, err error) {
mode uint32, quotaIds []uint32, fullPath string, ignoreExist bool,
) (status int, err error) {
bgTime := stat.BeginStat()
defer func() {
stat.EndStat("txDcreate", err, bgTime, 1)
@ -525,7 +529,8 @@ func (mw *MetaWrapper) txDcreate(tx *Transaction, mp *MetaPartition, parentID ui
}
func (mw *MetaWrapper) quotaDcreate(mp *MetaPartition, parentID uint64, name string, inode uint64, mode uint32,
quotaIds []uint32, fullPath string, ignoreExistError bool) (status int, err error) {
quotaIds []uint32, fullPath string, ignoreExistError bool,
) (status int, err error) {
bgTime := stat.BeginStat()
defer func() {
stat.EndStat("dcreate", err, bgTime, 1)
@ -570,7 +575,7 @@ func (mw *MetaWrapper) quotaDcreate(mp *MetaPartition, parentID uint64, name str
if (status != statusOK) && (status != statusExist) {
err = errors.New(packet.GetResultMsg())
log.LogErrorf("quotaDcreate: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg())
} else if status == statusExist && ignoreExistError == false {
} else if status == statusExist && !ignoreExistError {
log.LogWarnf("quotaDcreate: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg())
}
log.LogDebugf("quotaDcreate: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg())
@ -578,7 +583,8 @@ func (mw *MetaWrapper) quotaDcreate(mp *MetaPartition, parentID uint64, name str
}
func (mw *MetaWrapper) dcreate(mp *MetaPartition, parentID uint64, name string, inode uint64, mode uint32,
fullPath string, ignoreExistError bool) (status int, err error) {
fullPath string, ignoreExistError bool,
) (status int, err error) {
bgTime := stat.BeginStat()
defer func() {
stat.EndStat("dcreate", err, bgTime, 1)
@ -622,7 +628,7 @@ func (mw *MetaWrapper) dcreate(mp *MetaPartition, parentID uint64, name string,
if (status != statusOK) && (status != statusExist) {
err = errors.New(packet.GetResultMsg())
log.LogErrorf("dcreate: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg())
} else if status == statusExist && ignoreExistError == false {
} else if status == statusExist && !ignoreExistError {
log.LogWarnf("dcreate: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg())
}
if proto.IsDir(mode) {
@ -896,8 +902,9 @@ func (mw *MetaWrapper) canDeleteInode(mp *MetaPartition, info *proto.InodeInfo,
return true, nil
}
func (mw *MetaWrapper) ddeletes(mp *MetaPartition, parentID uint64, dentries []proto.Dentry, fullPaths []string,
) (status int, resp *proto.BatchDeleteDentryResponse, err error) {
func (mw *MetaWrapper) ddeletes(mp *MetaPartition, parentID uint64, dentries []proto.Dentry, fullPaths []string) (status int,
resp *proto.BatchDeleteDentryResponse, err error,
) {
bgTime := stat.BeginStat()
defer func() {
stat.EndStat("ddeletes", err, bgTime, 1)
@ -2489,7 +2496,8 @@ func (mw *MetaWrapper) updateXAttrs(mp *MetaPartition, inode uint64, filesInc in
return nil
}
func (mw *MetaWrapper) batchSetInodeQuota(mp *MetaPartition, inodes []uint64, quotaId uint32, IsRoot bool,
func (mw *MetaWrapper) batchSetInodeQuota(mp *MetaPartition, inodes []uint64, quotaId uint32,
IsRoot bool,
) (resp *proto.BatchSetMetaserverQuotaResponse, err error) {
bgTime := stat.BeginStat()
defer func() {
@ -2538,7 +2546,8 @@ func (mw *MetaWrapper) batchSetInodeQuota(mp *MetaPartition, inodes []uint64, qu
return
}
func (mw *MetaWrapper) batchDeleteInodeQuota(mp *MetaPartition, inodes []uint64, quotaId uint32,
func (mw *MetaWrapper) batchDeleteInodeQuota(mp *MetaPartition, inodes []uint64,
quotaId uint32,
) (resp *proto.BatchDeleteMetaserverQuotaResponse, err error) {
bgTime := stat.BeginStat()
defer func() {

9
sdk/meta/transaction_helper.go
View File

@ -70,7 +70,8 @@ func NewCreateTransaction(parentMp, inoMp *MetaPartition, parentID uint64, name
func NewDeleteTransaction(
denMp *MetaPartition, parentID uint64, name string,
inoMp *MetaPartition, ino uint64, txTimeout int64) (tx *Transaction, err error) {
inoMp *MetaPartition, ino uint64, txTimeout int64,
) (tx *Transaction, err error) {
tx = NewTransaction(txTimeout, proto.TxTypeRemove)
denMembers := getMembersFromMp(denMp)
@ -98,7 +99,8 @@ func NewDeleteTransaction(
}
func NewRenameTransaction(srcMp *MetaPartition, srcDenParentID uint64, srcName string,
dstMp *MetaPartition, dstDenParentID uint64, dstName string, txTimeout int64) (tx *Transaction, err error) {
dstMp *MetaPartition, dstDenParentID uint64, dstName string, txTimeout int64,
) (tx *Transaction, err error) {
tx = NewTransaction(txTimeout, proto.TxTypeRename)
srcMembers := getMembersFromMp(srcMp)
@ -139,7 +141,8 @@ func RenameTxReplaceInode(tx *Transaction, inoMp *MetaPartition, ino uint64) (er
func NewLinkTransaction(
denMp *MetaPartition, parentID uint64, name string,
inoMp *MetaPartition, ino uint64, txTimeout int64) (tx *Transaction, err error) {
inoMp *MetaPartition, ino uint64, txTimeout int64,
) (tx *Transaction, err error) {
tx = NewTransaction(txTimeout, proto.TxTypeLink)
denMembers := getMembersFromMp(denMp)

149
sdk/meta/trash.go
View File

@ -12,15 +12,17 @@ import (
"time"
"github.com/cubefs/cubefs/proto"
"github.com/google/uuid"
// "syscall"
"github.com/cubefs/cubefs/util/errors"
"github.com/cubefs/cubefs/util/log"
"github.com/google/uuid"
)
// put deleted files or directories into this folder under Trash
const CurrentName = "Current"
const (
CurrentName = "Current"
TrashPrefix = ".Trash"
ExpiredPrefix = "Expired"
ParentDirPrefix = "|__|"
@ -41,7 +43,6 @@ const (
type Trash struct {
mw *MetaWrapper
mountPath string
mountPoint string
trashRoot string
trashRootIno uint64
deleteInterval int64
@ -169,34 +170,6 @@ func (trash *Trash) CleanTrashPatchCache(parentPathAbsolute string, fileName str
log.LogDebugf("CleanTrashPatchCache: CleanTrashPatchCache(%v) ", dstPath)
}
func (trash *Trash) findFileFromExpired(fileName string) (info *proto.InodeInfo, err error) {
log.LogDebugf("action[findFileFromExpired]find %v", fileName)
entries, err := trash.mw.ReadDir_ll(trash.trashRootIno)
if err != nil {
log.LogWarnf("action[findFileFromExpired]ReadDir trashRoot failed: %v", err.Error())
return nil, err
}
for _, entry := range entries {
if !proto.IsDir(entry.Type) {
continue
}
// skip current
if strings.Compare(entry.Name, CurrentName) == 0 {
continue
}
dstPath := path.Join(trash.trashRoot, entry.Name, fileName)
log.LogDebugf("action[findFileFromExpired]try find %v", dstPath)
info, err = trash.LookupPath(dstPath, false)
if err != nil {
log.LogDebugf("action[findFileFromExpired]find %v failed: %v", dstPath, err.Error())
continue
} else {
return info, nil
}
}
return nil, err
}
func (trash *Trash) MoveToTrash(parentPathAbsolute string, parentIno uint64, fileName string, isDir bool) (err error) {
start := time.Now()
defer func() {
@ -262,12 +235,12 @@ func (trash *Trash) MoveToTrash(parentPathAbsolute string, parentIno uint64, fil
return
}
err = trash.mw.XAttrSet_ll(info.Inode, []byte(OriginalName), []byte(originName))
err = trash.mw.XAttrSet_ll(info.Inode, []byte(OriginalName), []byte(name))
if err != nil {
log.LogWarnf("action[MoveToTrash] set xattr for %v[%v] failed:%v", dstPath, info.Inode, err.Error())
return
}
log.LogDebugf("action[MoveToTrash] set xattr for %v [%v]success:%v", dstPath, info.Inode, originName)
log.LogDebugf("action[MoveToTrash] set xattr for %v [%v]success:%v", dstPath, info.Inode, name)
}(originName, dstPath, trashCurrentIno)
}
// nil to check tmp file exist
@ -279,10 +252,8 @@ func (trash *Trash) MoveToTrash(parentPathAbsolute string, parentIno uint64, fil
func transferLongFileName(filePath string) (newName, oldName string) {
oldName = path.Base(filePath)
parentPath := path.Dir(filePath)
if strings.HasPrefix(oldName, ParentDirPrefix) {
newName = strings.TrimPrefix(oldName, ParentDirPrefix)
}
newName = strings.ReplaceAll(oldName, ParentDirPrefix, "/")
newName = strings.ReplaceAll(newName, ParentDirPrefix, "/")
newBaseName := path.Base(newName)
return path.Join(parentPath, LongNamePrefix+newBaseName+ParentDirPrefix+uuid.New().String()), oldName
}
@ -364,8 +335,8 @@ func (trash *Trash) renameCurrent() {
return
}
checkPointInterval := trash.deleteInterval / 4
if time.Now().Sub(inoInfo.CreateTime) < (time.Duration(checkPointInterval) * time.Minute) {
log.LogDebugf("action[renameCurrent]trashCurrent keep for 1/4 interval %v", time.Now().Sub(inoInfo.CreateTime).String())
if time.Since(inoInfo.CreateTime) < (time.Duration(checkPointInterval) * time.Minute) {
log.LogDebugf("action[renameCurrent]trashCurrent keep for 1/4 interval %v", time.Since(inoInfo.CreateTime).String())
return
}
// ensure files in current is rebuild
@ -437,7 +408,6 @@ func (trash *Trash) removeAll(dirName string, dirIno uint64) {
}
batchNr := uint64(len(batches))
if batchNr == 0 || (from != "" && batchNr == 1) {
noMore = true
break
} else if batchNr < DefaultReaddirLimit {
noMore = true
@ -468,92 +438,18 @@ func (trash *Trash) removeAll(dirName string, dirIno uint64) {
wg.Wait()
from = batches[len(batches)-1].Name
}
noMore = false
from = ""
for !noMore {
batches, err := trash.mw.ReadDirLimit_ll(dirIno, from, DefaultReaddirLimit)
if err != nil {
log.LogErrorf("action[removeAll] ReadDirLimit_ll: ino(%v) err(%v) from(%v)", dirIno, err, from)
return
}
batchNr := uint64(len(batches))
if batchNr == 0 || (from != "" && batchNr == 1) {
noMore = true
break
} else if batchNr < DefaultReaddirLimit {
noMore = true
}
if from != "" {
batches = batches[1:]
}
for _, entry := range batches {
select {
case trash.traverseDirGoroutineLimit <- true:
wg.Add(1)
go func(parentIno uint64, entry string, isDir bool, fullPath string) {
defer wg.Done()
trash.deleteTask(parentIno, entry, isDir, fullPath)
}(dirIno, entry.Name, proto.IsDir(entry.Type), path.Join(dirName, entry.Name))
default:
trash.deleteTask(dirIno, entry.Name, proto.IsDir(entry.Type), path.Join(dirName, entry.Name))
}
}
wg.Wait()
from = batches[len(batches)-1].Name
}
// entries, err := trash.mw.ReadDir_ll(dirIno)
// if err != nil {
// log.LogWarnf("action[deleteDir]delete %v failed: %v", dirName, err)
// return
// }
// delete sub files
// for _, entry := range entries {
// log.LogDebugf("action[deleteDir]traverse %v", entry.Name)
// if !proto.IsDir(entry.Type) {
// continue
// }
// trash.mw.AddInoInfoCache(entry.Inode, dirIno, entry.Name)
// select {
// case trash.traverseDirGoroutineLimit <- true:
// log.LogDebugf("action[deleteDir]launch goroutine %v", entry.Name)
// wg.Add(1)
// go func(dirName string, dirIno uint64) {
// defer wg.Done()
// trash.removeAll(dirName, dirIno)
// trash.releaseTraverseToken()
// }(entry.Name, entry.Inode)
// default:
// log.LogDebugf("action[deleteDir]execute local %v", entry.Name)
// trash.removeAll(entry.Name, entry.Inode)
// }
// }
// wg.Wait()
// //all sub files is deleted
// for _, entry := range entries {
// select {
// case trash.traverseDirGoroutineLimit <- true:
// wg.Add(1)
// go func(parentIno uint64, entry string, isDir bool) {
// defer wg.Done()
// trash.deleteTask(parentIno, entry, isDir)
// }(dirIno, entry.Name, proto.IsDir(entry.Type))
// default:
// trash.deleteTask(dirIno, entry.Name, proto.IsDir(entry.Type))
// }
// }
// wg.Wait()
log.LogDebugf("action[deleteDir] delete complete %v", dirName)
}
func (trash *Trash) extractTimeStampFromName(fileName string) (err error, timeStamp int64) {
subs := strings.Split(fileName, "_")
if len(subs) != 2 {
return errors.New(fmt.Sprintf("fileName format is not valid")), 0
return errors.New("fileName format is not valid"), 0
}
parsedTime, err := time.ParseInLocation(ExpiredTimeFormat, subs[1], time.Local)
if err != nil {
return errors.New(fmt.Sprintf("fileName format is not valid")), 0
return errors.New("fileName format is not valid"), 0
}
return nil, parsedTime.Unix()
}
@ -590,10 +486,7 @@ func (trash *Trash) pathIsExistInTrash(filePath string) bool {
} else {
currentIno := info.Inode
_, _, err := trash.mw.Lookup_ll(currentIno, path.Base(filePath))
if err != nil {
return false
}
return true
return err == nil
}
}
@ -706,7 +599,6 @@ func (trash *Trash) createParentPathInTrash(parentPath, rootDir string) (err err
}
if info == nil {
panic(fmt.Sprintf("info should not be nil for parentPath %v", parentPath))
return
}
parentInfo, err = trash.CreateDirectory(parentIno, sub, info.Mode, info.Uid, info.Gid, path.Join(parentPath, sub), true)
if err != nil {
@ -765,18 +657,6 @@ func (trash *Trash) rename(oldPath, newPath string) error {
return trash.mw.Rename_ll(oldInfo.Inode, path.Base(oldPath), newInfo.Inode, path.Base(newPath), oldPath, newPath, true)
}
func (trash *Trash) deleteSrcDir(dirPath string) error {
parentDir := path.Dir(dirPath)
parentInfo, err := trash.LookupPath(parentDir, true)
if err != nil {
log.LogDebugf("action[deleteSrcDir] lookup %v failed %v", parentDir, err.Error())
return err
}
_, err = trash.mw.Delete_ll(parentInfo.Inode, path.Base(dirPath), true, dirPath)
return err
}
func (trash *Trash) IsTrashRoot(parentIno uint64, name string) bool {
info, _ := trash.LookupPath(trash.mountPath, true)
if info.Inode == parentIno && name == TrashPrefix {
@ -912,7 +792,6 @@ func (trash *Trash) buildDeletedFileParentDirs() {
}
batchNr := uint64(len(batches))
if batchNr == 0 || (from != "" && batchNr == 1) {
noMore = true
break
} else if batchNr < DefaultReaddirLimit {
noMore = true
@ -1075,9 +954,7 @@ func (trash *Trash) recoverPosixPathName(fileName string, fileIno uint64) string
log.LogDebugf("action[recoverPosixPathName] fileName %v is read from xattr ", fileName)
}
}
if strings.HasPrefix(fileName, ParentDirPrefix) {
fileName = strings.TrimPrefix(fileName, ParentDirPrefix)
}
fileName = strings.ReplaceAll(fileName, ParentDirPrefix, "/")
return fileName
}

23
storage/extent.go
View File

@ -88,6 +88,15 @@ func (ei *ExtentInfo) MarshalBinaryWithBuffer(buff *bytes.Buffer) (err error) {
if err = binary.Write(buff, binary.BigEndian, ei.AccessTime); err != nil {
return
}
if err = binary.Write(buff, binary.BigEndian, ei.SnapPreAllocDataOff); err != nil {
return
}
if err = binary.Write(buff, binary.BigEndian, ei.SnapshotDataOff); err != nil {
return
}
if err = binary.Write(buff, binary.BigEndian, ei.ApplyID); err != nil {
return
}
return
}
@ -116,6 +125,18 @@ func (ei *ExtentInfo) UnmarshalBinaryWithBuffer(buff *bytes.Buffer) (err error)
if err = binary.Read(buff, binary.BigEndian, &ei.AccessTime); err != nil {
return
}
if buff.Len() > 0 {
if err = binary.Read(buff, binary.BigEndian, &ei.SnapPreAllocDataOff); err != nil {
return
}
if err = binary.Read(buff, binary.BigEndian, &ei.SnapshotDataOff); err != nil {
return
}
if err = binary.Read(buff, binary.BigEndian, &ei.ApplyID); err != nil {
return
}
}
return
}
@ -261,7 +282,7 @@ func (e *Extent) GetDataSize(statSize int64) (dataSize int64) {
// RestoreFromFS restores the entity data and status from the file stored on the filesystem.
func (e *Extent) RestoreFromFS() (err error) {
if e.file, err = os.OpenFile(e.filePath, os.O_RDWR, 0666); err != nil {
if e.file, err = os.OpenFile(e.filePath, os.O_RDWR, 0o666); err != nil {
if os.IsNotExist(err) {
err = ExtentNotFoundError
}

111
storage/extent_store.go
View File

@ -20,12 +20,9 @@ import (
"fmt"
"hash/crc32"
"io"
"io/fs"
"os"
"path"
"time"
"hash/crc32"
"io"
"regexp"
"runtime"
"sort"
@ -70,9 +67,7 @@ const (
StaleExtStoreTimeFormat = "20060102150405.000000000"
)
var (
ErrStoreAlreadyClosed = errors.New("extent store already closed")
)
var ErrStoreAlreadyClosed = errors.New("extent store already closed")
var (
RegexpExtentFile, _ = regexp.Compile(`^(\d)+$`)
@ -208,6 +203,23 @@ func NewExtentStore(dataDir string, partitionID uint64, storeSize, dpType int, i
}
}
aId := 0
var vFp *os.File
for {
dataPath := path.Join(s.dataPath, ExtCrcHeaderFileName+"_"+strconv.Itoa(aId))
if _, err = os.Stat(dataPath); err != nil {
log.LogDebugf("NewExtentStore. partitionID [%v] dataPath not exist err %v. verifyExtentFpAppend init return", partitionID, err)
break
}
if vFp, err = os.OpenFile(dataPath, os.O_CREATE|os.O_RDWR, 0o666); err != nil {
log.LogErrorf("NewExtentStore. partitionID [%v] dataPath exist but open err %v. verifyExtentFpAppend init return", partitionID, err)
return
}
log.LogDebugf("NewExtentStore. partitionID [%v] dataPath exist and opened id %v", partitionID, aId)
s.verifyExtentFpAppend = append(s.verifyExtentFpAppend, vFp)
aId++
}
stat, err := s.tinyExtentDeleteFp.Stat()
if err != nil {
return
@ -220,7 +232,7 @@ func NewExtentStore(dataDir string, partitionID uint64, storeSize, dpType int, i
}
s.extentInfoMap = make(map[uint64]*ExtentInfo)
s.extentLockMap = make(map[uint64]proto.GcFlag, 0)
s.extentLockMap = make(map[uint64]proto.GcFlag)
s.cache = NewExtentCache(100)
if err = s.initBaseFileID(); err != nil {
err = fmt.Errorf("init base field ID: %v", err)
@ -387,8 +399,8 @@ func (s *ExtentStore) GetExtentInfoFromDisk(id uint64) (ei *ExtentInfo, err erro
}
if IsTinyExtent(id) {
watermark := ei.Size
if watermark%PageSize != 0 {
watermark = watermark + (PageSize - watermark%PageSize)
if watermark%util.PageSize != 0 {
watermark = watermark + (util.PageSize - watermark%util.PageSize)
}
ei.Size = watermark
}
@ -457,7 +469,7 @@ func (s *ExtentStore) writeReadDirHint() (err error) {
log.LogErrorf("[writeReadDirHint] store(%v) failed to marshal hint, err(%v)", s.dataPath, err)
return
}
if err = os.WriteFile(hintTempPath, buff.Bytes(), 0666); err != nil {
if err = os.WriteFile(hintTempPath, buff.Bytes(), 0o666); err != nil {
log.LogErrorf("[writeReadDirHint] store(%v) failed to write readdir hint, err(%v)", s.dataPath, err)
return
}
@ -529,9 +541,7 @@ func (s *ExtentStore) initBaseFileID() error {
defer func() {
log.LogInfof("[initBaseFileID] store(%v) init base file id using time(%v), count(%v)", s.dataPath, time.Since(begin), extNum)
}()
var (
baseFileID uint64
)
var baseFileID uint64
baseFileID, _ = s.GetPersistenceBaseExtentID()
log.LogInfof("[initBaseFileID] store(%v) init base file to persistence base extent id using time(%v)", s.dataPath, time.Since(begin))
@ -541,7 +551,6 @@ func (s *ExtentStore) initBaseFileID() error {
extMap, err = s.readReadDirHint()
if err != nil {
log.LogErrorf("[initBaseFileID] store(%v) failed to read hint, err(%v)", s.dataPath, err)
err = nil
}
// NOTE: remove hint
if err = s.removeReadDirHint(); err != nil {
@ -611,17 +620,10 @@ func (s *ExtentStore) Write(extentID uint64, offset, size int64, data []byte, cr
ei *ExtentInfo
)
ei, _ = s.GetExtentInfo(extentID)
e, err = s.extentWithHeader(ei)
s.eiMutex.Unlock()
if err != nil {
return status, err
}
s.elMutex.RLock()
if isBackupWrite {
// NOTE: meet an error is impossible
_, ok = s.GetExtentInfo(extentID)
_, ok := s.GetExtentInfo(extentID)
if !ok {
s.elMutex.RUnlock()
err = fmt.Errorf("extent(%v) is not locked", extentID)
@ -772,12 +774,8 @@ func (s *ExtentStore) punchDelete(extentID uint64, offset, size int64) (err erro
return
}
func (s *ExtentStore) CanGcDelete(extId uint64) (ok bool, err error) {
func (s *ExtentStore) CanGcDelete(extId uint64) (ok bool) {
ei, _ := s.GetExtentInfo(extId)
if err != nil {
log.LogErrorf("[IsMarkGc] failed to get extent(%v) info, err(%v)", extId, err)
return
}
if ei == nil || ei.IsDeleted {
return true
}
@ -810,12 +808,10 @@ func (s *ExtentStore) MarkDelete(extentID uint64, offset, size int64) (err error
return
}
var (
ei *ExtentInfo
)
var ei *ExtentInfo
if IsTinyExtent(extentID) {
return s.tinyDelete(extentID, offset, size)
return s.punchDelete(extentID, offset, size)
}
ei, _ = s.GetExtentInfo(extentID)
@ -941,14 +937,8 @@ func (s *ExtentStore) Close() {
// Watermark returns the extent info of the given extent on the record.
func (s *ExtentStore) Watermark(extentID uint64) (ei *ExtentInfo, err error) {
var (
has bool
)
var has bool
ei, has = s.GetExtentInfo(extentID)
if err != nil {
log.LogErrorf("[Watermark] failed to get extent(%v) watermark, err(%v)", extentID, err)
return
}
if !has {
err = fmt.Errorf("e %v not exist", s.getExtentKey(extentID))
return
@ -1085,10 +1075,6 @@ func (s *ExtentStore) getTinyExtentInfo() (extents []*ExtentInfo) {
for extentID = TinyExtentStartID; extentID < TinyExtentCount+TinyExtentStartID; extentID++ {
var ei *ExtentInfo
ei, _ = s.GetExtentInfo(extentID)
if err != nil {
log.LogErrorf("[getTinyExtentInfo] failed to get extent(%v) info, err(%v)", extentID, err)
return
}
if ei == nil {
continue
}
@ -1354,14 +1340,6 @@ func (s *ExtentStore) UpdateBaseExtentID(id uint64) (err error) {
return
}
func (s *ExtentStore) extent(extentID uint64) (e *Extent, err error) {
if e, err = s.LoadExtentFromDisk(extentID, false); err != nil {
err = fmt.Errorf("load extent from disk: %v", err)
return nil, err
}
return
}
func (s *ExtentStore) extentWithHeader(ei *ExtentInfo) (e *Extent, err error) {
var ok bool
if ei == nil || ei.IsDeleted {
@ -1595,9 +1573,7 @@ func (s *ExtentStore) TinyExtentRecover(extentID uint64, offset, size int64, dat
}
func (s *ExtentStore) TinyExtentGetFinfoSize(extentID uint64) (size uint64, err error) {
var (
e *Extent
)
var e *Extent
if !IsTinyExtent(extentID) {
return 0, fmt.Errorf("unavali extent id (%v)", extentID)
}
@ -1619,16 +1595,29 @@ func (s *ExtentStore) TinyExtentGetFinfoSize(extentID uint64) (size uint64, err
return
}
func (s *ExtentStore) GetExtentWithHoleAvailableOffset(extentID uint64, offset int64) (newOffset, newEnd int64, err error) {
func (s *ExtentStore) GetExtentFinfoSize(extentID uint64) (size uint64, err error) {
var e *Extent
if !IsTinyExtent(extentID) {
return 0, 0, fmt.Errorf("unavali extent(%v)", extentID)
}
ei, _ := s.GetExtentInfo(extentID)
if err != nil {
log.LogErrorf("[TinyExtentGetFinfoSize] failed to get extent(%v) info, err(%v)", extentID, err)
s.eiMutex.RLock()
ei := s.extentInfoMap[extentID]
s.eiMutex.RUnlock()
if e, err = s.extentWithHeader(ei); err != nil {
return
}
finfo, err := e.file.Stat()
if err != nil {
return 0, err
}
size = uint64(finfo.Size())
return
}
func (s *ExtentStore) GetExtentWithHoleAvailableOffset(extentID uint64, offset int64) (newOffset, newEnd int64, err error) {
var e *Extent
s.eiMutex.RLock()
ei := s.extentInfoMap[extentID]
s.eiMutex.RUnlock()
if e, err = s.extentWithHeader(ei); err != nil {
return
}

14
storage/extent_store_test.go
View File

@ -40,15 +40,15 @@ func extentStoreNormalRwTest(t *testing.T, s *storage.ExtentStore, id uint64) {
data := []byte(dataStr)
crc := crc32.ChecksumIEEE(data)
// append write
_, err := s.Write(id, 0, int64(len(data)), data, crc, storage.AppendWriteType, true, false, false)
_, err := s.Write(id, 0, int64(len(data)), data, crc, storage.AppendWriteType, true, false, false, false)
require.NoError(t, err)
actualCrc, err := s.Read(id, 0, int64(len(data)), data, false)
actualCrc, err := s.Read(id, 0, int64(len(data)), data, false, false)
require.NoError(t, err)
require.EqualValues(t, crc, actualCrc)
// random write
_, err = s.Write(id, 0, int64(len(data)), data, crc, storage.RandomWriteType, true, false, false)
_, err = s.Write(id, 0, int64(len(data)), data, crc, storage.RandomWriteType, true, false, false, false)
require.NoError(t, err)
actualCrc, err = s.Read(id, 0, int64(len(data)), data, false)
actualCrc, err = s.Read(id, 0, int64(len(data)), data, false, false)
require.NoError(t, err)
require.EqualValues(t, crc, actualCrc)
// TODO: append random write
@ -93,7 +93,7 @@ func extentMarkDeleteTinyTest(t *testing.T, s *storage.ExtentStore, id uint64) {
require.NotEqualValues(t, size, 0)
// write second file to extent
crc := crc32.ChecksumIEEE(data)
_, err = s.Write(id, size, int64(len(data)), data, crc, storage.AppendWriteType, true, false, false)
_, err = s.Write(id, size, int64(len(data)), data, crc, storage.AppendWriteType, true, false, false, false)
require.NoError(t, err)
// mark delete first file
extentStoreMarkDeleteTiny(t, s, id, 0, size)
@ -155,7 +155,7 @@ func reopenExtentStoreTest(t *testing.T, dpType int) {
data := []byte(dataStr)
crc := crc32.ChecksumIEEE(data)
// write some data
_, err = s.Write(id, 0, int64(len(data)), data, crc, storage.AppendWriteType, true, false, false)
_, err = s.Write(id, 0, int64(len(data)), data, crc, storage.AppendWriteType, true, false, false, false)
require.NoError(t, err)
firstSnap, err := s.SnapShot()
require.NoError(t, err)
@ -164,7 +164,7 @@ func reopenExtentStoreTest(t *testing.T, dpType int) {
require.NoError(t, err)
defer newStor.Close()
// read data
actualCrc, err := newStor.Read(id, 0, int64(len(data)), data, false)
actualCrc, err := newStor.Read(id, 0, int64(len(data)), data, false, false)
require.NoError(t, err)
require.EqualValues(t, crc, actualCrc)
secondSnap, err := newStor.SnapShot()

3
util/auditlog/auditlog.go
View File

@ -293,9 +293,6 @@ func (a *Audit) formatMetaAudit(ipAddr, hostName, op, src, dst string, err error
} else {
errStr = "nil"
}
curTime := time.Now()
curTimeStr := curTime.Format("2006-01-02 15:04:05.000000")
timeZone, _ := curTime.Zone()
latencyStr := strconv.FormatInt(latency, 10) + " us"
srcInodeStr := strconv.FormatUint(srcInode, 10)
dstInodeStr := strconv.FormatUint(dstInode, 10)

21
util/btree/btree.go
View File

@ -20,7 +20,9 @@
// It has a flatter structure than an equivalent red-black or other binary tree,
// which in some cases yields better memory usage and/or performance.
// See some discussion on the matter here:
//
// http://google-opensource.blogspot.com/2013/01/c-containers-that-save-memory-and-time.html
//
// Note, though, that this project is in no way related to the C++ B-Tree
// implementation written about there.
//
@ -28,13 +30,14 @@
// slice of children. For basic numeric values or raw structs, this can cause
// efficiency differences when compared to equivalent C++ template code that
// stores values in arrays within the node:
// * Due to the overhead of storing values as interfaces (each
// - Due to the overhead of storing values as interfaces (each
// value needs to be stored as the value itself, then 2 words for the
// interface pointing to that value and its type), resulting in higher
// memory use.
// * Since interfaces can point to values anywhere in memory, values are
// - Since interfaces can point to values anywhere in memory, values are
// most likely not stored in contiguous blocks, resulting in a higher
// number of cache misses.
//
// These issues don't tend to matter, though, when working with strings or other
// heap-allocated structures, since C++-equivalent structures also must store
// pointers and also distribute their values across the heap.
@ -253,8 +256,8 @@ func (s *children) truncate(index int) {
// node is an internal node in a tree.
//
// It must at all times maintain the invariant that either
// * len(children) == 0, len(items) unconstrained
// * len(children) == len(items) + 1
// - len(children) == 0, len(items) unconstrained
// - len(children) == len(items) + 1
type node struct {
items items
children children
@ -453,15 +456,20 @@ func (n *node) remove(item Item, minItems int, typ toRemove) Item {
// remove it.
//
// Most documentation says we have to do two sets of special casing:
// 1) item is in this node
// 2) item is in child
// 1. item is in this node
// 2. item is in child
//
// In both cases, we need to handle the two subcases:
//
// A) node has enough values that it can spare one
// B) node doesn't have enough values
//
// For the latter, we have to check:
//
// a) left sibling has node to spare
// b) right sibling has node to spare
// c) we must merge
//
// To simplify our code here, we handle cases #1 and #2 the same:
// If a node doesn't have enough items, we make sure it does (using a,b,c).
// We then simply redo our remove call, and the second time (regardless of
@ -900,6 +908,7 @@ func (t *BTree) Len() int {
// one, instead of being lost to the garbage collector.
//
// This call takes:
//
// O(1): when addNodesToFreelist is false, this is a single operation.
// O(1): when the freelist is already full, it breaks out immediately
// O(freelist size): when the freelist is empty and the nodes are all owned

4
util/fileutil/readdir_test.go
View File

@ -30,10 +30,10 @@ func TestReaddir(t *testing.T) {
file1 := "f1"
file2 := "f2"
err = os.WriteFile(path.Join(tmpDir, file1), []byte(""), 0644)
err = os.WriteFile(path.Join(tmpDir, file1), []byte(""), 0o644)
require.NoError(t, err)
os.WriteFile(path.Join(tmpDir, file2), []byte(""), 0644)
os.WriteFile(path.Join(tmpDir, file2), []byte(""), 0o644)
require.NoError(t, err)
names, err := fileutil.ReadDir(tmpDir)

4
util/log/log.go
View File

@ -328,9 +328,7 @@ func InitLog(dir, module string, level Level, rotate *LogRotate, logLeftSpaceLim
}
if rotate.headRoom == 0 {
var minLogLeftSpaceLimit float64
minLogLeftSpaceLimit = float64(fs.Blocks*uint64(fs.Bsize)) * logLeftSpaceLimitRatio / 1024 / 1024
minLogLeftSpaceLimit := float64(fs.Blocks*uint64(fs.Bsize)) * logLeftSpaceLimitRatio / 1024 / 1024
rotate.SetHeadRoomMb(int64(math.Min(minLogLeftSpaceLimit, DefaultHeadRoom)))
}

8
util/mocktest/raftstore/partition.go
View File

@ -82,14 +82,6 @@ func (mr *MockPartitionMockRecorder) CommittedIndex() *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CommittedIndex", reflect.TypeOf((*MockPartition)(nil).CommittedIndex))
}
// IsRestoring mocks base method.
func (m *MockPartition) IsRestoring() bool {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "IsRestoring")
ret0, _ := ret[0].(bool)
return ret0
}
// IsRestoring indicates an expected call of IsRestoring.
func (mr *MockPartitionMockRecorder) IsRestoring() *gomock.Call {
mr.mock.ctrl.T.Helper()