gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 02:00:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

style(all): format all codes with golangci tools

Browse Source 
closes #3371

@formatter:off

Signed-off-by: slasher <shenjie1@oppo.com>
This commit is contained in:
slasher 2024-05-13 10:39:56 +08:00 committed by 梁曟風
parent c2ea1de3d4
commit 5c7898354f
196 changed files with 754 additions and 1591 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
authnode/api_service.go
View File

@ -29,10 +29,6 @@ import (
"github.com/cubefs/cubefs/util/log"
)
const (
nodeType = "auth"
)
func (m *Server) getTicket(w http.ResponseWriter, r *http.Request) {
var (
plaintext []byte
@ -43,7 +39,7 @@ func (m *Server) getTicket(w http.ResponseWriter, r *http.Request) {
message string
)
if m.metaReady == false {
if !m.metaReady {
log.LogWarnf("action[handlerWithInterceptor] leader meta has not ready")
http.Error(w, m.leaderInfo.addr, http.StatusBadRequest)
}
@ -79,7 +75,6 @@ func (m *Server) getTicket(w http.ResponseWriter, r *http.Request) {
}
sendOkReply(w, r, newSuccessHTTPAuthReply(message))
return
}
func (m *Server) raftNodeOp(w http.ResponseWriter, r *http.Request) {
@ -145,21 +140,14 @@ func (m *Server) raftNodeOp(w http.ResponseWriter, r *http.Request) {
}
sendOkReply(w, r, newSuccessHTTPAuthReply(message))
return
}
func (m *Server) handleAddRaftNode(raftNodeInfo *proto.AuthRaftNodeInfo) (err error) {
if err = m.cluster.addRaftNode(raftNodeInfo.ID, raftNodeInfo.Addr); err != nil {
return
}
return
return m.cluster.addRaftNode(raftNodeInfo.ID, raftNodeInfo.Addr)
}
func (m *Server) handleRemoveRaftNode(raftNodeInfo *proto.AuthRaftNodeInfo) (err error) {
if err = m.cluster.removeRaftNode(raftNodeInfo.ID, raftNodeInfo.Addr); err != nil {
return
}
return
return m.cluster.removeRaftNode(raftNodeInfo.ID, raftNodeInfo.Addr)
}
func genAuthRaftNodeOpResp(req *proto.APIAccessReq, ts int64, key []byte, msg string) (message string, err error) {
@ -289,28 +277,18 @@ func (m *Server) apiAccessEntry(w http.ResponseWriter, r *http.Request) {
}
sendOkReply(w, r, newSuccessHTTPAuthReply(message))
return
}
func (m *Server) handleCreateKey(keyInfo *keystore.KeyInfo) (res *keystore.KeyInfo, err error) {
if res, err = m.cluster.CreateNewKey(keyInfo.ID, keyInfo); err != nil {
return
}
return
return m.cluster.CreateNewKey(keyInfo.ID, keyInfo)
}
func (m *Server) handleDeleteKey(keyInfo *keystore.KeyInfo) (res *keystore.KeyInfo, err error) {
if res, err = m.cluster.DeleteKey(keyInfo.ID); err != nil {
return
}
return
return m.cluster.DeleteKey(keyInfo.ID)
}
func (m *Server) handleGetKey(keyInfo *keystore.KeyInfo) (res *keystore.KeyInfo, err error) {
if res, err = m.getSecretKeyInfo(keyInfo.ID); err != nil {
return
}
return
return m.getSecretKeyInfo(keyInfo.ID)
}
func (m *Server) handleAddCaps(keyInfo *keystore.KeyInfo) (res *keystore.KeyInfo, err error) {
@ -437,7 +415,6 @@ func (m *Server) osCapsOp(w http.ResponseWriter, r *http.Request) {
}
sendOkReply(w, r, newSuccessHTTPAuthReply(message))
return
}
func (m *Server) genTicket(key []byte, serviceID string, IP string, caps []byte) (ticket cryptoutil.Ticket) {
@ -680,7 +657,6 @@ func send(w http.ResponseWriter, r *http.Request, reply []byte) {
return
}
log.LogInfof("URL[%v],remoteAddr[%v],response ok", r.URL, r.RemoteAddr)
return
}
func keyNotFound(name string) (err error) {
@ -700,5 +676,4 @@ func sendErrReply(w http.ResponseWriter, r *http.Request, HTTPAuthReply *proto.H
if _, err = w.Write(reply); err != nil {
log.LogErrorf("fail to write http reply[%s] len[%d].URL[%v],remoteAddr[%v] err:[%v]", string(reply), len(reply), r.URL, r.RemoteAddr, err)
}
return
}

3
authnode/authnode_manager.go
View File

@ -36,7 +36,7 @@ func (m *Server) handleLeaderChange(leader uint64) {
log.LogWarnf("action[handleLeaderChange] change leader to [%v] ", m.leaderInfo.addr)
m.authProxy = m.newAuthProxy() // TODO no lock?
if m.metaReady == false {
if !m.metaReady {
if err := m.cluster.loadKeystore(); err != nil {
panic(err)
}
@ -73,5 +73,4 @@ func (m *Server) handlePeerChange(confChange *proto.ConfChange) (err error) {
func (m *Server) handleApplySnapshot() {
log.LogInfof("clusterID[%v] peerID:%v action[handleApplySnapshot]", m.clusterName, m.id)
m.fsm.restore()
return
}

8
authnode/cluster.go
View File

@ -57,8 +57,8 @@ func newCluster(name string, leaderInfo *LeaderInfo, fsm *KeystoreFsm, partition
c.cfg = cfg
c.fsm = fsm
c.partition = partition
c.fsm.keystore = make(map[string]*keystore.KeyInfo, 0)
c.fsm.accessKeystore = make(map[string]*keystore.AccessKeyInfo, 0)
c.fsm.keystore = make(map[string]*keystore.KeyInfo)
c.fsm.accessKeystore = make(map[string]*keystore.AccessKeyInfo)
return
}
@ -66,10 +66,6 @@ func (c *Cluster) scheduleTask() {
c.scheduleToCheckHeartbeat()
}
func (c *Cluster) authAddr() (addr string) {
return c.leaderInfo.addr
}
func (c *Cluster) scheduleToCheckHeartbeat() {
go func() {
for {

6
authnode/const.go
View File

@ -37,3 +37,9 @@ const (
akAcronym = "ak"
akPrefix = keySeparator + akAcronym + keySeparator
)
// TODO: unused
var (
_ = opSyncGetKey
_ = opSyncGetCaps
)

2
authnode/http_server.go
View File

@ -54,7 +54,6 @@ func (m *Server) startHTTPService() {
}
}
}()
return
}
func (m *Server) newAuthProxy() *AuthProxy {
@ -122,7 +121,6 @@ func (m *Server) handleFunctions() {
http.Handle(proto.OSAddCaps, m.handlerWithInterceptor())
http.Handle(proto.OSDeleteCaps, m.handlerWithInterceptor())
http.Handle(proto.OSGetCaps, m.handlerWithInterceptor())
return
}
func (m *Server) handlerWithInterceptor() http.Handler {

2
authnode/keystore_cache_op.go
View File

@ -30,7 +30,6 @@ func (mf *KeystoreFsm) DeleteKey(id string) {
mf.ksMutex.Lock()
defer mf.ksMutex.Unlock()
delete(mf.keystore, id)
return
}
func (mf *KeystoreFsm) PutAKInfo(akInfo *keystore.AccessKeyInfo) {
@ -55,5 +54,4 @@ func (mf *KeystoreFsm) DeleteAKInfo(accessKey string) {
mf.aksMutex.Lock()
defer mf.aksMutex.Unlock()
delete(mf.accessKeystore, accessKey)
return
}

2
authnode/keystore_fsm.go
View File

@ -37,8 +37,6 @@ type raftLeaderChangeHandler func(leader uint64)
type raftPeerChangeHandler func(confChange *proto.ConfChange) (err error)
type raftCmdApplyHandler func(cmd *RaftCmd) (err error)
type raftApplySnapshotHandler func()
// KeystoreFsm represents the finite state machine of a keystore

16
authnode/keystore_fsm_op.go
View File

@ -118,7 +118,7 @@ func (c *Cluster) syncPutAccessKeyInfo(opType uint32, accessKeyInfo *keystore.Ac
}
func (c *Cluster) loadKeystore() (err error) {
ks := make(map[string]*keystore.KeyInfo, 0)
ks := make(map[string]*keystore.KeyInfo)
log.LogInfof("action[loadKeystore]")
result, err := c.fsm.store.SeekForPrefix([]byte(ksPrefix))
if err != nil {
@ -143,14 +143,8 @@ func (c *Cluster) loadKeystore() (err error) {
return
}
func (c *Cluster) clearKeystore() {
c.fsm.ksMutex.Lock()
defer c.fsm.ksMutex.Unlock()
c.fsm.keystore = nil
}
func (c *Cluster) loadAKstore() (err error) {
aks := make(map[string]*keystore.AccessKeyInfo, 0)
aks := make(map[string]*keystore.AccessKeyInfo)
log.LogInfof("action[loadAccessKeystore]")
result, err := c.fsm.store.SeekForPrefix([]byte(akPrefix))
if err != nil {
@ -175,12 +169,6 @@ func (c *Cluster) loadAKstore() (err error) {
return
}
func (c *Cluster) clearAKstore() {
c.fsm.aksMutex.Lock()
defer c.fsm.aksMutex.Unlock()
c.fsm.accessKeystore = nil
}
func (c *Cluster) addRaftNode(nodeID uint64, addr string) (err error) {
peer := proto.Peer{ID: nodeID}
_, err = c.partition.ChangeMember(proto.ConfAddNode, peer, []byte(addr))

2
authnode/server.go
View File

@ -212,7 +212,7 @@ func (m *Server) Start(cfg *config.Config) (err error) {
return fmt.Errorf("action[Start] failed %v,err: auth root Key invalid=%s", proto.ErrInvalidCfg, AuthRootKey)
}
if cfg.GetBool(EnableHTTPS) == true {
if cfg.GetBool(EnableHTTPS) {
m.cluster.PKIKey.EnableHTTPS = true
if m.cluster.PKIKey.AuthRootPublicKey, err = os.ReadFile("/app/server.crt"); err != nil {
return fmt.Errorf("action[Start] failed,err[%v]", err)

7
authtool/authtool.go
View File

@ -57,10 +57,7 @@ var action2PathMap = map[string]string{
OSGetCaps: proto.OSGetCaps,
}
var (
cflag string
flaginfo flagInfo
)
var flaginfo flagInfo
type ticketFlag struct {
key string
@ -364,7 +361,7 @@ func accessAuthServer() {
panic(err)
}
} else {
if res, err = resp.KeyInfo.DumpJSONStr(resp.AuthIDKey); err != nil {
if _, err = resp.KeyInfo.DumpJSONStr(resp.AuthIDKey); err != nil {
panic(err)
}
}

2
autofs/cfs.go
View File

@ -175,7 +175,7 @@ func cfsList() error {
for _, v := range mps {
if v.Type == "fuse" || v.Type == "fuse.cubefs" {
fmt.Println(fmt.Sprintf("%s on %s type %s (%s)", v.Device, v.Path, v.Type, strings.Join(v.Opts, ",")))
fmt.Printf("%s on %s type %s (%s)\n", v.Device, v.Path, v.Type, strings.Join(v.Opts, ","))
}
}

6
autofs/cfs_test.go
View File

@ -10,15 +10,11 @@ import (
func TestCfsOption_Parse(t *testing.T) {
c1 := cfsParse("tmp/mp", "subdir=sc1ch,volName=m1-project,owner=11111111,accessKey=ya0VE1xxxyyy===gY431,secretKey=5Clssf7831eXSmxxxyyyDQl2Is6J2x,masterAddr=10.0.0.1:22222,enablePosixACL")
if c1 == nil {
t.Error("parsed nil")
}
if c1.MountPoint != "tmp/mp" || c1.Subdir != "sc1ch" || c1.AccessKey != "ya0VE1xxxyyy===gY431" {
t.Error("parsedErr, c1: ", fmt.Sprintf("%+v", c1))
}
log.Println(fmt.Sprintf("option: %+v", c1))
log.Printf("option: %+v", c1)
}
func TestCfsOption_ConvertToCliOptions(t *testing.T) {

2
autofs/main.go
View File

@ -25,7 +25,7 @@ var rootCmd = &cobra.Command{
// version
v, _ := c.Flags().GetBool("version")
if v {
fmt.Println(fmt.Sprintf("Version: %s, BuildDate: %s", buildVersion, buildDate))
fmt.Printf("Version: %s, BuildDate: %s\n", buildVersion, buildDate)
return
}

3
blockcache/cmd.go
View File

@ -40,9 +40,6 @@ import (
"github.com/jacobsa/daemonize"
)
//TODO: remove this later.
//go:generate golangci-lint run --issues-exit-code=1 -D errcheck -E bodyclose ./...
const (
ConfigKeyLogDir = "logDir"
ConfigKeyLogLevel = "logLevel"

3
cli/cli.go
View File

@ -24,9 +24,6 @@ import (
"github.com/spf13/cobra"
)
//TODO: remove this later.
//go:generate golangci-lint run --issues-exit-code=1 -D errcheck -E bodyclose ./...
func runCLI() (err error) {
var cfg *cmd.Config
if cfg, err = cmd.LoadConfig(); err != nil {

3
client/fs/dir.go
View File

@ -49,7 +49,7 @@ type DirContexts struct {
func NewDirContexts() (dctx *DirContexts) {
dctx = &DirContexts{}
dctx.dirCtx = make(map[fuse.HandleID]*DirContext, 0)
dctx.dirCtx = make(map[fuse.HandleID]*DirContext)
return
}
@ -510,7 +510,6 @@ func (d *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
}
batchNr := uint64(len(batches))
if batchNr == 0 || (from != "" && batchNr == 1) {
noMore = true
break
} else if batchNr < DefaultReaddirLimit {
noMore = true

10
client/fs/summarycache.go
View File

@ -126,19 +126,13 @@ func (sc *SummaryCache) evict(foreground bool) {
func (sc *SummaryCache) backgroundEviction() {
t := time.NewTicker(SummaryBgEvictionInterval)
defer t.Stop()
for {
select {
case <-t.C:
for range t.C {
sc.Lock()
sc.evict(false)
sc.Unlock()
}
}
}
func cacheExpired(info *summaryCacheElement) bool {
if time.Now().UnixNano() > info.expiration {
return true
}
return false
return time.Now().UnixNano() > info.expiration
}

20
client/fs/super.go
View File

@ -90,7 +90,6 @@ type Super struct {
taskPool []common.TaskPool
closeC chan struct{}
enableVerRead bool
}
// Functions that Super needs to implement
@ -290,9 +289,8 @@ func NewSuper(opt *proto.MountOptions) (s *Super, err error) {
func (s *Super) scheduleFlush() {
t := time.NewTicker(2 * time.Second)
defer t.Stop()
for {
select {
case <-t.C:
for range t.C {
{
ctx := context.Background()
s.fslock.Lock()
for ino, node := range s.nodeCache {
@ -338,8 +336,8 @@ func (s *Super) Node(ino, pino uint64, mode uint32) (fs.Node, error) {
// the node is not evict. So we create a streamer for it,
// and streamer's refcnt is 0.
file := node.(*File)
file.Open(nil, nil, nil)
file.Release(nil, nil)
file.Open(context.TODO(), nil, nil)
file.Release(context.TODO(), nil)
}
s.fslock.Lock()
s.nodeCache[ino] = node
@ -398,10 +396,6 @@ func (s *Super) SetRate(w http.ResponseWriter, r *http.Request) {
}
}
func (s *Super) exporterKey(act string) string {
return fmt.Sprintf("%v_fuseclient_%v", s.cluster, act)
}
func (s *Super) umpKey(act string) string {
return fmt.Sprintf("%v_fuseclient_%v", s.cluster, act)
}
@ -456,11 +450,11 @@ func (s *Super) SetSuspend(w http.ResponseWriter, r *http.Request) {
// wait
msg := <-s.suspendCh
switch msg.(type) {
switch msgVal := msg.(type) {
case error:
err = msg.(error)
err = msgVal
case string:
ret = msg.(string)
ret = msgVal
default:
err = fmt.Errorf("Unknown return type: %v", msg)
}

4
client/fuse.go
View File

@ -70,12 +70,12 @@ const (
)
const (
LoggerDir = "client"
// LoggerDir = "client"
LoggerPrefix = "client"
LoggerOutput = "output.log"
ModuleName = "fuseclient"
ConfigKeyExporterPort = "exporterKey"
// ConfigKeyExporterPort = "exporterKey"
ControlCommandSetRate = "/rate/set"
ControlCommandGetRate = "/rate/get"

2
console/assets_generate_test.go
View File

@ -9,7 +9,7 @@ import (
func TestMakeHtml2GoBin(t *testing.T) {
// when you need rebuild html . please open it
return
_ = getAssets
/*
assets := getAssets()

22
console/cutil/handler.go
View File

@ -128,7 +128,7 @@ func (h *graphqlProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
}
UserID := ui.User_id
header.Set(proto.UserKey, UserID)
header.Set(string(proto.UserKey), UserID)
rep, err := h.client.Proxy(r.Context(), r, header)
if err != nil {
@ -273,26 +273,6 @@ func (h *httpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
runner.Stop()
}
func parseResponseNames(r *http.Request) ([]string, error) {
var params httpPostBody
if err := json.NewDecoder(r.Body).Decode(&params); err != nil {
return nil, err
}
query, err := graphql.Parse(params.Query, params.Variables)
if err != nil {
return nil, err
}
list := make([]string, 0)
for _, s := range query.Selections {
list = append(list, s.Name)
}
return list, nil
}
func IQLFun(writer http.ResponseWriter, request *http.Request) {
writer.Write([]byte(`<!DOCTYPE html>

2
console/server.go
View File

@ -120,7 +120,7 @@ func (c *ConsoleNode) loadConfig(cfg *config.Config) (err error) {
if len(c.listen) == 0 {
c.listen = "80"
}
if match := regexp.MustCompile("^(\\d)+$").MatchString(c.listen); !match {
if match := regexp.MustCompile(`^(\d)+$`).MatchString(c.listen); !match {
return fmt.Errorf("invalid listen configuration:[%s]", c.listen)
}
log.LogInfof("console loadConfig: setup config: %v(%v)", proto.ListenPort, c.listen)

4
console/server_test.go
View File

@ -8,8 +8,8 @@ import (
)
func TestServer(t *testing.T) {
return
_ = graphErr{}
_ = graphResponse{}
// if _, err := log.InitLog("/tmp", "console", log.DebugLevel, nil); err != nil {
// panic(err)
// }

12
console/service/monitor.go
View File

@ -32,7 +32,8 @@ func NewMonitorService(addr, app, cluster, dashboardAddr string) *MonitorService
func (fs *MonitorService) empty(ctx context.Context, args struct {
Empty bool
}) bool {
},
) bool {
return args.Empty
}
@ -60,8 +61,9 @@ func (ms *MonitorService) RangeQuery(ctx context.Context, args struct {
if err != nil {
return "", err
}
defer resp.Body.Close()
b, err := io.ReadAll(resp.Body)
b, _ := io.ReadAll(resp.Body)
return string(b), nil
}
@ -109,8 +111,9 @@ func (ms *MonitorService) Query(ctx context.Context, args struct {
if err != nil {
return "", err
}
defer resp.Body.Close()
b, err := io.ReadAll(resp.Body)
b, _ := io.ReadAll(resp.Body)
return string(b), nil
}
@ -150,8 +153,9 @@ func (ms *MonitorService) FuseClientList(ctx context.Context, args struct{}) ([]
if err != nil {
return nil, err
}
defer resp.Body.Close()
b, err := io.ReadAll(resp.Body)
b, _ := io.ReadAll(resp.Body)
result := struct {
Data struct {

17
cubefs.go Normal file
View File

@ -0,0 +1,17 @@
// Copyright 2024 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package cubefs
//go:generate golangci-lint run --issues-exit-code=1 --timeout 10m --skip-dirs depends --max-same-issues 1000 -D errcheck -E bodyclose ./...

8
datanode/const.go
View File

@ -103,3 +103,11 @@ const (
MaxFullSyncTinyDeleteTime = 3600 * 24
MinTinyExtentDeleteRecordSyncSize = 4 * 1024 * 1024
)
// TODO: to remove unused by golangci
var (
_ = (*DataPartition).canRemoveSelf
_ = (*DataPartition).getMemberExtentIDAndPartitionSize
_ = (*DataNode).detachDataPartition
_ = (*DataNode).loadDataPartition
)

13
datanode/data_partition_repair_test.go Executable file → Normal file
View File

@ -22,8 +22,8 @@ import (
type repairWorker struct {
proto.Packet
followersAddrs []string
followerPackets []*repl.FollowerPacket
// followersAddrs []string
// followerPackets []*repl.FollowerPacket
IsReleased int32 // TODO what is released?
Object interface{}
TpObject *exporter.TimePointCount
@ -166,8 +166,8 @@ func (p *repairWorker) WriteToConn(c net.Conn) (err error) {
}
func (p *repairWorker) ReadFromConnWithVer(c net.Conn, timeoutSec int) (err error) {
select {
case pr := <-p.packChannel:
pr := <-p.packChannel
{
p.CRC = pr.GetCRC()
p.Data = make([]byte, len(pr.GetData()))
copy(p.Data, pr.GetData())
@ -285,7 +285,6 @@ func extentReloadCheckNormalCrc(t *testing.T, s *storage.ExtentStore, id uint64,
assert.True(t, err == nil)
extCrc := e.GetCrc(0)
assert.True(t, crc == extCrc)
return
}
func extentStoreSnapshotRwTest(t *testing.T, s *storage.ExtentStore, id uint64, crc uint32, data []byte) {
@ -377,7 +376,7 @@ func mockInitWorker(t *testing.T, role string) *repairWorker {
return new(net.TCPConn), nil
}
worker.dp.dataNode.putRepairConnFunc = func(con net.Conn, force bool) {
return
_ = struct{}{}
}
return worker
}
@ -399,8 +398,6 @@ func workerInit(t *testing.T, id uint64, data []byte, crc uint32) {
recvWorker = mockInitWorker(t, "receiver")
sendWorker.dstWorker = recvWorker
recvWorker.dstWorker = sendWorker
return
}
func senderRepairWorker(t *testing.T, exitCh chan struct{}) {

7
datanode/disk.go
View File

@ -99,7 +99,8 @@ const (
type PartitionVisitor func(dp *DataPartition)
func NewDisk(path string, reservedSpace, diskRdonlySpace uint64, maxErrCnt int, space *SpaceManager,
diskEnableReadRepairExtentLimit bool) (d *Disk, err error) {
diskEnableReadRepairExtentLimit bool,
) (d *Disk, err error) {
d = new(Disk)
d.Path = path
d.ReservedSpace = reservedSpace
@ -127,7 +128,7 @@ func NewDisk(path string, reservedSpace, diskRdonlySpace uint64, maxErrCnt int,
}
d.startScheduleToUpdateSpaceInfo()
d.limitFactor = make(map[uint32]*rate.Limiter, 0)
d.limitFactor = make(map[uint32]*rate.Limiter)
d.limitFactor[proto.FlowReadType] = rate.NewLimiter(rate.Limit(proto.QosDefaultDiskMaxFLowLimit), proto.QosDefaultBurst)
d.limitFactor[proto.FlowWriteType] = rate.NewLimiter(rate.Limit(proto.QosDefaultDiskMaxFLowLimit), proto.QosDefaultBurst)
d.limitFactor[proto.IopsReadType] = rate.NewLimiter(rate.Limit(proto.QosDefaultDiskMaxIoLimit), defaultIOLimitBurst)
@ -135,7 +136,7 @@ func NewDisk(path string, reservedSpace, diskRdonlySpace uint64, maxErrCnt int,
d.limitRead = newIOLimiter(space.dataNode.diskReadFlow, space.dataNode.diskReadIocc)
d.limitWrite = newIOLimiter(space.dataNode.diskWriteFlow, space.dataNode.diskWriteIocc)
d.DiskErrPartitionSet = make(map[uint64]struct{}, 0)
d.DiskErrPartitionSet = make(map[uint64]struct{})
err = d.initDecommissionStatus()
if err != nil {

2
datanode/nodeinfo.go
View File

@ -5,11 +5,9 @@ import (
"time"
"github.com/cubefs/cubefs/util/log"
"golang.org/x/time/rate"
)
const (
defaultMarkDeleteLimitRate = rate.Inf
defaultMarkDeleteLimitBurst = 512
defaultIOLimitBurst = 512
UpdateNodeInfoTicket = 1 * time.Minute

5
datanode/partition.go
View File

@ -129,9 +129,9 @@ type DataPartition struct {
persistMetaMutex sync.RWMutex
// snapshot
// verSeqPrepare uint64
// verSeqCommitStatus int8
verSeq uint64
verSeqPrepare uint64
verSeqCommitStatus int8
volVersionInfoList *proto.VolVersionInfoList
decommissionRepairProgress float64 // record repair progress for decommission datapartition
stopRecover bool
@ -736,7 +736,6 @@ func (dp *DataPartition) checkIsDiskError(err error, rwFlag uint8) {
// must after change disk.status
dp.statusUpdate()
return
}
func newRaftApplyError(err error) error {

9
datanode/server.go
View File

@ -52,7 +52,7 @@ var (
ErrNewSpaceManagerFailed = errors.New("Creater new space manager failed")
ErrGetMasterDatanodeInfoFailed = errors.New("Failed to get datanode info from master")
LocalIP, serverPort string
LocalIP string
gConnPool = util.NewConnectPool()
// MasterClient = masterSDK.NewMasterClient(nil, false)
MasterClient *masterSDK.MasterCLientWithResolver
@ -133,7 +133,6 @@ type DataNode struct {
port string
zoneName string
clusterID string
localIP string
bindIp bool
localServerAddr string
nodeID uint64
@ -144,6 +143,7 @@ type DataNode struct {
tickInterval int
raftRecvBufSize int
startTime int64
// localIP string
tcpListener net.Listener
stopC chan bool
@ -174,12 +174,12 @@ type DataNode struct {
diskWriteIops int
diskWriteFlow int
dpMaxRepairErrCnt uint64
dpRepairTimeOut uint64
clusterUuid string
clusterUuidEnable bool
serviceIDKey string
cpuUtil atomicutil.Float64
cpuSamplerDone chan struct{}
// dpRepairTimeOut uint64
diskUnavailablePartitionErrorCount uint64 // disk status becomes unavailable when disk error partition count reaches this value
}
@ -189,7 +189,7 @@ type verOp2Phase struct {
verPrepare uint64
status uint32
step uint32
op uint8
// op uint8
sync.Mutex
}
@ -305,7 +305,6 @@ func (s *DataNode) parseConfig(cfg *config.Config) (err error) {
LocalIP = cfg.GetString(ConfigKeyLocalIP)
port = cfg.GetString(proto.ListenPort)
s.bindIp = cfg.GetBool(proto.BindIpKey)
serverPort = port
if regexpPort, err = regexp.Compile(`^(\d)+$`); err != nil {
return fmt.Errorf("Err:no port")
}

2
datanode/server_handler.go
View File

@ -727,8 +727,6 @@ func (s *DataNode) loadDataPartition(w http.ResponseWriter, r *http.Request) {
for _, fileInfo := range fileInfoList {
filename := fileInfo.Name()
if !disk.isPartitionDir(filename) {
if disk.isExpiredPartitionDir(filename) {
}
continue
}

13
datanode/space_manager.go
View File

@ -40,10 +40,8 @@ type SpaceManager struct {
partitionMutex sync.RWMutex
stats *Stats
stopC chan bool
selectedIndex int // TODO what is selected index
diskList []string
dataNode *DataNode
createPartitionMutex sync.RWMutex
diskUtils map[string]*atomicutil.Float64
samplerDone chan struct{}
allDisksLoaded bool
@ -227,7 +225,8 @@ func (manager *SpaceManager) Stats() *Stats {
}
func (manager *SpaceManager) LoadDisk(path string, reservedSpace, diskRdonlySpace uint64, maxErrCnt int,
diskEnableReadRepairExtentLimit bool) (err error) {
diskEnableReadRepairExtentLimit bool,
) (err error) {
var (
disk *Disk
visitor PartitionVisitor
@ -503,11 +502,3 @@ func (s *DataNode) buildHeartBeatResponse(response *proto.DataNodeHeartbeatRespo
response.DiskStats = append(response.DiskStats, bds)
}
}
func (manager *SpaceManager) getPartitionIds() []uint64 {
res := make([]uint64, 0)
for id := range manager.partitions {
res = append(res, id)
}
return res
}

8
datanode/stat.go
View File

@ -22,11 +22,6 @@ import (
// Stats defines various metrics that will be collected during the execution.
type Stats struct {
inDataSize uint64
outDataSize uint64
inFlow uint64
outFlow uint64
Zone string
ConnectionCnt int64
ClusterID string
@ -71,7 +66,8 @@ func (s *Stats) GetConnectionCount() int64 {
func (s *Stats) updateMetrics(
total, used, available, createdPartitionWeights, remainWeightsForCreatePartition,
maxWeightsForCreatePartition, dataPartitionCnt uint64) {
maxWeightsForCreatePartition, dataPartitionCnt uint64,
) {
s.Lock()
defer s.Unlock()

6
datanode/wrap_operator.go
View File

@ -954,6 +954,7 @@ func (s *DataNode) handleExtentRepairReadPacket(p *repl.Packet, connect net.Conn
partition := p.Object.(*DataPartition)
if !partition.disk.RequireReadExtentToken(partition.partitionID) {
err = storage.NoDiskReadRepairExtentTokenError
log.LogWarn("check the source code cos i don't understand the unuesd error,", err)
log.LogDebugf("dp(%v) disk(%v) extent(%v) wait for read extent token",
p.PartitionID, partition.disk.Path, p.ExtentID)
return
@ -1047,10 +1048,6 @@ func writeEmptyPacketOnExtentRepairRead(reply repl.PacketInterface, newOffset, c
return
}
func (s *DataNode) attachAvaliSizeOnExtentRepairRead(reply *repl.Packet, avaliSize uint64) {
binary.BigEndian.PutUint64(reply.Arg[9:17], avaliSize)
}
func (s *DataNode) NormalSnapshotExtentRepairRead(request *repl.Packet, connect net.Conn) {
replyFunc := func() repl.PacketInterface {
reply := repl.NewNormalExtentWithHoleStreamReadResponsePacket(request.ReqID, request.PartitionID, request.ExtentID)
@ -1082,7 +1079,6 @@ func (s *DataNode) tinyExtentRepairRead(request *repl.Packet, connect net.Conn)
return reply
}
s.ExtentWithHoleRepairRead(request, connect, replyFunc)
return
}
// Handle tinyExtentRepairRead packet.

19
deploy/cmd/firewall.go
View File

@ -1,7 +1,6 @@
package cmd
import (
"fmt"
"log"
"os/exec"
)
@ -61,12 +60,12 @@ func stopFirewall(nodeUser, node string) {
log.Println(cmd)
}
func checkPortStatus(nodeUser, node string, port string) (string, error) {
cmd := exec.Command("ssh", nodeUser+"@"+node, "firewall-cmd --list-all | grep "+port)
fmt.Println(cmd)
_, err := cmd.Output()
if err != nil {
return fmt.Sprintf("Port %s %s is closed", node, port), err
}
return fmt.Sprintf("Port %s is open", port), nil
}
// func checkPortStatus(nodeUser, node string, port string) (string, error) {
// cmd := exec.Command("ssh", nodeUser+"@"+node, "firewall-cmd --list-all | grep "+port)
// fmt.Println(cmd)
// _, err := cmd.Output()
// if err != nil {
// return fmt.Sprintf("Port %s %s is closed", node, port), err
// }
// return fmt.Sprintf("Port %s is open", port), nil
// }

3
deploy/cmd/transform.go
View File

@ -46,6 +46,9 @@ func transferDirectoryToRemote(localFilePath string, remoteFilePath string, remo
return nil
}
// TODO: to remove unused by golangci
var _ = copyFolder
func copyFolder(sourcePath, destinationPath string) error {
err := filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error {
if err != nil {

18
docker/script/run_format.sh
View File

@ -15,12 +15,16 @@ popd
export PATH=$PATH:/go/bin
for subdir in proto blockcache storage cli lcnode
do
pushd ${CurrentPath}/../../${subdir}
go generate ./...
if [[ $? -ne 0 ]]; then
golintFile=golint.diff
pushd ${CurrentPath}/../..
go generate . > ${golintFile}
cat ${golintFile}
if [[ $? -ne 0 ]]; then
exit 1
fi
fi
if [ "$(cat ${golintFile}|wc -l)" -gt 0 ]; then
popd
done
exit 1
fi
rm -f ${golintFile}
popd

13
fdstore/fdstore.go
View File

@ -126,7 +126,7 @@ func SendFuseFdToNewClient(udsListener net.Listener, file *os.File) (err error)
if file == nil {
err = fmt.Errorf("no file is received")
fmt.Fprintf(os.Stderr, err.Error())
fmt.Fprintln(os.Stderr, err.Error())
return
} else {
if err = util.SendFd(socket, file.Name(), file.Fd()); err != nil {
@ -176,11 +176,6 @@ func SendSuspendRequest(port string, udsListener net.Listener) (err error) {
return nil
}
func WaitSuspendFinish(ch chan error) error {
err := <-ch
return err
}
func doSuspend(port string) error {
var fud *os.File
@ -319,7 +314,6 @@ func doDump(filePathes string) {
)
rsize, err = handleListFile.Read(data)
if rsize == 0 || err == io.EOF {
err = nil
break
}
@ -357,7 +351,7 @@ func main() {
flag.Parse()
if *optVersion {
fmt.Printf(proto.DumpVersion("fdstore"))
fmt.Println(proto.DumpVersion("fdstore"))
os.Exit(0)
}
@ -385,6 +379,5 @@ func main() {
os.Exit(-1)
}
fmt.Printf("Done Successfully\n")
return
fmt.Println("Done Successfully")
}

30
fsck/cmd/clean.go
View File

@ -228,36 +228,6 @@ func doEvictInode(inode *Inode) error {
return nil
}
func doUnlinkInode(inode *Inode) error {
/*
* Do clean inode with the following exceptions:
* 1. nlink == 0, might be a temorary inode
* 2. size == 0 && ctime is close to current time, might be in the process of file creation
*/
// if inode.NLink == 0 ||
// (inode.Size == 0 &&
// time.Unix(inode.CreateTime, 0).Add(24*time.Hour).After(time.Now())) {
// return nil
// }
//
// err := gMetaWrapper.Unlink_ll(inode.Inode)
// if err != nil {
// if err != syscall.ENOENT {
// return err
// }
// err = nil
// }
//
// err = gMetaWrapper.Evict(inode.Inode)
// if err != nil {
// if err != syscall.ENOENT {
// return err
// }
// }
return nil
}
func cleanDentries() error {
// filePath := fmt.Sprintf("_export_%s/%s", VolName, obsoleteDentryDumpFileName)
// TODO: send request to meta node directly with pino, name and ino.

1
fsck/cmd/common.go
View File

@ -30,7 +30,6 @@ var (
var (
inodeDumpFileName string = "inode.dump"
dentryDumpFileName string = "dentry.dump"
inodeUpdateDumpFileName string = "inode.dump.update"
obsoleteInodeDumpFileName string = "inode.dump.obsolete"
obsoleteDentryDumpFileName string = "dentry.dump.obsolete"
pathDumpFileName string = "path.dump"

6
fsck/cmd/getInfo.go
View File

@ -38,12 +38,6 @@ const (
MaxFollowPathTaskNum = 120
)
type Summary struct {
files uint64
dirs uint64
bytes uint64
}
func newInfoCmd() *cobra.Command {
c := &cobra.Command{
Use: "get",

2
fsck/cmd/root.go
View File

@ -32,7 +32,7 @@ func NewRootCmd() *cobra.Command {
Args: cobra.MinimumNArgs(0),
Run: func(cmd *cobra.Command, args []string) {
if optShowVersion {
_, _ = fmt.Fprintf(os.Stdout, proto.DumpVersion("FSCK"))
fmt.Fprintln(os.Stdout, proto.DumpVersion("FSCK"))
return
}
},

3
lcnode/server.go
View File

@ -35,9 +35,6 @@ import (
"golang.org/x/time/rate"
)
//TODO: remove this later.
//go:generate golangci-lint run --issues-exit-code=1 -D errcheck -E bodyclose ./...
type LcNode struct {
listen string
localServerAddr string

7
libsdk/libsdk.go
View File

@ -130,7 +130,6 @@ var (
statusEISDIR = errorToStatus(syscall.EISDIR)
statusENOSPC = errorToStatus(syscall.ENOSPC)
)
var once sync.Once
func init() {
gClientManager = &clientManager{
@ -154,10 +153,6 @@ type clientManager struct {
mu sync.RWMutex
}
type pushConfig struct {
PushAddr string `json:"pushAddr"`
}
func newClient() *client {
id := atomic.AddInt64(&gClientManager.nextClientID, 1)
c := &client{
@ -851,7 +846,7 @@ func cfs_lsdir(id C.int64_t, fd C.int, direntsInfo []C.struct_cfs_dirent_info, c
}
dirp := f.dirp
inodeIDS := make([]uint64, count, count)
inodeIDS := make([]uint64, count)
inodeMap := make(map[uint64]C.int)
for dirp.pos < len(dirp.dirents) && n < count {
inodeIDS[n] = dirp.dirents[dirp.pos].Inode

5
master/admin_task_manager.go
View File

@ -84,7 +84,6 @@ func (sender *AdminTaskManager) doDeleteTasks() {
for _, t := range delTasks {
sender.DelTask(t)
}
return
}
func (sender *AdminTaskManager) getToBeDeletedTasks() (delTasks []*proto.AdminTask) {
@ -260,14 +259,14 @@ func (sender *AdminTaskManager) getToDoTasks() (tasks []*proto.AdminTask) {
// send heartbeat task first
for _, t := range sender.TaskMap {
if t.IsHeartbeatTask() && t.CheckTaskNeedSend() == true {
if t.IsHeartbeatTask() && t.CheckTaskNeedSend() {
tasks = append(tasks, t)
t.SendTime = time.Now().Unix()
}
}
// send urgent task immediately
for _, t := range sender.TaskMap {
if t.IsUrgentTask() && t.CheckTaskNeedSend() == true {
if t.IsUrgentTask() && t.CheckTaskNeedSend() {
tasks = append(tasks, t)
t.SendTime = time.Now().Unix()
}

79
master/api_args_parse.go
View File

@ -211,18 +211,6 @@ func parseAndExtractNodeAddr(r *http.Request) (nodeAddr string, err error) {
return extractNodeAddr(r)
}
func parseRequestToDecommissionNode(r *http.Request) (nodeAddr, diskPath string, err error) {
if err = r.ParseForm(); err != nil {
return
}
nodeAddr, err = extractNodeAddr(r)
if err != nil {
return
}
diskPath, err = extractDiskPath(r)
return
}
func parseRequestToGetTaskResponse(r *http.Request) (tr *proto.AdminTask, err error) {
var body []byte
if err = r.ParseForm(); err != nil {
@ -346,7 +334,7 @@ func extractUint64WithDefault(r *http.Request, key string, def uint64) (val uint
return def, nil
}
if val, err = strconv.ParseUint(str, 10, 64); err != nil || val < 0 {
if val, err = strconv.ParseUint(str, 10, 64); err != nil {
return 0, fmt.Errorf("parse [%s] is not valid uint [%d], err %v", key, val, err)
}
@ -783,7 +771,7 @@ func parseRequestToCreateVol(r *http.Request, req *createVolReq) (err error) {
if err != nil {
return
}
if followerExist && followerRead == false && proto.IsHot(req.volType) &&
if followerExist && !followerRead && proto.IsHot(req.volType) &&
(req.dpReplicaNum == 1 || req.dpReplicaNum == 2) {
return fmt.Errorf("vol with 1 ro 2 replia should enable followerRead")
}
@ -814,7 +802,7 @@ func parseRequestToCreateVol(r *http.Request, req *createVolReq) (err error) {
return
}
req.enablePosixAcl, err = extractPosixAcl(r)
req.enablePosixAcl, _ = extractPosixAcl(r)
if req.DpReadOnlyWhenVolFull, err = extractBoolWithDefault(r, dpReadOnlyWhenVolFull, false); err != nil {
return
@ -868,18 +856,6 @@ func parseRequestToCreateDataPartition(r *http.Request) (count int, name string,
return
}
func parseRequestToGetConcurrentLcNode(r *http.Request) (count uint64, err error) {
if err = r.ParseForm(); err != nil {
return
}
if count, err = extractUint64(r, countKey); err != nil || count == 0 {
err = unmatchedKey(countKey)
return
}
return
}
func parseRequestToGetDataPartition(r *http.Request) (ID uint64, volName string, err error) {
if err = r.ParseForm(); err != nil {
return
@ -1089,30 +1065,6 @@ func extractFollowerRead(r *http.Request) (followerRead bool, exist bool, err er
return
}
func extractAuthenticate(r *http.Request) (authenticate bool, err error) {
var value string
if value = r.FormValue(authenticateKey); value == "" {
authenticate = false
return
}
if authenticate, err = strconv.ParseBool(value); err != nil {
return
}
return
}
func extractCrossZone(r *http.Request) (crossZone bool, err error) {
var value string
if value = r.FormValue(crossZoneKey); value == "" {
crossZone = false
return
}
if crossZone, err = strconv.ParseBool(value); err != nil {
return
}
return
}
func parseAndExtractDirLimit(r *http.Request) (limit uint32, err error) {
if err = r.ParseForm(); err != nil {
return
@ -1194,7 +1146,6 @@ func parseAndExtractSetNodeSetInfoParams(r *http.Request) (params map[string]int
nodesetId := uint64(0)
nodesetId, err = strconv.ParseUint(value, 10, 64)
if err != nil {
err = unmatchedKey(idKey)
err = unmatchedKey(idKey)
return
}
@ -1472,7 +1423,7 @@ func extractUint64(r *http.Request, key string) (val uint64, err error) {
return 0, nil
}
if val, err = strconv.ParseUint(str, 10, 64); err != nil || val < 0 {
if val, err = strconv.ParseUint(str, 10, 64); err != nil {
return 0, fmt.Errorf("args [%s] is not legal, val %s", key, str)
}
@ -1486,7 +1437,7 @@ func extractUint32(r *http.Request, key string) (val uint32, err error) {
}
var tmp uint64
if tmp, err = strconv.ParseUint(str, 10, 32); err != nil || val < 0 {
if tmp, err = strconv.ParseUint(str, 10, 32); err != nil {
return 0, fmt.Errorf("args [%s] is not legal, val %s", key, str)
}
@ -1669,7 +1620,6 @@ func send(w http.ResponseWriter, r *http.Request, reply []byte) {
return
}
log.LogInfof("URL[%v],remoteAddr[%v],response ok", r.URL, r.RemoteAddr)
return
}
func sendErrReply(w http.ResponseWriter, r *http.Request, httpReply *proto.HTTPReply) {
@ -1686,8 +1636,6 @@ func sendErrReply(w http.ResponseWriter, r *http.Request, httpReply *proto.HTTPR
if _, err = w.Write(reply); err != nil {
log.LogErrorf("fail to write http len[%d].URL[%v],remoteAddr[%v] err:[%v]", len(reply), r.URL, r.RemoteAddr, err)
}
return
}
func parseRequestToUpdateDecommissionLimit(r *http.Request) (limit uint64, err error) {
@ -1817,14 +1765,6 @@ func parseGetQuotaParam(r *http.Request) (volName string, quotaId uint32, err er
return
}
func extractPath(r *http.Request) (fullPath string, err error) {
if fullPath = r.FormValue(fullPathKey); fullPath == "" {
err = keyNotFound(nameKey)
return
}
return
}
func extractQuotaId(r *http.Request) (quotaId uint32, err error) {
var value string
if value = r.FormValue(quotaKey); value == "" {
@ -1836,15 +1776,6 @@ func extractQuotaId(r *http.Request) (quotaId uint32, err error) {
return
}
func extractInodeId(r *http.Request) (inode uint64, err error) {
var value string
if value = r.FormValue(inodeKey); value == "" {
err = keyNotFound(inodeKey)
return
}
return strconv.ParseUint(value, 10, 64)
}
func parseRequestToUpdateDecommissionDiskFactor(r *http.Request) (factor float64, err error) {
if err = r.ParseForm(); err != nil {
return

61
master/api_service.go
View File

@ -104,7 +104,7 @@ type ZoneView struct {
}
func newZoneView(name string) *ZoneView {
return &ZoneView{NodeSet: make(map[uint64]*NodeSetView, 0), Name: name}
return &ZoneView{NodeSet: make(map[uint64]*NodeSetView), Name: name}
}
type badPartitionView = proto.BadPartitionView
@ -606,7 +606,7 @@ func (m *Server) clusterStat(w http.ResponseWriter, r *http.Request) {
cs := &proto.ClusterStatInfo{
DataNodeStatInfo: m.cluster.dataNodeStatInfo,
MetaNodeStatInfo: m.cluster.metaNodeStatInfo,
ZoneStatInfo: make(map[string]*proto.ZoneStat, 0),
ZoneStatInfo: make(map[string]*proto.ZoneStat),
}
for zoneName, zoneStat := range m.cluster.zoneStatInfos {
cs.ZoneStatInfo[zoneName] = zoneStat
@ -641,6 +641,7 @@ func (m *Server) UidOperate(w http.ResponseWriter, r *http.Request) {
op, err = strconv.ParseUint(value, 10, 64)
if err != nil {
err = fmt.Errorf("parseUintParam %s-%s is not legal, err %s", OperateKey, value, err.Error())
log.LogWarn(err)
return
}
@ -714,6 +715,7 @@ func (m *Server) aclOperate(w http.ResponseWriter, r *http.Request) {
op, err = strconv.ParseUint(value, 10, 64)
if err != nil {
err = fmt.Errorf("parseUintParam %s-%s is not legal, err %s", OperateKey, value, err.Error())
log.LogWarn(err)
return
}
@ -744,7 +746,7 @@ func (m *Server) aclOperate(w http.ResponseWriter, r *http.Request) {
}
case util.AclAddIP, util.AclDelIP:
if opAclRes != nil {
if err, res = opAclRes.(error); !res {
if _, res = opAclRes.(error); !res {
sendErrReply(w, r, newErrHTTPReply(fmt.Errorf("inner error")))
return
}
@ -854,7 +856,6 @@ func (m *Server) setApiQpsLimit(w http.ResponseWriter, r *http.Request) {
}
sendOkReply(w, r, newSuccessHTTPReply(fmt.Sprintf("set api qps limit success: name: %v, limit: %v, timeout: %v",
name, limit, timeout)))
return
}
func (m *Server) getApiQpsLimit(w http.ResponseWriter, r *http.Request) {
@ -983,7 +984,7 @@ func (m *Server) createMetaPartition(w http.ResponseWriter, r *http.Request) {
return
}
sendOkReply(w, r, newSuccessHTTPReply(fmt.Sprint("create meta partition successfully")))
sendOkReply(w, r, newSuccessHTTPReply("create meta partition successfully"))
}
func parsePreloadDpReq(r *http.Request, preload *DataPartitionPreLoad) (err error) {
@ -1437,7 +1438,6 @@ RET:
return
}
_ = sendOkReply(w, r, newSuccessHTTPReply("success"))
return
}
func (m *Server) createDataPartition(w http.ResponseWriter, r *http.Request) {
@ -1518,7 +1518,7 @@ func (m *Server) changeDataPartitionLeader(w http.ResponseWriter, r *http.Reques
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
return
}
rstMsg := fmt.Sprintf(" changeDataPartitionLeader command success send to dest host but need check. ")
rstMsg := " changeDataPartitionLeader command success send to dest host but need check. "
_ = sendOkReply(w, r, newSuccessHTTPReply(rstMsg))
}
@ -1788,7 +1788,7 @@ func (m *Server) changeMetaPartitionLeader(w http.ResponseWriter, r *http.Reques
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
return
}
rstMsg := fmt.Sprintf(" changeMetaPartitionLeader command success send to dest host but need check. ")
rstMsg := " changeMetaPartitionLeader command success send to dest host but need check. "
_ = sendOkReply(w, r, newSuccessHTTPReply(rstMsg))
}
@ -1838,7 +1838,7 @@ func (m *Server) balanceMetaPartitionLeader(w http.ResponseWriter, r *http.Reque
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
return
}
rstMsg := fmt.Sprintf("balanceMetaPartitionLeader command sucess")
rstMsg := "balanceMetaPartitionLeader command sucess"
_ = sendOkReply(w, r, newSuccessHTTPReply(rstMsg))
}
@ -2499,8 +2499,6 @@ func (m *Server) checkCreateReq(req *createVolReq) (err error) {
func (m *Server) createVol(w http.ResponseWriter, r *http.Request) {
req := &createVolReq{}
vol := &Vol{}
var err error
metric := exporter.NewTPCnt(apiToMetricsName(proto.AdminCreateVol))
@ -2523,6 +2521,7 @@ func (m *Server) createVol(w http.ResponseWriter, r *http.Request) {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
return
}
var vol *Vol
if vol, err = m.cluster.createVol(req); err != nil {
sendErrReply(w, r, newErrHTTPReply(err))
return
@ -3268,10 +3267,7 @@ func (m *Server) updateClusterSelector(dataNodesetSelector string, metaNodesetSe
return false
}
err = m.updateZoneNodeSelector(zone.name, dataNodeSelector, metaNodeSelector)
if err != nil {
return false
}
return true
return err == nil
})
return
}
@ -3552,7 +3548,6 @@ func (m *Server) setNodeRdOnlyHandler(w http.ResponseWriter, r *http.Request) {
}
sendOkReply(w, r, newSuccessHTTPReply(fmt.Sprintf("[setNodeRdOnlyHandler] set node %s to rdOnly(%v) success", addr, rdOnly)))
return
}
func (m *Server) setDpRdOnlyHandler(w http.ResponseWriter, r *http.Request) {
@ -3582,7 +3577,6 @@ func (m *Server) setDpRdOnlyHandler(w http.ResponseWriter, r *http.Request) {
}
sendOkReply(w, r, newSuccessHTTPReply(fmt.Sprintf("[setNodeRdOnlyHandler] set dpid %v to rdOnly(%v) success", dpId, rdOnly)))
return
}
func (m *Server) updateNodeSetCapacityHandler(w http.ResponseWriter, r *http.Request) {
@ -3712,7 +3706,7 @@ func (m *Server) updateNodeSetIdHandler(w http.ResponseWriter, r *http.Request)
return
}
sendOkReply(w, r, newSuccessHTTPReply(fmt.Sprintf("update node setid successfully")))
sendOkReply(w, r, newSuccessHTTPReply("update node setid successfully"))
}
func (m *Server) updateNodeSetNodeSelector(w http.ResponseWriter, r *http.Request) {
@ -3821,7 +3815,7 @@ func (m *Server) createDomainHandler(w http.ResponseWriter, r *http.Request) {
return
}
sendOkReply(w, r, newSuccessHTTPReply(fmt.Sprintf("successful")))
sendOkReply(w, r, newSuccessHTTPReply("successful"))
}
// get metanode some interval params
@ -4776,7 +4770,6 @@ func (m *Server) getMetaPartitions(w http.ResponseWriter, r *http.Request) {
mpsCache = vol.getMpsCache()
}
send(w, r, mpsCache)
return
}
func (m *Server) putDataPartitions(w http.ResponseWriter, r *http.Request) {
@ -5025,9 +5018,7 @@ func getMetaPartitionView(mp *MetaPartition) (mpView *proto.MetaPartitionView) {
mpView = proto.NewMetaPartitionView(mp.PartitionID, mp.Start, mp.End, mp.Status)
mp.RLock()
defer mp.RUnlock()
for _, host := range mp.Hosts {
mpView.Members = append(mpView.Members, host)
}
mpView.Members = append(mpView.Members, mp.Hosts...)
mr, err := mp.getMetaReplicaLeader()
if err != nil {
return
@ -5168,7 +5159,7 @@ func (m *Server) changeMasterLeader(w http.ResponseWriter, r *http.Request) {
sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
return
}
rstMsg := fmt.Sprintf(" changeMasterLeader. command success send to dest host but need check. ")
rstMsg := " changeMasterLeader. command success send to dest host but need check. "
_ = sendOkReply(w, r, newSuccessHTTPReply(rstMsg))
}
@ -5253,7 +5244,7 @@ func (m *Server) DelVersion(w http.ResponseWriter, r *http.Request) {
return
}
verSeq, err = extractUint64(r, verSeqKey)
verSeq, _ = extractUint64(r, verSeqKey)
log.LogDebugf("action[DelVersion] vol %v verSeq %v", name, verSeq)
if value = r.FormValue(forceKey); value != "" {
force, _ = strconv.ParseBool(value)
@ -5844,7 +5835,6 @@ func (m *Server) setConfigHandler(w http.ResponseWriter, r *http.Request) {
}
sendOkReply(w, r, newSuccessHTTPReply(fmt.Sprintf("set config key[%v], value[%v] success", key, value)))
return
}
func (m *Server) getConfigHandler(w http.ResponseWriter, r *http.Request) {
@ -6000,7 +5990,6 @@ func (m *Server) DeleteQuota(w http.ResponseWriter, r *http.Request) {
msg := fmt.Sprintf("delete quota successfully, vol [%v] quotaId [%v]", name, quotaId)
sendOkReply(w, r, newSuccessHTTPReply(msg))
return
}
func (m *Server) ListQuota(w http.ResponseWriter, r *http.Request) {
@ -6031,7 +6020,6 @@ func (m *Server) ListQuota(w http.ResponseWriter, r *http.Request) {
log.LogInfof("list quota vol [%v] resp [%v] success.", name, *resp)
sendOkReply(w, r, newSuccessHTTPReply(resp))
return
}
func (m *Server) ListQuotaAll(w http.ResponseWriter, r *http.Request) {
@ -6043,7 +6031,6 @@ func (m *Server) ListQuotaAll(w http.ResponseWriter, r *http.Request) {
volsInfo := m.cluster.listQuotaAll()
log.LogInfof("list all vol has quota [%v]", volsInfo)
sendOkReply(w, r, newSuccessHTTPReply(volsInfo))
return
}
func (m *Server) GetQuota(w http.ResponseWriter, r *http.Request) {
@ -6076,7 +6063,6 @@ func (m *Server) GetQuota(w http.ResponseWriter, r *http.Request) {
log.LogInfof("get quota vol [%v] quotaInfo [%v] success.", name, *quotaInfo)
sendOkReply(w, r, newSuccessHTTPReply(quotaInfo))
return
}
// func (m *Server) BatchModifyQuotaFullPath(w http.ResponseWriter, r *http.Request) {
@ -6188,7 +6174,6 @@ func (m *Server) setDpDiscardHandler(w http.ResponseWriter, r *http.Request) {
msg := fmt.Sprintf("[setDpDiscardHandler] set dpid %v to discard(%v) success", dpId, discard)
log.LogInfo(msg)
sendOkReply(w, r, newSuccessHTTPReply(msg))
return
}
func (m *Server) getDiscardDpHandler(w http.ResponseWriter, r *http.Request) {
@ -6201,8 +6186,7 @@ func (m *Server) getDiscardDpHandler(w http.ResponseWriter, r *http.Request) {
vols := m.cluster.copyVols()
for _, vol := range vols {
var dps *DataPartitionMap
dps = vol.dataPartitions
dps := vol.dataPartitions
for _, dp := range dps.partitions {
if dp.IsDiscard {
DiscardDpInfos.DiscardDps = append(DiscardDpInfos.DiscardDps, *dp.buildDpInfo(m.cluster))
@ -6213,7 +6197,6 @@ func (m *Server) getDiscardDpHandler(w http.ResponseWriter, r *http.Request) {
msg := fmt.Sprintf("[GetDiscardDpHandler] discard dp num:%v", len(DiscardDpInfos.DiscardDps))
log.LogInfo(msg)
sendOkReply(w, r, newSuccessHTTPReply(DiscardDpInfos))
return
}
func (m *Server) queryBadDisks(w http.ResponseWriter, r *http.Request) {
@ -6404,7 +6387,7 @@ func (m *Server) SetBucketLifecycle(w http.ResponseWriter, r *http.Request) {
return
}
_ = m.cluster.SetBucketLifecycle(&req)
sendOkReply(w, r, newSuccessHTTPReply(fmt.Sprintf("PutBucketLifecycleConfiguration successful ")))
sendOkReply(w, r, newSuccessHTTPReply("PutBucketLifecycleConfiguration successful"))
}
func (m *Server) GetBucketLifecycle(w http.ResponseWriter, r *http.Request) {
@ -6543,7 +6526,7 @@ func (m *Server) S3QosGet(w http.ResponseWriter, r *http.Request) {
doStatAndMetric(proto.S3QoSGet, metric, err, nil)
}()
apiLimitConf := make(map[string]*proto.UserLimitConf, 0)
apiLimitConf := make(map[string]*proto.UserLimitConf)
s3QosResponse := proto.S3QoSResponse{
ApiLimitConf: apiLimitConf,
}
@ -6561,9 +6544,9 @@ func (m *Server) S3QosGet(w http.ResponseWriter, r *http.Request) {
return true
}
if _, ok := apiLimitConf[api]; !ok {
bandWidthQuota := make(map[string]uint64, 0)
qpsQuota := make(map[string]uint64, 0)
concurrentQuota := make(map[string]uint64, 0)
bandWidthQuota := make(map[string]uint64)
qpsQuota := make(map[string]uint64)
concurrentQuota := make(map[string]uint64)
userLimitConf := &proto.UserLimitConf{
BandWidthQuota: bandWidthQuota,
QPSQuota: qpsQuota,

13
master/api_service_test.go
View File

@ -175,13 +175,13 @@ func createDefaultMasterServerForTest() *Server {
qosLimitArgs: &qosArgs{},
}
vol, err := testServer.cluster.createVol(req)
_, err = testServer.cluster.createVol(req)
if err != nil {
log.LogFlush()
panic(err)
}
vol, err = testServer.cluster.getVol(req.name)
vol, err := testServer.cluster.getVol(req.name)
if err != nil {
panic(err)
}
@ -307,11 +307,6 @@ func TestGetIpAndClusterName(t *testing.T) {
process(reqURL, t)
}
func fatal(t *testing.T, str string) {
log.LogFlush()
t.Fatal(str)
}
type httpReply = proto.HTTPReplyRaw
func processWithFatalV2(url string, success bool, req map[string]interface{}, t *testing.T) (reply *httpReply) {
@ -588,10 +583,6 @@ func setUpdateVolParm(key string, req map[string]interface{}, val interface{}, t
setParam(key, proto.AdminUpdateVol, req, val, t)
}
func checkUpdateVolParm(key string, req map[string]interface{}, wrong, correct interface{}, t *testing.T) {
checkParam(key, proto.AdminUpdateVol, req, wrong, correct, t)
}
func delVol(name string, t *testing.T) {
req := map[string]interface{}{
nameKey: name,

81
master/cluster.go
View File

@ -78,7 +78,6 @@ type Cluster struct {
MasterSecretKey []byte
lastZoneIdxForNode int
zoneIdxMux sync.Mutex //
zoneList []string
followerReadManager *followerReadManager
diskQosEnable bool
QosAcceptLimit *rate.Limiter
@ -172,7 +171,8 @@ func (mgr *followerReadManager) getVolumeDpView() {
}
for _, vv := range volViews {
if (vv.Status == proto.VolStatusMarkDelete && !vv.Forbidden) || (vv.Status == proto.VolStatusMarkDelete && vv.Forbidden && vv.DeleteExecTime.Sub(time.Now()) <= 0) {
if (vv.Status == proto.VolStatusMarkDelete && !vv.Forbidden) ||
(vv.Status == proto.VolStatusMarkDelete && vv.Forbidden && time.Until(vv.DeleteExecTime) <= 0) {
mgr.rwMutex.Lock()
mgr.lastUpdateTick[vv.Name] = time.Now()
mgr.status[vv.Name] = false
@ -194,7 +194,8 @@ func (mgr *followerReadManager) sendFollowerVolumeDpView() {
vols := mgr.c.copyVols()
for _, vol := range vols {
log.LogDebugf("followerReadManager.getVolumeDpView %v", vol.Name)
if (vol.Status == proto.VolStatusMarkDelete && !vol.Forbidden) || (vol.Status == proto.VolStatusMarkDelete && vol.Forbidden && vol.DeleteExecTime.Sub(time.Now()) <= 0) {
if (vol.Status == proto.VolStatusMarkDelete && !vol.Forbidden) ||
(vol.Status == proto.VolStatusMarkDelete && vol.Forbidden && time.Until(vol.DeleteExecTime) <= 0) {
continue
}
var body []byte
@ -225,12 +226,8 @@ func (mgr *followerReadManager) isVolRecordObsolete(volName string) bool {
// vol has been completely deleted
return true
}
if (volView.Status == proto.VolStatusMarkDelete && !volView.Forbidden) || (volView.Status == proto.VolStatusMarkDelete && volView.Forbidden && volView.DeleteExecTime.Sub(time.Now()) <= 0) {
return true
}
return false
return (volView.Status == proto.VolStatusMarkDelete && !volView.Forbidden) ||
(volView.Status == proto.VolStatusMarkDelete && volView.Forbidden && time.Until(volView.DeleteExecTime) <= 0)
}
func (mgr *followerReadManager) DelObsoleteVolRecord(obsoleteVolNames map[string]struct{}) {
@ -311,9 +308,9 @@ func (mgr *followerReadManager) getVolViewAsFollower(key string, compress bool)
defer mgr.rwMutex.RUnlock()
ok = true
if compress {
value, _ = mgr.volDataPartitionsCompress[key]
value = mgr.volDataPartitionsCompress[key]
} else {
value, _ = mgr.volDataPartitionsView[key]
value = mgr.volDataPartitionsView[key]
}
log.LogDebugf("getVolViewAsFollower. volume %v return!", key)
return
@ -332,9 +329,9 @@ func newCluster(name string, leaderInfo *LeaderInfo, fsm *MetadataFsm, partition
c = new(Cluster)
c.Name = name
c.leaderInfo = leaderInfo
c.vols = make(map[string]*Vol, 0)
c.vols = make(map[string]*Vol)
c.delayDeleteVolsInfo = make([]*delayDeleteVolInfo, 0)
c.stopc = make(chan bool, 0)
c.stopc = make(chan bool)
c.cfg = cfg
if c.cfg.MaxDpCntLimit == 0 {
c.cfg.MaxDpCntLimit = defaultMaxDpCntLimit
@ -484,7 +481,7 @@ func (c *Cluster) scheduleToCheckDelayDeleteVols() {
for index := 0; index < len(c.delayDeleteVolsInfo); index++ {
currentDeleteVol := c.delayDeleteVolsInfo[index]
log.LogDebugf("action[scheduleToCheckDelayDeleteVols] currentDeleteVol[%v]", currentDeleteVol)
if currentDeleteVol.execTime.Sub(time.Now()) > 0 {
if time.Until(currentDeleteVol.execTime) > 0 {
continue
}
go func() {
@ -578,7 +575,7 @@ func (c *Cluster) scheduleToCheckVolQos() {
func (c *Cluster) scheduleToCheckNodeSetGrpManagerStatus() {
go func() {
for {
if c.FaultDomain == false || !c.partition.IsRaftLeader() {
if !c.FaultDomain || !c.partition.IsRaftLeader() {
time.Sleep(time.Minute)
continue
}
@ -638,7 +635,8 @@ func (c *Cluster) doLoadDataPartitions() {
}()
vols := c.allVols()
for _, vol := range vols {
if (vol.Status == proto.VolStatusMarkDelete && !vol.Forbidden) || (vol.Status == proto.VolStatusMarkDelete && vol.Forbidden && vol.DeleteExecTime.Sub(time.Now()) <= 0) {
if (vol.Status == proto.VolStatusMarkDelete && !vol.Forbidden) ||
(vol.Status == proto.VolStatusMarkDelete && vol.Forbidden && time.Until(vol.DeleteExecTime) <= 0) {
continue
}
vol.loadDataPartition(c)
@ -703,10 +701,6 @@ func (c *Cluster) scheduleToCheckHeartbeat() {
}()
}
func (c *Cluster) passAclCheck(ip string) {
// do nothing
}
func (c *Cluster) checkLeaderAddr() {
leaderID, _ := c.partition.LeaderTerm()
c.leaderInfo.addr = AddrDatabase[leaderID]
@ -801,7 +795,6 @@ func (c *Cluster) checkLcNodeHeartbeat() {
log.LogInfof("checkLcNodeHeartbeat: deregister node(%v)", node)
_ = c.delLcNode(node)
}
return
}
func (c *Cluster) scheduleToCheckMetaPartitions() {
@ -1163,8 +1156,7 @@ func (c *Cluster) checkLackReplicaAndHostDataPartitions() (lackReplicaDataPartit
vols := c.copyVols()
var ids []uint64
for _, vol := range vols {
var dps *DataPartitionMap
dps = vol.dataPartitions
dps := vol.dataPartitions
for _, dp := range dps.partitions {
if dp.ReplicaNum > uint8(len(dp.Hosts)) && len(dp.Hosts) == len(dp.Replicas) && (dp.IsDecommissionInitial() || dp.IsRollbackFailed()) {
lackReplicaDataPartitions = append(lackReplicaDataPartitions, dp)
@ -1181,8 +1173,7 @@ func (c *Cluster) checkLackReplicaDataPartitions() (lackReplicaDataPartitions []
lackReplicaDataPartitions = make([]*DataPartition, 0)
vols := c.copyVols()
for _, vol := range vols {
var dps *DataPartitionMap
dps = vol.dataPartitions
dps := vol.dataPartitions
for _, dp := range dps.partitions {
if dp.ReplicaNum > uint8(len(dp.Hosts)) {
lackReplicaDataPartitions = append(lackReplicaDataPartitions, dp)
@ -1195,7 +1186,8 @@ func (c *Cluster) checkLackReplicaDataPartitions() (lackReplicaDataPartitions []
func (c *Cluster) checkReplicaOfDataPartitions(ignoreDiscardDp bool) (
lackReplicaDPs []*DataPartition, unavailableReplicaDPs []*DataPartition, repFileCountDifferDps []*DataPartition,
repUsedSizeDifferDps []*DataPartition, excessReplicaDPs []*DataPartition, noLeaderDPs []*DataPartition, err error) {
repUsedSizeDifferDps []*DataPartition, excessReplicaDPs []*DataPartition, noLeaderDPs []*DataPartition, err error,
) {
noLeaderDPs = make([]*DataPartition, 0)
lackReplicaDPs = make([]*DataPartition, 0)
unavailableReplicaDPs = make([]*DataPartition, 0)
@ -1203,14 +1195,14 @@ func (c *Cluster) checkReplicaOfDataPartitions(ignoreDiscardDp bool) (
vols := c.copyVols()
for _, vol := range vols {
var dps *DataPartitionMap
dps = vol.dataPartitions
dps := vol.dataPartitions
for _, dp := range dps.partitions {
if ignoreDiscardDp && dp.IsDiscard {
continue
}
if (vol.Status == proto.VolStatusMarkDelete && !vol.Forbidden) || (vol.Status == proto.VolStatusMarkDelete && vol.Forbidden && vol.DeleteExecTime.Sub(time.Now()) <= 0) {
if (vol.Status == proto.VolStatusMarkDelete && !vol.Forbidden) ||
(vol.Status == proto.VolStatusMarkDelete && vol.Forbidden && time.Until(vol.DeleteExecTime) <= 0) {
continue
}
@ -1376,7 +1368,6 @@ func (c *Cluster) deleteVol(name string) {
c.volMutex.Lock()
defer c.volMutex.Unlock()
delete(c.vols, name)
return
}
func (c *Cluster) markDeleteVol(name, authKey string, force bool, isNotCancel bool) (err error) {
@ -1652,7 +1643,8 @@ errHandler:
}
func (c *Cluster) syncCreateDataPartitionToDataNode(host string, size uint64, dp *DataPartition,
peers []proto.Peer, hosts []string, createType int, partitionType int, needRollBack, ignoreDecommissionDisk bool) (diskPath string, err error) {
peers []proto.Peer, hosts []string, createType int, partitionType int, needRollBack, ignoreDecommissionDisk bool,
) (diskPath string, err error) {
log.LogInfof("action[syncCreateDataPartitionToDataNode] dp [%v] createType[%v], partitionType[%v] ignoreDecommissionDisk[%v]",
dp.PartitionID, createType, partitionType, ignoreDecommissionDisk)
dataNode, err := c.dataNode(host)
@ -1766,7 +1758,8 @@ func (c *Cluster) chooseZone2Plus1(zones []*Zone, excludeNodeSets []uint64, excl
}
func (c *Cluster) chooseZoneNormal(zones []*Zone, excludeNodeSets []uint64, excludeHosts []string,
nodeType uint32, replicaNum int) (hosts []string, peers []proto.Peer, err error) {
nodeType uint32, replicaNum int,
) (hosts []string, peers []proto.Peer, err error) {
log.LogInfof("action[chooseZoneNormal] zones[%s] nodeType[%d] replicaNum[%d]", printZonesName(zones), nodeType, replicaNum)
c.zoneIdxMux.Lock()
@ -1834,8 +1827,6 @@ func (c *Cluster) getHostFromNormalZone(nodeType uint32, excludeZones []string,
goto result
}
hosts = make([]string, 0)
peers = make([]proto.Peer, 0)
if excludeHosts == nil {
excludeHosts = make([]string, 0)
}
@ -2085,7 +2076,7 @@ func (c *Cluster) delDecommissionDiskFromCache(dd *DecommissionDisk) {
func (c *Cluster) decommissionSingleDp(dp *DataPartition, newAddr, offlineAddr string) (err error) {
var (
dataNode *DataNode
decommContinue = false
decommContinue bool
newReplica *DataReplica
)
@ -2140,7 +2131,7 @@ func (c *Cluster) decommissionSingleDp(dp *DataPartition, newAddr, offlineAddr s
log.LogInfof("action[decommissionSingleDp] dp %v replica[%v] status %v",
dp.PartitionID, newReplica.Addr, newReplica.Status)
if newReplica.isRepairing() { // wait for repair
if time.Now().Sub(dp.RecoverStartTime) > c.GetDecommissionDataPartitionRecoverTimeOut() {
if time.Since(dp.RecoverStartTime) > c.GetDecommissionDataPartitionRecoverTimeOut() {
err = fmt.Errorf("action[decommissionSingleDp] dp %v new replica %v repair time out",
dp.PartitionID, newAddr)
dp.DecommissionNeedRollback = true
@ -3621,7 +3612,7 @@ func (c *Cluster) allVolNames() (vols []string) {
}
func (c *Cluster) copyVols() (vols map[string]*Vol) {
vols = make(map[string]*Vol, 0)
vols = make(map[string]*Vol)
c.volMutex.RLock()
defer c.volMutex.RUnlock()
@ -3634,7 +3625,7 @@ func (c *Cluster) copyVols() (vols map[string]*Vol) {
// Return all the volumes except the ones that have been marked to be deleted.
func (c *Cluster) allVols() (vols map[string]*Vol) {
vols = make(map[string]*Vol, 0)
vols = make(map[string]*Vol)
c.volMutex.RLock()
defer c.volMutex.RUnlock()
for name, vol := range c.vols {
@ -3793,11 +3784,6 @@ func (c *Cluster) setMetaNodeDeleteWorkerSleepMs(val uint64) (err error) {
return
}
func (c *Cluster) getMaxDpCntLimit() (dpCntInLimit uint64) {
dpCntInLimit = atomic.LoadUint64(&c.cfg.MaxDpCntLimit)
return
}
func (c *Cluster) setMaxDpCntLimit(val uint64) (err error) {
if val == 0 {
val = defaultMaxDpCntLimit
@ -3863,8 +3849,8 @@ func (c *Cluster) setMaxConcurrentLcNodes(count uint64) (err error) {
func (c *Cluster) clearVols() {
c.volMutex.Lock()
defer c.volMutex.Unlock()
c.vols = make(map[string]*Vol, 0)
c.vols = make(map[string]*Vol)
c.volMutex.Unlock()
}
func (c *Cluster) clearTopology() {
@ -3911,7 +3897,7 @@ func (c *Cluster) checkDecommissionDataNode() {
partitions := c.getAllDataPartitionByDataNode(dataNode.Addr)
// if only decommission part of data partitions, do not remove the data node
if len(partitions) != 0 {
if time.Now().Sub(time.Unix(dataNode.DecommissionCompleteTime, 0)) > (20 * time.Minute) {
if time.Since(time.Unix(dataNode.DecommissionCompleteTime, 0)) > (20 * time.Minute) {
log.LogWarnf("action[checkDecommissionDataNode] dataNode %v decommission completed, "+
"but has dp left, so only reset decommission status", dataNode.Addr)
dataNode.resetDecommissionStatus()
@ -4185,7 +4171,7 @@ func (c *Cluster) checkDecommissionDisk() {
// keep failed decommission disk in list for preventing the reuse of a
// term in future decommissioning operations
if status == DecommissionSuccess {
if time.Now().Sub(time.Unix(disk.DecommissionCompleteTime, 0)) > (20 * time.Minute) {
if time.Since(time.Unix(disk.DecommissionCompleteTime, 0)) > (20 * time.Minute) {
if err := c.syncDeleteDecommissionDisk(disk); err != nil {
msg := fmt.Sprintf("action[checkDecommissionDisk],clusterID[%v] node[%v] disk[%v],"+
"syncDeleteDecommissionDisk failed,err[%v]",
@ -4706,7 +4692,6 @@ func (c *Cluster) DelBucketLifecycle(VolName string) {
}
c.lcMgr.DelS3BucketLifecycle(VolName)
log.LogInfof("action[DelS3BucketLifecycle],clusterID[%v] vol:%v", c.Name, VolName)
return
}
func (c *Cluster) addDecommissionDiskToNodeset(dd *DecommissionDisk) (err error) {

12
master/cluster_task.go
View File

@ -295,7 +295,8 @@ type VolNameSet map[string]struct{}
func (c *Cluster) checkReplicaMetaPartitions() (
lackReplicaMetaPartitions []*MetaPartition, noLeaderMetaPartitions []*MetaPartition,
unavailableReplicaMPs []*MetaPartition, excessReplicaMetaPartitions, inodeCountNotEqualMPs, maxInodeNotEqualMPs, dentryCountNotEqualMPs []*MetaPartition, err error) {
unavailableReplicaMPs []*MetaPartition, excessReplicaMetaPartitions, inodeCountNotEqualMPs, maxInodeNotEqualMPs, dentryCountNotEqualMPs []*MetaPartition, err error,
) {
lackReplicaMetaPartitions = make([]*MetaPartition, 0)
noLeaderMetaPartitions = make([]*MetaPartition, 0)
excessReplicaMetaPartitions = make([]*MetaPartition, 0)
@ -501,10 +502,8 @@ func (c *Cluster) addMetaReplica(partition *MetaPartition, addr string) (err err
if err = c.addMetaPartitionRaftMember(partition, addPeer); err != nil {
return
}
newHosts := make([]string, 0, len(partition.Hosts)+1)
newPeers := make([]proto.Peer, 0, len(partition.Hosts)+1)
newHosts = append(partition.Hosts, addPeer.Addr)
newPeers = append(partition.Peers, addPeer)
newHosts := append(partition.Hosts, addPeer.Addr)
newPeers := append(partition.Peers, addPeer)
if err = partition.persistToRocksDB("addMetaReplica", partition.volName, newHosts, newPeers, c); err != nil {
return
}
@ -886,7 +885,6 @@ func (c *Cluster) adjustMetaNode(metaNode *MetaNode) {
return
}
err = c.t.putMetaNode(metaNode)
return
}
func (c *Cluster) handleDataNodeTaskResponse(nodeAddr string, task *proto.AdminTask) {
@ -935,7 +933,6 @@ func (c *Cluster) handleDataNodeTaskResponse(nodeAddr string, task *proto.AdminT
errHandler:
log.LogErrorf("process task[%v] failed,err:%v", task.ToString(), err)
return
}
func (c *Cluster) dealDeleteDataPartitionResponse(nodeAddr string, resp *proto.DeleteDataPartitionResponse) (err error) {
@ -1069,7 +1066,6 @@ func (c *Cluster) adjustDataNode(dataNode *DataNode) {
return
}
err = c.t.putDataNode(dataNode)
return
}
/*if node report data partition infos,so range data partition infos,then update data partition info*/

7
master/cluster_test.go
View File

@ -97,7 +97,6 @@ func TestPanicCheckMetaPartitions(t *testing.T) {
}
mp := newMetaPartition(partitionID, 1, defaultMaxMetaPartitionInodeID, vol.mpReplicaNum, vol.Name, vol.ID, 0)
vol.addMetaPartition(mp)
mp = nil
c.checkMetaPartitions()
t.Logf("catched panic")
}
@ -152,9 +151,7 @@ func TestCheckBadDiskRecovery(t *testing.T) {
}
vol.volLock.RLock()
dps := make([]*DataPartition, 0)
for _, dp := range vol.dataPartitions.partitions {
dps = append(dps, dp)
}
dps = append(dps, vol.dataPartitions.partitions...)
dpsMapLen := len(vol.dataPartitions.partitionMap)
vol.volLock.RUnlock()
dpsLen := len(dps)
@ -165,7 +162,6 @@ func TestCheckBadDiskRecovery(t *testing.T) {
for _, dp := range dps {
dp.RLock()
if len(dp.Replicas) == 0 {
dpsLen--
dp.RUnlock()
return
}
@ -241,7 +237,6 @@ func TestCheckBadMetaPartitionRecovery(t *testing.T) {
for _, mp := range mps {
mp.RLock()
if len(mp.Replicas) == 0 {
mpsLen--
mp.RUnlock()
return
}

2
master/config.go
View File

@ -90,9 +90,9 @@ const (
defaultMaxDpCntLimit = 3000
defaultIntervalToScanS3Expiration = 12 * 3600
defaultMaxConcurrentLcNodes = 3
defaultIntervalToCheckDelVerTaskExpiration = 3
metaPartitionInodeUsageThreshold float64 = 0.75 // inode usage threshold on a meta partition
lowerLimitRWMetaPartition = 3 // lower limit of RW meta partition, equal defaultReplicaNum
// defaultIntervalToCheckDelVerTaskExpiration = 3
)
// AddrDatabase is a map that stores the address of a given host (e.g., the leader)

29
master/const.go
View File

@ -149,7 +149,6 @@ const (
checkDataPartitionDiskErr = "checkDataPartitionDiskErr "
dataNodeOfflineErr = "dataNodeOfflineErr "
diskOfflineErr = "diskOfflineErr "
handleDataPartitionOfflineErr = "handleDataPartitionOffLineErr "
)
const (
@ -351,3 +350,31 @@ const (
DataNodeType = NodeType(0)
MetaNodeType = NodeType(iota)
)
// TODO: to remove unused by golangci
var (
_ = startKey
_ = nodeHostsKey
_ = fullPathKey
_ = inodeKey
_ = dataNodeOfflineErr
_ = defaultMigrateDpCnt
_ = idSeparator
_ = volCachePrefix
_ = opSyncDataPartitionsView
_ = opSyncUpdateLcNode
_ = (createVolReq{}).clientReqPeriod
_ = (createVolReq{}).clientHitTriggerCnt
_ = (*Server).createDomainHandler
_ = (*VolumeService).markDeleteVol
__c = (*Cluster)(nil)
_ = __c.checkLackReplicaDataPartitions
_ = __c.getAllMetaPartitionsByMetaNode
_ = __c.isRecovering
_ = __c.updateInodeIDRange
_ = __c.setMaxConcurrentLcNodes
_ = __c.checkCorruptMetaNode
)

13
master/data_node.go
View File

@ -101,8 +101,6 @@ func (dataNode *DataNode) checkLiveness() {
if time.Since(dataNode.ReportTime) > time.Second*time.Duration(defaultNodeTimeOutSec) {
dataNode.isActive = false
}
return
}
func (dataNode *DataNode) badPartitions(diskPath string, c *Cluster) (partitions []*DataPartition) {
@ -171,17 +169,10 @@ func (dataNode *DataNode) updateNodeMetric(resp *proto.DataNodeHeartbeatResponse
func (dataNode *DataNode) canAlloc() bool {
dataNode.RLock()
defer dataNode.RUnlock()
if !overSoldLimit() {
return true
}
maxCapacity := overSoldCap(dataNode.Total)
if maxCapacity < dataNode.TotalPartitionSize {
return false
}
return true
return overSoldCap(dataNode.Total) >= dataNode.TotalPartitionSize
}
func (dataNode *DataNode) isWriteAble() (ok bool) {
@ -224,7 +215,7 @@ func (dataNode *DataNode) isWriteAbleWithSize(size uint64) (ok bool) {
dataNode.RLock()
defer dataNode.RUnlock()
if dataNode.isActive == true && dataNode.AvailableSpace > size {
if dataNode.isActive && dataNode.AvailableSpace > size {
ok = true
}

58
master/data_partition.go
View File

@ -97,7 +97,7 @@ func newDataPartition(ID uint64, replicaNum uint8, volName string, volID uint64,
partition.Hosts = make([]string, 0)
partition.Peers = make([]proto.Peer, 0)
partition.Replicas = make([]*DataReplica, 0)
partition.FileInCoreMap = make(map[string]*FileInCore, 0)
partition.FileInCoreMap = make(map[string]*FileInCore)
partition.FilesWithMissingReplica = make(map[string]int64)
partition.MissingNodes = make(map[string]int64)
@ -130,14 +130,6 @@ func (partition *DataPartition) isSpecialReplicaCnt() bool {
return partition.ReplicaNum == 1 || partition.ReplicaNum == 2
}
func (partition *DataPartition) isSingleReplica() bool {
return partition.ReplicaNum == 1
}
func (partition *DataPartition) isTwoReplica() bool {
return partition.ReplicaNum == 2
}
func (partition *DataPartition) resetFilesWithMissingReplica() {
partition.Lock()
defer partition.Unlock()
@ -353,21 +345,6 @@ func (partition *DataPartition) canBeOffLine(offlineAddr string) (err error) {
return
}
// get all the valid replicas of the given data partition
func (partition *DataPartition) availableDataReplicas() (replicas []*DataReplica) {
replicas = make([]*DataReplica, 0)
for i := 0; i < len(partition.Replicas); i++ {
replica := partition.Replicas[i]
// the node reports heartbeat normally and the node is available
if replica.isLocationAvailable() == true && partition.hasHost(replica.Addr) == true {
replicas = append(replicas, replica)
}
}
return
}
// Remove the replica address from the memory.
func (partition *DataPartition) removeReplicaByAddr(addr string) {
delIndex := -1
@ -385,11 +362,9 @@ func (partition *DataPartition) removeReplicaByAddr(addr string) {
if delIndex == -1 {
return
}
partition.FileInCoreMap = make(map[string]*FileInCore, 0)
partition.FileInCoreMap = make(map[string]*FileInCore)
partition.deleteReplicaByIndex(delIndex)
partition.modifyTime = time.Now().Unix()
return
}
func (partition *DataPartition) deleteReplicaByIndex(index int) {
@ -409,7 +384,7 @@ func (partition *DataPartition) createLoadTasks() (tasks []*proto.AdminTask) {
defer partition.Unlock()
for _, addr := range partition.Hosts {
replica, err := partition.getReplica(addr)
if err != nil || replica.isLive(defaultDataPartitionTimeOutSec) == false {
if err != nil || !replica.isLive(defaultDataPartitionTimeOutSec) {
continue
}
replica.HasLoadResponse = false
@ -486,13 +461,13 @@ func (partition *DataPartition) checkLoadResponse(timeOutSec int64) (isResponse
return
}
timePassed := time.Now().Unix() - partition.LastLoadedTime
if replica.HasLoadResponse == false && timePassed > timeToWaitForResponse {
if !replica.HasLoadResponse && timePassed > timeToWaitForResponse {
msg := fmt.Sprintf("action[checkLoadResponse], partitionID:%v on node:%v no response, spent time %v s",
partition.PartitionID, addr, timePassed)
log.LogWarn(msg)
return
}
if replica.isLive(timeOutSec) == false || replica.HasLoadResponse == false {
if !replica.isLive(timeOutSec) || !replica.HasLoadResponse {
log.LogInfof("action[checkLoadResponse] partitionID:%v getReplica addr %v replica.isLive(timeOutSec) %v", partition.PartitionID, addr, replica.isLive(timeOutSec))
return
}
@ -540,7 +515,7 @@ func (partition *DataPartition) releaseDataPartition() {
fc.MetadataArray = nil
delete(partition.FileInCoreMap, name)
}
partition.FileInCoreMap = make(map[string]*FileInCore, 0)
partition.FileInCoreMap = make(map[string]*FileInCore)
for name, fileMissReplicaTime := range partition.FilesWithMissingReplica {
if time.Now().Unix()-fileMissReplicaTime > 2*intervalToLoadDataPartition {
delete(partition.FilesWithMissingReplica, name)
@ -624,7 +599,7 @@ func (partition *DataPartition) getLiveReplicasFromHosts(timeOutSec int64) (repl
if !ok {
continue
}
if replica.isLive(timeOutSec) == true {
if replica.isLive(timeOutSec) {
replicas = append(replicas, replica)
} else {
replica.Status = proto.Unavailable
@ -640,7 +615,7 @@ func (partition *DataPartition) getLiveReplicasFromHosts(timeOutSec int64) (repl
func (partition *DataPartition) getLiveReplicas(timeOutSec int64) (replicas []*DataReplica) {
replicas = make([]*DataReplica, 0)
for _, replica := range partition.Replicas {
if replica.isLive(timeOutSec) == true {
if replica.isLive(timeOutSec) {
replicas = append(replicas, replica)
} else {
replica.Status = proto.Unavailable
@ -653,9 +628,7 @@ func (partition *DataPartition) getLiveReplicas(timeOutSec int64) (replicas []*D
}
func (partition *DataPartition) checkAndRemoveMissReplica(addr string) {
if _, ok := partition.MissingNodes[addr]; ok {
delete(partition.MissingNodes, addr)
}
}
func (partition *DataPartition) loadFile(dataNode *DataNode, resp *proto.LoadDataPartitionResponse) {
@ -833,18 +806,6 @@ func (partition *DataPartition) getReplicaDisk(nodeAddr string) string {
return ""
}
func (partition *DataPartition) getMinus() (minus float64) {
partition.RLock()
defer partition.RUnlock()
used := partition.Replicas[0].Used
for _, replica := range partition.Replicas {
if math.Abs(float64(replica.Used)-float64(used)) > minus {
minus = math.Abs(float64(replica.Used) - float64(used))
}
}
return minus
}
func (partition *DataPartition) activeUsedSimilar() bool {
partition.RLock()
defer partition.RUnlock()
@ -1042,7 +1003,6 @@ const (
defaultDecommissionParallelLimit = 10
defaultDecommissionRetryLimit = 5
defaultDecommissionRollbackLimit = 3
defaultDecommissionDiskParallelFactor = 0
)
func GetDecommissionStatusMessage(status uint32) string {
@ -1501,7 +1461,7 @@ func (partition *DataPartition) pauseReplicaRepair(replicaAddr string, stop bool
log.LogDebugf("action[pauseReplicaRepair]dp[%v] replica %v RecoverStartTime sub %v seconds",
partition.PartitionID, replicaAddr, partition.RecoverLastConsumeTime.Seconds())
} else {
partition.RecoverLastConsumeTime = time.Now().Sub(partition.RecoverStartTime)
partition.RecoverLastConsumeTime = time.Since(partition.RecoverStartTime)
log.LogDebugf("action[pauseReplicaRepair]dp[%v] replica %v already recover %v seconds",
partition.PartitionID, replicaAddr, partition.RecoverLastConsumeTime.Seconds())
}

29
master/data_partition_check.go
View File

@ -26,7 +26,8 @@ import (
)
func (partition *DataPartition) checkStatus(clusterName string, needLog bool, dpTimeOutSec int64, c *Cluster,
shouldDpInhibitWriteByVolFull bool, forbiddenVol bool) {
shouldDpInhibitWriteByVolFull bool, forbiddenVol bool,
) {
partition.Lock()
defer partition.Unlock()
var liveReplicas []*DataReplica
@ -81,7 +82,7 @@ func (partition *DataPartition) checkStatus(clusterName string, needLog bool, dp
partition.Status = proto.Unavailable
}
if needLog == true && len(liveReplicas) != int(partition.ReplicaNum) {
if needLog && len(liveReplicas) != int(partition.ReplicaNum) {
msg := fmt.Sprintf("action[extractStatus],partitionID:%v replicaNum:%v liveReplicas:%v Status:%v RocksDBHost:%v ",
partition.PartitionID, partition.ReplicaNum, len(liveReplicas), partition.Status, partition.Hosts)
log.LogInfo(msg)
@ -94,22 +95,7 @@ func (partition *DataPartition) checkStatus(clusterName string, needLog bool, dp
func (partition *DataPartition) hasEnoughAvailableSpace() bool {
avail := partition.total - partition.used
if int64(avail) > 10*util.GB {
return true
}
return false
}
func (partition *DataPartition) checkReplicaNotHaveStatus(liveReplicas []*DataReplica, status int8) (equal bool) {
for _, replica := range liveReplicas {
if replica.Status == status {
log.LogInfof("action[checkReplicaNotHaveStatus] partition %v replica %v status %v dst status %v",
partition.PartitionID, replica.Addr, replica.Status, status)
return
}
}
return true
return int64(avail) > 10*util.GB
}
func (partition *DataPartition) checkReplicaEqualStatus(liveReplicas []*DataReplica, status int8) (equal bool) {
@ -165,7 +151,6 @@ func (partition *DataPartition) checkLeader(clusterID string, timeOut int64) {
if WarnMetrics != nil {
WarnMetrics.WarnDpNoLeader(clusterID, partition.PartitionID, report)
}
return
}
// Check if there is any missing replica for a data partition.
@ -269,7 +254,7 @@ func (partition *DataPartition) hasMissingDataPartition(addr string) (isMissing
}
func (partition *DataPartition) checkDiskError(clusterID, leaderAddr string) {
diskErrorAddrs := make(map[string]string, 0)
diskErrorAddrs := make(map[string]string)
partition.Lock()
defer partition.Unlock()
@ -299,8 +284,6 @@ func (partition *DataPartition) checkDiskError(clusterID, leaderAddr string) {
checkDataPartitionDiskErr, clusterID, partition.PartitionID, addr, leaderAddr, addr, diskPath)
Warn(clusterID, msg)
}
return
}
func (partition *DataPartition) checkReplicationTask(clusterID string, dataPartitionSize uint64) {
@ -324,8 +307,6 @@ func (partition *DataPartition) checkReplicationTask(clusterID string, dataParti
addMissingReplicaErr, partition.PartitionID, lackAddr, lackErr.Error(), partition.Hosts)
Warn(clusterID, msg)
}
return
}
func (partition *DataPartition) deleteIllegalReplica() (excessAddr string, err error) {

12
master/data_partition_map.go
View File

@ -43,7 +43,7 @@ type DataPartitionMap struct {
func newDataPartitionMap(volName string) (dpMap *DataPartitionMap) {
dpMap = new(DataPartitionMap)
dpMap.partitionMap = make(map[uint64]*DataPartition, 0)
dpMap.partitionMap = make(map[uint64]*DataPartition)
dpMap.partitions = make([]*DataPartition, 0)
dpMap.responseCache = make([]byte, 0)
dpMap.responseCompressCache = make([]byte, 0)
@ -55,13 +55,9 @@ func newDataPartitionMap(volName string) (dpMap *DataPartitionMap) {
// attention: it's not deep clone for element, dataPartition
func (dpMap *DataPartitionMap) clonePartitions() []*DataPartition {
dpMap.RLock()
defer dpMap.RUnlock()
partitions := make([]*DataPartition, 0)
for _, dp := range dpMap.partitions {
partitions = append(partitions, dp)
}
partitions := make([]*DataPartition, 0, len(dpMap.partitions))
partitions = append(partitions, dpMap.partitions...)
dpMap.RUnlock()
return partitions
}

1
master/data_partition_test.go
View File

@ -23,6 +23,7 @@ func TestDataPartition(t *testing.T) {
partition := commonVol.dataPartitions.partitions[0]
getDataPartition(partition.PartitionID, t)
loadDataPartitionTest(partition, t)
_ = decommissionDataPartition
// decommissionDataPartition(partition, t)
}

14
master/data_replica.go
View File

@ -25,7 +25,7 @@ import (
type DataReplica struct {
proto.DataReplica
dataNode *DataNode
loc uint8
// loc uint8
}
func newDataReplica(dataNode *DataNode) (replica *DataReplica) {
@ -65,18 +65,6 @@ func (replica *DataReplica) getReplicaNode() (node *DataNode) {
return replica.dataNode
}
// check if the replica's location is available
func (replica *DataReplica) isLocationAvailable() (isAvailable bool) {
dataNode := replica.getReplicaNode()
dataNode.Lock()
defer dataNode.Unlock()
if dataNode.isActive == true && replica.isActive(defaultDataPartitionTimeOutSec) == true {
isAvailable = true
}
return
}
func (replica *DataReplica) isRepairing() bool {
return replica.Status == proto.Recovering
}

6
master/disk_manager.go
View File

@ -94,7 +94,6 @@ func (c *Cluster) checkDiskRecoveryProgress() {
err = c.syncUpdateDataPartition(partition)
if err != nil {
log.LogErrorf("[checkDiskRecoveryProgress] update dp(%v) fail, err(%v)", partitionID, err)
err = nil
}
partition.RUnlock()
continue
@ -114,7 +113,6 @@ func (c *Cluster) checkDiskRecoveryProgress() {
err = c.syncUpdateDataPartition(partition)
if err != nil {
log.LogErrorf("[checkDiskRecoveryProgress] update dp(%v) fail, err(%v)", partitionID, err)
err = nil
}
partition.RUnlock()
} else {
@ -139,7 +137,6 @@ func (c *Cluster) checkDiskRecoveryProgress() {
err = c.syncUpdateDataPartition(partition)
if err != nil {
log.LogErrorf("[checkDiskRecoveryProgress] update dp(%v) fail, err(%v)", partitionID, err)
err = nil
}
partition.RUnlock()
}
@ -182,7 +179,8 @@ func (c *Cluster) deleteAndSyncDecommissionedDisk(dataNode *DataNode, diskPath s
}
func (c *Cluster) decommissionDisk(dataNode *DataNode, raftForce bool, badDiskPath string,
badPartitions []*DataPartition, diskDisable bool) (err error) {
badPartitions []*DataPartition, diskDisable bool,
) (err error) {
msg := fmt.Sprintf("action[decommissionDisk], Node[%v] OffLine,disk[%v]", dataNode.Addr, badDiskPath)
log.LogWarn(msg)

7
master/filecheck.go
View File

@ -48,7 +48,6 @@ func (partition *DataPartition) validateCRC(clusterID string) {
Warn(clusterID, fmt.Sprintf("vol[%v],dpId[%v],liveAddrs[%v],inactiveAddrs[%v]", partition.VolName, partition.PartitionID, liveAddrs, inactiveAddrs))
}
partition.doValidateCRC(liveReplicas, clusterID)
return
}
func (partition *DataPartition) doValidateCRC(liveReplicas []*DataReplica, clusterID string) {
@ -73,7 +72,7 @@ func (partition *DataPartition) doValidateCRC(liveReplicas []*DataReplica, clust
}
func (partition *DataPartition) checkTinyExtentFile(fc *FileInCore, liveReplicas []*DataReplica, clusterID string, getInfoCallback func() string) {
if fc.shouldCheckCrc() == false {
if !fc.shouldCheckCrc() {
return
}
fms, needRepair := fc.needCrcRepair(liveReplicas, getInfoCallback)
@ -93,11 +92,10 @@ func (partition *DataPartition) checkTinyExtentFile(fc *FileInCore, liveReplicas
msg = msg + fmt.Sprintf("fm[%v]:%v\n", fm.locIndex, fm)
}
Warn(clusterID, msg)
return
}
func (partition *DataPartition) checkExtentFile(fc *FileInCore, liveReplicas []*DataReplica, clusterID string, getInfoCallback func() string) {
if fc.shouldCheckCrc() == false {
if !fc.shouldCheckCrc() {
return
}
fms, needRepair := fc.needCrcRepair(liveReplicas, getInfoCallback)
@ -155,5 +153,4 @@ func (partition *DataPartition) checkExtentFile(fc *FileInCore, liveReplicas []*
Warn(clusterID, msg)
}
}
return
}

2
master/filecrc.go
View File

@ -122,7 +122,7 @@ func (fc *FileInCore) calculateCrc(badVfNodes []*FileMetadata) (fileCrcArr []*Fi
}
}
if isFound == false {
if !isFound {
crc = newFileCrc(crcKey)
crc.meta = badVfNodes[i]
fileCrcArr = append(fileCrcArr, crc)

2
master/filemeta.go
View File

@ -99,7 +99,7 @@ func (fc *FileInCore) updateFileInCore(volID uint64, vf *proto.File, volLoc *Dat
}
}
if isFind == false {
if !isFind {
fm := newFileMetadata(vf.Crc, volLoc.Addr, volLocIndex, vf.Size, vf.ApplyID)
fc.MetadataArray = append(fc.MetadataArray, fm)
}

54
master/gapi_cluster.go
View File

@ -227,7 +227,8 @@ func (m *ClusterService) decommissionDataNode(ctx context.Context, args struct {
func (m *ClusterService) decommissionMetaNode(ctx context.Context, args struct {
OffLineAddr string
}) (*proto.GeneralResp, error) {
},
) (*proto.GeneralResp, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
@ -244,7 +245,8 @@ func (m *ClusterService) decommissionMetaNode(ctx context.Context, args struct {
func (m *ClusterService) loadMetaPartition(ctx context.Context, args struct {
PartitionID uint64
}) (*proto.GeneralResp, error) {
},
) (*proto.GeneralResp, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
@ -261,7 +263,8 @@ func (m *ClusterService) loadMetaPartition(ctx context.Context, args struct {
func (m *ClusterService) decommissionMetaPartition(ctx context.Context, args struct {
PartitionID uint64
NodeAddr string
}) (*proto.GeneralResp, error) {
},
) (*proto.GeneralResp, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
@ -276,19 +279,6 @@ func (m *ClusterService) decommissionMetaPartition(ctx context.Context, args str
return proto.Success("success"), nil
}
func (m *ClusterService) getMetaNode(ctx context.Context, args struct {
NodeAddr string
}) (*MetaNode, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
metaNode, err := m.cluster.metaNode(args.NodeAddr)
if err != nil {
return nil, err
}
return metaNode, nil
}
// View the topology of the cluster.
func (m *ClusterService) getTopology(ctx context.Context, args struct{}) (*proto.GeneralResp, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
@ -361,7 +351,8 @@ func (s *ClusterService) masterList(ctx context.Context, args struct{}) ([]*Mast
func (s *ClusterService) dataNodeGet(ctx context.Context, args struct {
Addr string
}) (*DataNode, error) {
},
) (*DataNode, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
@ -382,7 +373,8 @@ func (s *ClusterService) dataNodeList(ctx context.Context, args struct{}) ([]*Da
func (s *ClusterService) dataNodeListTest(ctx context.Context, args struct {
Num int64
}) ([]*DataNode, error) {
},
) ([]*DataNode, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
@ -409,7 +401,8 @@ func (s *ClusterService) dataNodeListTest(ctx context.Context, args struct {
func (s *ClusterService) metaNodeGet(ctx context.Context, args struct {
Addr string
}) (*MetaNode, error) {
},
) (*MetaNode, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
@ -432,22 +425,12 @@ func (s *ClusterService) metaNodeList(ctx context.Context, args struct{}) ([]*Me
return all, nil
}
func (m *ClusterService) addMetaNode(ctx context.Context, args struct {
NodeAddr string
ZoneName string
}) (uint64, error) {
if id, err := m.cluster.addMetaNode(args.NodeAddr, args.ZoneName, 0); err != nil {
return 0, err
} else {
return id, nil
}
}
// Dynamically remove a master node. Similar to addRaftNode, this operation is performed online.
func (m *ClusterService) removeRaftNode(ctx context.Context, args struct {
Id uint64
Addr string
}) (*proto.GeneralResp, error) {
},
) (*proto.GeneralResp, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
@ -463,7 +446,8 @@ func (m *ClusterService) removeRaftNode(ctx context.Context, args struct {
func (m *ClusterService) addRaftNode(ctx context.Context, args struct {
Id uint64
Addr string
}) (*proto.GeneralResp, error) {
},
) (*proto.GeneralResp, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
@ -486,7 +470,8 @@ func (m *ClusterService) addRaftNode(ctx context.Context, args struct {
// 2. and the number of r&w data partition is less than 20.
func (m *ClusterService) clusterFreeze(ctx context.Context, args struct {
Status bool
}) (*proto.GeneralResp, error) {
},
) (*proto.GeneralResp, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
@ -508,7 +493,8 @@ type WarnMessage struct {
func (m *ClusterService) alarmList(ctx context.Context, args struct {
Size int32
}) ([]*WarnMessage, error) {
},
) ([]*WarnMessage, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}

17
master/gapi_user.go
View File

@ -85,8 +85,9 @@ func (s *UserService) registerQuery(schema *schemabuilder.Schema) {
func (m *UserService) getUserAKInfo(ctx context.Context, args struct {
AccessKey string
}) (*proto.UserInfo, error) {
uid, perm, err := permissions(ctx, ADMIN|USER)
},
) (*proto.UserInfo, error) {
uid, perm, _ := permissions(ctx, ADMIN|USER)
userInfo, err := m.user.getKeyInfo(args.AccessKey)
if err != nil {
return nil, err
@ -222,7 +223,8 @@ func (s *UserService) updateUser(ctx context.Context, args proto.UserUpdateParam
func (s *UserService) deleteUser(ctx context.Context, args struct {
UserID string
}) (*proto.GeneralResp, error) {
},
) (*proto.GeneralResp, error) {
uid, _, err := permissions(ctx, ADMIN)
if err != nil {
return nil, err
@ -239,7 +241,8 @@ func (s *UserService) deleteUser(ctx context.Context, args struct {
func (s *UserService) getUserInfo(ctx context.Context, args struct {
UserID string
}) (*proto.UserInfo, error) {
},
) (*proto.UserInfo, error) {
uid, perm, err := permissions(ctx, ADMIN|USER)
if err != nil {
return nil, err
@ -274,7 +277,8 @@ type UserUseSpace struct {
func (s *UserService) topNUser(ctx context.Context, args struct {
N int32
}) ([]*UserUseSpace, error) {
},
) ([]*UserUseSpace, error) {
if _, _, err := permissions(ctx, ADMIN); err != nil {
return nil, err
}
@ -334,7 +338,8 @@ func (s *UserService) topNUser(ctx context.Context, args struct {
func (s *UserService) validatePassword(ctx context.Context, args struct {
UserID string
Password string
}) (*proto.UserInfo, error) {
},
) (*proto.UserInfo, error) {
ui, err := s.user.getUserInfo(args.UserID)
if err != nil {
return nil, err

3
master/gapi_volume.go
View File

@ -366,7 +366,8 @@ func (s *VolumeService) listVolume(ctx context.Context, args struct {
continue
}
if (vol.Status == proto.VolStatusMarkDelete && !vol.Forbidden) || (vol.Status == proto.VolStatusMarkDelete && vol.Forbidden && vol.DeleteExecTime.Sub(time.Now()) <= 0) {
if (vol.Status == proto.VolStatusMarkDelete && !vol.Forbidden) ||
(vol.Status == proto.VolStatusMarkDelete && vol.Forbidden && time.Until(vol.DeleteExecTime) <= 0) {
continue
}

3
master/http_server.go
View File

@ -62,7 +62,6 @@ func (m *Server) startHTTPService(modulename string, cfg *config.Config) {
}
go serveAPI()
m.apiServer = server
return
}
func (m *Server) isClientPartitionsReq(r *http.Request) bool {
@ -771,7 +770,7 @@ func (m *Server) registerHandler(router *mux.Router, model string, schema *graph
gHandler := graphql.HTTPHandler(schema)
router.NewRoute().Name(model).Methods(http.MethodGet, http.MethodPost).Path(model).HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
userID := request.Header.Get(proto.UserKey)
userID := request.Header.Get(string(proto.UserKey))
if userID == "" {
ErrResponse(writer, fmt.Errorf("not found [%s] in header", proto.UserKey))
return

9
master/lifecycle_manager.go
View File

@ -90,7 +90,7 @@ func (lcMgr *lifecycleManager) scanning() bool {
}
for _, v := range lcMgr.lcRuleTaskStatus.Results {
if v.Done != true && time.Now().Before(v.UpdateTime.Add(time.Minute*10)) {
if !v.Done && time.Now().Before(v.UpdateTime.Add(time.Minute*10)) {
return true
}
}
@ -232,19 +232,16 @@ func (ns *lcNodeStatus) GetIdleNode() (nodeAddr string) {
func (ns *lcNodeStatus) RemoveNode(nodeAddr string) {
ns.Lock()
defer ns.Unlock()
delete(ns.WorkingCount, nodeAddr)
return
ns.Unlock()
}
func (ns *lcNodeStatus) UpdateNode(nodeAddr string, count int) {
ns.Lock()
defer ns.Unlock()
ns.WorkingCount[nodeAddr] = count
return
ns.Unlock()
}
// -----------------------------------------------
type lcRuleTaskStatus struct {
sync.RWMutex
ToBeScanned map[string]*proto.RuleTask

2
master/lifecycle_node.go
View File

@ -52,8 +52,6 @@ func (lcNode *LcNode) checkLiveness() {
if time.Since(lcNode.ReportTime) > time.Second*time.Duration(defaultNodeTimeOutSec) {
lcNode.IsActive = false
}
return
}
func (lcNode *LcNode) createHeartbeatTask(masterAddr string) (task *proto.AdminTask) {

1
master/lifecycle_task.go
View File

@ -63,7 +63,6 @@ func (c *Cluster) handleLcNodeTaskResponse(nodeAddr string, task *proto.AdminTas
errHandler:
log.LogWarnf("lc handleLcNodeTaskResponse failed, task: %v, err: %v", task.ToString(), err)
return
}
func (c *Cluster) handleLcNodeHeartbeatResp(nodeAddr string, resp *proto.LcNodeHeartbeatResponse) (err error) {

8
master/limiter.go
View File

@ -154,7 +154,8 @@ func (uMgr *UidSpaceManager) volUidUpdate(report *proto.MetaPartitionReport) {
}
}
if _, ok := uidInfo[space.Uid]; !ok {
uidInfo[space.Uid] = &(*uMgr.uidInfo[space.Uid])
infoCopy := *uMgr.uidInfo[space.Uid]
uidInfo[space.Uid] = &infoCopy
}
log.LogDebugf("volUidUpdate.vol %v uid %v from mpId %v useSize %v add %v", uMgr.vol, space.Uid, mpId, uidInfo[space.Uid].UsedSize, space.Size)
@ -682,7 +683,6 @@ func (qosManager *QosCtrlManager) updateServerLimitByClientsInfo(factorType uint
}
log.QosWriteDebugf("action[updateServerLimitByClientsInfo] vol [%v] type [%v] after adjust limitRatio serverLimit:(%v)",
qosManager.vol.Name, proto.QosTypeString(factorType), serverLimit)
return
}
func (qosManager *QosCtrlManager) assignClientsNewQos(factorType uint32) {
@ -800,14 +800,14 @@ func (vol *Vol) getClientLimitInfo(id uint64, ip string) (interface{}, error) {
Cli: &ClientReportOutput{
ID: info.Cli.ID,
Status: info.Cli.Status,
FactorMap: make(map[uint32]*proto.ClientLimitInfo, 0),
FactorMap: make(map[uint32]*proto.ClientLimitInfo),
},
Assign: &LimitOutput{
ID: info.Assign.ID,
Enable: info.Assign.Enable,
ReqPeriod: info.Assign.ReqPeriod,
HitTriggerCnt: info.Assign.HitTriggerCnt,
FactorMap: make(map[uint32]*proto.ClientLimitInfo, 0),
FactorMap: make(map[uint32]*proto.ClientLimitInfo),
},
Time: info.Time,
Host: info.Host,

1
master/master_manager.go
View File

@ -98,7 +98,6 @@ func (m *Server) handlePeerChange(confChange *proto.ConfChange) (err error) {
func (m *Server) handleApplySnapshot() {
m.fsm.restore()
m.restoreIDAlloc()
return
}
func (m *Server) handleRaftUserCmd(opt uint32, key string, cmdMap map[string][]byte) (err error) {

16
master/master_quota_manager.go
View File

@ -77,14 +77,11 @@ func (mqMgr *MasterQuotaManager) createQuota(req *proto.SetMasterQuotaReuqest) (
VolName: req.VolName,
QuotaId: quotaId,
CTime: time.Now().Unix(),
PathInfos: make([]proto.QuotaPathInfo, 0, 0),
PathInfos: make([]proto.QuotaPathInfo, 0),
MaxFiles: req.MaxFiles,
MaxBytes: req.MaxBytes,
}
for _, pathInfo := range req.PathInfos {
quotaInfo.PathInfos = append(quotaInfo.PathInfos, pathInfo)
}
quotaInfo.PathInfos = append(quotaInfo.PathInfos, req.PathInfos...)
var value []byte
if value, err = json.Marshal(quotaInfo); err != nil {
@ -230,7 +227,7 @@ func (mqMgr *MasterQuotaManager) quotaUpdate(report *proto.MetaPartitionReport)
quotaInfo.UsedInfo.UsedFiles = 0
quotaInfo.UsedInfo.UsedBytes = 0
}
deleteQuotaIds := make(map[uint32]bool, 0)
deleteQuotaIds := make(map[uint32]bool)
for mpId, reportInfos := range mqMgr.MpQuotaInfoMap {
for _, info := range reportInfos {
if _, isFind := mqMgr.IdQuotaInfoMap[info.QuotaId]; !isFind {
@ -258,7 +255,6 @@ func (mqMgr *MasterQuotaManager) quotaUpdate(report *proto.MetaPartitionReport)
}
log.LogDebugf("[quotaUpdate] quotaId [%v] quotaInfo [%v]", id, quotaInfo)
}
return
}
func (mqMgr *MasterQuotaManager) getQuotaHbInfos() (infos []*proto.QuotaHeartBeatInfo) {
@ -281,9 +277,5 @@ func (mqMgr *MasterQuotaManager) getQuotaHbInfos() (infos []*proto.QuotaHeartBea
func (mqMgr *MasterQuotaManager) HasQuota() bool {
mqMgr.RLock()
defer mqMgr.RUnlock()
if len(mqMgr.IdQuotaInfoMap) == 0 {
return false
}
return true
return len(mqMgr.IdQuotaInfoMap) > 0
}

37
master/meta_partition.go
View File

@ -92,7 +92,7 @@ func newMetaPartition(partitionID, start, end uint64, replicaNum uint8, volName
mp.Replicas = make([]*MetaReplica, 0)
mp.LeaderReportTime = time.Now().Unix()
mp.Status = proto.Unavailable
mp.MissNodes = make(map[string]int64, 0)
mp.MissNodes = make(map[string]int64)
mp.Peers = make([]proto.Peer, 0)
mp.Hosts = make([]string, 0)
mp.VerSeq = verSeq
@ -120,7 +120,6 @@ func (mp *MetaPartition) addReplica(mr *MetaReplica) {
}
}
mp.Replicas = append(mp.Replicas, mr)
return
}
func (mp *MetaPartition) removeReplica(mr *MetaReplica) {
@ -132,7 +131,6 @@ func (mp *MetaPartition) removeReplica(mr *MetaReplica) {
newReplicas = append(newReplicas, m)
}
mp.Replicas = newReplicas
return
}
func (mp *MetaPartition) removeReplicaByAddr(addr string) {
@ -144,7 +142,6 @@ func (mp *MetaPartition) removeReplicaByAddr(addr string) {
newReplicas = append(newReplicas, m)
}
mp.Replicas = newReplicas
return
}
func (mp *MetaPartition) updateInodeIDRangeForAllReplicas() {
@ -260,9 +257,7 @@ func (mp *MetaPartition) getMetaReplica(addr string) (mr *MetaReplica, err error
}
func (mp *MetaPartition) removeMissingReplica(addr string) {
if _, ok := mp.MissNodes[addr]; ok {
delete(mp.MissNodes, addr)
}
}
func (mp *MetaPartition) isLeaderExist() bool {
@ -292,7 +287,6 @@ func (mp *MetaPartition) checkLeader(clusterID string) {
if WarnMetrics != nil {
WarnMetrics.WarnMpNoLeader(clusterID, mp.PartitionID, report)
}
return
}
func (mp *MetaPartition) checkStatus(clusterID string, writeLog bool, replicaNum int, maxPartitionID uint64, metaPartitionInodeIdStep uint64, forbiddenVol bool) (doSplit bool) {
@ -487,7 +481,6 @@ func (mp *MetaPartition) checkReplicas() {
mr.Status = proto.Unavailable
}
}
return
}
func (mp *MetaPartition) persistToRocksDB(action, volName string, newHosts []string, newPeers []proto.Peer, c *Cluster) (err error) {
@ -600,7 +593,7 @@ func (mp *MetaPartition) replicaCreationTasks(clusterID, volName string) (tasks
func (mp *MetaPartition) buildNewMetaPartitionTasks(specifyAddrs []string, peers []proto.Peer, volName string) (tasks []*proto.AdminTask) {
tasks = make([]*proto.AdminTask, 0)
hosts := make([]string, 0)
var hosts []string
req := &proto.CreateMetaPartitionRequest{
Start: mp.Start,
@ -694,17 +687,6 @@ func (mp *MetaPartition) createTaskToRemoveRaftMember(removePeer proto.Peer) (t
return
}
func (mp *MetaPartition) createTaskToDecommissionReplica(volName string, removePeer proto.Peer, addPeer proto.Peer) (t *proto.AdminTask, err error) {
mr, err := mp.getMetaReplicaLeader()
if err != nil {
return nil, errors.NewError(err)
}
req := &proto.MetaPartitionDecommissionRequest{PartitionID: mp.PartitionID, VolName: volName, RemovePeer: removePeer, AddPeer: addPeer}
t = proto.NewAdminTask(proto.OpDecommissionMetaPartition, mr.Addr, req)
resetMetaPartitionTaskID(t, mp.PartitionID)
return
}
func resetMetaPartitionTaskID(t *proto.AdminTask, partitionID uint64) {
t.ID = fmt.Sprintf("%v_pid[%v]", t.ID, partitionID)
t.PartitionID = partitionID
@ -905,21 +887,6 @@ func (mp *MetaPartition) SetTxCnt() {
mp.TxCnt, mp.TxRbInoCnt, mp.TxRbDenCnt = txCnt, rbInoCnt, rbDenCnt
}
func (mp *MetaPartition) getAllNodeSets() (nodeSets []uint64) {
mp.RLock()
defer mp.RUnlock()
nodeSets = make([]uint64, 0)
for _, mr := range mp.Replicas {
if mr.metaNode == nil {
continue
}
if !containsID(nodeSets, mr.metaNode.NodeSetID) {
nodeSets = append(nodeSets, mr.metaNode.NodeSetID)
}
}
return
}
func (mp *MetaPartition) getLiveZones(offlineAddr string) (zones []string) {
mp.RLock()
defer mp.RUnlock()

20
master/metadata_fsm_op.go
View File

@ -886,11 +886,6 @@ func (c *Cluster) buildPutMetaNodeCmd(opType uint32, metaNode *MetaNode) (metada
return
}
func (c *Cluster) buildAddMetaNodeCmd(metaNode *MetaNode) (metadata *RaftCmd, err error) {
metadata, err = c.buildPutMetaNodeCmd(opSyncAddMetaNode, metaNode)
return
}
func (c *Cluster) buildDeleteMetaNodeCmd(metaNode *MetaNode) (metadata *RaftCmd, err error) {
metadata, err = c.buildPutMetaNodeCmd(opSyncDeleteMetaNode, metaNode)
return
@ -922,11 +917,6 @@ func (c *Cluster) syncUpdateDataNode(dataNode *DataNode) (err error) {
return c.syncPutDataNode(opSyncUpdateDataNode, dataNode)
}
func (c *Cluster) buildAddDataNodeCmd(dataNode *DataNode) (metadata *RaftCmd, err error) {
metadata, err = c.buildPutDataNodeCmd(opSyncAddDataNode, dataNode)
return
}
func (c *Cluster) buildDeleteDataNodeCmd(dataNode *DataNode) (metadata *RaftCmd, err error) {
metadata, err = c.buildPutDataNodeCmd(opSyncDeleteDataNode, dataNode)
return
@ -1094,7 +1084,6 @@ func (c *Cluster) checkPersistClusterValue() {
panic(err)
}
log.LogInfo("action[checkPersistClusterValue] add cluster value record")
return
}
func (c *Cluster) loadClusterValue() (err error) {
@ -1286,13 +1275,12 @@ func (c *Cluster) loadZoneDomain() (ok bool, err error) {
for zoneName, domainId := range c.domainManager.ZoneName2DomainIdMap {
log.LogInfof("action[loadZoneDomain] zoneName %v domainid %v", zoneName, domainId)
if domainIndex, ok := c.domainManager.domainId2IndexMap[domainId]; !ok {
if _, ok := c.domainManager.domainId2IndexMap[domainId]; !ok {
log.LogInfof("action[loadZoneDomain] zoneName %v domainid %v build new domainnodesetgrp manager", zoneName, domainId)
domainGrp := newDomainNodeSetGrpManager()
domainGrp.domainId = domainId
c.domainManager.domainNodeSetGrpVec = append(c.domainManager.domainNodeSetGrpVec, domainGrp)
domainIndex = len(c.domainManager.domainNodeSetGrpVec) - 1
c.domainManager.domainId2IndexMap[domainId] = domainIndex
c.domainManager.domainId2IndexMap[domainId] = len(c.domainManager.domainNodeSetGrpVec) - 1
}
}
@ -1732,10 +1720,6 @@ func (c *Cluster) syncDeleteLcNode(ln *LcNode) (err error) {
return c.syncPutLcNodeInfo(opSyncDeleteLcNode, ln)
}
func (c *Cluster) syncUpdateLcNode(ln *LcNode) (err error) {
return c.syncPutLcNodeInfo(opSyncUpdateLcNode, ln)
}
func (c *Cluster) syncPutLcNodeInfo(opType uint32, ln *LcNode) (err error) {
metadata := new(RaftCmd)
metadata.Op = opType

2
master/mocktest/meta_server.go
View File

@ -39,7 +39,7 @@ type MockMetaServer struct {
func NewMockMetaServer(addr string, zoneName string) *MockMetaServer {
mms := &MockMetaServer{
TcpAddr: addr, partitions: make(map[uint64]*MockMetaPartition, 0),
TcpAddr: addr, partitions: make(map[uint64]*MockMetaPartition),
ZoneName: zoneName,
mc: master.NewMasterClient([]string{hostAddr}, false),
}

4
master/mocktest/partition.go
View File

@ -43,10 +43,6 @@ type MockMetaPartition struct {
// MockMetaReplica defines the replica of a meta partition
type MockMetaReplica struct {
Addr string
start uint64 // lower bound of the inode id
end uint64 // upper bound of the inode id
dataSize uint64
nodeID uint64
MaxInodeID uint64
InodeCount uint64
DentryCount uint64

23
master/monitor_metrics.go
View File

@ -483,15 +483,14 @@ func (mm *monitorMetrics) start() {
func (mm *monitorMetrics) statMetrics() {
ticker := time.NewTicker(StatPeriod)
defer func() {
ticker.Stop()
if err := recover(); err != nil {
ticker.Stop()
log.LogErrorf("statMetrics panic,msg:%v", err)
}
}()
for {
select {
case <-ticker.C:
for range ticker.C {
partition := mm.cluster.partition
if partition != nil && partition.IsRaftLeader() {
mm.resetFollowerMetrics()
@ -501,7 +500,6 @@ func (mm *monitorMetrics) statMetrics() {
mm.doFollowerStat()
}
}
}
}
func (mm *monitorMetrics) doFollowerStat() {
@ -554,11 +552,11 @@ func (mm *monitorMetrics) setMpAndDpMetrics() {
vols := mm.cluster.copyVols()
for _, vol := range vols {
if (vol.Status == proto.VolStatusMarkDelete && !vol.Forbidden) || (vol.Status == proto.VolStatusMarkDelete && vol.Forbidden && vol.DeleteExecTime.Sub(time.Now()) <= 0) {
if (vol.Status == proto.VolStatusMarkDelete && !vol.Forbidden) ||
(vol.Status == proto.VolStatusMarkDelete && vol.Forbidden && time.Until(vol.DeleteExecTime) <= 0) {
continue
}
var dps *DataPartitionMap
dps = vol.dataPartitions
dps := vol.dataPartitions
dpCount += len(dps.partitions)
for _, dp := range dps.partitions {
if dp.ReplicaNum > uint8(len(dp.liveReplicas(defaultDataPartitionTimeOutSec))) {
@ -580,9 +578,7 @@ func (mm *monitorMetrics) setMpAndDpMetrics() {
mm.dataPartitionCount.Set(float64(dpCount))
mm.ReplicaMissingDPCount.Set(float64(dpMissingReplicaDpCount))
mm.DpMissingLeaderCount.Set(float64(dpMissingLeaderCount))
mm.MpMissingLeaderCount.Set(float64(mpMissingLeaderCount))
return
}
func (mm *monitorMetrics) setVolNoCacheMetrics() {
@ -598,7 +594,7 @@ func (mm *monitorMetrics) setVolNoCacheMetrics() {
continue
}
if stat == true {
if stat {
deleteVolNames[volName] = struct{}{}
log.LogDebugf("setVolNoCacheMetrics: to deleteVolNames volName %v for status becomes ok", volName)
continue
@ -632,9 +628,7 @@ func (mm *monitorMetrics) setVolMetrics() {
return true
}
mm.volNames[volName] = struct{}{}
if _, ok := deleteVolNames[volName]; ok {
delete(deleteVolNames, volName)
}
mm.volTotalSpace.SetWithLabelValues(float64(volStatInfo.TotalSize)/float64(util.GB), volName)
mm.volUsedSpace.SetWithLabelValues(float64(volStatInfo.UsedSize)/float64(util.GB), volName)
@ -709,7 +703,8 @@ func (mm *monitorMetrics) setMpInconsistentErrorMetric() {
defer mm.cluster.volMutex.RUnlock()
for _, vol := range mm.cluster.vols {
if (vol.Status == proto.VolStatusMarkDelete && !vol.Forbidden) || (vol.Status == proto.VolStatusMarkDelete && vol.Forbidden && vol.DeleteExecTime.Sub(time.Now()) <= 0) {
if (vol.Status == proto.VolStatusMarkDelete && !vol.Forbidden) ||
(vol.Status == proto.VolStatusMarkDelete && vol.Forbidden && time.Until(vol.DeleteExecTime) <= 0) {
continue
}
vol.mpsLock.RLock()
@ -906,7 +901,7 @@ func (mm *monitorMetrics) deleteS3LcVolMetric(volName string) {
func (mm *monitorMetrics) setLcMetrics() {
lcTaskStatus := mm.cluster.lcMgr.lcRuleTaskStatus
volumeScanStatistics := make(map[string]proto.LcNodeRuleTaskStatistics, 0)
volumeScanStatistics := make(map[string]proto.LcNodeRuleTaskStatistics)
lcTaskStatus.RLock()
for _, r := range lcTaskStatus.Results {
key := r.Volume + "[" + r.RuleId + "]"

14
master/multi_ver_snapshot.go
View File

@ -196,7 +196,7 @@ func (verMgr *VolVersionManager) SetVerStrategy(strategy proto.VolumeVerStrategy
log.LogWarnf("vol %v SetVerStrategy.keepCnt %v need in [1-%v], peroidic %v need in [1-%v], enable %v", verMgr.vol.Name,
strategy.KeepVerCnt, MaxSnapshotCount, strategy.GetPeriodic(), 24*7, strategy.Enable)
if strategy.Enable == true {
if strategy.Enable {
if strategy.KeepVerCnt > MaxSnapshotCount || strategy.GetPeriodic() > 24*7 || strategy.KeepVerCnt < 0 || strategy.GetPeriodic() < 0 {
return fmt.Errorf("SetVerStrategy.vol %v keepCnt %v need in [1-%v], peroidic %v need in [1-%v] not qualified",
verMgr.vol.Name, strategy.KeepVerCnt, MaxSnapshotCount, strategy.GetPeriodic(), 24*7)
@ -759,18 +759,6 @@ func (verMgr *VolVersionManager) getOldestVer() (ver uint64, status uint8) {
return verMgr.multiVersionList[0].Ver, verMgr.multiVersionList[0].Status
}
func (verMgr *VolVersionManager) getVolDelStatus() (status uint8) {
verMgr.RLock()
defer verMgr.RUnlock()
size := len(verMgr.multiVersionList)
if size == 0 {
return 0
}
log.LogInfof("action[getLatestVer] ver len %v verMgr %v", size, verMgr)
return verMgr.multiVersionList[size-1].Status
}
func (verMgr *VolVersionManager) getLatestVer() (ver uint64) {
verMgr.RLock()
defer verMgr.RUnlock()

28
master/node_selector_test.go
View File

@ -259,13 +259,11 @@ func TestCarryWeightNodeSelector(t *testing.T) {
if node == nil {
return
}
count, _ := dataSelectTimes[node.ID]
count += 1
dataSelectTimes[node.ID] = count
dataSelectTimes[node.ID]++
}
t.Logf("%v data node select times:\n", selector.GetName())
printNodeSelectTimes(t, dataSelectTimes)
count, _ := dataSelectTimes[dataNode.ID]
count := dataSelectTimes[dataNode.ID]
for _, c := range dataSelectTimes {
if count < c {
t.Errorf("%v failed to select data nodes", selector.GetName())
@ -290,13 +288,11 @@ func TestCarryWeightNodeSelector(t *testing.T) {
if node == nil {
return
}
count, _ := metaSelectTimes[node.ID]
count += 1
metaSelectTimes[node.ID] = count
metaSelectTimes[node.ID]++
}
t.Logf("%v meta node select times:\n", selector.GetName())
printNodeSelectTimes(t, metaSelectTimes)
count, _ = metaSelectTimes[metaNode.ID]
count = metaSelectTimes[metaNode.ID]
for _, c := range metaSelectTimes {
if count < c {
t.Errorf("%v failed to select meta nodes", selector.GetName())
@ -379,13 +375,11 @@ func TestStrawNodeSelector(t *testing.T) {
if node == nil {
return
}
count, _ := dataSelectTimes[node.ID]
count += 1
dataSelectTimes[node.ID] = count
dataSelectTimes[node.ID]++
}
t.Logf("%v data node select times:\n", selector.GetName())
printNodeSelectTimes(t, dataSelectTimes)
count, _ := dataSelectTimes[dataNode.ID]
count := dataSelectTimes[dataNode.ID]
for _, c := range dataSelectTimes {
if count < c {
t.Errorf("%v failed to select data nodes", selector.GetName())
@ -406,13 +400,11 @@ func TestStrawNodeSelector(t *testing.T) {
if node == nil {
return
}
count, _ := metaSelectTimes[node.ID]
count += 1
metaSelectTimes[node.ID] = count
metaSelectTimes[node.ID]++
}
t.Logf("%v meta node select times:\n", selector.GetName())
printNodeSelectTimes(t, metaSelectTimes)
count, _ = metaSelectTimes[metaNode.ID]
count = metaSelectTimes[metaNode.ID]
for _, c := range metaSelectTimes {
if count < c {
t.Errorf("%v failed to select meta nodes", selector.GetName())
@ -480,9 +472,7 @@ func nodeSelectorBench(selector NodeSelector, nset *nodeSet, onSelect func(addr
return nil, err
}
for _, peer := range peers {
count, _ := times[peer.ID]
count += 1
times[peer.ID] = count
times[peer.ID]++
if onSelect != nil {
onSelect(peer.Addr)
}

6
master/nodeset_selector_test.go
View File

@ -119,8 +119,6 @@ func prepareMetaNodesetForBench(count int, initTotal uint64, grow uint64) (nsc n
return
}
const loopNodesetSelectorTestCount = 100
func nodesetSelectorBench(selector NodesetSelector, nsc nodeSetCollection, onSelect func(id uint64)) (map[uint64]int, error) {
times := make(map[uint64]int)
for i := 0; i < loopNodeSelectorTestCount; i++ {
@ -128,9 +126,7 @@ func nodesetSelectorBench(selector NodesetSelector, nsc nodeSetCollection, onSel
if err != nil {
return nil, err
}
count, _ := times[ns.ID]
count += 1
times[ns.ID] = count
times[ns.ID]++
if onSelect != nil {
onSelect(ns.ID)
}

23
master/operate_util.go
View File

@ -31,7 +31,8 @@ import (
func newCreateDataPartitionRequest(volName string, ID uint64, replicaNum int, members []proto.Peer,
dataPartitionSize, leaderSize int, hosts []string, createType int, partitionType int,
decommissionedDisks []string, verSeq uint64) (req *proto.CreateDataPartitionRequest) {
decommissionedDisks []string, verSeq uint64,
) (req *proto.CreateDataPartitionRequest) {
req = &proto.CreateDataPartitionRequest{
PartitionTyp: partitionType,
PartitionId: ID,
@ -133,7 +134,7 @@ func unmarshalTaskResponse(task *proto.AdminTask) (err error) {
}
func contains(arr []string, element string) (ok bool) {
if arr == nil || len(arr) == 0 {
if len(arr) == 0 {
return
}
@ -147,7 +148,7 @@ func contains(arr []string, element string) (ok bool) {
}
func containsID(arr []uint64, element uint64) bool {
if arr == nil || len(arr) == 0 {
if len(arr) == 0 {
return false
}
@ -161,7 +162,7 @@ func containsID(arr []uint64, element uint64) bool {
}
func reshuffleHosts(oldHosts []string) (newHosts []string, err error) {
if oldHosts == nil || len(oldHosts) == 0 {
if len(oldHosts) == 0 {
log.LogError(fmt.Sprintf("action[reshuffleHosts],err:%v", proto.ErrReshuffleArray))
err = proto.ErrReshuffleArray
return
@ -203,10 +204,6 @@ func unmatchedKey(name string) (err error) {
return errors.NewErrorf("parameter %v not match", name)
}
func txInvalidMask() (err error) {
return errors.New("transaction mask key value pair should be: enableTxMaskKey=[create|mkdir|remove|rename|mknod|symlink|link]\n enableTxMaskKey=off \n enableTxMaskKey=all")
}
func notFoundMsg(name string) (err error) {
return errors.NewErrorf("%v not found", name)
}
@ -227,10 +224,6 @@ func dataReplicaNotFound(addr string) (err error) {
return notFoundMsg(fmt.Sprintf("data replica[%v]", addr))
}
func zoneNotFound(name string) (err error) {
return notFoundMsg(fmt.Sprintf("zone[%v]", name))
}
func nodeSetNotFound(id uint64) (err error) {
return notFoundMsg(fmt.Sprintf("node set[%v]", id))
}
@ -247,10 +240,6 @@ func lcNodeNotFound(addr string) (err error) {
return notFoundMsg(fmt.Sprintf("lc node[%v]", addr))
}
func volNotFound(name string) (err error) {
return notFoundMsg(fmt.Sprintf("vol[%v]", name))
}
func matchKey(serverKey, clientKey string) bool {
h := md5.New()
_, err := h.Write([]byte(serverKey))
@ -259,5 +248,5 @@ func matchKey(serverKey, clientKey string) bool {
return false
}
cipherStr := h.Sum(nil)
return strings.ToLower(clientKey) == strings.ToLower(hex.EncodeToString(cipherStr))
return strings.EqualFold(clientKey, hex.EncodeToString(cipherStr))
}

10
master/server.go
View File

@ -79,11 +79,7 @@ var (
var overSoldFactor = defaultOverSoldFactor
func overSoldLimit() bool {
if overSoldFactor <= 0 {
return false
}
return true
return overSoldFactor > 0
}
func overSoldCap(cap uint64) uint64 {
@ -100,8 +96,6 @@ func setOverSoldFactor(factor float32) {
}
}
var volNameErr = errors.New("name can only start and end with number or letters, and len can't less than 3")
// Server represents the server in a cluster
type Server struct {
id uint64
@ -180,7 +174,7 @@ func (m *Server) Start(cfg *config.Config) (err error) {
stat.DefaultTimeOutUs, true)
m.wg.Add(1)
return nil
return err
}
// Shutdown closes the server

4
master/snapshot_manager.go
View File

@ -176,12 +176,12 @@ func (vs *lcSnapshotVerStatus) DeleteOldResult() {
defer vs.Unlock()
for k, v := range vs.TaskResults {
// delete result that already done
if v.Done == true && time.Now().After(v.EndTime.Add(time.Minute*10)) {
if v.Done && time.Now().After(v.EndTime.Add(time.Minute*10)) {
delete(vs.TaskResults, k)
log.LogDebugf("delete result already done: %v", v)
}
// delete result that not done but no updating
if v.Done != true && time.Now().After(v.UpdateTime.Add(time.Minute*10)) {
if !v.Done && time.Now().After(v.UpdateTime.Add(time.Minute*10)) {
delete(vs.TaskResults, k)
log.LogWarnf("delete result that not done but no updating: %v", v)
}

73
master/topology.go
View File

@ -140,15 +140,6 @@ func (t *topology) deleteDataNode(dataNode *DataNode) {
t.dataNodes.Delete(dataNode.Addr)
}
func (t *topology) getZoneByDataNode(dataNode *DataNode) (zone *Zone, err error) {
_, ok := t.dataNodes.Load(dataNode.Addr)
if !ok {
return nil, errors.Trace(dataNodeNotFound(dataNode.Addr), "%v not found", dataNode.Addr)
}
return t.getZone(dataNode.ZoneName)
}
func (t *topology) putMetaNode(metaNode *MetaNode) (err error) {
if _, ok := t.metaNodes.Load(metaNode.Addr); ok {
return
@ -266,7 +257,7 @@ func (nsgm *DomainManager) start() {
}
func (nsgm *DomainManager) createDomain(zoneName string) (err error) {
if nsgm.init == false {
if !nsgm.init {
return fmt.Errorf("createDomain err [%v]", err)
}
log.LogInfof("zone name [%v] createDomain", zoneName)
@ -328,7 +319,7 @@ func (nsgm *DomainManager) checkExcludeZoneState() {
excludeNeedDomain)
nsgm.c.needFaultDomain = true
} else {
if nsgm.c.needFaultDomain == true {
if nsgm.c.needFaultDomain {
log.LogInfof("action[checkExcludeZoneState] needFaultDomain be set false")
}
nsgm.c.needFaultDomain = false
@ -456,8 +447,7 @@ func (nsgm *DomainManager) buildNodeSetGrp(domainGrpManager *DomainNodeSetGrpMan
return
}
var method map[int]buildNodeSetGrpMethod
method = make(map[int]buildNodeSetGrpMethod)
method := make(map[int]buildNodeSetGrpMethod)
method[3] = buildNodeSetGrp3Zone
method[2] = buildNodeSetGrp2Plus1
method[1] = buildNodeSetGrpOneZone
@ -578,7 +568,8 @@ func (nsgm *DomainManager) getHostFromNodeSetGrpSpecific(domainGrpManager *Domai
func (nsgm *DomainManager) getHostFromNodeSetGrp(domainId uint64, replicaNum uint8, createType uint32) (
hosts []string,
peers []proto.Peer,
err error) {
err error,
) {
var ok bool
var index int
@ -893,7 +884,7 @@ func (nsgm *DomainManager) putNodeSet(ns *nodeSet, load bool) (err error) {
nsgm.ZoneName2DomainIdMap[ns.zoneName] = 0
}
if index, ok = nsgm.domainId2IndexMap[domainId]; !ok {
if domainId > 0 && load == false { // domainId 0 can be created through nodeset create,others be created by createDomain
if domainId > 0 && !load { // domainId 0 can be created through nodeset create,others be created by createDomain
err = fmt.Errorf("inconsistent domainid exist in name map but node exist in index map")
log.LogErrorf("action[putNodeSet] %v", err)
return
@ -1276,12 +1267,6 @@ func (t *topology) getAllZones() (zones []*Zone) {
return
}
func (t *topology) getZoneByIndex(index int) (zone *Zone) {
t.zoneLock.RLock()
defer t.zoneLock.RUnlock()
return t.zones[index]
}
func (t *topology) getNodeSetByNodeSetId(nodeSetId uint64) (nodeSet *nodeSet, err error) {
zones := t.getAllZones()
for _, zone := range zones {
@ -1406,15 +1391,6 @@ func (t *topology) allocZonesForDataNode(zoneNum, replicaNum int, excludeZone []
return
}
func (ns *nodeSet) dataNodeCount() int {
var count int
ns.dataNodes.Range(func(key, value interface{}) bool {
count++
return true
})
return count
}
// Zone stores all the zone related information
type Zone struct {
name string
@ -1520,12 +1496,6 @@ func (zone *Zone) getStatusToString() string {
}
}
func (zone *Zone) isSingleNodeSet() bool {
zone.RLock()
defer zone.RUnlock()
return len(zone.nodeSetMap) == 1
}
func (zone *Zone) getNodeSet(setID uint64) (ns *nodeSet, err error) {
zone.nsLock.RLock()
defer zone.nsLock.RUnlock()
@ -1653,15 +1623,6 @@ func (zone *Zone) putDataNode(dataNode *DataNode) (err error) {
return
}
func (zone *Zone) getDataNode(addr string) (dataNode *DataNode, err error) {
value, ok := zone.dataNodes.Load(addr)
if !ok {
return nil, errors.Trace(dataNodeNotFound(addr), "%v not found", addr)
}
dataNode = value.(*DataNode)
return
}
func (zone *Zone) deleteDataNode(dataNode *DataNode) {
ns, err := zone.getNodeSet(dataNode.NodeSetID)
if err != nil {
@ -1770,7 +1731,7 @@ func (zone *Zone) isUsedRatio(ratio float64) (can bool) {
)
zone.dataNodes.Range(func(addr, value interface{}) bool {
dataNode := value.(*DataNode)
if dataNode.isActive == true {
if dataNode.isActive {
dataNodeUsed += dataNode.Used
} else {
dataNodeUsed += dataNode.Total
@ -1786,7 +1747,7 @@ func (zone *Zone) isUsedRatio(ratio float64) (can bool) {
zone.metaNodes.Range(func(addr, value interface{}) bool {
metaNode := value.(*MetaNode)
if metaNode.IsActive == true && metaNode.isWritable() == true {
if metaNode.IsActive && metaNode.isWritable() {
metaNodeUsed += metaNode.Used
} else {
metaNodeUsed += metaNode.Total
@ -1808,7 +1769,7 @@ func (zone *Zone) getDataUsed() (dataNodeUsed uint64, dataNodeTotal uint64) {
defer zone.RUnlock()
zone.dataNodes.Range(func(addr, value interface{}) bool {
dataNode := value.(*DataNode)
if dataNode.isActive == true {
if dataNode.isActive {
dataNodeUsed += dataNode.Used
} else {
dataNodeUsed += dataNode.Total
@ -1826,7 +1787,7 @@ func (zone *Zone) getMetaUsed() (metaNodeUsed uint64, metaNodeTotal uint64) {
zone.metaNodes.Range(func(addr, value interface{}) bool {
metaNode := value.(*MetaNode)
if metaNode.IsActive == true && metaNode.isWritable() == true {
if metaNode.IsActive && metaNode.isWritable() {
metaNodeUsed += metaNode.Used
} else {
metaNodeUsed += metaNode.Total
@ -1853,7 +1814,7 @@ func (zone *Zone) canWriteForMetaNode(replicaNum uint8) (can bool) {
var leastAlive uint8
zone.metaNodes.Range(func(addr, value interface{}) bool {
metaNode := value.(*MetaNode)
if metaNode.IsActive == true && metaNode.isWritable() == true {
if metaNode.IsActive && metaNode.isWritable() {
leastAlive++
}
if leastAlive >= replicaNum {
@ -1865,17 +1826,6 @@ func (zone *Zone) canWriteForMetaNode(replicaNum uint8) (can bool) {
return
}
func (zone *Zone) getDataNodeMaxTotal() (maxTotal uint64) {
zone.dataNodes.Range(func(key, value interface{}) bool {
dataNode := value.(*DataNode)
if dataNode.Total > maxTotal {
maxTotal = dataNode.Total
}
return true
})
return
}
func (zone *Zone) getAvailNodeHosts(nodeType uint32, excludeNodeSets []uint64, excludeHosts []string, replicaNum int) (newHosts []string, peers []proto.Peer, err error) {
if replicaNum == 0 {
return
@ -2169,7 +2119,6 @@ func (l *DecommissionDataPartitionList) pushFailedDp(value *DataPartition, c *Cl
l.mu.Unlock()
log.LogInfof("action[pushFailedDp] add dp[%v] status[%v] isRecover[%v]",
value.PartitionID, status, value.isRecover)
return
}
func (l *DecommissionDataPartitionList) Remove(value *DataPartition) {

4
master/topology_test.go
View File

@ -120,14 +120,14 @@ func TestAllocZones(t *testing.T) {
cluster.cfg = newClusterConfig()
// don't cross zone
hosts, _, err := cluster.getHostFromNormalZone(TypeDataPartition, nil, nil, nil, replicaNum, 1, "")
_, _, err = cluster.getHostFromNormalZone(TypeDataPartition, nil, nil, nil, replicaNum, 1, "")
if err != nil {
t.Error(err)
return
}
// cross zone
hosts, _, err = cluster.getHostFromNormalZone(TypeDataPartition, nil, nil, nil, replicaNum, 2, "")
hosts, _, err := cluster.getHostFromNormalZone(TypeDataPartition, nil, nil, nil, replicaNum, 2, "")
if err != nil {
t.Error(err)
return

4
master/user.go
View File

@ -460,9 +460,7 @@ func (u *User) getUsersOfVol(volName string) (userIDs []string, err error) {
}
volUser.Mu.RLock()
defer volUser.Mu.RUnlock()
for _, userID := range volUser.UserIDs {
userIDs = append(userIDs, userID)
}
userIDs = append(userIDs, volUser.UserIDs...)
log.LogInfof("action[getUsersOfVol], vol: %v, user numbers: %v", volName, len(userIDs))
return
}

47
master/vol.go
View File

@ -40,7 +40,6 @@ type VolVarargs struct {
dpSelectorName string
dpSelectorParm string
coldArgs *coldVolArgs
domainId uint64
dpReplicaNum uint8
enablePosixAcl bool
dpReadOnlyWhenVolFull bool
@ -121,8 +120,7 @@ type Vol struct {
}
func newVol(vv volValue) (vol *Vol) {
vol = &Vol{ID: vv.ID, Name: vv.Name, MetaPartitions: make(map[uint64]*MetaPartition, 0)}
vol = &Vol{ID: vv.ID, Name: vv.Name, MetaPartitions: make(map[uint64]*MetaPartition)}
if vol.threshold <= 0 {
vol.threshold = defaultMetaPartitionMemUsageThreshold
}
@ -284,9 +282,8 @@ func (mpsLock *mpsLockManager) RUnlock() {
func (mpsLock *mpsLockManager) CheckExceptionLock(interval time.Duration, expireTime time.Duration) {
ticker := time.NewTicker(interval)
for {
select {
case <-ticker.C:
for range ticker.C {
{
if mpsLock.vol.status() == proto.VolStatusMarkDelete || atomic.LoadInt32(&mpsLock.enable) == 0 {
break
}
@ -331,7 +328,7 @@ func (vol *Vol) CheckStrategy(c *Cluster) {
return
}
vol.VersionMgr.RLock()
if vol.VersionMgr.strategy.GetPeriodicSecond() == 0 || vol.VersionMgr.strategy.Enable == false { // strategy not be set
if vol.VersionMgr.strategy.GetPeriodicSecond() == 0 || !vol.VersionMgr.strategy.Enable { // strategy not be set
vol.VersionMgr.RUnlock()
continue
}
@ -371,8 +368,8 @@ func (vol *Vol) getPreloadCapacity() uint64 {
func (vol *Vol) initQosManager(limitArgs *qosArgs) {
vol.qosManager = &QosCtrlManager{
cliInfoMgrMap: make(map[uint64]*ClientInfoMgr, 0),
serverFactorLimitMap: make(map[uint32]*ServerFactorLimit, 0),
cliInfoMgrMap: make(map[uint64]*ClientInfoMgr),
serverFactorLimitMap: make(map[uint32]*ServerFactorLimit),
qosEnable: limitArgs.qosEnable,
vol: vol,
ClientHitTriggerCnt: defaultClientTriggerHitCnt,
@ -793,7 +790,6 @@ func (vol *Vol) checkSplitMetaPartition(c *Cluster, metaPartitionInodeStep uint6
log.LogInfof("volume[%v] split MaxMP[%v], MaxInodeID[%d] Start[%d] RWMPNum[%d] maxMPInodeUsedRatio[%.2f]",
vol.Name, maxPartitionID, maxMP.MaxInodeID, maxMP.Start, RWMPNum, maxMPInodeUsedRatio)
}
return
}
func (mp *MetaPartition) memUsedReachThreshold(clusterName, volName string) bool {
@ -820,7 +816,7 @@ func (mp *MetaPartition) memUsedReachThreshold(clusterName, volName string) bool
}
func (vol *Vol) cloneMetaPartitionMap() (mps map[uint64]*MetaPartition) {
mps = make(map[uint64]*MetaPartition, 0)
mps = make(map[uint64]*MetaPartition)
vol.mpsLock.RLock()
defer vol.mpsLock.RUnlock()
for _, mp := range vol.MetaPartitions {
@ -842,7 +838,7 @@ func (vol *Vol) setMpForbid() {
func (vol *Vol) cloneDataPartitionMap() (dps map[uint64]*DataPartition) {
vol.dataPartitions.RLock()
defer vol.dataPartitions.RUnlock()
dps = make(map[uint64]*DataPartition, 0)
dps = make(map[uint64]*DataPartition)
for _, dp := range vol.dataPartitions.partitionMap {
dps[dp.PartitionID] = dp
}
@ -939,11 +935,7 @@ func (vol *Vol) shouldInhibitWriteBySpaceFull() bool {
}
usedSpace := vol.totalUsedSpace() / util.GB
if usedSpace >= vol.capacity() {
return true
}
return false
return usedSpace >= vol.capacity()
}
func (vol *Vol) needCreateDataPartition() (ok bool, err error) {
@ -1065,21 +1057,6 @@ func (vol *Vol) cfsUsedSpace() uint64 {
return vol.dataPartitions.totalUsedSpace()
}
func (vol *Vol) sendViewCacheToFollower(c *Cluster) {
var err error
log.LogInfof("action[asyncSendPartitionsToFollower]")
metadata := new(RaftCmd)
metadata.Op = opSyncDataPartitionsView
metadata.K = vol.Name
metadata.V = vol.dataPartitions.getDataPartitionResponseCache()
if err = c.submit(metadata); err != nil {
log.LogErrorf("action[asyncSendPartitionsToFollower] error [%v]", err)
}
log.LogInfof("action[asyncSendPartitionsToFollower] finished")
}
func (vol *Vol) ebsUsedSpace() uint64 {
size := uint64(0)
vol.mpsLock.RLock()
@ -1231,8 +1208,6 @@ func (vol *Vol) checkStatus(c *Cluster) {
vol.deleteDataPartitionFromDataNode(c, dataTask)
}
}()
return
}
func (vol *Vol) deleteMetaPartitionFromMetaNode(c *Cluster, task *proto.AdminTask) {
@ -1262,7 +1237,6 @@ func (vol *Vol) deleteMetaPartitionFromMetaNode(c *Cluster, task *proto.AdminTas
mp.removeReplicaByAddr(metaNode.Addr)
mp.removeMissingReplica(metaNode.Addr)
mp.Unlock()
return
}
func (vol *Vol) deleteDataPartitionFromDataNode(c *Cluster, task *proto.AdminTask) (err error) {
@ -1325,7 +1299,6 @@ func (vol *Vol) deleteMetaPartitionsFromStore(c *Cluster) {
for _, mp := range vol.MetaPartitions {
c.syncDeleteMetaPartition(mp)
}
return
}
func (vol *Vol) deleteDataPartitionsFromStore(c *Cluster) {
@ -1385,7 +1358,7 @@ func (vol *Vol) doSplitMetaPartition(c *Cluster, mp *MetaPartition, end uint64,
}
log.LogWarnf("action[splitMetaPartition],partition[%v],start[%v],end[%v],new end[%v]", mp.PartitionID, mp.Start, mp.End, end)
cmdMap := make(map[string]*RaftCmd, 0)
cmdMap := make(map[string]*RaftCmd)
oldEnd := mp.End
mp.End = end

2
master/vol_test.go
View File

@ -218,11 +218,9 @@ func createVol(kv map[string]interface{}, t *testing.T) {
switch kv[volTypeKey].(int) {
case proto.VolumeTypeHot:
checkWithDefault(kv, replicaNumKey, 3)
break
case proto.VolumeTypeCold:
checkWithDefault(kv, cacheCapacity, 80)
checkWithDefault(kv, replicaNumKey, 1)
break
default:
// do nothing
}

35
metanode/api_handler.go
View File

@ -78,8 +78,7 @@ func (m *MetaNode) registerAPIHandler() (err error) {
return
}
func (m *MetaNode) getParamsHandler(w http.ResponseWriter,
r *http.Request) {
func (m *MetaNode) getParamsHandler(w http.ResponseWriter, r *http.Request) {
resp := NewAPIResponse(http.StatusOK, http.StatusText(http.StatusOK))
params := make(map[string]interface{})
params[metaNodeDeleteBatchCountKey] = DeleteBatchCount()
@ -90,8 +89,7 @@ func (m *MetaNode) getParamsHandler(w http.ResponseWriter,
}
}
func (m *MetaNode) getSmuxStatHandler(w http.ResponseWriter,
r *http.Request) {
func (m *MetaNode) getSmuxStatHandler(w http.ResponseWriter, r *http.Request) {
resp := NewAPIResponse(http.StatusOK, http.StatusText(http.StatusOK))
resp.Data = smuxPool.GetStat()
data, _ := resp.Marshal()
@ -100,8 +98,7 @@ func (m *MetaNode) getSmuxStatHandler(w http.ResponseWriter,
}
}
func (m *MetaNode) getPartitionsHandler(w http.ResponseWriter,
r *http.Request) {
func (m *MetaNode) getPartitionsHandler(w http.ResponseWriter, r *http.Request) {
resp := NewAPIResponse(http.StatusOK, http.StatusText(http.StatusOK))
resp.Data = m.metadataManager
data, _ := resp.Marshal()
@ -288,7 +285,6 @@ func (m *MetaNode) getSplitKeyHandler(w http.ResponseWriter, r *http.Request) {
} else {
log.LogDebugf("getSplitKeyHandler")
}
return
}
func (m *MetaNode) getInodeHandler(w http.ResponseWriter, r *http.Request) {
@ -343,7 +339,6 @@ func (m *MetaNode) getInodeHandler(w http.ResponseWriter, r *http.Request) {
if len(p.Data) > 0 {
resp.Data = json.RawMessage(p.Data)
}
return
}
func (m *MetaNode) getRaftStatusHandler(w http.ResponseWriter, r *http.Request) {
@ -371,8 +366,7 @@ func (m *MetaNode) getRaftStatusHandler(w http.ResponseWriter, r *http.Request)
resp.Data = raftStatus
}
func (m *MetaNode) getEbsExtentsByInodeHandler(w http.ResponseWriter,
r *http.Request) {
func (m *MetaNode) getEbsExtentsByInodeHandler(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
resp := NewAPIResponse(http.StatusBadRequest, "")
defer func() {
@ -412,11 +406,9 @@ func (m *MetaNode) getEbsExtentsByInodeHandler(w http.ResponseWriter,
if len(p.Data) > 0 {
resp.Data = json.RawMessage(p.Data)
}
return
}
func (m *MetaNode) getExtentsByInodeHandler(w http.ResponseWriter,
r *http.Request) {
func (m *MetaNode) getExtentsByInodeHandler(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
resp := NewAPIResponse(http.StatusBadRequest, "")
defer func() {
@ -466,7 +458,6 @@ func (m *MetaNode) getExtentsByInodeHandler(w http.ResponseWriter,
if len(p.Data) > 0 {
resp.Data = json.RawMessage(p.Data)
}
return
}
func (m *MetaNode) getDentryHandler(w http.ResponseWriter, r *http.Request) {
@ -524,7 +515,6 @@ func (m *MetaNode) getDentryHandler(w http.ResponseWriter, r *http.Request) {
if len(p.Data) > 0 {
resp.Data = json.RawMessage(p.Data)
}
return
}
func (m *MetaNode) getTxHandler(w http.ResponseWriter, r *http.Request) {
@ -573,7 +563,6 @@ func (m *MetaNode) getTxHandler(w http.ResponseWriter, r *http.Request) {
if len(p.Data) > 0 {
resp.Data = json.RawMessage(p.Data)
}
return
}
func (m *MetaNode) getRealVerSeq(w http.ResponseWriter, r *http.Request) (verSeq uint64, err error) {
@ -661,7 +650,6 @@ func (m *MetaNode) getAllDentriesHandler(w http.ResponseWriter, r *http.Request)
if _, err = w.Write(buff.Bytes()); err != nil {
log.LogErrorf("[getAllDentriesHandler] response %s", err)
}
return
}
func (m *MetaNode) getAllTxHandler(w http.ResponseWriter, r *http.Request) {
@ -710,17 +698,11 @@ func (m *MetaNode) getAllTxHandler(w http.ResponseWriter, r *http.Request) {
if ino, ok := i.(*TxRollbackInode); ok {
_, err = w.Write([]byte(ino.ToString()))
if err != nil {
return false
}
return true
return err == nil
}
if den, ok := i.(*TxRollbackDentry); ok {
_, err = w.Write([]byte(den.ToString()))
if err != nil {
return false
}
return true
return err == nil
}
val, err = json.Marshal(i)
@ -745,7 +727,6 @@ func (m *MetaNode) getAllTxHandler(w http.ResponseWriter, r *http.Request) {
if _, err = w.Write(buff.Bytes()); err != nil {
log.LogErrorf("[getAllTxHandler] response %s", err)
}
return
}
func (m *MetaNode) getDirectoryHandler(w http.ResponseWriter, r *http.Request) {
@ -795,7 +776,6 @@ func (m *MetaNode) getDirectoryHandler(w http.ResponseWriter, r *http.Request) {
if len(p.Data) > 0 {
resp.Data = json.RawMessage(p.Data)
}
return
}
func (m *MetaNode) genClusterVersionFileHandler(w http.ResponseWriter, r *http.Request) {
@ -823,7 +803,6 @@ func (m *MetaNode) genClusterVersionFileHandler(w http.ResponseWriter, r *http.R
return
}
}
return
}
func (m *MetaNode) getInodeSnapshotHandler(w http.ResponseWriter, r *http.Request) {

5
metanode/api_handler_test.go
View File

@ -134,15 +134,14 @@ func getSnapshot(t *testing.T, snapshotFile string, url string) {
func TestGetInodeSnapshot(t *testing.T) {
url := fmt.Sprintf("http://127.0.0.1:%v%v?pid=%v",
PROF_PORT, "/getInodeSnapshot", METAPARTITION_ID)
fmt.Printf(url)
fmt.Printf("\n")
fmt.Println(url)
getSnapshot(t, inodeFile, url)
}
func TestGetDentrySnapshot(t *testing.T) {
url := fmt.Sprintf("http://127.0.0.1:%v%v?pid=%v",
PROF_PORT, "/getDentrySnapshot", METAPARTITION_ID)
fmt.Printf(url)
fmt.Println(url)
getSnapshot(t, dentryFile, url)
}

23
metanode/const.go
View File

@ -189,8 +189,6 @@ const (
opFSMVerListSnapShot = 73
)
var exporterKey string
var (
ErrNoLeader = errors.New("no leader")
ErrNotALeader = errors.New("not a leader")
@ -200,16 +198,13 @@ var (
const (
defaultMetadataDir = "metadataDir"
defaultRaftDir = "raftDir"
defaultAuthTimeout = 5 // seconds
)
// Configuration keys
const (
cfgLocalIP = "localIP"
cfgListen = "listen"
cfgMetadataDir = "metadataDir"
cfgRaftDir = "raftDir"
cfgMasterAddrs = "masterAddrs" // will be deprecated
cfgRaftHeartbeatPort = "raftHeartbeatPort"
cfgRaftReplicaPort = "raftReplicaPort"
cfgDeleteBatchCount = "deleteBatchCount"
@ -248,3 +243,21 @@ const (
MB
GB
)
// TODO: to remove unused by golangci
var (
_ = opFSMDelVer
_ = opFSMDeletePartition
_ = opFSMSentToChanV1
_ = opFSMStoreTickV1
_ = opFSMUpdateSummaryInfo
_ = (*metaPartition).doDeleteMarkedInodes
_ = (*Dentry).getLastestVer
_ = (*Inode).isEkInRefMap
_ = (*metaPartition).notifyRaftFollowerToFreeInodes
_ = (*metaPartition).decommissionPartition
_ = (*metaPartition).getDentryTree
_ = (*metaPartition).internalHasInode
_ = (*metaPartition).fsmDelVerExtents
_ = (*TransactionResource).copyGetTxRbInode
)

2
metanode/dentry.go
View File

@ -703,7 +703,7 @@ func (d *Dentry) UnmarshalKey(k []byte) (err error) {
if err = binary.Read(buff, binary.BigEndian, &d.ParentId); err != nil {
return
}
d.Name = string(buff.Bytes())
d.Name = buff.String()
return
}

8
metanode/extend.go
View File

@ -23,11 +23,6 @@ import (
"github.com/cubefs/cubefs/util/btree"
)
type ExtentVal struct {
dataMap map[string][]byte
verSeq uint64
}
type Extend struct {
inode uint64
dataMap map[string][]byte
@ -178,9 +173,8 @@ func (e *Extend) Get(key []byte) (value []byte, exist bool) {
func (e *Extend) Remove(key []byte) {
e.mu.Lock()
defer e.mu.Unlock()
delete(e.dataMap, string(key))
return
e.mu.Unlock()
}
func (e *Extend) Range(visitor func(key, value []byte) bool) {

4
metanode/inode.go
View File

@ -136,7 +136,7 @@ func (i *Inode) setVerNoCheck(seq uint64) {
func (i *Inode) setVer(seq uint64) {
if i.getVer() > seq {
syslog.Println(fmt.Sprintf("inode[%v] old seq [%v] cann't use seq [%v]", i.getVer(), seq, string(debug.Stack())))
syslog.Printf("inode[%v] old seq [%v] cann't use seq [%v]\n", i.getVer(), seq, string(debug.Stack()))
log.LogFatalf("inode[%v] old seq [%v] cann't use seq [%v] stack %v", i.Inode, i.getVer(), seq, string(debug.Stack()))
}
i.verUpdate(seq)
@ -704,8 +704,6 @@ func (i *Inode) MarshalInodeValue(buff *bytes.Buffer) {
if err = binary.Write(buff, binary.BigEndian, i.getVer()); err != nil {
panic(err)
}
return
}
// MarshalValue marshals the value to bytes.

9
metanode/manager.go
View File

@ -83,7 +83,6 @@ type metadataManager struct {
mu sync.RWMutex
partitions map[uint64]MetaPartition // Key: metaRangeId, Val: metaPartition
metaNode *MetaNode
flDeleteBatchCount atomic.Value
fileStatsEnable bool
curQuotaGoroutineNum int32
maxQuotaGoroutineNum int32
@ -377,7 +376,6 @@ func (m *metadataManager) onStop() {
// stop sampler
close(m.stopC)
}
return
}
// LoadMetaPartition returns the meta partition with the specified volName.
@ -511,7 +509,7 @@ func (m *metadataManager) loadPartitions() (err error) {
}
func (m *metadataManager) attachPartition(id uint64, partition MetaPartition) (err error) {
syslog.Println(fmt.Sprintf("start load metaPartition %v", id))
syslog.Printf("start load metaPartition %v\n", id)
partition.ForceSetMetaPartitionToLoadding()
if err = partition.Start(false); err != nil {
msg := fmt.Sprintf("load meta partition %v fail: %v", id, err)
@ -639,10 +637,7 @@ func (m *metadataManager) MarshalJSON() (data []byte, err error) {
func (m *metadataManager) QuotaGoroutineIsOver() (lsOver bool) {
log.LogInfof("QuotaGoroutineIsOver cur [%v] max [%v]", m.curQuotaGoroutineNum, m.maxQuotaGoroutineNum)
if atomic.LoadInt32(&m.curQuotaGoroutineNum) >= m.maxQuotaGoroutineNum {
return true
}
return false
return atomic.LoadInt32(&m.curQuotaGoroutineNum) >= m.maxQuotaGoroutineNum
}
func (m *metadataManager) QuotaGoroutineInc(num int32) {

150
metanode/manager_op.go
View File

@ -47,7 +47,6 @@ func (m *metadataManager) checkFollowerRead(volNames []string, partition MetaPar
}
}
partition.SetFollowerRead(false)
return
}
func (m *metadataManager) checkForbiddenVolume(volNames []string, partition MetaPartition) {
@ -59,7 +58,6 @@ func (m *metadataManager) checkForbiddenVolume(volNames []string, partition Meta
}
}
partition.SetForbidden(false)
return
}
func (m *metadataManager) checkDisableAuditLogVolume(volNames []string, partition MetaPartition) {
@ -71,11 +69,9 @@ func (m *metadataManager) checkDisableAuditLogVolume(volNames []string, partitio
}
}
partition.SetEnableAuditLog(true)
return
}
func (m *metadataManager) opMasterHeartbeat(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opMasterHeartbeat(conn net.Conn, p *Packet, remoteAddr string) (err error) {
// For ack to master
data := p.Data
m.responseAckOKToMaster(conn, p)
@ -164,8 +160,7 @@ func (m *metadataManager) opMasterHeartbeat(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opCreateMetaPartition(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opCreateMetaPartition(conn net.Conn, p *Packet, remoteAddr string) (err error) {
defer func() {
var buf []byte
status := proto.OpOk
@ -201,8 +196,7 @@ func (m *metadataManager) opCreateMetaPartition(conn net.Conn, p *Packet,
}
// Handle OpCreate inode.
func (m *metadataManager) opCreateInode(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opCreateInode(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &CreateInoReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -300,8 +294,7 @@ func (m *metadataManager) opTxMetaLinkInode(conn net.Conn, p *Packet, remoteAddr
return
}
func (m *metadataManager) opMetaLinkInode(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opMetaLinkInode(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &LinkInodeReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -333,8 +326,7 @@ func (m *metadataManager) opMetaLinkInode(conn net.Conn, p *Packet,
}
// Handle OpCreate
func (m *metadataManager) opFreeInodeOnRaftFollower(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opFreeInodeOnRaftFollower(conn net.Conn, p *Packet, remoteAddr string) (err error) {
mp, err := m.getPartition(p.PartitionID)
if err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -350,8 +342,7 @@ func (m *metadataManager) opFreeInodeOnRaftFollower(conn net.Conn, p *Packet,
}
// Handle OpCreate
func (m *metadataManager) opTxCreateDentry(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opTxCreateDentry(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.TxCreateDentryRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -385,8 +376,7 @@ func (m *metadataManager) opTxCreateDentry(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opTxCreate(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opTxCreate(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.TxCreateRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -415,8 +405,7 @@ func (m *metadataManager) opTxCreate(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opTxGet(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opTxGet(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.TxGetInfoRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -447,8 +436,7 @@ func (m *metadataManager) opTxGet(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opTxCommitRM(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opTxCommitRM(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.TxApplyRMRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -477,8 +465,7 @@ func (m *metadataManager) opTxCommitRM(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opTxRollbackRM(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opTxRollbackRM(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.TxApplyRMRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -507,8 +494,7 @@ func (m *metadataManager) opTxRollbackRM(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opTxCommit(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opTxCommit(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.TxApplyRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -537,8 +523,7 @@ func (m *metadataManager) opTxCommit(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opTxRollback(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opTxRollback(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.TxApplyRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -568,8 +553,7 @@ func (m *metadataManager) opTxRollback(conn net.Conn, p *Packet,
}
// Handle OpCreate
func (m *metadataManager) opCreateDentry(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opCreateDentry(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &CreateDentryReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -601,8 +585,7 @@ func (m *metadataManager) opCreateDentry(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opQuotaCreateDentry(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opQuotaCreateDentry(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.QuotaCreateDentryRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -635,8 +618,7 @@ func (m *metadataManager) opQuotaCreateDentry(conn net.Conn, p *Packet,
}
// Handle OpDelete Dentry
func (m *metadataManager) opTxDeleteDentry(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opTxDeleteDentry(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.TxDeleteDentryRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -666,8 +648,7 @@ func (m *metadataManager) opTxDeleteDentry(conn net.Conn, p *Packet,
}
// Handle OpDelete Dentry
func (m *metadataManager) opDeleteDentry(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opDeleteDentry(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &DeleteDentryReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -701,8 +682,7 @@ func (m *metadataManager) opDeleteDentry(conn net.Conn, p *Packet,
}
// Handle Op batch Delete Dentry
func (m *metadataManager) opBatchDeleteDentry(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opBatchDeleteDentry(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &BatchDeleteDentryReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -767,8 +747,7 @@ func (m *metadataManager) opTxUpdateDentry(conn net.Conn, p *Packet, remoteAddr
return
}
func (m *metadataManager) opUpdateDentry(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opUpdateDentry(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &UpdateDentryReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -834,8 +813,7 @@ func (m *metadataManager) opTxMetaUnlinkInode(conn net.Conn, p *Packet, remoteAd
return
}
func (m *metadataManager) opMetaUnlinkInode(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opMetaUnlinkInode(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &UnlinkInoReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -866,8 +844,7 @@ func (m *metadataManager) opMetaUnlinkInode(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opMetaBatchUnlinkInode(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opMetaBatchUnlinkInode(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &BatchUnlinkInoReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -898,8 +875,7 @@ func (m *metadataManager) opMetaBatchUnlinkInode(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opReadDirOnly(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opReadDirOnly(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.ReadDirOnlyRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -925,8 +901,7 @@ func (m *metadataManager) opReadDirOnly(conn net.Conn, p *Packet,
}
// Handle OpReadDir
func (m *metadataManager) opReadDir(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opReadDir(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.ReadDirRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -952,8 +927,7 @@ func (m *metadataManager) opReadDir(conn net.Conn, p *Packet,
}
// Handle OpReadDirLimit
func (m *metadataManager) opReadDirLimit(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opReadDirLimit(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.ReadDirLimitRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -978,9 +952,7 @@ func (m *metadataManager) opReadDirLimit(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opMetaInodeGet(conn net.Conn, p *Packet,
remoteAddr string) (err error,
) {
func (m *metadataManager) opMetaInodeGet(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &InodeGetReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1001,6 +973,7 @@ func (m *metadataManager) opMetaInodeGet(conn net.Conn, p *Packet,
}
if err = mp.InodeGet(req, p); err != nil {
err = errors.NewErrorf("InodeGet [%v],req[%v],err[%v]", p.GetOpMsgWithReqAndResult(), req, string(p.Data))
log.LogDebug(err)
}
if err = m.respondToClient(conn, p); err != nil {
@ -1023,8 +996,7 @@ func (m *metadataManager) opMetaInodeGet(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opBatchMetaEvictInode(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opBatchMetaEvictInode(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.BatchEvictInodeRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
@ -1057,8 +1029,7 @@ func (m *metadataManager) opBatchMetaEvictInode(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opMetaEvictInode(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opMetaEvictInode(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.EvictInodeRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1091,8 +1062,7 @@ func (m *metadataManager) opMetaEvictInode(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opSetAttr(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opSetAttr(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &SetattrRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1128,8 +1098,7 @@ func (m *metadataManager) opSetAttr(conn net.Conn, p *Packet,
}
// Lookup request
func (m *metadataManager) opMetaLookup(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opMetaLookup(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.LookupRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1162,8 +1131,7 @@ func (m *metadataManager) opMetaLookup(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opMetaExtentsAdd(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opMetaExtentsAdd(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.AppendExtentKeyRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1199,8 +1167,7 @@ func (m *metadataManager) opMetaExtentsAdd(conn net.Conn, p *Packet,
}
// Append one extent with discard check
func (m *metadataManager) opMetaExtentAddWithCheck(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opMetaExtentAddWithCheck(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.AppendExtentKeyWithCheckRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1237,8 +1204,7 @@ func (m *metadataManager) opMetaExtentAddWithCheck(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opMetaExtentsList(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opMetaExtentsList(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.GetExtentsRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1266,8 +1232,7 @@ func (m *metadataManager) opMetaExtentsList(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opMetaObjExtentsList(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opMetaObjExtentsList(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.GetExtentsRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1293,8 +1258,7 @@ func (m *metadataManager) opMetaObjExtentsList(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opMetaExtentsDel(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opMetaExtentsDel(conn net.Conn, p *Packet, remoteAddr string) (err error) {
panic("not implemented yet")
// req := &proto.DelExtentKeyRequest{}
// if err = json.Unmarshal(p.Data, req); err != nil {
@ -1320,8 +1284,7 @@ func (m *metadataManager) opMetaExtentsDel(conn net.Conn, p *Packet,
// return
}
func (m *metadataManager) opMetaExtentsTruncate(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opMetaExtentsTruncate(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &ExtentsTruncateReq{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1352,8 +1315,7 @@ func (m *metadataManager) opMetaExtentsTruncate(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opMetaClearInodeCache(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opMetaClearInodeCache(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.ClearInodeCacheRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1379,8 +1341,7 @@ func (m *metadataManager) opMetaClearInodeCache(conn net.Conn, p *Packet,
}
// Delete a meta partition.
func (m *metadataManager) opDeleteMetaPartition(conn net.Conn,
p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opDeleteMetaPartition(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.DeleteMetaPartitionRequest{}
adminTask := &proto.AdminTask{
Request: req,
@ -1413,8 +1374,7 @@ func (m *metadataManager) opDeleteMetaPartition(conn net.Conn,
return
}
func (m *metadataManager) opUpdateMetaPartition(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opUpdateMetaPartition(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := new(UpdatePartitionReq)
adminTask := &proto.AdminTask{
Request: req,
@ -1453,8 +1413,7 @@ func (m *metadataManager) opUpdateMetaPartition(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opLoadMetaPartition(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opLoadMetaPartition(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.MetaPartitionLoadRequest{}
adminTask := &proto.AdminTask{
Request: req,
@ -1488,8 +1447,7 @@ func (m *metadataManager) opLoadMetaPartition(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opDecommissionMetaPartition(conn net.Conn,
p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opDecommissionMetaPartition(conn net.Conn, p *Packet, remoteAddr string) (err error) {
var reqData []byte
req := &proto.MetaPartitionDecommissionRequest{}
adminTask := &proto.AdminTask{
@ -1547,8 +1505,7 @@ func (m *metadataManager) opDecommissionMetaPartition(conn net.Conn,
return
}
func (m *metadataManager) opAddMetaPartitionRaftMember(conn net.Conn,
p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opAddMetaPartitionRaftMember(conn net.Conn, p *Packet, remoteAddr string) (err error) {
var reqData []byte
req := &proto.AddMetaPartitionRaftMemberRequest{}
adminTask := &proto.AdminTask{
@ -1616,8 +1573,7 @@ func (m *metadataManager) opAddMetaPartitionRaftMember(conn net.Conn,
return
}
func (m *metadataManager) opRemoveMetaPartitionRaftMember(conn net.Conn,
p *Packet, remoteAddr string) (err error) {
func (m *metadataManager) opRemoveMetaPartitionRaftMember(conn net.Conn, p *Packet, remoteAddr string) (err error) {
var reqData []byte
req := &proto.RemoveMetaPartitionRaftMemberRequest{}
adminTask := &proto.AdminTask{
@ -1694,8 +1650,7 @@ func (m *metadataManager) opRemoveMetaPartitionRaftMember(conn net.Conn,
return
}
func (m *metadataManager) opMetaBatchInodeGet(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opMetaBatchInodeGet(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.BatchInodeGetRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1721,8 +1676,7 @@ func (m *metadataManager) opMetaBatchInodeGet(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opMetaPartitionTryToLeader(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opMetaPartitionTryToLeader(conn net.Conn, p *Packet, remoteAddr string) (err error) {
mp, err := m.getPartition(p.PartitionID)
if err != nil {
goto errDeal
@ -1739,8 +1693,7 @@ errDeal:
return
}
func (m *metadataManager) opMetaDeleteInode(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opMetaDeleteInode(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.DeleteInodeRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -1765,8 +1718,7 @@ func (m *metadataManager) opMetaDeleteInode(conn net.Conn, p *Packet,
return
}
func (m *metadataManager) opMetaBatchDeleteInode(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opMetaBatchDeleteInode(conn net.Conn, p *Packet, remoteAddr string) (err error) {
var req *proto.DeleteInodeBatchRequest
if err = json.Unmarshal(p.Data, &req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -2214,8 +2166,7 @@ func (m *metadataManager) opListMultipart(conn net.Conn, p *Packet, remoteAddr s
}
// Handle OpMetaTxCreateInode inode.
func (m *metadataManager) opTxCreateInode(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opTxCreateInode(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.TxCreateInodeRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -2351,8 +2302,7 @@ func (m *metadataManager) opMetaGetInodeQuota(conn net.Conn, p *Packet, remote s
return
}
func (m *metadataManager) opMetaGetUniqID(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opMetaGetUniqID(conn net.Conn, p *Packet, remoteAddr string) (err error) {
req := &proto.GetUniqIDRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
@ -2539,7 +2489,6 @@ func (m *metadataManager) updatePackRspSeq(mp MetaPartition, p *Packet) {
p.VerList = make([]*proto.VolVersionInfo, len(mp.GetVerList()))
copy(p.VerList, mp.GetVerList())
}
return
}
func (m *metadataManager) checkMultiVersionStatus(mp MetaPartition, p *Packet) (err error) {
@ -2628,8 +2577,7 @@ func (m *metadataManager) checkAndPromoteVersion(volName string) (err error) {
return
}
func (m *metadataManager) opMultiVersionOp(conn net.Conn, p *Packet,
remoteAddr string) (err error) {
func (m *metadataManager) opMultiVersionOp(conn net.Conn, p *Packet, remoteAddr string) (err error) {
// For ack to master
data := p.Data
m.responseAckOKToMaster(conn, p)

2
metanode/meta_quota_manager.go
View File

@ -194,7 +194,6 @@ func (mqMgr *MetaQuotaManager) setQuotaHbInfo(infos []*proto.QuotaHeartBeatInfo)
}
return true
})
return
}
func (mqMgr *MetaQuotaManager) getQuotaReportInfos() (infos []*proto.QuotaReportInfo) {
@ -331,7 +330,6 @@ func (mqMgr *MetaQuotaManager) updateUsedInfo(size int64, files int64, quotaId u
mqMgr.statisticRebuildTemp.Store(quotaId, baseTemp)
}
log.LogDebugf("updateUsedInfo mpId [%v] quotaId [%v] baseInfo [%v] baseTemp[%v]", mqMgr.mpID, quotaId, baseInfo, baseTemp)
return
}
func (mqMgr *MetaQuotaManager) EnableQuota() bool {

2
metanode/metanode.go
View File

@ -271,7 +271,7 @@ func (m *MetaNode) parseConfig(cfg *config.Config) (err error) {
if cfg.HasKey(cfgRaftSyncSnapFormatVersion) {
raftSyncSnapFormatVersion := uint32(cfg.GetInt64(cfgRaftSyncSnapFormatVersion))
if raftSyncSnapFormatVersion < 0 || raftSyncSnapFormatVersion > SnapFormatVersion_1 {
if raftSyncSnapFormatVersion > SnapFormatVersion_1 {
m.raftSyncSnapFormatVersion = SnapFormatVersion_1
log.LogInfof("invalid config raftSyncSnapFormatVersion, using default[%v]", m.raftSyncSnapFormatVersion)
} else {

Some files were not shown because too many files have changed in this diff Show More