From 03f79e737099cd3f6e782f6abbb781b8ab871aa7 Mon Sep 17 00:00:00 2001 From: awzhgw Date: Mon, 17 Jun 2019 11:48:39 +0800 Subject: [PATCH] Specification code and metanode config file Signed-off-by: awzhgw --- cmd/cfg/datanode.json | 31 +++++++++++++++---------------- cmd/cfg/master.json | 24 ++++++++++-------------- cmd/cfg/meta.json | 22 ++++++++++++---------- cmd/cmd.go | 4 ++-- datanode/disk.go | 6 +++--- datanode/partition.go | 8 ++++---- datanode/wrap_prepare.go | 12 ++++++------ metanode/metanode.go | 16 ++++++++++++++++ metanode/partition.go | 2 +- metanode/partition_fsm.go | 2 +- metanode/partition_store.go | 16 ++++++++-------- storage/extent_store.go | 30 +++++++++++++++--------------- util/ump/ump.go | 2 +- util/ump/ump_async_log.go | 2 +- 14 files changed, 95 insertions(+), 82 deletions(-) diff --git a/cmd/cfg/datanode.json b/cmd/cfg/datanode.json index 26f74dd40..38bdd8105 100644 --- a/cmd/cfg/datanode.json +++ b/cmd/cfg/datanode.json @@ -2,22 +2,21 @@ "role": "datanode", "port": "6000", "prof": "6001", - "clusterID": "data", - "logDir": "/var/logs", - "logLevel": "debug", - "masterAddr": [ - "10.196.30.200:80", - "10.196.31.141:80", - "10.196.31.173:80" - ], - "rack": "rack-01", - "consulAddr": "http://cbconsul-cfs01.cbmonitor.svc.ht7.n.jd.local", + "logDir": "/export/Logs/datanode", + "logLevel": "info", + "raftHeartbeat": "9095", + "raftReplica": "9096", + "warnLogDir":"/export/home/tomcat/UMP-Monitor/logs/", + "consulAddr": "http://consul.prometheus-cfs.local", "exporterPort": 9512, + "masterAddr": [ + "192.168.31.173:80", + "192.168.31.141:80", + "192.168.30.200:80" + ], + "rack": "", "disks": [ - "/data0:1:20000", - "/data1:1:20000", - "/data2:1:20000", - "/data3:1:20000", - "/data4:1:20000" + "/data0:21474836480", + "/data1:21474836480" ] -} \ No newline at end of file +} diff --git a/cmd/cfg/master.json b/cmd/cfg/master.json index 75e7bd1e2..ad0e931e4 100644 --- a/cmd/cfg/master.json +++ b/cmd/cfg/master.json @@ -1,21 +1,17 @@ { "role": "master", - "ip": "10.196.31.173", + "ip": "192.168.31.173", "port": "80", "prof":"10088", "id":"1", - "peers": "1:10.196.31.173:80,2:10.196.31.141:80,3:10.196.30.200:80", + "peers": "1:192.168.31.173:80,2:192.168.31.141:80,3:192.168.30.200:80", "retainLogs":"20000", - "logDir": "/export/Logs/baudstorage/master", - "logLevel":"DEBUG", - "walDir":"/export/Logs/baudstorage/raft", - "storeDir":"/export/baudstorage/rocksdbstore", - "consulAddr": "http://cbconsul-cfs01.cbmonitor.svc.ht7.n.jd.local", + "logDir": "/export/Logs/master", + "logLevel":"info", + "walDir":"/export/Data/master/raft", + "storeDir":"/export/Data/master/rocksdbstore", + "warnLogDir":"/export/home/tomcat/UMP-Monitor/logs/", + "consulAddr": "http://consul.prometheus-cfs.local", "exporterPort": 9510, - "clusterName":"baudStorage" -} - - - - - + "clusterName":"cfs" +} \ No newline at end of file diff --git a/cmd/cfg/meta.json b/cmd/cfg/meta.json index 2d3b414ef..ecc778230 100644 --- a/cmd/cfg/meta.json +++ b/cmd/cfg/meta.json @@ -2,17 +2,19 @@ "role": "metanode", "listen": "9021", "prof": "9092", - "logLevel": "debug", - "metadataDir": "/export/Data/bdfs/metanode_meta", - "logDir": "/export/Logs/bdfs", - "raftDir": "/export/Data/bdfs/metanode_raft", + "logLevel": "info", + "metadataDir": "/export/Data/metanode", + "logDir": "/export/Logs/metanode", + "raftDir": "/export/Data/metanode/raft", "raftHeartbeatPort": "9093", - "raftReplicaPort": "9094", - "consulAddr": "http://cbconsul-cfs01.cbmonitor.svc.ht7.n.jd.local", + "raftReplicatePort": "9094", + "totalMem": "17179869184", + "warnLogDir":"/export/home/tomcat/UMP-Monitor/logs/", + "consulAddr": "http://consul.prometheus-cfs.local", "exporterPort": 9511, "masterAddrs": [ - "10.196.31.141:80", - "10.196.30.200:80", - "10.196.31.173:80" + "192.168.31.173:80", + "192.168.31.141:80", + "192.168.30.200:80" ] -} +} \ No newline at end of file diff --git a/cmd/cmd.go b/cmd/cmd.go index 4e44d3be9..27c608b4f 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -194,8 +194,8 @@ func main() { interceptSignal(server) err = server.Start(cfg) if err != nil { - fmt.Println("Fatal: failed to start the baud storage daemon - ", err) - log.LogFatal("Fatal: failed to start the baud storage daemon - ", err) + fmt.Println(fmt.Sprintf("Fatal: failed to start the ChubaoFS %v daemon err %v - ", role, err)) + log.LogFatal(fmt.Sprintf("Fatal: failed to start the ChubaoFS %v daemon err %v - ", role, err)) log.LogFlush() os.Exit(1) return diff --git a/datanode/disk.go b/datanode/disk.go index ebacd21eb..dc4588664 100644 --- a/datanode/disk.go +++ b/datanode/disk.go @@ -209,11 +209,11 @@ func (d *Disk) ForceLoadPartitionHeader() { } } -func (d *Disk) ForceExitRaftStore(){ +func (d *Disk) ForceExitRaftStore() { partitionList := d.DataPartitionList() for _, partitionID := range partitionList { partition := d.GetDataPartition(partitionID) - partition.partitionStatus=proto.Unavailable + partition.partitionStatus = proto.Unavailable partition.stopRaft() } } @@ -289,7 +289,7 @@ func (d *Disk) RestorePartition(visitor PartitionVisitor) { ) defer wg.Done() if dp, err = LoadDataPartition(path.Join(d.Path, filename), d); err != nil { - mesg:=fmt.Sprintf("action[RestorePartition] new partition(%v) err(%v) ", + mesg := fmt.Sprintf("action[RestorePartition] new partition(%v) err(%v) ", partitionID, err.Error()) log.LogError(mesg) exporter.NewAlarm(mesg) diff --git a/datanode/partition.go b/datanode/partition.go index 14b6cd895..b5e064493 100644 --- a/datanode/partition.go +++ b/datanode/partition.go @@ -662,15 +662,15 @@ func (dp *DataPartition) doStreamFixTinyDeleteRecord(repairTask *DataPartitionRe err = fmt.Errorf("crc not match") return } - if p.Size%storage.EveryTinyDeleteRecordSize != 0 { + if p.Size%storage.DeleteTinyRecordSize != 0 { err = fmt.Errorf("unavali size") return } var index int - for (index+1)*storage.EveryTinyDeleteRecordSize <= int(p.Size) { - record := p.Data[index*storage.EveryTinyDeleteRecordSize : (index+1)*storage.EveryTinyDeleteRecordSize] + for (index+1)*storage.DeleteTinyRecordSize <= int(p.Size) { + record := p.Data[index*storage.DeleteTinyRecordSize : (index+1)*storage.DeleteTinyRecordSize] extentID, offset, size := storage.UnMarshalTinyExtent(record) - localTinyDeleteFileSize += storage.EveryTinyDeleteRecordSize + localTinyDeleteFileSize += storage.DeleteTinyRecordSize index++ if !storage.IsTinyExtent(extentID) { continue diff --git a/datanode/wrap_prepare.go b/datanode/wrap_prepare.go index 8ef709270..4f26b73b5 100644 --- a/datanode/wrap_prepare.go +++ b/datanode/wrap_prepare.go @@ -92,25 +92,25 @@ func (s *DataNode) addExtentInfo(p *repl.Packet) error { store := p.Object.(*DataPartition).ExtentStore() var ( extentID uint64 - err error + err error ) if isLeaderPacket(p) && p.ExtentType == proto.TinyExtentType && isWriteOperation(p) { extentID, err = store.GetAvailableTinyExtent() if err != nil { - return fmt.Errorf("addExtentInfo partition %v GetAvailableTinyExtent error %v", p.PartitionID,err.Error()) + return fmt.Errorf("addExtentInfo partition %v GetAvailableTinyExtent error %v", p.PartitionID, err.Error()) } p.ExtentID = extentID p.ExtentOffset, err = store.GetTinyExtentOffset(extentID) if err != nil { - return fmt.Errorf("addExtentInfo partition %v %v GetTinyExtentOffset error %v", p.PartitionID,extentID,err.Error()) + return fmt.Errorf("addExtentInfo partition %v %v GetTinyExtentOffset error %v", p.PartitionID, extentID, err.Error()) } } else if isLeaderPacket(p) && isCreateExtentOperation(p) { if partition.GetExtentCount() >= storage.MaxExtentCount*3 { return fmt.Errorf("addExtentInfo partition %v has reached maxExtentId", p.PartitionID) } - p.ExtentID,err = store.NextExtentID() - if err!=nil { - return fmt.Errorf("addExtentInfo partition %v alloc NextExtentId error %v", p.PartitionID,err) + p.ExtentID, err = store.NextExtentID() + if err != nil { + return fmt.Errorf("addExtentInfo partition %v alloc NextExtentId error %v", p.PartitionID, err) } } else if isLeaderPacket(p) && isMarkDeleteExtentOperation(p) && isTinyExtentType(p) { record := new(proto.TinyExtentDeleteRecord) diff --git a/metanode/metanode.go b/metanode/metanode.go index 076e4be13..329b1b675 100644 --- a/metanode/metanode.go +++ b/metanode/metanode.go @@ -22,6 +22,7 @@ import ( "sync/atomic" "time" + "fmt" "github.com/chubaofs/chubaofs/proto" "github.com/chubaofs/chubaofs/raftstore" "github.com/chubaofs/chubaofs/util" @@ -149,6 +150,21 @@ func (m *MetaNode) parseConfig(cfg *config.Config) (err error) { if configTotalMem > total { configTotalMem = total } + if m.metadataDir == "" { + return fmt.Errorf("bad metadataDir config") + } + if m.listen == "" { + return fmt.Errorf("bad listen config") + } + if m.raftDir == "" { + return fmt.Errorf("bad raftDir config") + } + if m.raftHeartbeatPort == "" { + return fmt.Errorf("bad raftHeartbeatPort config") + } + if m.raftReplicatePort == "" { + return fmt.Errorf("bad cfgRaftReplicaPort config") + } log.LogInfof("[parseConfig] load localAddr[%v].", m.localAddr) log.LogInfof("[parseConfig] load listen[%v].", m.listen) diff --git a/metanode/partition.go b/metanode/partition.go index 51915f0bc..da89de320 100644 --- a/metanode/partition.go +++ b/metanode/partition.go @@ -298,7 +298,7 @@ func (mp *metaPartition) stopRaft() { if mp.raftPartition != nil { // TODO Unhandled errors mp.raftPartition.Stop() - mp.raftPartition=nil + mp.raftPartition = nil } return } diff --git a/metanode/partition_fsm.go b/metanode/partition_fsm.go index a9f22758b..663a710d9 100644 --- a/metanode/partition_fsm.go +++ b/metanode/partition_fsm.go @@ -36,7 +36,7 @@ import ( func (mp *metaPartition) Apply(command []byte, index uint64) (resp interface{}, err error) { msg := &MetaItem{} defer func() { - if err==nil { + if err == nil { mp.uploadApplyID(index) } }() diff --git a/metanode/partition_store.go b/metanode/partition_store.go index 929165ebe..ee5aebb2d 100644 --- a/metanode/partition_store.go +++ b/metanode/partition_store.go @@ -26,8 +26,8 @@ import ( "io/ioutil" "os" "path" - "sync/atomic" "strings" + "sync/atomic" ) const ( @@ -206,18 +206,18 @@ func (mp *metaPartition) loadApplyID(rootDir string) (err error) { return } var cursor uint64 - if strings.Contains(string(data),"|"){ - _, err = fmt.Sscanf(string(data), "%d|%d", &mp.applyID,&cursor) - }else { + if strings.Contains(string(data), "|") { + _, err = fmt.Sscanf(string(data), "%d|%d", &mp.applyID, &cursor) + } else { _, err = fmt.Sscanf(string(data), "%d", &mp.applyID) } - if err!=nil { + if err != nil { err = errors.NewErrorf("[loadApplyID] ReadApplyID: %s", err.Error()) return } - if cursor>atomic.LoadUint64(&mp.config.Cursor){ - atomic.StoreUint64(&mp.config.Cursor,cursor) + if cursor > atomic.LoadUint64(&mp.config.Cursor) { + atomic.StoreUint64(&mp.config.Cursor, cursor) } return @@ -265,7 +265,7 @@ func (mp *metaPartition) storeApplyID(rootDir string, sm *storeMsg) (err error) err = fp.Sync() fp.Close() }() - if _, err = fp.WriteString(fmt.Sprintf("%d|%d", sm.applyIndex,atomic.LoadUint64(&mp.config.Cursor))); err != nil { + if _, err = fp.WriteString(fmt.Sprintf("%d|%d", sm.applyIndex, atomic.LoadUint64(&mp.config.Cursor))); err != nil { return } return diff --git a/storage/extent_store.go b/storage/extent_store.go index d6b02b489..d0b706607 100644 --- a/storage/extent_store.go +++ b/storage/extent_store.go @@ -39,17 +39,17 @@ import ( ) const ( - ExtCrcHeaderFileName = "EXTENT_CRC" - ExtBaseExtentIDFileName = "EXTENT_META" - TinyDeleteFileOpt = os.O_CREATE | os.O_RDWR - TinyExtDeletedFileName = "TINYEXTENT_DELETE" - NormalExtDeletedFileName = "NORMALEXTENT_DELETE" - MaxExtentCount = 20000 - TinyExtentCount = 64 - TinyExtentStartID = 1 - MinExtentID = 1024 - EveryTinyDeleteRecordSize = 24 - UpdateCrcInterval = 600 + ExtCrcHeaderFileName = "EXTENT_CRC" + ExtBaseExtentIDFileName = "EXTENT_META" + TinyDeleteFileOpt = os.O_CREATE | os.O_RDWR + TinyExtDeletedFileName = "TINYEXTENT_DELETE" + NormalExtDeletedFileName = "NORMALEXTENT_DELETE" + MaxExtentCount = 20000 + TinyExtentCount = 64 + TinyExtentStartID = 1 + MinExtentID = 1024 + DeleteTinyRecordSize = 24 + UpdateCrcInterval = 600 ) var ( @@ -591,17 +591,17 @@ func (s *ExtentStore) StoreSize() (totalSize uint64) { } func MarshalTinyExtent(extentID uint64, offset, size int64) (data []byte) { - data = make([]byte, EveryTinyDeleteRecordSize) + data = make([]byte, DeleteTinyRecordSize) binary.BigEndian.PutUint64(data[0:8], extentID) binary.BigEndian.PutUint64(data[8:16], uint64(offset)) - binary.BigEndian.PutUint64(data[16:EveryTinyDeleteRecordSize], uint64(size)) + binary.BigEndian.PutUint64(data[16:DeleteTinyRecordSize], uint64(size)) return data } func UnMarshalTinyExtent(data []byte) (extentID, offset, size uint64) { extentID = binary.BigEndian.Uint64(data[0:8]) offset = binary.BigEndian.Uint64(data[8:16]) - size = binary.BigEndian.Uint64(data[16:EveryTinyDeleteRecordSize]) + size = binary.BigEndian.Uint64(data[16:DeleteTinyRecordSize]) return } @@ -637,7 +637,7 @@ func (s *ExtentStore) NextExtentID() (extentID uint64,err error) { } func (s *ExtentStore) NextTinyDeleteFileOffset() (offset int64) { - return atomic.AddInt64(&s.baseTinyDeleteOffset, EveryTinyDeleteRecordSize) + return atomic.AddInt64(&s.baseTinyDeleteOffset, DeleteTinyRecordSize) } func (s *ExtentStore) LoadTinyDeleteFileOffset() (offset int64) { diff --git a/util/ump/ump.go b/util/ump/ump.go index 009eab5b0..3960056c8 100644 --- a/util/ump/ump.go +++ b/util/ump/ump.go @@ -1,4 +1,4 @@ -// Copyright 2018 The ChubaoFS Authors. +// Copyright 2018 The Chubao Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/util/ump/ump_async_log.go b/util/ump/ump_async_log.go index 4fda2831a..c48196119 100644 --- a/util/ump/ump_async_log.go +++ b/util/ump/ump_async_log.go @@ -1,4 +1,4 @@ -// Copyright 2018 The ChubaoFS Authors. +// Copyright 2018 The Chubao Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License.