gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 02:00:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

enhance(log): Modify the threshold triggering log to logLeftSpaceLimitRatio of the total disk space

Browse Source 
@formatter:off

Signed-off-by: shuqiang-zheng <zhengshuqiang@oppo.com>
This commit is contained in:
shuqiang-zheng 2024-05-20 11:02:39 +08:00 committed by longerfly
parent 76121f501d
commit eed62c913f
15 changed files with 34 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
blockcache/cmd.go
View File

@ -166,7 +166,7 @@ func main() {
level = log.ErrorLevel level = log.ErrorLevel
} }
_, err = log.InitLog(logDir, module, level, nil, log.DefaultLogLeftSpaceLimit) _, err = log.InitLog(logDir, module, level, nil, log.DefaultLogLeftSpaceLimitRatio)
if err != nil { if err != nil {
err = errors.NewErrorf("Fatal: failed to init log - %v", err) err = errors.NewErrorf("Fatal: failed to init log - %v", err)
fmt.Println(err) fmt.Println(err)

2
cli/cli.go
View File

@ -71,7 +71,7 @@ func setupCommands(cfg *cmd.Config) *cobra.Command {
func main() { func main() {
var err error var err error
_, err = log.InitLog("/tmp/cfs", "cli", log.DebugLevel, nil, log.DefaultLogLeftSpaceLimit) _, err = log.InitLog("/tmp/cfs", "cli", log.DebugLevel, nil, log.DefaultLogLeftSpaceLimitRatio)
if err != nil { if err != nil {
fmt.Fprintln(os.Stderr, "Error:", err) fmt.Fprintln(os.Stderr, "Error:", err)
os.Exit(1) os.Exit(1)

2
client/fuse.go
View File

@ -308,7 +308,7 @@ func main() {
// use uber automaxprocs: get real cpu number to k8s pod" // use uber automaxprocs: get real cpu number to k8s pod"
level := parseLogLevel(opt.Loglvl) level := parseLogLevel(opt.Loglvl)
_, err = log.InitLog(opt.Logpath, opt.Volname, level, nil, log.DefaultLogLeftSpaceLimit) _, err = log.InitLog(opt.Logpath, opt.Volname, level, nil, log.DefaultLogLeftSpaceLimitRatio)
if err != nil { if err != nil {
err = errors.NewErrorf("Init log dir fail: %v\n", err) err = errors.NewErrorf("Init log dir fail: %v\n", err)
fmt.Println(err) fmt.Println(err)

32
cmd/cmd.go
View File

@ -48,16 +48,16 @@ import (
) )
const ( const (
ConfigKeyRole = "role" ConfigKeyRole = "role"
ConfigKeyLogDir = "logDir" ConfigKeyLogDir = "logDir"
ConfigKeyLogLevel = "logLevel" ConfigKeyLogLevel = "logLevel"
ConfigKeyLogRotateSize = "logRotateSize" ConfigKeyLogRotateSize = "logRotateSize"
ConfigKeyLogRotateHeadRoom = "logRotateHeadRoom" ConfigKeyLogRotateHeadRoom = "logRotateHeadRoom"
ConfigKeyProfPort = "prof" ConfigKeyProfPort = "prof"
ConfigKeyWarnLogDir = "warnLogDir" ConfigKeyWarnLogDir = "warnLogDir"
ConfigKeyBuffersTotalLimit = "buffersTotalLimit" ConfigKeyBuffersTotalLimit = "buffersTotalLimit"
ConfigKeyLogLeftSpaceLimit = "logLeftSpaceLimit" ConfigKeyLogLeftSpaceLimitRatio = "logLeftSpaceLimitRatio"
ConfigKeyEnableLogPanicHook = "enableLogPanicHook" ConfigKeyEnableLogPanicHook = "enableLogPanicHook"
) )
const ( const (
@ -165,12 +165,12 @@ func main() {
profPort := cfg.GetString(ConfigKeyProfPort) profPort := cfg.GetString(ConfigKeyProfPort)
umpDatadir := cfg.GetString(ConfigKeyWarnLogDir) umpDatadir := cfg.GetString(ConfigKeyWarnLogDir)
buffersTotalLimit := cfg.GetInt64(ConfigKeyBuffersTotalLimit) buffersTotalLimit := cfg.GetInt64(ConfigKeyBuffersTotalLimit)
logLeftSpaceLimitStr := cfg.GetString(ConfigKeyLogLeftSpaceLimit) logLeftSpaceLimitRatioStr := cfg.GetString(ConfigKeyLogLeftSpaceLimitRatio)
logLeftSpaceLimit, err := strconv.ParseInt(logLeftSpaceLimitStr, 10, 64) logLeftSpaceLimitRatio, err := strconv.ParseFloat(logLeftSpaceLimitRatioStr, 64)
enableLogPanicHook := cfg.GetBool(ConfigKeyEnableLogPanicHook) enableLogPanicHook := cfg.GetBool(ConfigKeyEnableLogPanicHook)
if err != nil || logLeftSpaceLimit == 0 { if err != nil || logLeftSpaceLimitRatio <= 0 || logLeftSpaceLimitRatio > 1.0 {
log.LogErrorf("logLeftSpaceLimit is not a legal int value: %v", err.Error()) log.LogErrorf("logLeftSpaceLimitRatio is not a legal float value: %v", err.Error())
logLeftSpaceLimit = log.DefaultLogLeftSpaceLimit logLeftSpaceLimitRatio = log.DefaultLogLeftSpaceLimitRatio
} }
// Init server instance with specified role configuration. // Init server instance with specified role configuration.
var ( var (
@ -231,7 +231,7 @@ func main() {
if logRotateHeadRoom > 0 { if logRotateHeadRoom > 0 {
rotate.SetHeadRoomMb(logRotateHeadRoom) rotate.SetHeadRoomMb(logRotateHeadRoom)
} }
_, err = log.InitLog(logDir, module, level, rotate, logLeftSpaceLimit) _, err = log.InitLog(logDir, module, level, rotate, logLeftSpaceLimitRatio)
if err != nil { if err != nil {
err = errors.NewErrorf("Fatal: failed to init log - %v", err) err = errors.NewErrorf("Fatal: failed to init log - %v", err)
fmt.Println(err) fmt.Println(err)

2
deploy/deploy_cli.go
View File

@ -17,7 +17,7 @@ func init() {
} }
func main() { func main() {
_, err := log.InitLog("/tmp/cfs", "deploy", log.DebugLevel, nil, log.DefaultLogLeftSpaceLimit) _, err := log.InitLog("/tmp/cfs", "deploy", log.DebugLevel, nil, log.DefaultLogLeftSpaceLimitRatio)
if err != nil { if err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err) fmt.Fprintf(os.Stderr, "Error: %v\n", err)
log.LogFlush() log.LogFlush()

2
fsck/cmd/clean.go
View File

@ -105,7 +105,7 @@ func Clean(opt string) error {
ump.InitUmp("fsck", "") ump.InitUmp("fsck", "")
_, err := log.InitLog("fscklog", "fsck", log.InfoLevel, nil, log.DefaultLogLeftSpaceLimit) _, err := log.InitLog("fscklog", "fsck", log.InfoLevel, nil, log.DefaultLogLeftSpaceLimitRatio)
if err != nil { if err != nil {
return fmt.Errorf("Init log failed: %v", err) return fmt.Errorf("Init log failed: %v", err)
} }

2
libsdk/libsdk.go
View File

@ -1158,7 +1158,7 @@ func (c *client) start() (err error) {
c.logLevel = "WARN" c.logLevel = "WARN"
} }
level := parseLogLevel(c.logLevel) level := parseLogLevel(c.logLevel)
log.InitLog(c.logDir, "libcfs", level, nil, log.DefaultLogLeftSpaceLimit) log.InitLog(c.logDir, "libcfs", level, nil, log.DefaultLogLeftSpaceLimitRatio)
stat.NewStatistic(c.logDir, "libcfs", int64(stat.DefaultStatLogSize), stat.DefaultTimeOutUs, true) stat.NewStatistic(c.logDir, "libcfs", int64(stat.DefaultStatLogSize), stat.DefaultTimeOutUs, true)
} }
proto.InitBufferPool(int64(32768)) proto.InitBufferPool(int64(32768))

2
master/api_service_test.go
View File

@ -238,7 +238,7 @@ func createMasterServer(cfgJSON string) (server *Server, err error) {
level = log.ErrorLevel level = log.ErrorLevel
} }
if mocktest.LogOn { if mocktest.LogOn {
if _, err = log.InitLog(logDir, "master", level, nil, log.DefaultLogLeftSpaceLimit); err != nil { if _, err = log.InitLog(logDir, "master", level, nil, log.DefaultLogLeftSpaceLimitRatio); err != nil {
fmt.Println("Fatal: failed to start the cubefs daemon - ", err) fmt.Println("Fatal: failed to start the cubefs daemon - ", err)
return return
} }

2
metanode/multi_ver_test.go
View File

@ -93,7 +93,7 @@ func init() {
logDir := cfg.GetString(ConfigKeyLogDir) logDir := cfg.GetString(ConfigKeyLogDir)
os.RemoveAll(logDir) os.RemoveAll(logDir)
if _, err := log.InitLog(logDir, "metanode", log.DebugLevel, nil, log.DefaultLogLeftSpaceLimit); err != nil { if _, err := log.InitLog(logDir, "metanode", log.DebugLevel, nil, log.DefaultLogLeftSpaceLimitRatio); err != nil {
fmt.Println("Fatal: failed to start the cubefs daemon - ", err) fmt.Println("Fatal: failed to start the cubefs daemon - ", err)
return return
} }

2
metanode/transaction_test.go
View File

@ -45,7 +45,7 @@ const (
) )
func init() { func init() {
log.InitLog("/tmp/cfs/logs/", "test", log.DebugLevel, nil, log.DefaultLogLeftSpaceLimit) log.InitLog("/tmp/cfs/logs/", "test", log.DebugLevel, nil, log.DefaultLogLeftSpaceLimitRatio)
} }
func newMetaPartition(PartitionId uint64, manager *metadataManager) (mp *metaPartition) { func newMetaPartition(PartitionId uint64, manager *metadataManager) (mp *metaPartition) {

2
preload/sdk/preloadsdk.go
View File

@ -89,7 +89,7 @@ func NewClient(config PreloadConfig) *PreLoadClient {
} }
if config.LogDir != "" { if config.LogDir != "" {
log.InitLog(config.LogDir, "preload", convertLogLevel(config.LogLevel), nil, log.DefaultLogLeftSpaceLimit) log.InitLog(config.LogDir, "preload", convertLogLevel(config.LogLevel), nil, log.DefaultLogLeftSpaceLimitRatio)
stat.NewStatistic(config.LogDir, "preload", int64(stat.DefaultStatLogSize), stat.DefaultTimeOutUs, true) stat.NewStatistic(config.LogDir, "preload", int64(stat.DefaultStatLogSize), stat.DefaultTimeOutUs, true)
} }

2
snapshot/cmd/clean.go
View File

@ -125,7 +125,7 @@ func Clean(opt string, args []string) error {
ump.InitUmp("snapshot", "") ump.InitUmp("snapshot", "")
_, err := log.InitLog("snapshotlog", "snapshot", log.DebugLevel, nil, log.DefaultLogLeftSpaceLimit) _, err := log.InitLog("snapshotlog", "snapshot", log.DebugLevel, nil, log.DefaultLogLeftSpaceLimitRatio)
if err != nil { if err != nil {
return fmt.Errorf("Init log failed: %v", err) return fmt.Errorf("Init log failed: %v", err)
} }

9
util/log/log.go
View File

@ -302,7 +302,7 @@ func (l *Log) outputStderr(calldepth int, s string) {
} }
// InitLog initializes the log. // InitLog initializes the log.
func InitLog(dir, module string, level Level, rotate *LogRotate, logLeftSpaceLimit int64) (*Log, error) { func InitLog(dir, module string, level Level, rotate *LogRotate, logLeftSpaceLimitRatio float64) (*Log, error) {
l := new(Log) l := new(Log)
l.printStderr = 1 l.printStderr = 1
dir = path.Join(dir, module) dir = path.Join(dir, module)
@ -329,13 +329,8 @@ func InitLog(dir, module string, level Level, rotate *LogRotate, logLeftSpaceLim
if rotate.headRoom == 0 { if rotate.headRoom == 0 {
var minLogLeftSpaceLimit float64 var minLogLeftSpaceLimit float64
if float64(fs.Bavail*uint64(fs.Bsize)) < float64(fs.Blocks*uint64(fs.Bsize))*DefaultHeadRatio {
minLogLeftSpaceLimit = float64(fs.Bavail*uint64(fs.Bsize)) * DefaultHeadRatio / 1024 / 1024
} else {
minLogLeftSpaceLimit = float64(fs.Blocks*uint64(fs.Bsize)) * DefaultHeadRatio / 1024 / 1024
}
minLogLeftSpaceLimit = math.Max(minLogLeftSpaceLimit, float64(logLeftSpaceLimit)) minLogLeftSpaceLimit = float64(fs.Blocks*uint64(fs.Bsize)) * logLeftSpaceLimitRatio / 1024 / 1024
rotate.SetHeadRoomMb(int64(math.Min(minLogLeftSpaceLimit, DefaultHeadRoom))) rotate.SetHeadRoomMb(int64(math.Min(minLogLeftSpaceLimit, DefaultHeadRoom)))
} }

6
util/log/log_test.go
View File

@ -53,7 +53,7 @@ func TestLog(t *testing.T) {
return return
} }
InitLog("/tmp/cfs", "cfs", DebugLevel, nil, DefaultLogLeftSpaceLimit) InitLog("/tmp/cfs", "cfs", DebugLevel, nil, DefaultLogLeftSpaceLimitRatio)
for i := 0; i < 10; i++ { for i := 0; i < 10; i++ {
LogDebugf("[debug] current time %v.", time.Now()) LogDebugf("[debug] current time %v.", time.Now())
LogWarnf("[warn] current time %v.", time.Now()) LogWarnf("[warn] current time %v.", time.Now())
@ -107,7 +107,7 @@ func TestLogLeftSpaceLimit01(t *testing.T) {
t.Errorf("create file[%v] err[%v]", logFilePath, err) t.Errorf("create file[%v] err[%v]", logFilePath, err)
return return
} }
log, err := InitLog("/tmp/cfs", "cfs", DebugLevel, nil, DefaultLogLeftSpaceLimit) log, err := InitLog("/tmp/cfs", "cfs", DebugLevel, nil, DefaultLogLeftSpaceLimitRatio)
if err != nil { if err != nil {
t.Errorf("init log err[%v]", err) t.Errorf("init log err[%v]", err)
return return
@ -132,7 +132,7 @@ func TestLogLeftSpaceLimit02(t *testing.T) {
return return
} }
log, err := InitLog("/tmp/cfs", "cfs", DebugLevel, nil, DefaultLogLeftSpaceLimit) log, err := InitLog("/tmp/cfs", "cfs", DebugLevel, nil, DefaultLogLeftSpaceLimitRatio)
if err != nil { if err != nil {
t.Errorf("init log err[%v]", err) t.Errorf("init log err[%v]", err)
return return

4
util/log/rotate.go
View File

@ -22,8 +22,8 @@ const (
// DefaultHeadRoom The tolerance for the log space limit (in megabytes) // DefaultHeadRoom The tolerance for the log space limit (in megabytes)
DefaultHeadRoom = 50 * 1024 DefaultHeadRoom = 50 * 1024
// DefaultHeadRatio The disk reserve space ratio // DefaultHeadRatio The disk reserve space ratio
DefaultHeadRatio = 0.2 DefaultHeadRatio = 0.2
DefaultLogLeftSpaceLimit = 5 * 1024 DefaultLogLeftSpaceLimitRatio = 0.05
) )
// LogRotate A log can be rotated by the size or time. // LogRotate A log can be rotated by the size or time.