gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 02:00:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

enhance(log):Modify the threshold for triggering log cleanup to logLeftSpaceLimitRatio of the total disk space.

Browse Source 
@formatter:off

Signed-off-by: shuqiang-zheng <zhengshuqiang@oppo.com>
This commit is contained in:
shuqiang-zheng 2024-05-20 11:02:39 +08:00
parent 52067ffcde
commit 0f0211b28c
15 changed files with 34 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
blockcache/cmd.go
View File

@ -169,7 +169,7 @@ func main() {
level = log.ErrorLevel
}
_, err = log.InitLog(logDir, module, level, nil, log.DefaultLogLeftSpaceLimit)
_, err = log.InitLog(logDir, module, level, nil, log.DefaultLogLeftSpaceLimitRatio)
if err != nil {
err = errors.NewErrorf("Fatal: failed to init log - %v", err)
fmt.Println(err)

2
cli/cli.go
View File

@ -74,7 +74,7 @@ func setupCommands(cfg *cmd.Config) *cobra.Command {
func main() {
var err error
_, err = log.InitLog("/tmp/cfs", "cli", log.DebugLevel, nil, log.DefaultLogLeftSpaceLimit)
_, err = log.InitLog("/tmp/cfs", "cli", log.DebugLevel, nil, log.DefaultLogLeftSpaceLimitRatio)
if err != nil {
fmt.Fprintln(os.Stderr, "Error:", err)
os.Exit(1)

2
client/fuse.go
View File

@ -308,7 +308,7 @@ func main() {
// use uber automaxprocs: get real cpu number to k8s pod"
level := parseLogLevel(opt.Loglvl)
_, err = log.InitLog(opt.Logpath, opt.Volname, level, nil, log.DefaultLogLeftSpaceLimit)
_, err = log.InitLog(opt.Logpath, opt.Volname, level, nil, log.DefaultLogLeftSpaceLimitRatio)
if err != nil {
err = errors.NewErrorf("Init log dir fail: %v\n", err)
fmt.Println(err)

32
cmd/cmd.go
View File

@ -48,16 +48,16 @@ import (
)
const (
ConfigKeyRole = "role"
ConfigKeyLogDir = "logDir"
ConfigKeyLogLevel = "logLevel"
ConfigKeyLogRotateSize = "logRotateSize"
ConfigKeyLogRotateHeadRoom = "logRotateHeadRoom"
ConfigKeyProfPort = "prof"
ConfigKeyWarnLogDir = "warnLogDir"
ConfigKeyBuffersTotalLimit = "buffersTotalLimit"
ConfigKeyLogLeftSpaceLimit = "logLeftSpaceLimit"
ConfigKeyEnableLogPanicHook = "enableLogPanicHook"
ConfigKeyRole = "role"
ConfigKeyLogDir = "logDir"
ConfigKeyLogLevel = "logLevel"
ConfigKeyLogRotateSize = "logRotateSize"
ConfigKeyLogRotateHeadRoom = "logRotateHeadRoom"
ConfigKeyProfPort = "prof"
ConfigKeyWarnLogDir = "warnLogDir"
ConfigKeyBuffersTotalLimit = "buffersTotalLimit"
ConfigKeyLogLeftSpaceLimitRatio = "logLeftSpaceLimitRatio"
ConfigKeyEnableLogPanicHook = "enableLogPanicHook"
)
const (
@ -165,12 +165,12 @@ func main() {
profPort := cfg.GetString(ConfigKeyProfPort)
umpDatadir := cfg.GetString(ConfigKeyWarnLogDir)
buffersTotalLimit := cfg.GetInt64(ConfigKeyBuffersTotalLimit)
logLeftSpaceLimitStr := cfg.GetString(ConfigKeyLogLeftSpaceLimit)
logLeftSpaceLimit, err := strconv.ParseInt(logLeftSpaceLimitStr, 10, 64)
logLeftSpaceLimitRatioStr := cfg.GetString(ConfigKeyLogLeftSpaceLimitRatio)
logLeftSpaceLimitRatio, err := strconv.ParseFloat(logLeftSpaceLimitRatioStr, 64)
enableLogPanicHook := cfg.GetBool(ConfigKeyEnableLogPanicHook)
if err != nil || logLeftSpaceLimit == 0 {
log.LogErrorf("logLeftSpaceLimit is not a legal int value: %v", err.Error())
logLeftSpaceLimit = log.DefaultLogLeftSpaceLimit
if err != nil || logLeftSpaceLimitRatio <= 0 || logLeftSpaceLimitRatio > 1.0 {
log.LogErrorf("logLeftSpaceLimitRatio is not a legal float value: %v", err.Error())
logLeftSpaceLimitRatio = log.DefaultLogLeftSpaceLimitRatio
}
// Init server instance with specified role configuration.
var (
@ -231,7 +231,7 @@ func main() {
if logRotateHeadRoom > 0 {
rotate.SetHeadRoomMb(logRotateHeadRoom)
}
_, err = log.InitLog(logDir, module, level, rotate, logLeftSpaceLimit)
_, err = log.InitLog(logDir, module, level, rotate, logLeftSpaceLimitRatio)
if err != nil {
err = errors.NewErrorf("Fatal: failed to init log - %v", err)
fmt.Println(err)

2
deploy/deploy_cli.go
View File

@ -17,7 +17,7 @@ func init() {
}
func main() {
_, err := log.InitLog("/tmp/cfs", "deploy", log.DebugLevel, nil, log.DefaultLogLeftSpaceLimit)
_, err := log.InitLog("/tmp/cfs", "deploy", log.DebugLevel, nil, log.DefaultLogLeftSpaceLimitRatio)
if err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
log.LogFlush()

2
fsck/cmd/clean.go
View File

@ -105,7 +105,7 @@ func Clean(opt string) error {
ump.InitUmp("fsck", "")
_, err := log.InitLog("fscklog", "fsck", log.InfoLevel, nil, log.DefaultLogLeftSpaceLimit)
_, err := log.InitLog("fscklog", "fsck", log.InfoLevel, nil, log.DefaultLogLeftSpaceLimitRatio)
if err != nil {
return fmt.Errorf("Init log failed: %v", err)
}

2
libsdk/libsdk.go
View File

@ -1163,7 +1163,7 @@ func (c *client) start() (err error) {
c.logLevel = "WARN"
}
level := parseLogLevel(c.logLevel)
log.InitLog(c.logDir, "libcfs", level, nil, log.DefaultLogLeftSpaceLimit)
log.InitLog(c.logDir, "libcfs", level, nil, log.DefaultLogLeftSpaceLimitRatio)
stat.NewStatistic(c.logDir, "libcfs", int64(stat.DefaultStatLogSize), stat.DefaultTimeOutUs, true)
}
proto.InitBufferPool(int64(32768))

2
master/api_service_test.go
View File

@ -238,7 +238,7 @@ func createMasterServer(cfgJSON string) (server *Server, err error) {
level = log.ErrorLevel
}
if mocktest.LogOn {
if _, err = log.InitLog(logDir, "master", level, nil, log.DefaultLogLeftSpaceLimit); err != nil {
if _, err = log.InitLog(logDir, "master", level, nil, log.DefaultLogLeftSpaceLimitRatio); err != nil {
fmt.Println("Fatal: failed to start the cubefs daemon - ", err)
return
}

2
metanode/multi_ver_test.go
View File

@ -97,7 +97,7 @@ func init() {
logDir := cfg.GetString(ConfigKeyLogDir)
os.RemoveAll(logDir)
if _, err := log.InitLog(logDir, "metanode", log.DebugLevel, nil, log.DefaultLogLeftSpaceLimit); err != nil {
if _, err := log.InitLog(logDir, "metanode", log.DebugLevel, nil, log.DefaultLogLeftSpaceLimitRatio); err != nil {
fmt.Println("Fatal: failed to start the cubefs daemon - ", err)
return
}

2
metanode/transaction_test.go
View File

@ -46,7 +46,7 @@ const (
)
func init() {
log.InitLog("/tmp/cfs/logs/", "test", log.DebugLevel, nil, log.DefaultLogLeftSpaceLimit)
log.InitLog("/tmp/cfs/logs/", "test", log.DebugLevel, nil, log.DefaultLogLeftSpaceLimitRatio)
}
func newMetaPartition(PartitionId uint64, manager *metadataManager) (mp *metaPartition) {

2
preload/sdk/preloadsdk.go
View File

@ -89,7 +89,7 @@ func NewClient(config PreloadConfig) *PreLoadClient {
}
if config.LogDir != "" {
log.InitLog(config.LogDir, "preload", convertLogLevel(config.LogLevel), nil, log.DefaultLogLeftSpaceLimit)
log.InitLog(config.LogDir, "preload", convertLogLevel(config.LogLevel), nil, log.DefaultLogLeftSpaceLimitRatio)
stat.NewStatistic(config.LogDir, "preload", int64(stat.DefaultStatLogSize), stat.DefaultTimeOutUs, true)
}

2
snapshot/cmd/clean.go
View File

@ -119,7 +119,7 @@ func Clean(opt string, args []string) error {
ump.InitUmp("snapshot", "")
_, err := log.InitLog("snapshotlog", "snapshot", log.DebugLevel, nil, log.DefaultLogLeftSpaceLimit)
_, err := log.InitLog("snapshotlog", "snapshot", log.DebugLevel, nil, log.DefaultLogLeftSpaceLimitRatio)
if err != nil {
return fmt.Errorf("Init log failed: %v", err)
}

9
util/log/log.go
View File

@ -303,7 +303,7 @@ func (l *Log) outputStderr(calldepth int, s string) {
}
// InitLog initializes the log.
func InitLog(dir, module string, level Level, rotate *LogRotate, logLeftSpaceLimit int64) (*Log, error) {
func InitLog(dir, module string, level Level, rotate *LogRotate, logLeftSpaceLimitRatio float64) (*Log, error) {
l := new(Log)
l.printStderr = 1
dir = path.Join(dir, module)
@ -330,13 +330,8 @@ func InitLog(dir, module string, level Level, rotate *LogRotate, logLeftSpaceLim
if rotate.headRoom == 0 {
var minLogLeftSpaceLimit float64
if float64(fs.Bavail*uint64(fs.Bsize)) < float64(fs.Blocks*uint64(fs.Bsize))*DefaultHeadRatio {
minLogLeftSpaceLimit = float64(fs.Bavail*uint64(fs.Bsize)) * DefaultHeadRatio / 1024 / 1024
} else {
minLogLeftSpaceLimit = float64(fs.Blocks*uint64(fs.Bsize)) * DefaultHeadRatio / 1024 / 1024
}
minLogLeftSpaceLimit = math.Max(minLogLeftSpaceLimit, float64(logLeftSpaceLimit))
minLogLeftSpaceLimit = float64(fs.Blocks*uint64(fs.Bsize)) * logLeftSpaceLimitRatio / 1024 / 1024
rotate.SetHeadRoomMb(int64(math.Min(minLogLeftSpaceLimit, DefaultHeadRoom)))
}

6
util/log/log_test.go
View File

@ -53,7 +53,7 @@ func TestLog(t *testing.T) {
return
}
InitLog("/tmp/cfs", "cfs", DebugLevel, nil, DefaultLogLeftSpaceLimit)
InitLog("/tmp/cfs", "cfs", DebugLevel, nil, DefaultLogLeftSpaceLimitRatio)
for i := 0; i < 10; i++ {
LogDebugf("[debug] current time %v.", time.Now())
LogWarnf("[warn] current time %v.", time.Now())
@ -107,7 +107,7 @@ func TestLogLeftSpaceLimit01(t *testing.T) {
t.Errorf("create file[%v] err[%v]", logFilePath, err)
return
}
log, err := InitLog("/tmp/cfs", "cfs", DebugLevel, nil, DefaultLogLeftSpaceLimit)
log, err := InitLog("/tmp/cfs", "cfs", DebugLevel, nil, DefaultLogLeftSpaceLimitRatio)
if err != nil {
t.Errorf("init log err[%v]", err)
return
@ -132,7 +132,7 @@ func TestLogLeftSpaceLimit02(t *testing.T) {
return
}
log, err := InitLog("/tmp/cfs", "cfs", DebugLevel, nil, DefaultLogLeftSpaceLimit)
log, err := InitLog("/tmp/cfs", "cfs", DebugLevel, nil, DefaultLogLeftSpaceLimitRatio)
if err != nil {
t.Errorf("init log err[%v]", err)
return

4
util/log/rotate.go
View File

@ -22,8 +22,8 @@ const (
// DefaultHeadRoom The tolerance for the log space limit (in megabytes)
DefaultHeadRoom = 50 * 1024
// DefaultHeadRatio The disk reserve space ratio
DefaultHeadRatio = 0.2
DefaultLogLeftSpaceLimit = 5 * 1024
DefaultHeadRatio = 0.2
DefaultLogLeftSpaceLimitRatio = 0.05
)
// LogRotate A log can be rotated by the size or time.