diff --git a/datanode/disk.go b/datanode/disk.go index 4b569a5ad..7768b6f23 100644 --- a/datanode/disk.go +++ b/datanode/disk.go @@ -616,6 +616,10 @@ func (d *Disk) triggerDiskError(rwFlag uint8, dpId uint64) { // exporter.Warning(mesg) log.LogWarnf(mesg) + if d.HasDiskErrPartition(dpId) { + return + } + if rwFlag == WriteFlag { d.incWriteErrCnt() } else if rwFlag == ReadFlag { @@ -628,7 +632,7 @@ func (d *Disk) triggerDiskError(rwFlag uint8, dpId uint64) { d.AddDiskErrPartition(dpId) diskErrCnt := d.getTotalErrCnt() diskErrPartitionCnt := d.GetDiskErrPartitionCount() - if diskErrPartitionCnt >= d.dataNode.diskUnavailablePartitionErrorCount { + if diskErrCnt >= d.dataNode.diskUnavailableErrorCount || diskErrPartitionCnt >= d.dataNode.diskUnavailablePartitionErrorCount { msg := fmt.Sprintf("set disk unavailable for too many disk error, "+ "disk path(%v), ip(%v), diskErrCnt(%v), diskErrPartitionCnt(%v) threshold(%v)", d.Path, LocalIP, diskErrCnt, diskErrPartitionCnt, d.dataNode.diskUnavailablePartitionErrorCount) diff --git a/datanode/server.go b/datanode/server.go index 20a19a30a..ceea48fd7 100644 --- a/datanode/server.go +++ b/datanode/server.go @@ -145,6 +145,7 @@ const ( ConfigGcRecyclePercent = "gcRecyclePercent" // disk status becomes unavailable if disk error partition count reaches this value + ConfigKeyDiskUnavailableErrorCount = "diskUnavailableErrorCount" ConfigKeyDiskUnavailablePartitionErrorCount = "diskUnavailablePartitionErrorCount" ConfigKeyCacheCap = "cacheCap" ConfigExtentCacheTtlByMin = "extentCacheTtlByMin" @@ -234,6 +235,7 @@ type DataNode struct { gcRecyclePercent float64 gcTimer *util.RecycleTimer + diskUnavailableErrorCount uint64 // disk status becomes unavailable when disk error count reaches this value diskUnavailablePartitionErrorCount uint64 // disk status becomes unavailable when disk error partition count reaches this value started int32 dpBackupTimeout time.Duration @@ -472,6 +474,15 @@ func (s *DataNode) parseConfig(cfg *config.Config) (err error) { s.gcRecyclePercent = defaultGcRecyclePercent } + diskUnavailableErrorCount := cfg.GetInt64(ConfigKeyDiskUnavailableErrorCount) + if diskUnavailableErrorCount <= 0 || diskUnavailableErrorCount > 100 { + diskUnavailableErrorCount = DefaultDiskUnavailableErrorCount + log.LogDebugf("action[parseConfig] ConfigKeyDiskUnavailableErrorCount(%v) out of range, set as default(%v)", + diskUnavailableErrorCount, DefaultDiskUnavailableErrorCount) + } + s.diskUnavailableErrorCount = uint64(diskUnavailableErrorCount) + log.LogDebugf("action[parseConfig] load diskUnavailableErrorCount(%v)", s.diskUnavailableErrorCount) + diskUnavailablePartitionErrorCount := cfg.GetInt64(ConfigKeyDiskUnavailablePartitionErrorCount) if diskUnavailablePartitionErrorCount <= 0 || diskUnavailablePartitionErrorCount > 100 { diskUnavailablePartitionErrorCount = DefaultDiskUnavailablePartitionErrorCount