fix(flash): select availbale disk by real left space

with: #1000444307

Signed-off-by: clinx <chenlin1@oppo.com>

remotecache/flashnode/cachengine/engine.go

@@ -19,6 +19,7 @@ import (
 	"hash/crc32"
 	syslog "log"
 	"math"
+	"math/rand"
 	"os"
 	"path"
 	"path/filepath"
@@ -51,9 +52,10 @@ const (
 	DefaultCacheMaxUsedRatio = 0.99
 	DefaultEnableTmpfs       = true
 
-	LRUCacheBlockCacheType = 0
-	LRUFileHandleCacheType = 1
-	MaxEvictCountPerRound  = 100
+	LRUCacheBlockCacheType       = 0
+	LRUFileHandleCacheType       = 1
+	MaxEvictCountPerRound        = 100
+	OneGiB                 int64 = 1 << 30
 )
 
 var (
@@ -640,34 +642,45 @@ func (c *CacheEngine) PeekCacheBlock(key string) (block *CacheBlock, err error)
 
 func (c *CacheEngine) selectAvailableLruCache() (cacheItem *lruCacheItem, err error) {
 	var maxLeftSpace int64 = math.MinInt64
-	var leftSpace int64 = 0
+	lowSpaceCandidates := make([]*lruCacheItem, 0)
+	threshold := c.reservedSpace + OneGiB // reservedSpace + 1GB
 	c.lruCacheMap.Range(func(key, value interface{}) bool {
 		item := value.(*lruCacheItem)
 		if atomic.LoadInt32(&item.disk.Status) == proto.ReadWrite {
-			expectedLeftSpace := item.config.MaxAlloc - item.lruCache.GetAllocated()
 			fs := syscall.Statfs_t{}
 			if err = syscall.Statfs(item.disk.Path, &fs); err != nil {
 				log.LogErrorf("get disk(%s) stat err:%v", item.disk.Path, err)
 				return true
 			}
 			realLeftSpace := int64(fs.Bavail * uint64(fs.Bsize))
-
-			if realLeftSpace < expectedLeftSpace {
-				leftSpace = realLeftSpace
-			} else {
-				leftSpace = expectedLeftSpace
+			// Collect low-space items and continue scanning
+			if realLeftSpace < threshold {
+				lowSpaceCandidates = append(lowSpaceCandidates, item)
+				return true
 			}
-
-			if leftSpace >= maxLeftSpace {
-				maxLeftSpace = leftSpace
+			if realLeftSpace >= maxLeftSpace {
+				maxLeftSpace = realLeftSpace
 				cacheItem = item
 			}
-
 		}
 		return true
 	})
 	if cacheItem != nil {
-		log.LogInfof("select disk(%v) success", cacheItem.config.Path)
+		if log.EnableInfo() {
+			log.LogInfof("select disk(%v) success", cacheItem.config.Path)
+		}
+		return
+	}
+	if len(lowSpaceCandidates) != 0 {
+		if len(lowSpaceCandidates) == 1 {
+			cacheItem = lowSpaceCandidates[0]
+		} else {
+			idx := rand.Intn(len(lowSpaceCandidates))
+			cacheItem = lowSpaceCandidates[idx]
+		}
+		if log.EnableInfo() {
+			log.LogInfof("choose disk(%v) success from low-space candidates", cacheItem.config.Path)
+		}
 		return
 	}
 	return nil, errors.NewErrorf("no available disk can select")
@@ -1060,6 +1073,19 @@ func (c *CacheEngine) GetLruUsageRatio() float64 {
 	return 0
 }
 
+func (c *CacheEngine) GetCacheLengths() (totalLRULen int, fhLRULen int, keyToDiskLen int) {
+	c.lruCacheMap.Range(func(key, value interface{}) bool {
+		cacheItem := value.(*lruCacheItem)
+		totalLRULen += cacheItem.lruCache.Len()
+		return true
+	})
+	if c.lruFhCache != nil {
+		fhLRULen = c.lruFhCache.Len()
+	}
+	keyToDiskLen = len(c.keyToDiskMap)
+	return
+}
+
 func (c *CacheEngine) GetDiskUsageRatio() map[string]float64 {
 	result := make(map[string]float64)
 	c.lruCacheMap.Range(func(key, value interface{}) bool {

remotecache/flashnode/flashnode.go

@@ -15,6 +15,7 @@
 package flashnode
 
 import (
+	"bufio"
 	"fmt"
 	"net"
 	"os"
@@ -561,6 +562,14 @@ func (f *FlashNode) startCacheEngine() (err error) {
 		log.LogErrorf("startCacheEngine failed:%v", err)
 		return
 	}
+	stat.PrintModuleStat = func(writer *bufio.Writer) {
+		if f.cacheEngine != nil {
+			lruSum, fhLen, keyMapLen := f.cacheEngine.GetCacheLengths()
+			fmt.Fprintf(writer, "lruSum:%d fhLru:%d keyToDisk:%d\n", lruSum, fhLen, keyMapLen)
+		} else {
+			fmt.Fprintf(writer, "lruSum:%d fhLru:%d keyToDisk:%d\n", 0, 0, 0)
+		}
+	}
 	f.cacheEngine.SetReadDataNodeTimeout(proto.DefaultRemoteCacheExtentReadTimeout)
 	f.cacheEngine.StartCachePrepareWorkers(f.limitWrite, f.prepareLoadRoutineNum)
 	return f.cacheEngine.Start()

remotecache/flashnode/manual_scanner.go

@@ -275,7 +275,7 @@ func (s *ManualScanner) handleFile(dentry *proto.ScanItem) {
 	s.limiter.Wait(context.Background())
 	info, err := s.mw.InodeGet_ll(dentry.Inode)
 	if err != nil {
-		log.LogErrorf("handleFile InodeGet_ll err: %v, dentry: %+v", err, dentry)
+		log.LogWarnf("handleFile InodeGet_ll err: %v, dentry: %+v", err, dentry)
 		return
 	}
 	op := s.manualTask.Action