diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b24db831d..27f0e0baf 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -471,6 +471,7 @@ services: environment: - TZ=Asia/Shanghai - DOCKER_TESTING_LOG_OFF="on" + - DOCKER_FLASHNODE_TMPFS_OFF="on" - http_proxy= - https_proxy= - HTTP_PROXY= diff --git a/flashnode/cachengine/block_test.go b/flashnode/cachengine/block_test.go index 734b06d0a..c322ce990 100644 --- a/flashnode/cachengine/block_test.go +++ b/flashnode/cachengine/block_test.go @@ -29,12 +29,16 @@ import ( "github.com/stretchr/testify/require" ) -func initTestTmpfs(size int64) (err error) { +func initTestTmpfs() (umount func() error, err error) { + os.MkdirAll(testTmpFS, 0o777) + if !enabledTmpfs() { + return func() error { return os.RemoveAll(testTmpFS) }, nil + } _, err = os.Stat(testTmpFS) if err == nil { if tmpfs.IsTmpfs(testTmpFS) { if err = tmpfs.Umount(testTmpFS); err != nil { - return err + return } } } else { @@ -43,13 +47,17 @@ func initTestTmpfs(size int64) (err error) { } _ = os.MkdirAll(testTmpFS, 0o777) } - err = tmpfs.MountTmpfs(testTmpFS, size) - return + if err = tmpfs.MountTmpfs(testTmpFS, 200*util.MB); err != nil { + return + } + return func() error { return tmpfs.Umount(testTmpFS) }, nil } func TestBlockWriteCache(t *testing.T) { - require.NoError(t, initTestTmpfs(200*util.MB)) - defer func() { require.NoError(t, tmpfs.Umount(testTmpFS)) }() + umount, err := initTestTmpfs() + require.NoError(t, err) + defer func() { require.NoError(t, umount()) }() + testWriteSingleFile(t) testWriteSingleFileError(t) testWriteCacheBlockFull(t) @@ -139,8 +147,9 @@ func testWriteMultiCacheBlock(t *testing.T, newMultiCacheFunc func(volume string } func TestBlockReadCache(t *testing.T) { - require.NoError(t, initTestTmpfs(200*util.MB)) - defer func() { require.NoError(t, tmpfs.Umount(testTmpFS)) }() + umount, err := initTestTmpfs() + require.NoError(t, err) + defer func() { require.NoError(t, umount()) }() cacheBlock := NewCacheBlock(testTmpFS, t.Name(), 1, 1024, 2568748711, proto.CACHE_BLOCK_SIZE, nil) require.NoError(t, cacheBlock.initFilePath()) @@ -151,7 +160,7 @@ func TestBlockReadCache(t *testing.T) { require.NoError(t, cacheBlock.WriteAt(bytes, offset, 1024)) cacheBlock.notifyReady() bytesRead := make([]byte, 1024) - _, err := cacheBlock.Read(context.Background(), bytesRead, offset, 1024) + _, err = cacheBlock.Read(context.Background(), bytesRead, offset, 1024) require.NoError(t, err) require.Equal(t, bytesRead, bytes) } @@ -163,8 +172,9 @@ func TestParallelOperation(t *testing.T) { } func testParallelOperation(t *testing.T) { - require.NoError(t, initTestTmpfs(200*util.MB)) - defer func() { require.NoError(t, tmpfs.Umount(testTmpFS)) }() + umount, err := initTestTmpfs() + require.NoError(t, err) + defer func() { require.NoError(t, umount()) }() cacheBlock := NewCacheBlock(testTmpFS, t.Name(), 1, 1024, 112456871, proto.CACHE_BLOCK_SIZE, nil) require.NoError(t, cacheBlock.initFilePath()) diff --git a/flashnode/cachengine/engine.go b/flashnode/cachengine/engine.go index 3823d6ed4..3f0ab6662 100644 --- a/flashnode/cachengine/engine.go +++ b/flashnode/cachengine/engine.go @@ -31,13 +31,9 @@ import ( "github.com/cubefs/cubefs/util/tmpfs" ) -//TODO: remove this later. -//go:generate gofumpt -l -w . -//go:generate git diff --exit-code -//go:generate golangci-lint run --issues-exit-code=1 -D errcheck -E bodyclose ./... - const ( prepareWorkers = 20 + EnvDockerTmpfs = "DOCKER_FLASHNODE_TMPFS_OFF" DefaultExpireTime = 60 * 60 InitFileName = "flash.init" @@ -70,6 +66,8 @@ type CacheEngine struct { closeOnce sync.Once closeCh chan struct{} + + enableTmpfs bool // for testing in docker } type ( @@ -82,6 +80,7 @@ func NewCacheEngine(dataDir string, totalSize int64, maxUseRatio float64, ) (s *CacheEngine, err error) { s = new(CacheEngine) s.dataPath = dataDir + s.enableTmpfs = enabledTmpfs() if maxUseRatio < 1e-1 { maxUseRatio = DefaultCacheMaxUsedRatio } @@ -96,8 +95,10 @@ func NewCacheEngine(dataDir string, totalSize int64, maxUseRatio float64, if err = os.MkdirAll(dataDir, 0o755); err != nil { return nil, fmt.Errorf("NewCacheEngine [%v] err[%v]", dataDir, err) } - if err = s.doMount(); err != nil { - return + if s.enableTmpfs { + if err = s.doMount(); err != nil { + return + } } s.cachePrepareTaskCh = make(chan cachePrepareTask, 1024) @@ -123,6 +124,10 @@ func (c *CacheEngine) Stop() (err error) { if err = c.lruCache.Close(); err != nil { return err } + if !c.enableTmpfs { + log.LogInfof("CacheEngine stopped, remove tmpfs dir: %s", c.dataPath) + return os.RemoveAll(c.dataPath) + } time.Sleep(time.Second) log.LogInfof("CacheEngine stopped, umount tmpfs: %v", c.dataPath) return tmpfs.Umount(c.dataPath) @@ -324,3 +329,7 @@ func GenCacheBlockKey(volume string, inode, offset uint64, version uint32) strin u := strconv.FormatUint return path.Join(volume, u(inode, 10)+"#"+u(offset, 10)+"#"+u(uint64(version), 10)) } + +func enabledTmpfs() bool { + return os.Getenv(EnvDockerTmpfs) == "" +} diff --git a/flashnode/cachengine/engine_test.go b/flashnode/cachengine/engine_test.go index 883d96b78..2d143f75d 100644 --- a/flashnode/cachengine/engine_test.go +++ b/flashnode/cachengine/engine_test.go @@ -52,6 +52,9 @@ func TestEngineNew(t *testing.T) { } func TestEngineOverFlow(t *testing.T) { + if !enabledTmpfs() { + t.Skip("disabled tmpfs") + } ce, err := NewCacheEngine(testTmpFS, util.GB, 1.1, 1024, DefaultExpireTime, nil) require.NoError(t, err) defer func() { require.NoError(t, ce.Stop()) }()