diff --git a/client/fs/super.go b/client/fs/super.go index 8377d7647..aaf2276b1 100644 --- a/client/fs/super.go +++ b/client/fs/super.go @@ -230,6 +230,7 @@ func NewSuper(opt *proto.MountOptions) (s *Super, err error) { BcacheDir: opt.BcacheDir, MaxStreamerLimit: opt.MaxStreamerLimit, VerReadSeq: opt.VerReadSeq, + MetaWrapper: s.mw, OnAppendExtentKey: s.mw.AppendExtentKey, OnSplitExtentKey: s.mw.SplitExtentKey, OnGetExtents: s.mw.GetExtents, diff --git a/cmd/common/args.go b/cmd/common/args.go index 6e87e93da..c9ea44f2f 100644 --- a/cmd/common/args.go +++ b/cmd/common/args.go @@ -25,10 +25,12 @@ type ( func (b *Bool) Key(key string) *Argument { return NewArgument(key, &b.V) } func (b *Bool) Enable() *Argument { return b.Key("enable") } func (b *Bool) Status() *Argument { return b.Key("status") } +func (b *Bool) All() *Argument { return b.Key("all") } func (i *Int) Key(key string) *Argument { return NewArgument(key, &i.V) } func (i *Int) ID() *Argument { return i.Key("id") } func (i *Int) ExtentID() *Argument { return i.Key("extentID") } +func (i *Int) Count() *Argument { return i.Key("count") } func (u *Uint) Key(key string) *Argument { return NewArgument(key, &u.V) } func (u *Uint) ID() *Argument { return u.Key("id") } @@ -42,3 +44,5 @@ func (f *Float) Key(key string) *Argument { return NewArgument(key, &f.V) } func (s *String) Key(key string) *Argument { return NewArgument(key, &s.V) } func (s *String) Disk() *Argument { return s.Key("disk") } func (s *String) DiskPath() *Argument { return s.Key("diskPath") } +func (s *String) Addr() *Argument { return s.Key("addr") } +func (s *String) ZoneName() *Argument { return s.Key("zoneName") } diff --git a/cmd/common/argument.go b/cmd/common/argument.go index b20cc6cf8..4fc6913d8 100644 --- a/cmd/common/argument.go +++ b/cmd/common/argument.go @@ -67,6 +67,11 @@ func (a *Argument) OnValue(f func() error) *Argument { return a } +// Key returns key of the argument. +func (a *Argument) Key() string { + return a.key +} + func ParseArguments(r *http.Request, args ...*Argument) error { if err := r.ParseForm(); err != nil { return err diff --git a/cmd/common/argument_test.go b/cmd/common/argument_test.go index 5580cc7fa..46f2ca93a 100644 --- a/cmd/common/argument_test.go +++ b/cmd/common/argument_test.go @@ -112,3 +112,31 @@ func TestCmdCommonArgs(t *testing.T) { require.Equal(t, "11111", s.V) } } + +func TestCmdCommonKeys(t *testing.T) { + b := new(Bool) + require.Equal(t, "enable", b.Enable().Key()) + require.Equal(t, "status", b.Status().Key()) + require.Equal(t, "all", b.All().Key()) + + i := new(Int) + require.Equal(t, "id", i.ID().Key()) + require.Equal(t, "extentID", i.ExtentID().Key()) + require.Equal(t, "count", i.Count().Key()) + + u := new(Uint) + require.Equal(t, "id", u.ID().Key()) + require.Equal(t, "pid", u.PID().Key()) + require.Equal(t, "partitionID", u.PartitionID().Key()) + require.Equal(t, "ino", u.Ino().Key()) + require.Equal(t, "parentIno", u.ParentIno().Key()) + + f := new(Float) + require.Equal(t, "fff", f.Key("fff").Key()) + + s := new(String) + require.Equal(t, "disk", s.Disk().Key()) + require.Equal(t, "diskPath", s.DiskPath().Key()) + require.Equal(t, "addr", s.Addr().Key()) + require.Equal(t, "zoneName", s.ZoneName().Key()) +} diff --git a/master/const.go b/master/const.go index d025126e4..ea30ca55c 100644 --- a/master/const.go +++ b/master/const.go @@ -287,6 +287,7 @@ const ( opSyncDeleteVolUser uint32 = 0x1D opSyncUpdateVolUser uint32 = 0x1E opSyncNodeSetGrp uint32 = 0x1F + opSyncDataPartitionsView uint32 = 0x20 opSyncExclueDomain uint32 = 0x23 opSyncUpdateZone uint32 = 0x24 opSyncAllocClientID uint32 = 0x25 @@ -299,6 +300,7 @@ const ( opSyncAddLcNode uint32 = 0x30 opSyncDeleteLcNode uint32 = 0x31 + opSyncUpdateLcNode uint32 = 0x32 opSyncAddLcConf uint32 = 0x33 opSyncDeleteLcConf uint32 = 0x34 opSyncUpdateLcConf uint32 = 0x35 diff --git a/proto/admin_proto.go b/proto/admin_proto.go index cffa9080a..03e96d160 100644 --- a/proto/admin_proto.go +++ b/proto/admin_proto.go @@ -1274,6 +1274,13 @@ type SimpleVolView struct { CacheDpStorageClass uint32 ForbidWriteOpOfProtoVer0 bool QuotaOfStorageClass []*StatOfStorageClass + + RemoteCacheBoostEnable bool + RemoteCacheBoostPath string + RemoteCacheAutoPrepare bool + RemoteCacheTTL int64 + RemoteCacheReadTimeoutMs int64 + EnableRemoveDupReq bool // TODO: using it in metanode } type NodeSetInfo struct { diff --git a/sdk/data/stream/extent_client.go b/sdk/data/stream/extent_client.go index b88e55331..6b92db10a 100644 --- a/sdk/data/stream/extent_client.go +++ b/sdk/data/stream/extent_client.go @@ -18,6 +18,7 @@ import ( "container/list" "context" "fmt" + "runtime/debug" "strings" "sync" "sync/atomic" @@ -30,6 +31,7 @@ import ( "github.com/cubefs/cubefs/sdk/data/wrapper" "github.com/cubefs/cubefs/sdk/meta" "github.com/cubefs/cubefs/util" + "github.com/cubefs/cubefs/util/bloom" "github.com/cubefs/cubefs/util/errors" "github.com/cubefs/cubefs/util/exporter" "github.com/cubefs/cubefs/util/log" @@ -42,6 +44,11 @@ var reqChanSize = defaultChanSize const defaultChanSize = 64 +const ( + PrepareWorkerNum = 5 + PrepareReqChanCap = 1024 +) + type ( SplitExtentKeyFunc func(parentInode, inode uint64, key proto.ExtentKey, storageClass uint32) error AppendExtentKeyFunc func(parentInode, inode uint64, key proto.ExtentKey, discard []proto.ExtentKey, isCache bool, storageClass uint32, isMigration bool) (int, error) @@ -124,6 +131,7 @@ type ExtentConfig struct { BcacheDir string MaxStreamerLimit int64 VerReadSeq uint64 + MetaWrapper *meta.MetaWrapper OnAppendExtentKey AppendExtentKeyFunc OnSplitExtentKey SplitExtentKeyFunc OnGetExtents GetExtentsFunc @@ -178,6 +186,7 @@ type ExtentClient struct { preload bool LimitManager *manager.LimitManager dataWrapper *wrapper.Wrapper + metaWrapper *meta.MetaWrapper appendExtentKey AppendExtentKeyFunc splitExtentKey SplitExtentKeyFunc getExtents GetExtentsFunc @@ -197,6 +206,20 @@ type ExtentClient struct { bcacheOnlyForNotSSD bool InnerReq bool AheadRead *AheadReadCache + + extentConfig *ExtentConfig + OldCacheBoostStatus bool + EnableClusterCacheBoost bool + EnableVolCacheBoost bool + EnableCacheAutoPrepare bool + CacheBoostPath string + CacheTTL int64 + CacheReadTimeoutMs int64 + RemoteCache *RemoteCache + prepareRemoteCacheCh chan *PrepareRemoteCacheRequest + stopOnce sync.Once + stopCh chan struct{} + wg sync.WaitGroup } func (client *ExtentClient) UidIsLimited(uid uint32) bool { @@ -370,6 +393,19 @@ retry: log.LogInfof("max streamer limit %d", client.maxStreamerLimit) client.streamerList = list.New() + if client.metaWrapper != nil { + client.metaWrapper.RemoteCacheBloom = client.RemoteCacheBloom + } + + client.extentConfig = config + if client.IsCacheBoostEnabled() { + client.InitRemoteCache() + } + client.prepareRemoteCacheCh = make(chan *PrepareRemoteCacheRequest, PrepareReqChanCap) + client.stopCh = make(chan struct{}) + client.wg.Add(1) + go client.DoPrepare() + go client.backgroundEvictStream() return @@ -394,6 +430,84 @@ func (client *ExtentClient) SetClientID(id uint64) (err error) { return } +func (client *ExtentClient) setClusterBoostEnable(enableBoost bool) { + client.OldCacheBoostStatus = client.IsCacheBoostEnabled() + + oldEnableBoost := client.EnableClusterCacheBoost + client.EnableClusterCacheBoost = enableBoost + if oldEnableBoost != enableBoost { + log.LogInfof("setClusterBoostEnable: from old(%v) to new(%v)", oldEnableBoost, enableBoost) + } +} + +func (client *ExtentClient) IsCacheBoostEnabled() bool { + return client.EnableClusterCacheBoost && client.EnableVolCacheBoost +} + +func (client *ExtentClient) InitRemoteCache() (err error) { + cacheConfig := &CacheConfig{ + Cluster: client.dataWrapper.ClusterName, + Volume: client.extentConfig.Volume, + Masters: client.extentConfig.Masters, + MW: client.metaWrapper, + readTimeoutSec: client.CacheReadTimeoutMs / 1000, + } + if client.RemoteCache, err = NewRemoteCache(cacheConfig); err != nil { + return + } + if !client.RemoteCache.ResetCacheBoostPathToBloom(client.CacheBoostPath) { + client.CacheBoostPath = "" + } + return +} + +func (client *ExtentClient) UpdateRemoteCacheConfig(view *proto.SimpleVolView) { + if client.EnableVolCacheBoost != view.RemoteCacheBoostEnable { + log.LogInfof("updateRemoteCacheConfig: RemoteCacheBoostEnable from old(%v) to new(%v)", client.EnableVolCacheBoost, view.RemoteCacheBoostEnable) + client.EnableVolCacheBoost = view.RemoteCacheBoostEnable + } + if client.OldCacheBoostStatus != client.IsCacheBoostEnabled() { + log.LogInfof("updateRemoteCacheConfig: enable from old(%v) to new(%v)", client.OldCacheBoostStatus, client.IsCacheBoostEnabled()) + } + // RemoteCache may be nil if the first initialization failed, it will not be set nil anymore even if remote cache is disabled + if client.IsCacheBoostEnabled() { + if !client.OldCacheBoostStatus || client.RemoteCache == nil { + log.LogInfof("updateRemoteCacheConfig: initRemoteCache: enable(%v -> %v) RemoteCache isNil(%v)", client.OldCacheBoostStatus, client.IsCacheBoostEnabled(), client.RemoteCache == nil) + if err := client.InitRemoteCache(); err != nil { + log.LogErrorf("updateRemoteCacheConfig: initRemoteCache failed, err: %v", err) + } + } + } else if client.OldCacheBoostStatus && client.RemoteCache != nil { + client.RemoteCache.Stop() + log.LogInfof("updateRemoteCacheConfig: stop RemoteCache") + } + + if client.CacheBoostPath != view.RemoteCacheBoostPath { + oldBoostPath := client.CacheBoostPath + client.CacheBoostPath = view.RemoteCacheBoostPath + if client.IsCacheBoostEnabled() && client.RemoteCache != nil { + if !client.RemoteCache.ResetCacheBoostPathToBloom(view.RemoteCacheBoostPath) { + client.CacheBoostPath = "" + } + } + log.LogInfof("updateRemoteCacheConfig: RemoteCacheBoostPath from old(%v) to want(%v), but(%v)", oldBoostPath, view.RemoteCacheBoostPath, client.CacheBoostPath) + } + + if client.EnableCacheAutoPrepare != view.RemoteCacheAutoPrepare { + log.LogInfof("updateRemoteCacheConfig: RemoteCacheAutoPrepare from old(%v) to new(%v)", client.EnableCacheAutoPrepare, view.RemoteCacheAutoPrepare) + client.EnableCacheAutoPrepare = view.RemoteCacheAutoPrepare + } + + if client.CacheTTL != view.RemoteCacheTTL { + log.LogInfof("updateRemoteCacheConfig: RemoteCacheTTL from old(%d) to new(%d)", client.CacheTTL, view.RemoteCacheTTL) + client.CacheTTL = view.RemoteCacheTTL + } + + if client.CacheReadTimeoutMs != view.RemoteCacheReadTimeoutMs { + client.CacheReadTimeoutMs = view.RemoteCacheReadTimeoutMs + } +} + func (client *ExtentClient) GetVolumeName() string { return client.volumeName } @@ -877,6 +991,8 @@ func setRate(lim *rate.Limiter, val int) string { func (client *ExtentClient) Close() error { // release streamers + client.stopOnce.Do(func() { close(client.stopCh) }) + client.wg.Wait() var inodes []uint64 client.streamerLock.Lock() inodes = make([]uint64, 0, len(client.streamers)) @@ -917,3 +1033,54 @@ func (client *ExtentClient) IsPreloadMode() bool { func (client *ExtentClient) UploadFlowInfo(clientInfo wrapper.SimpleClientInfo) (bWork bool, err error) { return client.dataWrapper.UploadFlowInfo(clientInfo, false) } + +func (c *ExtentClient) RemoteCacheBloom() *bloom.BloomFilter { + if c.RemoteCache != nil { + return c.RemoteCache.GetRemoteCacheBloom() + } + return nil +} + +func (c *ExtentClient) GetInodeBloomStatus(ino uint64) bool { + cacheBloom := c.RemoteCacheBloom() + if cacheBloom == nil { + return false + } + cacheBloom.AddUint64(ino) + return true +} + +func (c *ExtentClient) DoPrepare() { + defer c.wg.Done() + + workerWg := sync.WaitGroup{} + for i := 0; i < PrepareWorkerNum; i++ { + workerWg.Add(1) + go func() { + defer workerWg.Done() + for { + select { + case <-c.stopCh: + return + case req := <-c.prepareRemoteCacheCh: + c.servePrepareRequest(req) + } + } + }() + } + workerWg.Wait() +} + +func (c *ExtentClient) servePrepareRequest(prepareReq *PrepareRemoteCacheRequest) { + defer func() { + if err := recover(); err != nil { + log.LogWarnf("servePrepareRequest: panic occurs, stack(%v)", string(debug.Stack())) + } + }() + s := c.GetStreamer(prepareReq.inode) + if s == nil { + log.LogWarnf("servePrepareRequest: streamer is nil, prepare request: %v)", prepareReq) + return + } + s.prepareRemoteCache(prepareReq.ctx, prepareReq.ek) +} diff --git a/sdk/data/stream/extent_flash_cache.go b/sdk/data/stream/extent_flash_cache.go index 5276e7577..87a860e18 100644 --- a/sdk/data/stream/extent_flash_cache.go +++ b/sdk/data/stream/extent_flash_cache.go @@ -72,7 +72,7 @@ type CacheConfig struct { MW *meta.MetaWrapper SameZoneWeight int - ReadTimeoutSec int + readTimeoutSec int64 } type RemoteCache struct { @@ -105,7 +105,7 @@ func NewRemoteCache(config *CacheConfig) (*RemoteCache, error) { } else { rc.sameZoneWeight = config.SameZoneWeight } - rc.readTimeoutSec = config.ReadTimeoutSec + rc.readTimeoutSec = int(config.readTimeoutSec) if rc.readTimeoutSec <= 0 { rc.readTimeoutSec = _connReadTimeoutSec } diff --git a/sdk/data/stream/extent_handler.go b/sdk/data/stream/extent_handler.go index 7d9760e34..4c561e03d 100644 --- a/sdk/data/stream/extent_handler.go +++ b/sdk/data/stream/extent_handler.go @@ -15,6 +15,7 @@ package stream import ( + "context" "fmt" "math" "math/rand" @@ -609,6 +610,16 @@ func (eh *ExtentHandler) appendExtentKey() (err error) { */ _ = eh.stream.extents.Append(eh.key, false) } + + if eh.key.PartitionId > 0 && eh.stream.enableCacheAutoPrepare() { + prepareReq := &PrepareRemoteCacheRequest{ + ctx: context.Background(), + ek: eh.key, + inode: eh.stream.inode, + } + eh.stream.sendToPrepareRomoteCacheChan(prepareReq) + // eh.stream.prepareRemoteCache(ctx, ek) + } } if err == nil { eh.dirty = false diff --git a/sdk/data/stream/stream_reader.go b/sdk/data/stream/stream_reader.go index b321b43d9..f3feeca15 100644 --- a/sdk/data/stream/stream_reader.go +++ b/sdk/data/stream/stream_reader.go @@ -56,10 +56,10 @@ type Streamer struct { needUpdateVer int32 isCache bool openForWrite bool - - rdonly bool - aheadReadEnable bool - aheadReadWindow *AheadReadWindow + rdonly bool + aheadReadEnable bool + bloomStatus bool + aheadReadWindow *AheadReadWindow } type bcacheKey struct { @@ -254,6 +254,17 @@ func (s *Streamer) read(data []byte, offset int, size int, storageClass uint32) log.LogDebugf("Streamer not read from bcache, ino(%v) storageClass(%v) s.client.bcacheEnable(%v) bcacheOnlyForNotSSD(%v)", s.inode, proto.StorageClassString(inodeInfo.StorageClass), s.client.bcacheEnable, s.client.bcacheOnlyForNotSSD) } + log.LogDebugf("TRACE Stream read. miss blockCache cacheKey(%v) loadBcache(%v)", cacheKey, s.client.loadBcache) + } else if s.enableRemoteCache() { + var cacheReadRequests []*CacheReadRequest + cacheReadRequests, err = s.prepareCacheRequests(uint64(offset), uint64(size), data) + if err == nil { + var read int + if read, err = s.readFromRemoteCache(ctx, uint64(offset), uint64(size), cacheReadRequests); err == nil { + return read, err + } + } + log.LogWarnf("Stream read: readFromRemoteCache failed: ino(%v) offset(%v) size(%v), err(%v)", s.inode, offset, size, err) } if s.needBCache { diff --git a/sdk/data/stream/stream_remote_cache.go b/sdk/data/stream/stream_remote_cache.go new file mode 100644 index 000000000..92aaf7190 --- /dev/null +++ b/sdk/data/stream/stream_remote_cache.go @@ -0,0 +1,235 @@ +// Copyright 2023 The CubeFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package stream + +import ( + "context" + "fmt" + + "github.com/cubefs/cubefs/proto" + "github.com/cubefs/cubefs/util/log" + "github.com/cubefs/cubefs/util/stat" +) + +type PrepareRemoteCacheRequest struct { + ctx context.Context + inode uint64 + ek *proto.ExtentKey +} + +func (pr *PrepareRemoteCacheRequest) String() string { + if pr == nil { + return "" + } + return fmt.Sprintf("PrepareRemoteCacheRequest{ino: %v, ek: %v}", pr.inode, pr.ek) +} + +func (s *Streamer) enableRemoteCache() bool { + if !s.client.IsCacheBoostEnabled() || s.client.RemoteCache == nil { + return false + } + return s.bloomStatus +} + +func (s *Streamer) enableCacheAutoPrepare() bool { + if !s.client.EnableCacheAutoPrepare { + return false + } + return s.enableRemoteCache() +} + +func (s *Streamer) sendToPrepareRomoteCacheChan(req *PrepareRemoteCacheRequest) { + select { + case s.client.prepareRemoteCacheCh <- req: + default: + log.LogWarnf("sendToPrepareRomoteCacheChan: chan is full, discard req(%v)", req) + } +} + +func (s *Streamer) prepareRemoteCache(ctx context.Context, ek *proto.ExtentKey) { + cReadRequests, err := s.prepareCacheRequests(ek.FileOffset, uint64(ek.Size), nil) + if err != nil { + log.LogWarnf("Streamer prepareRemoteCache: prepareCacheRequests failed. start(%v), size(%v), err(%v)", ek.FileOffset, ek.Size, err) + return + } + + for _, req := range cReadRequests { + fg := s.getFlashGroup(req.CacheRequest.FixedFileOffset) + if fg == nil { + err = fmt.Errorf("cannot find any flashGroups") + log.LogWarnf("Streamer prepareRemoteCache failed: %v", err) + break + } + prepareReq := &proto.CachePrepareRequest{ + CacheRequest: req.CacheRequest, + FlashNodes: fg.Hosts, + } + + if err = s.client.RemoteCache.Prepare(ctx, fg, s.inode, prepareReq); err != nil { + log.LogWarnf("Streamer prepareRemoteCache: flashGroup prepare failed. fg(%v) req(%v) err(%v)", fg, prepareReq, err) + } + } + + log.LogDebugf("prepareRemoteCache: inode(%d), err(%v)", s.inode, err) +} + +func (s *Streamer) readFromRemoteCache(ctx context.Context, offset, size uint64, cReadRequests []*CacheReadRequest) (total int, err error) { + bgTime := stat.BeginStat() + defer func() { + stat.EndStat("remote-cache-read", err, bgTime, 1) + }() + + var read int + for _, req := range cReadRequests { + if len(req.CacheRequest.Sources) == 0 { + total += int(req.Size_) + continue + } + fg := s.getFlashGroup(req.CacheRequest.FixedFileOffset) + if fg == nil { + err = fmt.Errorf("readFromRemoteCache failed: Cannot find any flashGroups") + return + } + + if read, err = s.client.RemoteCache.Read(ctx, fg, s.inode, req); err != nil { + log.LogWarnf("readFromRemoteCache: flashGroup read failed. offset(%v) size(%v) fg(%v) req(%v) err(%v)", offset, size, fg, req, err) + return + } else { + total += read + } + } + log.LogDebugf("readFromRemoteCache: inode(%d), cacheReadRequests(%v) offset(%v) size(%v) total(%v)", s.inode, cReadRequests, offset, size, total) + return total, nil +} + +func (s *Streamer) getFlashGroup(fixedFileOffset uint64) *FlashGroup { + slot := proto.ComputeCacheBlockSlot(s.client.dataWrapper.VolName, s.inode, fixedFileOffset) + return s.client.RemoteCache.GetFlashGroupBySlot(slot) +} + +func (s *Streamer) getDataSource(start, size, fixedFileOffset uint64, isRead bool) ([]*proto.DataSource, error) { + sources := make([]*proto.DataSource, 0) + var revisedRequests []*ExtentRequest + var data []byte + eReqs := s.extents.PrepareReadRequests(int(fixedFileOffset), proto.CACHE_BLOCK_SIZE, nil) + for _, req := range eReqs { + if req.ExtentKey == nil { + continue + } + if req.ExtentKey.PartitionId == 0 || req.ExtentKey.ExtentId == 0 { + s.writeLock.Lock() + if err := s.IssueFlushRequest(); err != nil { + s.writeLock.Unlock() + return nil, err + } + revisedRequests = s.extents.PrepareReadRequests(int(fixedFileOffset), proto.CACHE_BLOCK_SIZE, data) + s.writeLock.Unlock() + break + } + } + if revisedRequests != nil { + eReqs = revisedRequests + } + + for _, eReq := range eReqs { + if eReq.ExtentKey == nil { + continue + } + + dp, err := s.client.dataWrapper.GetDataPartition(eReq.ExtentKey.PartitionId) + if err != nil { + log.LogWarnf("Streamer getDataSource: GetDataPartition failed. PartitionId(%v), err(%v)", eReq.ExtentKey.PartitionId, err) + return nil, err + } + + sortedHosts := dp.SortHostsByPingElapsed() + + source := &proto.DataSource{ + FileOffset: uint64(eReq.FileOffset), + Size_: uint64(eReq.Size), + PartitionID: eReq.ExtentKey.PartitionId, + ExtentID: eReq.ExtentKey.ExtentId, + ExtentOffset: uint64(eReq.FileOffset) - eReq.ExtentKey.FileOffset + eReq.ExtentKey.ExtentOffset, + Hosts: sortedHosts, + } + sources = append(sources, source) + } + return sources, nil +} + +func (s *Streamer) prepareCacheRequests(offset, size uint64, data []byte) ([]*CacheReadRequest, error) { + var ( + cReadRequests []*CacheReadRequest + cRequests = make([]*proto.CacheRequest, 0) + isRead = data != nil + ) + for fixedOff := offset / proto.CACHE_BLOCK_SIZE * proto.CACHE_BLOCK_SIZE; fixedOff < offset+size; fixedOff += proto.CACHE_BLOCK_SIZE { + sources, err := s.getDataSource(offset, size, fixedOff, isRead) + if err != nil { + log.LogWarnf("Streamer prepareCacheRequests: getDataSource failed. fixedOff(%v) err(%v)", fixedOff, err) + return nil, err + } + cReq := &proto.CacheRequest{ + Volume: s.client.dataWrapper.VolName, + Inode: s.inode, + FixedFileOffset: fixedOff, + TTL: s.client.CacheTTL, + Sources: sources, + Version: proto.ComputeSourcesVersion(sources), + } + cRequests = append(cRequests, cReq) + } + if isRead { + cReadRequests = getCacheReadRequests(offset, size, data, cRequests) + } else { + cReadRequests = make([]*CacheReadRequest, 0, len(cRequests)) + for _, cReq := range cRequests { + if len(cReq.Sources) == 0 { + continue + } + cReadRequest := new(CacheReadRequest) + cReadRequest.CacheRequest = cReq + cReadRequests = append(cReadRequests, cReadRequest) + } + } + return cReadRequests, nil +} + +func getCacheReadRequests(offset uint64, size uint64, data []byte, cRequests []*proto.CacheRequest) (cReadRequests []*CacheReadRequest) { + cReadRequests = make([]*CacheReadRequest, 0, len(cRequests)) + startFixedOff := offset / proto.CACHE_BLOCK_SIZE * proto.CACHE_BLOCK_SIZE + endFixedOff := (offset + size - 1) / proto.CACHE_BLOCK_SIZE * proto.CACHE_BLOCK_SIZE + + for _, cReq := range cRequests { + cReadReq := new(CacheReadRequest) + cReadReq.CacheRequest = cReq + if cReq.FixedFileOffset == startFixedOff { + cReadReq.Offset = offset - startFixedOff + } else { + cReadReq.Offset = 0 + } + + if cReq.FixedFileOffset == endFixedOff { + cReadReq.Size_ = offset + size - cReq.FixedFileOffset - cReadReq.Offset + } else { + cReadReq.Size_ = proto.CACHE_BLOCK_SIZE - cReadReq.Offset + } + + dataStart := cReadReq.Offset + cReq.FixedFileOffset - offset + cReadReq.Data = data[dataStart : dataStart+cReadReq.Size_] + cReadRequests = append(cReadRequests, cReadReq) + } + return cReadRequests +} diff --git a/sdk/data/wrapper/data_partition.go b/sdk/data/wrapper/data_partition.go index 803a8befd..96da44582 100644 --- a/sdk/data/wrapper/data_partition.go +++ b/sdk/data/wrapper/data_partition.go @@ -17,6 +17,7 @@ package wrapper import ( "fmt" "net" + "sort" "strings" "sync" "syscall" @@ -27,6 +28,60 @@ import ( "github.com/cubefs/cubefs/util/log" ) +type hostPingElapsed struct { + host string + elapsed time.Duration +} + +type PingElapsedSortedHosts struct { + sortedHosts []string + updateTSUnix int64 // Timestamp (unix second) of latest update. + getHosts func() (hosts []string) + getElapsed func(host string) (elapsed time.Duration, ok bool) +} + +func (h *PingElapsedSortedHosts) isNeedUpdate() bool { + return h.updateTSUnix == 0 || time.Now().Unix()-h.updateTSUnix > 10 +} + +func (h *PingElapsedSortedHosts) update(getHosts func() []string, getElapsed func(host string) (time.Duration, bool)) []string { + hosts := getHosts() + hostElapses := make([]*hostPingElapsed, 0, len(hosts)) + for _, host := range hosts { + var hostElapsed *hostPingElapsed + if elapsed, ok := getElapsed(host); ok { + hostElapsed = &hostPingElapsed{host: host, elapsed: elapsed} + } else { + hostElapsed = &hostPingElapsed{host: host, elapsed: time.Duration(0)} + } + hostElapses = append(hostElapses, hostElapsed) + } + sort.SliceStable(hostElapses, func(i, j int) bool { + return hostElapses[j].elapsed == 0 || hostElapses[i].elapsed < hostElapses[j].elapsed + }) + sorted := make([]string, len(hostElapses)) + for i, hotElapsed := range hostElapses { + sorted[i] = hotElapsed.host + } + h.sortedHosts = sorted + h.updateTSUnix = time.Now().Unix() + return sorted +} + +func (h *PingElapsedSortedHosts) GetSortedHosts() []string { + if h.isNeedUpdate() { + return h.update(h.getHosts, h.getElapsed) + } + return h.sortedHosts +} + +func NewPingElapsedSortHosts(getHosts func() []string, getElapsed func(host string) (time.Duration, bool)) *PingElapsedSortedHosts { + return &PingElapsedSortedHosts{ + getHosts: getHosts, + getElapsed: getElapsed, + } +} + // DataPartition defines the wrapper of the data partition. type DataPartition struct { // Will not be changed @@ -35,6 +90,8 @@ type DataPartition struct { NearHosts []string ClientWrapper *Wrapper Metrics *DataPartitionMetrics + + pingElapsedSortedHosts *PingElapsedSortedHosts } // DataPartitionMetrics defines the wrapper of the metrics related to the data partition. @@ -155,3 +212,20 @@ func isExcluded(dp *DataPartition, exclude map[string]struct{}) bool { } return false } + +func (dp *DataPartition) SortHostsByPingElapsed() []string { + if dp.pingElapsedSortedHosts == nil { + getHosts := func() []string { + return dp.Hosts + } + getElapsed := func(host string) (time.Duration, bool) { + delay, ok := dp.ClientWrapper.HostsDelay.Load(host) + if !ok { + return 0, false + } + return delay.(time.Duration), true + } + dp.pingElapsedSortedHosts = NewPingElapsedSortHosts(getHosts, getElapsed) + } + return dp.pingElapsedSortedHosts.GetSortedHosts() +} diff --git a/sdk/data/wrapper/wrapper.go b/sdk/data/wrapper/wrapper.go index 8b1161a54..6f508afa6 100644 --- a/sdk/data/wrapper/wrapper.go +++ b/sdk/data/wrapper/wrapper.go @@ -49,13 +49,14 @@ type SimpleClientInfo interface { GetReadVer() uint64 GetLatestVer() uint64 GetVerMgr() *proto.VolVersionInfoList + UpdateRemoteCacheConfig(view *proto.SimpleVolView) } // Wrapper TODO rename. This name does not reflect what it is doing. type Wrapper struct { Lock sync.RWMutex - clusterName string - volName string + ClusterName string + VolName string volType int EnablePosixAcl bool masters []string @@ -86,6 +87,7 @@ type Wrapper struct { volStorageClass uint32 volAllowedStorageClass []uint32 volStatByClass map[uint32]*proto.StatOfStorageClass + HostsDelay sync.Map } // NewDataPartitionWrapper returns a new data partition wrapper. @@ -98,7 +100,7 @@ func NewDataPartitionWrapper(client SimpleClientInfo, volName string, masters [] w.stopC = make(chan struct{}) w.masters = masters w.mc = masterSDK.NewMasterClient(masters, false) - w.volName = volName + w.VolName = volName w.partitions = make(map[uint64]*DataPartition) w.HostsStatus = make(map[string]bool) w.preload = preload @@ -178,7 +180,7 @@ func (w *Wrapper) updateClusterInfo() (err error) { return } log.LogInfof("UpdateClusterInfo: get cluster info: cluster(%v) localIP(%v)", info.Cluster, info.Ip) - w.clusterName = info.Cluster + w.ClusterName = info.Cluster LocalIP = info.Ip w.IsSnapshotEnabled = info.ClusterEnableSnapshot return @@ -200,14 +202,14 @@ func (w *Wrapper) UpdateUidsView(view *proto.SimpleVolView) { func (w *Wrapper) GetSimpleVolView() (err error) { var view *proto.SimpleVolView - if view, err = w.mc.AdminAPI().GetVolumeSimpleInfo(w.volName); err != nil { - log.LogWarnf("GetSimpleVolView: get volume simple info fail: volume(%v) err(%v)", w.volName, err) + if view, err = w.mc.AdminAPI().GetVolumeSimpleInfo(w.VolName); err != nil { + log.LogWarnf("GetSimpleVolView: get volume simple info fail: volume(%v) err(%v)", w.VolName, err) return } if view.Status == 1 { log.LogWarnf("GetSimpleVolView: volume has been marked for deletion: volume(%v) status(%v - 0:normal/1:markDelete)", - w.volName, view.Status) + w.VolName, view.Status) return proto.ErrVolNotExists } @@ -253,7 +255,7 @@ func (w *Wrapper) uploadFlowInfoByTick(clientInfo SimpleClientInfo) { func (w *Wrapper) update(clientInfo SimpleClientInfo) { ticker := time.NewTicker(time.Minute) taskFunc := func() { - w.updateSimpleVolView() + w.updateSimpleVolView(clientInfo) w.updateDataPartition(false) w.updateDataNodeStatus() w.CheckPermission() @@ -277,8 +279,8 @@ func (w *Wrapper) UploadFlowInfo(clientInfo SimpleClientInfo, init bool) (work b ) flowInfo, work = clientInfo.GetFlowInfo() - if limitRsp, err = w.mc.AdminAPI().UploadFlowInfo(w.volName, flowInfo); err != nil { - log.LogWarnf("UpdateSimpleVolView: get volume simple info fail: volume(%v) err(%v)", w.volName, err) + if limitRsp, err = w.mc.AdminAPI().UploadFlowInfo(w.VolName, flowInfo); err != nil { + log.LogWarnf("UpdateSimpleVolView: get volume simple info fail: volume(%v) err(%v)", w.VolName, err) return } @@ -296,7 +298,7 @@ func (w *Wrapper) UploadFlowInfo(clientInfo SimpleClientInfo, init bool) (work b } func (w *Wrapper) CheckPermission() { - if info, err := w.mc.UserAPI().AclOperation(w.volName, w.LocalIp, util.AclCheckIP); err != nil { + if info, err := w.mc.UserAPI().AclOperation(w.VolName, w.LocalIp, util.AclCheckIP); err != nil { syslog.Println(err) } else if !info.OK { syslog.Println(err) @@ -308,20 +310,20 @@ func (w *Wrapper) updateVerlist(client SimpleClientInfo) (err error) { if !w.IsSnapshotEnabled { return } - verList, err := w.mc.AdminAPI().GetVerList(w.volName) + verList, err := w.mc.AdminAPI().GetVerList(w.VolName) if err != nil { log.LogErrorf("CheckReadVerSeq: get cluster fail: err(%v)", err) return err } if verList == nil { - msg := fmt.Sprintf("get verList nil, vol [%v] reqd seq [%v]", w.volName, w.verReadSeq) + msg := fmt.Sprintf("get verList nil, vol [%v] reqd seq [%v]", w.VolName, w.verReadSeq) log.LogErrorf("action[CheckReadVerSeq] %v", msg) return fmt.Errorf("%v", msg) } if w.verReadSeq > 0 { - if _, err = w.CheckReadVerSeq(w.volName, w.verConfReadSeq, verList); err != nil { + if _, err = w.CheckReadVerSeq(w.VolName, w.verConfReadSeq, verList); err != nil { log.LogFatalf("updateSimpleVolView: readSeq abnormal %v", err) } return @@ -335,10 +337,10 @@ func (w *Wrapper) updateVerlist(client SimpleClientInfo) (err error) { return } -func (w *Wrapper) updateSimpleVolView() (err error) { +func (w *Wrapper) updateSimpleVolView(clientInfo SimpleClientInfo) (err error) { var view *proto.SimpleVolView - if view, err = w.mc.AdminAPI().GetVolumeSimpleInfo(w.volName); err != nil { - log.LogWarnf("updateSimpleVolView: get volume simple info fail: volume(%v) err(%v)", w.volName, err) + if view, err = w.mc.AdminAPI().GetVolumeSimpleInfo(w.VolName); err != nil { + log.LogWarnf("updateSimpleVolView: get volume simple info fail: volume(%v) err(%v)", w.VolName, err) return } @@ -359,7 +361,7 @@ func (w *Wrapper) updateSimpleVolView() (err error) { w.dpSelectorChanged = true w.Lock.Unlock() } - + clientInfo.UpdateRemoteCacheConfig(view) return nil } @@ -418,7 +420,7 @@ func (w *Wrapper) updateDataPartitionByRsp(forceUpdate bool, refreshPolicy Refre if forceUpdate || len(rwPartitionGroups) >= 1 { log.LogInfof("updateDataPartition: volume(%v) refresh dpSelector, forceUpdate(%v) policy(%v), "+ "allDp(%v) allWritableDp(%v), SsdDp(%v) SsdWritableDp(%v), hddDp(%v) hddWritableDp(%v)", - w.volName, forceUpdate, refreshPolicy, len(DataPartitions), len(rwPartitionGroups), + w.VolName, forceUpdate, refreshPolicy, len(DataPartitions), len(rwPartitionGroups), ssdDpCount, ssdDpWritableCount, hddDpCount, hddDpWritableCount) w.refreshDpSelector(refreshPolicy, rwPartitionGroups) } else { @@ -427,7 +429,7 @@ func (w *Wrapper) updateDataPartitionByRsp(forceUpdate bool, refreshPolicy Refre } else { err = errors.New("updateDataPartition: no writable data partition") log.LogWarnf("updateDataPartition: no enough writable data partitions, volume(%v) with %v rw partitions(%v all), forceUpdate(%v)", - w.volName, len(rwPartitionGroups), len(DataPartitions), forceUpdate) + w.VolName, len(rwPartitionGroups), len(DataPartitions), forceUpdate) } } @@ -440,12 +442,12 @@ func (w *Wrapper) updateDataPartition(isInit bool) (err error) { return } var dpv *proto.DataPartitionsView - if dpv, err = w.mc.ClientAPI().EncodingGzip().GetDataPartitions(w.volName); err != nil { - log.LogErrorf("updateDataPartition: get data partitions fail: volume(%v) err(%v)", w.volName, err) + if dpv, err = w.mc.ClientAPI().EncodingGzip().GetDataPartitions(w.VolName); err != nil { + log.LogErrorf("updateDataPartition: get data partitions fail: volume(%v) err(%v)", w.VolName, err) return } log.LogInfof("updateDataPartition: get data partitions: volume(%v) partitions(%v) VolReadOnly(%v)", - w.volName, len(dpv.DataPartitions), dpv.VolReadOnly) + w.VolName, len(dpv.DataPartitions), dpv.VolReadOnly) forceUpdate := false if isInit || dpv.VolReadOnly { @@ -457,7 +459,7 @@ func (w *Wrapper) updateDataPartition(isInit bool) (err error) { for _, st := range dpv.StatByClass { m[st.StorageClass] = st log.LogInfof("updateDataPartition: get storage class stat info: volume(%v) stat(%s) VolReadOnly(%v)", - w.volName, st.String(), dpv.VolReadOnly) + w.VolName, st.String(), dpv.VolReadOnly) } w.volStatByClass = m w.Lock.Unlock() @@ -485,14 +487,14 @@ func (w *Wrapper) UpdateDataPartition() (err error) { // updateDataPartition which may not take effect if nginx be placed for reduce the pressure of master func (w *Wrapper) getDataPartitionFromMaster(dpId uint64) (err error) { var dpInfo *proto.DataPartitionInfo - if dpInfo, err = w.mc.AdminAPI().GetDataPartition(w.volName, dpId); err != nil { + if dpInfo, err = w.mc.AdminAPI().GetDataPartition(w.VolName, dpId); err != nil { log.LogErrorf("getDataPartitionFromMaster: get data partitions fail: volume(%v) dpId(%v) err(%v)", - w.volName, dpId, err) + w.VolName, dpId, err) return } log.LogInfof("getDataPartitionFromMaster: get data partitions: volume(%v), dpId(%v) mediaType(%v)", - w.volName, dpId, proto.MediaTypeString(dpInfo.MediaType)) + w.VolName, dpId, proto.MediaTypeString(dpInfo.MediaType)) var leaderAddr string for _, replica := range dpInfo.Replicas { if replica.IsLeader { @@ -642,7 +644,7 @@ func (w *Wrapper) CheckReadVerSeq(volName string, verReadSeq uint64, verList *pr // WarningMsg returns the warning message that contains the cluster name. func (w *Wrapper) WarningMsg() string { - return fmt.Sprintf("%s_client_warning", w.clusterName) + return fmt.Sprintf("%s_client_warning", w.ClusterName) } func (w *Wrapper) updateDataNodeStatus() (err error) { diff --git a/sdk/meta/api.go b/sdk/meta/api.go index 4cec1f8ca..4da171ff9 100644 --- a/sdk/meta/api.go +++ b/sdk/meta/api.go @@ -238,7 +238,14 @@ func (mw *MetaWrapper) create_ll(parentID uint64, name string, mode, uid, gid ui mp *MetaPartition rwPartitions []*MetaPartition ) - + defer func() { + if info != nil && mw.RemoteCacheBloom != nil { + cacheBloom := mw.RemoteCacheBloom() + if cacheBloom.TestUint64(parentID) { + cacheBloom.AddUint64(info.Inode) + } + } + }() parentMP := mw.getPartitionByInode(parentID) if parentMP == nil { log.LogErrorf("Create_ll: No parent partition, parentID(%v)", parentID) @@ -360,6 +367,15 @@ create_dentry: } func (mw *MetaWrapper) Lookup_ll(parentID uint64, name string) (inode uint64, mode uint32, err error) { + defer func() { + if err == nil && mw.RemoteCacheBloom != nil { + cacheBloom := mw.RemoteCacheBloom() + if cacheBloom.TestUint64(parentID) { + cacheBloom.AddUint64(inode) + } + } + }() + parentMP := mw.getPartitionByInode(parentID) if parentMP == nil { log.LogErrorf("Lookup_ll: No parent partition, parentID(%v) name(%v)", parentID, name) diff --git a/sdk/meta/meta.go b/sdk/meta/meta.go index a4bfaf466..8bf9b36b7 100644 --- a/sdk/meta/meta.go +++ b/sdk/meta/meta.go @@ -29,6 +29,7 @@ import ( masterSDK "github.com/cubefs/cubefs/sdk/master" "github.com/cubefs/cubefs/util" "github.com/cubefs/cubefs/util/auth" + "github.com/cubefs/cubefs/util/bloom" "github.com/cubefs/cubefs/util/btree" "github.com/cubefs/cubefs/util/errors" "github.com/cubefs/cubefs/util/log" @@ -192,6 +193,8 @@ type MetaWrapper struct { CacheDpStorageClass uint32 InnerReq bool FollowerRead bool + + RemoteCacheBloom func() *bloom.BloomFilter } type uniqidRange struct { diff --git a/sdk/meta/operation.go b/sdk/meta/operation.go index 3b9cc29db..90ea86357 100644 --- a/sdk/meta/operation.go +++ b/sdk/meta/operation.go @@ -1142,6 +1142,14 @@ func (mw *MetaWrapper) readDir(mp *MetaPartition, parentID uint64) (status int, bgTime := stat.BeginStat() defer func() { stat.EndStat("readDir", err, bgTime, 1) + if err == nil && mw.RemoteCacheBloom != nil { + cacheBloom := mw.RemoteCacheBloom() + if cacheBloom.TestUint64(parentID) { + for _, c := range children { + cacheBloom.AddUint64(c.Inode) + } + } + } }() req := &proto.ReadDirRequest{