gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 02:00:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

update(flashsdk): enable remote cache

Browse Source 
Signed-off-by: leonrayang <chl696@sina.com>
This commit is contained in:
leonrayang 2023-12-22 17:21:32 +08:00 committed by zhumingze1108
parent d8396639d5
commit 2e497bcd73
16 changed files with 609 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
client/fs/super.go
View File

@ -230,6 +230,7 @@ func NewSuper(opt *proto.MountOptions) (s *Super, err error) {
BcacheDir: opt.BcacheDir,
MaxStreamerLimit: opt.MaxStreamerLimit,
VerReadSeq: opt.VerReadSeq,
MetaWrapper: s.mw,
OnAppendExtentKey: s.mw.AppendExtentKey,
OnSplitExtentKey: s.mw.SplitExtentKey,
OnGetExtents: s.mw.GetExtents,

4
cmd/common/args.go
View File

@ -25,10 +25,12 @@ type (
func (b *Bool) Key(key string) *Argument { return NewArgument(key, &b.V) }
func (b *Bool) Enable() *Argument { return b.Key("enable") }
func (b *Bool) Status() *Argument { return b.Key("status") }
func (b *Bool) All() *Argument { return b.Key("all") }
func (i *Int) Key(key string) *Argument { return NewArgument(key, &i.V) }
func (i *Int) ID() *Argument { return i.Key("id") }
func (i *Int) ExtentID() *Argument { return i.Key("extentID") }
func (i *Int) Count() *Argument { return i.Key("count") }
func (u *Uint) Key(key string) *Argument { return NewArgument(key, &u.V) }
func (u *Uint) ID() *Argument { return u.Key("id") }
@ -42,3 +44,5 @@ func (f *Float) Key(key string) *Argument { return NewArgument(key, &f.V) }
func (s *String) Key(key string) *Argument { return NewArgument(key, &s.V) }
func (s *String) Disk() *Argument { return s.Key("disk") }
func (s *String) DiskPath() *Argument { return s.Key("diskPath") }
func (s *String) Addr() *Argument { return s.Key("addr") }
func (s *String) ZoneName() *Argument { return s.Key("zoneName") }

5
cmd/common/argument.go
View File

@ -67,6 +67,11 @@ func (a *Argument) OnValue(f func() error) *Argument {
return a
}
// Key returns key of the argument.
func (a *Argument) Key() string {
return a.key
}
func ParseArguments(r *http.Request, args ...*Argument) error {
if err := r.ParseForm(); err != nil {
return err

28
cmd/common/argument_test.go
View File

@ -112,3 +112,31 @@ func TestCmdCommonArgs(t *testing.T) {
require.Equal(t, "11111", s.V)
}
}
func TestCmdCommonKeys(t *testing.T) {
b := new(Bool)
require.Equal(t, "enable", b.Enable().Key())
require.Equal(t, "status", b.Status().Key())
require.Equal(t, "all", b.All().Key())
i := new(Int)
require.Equal(t, "id", i.ID().Key())
require.Equal(t, "extentID", i.ExtentID().Key())
require.Equal(t, "count", i.Count().Key())
u := new(Uint)
require.Equal(t, "id", u.ID().Key())
require.Equal(t, "pid", u.PID().Key())
require.Equal(t, "partitionID", u.PartitionID().Key())
require.Equal(t, "ino", u.Ino().Key())
require.Equal(t, "parentIno", u.ParentIno().Key())
f := new(Float)
require.Equal(t, "fff", f.Key("fff").Key())
s := new(String)
require.Equal(t, "disk", s.Disk().Key())
require.Equal(t, "diskPath", s.DiskPath().Key())
require.Equal(t, "addr", s.Addr().Key())
require.Equal(t, "zoneName", s.ZoneName().Key())
}

2
master/const.go
View File

@ -287,6 +287,7 @@ const (
opSyncDeleteVolUser uint32 = 0x1D
opSyncUpdateVolUser uint32 = 0x1E
opSyncNodeSetGrp uint32 = 0x1F
opSyncDataPartitionsView uint32 = 0x20
opSyncExclueDomain uint32 = 0x23
opSyncUpdateZone uint32 = 0x24
opSyncAllocClientID uint32 = 0x25
@ -299,6 +300,7 @@ const (
opSyncAddLcNode uint32 = 0x30
opSyncDeleteLcNode uint32 = 0x31
opSyncUpdateLcNode uint32 = 0x32
opSyncAddLcConf uint32 = 0x33
opSyncDeleteLcConf uint32 = 0x34
opSyncUpdateLcConf uint32 = 0x35

7
proto/admin_proto.go
View File

@ -1274,6 +1274,13 @@ type SimpleVolView struct {
CacheDpStorageClass uint32
ForbidWriteOpOfProtoVer0 bool
QuotaOfStorageClass []*StatOfStorageClass
RemoteCacheBoostEnable bool
RemoteCacheBoostPath string
RemoteCacheAutoPrepare bool
RemoteCacheTTL int64
RemoteCacheReadTimeoutMs int64
EnableRemoveDupReq bool // TODO: using it in metanode
}
type NodeSetInfo struct {

167
sdk/data/stream/extent_client.go
View File

@ -18,6 +18,7 @@ import (
"container/list"
"context"
"fmt"
"runtime/debug"
"strings"
"sync"
"sync/atomic"
@ -30,6 +31,7 @@ import (
"github.com/cubefs/cubefs/sdk/data/wrapper"
"github.com/cubefs/cubefs/sdk/meta"
"github.com/cubefs/cubefs/util"
"github.com/cubefs/cubefs/util/bloom"
"github.com/cubefs/cubefs/util/errors"
"github.com/cubefs/cubefs/util/exporter"
"github.com/cubefs/cubefs/util/log"
@ -42,6 +44,11 @@ var reqChanSize = defaultChanSize
const defaultChanSize = 64
const (
PrepareWorkerNum = 5
PrepareReqChanCap = 1024
)
type (
SplitExtentKeyFunc func(parentInode, inode uint64, key proto.ExtentKey, storageClass uint32) error
AppendExtentKeyFunc func(parentInode, inode uint64, key proto.ExtentKey, discard []proto.ExtentKey, isCache bool, storageClass uint32, isMigration bool) (int, error)
@ -124,6 +131,7 @@ type ExtentConfig struct {
BcacheDir string
MaxStreamerLimit int64
VerReadSeq uint64
MetaWrapper *meta.MetaWrapper
OnAppendExtentKey AppendExtentKeyFunc
OnSplitExtentKey SplitExtentKeyFunc
OnGetExtents GetExtentsFunc
@ -178,6 +186,7 @@ type ExtentClient struct {
preload bool
LimitManager *manager.LimitManager
dataWrapper *wrapper.Wrapper
metaWrapper *meta.MetaWrapper
appendExtentKey AppendExtentKeyFunc
splitExtentKey SplitExtentKeyFunc
getExtents GetExtentsFunc
@ -197,6 +206,20 @@ type ExtentClient struct {
bcacheOnlyForNotSSD bool
InnerReq bool
AheadRead *AheadReadCache
extentConfig *ExtentConfig
OldCacheBoostStatus bool
EnableClusterCacheBoost bool
EnableVolCacheBoost bool
EnableCacheAutoPrepare bool
CacheBoostPath string
CacheTTL int64
CacheReadTimeoutMs int64
RemoteCache *RemoteCache
prepareRemoteCacheCh chan *PrepareRemoteCacheRequest
stopOnce sync.Once
stopCh chan struct{}
wg sync.WaitGroup
}
func (client *ExtentClient) UidIsLimited(uid uint32) bool {
@ -370,6 +393,19 @@ retry:
log.LogInfof("max streamer limit %d", client.maxStreamerLimit)
client.streamerList = list.New()
if client.metaWrapper != nil {
client.metaWrapper.RemoteCacheBloom = client.RemoteCacheBloom
}
client.extentConfig = config
if client.IsCacheBoostEnabled() {
client.InitRemoteCache()
}
client.prepareRemoteCacheCh = make(chan *PrepareRemoteCacheRequest, PrepareReqChanCap)
client.stopCh = make(chan struct{})
client.wg.Add(1)
go client.DoPrepare()
go client.backgroundEvictStream()
return
@ -394,6 +430,84 @@ func (client *ExtentClient) SetClientID(id uint64) (err error) {
return
}
func (client *ExtentClient) setClusterBoostEnable(enableBoost bool) {
client.OldCacheBoostStatus = client.IsCacheBoostEnabled()
oldEnableBoost := client.EnableClusterCacheBoost
client.EnableClusterCacheBoost = enableBoost
if oldEnableBoost != enableBoost {
log.LogInfof("setClusterBoostEnable: from old(%v) to new(%v)", oldEnableBoost, enableBoost)
}
}
func (client *ExtentClient) IsCacheBoostEnabled() bool {
return client.EnableClusterCacheBoost && client.EnableVolCacheBoost
}
func (client *ExtentClient) InitRemoteCache() (err error) {
cacheConfig := &CacheConfig{
Cluster: client.dataWrapper.ClusterName,
Volume: client.extentConfig.Volume,
Masters: client.extentConfig.Masters,
MW: client.metaWrapper,
readTimeoutSec: client.CacheReadTimeoutMs / 1000,
}
if client.RemoteCache, err = NewRemoteCache(cacheConfig); err != nil {
return
}
if !client.RemoteCache.ResetCacheBoostPathToBloom(client.CacheBoostPath) {
client.CacheBoostPath = ""
}
return
}
func (client *ExtentClient) UpdateRemoteCacheConfig(view *proto.SimpleVolView) {
if client.EnableVolCacheBoost != view.RemoteCacheBoostEnable {
log.LogInfof("updateRemoteCacheConfig: RemoteCacheBoostEnable from old(%v) to new(%v)", client.EnableVolCacheBoost, view.RemoteCacheBoostEnable)
client.EnableVolCacheBoost = view.RemoteCacheBoostEnable
}
if client.OldCacheBoostStatus != client.IsCacheBoostEnabled() {
log.LogInfof("updateRemoteCacheConfig: enable from old(%v) to new(%v)", client.OldCacheBoostStatus, client.IsCacheBoostEnabled())
}
// RemoteCache may be nil if the first initialization failed, it will not be set nil anymore even if remote cache is disabled
if client.IsCacheBoostEnabled() {
if !client.OldCacheBoostStatus || client.RemoteCache == nil {
log.LogInfof("updateRemoteCacheConfig: initRemoteCache: enable(%v -> %v) RemoteCache isNil(%v)", client.OldCacheBoostStatus, client.IsCacheBoostEnabled(), client.RemoteCache == nil)
if err := client.InitRemoteCache(); err != nil {
log.LogErrorf("updateRemoteCacheConfig: initRemoteCache failed, err: %v", err)
}
}
} else if client.OldCacheBoostStatus && client.RemoteCache != nil {
client.RemoteCache.Stop()
log.LogInfof("updateRemoteCacheConfig: stop RemoteCache")
}
if client.CacheBoostPath != view.RemoteCacheBoostPath {
oldBoostPath := client.CacheBoostPath
client.CacheBoostPath = view.RemoteCacheBoostPath
if client.IsCacheBoostEnabled() && client.RemoteCache != nil {
if !client.RemoteCache.ResetCacheBoostPathToBloom(view.RemoteCacheBoostPath) {
client.CacheBoostPath = ""
}
}
log.LogInfof("updateRemoteCacheConfig: RemoteCacheBoostPath from old(%v) to want(%v), but(%v)", oldBoostPath, view.RemoteCacheBoostPath, client.CacheBoostPath)
}
if client.EnableCacheAutoPrepare != view.RemoteCacheAutoPrepare {
log.LogInfof("updateRemoteCacheConfig: RemoteCacheAutoPrepare from old(%v) to new(%v)", client.EnableCacheAutoPrepare, view.RemoteCacheAutoPrepare)
client.EnableCacheAutoPrepare = view.RemoteCacheAutoPrepare
}
if client.CacheTTL != view.RemoteCacheTTL {
log.LogInfof("updateRemoteCacheConfig: RemoteCacheTTL from old(%d) to new(%d)", client.CacheTTL, view.RemoteCacheTTL)
client.CacheTTL = view.RemoteCacheTTL
}
if client.CacheReadTimeoutMs != view.RemoteCacheReadTimeoutMs {
client.CacheReadTimeoutMs = view.RemoteCacheReadTimeoutMs
}
}
func (client *ExtentClient) GetVolumeName() string {
return client.volumeName
}
@ -877,6 +991,8 @@ func setRate(lim *rate.Limiter, val int) string {
func (client *ExtentClient) Close() error {
// release streamers
client.stopOnce.Do(func() { close(client.stopCh) })
client.wg.Wait()
var inodes []uint64
client.streamerLock.Lock()
inodes = make([]uint64, 0, len(client.streamers))
@ -917,3 +1033,54 @@ func (client *ExtentClient) IsPreloadMode() bool {
func (client *ExtentClient) UploadFlowInfo(clientInfo wrapper.SimpleClientInfo) (bWork bool, err error) {
return client.dataWrapper.UploadFlowInfo(clientInfo, false)
}
func (c *ExtentClient) RemoteCacheBloom() *bloom.BloomFilter {
if c.RemoteCache != nil {
return c.RemoteCache.GetRemoteCacheBloom()
}
return nil
}
func (c *ExtentClient) GetInodeBloomStatus(ino uint64) bool {
cacheBloom := c.RemoteCacheBloom()
if cacheBloom == nil {
return false
}
cacheBloom.AddUint64(ino)
return true
}
func (c *ExtentClient) DoPrepare() {
defer c.wg.Done()
workerWg := sync.WaitGroup{}
for i := 0; i < PrepareWorkerNum; i++ {
workerWg.Add(1)
go func() {
defer workerWg.Done()
for {
select {
case <-c.stopCh:
return
case req := <-c.prepareRemoteCacheCh:
c.servePrepareRequest(req)
}
}
}()
}
workerWg.Wait()
}
func (c *ExtentClient) servePrepareRequest(prepareReq *PrepareRemoteCacheRequest) {
defer func() {
if err := recover(); err != nil {
log.LogWarnf("servePrepareRequest: panic occurs, stack(%v)", string(debug.Stack()))
}
}()
s := c.GetStreamer(prepareReq.inode)
if s == nil {
log.LogWarnf("servePrepareRequest: streamer is nil, prepare request: %v)", prepareReq)
return
}
s.prepareRemoteCache(prepareReq.ctx, prepareReq.ek)
}

4
sdk/data/stream/extent_flash_cache.go
View File

@ -72,7 +72,7 @@ type CacheConfig struct {
MW *meta.MetaWrapper
SameZoneWeight int
ReadTimeoutSec int
readTimeoutSec int64
}
type RemoteCache struct {
@ -105,7 +105,7 @@ func NewRemoteCache(config *CacheConfig) (*RemoteCache, error) {
} else {
rc.sameZoneWeight = config.SameZoneWeight
}
rc.readTimeoutSec = config.ReadTimeoutSec
rc.readTimeoutSec = int(config.readTimeoutSec)
if rc.readTimeoutSec <= 0 {
rc.readTimeoutSec = _connReadTimeoutSec
}

11
sdk/data/stream/extent_handler.go
View File

@ -15,6 +15,7 @@
package stream
import (
"context"
"fmt"
"math"
"math/rand"
@ -609,6 +610,16 @@ func (eh *ExtentHandler) appendExtentKey() (err error) {
*/
_ = eh.stream.extents.Append(eh.key, false)
}
if eh.key.PartitionId > 0 && eh.stream.enableCacheAutoPrepare() {
prepareReq := &PrepareRemoteCacheRequest{
ctx: context.Background(),
ek: eh.key,
inode: eh.stream.inode,
}
eh.stream.sendToPrepareRomoteCacheChan(prepareReq)
// eh.stream.prepareRemoteCache(ctx, ek)
}
}
if err == nil {
eh.dirty = false

19
sdk/data/stream/stream_reader.go
View File

@ -56,10 +56,10 @@ type Streamer struct {
needUpdateVer int32
isCache bool
openForWrite bool
rdonly bool
aheadReadEnable bool
aheadReadWindow *AheadReadWindow
rdonly bool
aheadReadEnable bool
bloomStatus bool
aheadReadWindow *AheadReadWindow
}
type bcacheKey struct {
@ -254,6 +254,17 @@ func (s *Streamer) read(data []byte, offset int, size int, storageClass uint32)
log.LogDebugf("Streamer not read from bcache, ino(%v) storageClass(%v) s.client.bcacheEnable(%v) bcacheOnlyForNotSSD(%v)",
s.inode, proto.StorageClassString(inodeInfo.StorageClass), s.client.bcacheEnable, s.client.bcacheOnlyForNotSSD)
}
log.LogDebugf("TRACE Stream read. miss blockCache cacheKey(%v) loadBcache(%v)", cacheKey, s.client.loadBcache)
} else if s.enableRemoteCache() {
var cacheReadRequests []*CacheReadRequest
cacheReadRequests, err = s.prepareCacheRequests(uint64(offset), uint64(size), data)
if err == nil {
var read int
if read, err = s.readFromRemoteCache(ctx, uint64(offset), uint64(size), cacheReadRequests); err == nil {
return read, err
}
}
log.LogWarnf("Stream read: readFromRemoteCache failed: ino(%v) offset(%v) size(%v), err(%v)", s.inode, offset, size, err)
}
if s.needBCache {

235
sdk/data/stream/stream_remote_cache.go Normal file
View File

@ -0,0 +1,235 @@
// Copyright 2023 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package stream
import (
"context"
"fmt"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/util/log"
"github.com/cubefs/cubefs/util/stat"
)
type PrepareRemoteCacheRequest struct {
ctx context.Context
inode uint64
ek *proto.ExtentKey
}
func (pr *PrepareRemoteCacheRequest) String() string {
if pr == nil {
return ""
}
return fmt.Sprintf("PrepareRemoteCacheRequest{ino: %v, ek: %v}", pr.inode, pr.ek)
}
func (s *Streamer) enableRemoteCache() bool {
if !s.client.IsCacheBoostEnabled() || s.client.RemoteCache == nil {
return false
}
return s.bloomStatus
}
func (s *Streamer) enableCacheAutoPrepare() bool {
if !s.client.EnableCacheAutoPrepare {
return false
}
return s.enableRemoteCache()
}
func (s *Streamer) sendToPrepareRomoteCacheChan(req *PrepareRemoteCacheRequest) {
select {
case s.client.prepareRemoteCacheCh <- req:
default:
log.LogWarnf("sendToPrepareRomoteCacheChan: chan is full, discard req(%v)", req)
}
}
func (s *Streamer) prepareRemoteCache(ctx context.Context, ek *proto.ExtentKey) {
cReadRequests, err := s.prepareCacheRequests(ek.FileOffset, uint64(ek.Size), nil)
if err != nil {
log.LogWarnf("Streamer prepareRemoteCache: prepareCacheRequests failed. start(%v), size(%v), err(%v)", ek.FileOffset, ek.Size, err)
return
}
for _, req := range cReadRequests {
fg := s.getFlashGroup(req.CacheRequest.FixedFileOffset)
if fg == nil {
err = fmt.Errorf("cannot find any flashGroups")
log.LogWarnf("Streamer prepareRemoteCache failed: %v", err)
break
}
prepareReq := &proto.CachePrepareRequest{
CacheRequest: req.CacheRequest,
FlashNodes: fg.Hosts,
}
if err = s.client.RemoteCache.Prepare(ctx, fg, s.inode, prepareReq); err != nil {
log.LogWarnf("Streamer prepareRemoteCache: flashGroup prepare failed. fg(%v) req(%v) err(%v)", fg, prepareReq, err)
}
}
log.LogDebugf("prepareRemoteCache: inode(%d), err(%v)", s.inode, err)
}
func (s *Streamer) readFromRemoteCache(ctx context.Context, offset, size uint64, cReadRequests []*CacheReadRequest) (total int, err error) {
bgTime := stat.BeginStat()
defer func() {
stat.EndStat("remote-cache-read", err, bgTime, 1)
}()
var read int
for _, req := range cReadRequests {
if len(req.CacheRequest.Sources) == 0 {
total += int(req.Size_)
continue
}
fg := s.getFlashGroup(req.CacheRequest.FixedFileOffset)
if fg == nil {
err = fmt.Errorf("readFromRemoteCache failed: Cannot find any flashGroups")
return
}
if read, err = s.client.RemoteCache.Read(ctx, fg, s.inode, req); err != nil {
log.LogWarnf("readFromRemoteCache: flashGroup read failed. offset(%v) size(%v) fg(%v) req(%v) err(%v)", offset, size, fg, req, err)
return
} else {
total += read
}
}
log.LogDebugf("readFromRemoteCache: inode(%d), cacheReadRequests(%v) offset(%v) size(%v) total(%v)", s.inode, cReadRequests, offset, size, total)
return total, nil
}
func (s *Streamer) getFlashGroup(fixedFileOffset uint64) *FlashGroup {
slot := proto.ComputeCacheBlockSlot(s.client.dataWrapper.VolName, s.inode, fixedFileOffset)
return s.client.RemoteCache.GetFlashGroupBySlot(slot)
}
func (s *Streamer) getDataSource(start, size, fixedFileOffset uint64, isRead bool) ([]*proto.DataSource, error) {
sources := make([]*proto.DataSource, 0)
var revisedRequests []*ExtentRequest
var data []byte
eReqs := s.extents.PrepareReadRequests(int(fixedFileOffset), proto.CACHE_BLOCK_SIZE, nil)
for _, req := range eReqs {
if req.ExtentKey == nil {
continue
}
if req.ExtentKey.PartitionId == 0 || req.ExtentKey.ExtentId == 0 {
s.writeLock.Lock()
if err := s.IssueFlushRequest(); err != nil {
s.writeLock.Unlock()
return nil, err
}
revisedRequests = s.extents.PrepareReadRequests(int(fixedFileOffset), proto.CACHE_BLOCK_SIZE, data)
s.writeLock.Unlock()
break
}
}
if revisedRequests != nil {
eReqs = revisedRequests
}
for _, eReq := range eReqs {
if eReq.ExtentKey == nil {
continue
}
dp, err := s.client.dataWrapper.GetDataPartition(eReq.ExtentKey.PartitionId)
if err != nil {
log.LogWarnf("Streamer getDataSource: GetDataPartition failed. PartitionId(%v), err(%v)", eReq.ExtentKey.PartitionId, err)
return nil, err
}
sortedHosts := dp.SortHostsByPingElapsed()
source := &proto.DataSource{
FileOffset: uint64(eReq.FileOffset),
Size_: uint64(eReq.Size),
PartitionID: eReq.ExtentKey.PartitionId,
ExtentID: eReq.ExtentKey.ExtentId,
ExtentOffset: uint64(eReq.FileOffset) - eReq.ExtentKey.FileOffset + eReq.ExtentKey.ExtentOffset,
Hosts: sortedHosts,
}
sources = append(sources, source)
}
return sources, nil
}
func (s *Streamer) prepareCacheRequests(offset, size uint64, data []byte) ([]*CacheReadRequest, error) {
var (
cReadRequests []*CacheReadRequest
cRequests = make([]*proto.CacheRequest, 0)
isRead = data != nil
)
for fixedOff := offset / proto.CACHE_BLOCK_SIZE * proto.CACHE_BLOCK_SIZE; fixedOff < offset+size; fixedOff += proto.CACHE_BLOCK_SIZE {
sources, err := s.getDataSource(offset, size, fixedOff, isRead)
if err != nil {
log.LogWarnf("Streamer prepareCacheRequests: getDataSource failed. fixedOff(%v) err(%v)", fixedOff, err)
return nil, err
}
cReq := &proto.CacheRequest{
Volume: s.client.dataWrapper.VolName,
Inode: s.inode,
FixedFileOffset: fixedOff,
TTL: s.client.CacheTTL,
Sources: sources,
Version: proto.ComputeSourcesVersion(sources),
}
cRequests = append(cRequests, cReq)
}
if isRead {
cReadRequests = getCacheReadRequests(offset, size, data, cRequests)
} else {
cReadRequests = make([]*CacheReadRequest, 0, len(cRequests))
for _, cReq := range cRequests {
if len(cReq.Sources) == 0 {
continue
}
cReadRequest := new(CacheReadRequest)
cReadRequest.CacheRequest = cReq
cReadRequests = append(cReadRequests, cReadRequest)
}
}
return cReadRequests, nil
}
func getCacheReadRequests(offset uint64, size uint64, data []byte, cRequests []*proto.CacheRequest) (cReadRequests []*CacheReadRequest) {
cReadRequests = make([]*CacheReadRequest, 0, len(cRequests))
startFixedOff := offset / proto.CACHE_BLOCK_SIZE * proto.CACHE_BLOCK_SIZE
endFixedOff := (offset + size - 1) / proto.CACHE_BLOCK_SIZE * proto.CACHE_BLOCK_SIZE
for _, cReq := range cRequests {
cReadReq := new(CacheReadRequest)
cReadReq.CacheRequest = cReq
if cReq.FixedFileOffset == startFixedOff {
cReadReq.Offset = offset - startFixedOff
} else {
cReadReq.Offset = 0
}
if cReq.FixedFileOffset == endFixedOff {
cReadReq.Size_ = offset + size - cReq.FixedFileOffset - cReadReq.Offset
} else {
cReadReq.Size_ = proto.CACHE_BLOCK_SIZE - cReadReq.Offset
}
dataStart := cReadReq.Offset + cReq.FixedFileOffset - offset
cReadReq.Data = data[dataStart : dataStart+cReadReq.Size_]
cReadRequests = append(cReadRequests, cReadReq)
}
return cReadRequests
}

74
sdk/data/wrapper/data_partition.go
View File

@ -17,6 +17,7 @@ package wrapper
import (
"fmt"
"net"
"sort"
"strings"
"sync"
"syscall"
@ -27,6 +28,60 @@ import (
"github.com/cubefs/cubefs/util/log"
)
type hostPingElapsed struct {
host string
elapsed time.Duration
}
type PingElapsedSortedHosts struct {
sortedHosts []string
updateTSUnix int64 // Timestamp (unix second) of latest update.
getHosts func() (hosts []string)
getElapsed func(host string) (elapsed time.Duration, ok bool)
}
func (h *PingElapsedSortedHosts) isNeedUpdate() bool {
return h.updateTSUnix == 0 || time.Now().Unix()-h.updateTSUnix > 10
}
func (h *PingElapsedSortedHosts) update(getHosts func() []string, getElapsed func(host string) (time.Duration, bool)) []string {
hosts := getHosts()
hostElapses := make([]*hostPingElapsed, 0, len(hosts))
for _, host := range hosts {
var hostElapsed *hostPingElapsed
if elapsed, ok := getElapsed(host); ok {
hostElapsed = &hostPingElapsed{host: host, elapsed: elapsed}
} else {
hostElapsed = &hostPingElapsed{host: host, elapsed: time.Duration(0)}
}
hostElapses = append(hostElapses, hostElapsed)
}
sort.SliceStable(hostElapses, func(i, j int) bool {
return hostElapses[j].elapsed == 0 || hostElapses[i].elapsed < hostElapses[j].elapsed
})
sorted := make([]string, len(hostElapses))
for i, hotElapsed := range hostElapses {
sorted[i] = hotElapsed.host
}
h.sortedHosts = sorted
h.updateTSUnix = time.Now().Unix()
return sorted
}
func (h *PingElapsedSortedHosts) GetSortedHosts() []string {
if h.isNeedUpdate() {
return h.update(h.getHosts, h.getElapsed)
}
return h.sortedHosts
}
func NewPingElapsedSortHosts(getHosts func() []string, getElapsed func(host string) (time.Duration, bool)) *PingElapsedSortedHosts {
return &PingElapsedSortedHosts{
getHosts: getHosts,
getElapsed: getElapsed,
}
}
// DataPartition defines the wrapper of the data partition.
type DataPartition struct {
// Will not be changed
@ -35,6 +90,8 @@ type DataPartition struct {
NearHosts []string
ClientWrapper *Wrapper
Metrics *DataPartitionMetrics
pingElapsedSortedHosts *PingElapsedSortedHosts
}
// DataPartitionMetrics defines the wrapper of the metrics related to the data partition.
@ -155,3 +212,20 @@ func isExcluded(dp *DataPartition, exclude map[string]struct{}) bool {
}
return false
}
func (dp *DataPartition) SortHostsByPingElapsed() []string {
if dp.pingElapsedSortedHosts == nil {
getHosts := func() []string {
return dp.Hosts
}
getElapsed := func(host string) (time.Duration, bool) {
delay, ok := dp.ClientWrapper.HostsDelay.Load(host)
if !ok {
return 0, false
}
return delay.(time.Duration), true
}
dp.pingElapsedSortedHosts = NewPingElapsedSortHosts(getHosts, getElapsed)
}
return dp.pingElapsedSortedHosts.GetSortedHosts()
}

58
sdk/data/wrapper/wrapper.go
View File

@ -49,13 +49,14 @@ type SimpleClientInfo interface {
GetReadVer() uint64
GetLatestVer() uint64
GetVerMgr() *proto.VolVersionInfoList
UpdateRemoteCacheConfig(view *proto.SimpleVolView)
}
// Wrapper TODO rename. This name does not reflect what it is doing.
type Wrapper struct {
Lock sync.RWMutex
clusterName string
volName string
ClusterName string
VolName string
volType int
EnablePosixAcl bool
masters []string
@ -86,6 +87,7 @@ type Wrapper struct {
volStorageClass uint32
volAllowedStorageClass []uint32
volStatByClass map[uint32]*proto.StatOfStorageClass
HostsDelay sync.Map
}
// NewDataPartitionWrapper returns a new data partition wrapper.
@ -98,7 +100,7 @@ func NewDataPartitionWrapper(client SimpleClientInfo, volName string, masters []
w.stopC = make(chan struct{})
w.masters = masters
w.mc = masterSDK.NewMasterClient(masters, false)
w.volName = volName
w.VolName = volName
w.partitions = make(map[uint64]*DataPartition)
w.HostsStatus = make(map[string]bool)
w.preload = preload
@ -178,7 +180,7 @@ func (w *Wrapper) updateClusterInfo() (err error) {
return
}
log.LogInfof("UpdateClusterInfo: get cluster info: cluster(%v) localIP(%v)", info.Cluster, info.Ip)
w.clusterName = info.Cluster
w.ClusterName = info.Cluster
LocalIP = info.Ip
w.IsSnapshotEnabled = info.ClusterEnableSnapshot
return
@ -200,14 +202,14 @@ func (w *Wrapper) UpdateUidsView(view *proto.SimpleVolView) {
func (w *Wrapper) GetSimpleVolView() (err error) {
var view *proto.SimpleVolView
if view, err = w.mc.AdminAPI().GetVolumeSimpleInfo(w.volName); err != nil {
log.LogWarnf("GetSimpleVolView: get volume simple info fail: volume(%v) err(%v)", w.volName, err)
if view, err = w.mc.AdminAPI().GetVolumeSimpleInfo(w.VolName); err != nil {
log.LogWarnf("GetSimpleVolView: get volume simple info fail: volume(%v) err(%v)", w.VolName, err)
return
}
if view.Status == 1 {
log.LogWarnf("GetSimpleVolView: volume has been marked for deletion: volume(%v) status(%v - 0:normal/1:markDelete)",
w.volName, view.Status)
w.VolName, view.Status)
return proto.ErrVolNotExists
}
@ -253,7 +255,7 @@ func (w *Wrapper) uploadFlowInfoByTick(clientInfo SimpleClientInfo) {
func (w *Wrapper) update(clientInfo SimpleClientInfo) {
ticker := time.NewTicker(time.Minute)
taskFunc := func() {
w.updateSimpleVolView()
w.updateSimpleVolView(clientInfo)
w.updateDataPartition(false)
w.updateDataNodeStatus()
w.CheckPermission()
@ -277,8 +279,8 @@ func (w *Wrapper) UploadFlowInfo(clientInfo SimpleClientInfo, init bool) (work b
)
flowInfo, work = clientInfo.GetFlowInfo()
if limitRsp, err = w.mc.AdminAPI().UploadFlowInfo(w.volName, flowInfo); err != nil {
log.LogWarnf("UpdateSimpleVolView: get volume simple info fail: volume(%v) err(%v)", w.volName, err)
if limitRsp, err = w.mc.AdminAPI().UploadFlowInfo(w.VolName, flowInfo); err != nil {
log.LogWarnf("UpdateSimpleVolView: get volume simple info fail: volume(%v) err(%v)", w.VolName, err)
return
}
@ -296,7 +298,7 @@ func (w *Wrapper) UploadFlowInfo(clientInfo SimpleClientInfo, init bool) (work b
}
func (w *Wrapper) CheckPermission() {
if info, err := w.mc.UserAPI().AclOperation(w.volName, w.LocalIp, util.AclCheckIP); err != nil {
if info, err := w.mc.UserAPI().AclOperation(w.VolName, w.LocalIp, util.AclCheckIP); err != nil {
syslog.Println(err)
} else if !info.OK {
syslog.Println(err)
@ -308,20 +310,20 @@ func (w *Wrapper) updateVerlist(client SimpleClientInfo) (err error) {
if !w.IsSnapshotEnabled {
return
}
verList, err := w.mc.AdminAPI().GetVerList(w.volName)
verList, err := w.mc.AdminAPI().GetVerList(w.VolName)
if err != nil {
log.LogErrorf("CheckReadVerSeq: get cluster fail: err(%v)", err)
return err
}
if verList == nil {
msg := fmt.Sprintf("get verList nil, vol [%v] reqd seq [%v]", w.volName, w.verReadSeq)
msg := fmt.Sprintf("get verList nil, vol [%v] reqd seq [%v]", w.VolName, w.verReadSeq)
log.LogErrorf("action[CheckReadVerSeq] %v", msg)
return fmt.Errorf("%v", msg)
}
if w.verReadSeq > 0 {
if _, err = w.CheckReadVerSeq(w.volName, w.verConfReadSeq, verList); err != nil {
if _, err = w.CheckReadVerSeq(w.VolName, w.verConfReadSeq, verList); err != nil {
log.LogFatalf("updateSimpleVolView: readSeq abnormal %v", err)
}
return
@ -335,10 +337,10 @@ func (w *Wrapper) updateVerlist(client SimpleClientInfo) (err error) {
return
}
func (w *Wrapper) updateSimpleVolView() (err error) {
func (w *Wrapper) updateSimpleVolView(clientInfo SimpleClientInfo) (err error) {
var view *proto.SimpleVolView
if view, err = w.mc.AdminAPI().GetVolumeSimpleInfo(w.volName); err != nil {
log.LogWarnf("updateSimpleVolView: get volume simple info fail: volume(%v) err(%v)", w.volName, err)
if view, err = w.mc.AdminAPI().GetVolumeSimpleInfo(w.VolName); err != nil {
log.LogWarnf("updateSimpleVolView: get volume simple info fail: volume(%v) err(%v)", w.VolName, err)
return
}
@ -359,7 +361,7 @@ func (w *Wrapper) updateSimpleVolView() (err error) {
w.dpSelectorChanged = true
w.Lock.Unlock()
}
clientInfo.UpdateRemoteCacheConfig(view)
return nil
}
@ -418,7 +420,7 @@ func (w *Wrapper) updateDataPartitionByRsp(forceUpdate bool, refreshPolicy Refre
if forceUpdate || len(rwPartitionGroups) >= 1 {
log.LogInfof("updateDataPartition: volume(%v) refresh dpSelector, forceUpdate(%v) policy(%v), "+
"allDp(%v) allWritableDp(%v), SsdDp(%v) SsdWritableDp(%v), hddDp(%v) hddWritableDp(%v)",
w.volName, forceUpdate, refreshPolicy, len(DataPartitions), len(rwPartitionGroups),
w.VolName, forceUpdate, refreshPolicy, len(DataPartitions), len(rwPartitionGroups),
ssdDpCount, ssdDpWritableCount, hddDpCount, hddDpWritableCount)
w.refreshDpSelector(refreshPolicy, rwPartitionGroups)
} else {
@ -427,7 +429,7 @@ func (w *Wrapper) updateDataPartitionByRsp(forceUpdate bool, refreshPolicy Refre
} else {
err = errors.New("updateDataPartition: no writable data partition")
log.LogWarnf("updateDataPartition: no enough writable data partitions, volume(%v) with %v rw partitions(%v all), forceUpdate(%v)",
w.volName, len(rwPartitionGroups), len(DataPartitions), forceUpdate)
w.VolName, len(rwPartitionGroups), len(DataPartitions), forceUpdate)
}
}
@ -440,12 +442,12 @@ func (w *Wrapper) updateDataPartition(isInit bool) (err error) {
return
}
var dpv *proto.DataPartitionsView
if dpv, err = w.mc.ClientAPI().EncodingGzip().GetDataPartitions(w.volName); err != nil {
log.LogErrorf("updateDataPartition: get data partitions fail: volume(%v) err(%v)", w.volName, err)
if dpv, err = w.mc.ClientAPI().EncodingGzip().GetDataPartitions(w.VolName); err != nil {
log.LogErrorf("updateDataPartition: get data partitions fail: volume(%v) err(%v)", w.VolName, err)
return
}
log.LogInfof("updateDataPartition: get data partitions: volume(%v) partitions(%v) VolReadOnly(%v)",
w.volName, len(dpv.DataPartitions), dpv.VolReadOnly)
w.VolName, len(dpv.DataPartitions), dpv.VolReadOnly)
forceUpdate := false
if isInit || dpv.VolReadOnly {
@ -457,7 +459,7 @@ func (w *Wrapper) updateDataPartition(isInit bool) (err error) {
for _, st := range dpv.StatByClass {
m[st.StorageClass] = st
log.LogInfof("updateDataPartition: get storage class stat info: volume(%v) stat(%s) VolReadOnly(%v)",
w.volName, st.String(), dpv.VolReadOnly)
w.VolName, st.String(), dpv.VolReadOnly)
}
w.volStatByClass = m
w.Lock.Unlock()
@ -485,14 +487,14 @@ func (w *Wrapper) UpdateDataPartition() (err error) {
// updateDataPartition which may not take effect if nginx be placed for reduce the pressure of master
func (w *Wrapper) getDataPartitionFromMaster(dpId uint64) (err error) {
var dpInfo *proto.DataPartitionInfo
if dpInfo, err = w.mc.AdminAPI().GetDataPartition(w.volName, dpId); err != nil {
if dpInfo, err = w.mc.AdminAPI().GetDataPartition(w.VolName, dpId); err != nil {
log.LogErrorf("getDataPartitionFromMaster: get data partitions fail: volume(%v) dpId(%v) err(%v)",
w.volName, dpId, err)
w.VolName, dpId, err)
return
}
log.LogInfof("getDataPartitionFromMaster: get data partitions: volume(%v), dpId(%v) mediaType(%v)",
w.volName, dpId, proto.MediaTypeString(dpInfo.MediaType))
w.VolName, dpId, proto.MediaTypeString(dpInfo.MediaType))
var leaderAddr string
for _, replica := range dpInfo.Replicas {
if replica.IsLeader {
@ -642,7 +644,7 @@ func (w *Wrapper) CheckReadVerSeq(volName string, verReadSeq uint64, verList *pr
// WarningMsg returns the warning message that contains the cluster name.
func (w *Wrapper) WarningMsg() string {
return fmt.Sprintf("%s_client_warning", w.clusterName)
return fmt.Sprintf("%s_client_warning", w.ClusterName)
}
func (w *Wrapper) updateDataNodeStatus() (err error) {

18
sdk/meta/api.go
View File

@ -238,7 +238,14 @@ func (mw *MetaWrapper) create_ll(parentID uint64, name string, mode, uid, gid ui
mp *MetaPartition
rwPartitions []*MetaPartition
)
defer func() {
if info != nil && mw.RemoteCacheBloom != nil {
cacheBloom := mw.RemoteCacheBloom()
if cacheBloom.TestUint64(parentID) {
cacheBloom.AddUint64(info.Inode)
}
}
}()
parentMP := mw.getPartitionByInode(parentID)
if parentMP == nil {
log.LogErrorf("Create_ll: No parent partition, parentID(%v)", parentID)
@ -360,6 +367,15 @@ create_dentry:
}
func (mw *MetaWrapper) Lookup_ll(parentID uint64, name string) (inode uint64, mode uint32, err error) {
defer func() {
if err == nil && mw.RemoteCacheBloom != nil {
cacheBloom := mw.RemoteCacheBloom()
if cacheBloom.TestUint64(parentID) {
cacheBloom.AddUint64(inode)
}
}
}()
parentMP := mw.getPartitionByInode(parentID)
if parentMP == nil {
log.LogErrorf("Lookup_ll: No parent partition, parentID(%v) name(%v)", parentID, name)

3
sdk/meta/meta.go
View File

@ -29,6 +29,7 @@ import (
masterSDK "github.com/cubefs/cubefs/sdk/master"
"github.com/cubefs/cubefs/util"
"github.com/cubefs/cubefs/util/auth"
"github.com/cubefs/cubefs/util/bloom"
"github.com/cubefs/cubefs/util/btree"
"github.com/cubefs/cubefs/util/errors"
"github.com/cubefs/cubefs/util/log"
@ -192,6 +193,8 @@ type MetaWrapper struct {
CacheDpStorageClass uint32
InnerReq bool
FollowerRead bool
RemoteCacheBloom func() *bloom.BloomFilter
}
type uniqidRange struct {

8
sdk/meta/operation.go
View File

@ -1142,6 +1142,14 @@ func (mw *MetaWrapper) readDir(mp *MetaPartition, parentID uint64) (status int,
bgTime := stat.BeginStat()
defer func() {
stat.EndStat("readDir", err, bgTime, 1)
if err == nil && mw.RemoteCacheBloom != nil {
cacheBloom := mw.RemoteCacheBloom()
if cacheBloom.TestUint64(parentID) {
for _, c := range children {
cacheBloom.AddUint64(c.Inode)
}
}
}
}()
req := &proto.ReadDirRequest{