gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 02:00:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix(client): client data modification is prohibited if renewal failed

Browse Source 
close:#22676203

Signed-off-by: chihe <chihe@oppo.com>
This commit is contained in:
chihe 2024-10-14 09:41:00 +08:00 committed by AmazingChi
parent 0eb8f4127d
commit 0a3f53cdfa
22 changed files with 217 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
client/fs/super.go
View File

@ -249,6 +249,7 @@ func NewSuper(opt *proto.MountOptions) (s *Super, err error) {
VolStorageClass: opt.VolStorageClass,
VolAllowedStorageClass: opt.VolAllowedStorageClass,
VolCacheDpStorageClass: s.cacheDpStorageClass,
OnForbiddenMigration: s.mw.ForbiddenMigration,
}
s.ec, err = stream.NewExtentClient(extentConfig)

30
gosdk/cfs_client.go
View File

@ -256,20 +256,22 @@ func (c *Client) Start() (err error) {
}
var ec *stream.ExtentClient
if ec, err = stream.NewExtentClient(&stream.ExtentConfig{
Volume: c.cfg.VolName,
Masters: masters,
FollowerRead: c.cfg.FollowerRead,
OnAppendExtentKey: mw.AppendExtentKey,
OnGetExtents: mw.GetExtents,
OnTruncate: mw.Truncate,
BcacheEnable: c.cfg.EnableBcache,
OnLoadBcache: c.bc.Get,
OnCacheBcache: c.bc.Put,
OnEvictBcache: c.bc.Evict,
DisableMetaCache: true,
VolStorageClass: c.volStorageClass,
VolAllowedStorageClass: c.volAllowedStorageClass,
VolCacheDpStorageClass: c.volCacheDpStorageClass,
Volume: c.cfg.VolName,
Masters: masters,
FollowerRead: c.cfg.FollowerRead,
OnAppendExtentKey: mw.AppendExtentKey,
OnGetExtents: mw.GetExtents,
OnTruncate: mw.Truncate,
BcacheEnable: c.cfg.EnableBcache,
OnLoadBcache: c.bc.Get,
OnCacheBcache: c.bc.Put,
OnEvictBcache: c.bc.Evict,
DisableMetaCache: true,
VolStorageClass: c.volStorageClass,
VolAllowedStorageClass: c.volAllowedStorageClass,
VolCacheDpStorageClass: c.volCacheDpStorageClass,
OnRenewalForbiddenMigration: mw.RenewalForbiddenMigration,
OnForbiddenMigration: mw.ForbiddenMigration,
}); err != nil {
log.LogErrorf("newClient NewExtentClient failed(%v)", err)
return

1
lcnode/lc_scanner.go
View File

@ -147,6 +147,7 @@ func NewS3Scanner(adminTask *proto.AdminTask, l *LcNode) (*LcScanner, error) {
VolStorageClass: volumeInfo.VolStorageClass,
VolAllowedStorageClass: volumeInfo.AllowedStorageClass,
VolCacheDpStorageClass: volumeInfo.CacheDpStorageClass,
OnForbiddenMigration: metaWrapper.ForbiddenMigration,
}
log.LogInfof("[NewS3Scanner] extentConfig: vol(%v) volStorageClass(%v) allowedStorageClass(%v), followerRead(%v)",
extentConfig.Volume, extentConfig.VolStorageClass, extentConfig.VolAllowedStorageClass, extentConfig.FollowerRead)

1
lcnode/server.go
View File

@ -489,6 +489,7 @@ func (l *LcNode) httpServiceGetFile(w http.ResponseWriter, r *http.Request) {
OnRenewalForbiddenMigration: metaWrapper.RenewalForbiddenMigration,
VolStorageClass: uint32(vsc),
VolAllowedStorageClass: asc,
OnForbiddenMigration: metaWrapper.ForbiddenMigration,
}
var extentClient *stream.ExtentClient
if extentClient, err = stream.NewExtentClient(extentConfig); err != nil {

32
libsdk/libsdk.go
View File

@ -1483,21 +1483,23 @@ func (c *client) start() (err error) {
}
var ec *stream.ExtentClient
if ec, err = stream.NewExtentClient(&stream.ExtentConfig{
Volume: c.volName,
Masters: masters,
FollowerRead: c.followerRead,
OnAppendExtentKey: mw.AppendExtentKey,
OnSplitExtentKey: mw.SplitExtentKey,
OnGetExtents: mw.GetExtents,
OnTruncate: mw.Truncate,
BcacheEnable: c.enableBcache,
OnLoadBcache: c.bc.Get,
OnCacheBcache: c.bc.Put,
OnEvictBcache: c.bc.Evict,
DisableMetaCache: true,
VolStorageClass: c.volStorageClass,
VolAllowedStorageClass: c.volAllowedStorageClass,
VolCacheDpStorageClass: c.cacheDpStorageClass,
Volume: c.volName,
Masters: masters,
FollowerRead: c.followerRead,
OnAppendExtentKey: mw.AppendExtentKey,
OnSplitExtentKey: mw.SplitExtentKey,
OnGetExtents: mw.GetExtents,
OnTruncate: mw.Truncate,
BcacheEnable: c.enableBcache,
OnLoadBcache: c.bc.Get,
OnCacheBcache: c.bc.Put,
OnEvictBcache: c.bc.Evict,
DisableMetaCache: true,
VolStorageClass: c.volStorageClass,
VolAllowedStorageClass: c.volAllowedStorageClass,
VolCacheDpStorageClass: c.cacheDpStorageClass,
OnRenewalForbiddenMigration: mw.RenewalForbiddenMigration,
OnForbiddenMigration: mw.ForbiddenMigration,
}); err != nil {
log.LogErrorf("newClient NewExtentClient failed(%v)", err)
return

2
metanode/const.go
View File

@ -103,6 +103,8 @@ type (
SetCreateTimeRequest = proto.SetCreateTimeRequest
DeleteMigrationExtentKeyRequest = proto.DeleteMigrationExtentKeyRequest
ForbiddenMigrationRequest = proto.ForbiddenMigrationRequest
)
// op code should be fixed, order change will cause raft fsm log apply fail

2
metanode/manager.go
View File

@ -389,6 +389,8 @@ func (m *metadataManager) HandleMetadataOperation(conn net.Conn, p *Packet, remo
err = m.opMetaUpdateExtentKeyAfterMigration(conn, p, remoteAddr)
case proto.OpDeleteMigrationExtentKey:
err = m.opDeleteMigrationExtentKey(conn, p, remoteAddr)
case proto.OpMetaForbiddenMigration:
err = m.opMetaForbiddenMigration(conn, p, remoteAddr)
default:
err = fmt.Errorf("%s unknown Opcode: %d, reqId: %d", remoteAddr,
p.Opcode, p.GetReqID())

32
metanode/manager_op.go
View File

@ -2963,3 +2963,35 @@ func (m *metadataManager) opDeleteMigrationExtentKey(conn net.Conn, p *Packet,
"resp body: %s", remoteAddr, p.GetReqID(), req, p.GetResultMsg(), p.Data)
return
}
func (m *metadataManager) opMetaForbiddenMigration(conn net.Conn, p *Packet,
remoteAddr string,
) (err error) {
req := &ForbiddenMigrationRequest{}
if err = json.Unmarshal(p.Data, req); err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
m.respondToClientWithVer(conn, p)
err = errors.NewErrorf("[%v] req: %v, resp: %v", p.GetOpMsgWithReqAndResult(), req, err.Error())
return
}
mp, err := m.getPartition(req.PartitionID)
if err != nil {
p.PacketErrorWithBody(proto.OpErr, ([]byte)(err.Error()))
m.respondToClientWithVer(conn, p)
err = errors.NewErrorf("[%v] req: %v, resp: %v", p.GetOpMsgWithReqAndResult(), req, err.Error())
return
}
if !m.serveProxy(conn, mp, p) {
return
}
if err = m.checkMultiVersionStatus(mp, p); err != nil {
err = errors.NewErrorf("[%v],req[%v],err[%v]", p.GetOpMsgWithReqAndResult(), req, string(p.Data))
return
}
mp.ForbiddenMigration(req, p, remoteAddr)
m.updatePackRspSeq(mp, p)
m.respondToClientWithVer(conn, p)
log.LogDebugf("%s [opMetaForbiddenMigration] req: %d - %v, resp body: %v, "+
"resp body: %s", remoteAddr, p.GetReqID(), req, p.GetResultMsg(), p.Data)
return
}

1
metanode/partition.go
View File

@ -153,6 +153,7 @@ type OpInode interface {
InodeGetWithEk(req *InodeGetReq, p *Packet) (err error)
SetCreateTime(req *SetCreateTimeRequest, reqData []byte, p *Packet) (err error) // for debugging
DeleteMigrationExtentKey(req *proto.DeleteMigrationExtentKeyRequest, p *Packet, remoteAddr string) (err error)
ForbiddenMigration(req *proto.ForbiddenMigrationRequest, p *Packet, remoteAddr string) (err error)
}
type OpExtend interface {

35
metanode/partition_op_inode.go
View File

@ -1453,3 +1453,38 @@ func (mp *metaPartition) innerCleanMigrationExtentKeyAfterError(ino *Inode) {
log.LogWarnf("action[innerCleanMigrationExtentKeyAfterError] mpId(%v) ino(%v) storageClass(%v) migrateStorageClass(%v) submit",
mp.config.PartitionId, ino.Inode, ino.StorageClass, ino.HybridCouldExtentsMigration.storageClass)
}
func (mp *metaPartition) ForbiddenMigration(req *proto.ForbiddenMigrationRequest,
p *Packet, remoteAddr string,
) (err error) {
log.LogDebugf("action[ForbiddenMigration] inode[%v] ", req.Inode)
// note:don't need set reqSeq, extents get be done in next step
ino := NewInode(req.Inode, 0)
if item := mp.inodeTree.Get(ino); item == nil {
err = fmt.Errorf("mp %v inode %v reqeust cann't found", mp.config.PartitionId, ino)
log.LogErrorf("action[ForbiddenMigration] %v", err)
p.PacketErrorWithBody(proto.OpNotExistErr, []byte(err.Error()))
return
} else {
ino.UpdateHybridCloudParams(item.(*Inode))
}
var (
val []byte
reply []byte
status = proto.OpOk
)
val, err = ino.Marshal()
if err != nil {
p.PacketErrorWithBody(proto.OpErr, []byte(err.Error()))
return
}
_, err = mp.submit(opFSMForbiddenMigrationInode, val)
if err != nil {
status = proto.OpErr
reply = []byte(err.Error())
}
// mark inode as ForbiddenMigration
p.PacketErrorWithBody(status, reply)
return
}

1
objectnode/fs_volume.go
View File

@ -3057,6 +3057,7 @@ func NewVolume(config *VolumeConfig) (*Volume, error) {
VolStorageClass: volumeInfo.VolStorageClass,
VolAllowedStorageClass: volumeInfo.AllowedStorageClass,
VolCacheDpStorageClass: volumeInfo.CacheDpStorageClass,
OnForbiddenMigration: metaWrapper.ForbiddenMigration,
}
if proto.IsCold(volumeInfo.VolType) || proto.IsStorageClassBlobStore(volumeInfo.VolStorageClass) {

22
preload/sdk/preloadsdk.go
View File

@ -147,16 +147,18 @@ func NewClient(config PreloadConfig) *PreLoadClient {
}
if ec, err = stream.NewExtentClient(&stream.ExtentConfig{
Volume: config.Volume,
Masters: config.Masters,
Preload: true,
OnAppendExtentKey: mw.AppendExtentKey,
OnSplitExtentKey: mw.SplitExtentKey,
OnGetExtents: mw.GetExtents,
OnTruncate: mw.Truncate,
VolStorageClass: view.VolStorageClass,
VolAllowedStorageClass: view.AllowedStorageClass,
VolCacheDpStorageClass: view.CacheDpStorageClass,
Volume: config.Volume,
Masters: config.Masters,
Preload: true,
OnAppendExtentKey: mw.AppendExtentKey,
OnSplitExtentKey: mw.SplitExtentKey,
OnGetExtents: mw.GetExtents,
OnTruncate: mw.Truncate,
VolStorageClass: view.VolStorageClass,
VolAllowedStorageClass: view.AllowedStorageClass,
VolCacheDpStorageClass: view.CacheDpStorageClass,
OnRenewalForbiddenMigration: mw.RenewalForbiddenMigration,
OnForbiddenMigration: mw.ForbiddenMigration,
}); err != nil {
log.LogErrorf("newClient NewExtentClient failed(%v)", err)
return nil

2
proto/admin_proto.go
View File

@ -1516,6 +1516,6 @@ func GetStorageClassByMediaType(mediaType uint32) (storageClass uint32) {
return
}
const ForbiddenMigrationRenewalPeriod = 2 * time.Minute
const ForbiddenMigrationRenewalPeriod = 1 * time.Hour
// const ForbiddenMigrationRenewalPeriod = 10 * time.Second // for debug

5
proto/fs_proto.go
View File

@ -1114,3 +1114,8 @@ type DeleteMigrationExtentKeyRequest struct {
Inode uint64 `json:"ino"`
RequestExtend
}
type ForbiddenMigrationRequest struct {
PartitionID uint64 `json:"pid"`
Inode uint64 `json:"ino"`
}

3
proto/packet.go
View File

@ -276,6 +276,7 @@ const (
OpLeaseOccupiedByOthers uint8 = 0x86
OpLeaseGenerationNotMatch uint8 = 0x87
OpWriteOpOfProtoVerForbidden uint8 = 0x88
OpMetaForbiddenMigration uint8 = 0x89
)
const (
@ -673,6 +674,8 @@ func (p *Packet) GetOpMsg() (m string) {
m = "OpMetaUpdateExtentKeyAfterMigration"
case OpDeleteMigrationExtentKey:
m = "OpDeleteMigrationExtentKey"
case OpMetaForbiddenMigration:
m = "OpMetaForbiddenMigration"
default:
m = fmt.Sprintf("op:%v not found", p.Opcode)
}

6
sdk/data/stream/extent_client.go
View File

@ -67,6 +67,7 @@ type (
CacheBcacheFunc func(key string, buf []byte) error
EvictBacheFunc func(key string) error
RenewalForbiddenMigrationFunc func(inode uint64) error
ForbiddenMigrationFunc func(inode uint64) error
)
const (
@ -144,6 +145,7 @@ type ExtentConfig struct {
StreamRetryTimeout int
OnRenewalForbiddenMigration RenewalForbiddenMigrationFunc
OnForbiddenMigration ForbiddenMigrationFunc
VolStorageClass uint32
VolAllowedStorageClass []uint32
@ -188,6 +190,7 @@ type ExtentClient struct {
inflightL1BigBlock int32
multiVerMgr *MultiVerMgr
renewalForbiddenMigration RenewalForbiddenMigrationFunc
forbiddenMigration ForbiddenMigrationFunc
CacheDpStorageClass uint32
}
@ -313,8 +316,9 @@ retry:
client.disableMetaCache = config.DisableMetaCache
client.renewalForbiddenMigration = config.OnRenewalForbiddenMigration
client.CacheDpStorageClass = config.VolCacheDpStorageClass
client.forbiddenMigration = config.OnForbiddenMigration
if config.StreamRetryTimeout <= 0 {
if config.StreamRetryTimeout <= 0 || config.StreamRetryTimeout >= 600 {
client.streamRetryTimeout = StreamSendMaxTimeout
} else {
client.streamRetryTimeout = time.Duration(config.StreamRetryTimeout) * time.Second

2
sdk/data/stream/extent_handler.go
View File

@ -835,7 +835,7 @@ func (eh *ExtentHandler) setRecovery() bool {
func (eh *ExtentHandler) setError() bool {
// log.LogDebugf("action[ExtentHandler.setError] stack (%v)", string(debug.Stack()))
if proto.IsHot(eh.stream.client.volumeType) || proto.IsStorageClassReplica(eh.storageClass) {
atomic.StoreInt32(&eh.stream.status, StreamerError)
eh.stream.setError()
}
return atomic.CompareAndSwapInt32(&eh.status, ExtentStatusRecovery, ExtentStatusError)
}

10
sdk/data/stream/stream_reader.go
View File

@ -81,6 +81,13 @@ func NewStreamer(client *ExtentClient, inode uint64, openForWrite, isCache bool)
s.openForWrite = openForWrite
s.isCache = isCache
log.LogDebugf("NewStreamer: streamer(%v)", s)
if s.openForWrite {
err := s.client.forbiddenMigration(s.inode)
if err != nil {
log.LogWarnf("ino(%v) forbiddenMigration failed err %v", s.inode, err.Error())
s.setError()
}
}
go s.server()
go s.asyncBlockCache()
return s
@ -92,7 +99,8 @@ func (s *Streamer) SetParentInode(inode uint64) {
// String returns the string format of the streamer.
func (s *Streamer) String() string {
return fmt.Sprintf("Streamer{ino(%v), refcnt(%v), isOpen(%v), inflight(%v), eh(%v) addr(%p)}", s.inode, s.refcnt, s.isOpen, len(s.request), s.handler, s)
return fmt.Sprintf("Streamer{ino(%v), refcnt(%v), isOpen(%v) openForWrite(%v), inflight(%v), eh(%v) addr(%p)}",
s.inode, s.refcnt, s.isOpen, s.openForWrite, len(s.request), s.handler, s)
}
// TODO should we call it RefreshExtents instead?

17
sdk/data/stream/stream_writer.go
View File

@ -200,9 +200,8 @@ func (s *Streamer) server() {
t := time.NewTicker(2 * time.Second)
defer t.Stop()
// only file opened with write request needs to forbidden migration
renewalTimer := time.NewTicker(proto.ForbiddenMigrationRenewalPeriod / 3)
renewalTimer := time.NewTicker(proto.ForbiddenMigrationRenewalPeriod / 5)
defer renewalTimer.Stop()
log.LogDebugf("start server: streamer(%v)", s)
for {
select {
@ -245,6 +244,7 @@ func (s *Streamer) server() {
err := s.client.renewalForbiddenMigration(s.inode)
if err != nil {
log.LogWarnf("ino(%v) renewalForbiddenMigration failed err %v", s.inode, err.Error())
s.setError()
}
}
}
@ -325,7 +325,9 @@ func (s *Streamer) write(data []byte, offset, size, flags int, checkFunc func()
direct bool
retryTimes int8
)
if atomic.LoadInt32(&s.status) >= StreamerError {
return 0, errors.New(fmt.Sprintf("IssueWriteRequest: stream writer in error status, ino(%v)", s.inode))
}
if flags&proto.FlagsSyncWrite != 0 {
direct = true
}
@ -961,7 +963,7 @@ func (s *Streamer) traverse() (err error) {
log.LogWarnf("Streamer traverse abort: appendExtentKey failed, eh(%v) err(%v)", eh, err)
// set the streamer to error status to avoid further writes
if err == syscall.EIO {
atomic.StoreInt32(&eh.stream.status, StreamerError)
eh.stream.setError()
}
return
}
@ -1058,6 +1060,9 @@ func (s *Streamer) abort() {
}
func (s *Streamer) truncate(size int, fullPath string) error {
if atomic.LoadInt32(&s.status) >= StreamerError {
return errors.New(fmt.Sprintf("IssueWriteRequest: stream writer in error status, ino(%v)", s.inode))
}
err := s.closeOpenHandler()
if err != nil {
return err
@ -1091,3 +1096,7 @@ func (s *Streamer) updateVer(verSeq uint64) (err error) {
func (s *Streamer) tinySizeLimit() int {
return util.DefaultTinySizeLimit
}
func (s *Streamer) setError() {
atomic.StoreInt32(&s.status, StreamerError)
}

14
sdk/meta/api.go
View File

@ -2957,3 +2957,17 @@ func (mw *MetaWrapper) ListVols(keywords string) (volsInfo []*proto.VolInfo, err
volsInfo, err = mw.mc.AdminAPI().ListVols(keywords)
return
}
func (mw *MetaWrapper) ForbiddenMigration(inode uint64) error {
mp := mw.getPartitionByInode(inode)
if mp == nil {
log.LogErrorf("[ForbiddenMigration]: No inode partition, ino(%v)", inode)
return syscall.ENOENT
}
status, err := mw.forbiddenMigration(mp, inode)
if err != nil || status != statusOK {
return statusToErrno(status)
}
return nil
}

3
sdk/meta/conn.go
View File

@ -28,6 +28,7 @@ import (
const (
SendRetryLimit = 200 // times
SendRetryInterval = 100 // ms
MaxRetryTime = 10 * 60
)
type MetaConn struct {
@ -68,6 +69,8 @@ func (mw *MetaWrapper) sendToMetaPartition(mp *MetaPartition, req *proto.Packet)
var sendTimeLimit int
if mw.metaSendTimeout < 20 {
sendTimeLimit = 20 * 1000 // ms
} else if mw.metaSendTimeout >= MaxRetryTime {
sendTimeLimit = MaxRetryTime * 1000
} else {
sendTimeLimit = int(mw.metaSendTimeout) * 1000 // ms
}

42
sdk/meta/operation.go
View File

@ -3135,3 +3135,45 @@ func (mw *MetaWrapper) deleteMigrationExtentKey(mp *MetaPartition, inode uint64,
log.LogDebugf("deleteMigrationExtentKey exit: packet(%v) mp(%v) req(%v)", packet, mp, *req)
return statusOK, nil
}
func (mw *MetaWrapper) forbiddenMigration(mp *MetaPartition, inode uint64) (status int, err error) {
bgTime := stat.BeginStat()
defer func() {
stat.EndStat("forbiddenMigration", err, bgTime, 1)
}()
req := &proto.ForbiddenMigrationRequest{
PartitionID: mp.PartitionID,
Inode: inode,
}
packet := proto.NewPacketReqID()
packet.Opcode = proto.OpMetaForbiddenMigration
packet.PartitionID = mp.PartitionID
err = packet.MarshalData(req)
if err != nil {
log.LogErrorf("forbiddenMigration: ino(%v) err(%v)", inode, err)
return
}
metric := exporter.NewTPCnt(packet.GetOpMsg())
defer func() {
metric.SetWithLabels(err, map[string]string{exporter.Vol: mw.volname})
}()
packet, err = mw.sendToMetaPartition(mp, packet)
if err != nil {
log.LogErrorf("forbiddenMigration: packet(%v) mp(%v) req(%v) err(%v)", packet, mp, *req, err)
return
}
status = parseStatus(packet.ResultCode)
if status != statusOK {
err = errors.New(packet.GetResultMsg())
log.LogErrorf("forbiddenMigration: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg())
return
}
log.LogDebugf("forbiddenMigration exit: packet(%v) mp(%v) req(%v)", packet, mp, *req)
return statusOK, nil
}