diff --git a/cli/cmd/const.go b/cli/cmd/const.go index ad82e2d48..e25df7158 100644 --- a/cli/cmd/const.go +++ b/cli/cmd/const.go @@ -171,6 +171,7 @@ const ( CliFlagRemoteCacheMaxFileSizeGB = "remoteCacheMaxFileSizeGB" CliFlagRemoteCacheOnlyForNotSSD = "remoteCacheOnlyForNotSSD" CliFlagRemoteCacheMultiRead = "remoteCacheMultiRead" + CliFlagFlashNodeTimeoutCount = "flashNodeTimeoutCount" // CliFlagSetDataPartitionCount = "count" use dp-count instead diff --git a/cli/cmd/fmt.go b/cli/cmd/fmt.go index 17b0efe57..917c03ba7 100644 --- a/cli/cmd/fmt.go +++ b/cli/cmd/fmt.go @@ -307,6 +307,7 @@ func formatSimpleVolView(svv *proto.SimpleVolView) string { sb.WriteString(fmt.Sprintf(" remoteCacheMaxFileSizeGB : %v G\n", svv.RemoteCacheMaxFileSizeGB)) sb.WriteString(fmt.Sprintf(" remoteCacheOnlyForNotSSD : %v\n", svv.RemoteCacheOnlyForNotSSD)) sb.WriteString(fmt.Sprintf(" remoteCacheMultiRead : %v\n", svv.RemoteCacheMultiRead)) + sb.WriteString(fmt.Sprintf(" flashNodeTimeoutCount : %v\n", svv.FlashNodeTimeoutCount)) return sb.String() } diff --git a/cli/cmd/vol.go b/cli/cmd/vol.go index 2e28ec3ff..099dfadd9 100644 --- a/cli/cmd/vol.go +++ b/cli/cmd/vol.go @@ -120,6 +120,7 @@ const ( cmdVolDefaultRemoteCacheTTL = proto.DefaultRemoteCacheTTL cmdVolDefaultRemoteCacheReadTimeout = proto.DefaultRemoteCacheClientReadTimeout cmdVolDefaultRemoteCacheMaxFileSizeGB = 128 + cmdVolDefaultFlashNodeTimeoutCount = 5 ) func newVolCreateCmd(client *master.MasterClient) *cobra.Command { @@ -163,6 +164,7 @@ func newVolCreateCmd(client *master.MasterClient) *cobra.Command { var optRemoteCacheMaxFileSizeGB int64 var optRemoteCacheOnlyForNotSSD string var optRemoteCacheMultiRead string + var optFlashNodeTimeoutCount int64 cmd := &cobra.Command{ Use: cmdVolCreateUse, @@ -229,6 +231,11 @@ func newVolCreateCmd(client *master.MasterClient) *cobra.Command { return } + if optFlashNodeTimeoutCount <= 0 { + err = fmt.Errorf("param flashNodeTimeoutCount(%v) must greater than 0", optFlashNodeTimeoutCount) + return + } + // ask user for confirm if !optYes { stdout("Create a new volume:\n") @@ -272,6 +279,7 @@ func newVolCreateCmd(client *master.MasterClient) *cobra.Command { stdout(" remoteCacheMaxFileSizeGB : %v G\n", optRemoteCacheMaxFileSizeGB) stdout(" remoteCacheOnlyForNotSSD : %v\n", optRemoteCacheOnlyForNotSSD) stdout(" remoteCacheMultiRead : %v\n", optRemoteCacheMultiRead) + stdout(" flashNodeTimeoutCount : %v\n", optFlashNodeTimeoutCount) stdout("\nConfirm (yes/no)[yes]: ") var userConfirm string @@ -290,7 +298,8 @@ func newVolCreateCmd(client *master.MasterClient) *cobra.Command { optCacheLowWater, optCacheLRUInterval, dpReadOnlyWhenVolFull, optTxMask, optTxTimeout, optTxConflictRetryNum, optTxConflictRetryInterval, optEnableQuota, clientIDKey, optVolStorageClass, optAllowedStorageClass, optMetaFollowerRead, optMaximallyRead, - optRcEnable, optRcAutoPrepare, optRcPath, optRcTTL, optRcReadTimeout, optRemoteCacheMaxFileSizeGB, optRemoteCacheOnlyForNotSSD, optRemoteCacheMultiRead) + optRcEnable, optRcAutoPrepare, optRcPath, optRcTTL, optRcReadTimeout, optRemoteCacheMaxFileSizeGB, + optRemoteCacheOnlyForNotSSD, optRemoteCacheMultiRead, optFlashNodeTimeoutCount) if err != nil { err = fmt.Errorf("Create volume failed case:\n%v\n", err) return @@ -342,6 +351,7 @@ func newVolCreateCmd(client *master.MasterClient) *cobra.Command { cmd.Flags().Int64Var(&optRemoteCacheMaxFileSizeGB, CliFlagRemoteCacheMaxFileSizeGB, cmdVolDefaultRemoteCacheMaxFileSizeGB, "Remote cache max file size[Unit: GB](must > 0)") cmd.Flags().StringVar(&optRemoteCacheOnlyForNotSSD, CliFlagRemoteCacheOnlyForNotSSD, "false", "Remote cache only for not ssd(true|false)") cmd.Flags().StringVar(&optRemoteCacheMultiRead, CliFlagRemoteCacheMultiRead, "false", "Remote cache follower read(true|false)") + cmd.Flags().Int64Var(&optFlashNodeTimeoutCount, CliFlagFlashNodeTimeoutCount, cmdVolDefaultFlashNodeTimeoutCount, "FlashNode timeout count, flashNode will be removed by client if it's timeout count exceeds this value") return cmd } @@ -378,6 +388,7 @@ func newVolUpdateCmd(client *master.MasterClient) *cobra.Command { var optRemoteCacheMaxFileSizeGB int64 var optRemoteCacheOnlyForNotSSD string var optRemoteCacheFollowerRead string + var optFlashNodeTimeoutCount int64 var optYes bool var optTxMask string @@ -920,6 +931,11 @@ func newVolUpdateCmd(client *master.MasterClient) *cobra.Command { return } + if cmd.Flags().Changed(CliFlagFlashNodeTimeoutCount) && optFlashNodeTimeoutCount <= 0 { + err = fmt.Errorf("param flashNodeTimeoutCount(%v) must greater than 0", optFlashNodeTimeoutCount) + return + } + for _, rcOpt := range []struct { val, opt interface{} name string @@ -932,6 +948,7 @@ func newVolUpdateCmd(client *master.MasterClient) *cobra.Command { {&vv.RemoteCacheMaxFileSizeGB, optRemoteCacheMaxFileSizeGB, CliFlagRemoteCacheMaxFileSizeGB}, {&vv.RemoteCacheOnlyForNotSSD, optRemoteCacheOnlyForNotSSD, CliFlagRemoteCacheOnlyForNotSSD}, {&vv.RemoteCacheMultiRead, optRemoteCacheFollowerRead, CliFlagRemoteCacheMultiRead}, + {&vv.FlashNodeTimeoutCount, optFlashNodeTimeoutCount, CliFlagFlashNodeTimeoutCount}, } { if err = checkChangedFlag(rcOpt.val, rcOpt.opt, rcOpt.name); err != nil { return @@ -1019,6 +1036,7 @@ func newVolUpdateCmd(client *master.MasterClient) *cobra.Command { cmd.Flags().Int64Var(&optRemoteCacheMaxFileSizeGB, CliFlagRemoteCacheMaxFileSizeGB, 0, "Remote cache max file size[Unit: GB](must > 0)") cmd.Flags().StringVar(&optRemoteCacheOnlyForNotSSD, CliFlagRemoteCacheOnlyForNotSSD, "", "Remote cache only for not ssd(true|false), default false") cmd.Flags().StringVar(&optRemoteCacheFollowerRead, CliFlagRemoteCacheMultiRead, "", "Remote cache follower read(true|false), default true") + cmd.Flags().Int64Var(&optFlashNodeTimeoutCount, CliFlagFlashNodeTimeoutCount, 0, "FlashNode timeout count, flashNode will be removed by client if it's timeout count exceeds this value") return cmd } diff --git a/flashnode/flashnode_op.go b/flashnode/flashnode_op.go index bd150ad4a..f8b20ab93 100644 --- a/flashnode/flashnode_op.go +++ b/flashnode/flashnode_op.go @@ -466,6 +466,7 @@ func (f *FlashNode) doStreamReadRequest(ctx context.Context, conn net.Conn, req func (f *FlashNode) opCachePrepare(conn net.Conn, p *proto.Packet) (err error) { action := "action[opCachePrepare]" var volume string + bgTime := stat.BeginStat() defer func() { if err != nil { log.LogErrorf("%s volume:[%s] %s", action, volume, @@ -475,6 +476,7 @@ func (f *FlashNode) opCachePrepare(conn net.Conn, p *proto.Packet) (err error) { log.LogErrorf("%s write to conn %v", action, e) } } + stat.EndStat("FlashNode:opCachePrepare", err, bgTime, 1) }() req := new(proto.CachePrepareRequest) diff --git a/master/api_args_parse.go b/master/api_args_parse.go index 216ad9ae1..0362d2a53 100644 --- a/master/api_args_parse.go +++ b/master/api_args_parse.go @@ -829,6 +829,7 @@ type createVolReq struct { remoteCacheMaxFileSizeGB int64 remoteCacheOnlyForNotSSD bool remoteCacheMultiRead bool + flashNodeTimeoutCount int64 } func checkCacheAction(action int) error { @@ -1104,6 +1105,9 @@ func parseRequestToCreateVol(r *http.Request, req *createVolReq) (err error) { if req.remoteCacheMultiRead, err = extractBoolWithDefault(r, remoteCacheMultiRead, false); err != nil { return } + if req.flashNodeTimeoutCount, err = extractInt64WithDefault(r, flashNodeTimeoutCount, DefaultFlashNodeTimeoutCount); err != nil { + return + } return } diff --git a/master/api_service.go b/master/api_service.go index 5deba8c32..7dd3a745d 100644 --- a/master/api_service.go +++ b/master/api_service.go @@ -2565,6 +2565,7 @@ func (m *Server) updateVol(w http.ResponseWriter, r *http.Request) { newArg("remoteCacheMaxFileSizeGB", &newArgs.remoteCacheMaxFileSizeGB).OmitEmpty(), newArg("remoteCacheOnlyForNotSSD", &newArgs.remoteCacheOnlyForNotSSD).OmitEmpty(), newArg("remoteCacheMultiRead", &newArgs.remoteCacheMultiRead).OmitEmpty(), + newArg("flashNodeTimeoutCount", &newArgs.flashNodeTimeoutCount).OmitEmpty(), ); err != nil { sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()}) return @@ -3246,6 +3247,7 @@ func newSimpleView(vol *Vol) (view *proto.SimpleVolView) { RemoteCacheMaxFileSizeGB: vol.remoteCacheMaxFileSizeGB, RemoteCacheOnlyForNotSSD: vol.remoteCacheOnlyForNotSSD, RemoteCacheMultiRead: vol.remoteCacheMultiRead, + FlashNodeTimeoutCount: vol.flashNodeTimeoutCount, } view.AllowedStorageClass = make([]uint32, len(vol.allowedStorageClass)) copy(view.AllowedStorageClass, vol.allowedStorageClass) diff --git a/master/cluster.go b/master/cluster.go index 5c78252bb..e50fd1227 100644 --- a/master/cluster.go +++ b/master/cluster.go @@ -4038,6 +4038,7 @@ func (c *Cluster) doCreateVol(req *createVolReq) (vol *Vol, err error) { RemoteCacheMaxFileSizeGB: req.remoteCacheMaxFileSizeGB, RemoteCacheOnlyForNotSSD: req.remoteCacheOnlyForNotSSD, RemoteCacheMultiRead: req.remoteCacheMultiRead, + FlashNodeTimeoutCount: req.flashNodeTimeoutCount, } vv.QuotaOfClass = make([]*proto.StatOfStorageClass, 0) diff --git a/master/const.go b/master/const.go index bd00bba85..cd01e923d 100644 --- a/master/const.go +++ b/master/const.go @@ -168,6 +168,7 @@ const ( remoteCacheMaxFileSizeGB = "remoteCacheMaxFileSizeGB" remoteCacheOnlyForNotSSD = "remoteCacheOnlyForNotSSD" remoteCacheMultiRead = "remoteCacheMultiRead" + flashNodeTimeoutCount = "flashNodeTimeoutCount" ) const ( @@ -243,6 +244,7 @@ const ( maxMpCreationCount = 10 defaultVolForbidWriteOpOfProtoVersion0 = true DefaultRemoteCacheMaxFileSizeGB = 128 + DefaultFlashNodeTimeoutCount = 5 defaultMetaNodeMemHighPer = 0.75 defaultMetaNodeMemLowPer = 0.3 metaNodeReserveMemorySize = 3 * 1024 * 1024 * 1024 diff --git a/master/metadata_fsm_op.go b/master/metadata_fsm_op.go index e61a09102..16e4fd238 100644 --- a/master/metadata_fsm_op.go +++ b/master/metadata_fsm_op.go @@ -374,6 +374,7 @@ type volValue struct { RemoteCacheMaxFileSizeGB int64 RemoteCacheOnlyForNotSSD bool RemoteCacheMultiRead bool + FlashNodeTimeoutCount int64 } func (v *volValue) Bytes() (raw []byte, err error) { @@ -467,6 +468,7 @@ func newVolValue(vol *Vol) (vv *volValue) { RemoteCacheMaxFileSizeGB: vol.remoteCacheMaxFileSizeGB, RemoteCacheOnlyForNotSSD: vol.remoteCacheOnlyForNotSSD, RemoteCacheMultiRead: vol.remoteCacheMultiRead, + FlashNodeTimeoutCount: vol.flashNodeTimeoutCount, } vv.AllowedStorageClass = make([]uint32, len(vol.allowedStorageClass)) copy(vv.AllowedStorageClass, vol.allowedStorageClass) diff --git a/master/vol.go b/master/vol.go index a54c0ed2b..135bbb70a 100644 --- a/master/vol.go +++ b/master/vol.go @@ -75,6 +75,7 @@ type VolVarargs struct { remoteCacheMaxFileSizeGB int64 remoteCacheOnlyForNotSSD bool remoteCacheMultiRead bool + flashNodeTimeoutCount int64 } // nolint: structcheck @@ -102,6 +103,7 @@ type TxSubItem struct { remoteCacheMaxFileSizeGB int64 remoteCacheOnlyForNotSSD bool remoteCacheMultiRead bool + flashNodeTimeoutCount int64 PreloadCacheOn bool NeedToLowerReplica bool @@ -260,6 +262,7 @@ func newVol(vv volValue) (vol *Vol) { vol.remoteCacheMaxFileSizeGB = vv.RemoteCacheMaxFileSizeGB vol.remoteCacheOnlyForNotSSD = vv.RemoteCacheOnlyForNotSSD vol.remoteCacheMultiRead = vv.RemoteCacheMultiRead + vol.flashNodeTimeoutCount = vv.FlashNodeTimeoutCount limitQosVal := &qosArgs{ qosEnable: vv.VolQosEnable, @@ -1964,6 +1967,7 @@ func setVolFromArgs(args *VolVarargs, vol *Vol) { vol.remoteCacheMaxFileSizeGB = args.remoteCacheMaxFileSizeGB vol.remoteCacheOnlyForNotSSD = args.remoteCacheOnlyForNotSSD vol.remoteCacheMultiRead = args.remoteCacheMultiRead + vol.flashNodeTimeoutCount = args.flashNodeTimeoutCount } func getVolVarargs(vol *Vol) *VolVarargs { @@ -2028,6 +2032,7 @@ func getVolVarargs(vol *Vol) *VolVarargs { remoteCacheMaxFileSizeGB: vol.remoteCacheMaxFileSizeGB, remoteCacheOnlyForNotSSD: vol.remoteCacheOnlyForNotSSD, remoteCacheMultiRead: vol.remoteCacheMultiRead, + flashNodeTimeoutCount: vol.flashNodeTimeoutCount, } } diff --git a/proto/admin_proto.go b/proto/admin_proto.go index f697ce52c..307f1c7c6 100644 --- a/proto/admin_proto.go +++ b/proto/admin_proto.go @@ -1353,6 +1353,7 @@ type SimpleVolView struct { RemoteCacheMaxFileSizeGB int64 RemoteCacheOnlyForNotSSD bool RemoteCacheMultiRead bool + FlashNodeTimeoutCount int64 RemoteCacheRemoveDupReq bool // TODO: using it in metanode, origin was named EnableRemoveDupReq } diff --git a/sdk/data/stream/extent_flash_cache.go b/sdk/data/stream/extent_flash_cache.go index 8311f0319..defb0391d 100644 --- a/sdk/data/stream/extent_flash_cache.go +++ b/sdk/data/stream/extent_flash_cache.go @@ -101,6 +101,7 @@ type RemoteCache struct { remoteCacheMaxFileSizeGB int64 remoteCacheOnlyForNotSSD bool remoteCacheMultiRead bool + flashNodeTimeoutCount int32 } func (rc *RemoteCache) UpdateRemoteCacheConfig(client *ExtentClient, view *proto.SimpleVolView) { @@ -170,6 +171,11 @@ func (rc *RemoteCache) UpdateRemoteCacheConfig(client *ExtentClient, view *proto log.LogInfof("RcFollowerRead: %v -> %v", rc.remoteCacheMultiRead, view.RemoteCacheMultiRead) rc.remoteCacheMultiRead = view.RemoteCacheMultiRead } + + if rc.flashNodeTimeoutCount != int32(view.FlashNodeTimeoutCount) { + log.LogInfof("RcFlashNodeTimeoutCount: %d -> %d", rc.flashNodeTimeoutCount, int32(view.FlashNodeTimeoutCount)) + rc.flashNodeTimeoutCount = int32(view.FlashNodeTimeoutCount) + } } func (rc *RemoteCache) DoRemoteCachePrepare(c *ExtentClient) { @@ -261,7 +267,7 @@ func (rc *RemoteCache) Read(ctx context.Context, fg *FlashGroup, inode uint64, r } if conn, err = rc.conns.GetConnect(addr); err != nil { log.LogWarnf("FlashGroup Read: get connection failed, addr(%v) reqPacket(%v) err(%v) remoteCacheMultiRead(%v)", addr, req, err, rc.remoteCacheMultiRead) - moved = fg.moveToUnknownRank(addr, err) + moved = fg.moveToUnknownRank(addr, err, rc.flashNodeTimeoutCount) if rc.remoteCacheMultiRead { log.LogInfof("Retrying due to GetConnect of addr(%v) failure err(%v)", addr, err) continue @@ -272,7 +278,7 @@ func (rc *RemoteCache) Read(ctx context.Context, fg *FlashGroup, inode uint64, r if err = reqPacket.WriteToConn(conn); err != nil { log.LogWarnf("FlashGroup Read: failed to write to addr(%v) err(%v) remoteCacheMultiRead(%v)", addr, err, rc.remoteCacheMultiRead) rc.conns.PutConnect(conn, err != nil) - moved = fg.moveToUnknownRank(addr, err) + moved = fg.moveToUnknownRank(addr, err, rc.flashNodeTimeoutCount) if rc.remoteCacheMultiRead { log.LogInfof("Retrying due to write to addr(%v) failure err(%v)", addr, err) continue @@ -286,7 +292,7 @@ func (rc *RemoteCache) Read(ctx context.Context, fg *FlashGroup, inode uint64, r } log.LogWarnf("FlashGroup Read: getReadReply from addr(%v) err(%v) remoteCacheMultiRead(%v)", addr, err, rc.remoteCacheMultiRead) rc.conns.PutConnect(conn, err != nil) - moved = fg.moveToUnknownRank(addr, err) + moved = fg.moveToUnknownRank(addr, err, rc.flashNodeTimeoutCount) if rc.remoteCacheMultiRead { log.LogInfof("Retrying due to getReadReply from addr(%v) failure err(%v)", addr, err) continue @@ -346,7 +352,7 @@ func (rc *RemoteCache) Prepare(ctx context.Context, fg *FlashGroup, inode uint64 } defer func() { if err != nil { - moved = fg.moveToUnknownRank(addr, err) + moved = fg.moveToUnknownRank(addr, err, rc.flashNodeTimeoutCount) } }() if conn, err = rc.conns.GetConnect(addr); err != nil { @@ -547,6 +553,49 @@ func (rc *RemoteCache) ClassifyHostsByAvgDelay(fgID uint64, hosts []string) (sor return sortedHosts } +func (rc *RemoteCache) resetFlashNodeTimeoutCount() { + fgSet := make([]uint64, 0) + + isInSet := func(fg uint64, fgSet []uint64) bool { + for _, v := range fgSet { + if v == fg { + return true + } + } + return false + } + + rangeFunc := func(i btree.Item) bool { + item := i.(*SlotItem) + fg := item.FlashGroup + + if isInSet(fg.ID, fgSet) { + return true + } + fgSet = append(fgSet, fg.ID) + fg.hostLock.Lock() + for addr := range fg.hostTimeoutCount { + if fg.hostTimeoutCount[addr] == 0 { + continue + } + // Check if the timeout host ping was successful + // if it was successful and within the sameZone and sameRegion ranges, reset the host timeout count to 0 + v, ok := rc.hostLatency.Load(addr) + if ok { + avgTime := v.(time.Duration) + if avgTime > 0 && avgTime < SameRegionTimeout { + log.LogDebugf("resetFlashNodeTimeoutCount fgId(%v) flashnode(%v) from %v to 0", + fg.ID, addr, fg.hostTimeoutCount[addr]) + fg.hostTimeoutCount[addr] = 0 + } + } + } + fg.hostLock.Unlock() + return true + } + rc.rangeFlashGroups(nil, rangeFunc) +} + func (rc *RemoteCache) refreshHostLatency() { hosts := rc.getFlashHostsMap() @@ -576,6 +625,7 @@ func (rc *RemoteCache) updateHostLatency(hosts []string) { log.LogWarnf("updateHostLatency: host(%v) err(%v)", host, err) } } + rc.resetFlashNodeTimeoutCount() } func (rc *RemoteCache) GetFlashGroupBySlot(slot uint32) (*FlashGroup, uint32) { diff --git a/sdk/data/stream/flash_group.go b/sdk/data/stream/flash_group.go index d474969c0..688617746 100644 --- a/sdk/data/stream/flash_group.go +++ b/sdk/data/stream/flash_group.go @@ -29,9 +29,10 @@ import ( type FlashGroup struct { *proto.FlashGroupInfo - rankedHost map[ZoneRankType][]string - hostLock sync.RWMutex - epoch uint64 + rankedHost map[ZoneRankType][]string + hostLock sync.RWMutex + epoch uint64 + hostTimeoutCount map[string]int32 } func (fg *FlashGroup) String() string { @@ -43,8 +44,9 @@ func (fg *FlashGroup) String() string { func NewFlashGroup(flashGroupInfo *proto.FlashGroupInfo, rankedHost map[ZoneRankType][]string) *FlashGroup { return &FlashGroup{ - FlashGroupInfo: flashGroupInfo, - rankedHost: rankedHost, + FlashGroupInfo: flashGroupInfo, + rankedHost: rankedHost, + hostTimeoutCount: make(map[string]int32), } } @@ -77,13 +79,19 @@ func (fg *FlashGroup) getFlashHost() (host string) { return } -func (fg *FlashGroup) moveToUnknownRank(addr string, err error) bool { +func (fg *FlashGroup) moveToUnknownRank(addr string, err error, timeoutCount int32) bool { if !(err != nil && (os.IsTimeout(err) || strings.Contains(err.Error(), syscall.ECONNREFUSED.Error()))) { return false } fg.hostLock.Lock() defer fg.hostLock.Unlock() + fg.hostTimeoutCount[addr]++ + if fg.hostTimeoutCount[addr] < timeoutCount { + log.LogInfof("moveToUnknownRank: fgID(%v) host(%v) timeoutCount(%v) not reached maxTimeoutCount(%v)", fg.ID, addr, fg.hostTimeoutCount[addr], timeoutCount) + return false + } + moved := false for rank := SameZoneRank; rank <= SameRegionRank; rank++ { hosts := fg.rankedHost[rank] @@ -104,7 +112,7 @@ func (fg *FlashGroup) moveToUnknownRank(addr string, err error) bool { unknowns = append(unknowns, addr) fg.rankedHost[UnknownZoneRank] = unknowns - log.LogWarnf("moveToUnknownRank: fgID(%v) host: %v by err %v", fg.ID, addr, err.Error()) + log.LogWarnf("moveToUnknownRank: fgID(%v) host(%v) timeoutCount(%v) by err %v", fg.ID, addr, fg.hostTimeoutCount[addr], err.Error()) return moved } diff --git a/sdk/master/api_admin.go b/sdk/master/api_admin.go index 4009c1418..0bb9ac89f 100644 --- a/sdk/master/api_admin.go +++ b/sdk/master/api_admin.go @@ -327,6 +327,7 @@ func (api *AdminAPI) UpdateVolume( request.addParam("remoteCacheMaxFileSizeGB", strconv.FormatInt(vv.RemoteCacheMaxFileSizeGB, 10)) request.addParamAny("remoteCacheOnlyForNotSSD", vv.RemoteCacheOnlyForNotSSD) request.addParamAny("remoteCacheMultiRead", vv.RemoteCacheMultiRead) + request.addParamAny("flashNodeTimeoutCount", vv.FlashNodeTimeoutCount) if txMask != "" { request.addParam("enableTxMask", txMask) @@ -397,7 +398,7 @@ func (api *AdminAPI) CreateVolName(volName, owner string, capacity uint64, delet dpReadOnlyWhenVolFull bool, txMask string, txTimeout uint32, txConflictRetryNum int64, txConflictRetryInterval int64, optEnableQuota string, clientIDKey string, volStorageClass uint32, allowedStorageClass string, optMetaFollowerRead string, optMaximallyRead string, remoteCacheEnable string, remoteCacheAutoPrepare string, remoteCachePath string, remoteCacheTTL int64, remoteCacheReadTimeout int64, - remoteCacheMaxFileSizeGB int64, remoteCacheOnlyForNotSSD string, remoteCacheMultiRead string, + remoteCacheMaxFileSizeGB int64, remoteCacheOnlyForNotSSD string, remoteCacheMultiRead string, flashNodeTimeoutCount int64, ) (err error) { request := newRequest(get, proto.AdminCreateVol).Header(api.h) request.addParam("name", volName) @@ -437,6 +438,7 @@ func (api *AdminAPI) CreateVolName(volName, owner string, capacity uint64, delet request.addParam("remoteCacheMaxFileSizeGB", strconv.FormatInt(remoteCacheMaxFileSizeGB, 10)) request.addParam("remoteCacheOnlyForNotSSD", remoteCacheOnlyForNotSSD) request.addParam("remoteCacheMultiRead", remoteCacheMultiRead) + request.addParamAny("flashNodeTimeoutCount", flashNodeTimeoutCount) if txMask != "" { request.addParam("enableTxMask", txMask)