gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 02:00:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix(master): fix port inconsistent when create dp/mp during upgrading process

Browse Source 
Signed-off-by: morphes1995 <morphes1995@gmail.com>
This commit is contained in:
morphes1995 2024-10-30 22:37:27 +08:00 committed by leonrayang
parent 848384c2a2
commit 00a3ee193f
12 changed files with 195 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cli/cmd/fmt.go
View File

@ -638,6 +638,7 @@ func formatMetaPartitionInfo(partition *proto.MetaPartitionInfo) string {
}
sb.WriteString("\n")
sb.WriteString("Peers :\n")
sb.WriteString(fmt.Sprintf("%v\n", formatPeerTableHeader()))
for _, peer := range partition.Peers {
sb.WriteString(fmt.Sprintf("%v\n", formatPeer(peer)))
}
@ -916,14 +917,14 @@ func formatMetaReplica(indentation string, replica *proto.MetaReplicaInfo, rowTa
return sb.String()
}
var peerTableRowPattern = "%-6v %-18v"
var peerTableRowPattern = "%-6v %-18v %-12v %-12v"
func formatPeerTableHeader() string {
return fmt.Sprintf(peerTableRowPattern, "ID", "PEER")
return fmt.Sprintf(peerTableRowPattern, "ID", "ADDR", "HEARTBEATPORT", "REPLICAPORT")
}
func formatPeer(peer proto.Peer) string {
return fmt.Sprintf(peerTableRowPattern, peer.ID, peer.Addr)
return fmt.Sprintf(peerTableRowPattern, peer.ID, peer.Addr, peer.HeartbeatPort, peer.ReplicaPort)
}
var dataNodeDetailTableRowPattern = "%-6v %-6v %-65v %-6v %-6v %-6v %-10v"

13
datanode/partition.go
View File

@ -25,6 +25,7 @@ import (
"os"
"path"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
@ -416,6 +417,18 @@ func newDataPartition(dpCfg *dataPartitionCfg, disk *Disk, isCreate bool) (dp *D
responseStatus: responseInitial,
PersistApplyIdChan: make(chan PersistApplyIdRequest),
}
// during upgrade process, create partition request may lack raft ports info
defaultHeartbeatPort, defaultReplicaPort, err := partition.raftPort()
if err == nil {
for i := range partition.config.Peers {
if len(partition.config.Peers[i].ReplicaPort) == 0 || len(partition.config.Peers[i].HeartbeatPort) == 0 {
partition.config.Peers[i].ReplicaPort = strconv.FormatInt(int64(defaultReplicaPort), 10)
partition.config.Peers[i].HeartbeatPort = strconv.FormatInt(int64(defaultHeartbeatPort), 10)
}
}
}
atomic.StoreUint64(&partition.recoverErrCnt, 0)
log.LogInfof("action[newDataPartition] dp %v replica num %v", partitionID, dpCfg.ReplicaNum)

85
datanode/partition_raft.go
View File

@ -56,28 +56,28 @@ type dataPartitionCfg struct {
ForbidWriteOpOfProtoVer0 bool
}
//func (dp *DataPartition) raftPort() (heartbeat, replica int, err error) {
// raftConfig := dp.config.RaftStore.RaftConfig()
// heartbeatAddrSplits := strings.Split(raftConfig.HeartbeatAddr, ":")
// replicaAddrSplits := strings.Split(raftConfig.ReplicateAddr, ":")
// if len(heartbeatAddrSplits) != 2 {
// err = errors.New("illegal heartbeat address")
// return
// }
// if len(replicaAddrSplits) != 2 {
// err = errors.New("illegal replica address")
// return
// }
// heartbeat, err = strconv.Atoi(heartbeatAddrSplits[1])
// if err != nil {
// return
// }
// replica, err = strconv.Atoi(replicaAddrSplits[1])
// if err != nil {
// return
// }
// return
//}
func (dp *DataPartition) raftPort() (heartbeat, replica int, err error) {
raftConfig := dp.config.RaftStore.RaftConfig()
heartbeatAddrSplits := strings.Split(raftConfig.HeartbeatAddr, ":")
replicaAddrSplits := strings.Split(raftConfig.ReplicateAddr, ":")
if len(heartbeatAddrSplits) != 2 {
err = errors.New("illegal heartbeat address")
return
}
if len(replicaAddrSplits) != 2 {
err = errors.New("illegal replica address")
return
}
heartbeat, err = strconv.Atoi(heartbeatAddrSplits[1])
if err != nil {
return
}
replica, err = strconv.Atoi(replicaAddrSplits[1])
if err != nil {
return
}
return
}
// StartRaft start raft instance when data partition start or restore.
func (dp *DataPartition) StartRaft(isLoad bool) (err error) {
@ -92,9 +92,9 @@ func (dp *DataPartition) StartRaft(isLoad bool) (err error) {
}
var (
heartbeatPort int
replicaPort int
peers []raftstore.PeerAddress
defaultHeartbeatPort int
defaultReplicaPort int
peers []raftstore.PeerAddress
)
defer func() {
if r := recover(); r != nil {
@ -109,13 +109,21 @@ func (dp *DataPartition) StartRaft(isLoad bool) (err error) {
}
}()
//if heartbeatPort, replicaPort, err = dp.raftPort(); err != nil {
// return
//}
if defaultHeartbeatPort, defaultReplicaPort, err = dp.raftPort(); err != nil {
return
}
for _, peer := range dp.config.Peers {
addr := strings.Split(peer.Addr, ":")[0]
heartbeatPort, _ = strconv.Atoi(peer.HeartbeatPort)
replicaPort, _ = strconv.Atoi(peer.ReplicaPort)
heartbeatPort, perr := strconv.Atoi(peer.HeartbeatPort)
if perr != nil {
heartbeatPort = defaultHeartbeatPort
}
replicaPort, perr := strconv.Atoi(peer.ReplicaPort)
if perr != nil {
replicaPort = defaultReplicaPort
}
rp := raftstore.PeerAddress{
Peer: raftproto.Peer{
ID: peer.ID,
@ -409,16 +417,21 @@ func (dp *DataPartition) addRaftNode(req *proto.AddDataPartitionRaftMemberReques
}
var (
heartbeatPort int
replicaPort int
defaultHeartbeatPort int
defaultReplicaPort int
)
heartbeatPort, err = strconv.Atoi(req.AddPeer.HeartbeatPort)
if err != nil {
if defaultHeartbeatPort, defaultReplicaPort, err = dp.raftPort(); err != nil {
return
}
replicaPort, err = strconv.Atoi(req.AddPeer.ReplicaPort)
if err != nil {
heartbeatPort, perr := strconv.Atoi(req.AddPeer.HeartbeatPort)
if perr != nil {
heartbeatPort = defaultHeartbeatPort
}
replicaPort, perr := strconv.Atoi(req.AddPeer.ReplicaPort)
if perr != nil {
replicaPort = defaultReplicaPort
return
}

13
master/api_args_parse.go
View File

@ -26,8 +26,6 @@ import (
"strings"
"time"
"github.com/cubefs/cubefs/raftstore"
"github.com/cubefs/cubefs/proto"
"github.com/cubefs/cubefs/util"
"github.com/cubefs/cubefs/util/compressor"
@ -177,14 +175,9 @@ func parseRequestForAddNode(r *http.Request) (nodeAddr, raftHeartbeatPort, raftR
if zoneName = r.FormValue(zoneNameKey); zoneName == "" {
zoneName = DefaultZoneName
}
if raftHeartbeatPort = r.FormValue(heartbeatPortKey); raftHeartbeatPort == "" {
raftHeartbeatPort = strconv.Itoa(raftstore.DefaultHeartbeatPort)
}
if raftReplicaPort = r.FormValue(replicaPortKey); raftReplicaPort == "" {
raftReplicaPort = strconv.Itoa(raftstore.DefaultReplicaPort)
}
// for old version node registration, heartbeat port and replica port may be empty
raftHeartbeatPort = r.FormValue(heartbeatPortKey)
raftReplicaPort = r.FormValue(replicaPortKey)
if mediaType, err = extractMediaType(r); err != nil {
return

28
master/cluster.go
View File

@ -1134,12 +1134,14 @@ func (c *Cluster) addMetaNode(nodeAddr, heartbeatPort, replicaPort, zoneName str
}
// compatible with old version in which raft heartbeat port and replica port did not persist
metaNode.Lock()
defer metaNode.Unlock()
metaNode.HeartbeatPort = heartbeatPort
metaNode.ReplicaPort = replicaPort
if err = c.syncUpdateMetaNode(metaNode); err != nil {
return metaNode.ID, err
if len(heartbeatPort) > 0 && len(replicaPort) > 0 {
metaNode.Lock()
defer metaNode.Unlock()
metaNode.HeartbeatPort = heartbeatPort
metaNode.ReplicaPort = replicaPort
if err = c.syncUpdateMetaNode(metaNode); err != nil {
return metaNode.ID, err
}
}
return metaNode.ID, nil
@ -1288,12 +1290,14 @@ func (c *Cluster) addDataNode(nodeAddr, raftHeartbeatPort, raftReplicaPort, zone
}
// compatible with old version in which raft heartbeat port and replica port did not persist
dataNode.Lock()
defer dataNode.Unlock()
dataNode.HeartbeatPort = raftHeartbeatPort
dataNode.ReplicaPort = raftReplicaPort
if err = c.syncUpdateDataNode(dataNode); err != nil {
return dataNode.ID, err
if len(raftHeartbeatPort) > 0 && len(raftReplicaPort) > 0 {
dataNode.Lock()
defer dataNode.Unlock()
dataNode.HeartbeatPort = raftHeartbeatPort
dataNode.ReplicaPort = raftReplicaPort
if err = c.syncUpdateDataNode(dataNode); err != nil {
return dataNode.ID, err
}
}
return dataNode.ID, nil

2
master/cluster_task.go
View File

@ -1117,7 +1117,7 @@ func (c *Cluster) updateMetaNode(metaNode *MetaNode, metaPartitions []*proto.Met
mp.addUpdateMetaReplicaTask(c)
}
mp.updateMetaPartition(mr, metaNode)
mp.updateMetaPartition(mr, metaNode, c)
vol.uidSpaceManager.pushUidMsg(mr)
vol.quotaManager.quotaUpdate(mr)
c.updateInodeIDUpperBound(mp, mr, threshold, metaNode)

22
master/data_partition.go
View File

@ -770,6 +770,28 @@ func (partition *DataPartition) updateMetric(vr *proto.DataPartitionReport, data
replica.DiskPath = oldDiskPath
}
}
// update old partition peers, add raft ports
localPeersWithPort := make(map[string]proto.Peer)
for _, peer := range vr.LocalPeers {
if len(peer.ReplicaPort) > 0 && len(peer.HeartbeatPort) > 0 {
localPeersWithPort[peer.Addr] = peer
}
}
needUpdate := false
for i, peer := range partition.Peers {
if len(peer.ReplicaPort) == 0 || len(peer.HeartbeatPort) == 0 {
if localPeer, exist := localPeersWithPort[peer.Addr]; exist {
partition.Peers[i].ReplicaPort = localPeer.ReplicaPort
partition.Peers[i].HeartbeatPort = localPeer.HeartbeatPort
needUpdate = true
}
}
}
if needUpdate {
c.syncUpdateDataPartition(partition)
}
partition.checkAndRemoveMissReplica(dataNode.Addr)
if replica.Status == proto.ReadWrite && (partition.RdOnly || replica.dataNode.RdOnly) {

23
master/meta_partition.go
View File

@ -407,7 +407,7 @@ func (mp *MetaPartition) missingReplicaAddrs() (lackAddrs []string) {
return
}
func (mp *MetaPartition) updateMetaPartition(mgr *proto.MetaPartitionReport, metaNode *MetaNode) {
func (mp *MetaPartition) updateMetaPartition(mgr *proto.MetaPartitionReport, metaNode *MetaNode, c *Cluster) {
if !contains(mp.Hosts, metaNode.Addr) {
return
}
@ -432,6 +432,27 @@ func (mp *MetaPartition) updateMetaPartition(mgr *proto.MetaPartitionReport, met
mp.setStatByStorageClass()
mp.setHeartBeatDone()
mp.SetForbidWriteOpOfProtoVer0()
// update old partition peers, add raft ports
localPeersWithPort := make(map[string]proto.Peer)
for _, peer := range mgr.LocalPeers {
if len(peer.ReplicaPort) > 0 && len(peer.HeartbeatPort) > 0 {
localPeersWithPort[peer.Addr] = peer
}
}
needUpdate := false
for i, peer := range mp.Peers {
if len(peer.ReplicaPort) == 0 || len(peer.HeartbeatPort) == 0 {
if localPeer, exist := localPeersWithPort[peer.Addr]; exist {
mp.Peers[i].ReplicaPort = localPeer.ReplicaPort
mp.Peers[i].HeartbeatPort = localPeer.HeartbeatPort
needUpdate = true
}
}
}
if needUpdate {
c.syncUpdateMetaPartition(mp)
}
}
func (mp *MetaPartition) canBeOffline(nodeAddr string, replicaNum int) (err error) {

1
metanode/manager_op.go
View File

@ -178,6 +178,7 @@ func (m *metadataManager) opMasterHeartbeat(conn net.Conn, p *Packet,
StatByStorageClass: partition.GetStatByStorageClass(),
StatByMigrateStorageClass: partition.GetMigrateStatByStorageClass(),
ForbidWriteOpOfProtoVer0: mpForbidWriteVer0,
LocalPeers: mConf.Peers,
}
mpr.TxCnt, mpr.TxRbInoCnt, mpr.TxRbDenCnt = partition.TxGetCnt()

80
metanode/partition.go
View File

@ -971,18 +971,26 @@ func (mp *metaPartition) onStop() {
func (mp *metaPartition) startRaft() (err error) {
var (
heartbeatPort int
replicaPort int
peers []raftstore.PeerAddress
defaultHeartbeatPort int
defaultReplicaPort int
peers []raftstore.PeerAddress
)
//if heartbeatPort, replicaPort, err = mp.getRaftPort(); err != nil {
// return
//}
if defaultHeartbeatPort, defaultReplicaPort, err = mp.getRaftPort(); err != nil {
return
}
for _, peer := range mp.config.Peers {
addr := strings.Split(peer.Addr, ":")[0]
heartbeatPort, _ = strconv.Atoi(peer.HeartbeatPort)
replicaPort, _ = strconv.Atoi(peer.ReplicaPort)
heartbeatPort, perr := strconv.Atoi(peer.HeartbeatPort)
if perr != nil {
heartbeatPort = defaultHeartbeatPort
}
replicaPort, perr := strconv.Atoi(peer.ReplicaPort)
if perr != nil {
replicaPort = defaultReplicaPort
}
rp := raftstore.PeerAddress{
Peer: raftproto.Peer{
ID: peer.ID,
@ -1016,28 +1024,28 @@ func (mp *metaPartition) stopRaft() {
}
}
//func (mp *metaPartition) getRaftPort() (heartbeat, replica int, err error) {
// raftConfig := mp.config.RaftStore.RaftConfig()
// heartbeatAddrSplits := strings.Split(raftConfig.HeartbeatAddr, ":")
// replicaAddrSplits := strings.Split(raftConfig.ReplicateAddr, ":")
// if len(heartbeatAddrSplits) != 2 {
// err = ErrIllegalHeartbeatAddress
// return
// }
// if len(replicaAddrSplits) != 2 {
// err = ErrIllegalReplicateAddress
// return
// }
// heartbeat, err = strconv.Atoi(heartbeatAddrSplits[1])
// if err != nil {
// return
// }
// replica, err = strconv.Atoi(replicaAddrSplits[1])
// if err != nil {
// return
// }
// return
//}
func (mp *metaPartition) getRaftPort() (heartbeat, replica int, err error) {
raftConfig := mp.config.RaftStore.RaftConfig()
heartbeatAddrSplits := strings.Split(raftConfig.HeartbeatAddr, ":")
replicaAddrSplits := strings.Split(raftConfig.ReplicateAddr, ":")
if len(heartbeatAddrSplits) != 2 {
err = ErrIllegalHeartbeatAddress
return
}
if len(replicaAddrSplits) != 2 {
err = ErrIllegalReplicateAddress
return
}
heartbeat, err = strconv.Atoi(heartbeatAddrSplits[1])
if err != nil {
return
}
replica, err = strconv.Atoi(replicaAddrSplits[1])
if err != nil {
return
}
return
}
// NewMetaPartition creates a new meta partition with the specified configuration.
func NewMetaPartition(conf *MetaPartitionConfig, manager *metadataManager) MetaPartition {
@ -1061,9 +1069,19 @@ func NewMetaPartition(conf *MetaPartitionConfig, manager *metadataManager) MetaP
multiVersionList: &proto.VolVersionInfoList{
TemporaryVerMap: make(map[uint64]*proto.VolVersionInfo),
},
enableAuditLog: true,
}
// during upgrade process, create partition request may lack raft ports info
defaultHeartbeatPort, defaultReplicaPort, err := mp.getRaftPort()
if err == nil {
for i := range mp.config.Peers {
if len(mp.config.Peers[i].ReplicaPort) == 0 || len(mp.config.Peers[i].HeartbeatPort) == 0 {
mp.config.Peers[i].ReplicaPort = strconv.FormatInt(int64(defaultReplicaPort), 10)
mp.config.Peers[i].HeartbeatPort = strconv.FormatInt(int64(defaultHeartbeatPort), 10)
}
}
}
if manager != nil {
mp.config.ForbidWriteOpOfProtoVer0 = manager.isVolForbidWriteOpOfProtoVer0(mp.config.VolName)
}

18
metanode/partition_fsmop.go
View File

@ -96,11 +96,21 @@ func (mp *metaPartition) fsmUpdatePartition(end uint64) (status uint8,
func (mp *metaPartition) confAddNode(req *proto.AddMetaPartitionRaftMemberRequest, index uint64) (updated bool, err error) {
var (
heartbeatPort int
replicaPort int
defaultHeartbeatPort int
defaultreplicaPort int
)
heartbeatPort, _ = strconv.Atoi(req.AddPeer.HeartbeatPort)
replicaPort, _ = strconv.Atoi(req.AddPeer.ReplicaPort)
if defaultHeartbeatPort, defaultreplicaPort, err = mp.getRaftPort(); err != nil {
return
}
heartbeatPort, perr := strconv.Atoi(req.AddPeer.HeartbeatPort)
if perr != nil {
heartbeatPort = defaultHeartbeatPort
}
replicaPort, perr := strconv.Atoi(req.AddPeer.ReplicaPort)
if perr != nil {
replicaPort = defaultreplicaPort
}
addPeer := false
for _, peer := range mp.config.Peers {

1
proto/admin_proto.go
View File

@ -880,6 +880,7 @@ type MetaPartitionReport struct {
QuotaReportInfos []*QuotaReportInfo
StatByStorageClass []*StatOfStorageClass
StatByMigrateStorageClass []*StatOfStorageClass
LocalPeers []Peer
}
// MetaNodeHeartbeatResponse defines the response to the meta node heartbeat request.