mirror of
https://github.com/cubefs/cubefs.git
synced 2026-08-02 02:00:56 +00:00
feat(metanode): Add raft members consistent check with master hosts in metanode member check
. #1000165698 Signed-off-by: leonrayang <changliang@oppo.com>
This commit is contained in:
parent
e944cecec5
commit
7d5ad3adf4
@ -412,6 +412,14 @@ var (
|
||||
partitionInfoTableHeader = fmt.Sprintf(partitionInfoTablePattern,
|
||||
"ID", "VOLUME", "REPLICAS", "STATUS", "MediaType", "MEMBERS")
|
||||
|
||||
PeerAbnormalRaftPartitionInfoTablePattern = "%-8v %-32v %-8v %-12v %-12v %-12v %v%v%v"
|
||||
PeerAbnormalRaftPartitionInfoHeader = fmt.Sprintf(PeerAbnormalRaftPartitionInfoTablePattern,
|
||||
"ID", "VOLUME", "REPLICAS", "STATUS", "MediaType", "MEMBERS", "DIFF-RAFT-MEMBERS", "|DOWN-MEMBERS", "|PEND-MEMBERS")
|
||||
|
||||
badMpReplicaPartitionInfoTablePattern = "%-8v %-32v %-8v %-12v %-64v %-24v"
|
||||
badMpReplicaPartitionInfoTableHeader = fmt.Sprintf(badMpReplicaPartitionInfoTablePattern,
|
||||
"ID", "VOLUME", "REPLICAS", "STATUS", "MEMBERS", "UNAVAILABLE_REPLICAS")
|
||||
|
||||
badReplicaPartitionInfoTablePattern = "%-8v %-32v %-8v %-12v %-64v %-24v"
|
||||
badReplicaPartitionInfoTableHeader = fmt.Sprintf(badReplicaPartitionInfoTablePattern,
|
||||
"ID", "VOLUME", "REPLICAS", "STATUS", "MEMBERS", "UNAVAILABLE_REPLICAS")
|
||||
@ -553,6 +561,65 @@ func formatMetaPartitionInfoRow(partition *proto.MetaPartitionInfo) string {
|
||||
formatDataPartitionStatus(partition.Status), "N/A", strings.Join(partition.Hosts, ", "))
|
||||
}
|
||||
|
||||
func formatMetaPartitionInfoRowWithRaft(partition *proto.MetaPartitionInfo) string {
|
||||
var (
|
||||
info *proto.MetaPartitionLoadResponse
|
||||
addrArr = make(map[uint64]string)
|
||||
diffHosts []string
|
||||
downHosts []string
|
||||
pendingHosts []string
|
||||
)
|
||||
for _, peer := range partition.Peers {
|
||||
addrArr[peer.ID] = peer.Addr
|
||||
}
|
||||
|
||||
for _, info = range partition.LoadResponse {
|
||||
if info.RaftInfo.RaftStatus.Leader == info.RaftInfo.RaftStatus.NodeID {
|
||||
break
|
||||
}
|
||||
}
|
||||
if info == nil {
|
||||
return ""
|
||||
}
|
||||
for _, peer := range info.RaftInfo.Hosts {
|
||||
if _, ok := addrArr[peer.ID]; ok {
|
||||
continue
|
||||
}
|
||||
diffHosts = append(diffHosts, peer.Addr)
|
||||
addrArr[peer.ID] = peer.Addr
|
||||
}
|
||||
|
||||
for _, dr := range info.RaftInfo.DownReplicas {
|
||||
if host, ok := addrArr[dr.NodeID]; ok {
|
||||
downHosts = append(downHosts, host)
|
||||
}
|
||||
}
|
||||
for _, pr := range info.RaftInfo.PendingPeers {
|
||||
if host, ok := addrArr[pr]; ok {
|
||||
pendingHosts = append(pendingHosts, host)
|
||||
}
|
||||
}
|
||||
var (
|
||||
diffInfo = "N/A"
|
||||
downInfo = "N/A"
|
||||
pendingInfo = "N/A"
|
||||
)
|
||||
if len(diffHosts) != 0 {
|
||||
diffInfo = strings.Join(diffHosts, ", ")
|
||||
}
|
||||
if len(downHosts) != 0 {
|
||||
downInfo = strings.Join(downHosts, ", ")
|
||||
}
|
||||
if len(pendingHosts) != 0 {
|
||||
pendingInfo = strings.Join(pendingHosts, ", ")
|
||||
}
|
||||
|
||||
return fmt.Sprintf(PeerAbnormalRaftPartitionInfoTablePattern,
|
||||
partition.PartitionID, partition.VolName, partition.ReplicaNum,
|
||||
formatDataPartitionStatus(partition.Status), "N/A", strings.Join(partition.Hosts, ", "),
|
||||
diffInfo+"|", downInfo+"|", pendingInfo)
|
||||
}
|
||||
|
||||
func formatBadReplicaMpInfoRow(partition *proto.MetaPartitionInfo) string {
|
||||
sb := strings.Builder{}
|
||||
sb.WriteString("[")
|
||||
|
||||
@ -90,7 +90,7 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on
|
||||
"reset" command will be released in next version.`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
var (
|
||||
diagnosis *proto.MetaPartitionDiagnosis
|
||||
diagnosis *proto.MetaPartitionDiagnosisV1
|
||||
metaNodes []*proto.MetaNodeInfo
|
||||
err error
|
||||
)
|
||||
@ -120,10 +120,10 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on
|
||||
stdout("\n")
|
||||
stdout("[Corrupt meta partitions](no leader):\n")
|
||||
stdout("%v\n", partitionInfoTableHeader)
|
||||
sort.SliceStable(diagnosis.CorruptMetaPartitionIDs, func(i, j int) bool {
|
||||
return diagnosis.CorruptMetaPartitionIDs[i] < diagnosis.CorruptMetaPartitionIDs[j]
|
||||
sort.SliceStable(diagnosis.NoLeaderMetaPartitionIDs, func(i, j int) bool {
|
||||
return diagnosis.NoLeaderMetaPartitionIDs[i] < diagnosis.NoLeaderMetaPartitionIDs[j]
|
||||
})
|
||||
for _, pid := range diagnosis.CorruptMetaPartitionIDs {
|
||||
for _, pid := range diagnosis.NoLeaderMetaPartitionIDs {
|
||||
var partition *proto.MetaPartitionInfo
|
||||
if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil {
|
||||
err = fmt.Errorf("Partition not found, err:[%v] ", err)
|
||||
@ -149,6 +149,23 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on
|
||||
}
|
||||
}
|
||||
|
||||
stdout("\n")
|
||||
stdout("%v\n", "[Meta partition Abnormal Raft Info]:")
|
||||
stdout("%v\n", PeerAbnormalRaftPartitionInfoHeader)
|
||||
sort.SliceStable(diagnosis.AbnormalRaftIDs, func(i, j int) bool {
|
||||
return diagnosis.AbnormalRaftIDs[i] < diagnosis.AbnormalRaftIDs[j]
|
||||
})
|
||||
for _, pid := range diagnosis.AbnormalRaftIDs {
|
||||
var partition *proto.MetaPartitionInfo
|
||||
if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil {
|
||||
err = fmt.Errorf("Partition not found, err:[%v] ", err)
|
||||
return
|
||||
}
|
||||
if partition != nil {
|
||||
stdout("%v\n", formatMetaPartitionInfoRowWithRaft(partition))
|
||||
}
|
||||
}
|
||||
|
||||
stdout("\n")
|
||||
stdout("%v\n", "[Bad meta partitions(decommission not completed)]:")
|
||||
badPartitionTablePattern := "%-8v %-10v\n"
|
||||
@ -164,11 +181,11 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on
|
||||
|
||||
stdout("\n")
|
||||
stdout("%v\n", "[Meta Partition has unavailable replica]:")
|
||||
stdout("%v\n", badReplicaPartitionInfoTableHeader)
|
||||
sort.SliceStable(diagnosis.BadReplicaMetaPartitionIDs, func(i, j int) bool {
|
||||
return diagnosis.BadReplicaMetaPartitionIDs[i] < diagnosis.BadReplicaMetaPartitionIDs[j]
|
||||
stdout("%v\n", badMpReplicaPartitionInfoTableHeader)
|
||||
sort.SliceStable(diagnosis.UnavailableMetaPartitionIDs, func(i, j int) bool {
|
||||
return diagnosis.UnavailableMetaPartitionIDs[i] < diagnosis.UnavailableMetaPartitionIDs[j]
|
||||
})
|
||||
for _, pid := range diagnosis.BadReplicaMetaPartitionIDs {
|
||||
for _, pid := range diagnosis.UnavailableMetaPartitionIDs {
|
||||
var partition *proto.MetaPartitionInfo
|
||||
if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil {
|
||||
err = fmt.Errorf("Partition not found, err:[%v] ", err)
|
||||
@ -185,10 +202,10 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on
|
||||
stdout("\n")
|
||||
stdout("%v\n", "[Partition with replica inode count not equal]:")
|
||||
stdout("%v\n", inodeCountNotEqualInfoTableHeader)
|
||||
sort.SliceStable(diagnosis.InodeCountNotEqualReplicaMetaPartitionIDs, func(i, j int) bool {
|
||||
return diagnosis.InodeCountNotEqualReplicaMetaPartitionIDs[i] < diagnosis.InodeCountNotEqualReplicaMetaPartitionIDs[j]
|
||||
sort.SliceStable(diagnosis.InodeCountNotEqualIDs, func(i, j int) bool {
|
||||
return diagnosis.InodeCountNotEqualIDs[i] < diagnosis.InodeCountNotEqualIDs[j]
|
||||
})
|
||||
for _, pid := range diagnosis.InodeCountNotEqualReplicaMetaPartitionIDs {
|
||||
for _, pid := range diagnosis.InodeCountNotEqualIDs {
|
||||
var partition *proto.MetaPartitionInfo
|
||||
if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil {
|
||||
err = fmt.Errorf("Partition not found, err:[%v] ", err)
|
||||
@ -202,10 +219,10 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on
|
||||
stdout("\n")
|
||||
stdout("%v\n", "[Partition with replica max inode not equal]:")
|
||||
stdout("%v\n", maxInodeNotEqualInfoTableHeader)
|
||||
sort.SliceStable(diagnosis.MaxInodeNotEqualReplicaMetaPartitionIDs, func(i, j int) bool {
|
||||
return diagnosis.MaxInodeNotEqualReplicaMetaPartitionIDs[i] < diagnosis.MaxInodeNotEqualReplicaMetaPartitionIDs[j]
|
||||
sort.SliceStable(diagnosis.MaxInodeNotEqualIDs, func(i, j int) bool {
|
||||
return diagnosis.MaxInodeNotEqualIDs[i] < diagnosis.MaxInodeNotEqualIDs[j]
|
||||
})
|
||||
for _, pid := range diagnosis.MaxInodeNotEqualReplicaMetaPartitionIDs {
|
||||
for _, pid := range diagnosis.MaxInodeNotEqualIDs {
|
||||
var partition *proto.MetaPartitionInfo
|
||||
if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil {
|
||||
err = fmt.Errorf("Partition not found, err:[%v] ", err)
|
||||
@ -219,10 +236,10 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on
|
||||
stdout("\n")
|
||||
stdout("%v\n", "[Partition with replica dentry count not equal]:")
|
||||
stdout("%v\n", dentryCountNotEqualInfoTableHeader)
|
||||
sort.SliceStable(diagnosis.DentryCountNotEqualReplicaMetaPartitionIDs, func(i, j int) bool {
|
||||
return diagnosis.DentryCountNotEqualReplicaMetaPartitionIDs[i] < diagnosis.DentryCountNotEqualReplicaMetaPartitionIDs[j]
|
||||
sort.SliceStable(diagnosis.DentryCountNotEqualIDs, func(i, j int) bool {
|
||||
return diagnosis.DentryCountNotEqualIDs[i] < diagnosis.DentryCountNotEqualIDs[j]
|
||||
})
|
||||
for _, pid := range diagnosis.DentryCountNotEqualReplicaMetaPartitionIDs {
|
||||
for _, pid := range diagnosis.DentryCountNotEqualIDs {
|
||||
var partition *proto.MetaPartitionInfo
|
||||
if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil {
|
||||
err = fmt.Errorf("Partition not found, err:[%v] ", err)
|
||||
@ -236,10 +253,10 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on
|
||||
stdout("\n")
|
||||
stdout("%v\n", "[Partition with excessive replicas]:")
|
||||
stdout("%v\n", partitionInfoTableHeader)
|
||||
sort.SliceStable(diagnosis.ExcessReplicaMetaPartitionIDs, func(i, j int) bool {
|
||||
return diagnosis.ExcessReplicaMetaPartitionIDs[i] < diagnosis.ExcessReplicaMetaPartitionIDs[j]
|
||||
sort.SliceStable(diagnosis.InConsistRreplicaCntMetaPartitionIDs, func(i, j int) bool {
|
||||
return diagnosis.InConsistRreplicaCntMetaPartitionIDs[i] < diagnosis.InConsistRreplicaCntMetaPartitionIDs[j]
|
||||
})
|
||||
for _, pid := range diagnosis.ExcessReplicaMetaPartitionIDs {
|
||||
for _, pid := range diagnosis.InConsistRreplicaCntMetaPartitionIDs {
|
||||
var partition *proto.MetaPartitionInfo
|
||||
if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil {
|
||||
err = fmt.Errorf("Partition not found, err:[%v] ", err)
|
||||
|
||||
@ -4583,7 +4583,7 @@ func (m *Server) getNodeInfoHandler(w http.ResponseWriter, r *http.Request) {
|
||||
sendOkReply(w, r, newSuccessHTTPReply(resp))
|
||||
}
|
||||
|
||||
func (m *Server) diagnoseMetaPartition(w http.ResponseWriter, r *http.Request) {
|
||||
func (m *Server) diagnoseMetaPartitionDelayDelted(w http.ResponseWriter, r *http.Request) {
|
||||
var (
|
||||
err error
|
||||
rstMsg *proto.MetaPartitionDiagnosis
|
||||
@ -4608,17 +4608,14 @@ func (m *Server) diagnoseMetaPartition(w http.ResponseWriter, r *http.Request) {
|
||||
defer func() {
|
||||
doStatAndMetric(proto.AdminDiagnoseMetaPartition, metric, err, nil)
|
||||
}()
|
||||
|
||||
corruptMpIDs = make([]uint64, 0)
|
||||
lackReplicaMpIDs = make([]uint64, 0)
|
||||
badReplicaMpIDs = make([]uint64, 0)
|
||||
excessReplicaMpIDs = make([]uint64, 0)
|
||||
|
||||
if inactiveNodes, err = m.cluster.checkInactiveMetaNodes(); err != nil {
|
||||
sendErrReply(w, r, newErrHTTPReply(err))
|
||||
return
|
||||
}
|
||||
|
||||
if lackReplicaMps, noLeaderMps, badReplicaMps, excessReplicaMPs,
|
||||
inodeCountNotEqualReplicaMps, maxInodeNotEqualMPs, dentryCountNotEqualReplicaMps, err = m.cluster.checkReplicaMetaPartitions(); err != nil {
|
||||
sendErrReply(w, r, newErrHTTPReply(err))
|
||||
@ -4636,14 +4633,12 @@ func (m *Server) diagnoseMetaPartition(w http.ResponseWriter, r *http.Request) {
|
||||
for _, mp := range excessReplicaMPs {
|
||||
excessReplicaMpIDs = append(excessReplicaMpIDs, mp.PartitionID)
|
||||
}
|
||||
|
||||
for _, mp := range inodeCountNotEqualReplicaMps {
|
||||
inodeCountNotEqualReplicaMpIDs = append(inodeCountNotEqualReplicaMpIDs, mp.PartitionID)
|
||||
}
|
||||
for _, mp := range maxInodeNotEqualMPs {
|
||||
maxInodeNotEqualReplicaMpIDs = append(maxInodeNotEqualReplicaMpIDs, mp.PartitionID)
|
||||
}
|
||||
|
||||
for _, mp := range dentryCountNotEqualReplicaMps {
|
||||
dentryCountNotEqualReplicaMpIDs = append(dentryCountNotEqualReplicaMpIDs, mp.PartitionID)
|
||||
}
|
||||
@ -4667,6 +4662,30 @@ func (m *Server) diagnoseMetaPartition(w http.ResponseWriter, r *http.Request) {
|
||||
sendOkReply(w, r, newSuccessHTTPReply(rstMsg))
|
||||
}
|
||||
|
||||
func (m *Server) diagnoseMetaPartition(w http.ResponseWriter, r *http.Request) {
|
||||
var (
|
||||
err error
|
||||
diagnosis interface{}
|
||||
)
|
||||
if err = r.ParseForm(); err != nil {
|
||||
return
|
||||
}
|
||||
if v1 := r.FormValue("v1"); v1 == "" {
|
||||
m.diagnoseMetaPartitionDelayDelted(w, r)
|
||||
return
|
||||
} else {
|
||||
metric := exporter.NewTPCnt(apiToMetricsName(proto.AdminDiagnoseMetaPartition))
|
||||
defer func() {
|
||||
doStatAndMetric(proto.AdminDiagnoseMetaPartition, metric, err, nil)
|
||||
}()
|
||||
if diagnosis, err = m.cluster.checkReplicaMetaPartitionsV1(); err != nil {
|
||||
sendErrReply(w, r, newErrHTTPReply(err))
|
||||
return
|
||||
}
|
||||
}
|
||||
sendOkReply(w, r, newSuccessHTTPReply(diagnosis))
|
||||
}
|
||||
|
||||
// Decommission a disk. This will decommission all the data partitions on this disk.
|
||||
// If parameter diskDisable is true, creating data partitions on this disk will be not allowed.
|
||||
func (m *Server) decommissionDisk(w http.ResponseWriter, r *http.Request) {
|
||||
@ -5991,6 +6010,7 @@ func (m *Server) getMetaPartition(w http.ResponseWriter, r *http.Request) {
|
||||
for i := 0; i < len(replicas); i++ {
|
||||
replicas[i] = &proto.MetaReplicaInfo{
|
||||
Addr: mp.Replicas[i].Addr,
|
||||
NodeID: mp.Replicas[i].nodeID,
|
||||
DomainAddr: mp.Replicas[i].metaNode.DomainAddr,
|
||||
MaxInodeID: mp.Replicas[i].MaxInodeID,
|
||||
ReportTime: mp.Replicas[i].ReportTime,
|
||||
|
||||
@ -81,6 +81,7 @@ type ClusterTopoSubItem struct {
|
||||
inodeCountNotEqualMP *sync.Map
|
||||
maxInodeNotEqualMP *sync.Map
|
||||
dentryCountNotEqualMP *sync.Map
|
||||
AbnormalRaftMP *sync.Map
|
||||
|
||||
mnMutex sync.RWMutex // meta node mutex
|
||||
dnMutex sync.RWMutex // data node mutex
|
||||
@ -465,6 +466,7 @@ func newCluster(name string, leaderInfo *LeaderInfo, fsm *MetadataFsm, partition
|
||||
c.inodeCountNotEqualMP = new(sync.Map)
|
||||
c.maxInodeNotEqualMP = new(sync.Map)
|
||||
c.dentryCountNotEqualMP = new(sync.Map)
|
||||
c.AbnormalRaftMP = new(sync.Map)
|
||||
c.lcMgr = newLifecycleManager()
|
||||
c.lcMgr.cluster = c
|
||||
c.snapshotMgr = newSnapshotManager()
|
||||
|
||||
@ -282,30 +282,24 @@ func (c *Cluster) checkReplicaMetaPartitions() (
|
||||
inodeCountNotEqualMPs = make([]*MetaPartition, 0)
|
||||
maxInodeNotEqualMPs = make([]*MetaPartition, 0)
|
||||
dentryCountNotEqualMPs = make([]*MetaPartition, 0)
|
||||
|
||||
markDeleteVolNames := make(VolNameSet)
|
||||
|
||||
vols := c.copyVols()
|
||||
for _, vol := range vols {
|
||||
if vol.Status == proto.VolStatusMarkDelete {
|
||||
markDeleteVolNames[vol.Name] = struct{}{}
|
||||
continue
|
||||
}
|
||||
|
||||
vol.mpsLock.RLock()
|
||||
for _, mp := range vol.MetaPartitions {
|
||||
if uint8(len(mp.Hosts)) < mp.ReplicaNum || uint8(len(mp.getActiveAddrs(defaultMetaPartitionTimeOutSec))) < mp.ReplicaNum {
|
||||
lackReplicaMetaPartitions = append(lackReplicaMetaPartitions, mp)
|
||||
}
|
||||
|
||||
if !mp.isLeaderExist() && (time.Now().Unix()-mp.LeaderReportTime > c.cfg.MpNoLeaderReportIntervalSec) {
|
||||
noLeaderMetaPartitions = append(noLeaderMetaPartitions, mp)
|
||||
}
|
||||
|
||||
if uint8(len(mp.Hosts)) > mp.ReplicaNum || uint8(len(mp.Replicas)) > mp.ReplicaNum {
|
||||
excessReplicaMetaPartitions = append(excessReplicaMetaPartitions, mp)
|
||||
}
|
||||
|
||||
for _, replica := range mp.Replicas {
|
||||
if replica.Status == proto.Unavailable {
|
||||
unavailableReplicaMPs = append(unavailableReplicaMPs, mp)
|
||||
@ -343,6 +337,69 @@ func (c *Cluster) checkReplicaMetaPartitions() (
|
||||
return
|
||||
}
|
||||
|
||||
func (c *Cluster) checkReplicaMetaPartitionsV1() (diagnosis *proto.MetaPartitionDiagnosisV1, err error) {
|
||||
diagnosis = &proto.MetaPartitionDiagnosisV1{}
|
||||
markDeleteVolNames := make(VolNameSet)
|
||||
vols := c.copyVols()
|
||||
for _, vol := range vols {
|
||||
if vol.Status == proto.VolStatusMarkDelete {
|
||||
markDeleteVolNames[vol.Name] = struct{}{}
|
||||
continue
|
||||
}
|
||||
|
||||
vol.mpsLock.RLock()
|
||||
for _, mp := range vol.MetaPartitions {
|
||||
if uint8(len(mp.Hosts)) < mp.ReplicaNum || uint8(len(mp.getActiveAddrs(defaultMetaPartitionTimeOutSec))) < mp.ReplicaNum {
|
||||
diagnosis.LackReplicaMetaPartitionIDs = append(diagnosis.LackReplicaMetaPartitionIDs, mp.PartitionID)
|
||||
}
|
||||
|
||||
if !mp.isLeaderExist() && (time.Now().Unix()-mp.LeaderReportTime > c.cfg.MpNoLeaderReportIntervalSec) {
|
||||
diagnosis.NoLeaderMetaPartitionIDs = append(diagnosis.NoLeaderMetaPartitionIDs, mp.PartitionID)
|
||||
}
|
||||
|
||||
if uint8(len(mp.Hosts)) > mp.ReplicaNum || uint8(len(mp.Replicas)) > mp.ReplicaNum {
|
||||
diagnosis.InConsistRreplicaCntMetaPartitionIDs = append(diagnosis.InConsistRreplicaCntMetaPartitionIDs, mp.PartitionID)
|
||||
}
|
||||
|
||||
for _, replica := range mp.Replicas {
|
||||
if replica.Status == proto.Unavailable {
|
||||
diagnosis.UnavailableMetaPartitionIDs = append(diagnosis.UnavailableMetaPartitionIDs, mp.PartitionID)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
vol.mpsLock.RUnlock()
|
||||
}
|
||||
|
||||
setAbnormalIDs := func(mpMap *sync.Map) []uint64 {
|
||||
var resultIDs []uint64
|
||||
mpMap.Range(func(key, value interface{}) bool {
|
||||
mp := value.(*MetaPartition)
|
||||
if _, ok := markDeleteVolNames[mp.volName]; !ok {
|
||||
resultIDs = append(resultIDs, mp.PartitionID)
|
||||
}
|
||||
return true
|
||||
})
|
||||
return resultIDs
|
||||
}
|
||||
|
||||
diagnosis.InodeCountNotEqualIDs = setAbnormalIDs(c.inodeCountNotEqualMP)
|
||||
diagnosis.MaxInodeNotEqualIDs = setAbnormalIDs(c.maxInodeNotEqualMP)
|
||||
diagnosis.DentryCountNotEqualIDs = setAbnormalIDs(c.dentryCountNotEqualMP)
|
||||
diagnosis.AbnormalRaftIDs = setAbnormalIDs(c.AbnormalRaftMP)
|
||||
|
||||
log.LogInfof("clusterID[%v], lackReplicaMetaPartitions count:[%v], noLeaderMetaPartitions count[%v]"+
|
||||
"unavailableReplicaMPs count:[%v], excessReplicaMp count:[%v], AbnormalRaftIDs count:[%v]",
|
||||
c.Name, len(diagnosis.LackReplicaMetaPartitionIDs), len(diagnosis.NoLeaderMetaPartitionIDs),
|
||||
len(diagnosis.UnavailableMetaPartitionIDs), len(diagnosis.InConsistRreplicaCntMetaPartitionIDs), len(diagnosis.AbnormalRaftIDs))
|
||||
|
||||
if diagnosis.InactiveMetaNodes, err = c.checkInactiveMetaNodes(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (c *Cluster) deleteMetaReplica(partition *MetaPartition, addr string, validate bool, forceDel bool) (err error) {
|
||||
defer func() {
|
||||
if err != nil {
|
||||
@ -619,6 +676,7 @@ func (c *Cluster) doLoadMetaPartition(mp *MetaPartition) {
|
||||
}(host)
|
||||
}
|
||||
wg.Wait()
|
||||
mp.checkPeerDiffWithRaft(c)
|
||||
select {
|
||||
case err := <-errChannel:
|
||||
msg := fmt.Sprintf("action[doLoadMetaPartition] vol[%v],mpID[%v],err[%v]", mp.volName, mp.PartitionID, err.Error())
|
||||
|
||||
@ -52,6 +52,48 @@ func (c *Cluster) checkLoadMetaPartitions() {
|
||||
}
|
||||
}
|
||||
|
||||
func (mp *MetaPartition) checkPeerDiffWithRaft(c *Cluster) {
|
||||
if len(mp.LoadResponse) == 0 {
|
||||
return
|
||||
}
|
||||
if !mp.doCompare() {
|
||||
return
|
||||
}
|
||||
var (
|
||||
addrArr = make(map[uint64]string)
|
||||
leaderID uint64
|
||||
)
|
||||
for _, mr := range mp.Replicas {
|
||||
addrArr[mr.nodeID] = mr.Addr
|
||||
if mr.IsLeader {
|
||||
leaderID = mr.nodeID
|
||||
}
|
||||
}
|
||||
|
||||
for _, info := range mp.LoadResponse {
|
||||
if info.RaftInfo.RaftStatus.NodeID != leaderID {
|
||||
continue
|
||||
}
|
||||
for peer := range info.RaftInfo.RaftStatus.Replicas {
|
||||
if _, ok := addrArr[peer]; !ok {
|
||||
c.AbnormalRaftMP.Store(mp.PartitionID, mp)
|
||||
return
|
||||
}
|
||||
}
|
||||
if len(info.RaftInfo.PendingPeers) != 0 {
|
||||
c.AbnormalRaftMP.Store(mp.PartitionID, mp)
|
||||
return
|
||||
}
|
||||
if len(info.RaftInfo.DownReplicas) != 0 {
|
||||
c.AbnormalRaftMP.Store(mp.PartitionID, mp)
|
||||
return
|
||||
}
|
||||
c.AbnormalRaftMP.Delete(mp.PartitionID)
|
||||
return
|
||||
}
|
||||
c.AbnormalRaftMP.Delete(mp.PartitionID)
|
||||
}
|
||||
|
||||
func (mp *MetaPartition) checkSnapshot(c *Cluster) {
|
||||
if len(mp.LoadResponse) == 0 {
|
||||
return
|
||||
|
||||
@ -1431,6 +1431,14 @@ func (mp *metaPartition) ResponseLoadMetaPartition(p *Packet) (err error) {
|
||||
resp.DentryCount = uint64(mp.GetDentryTreeLen())
|
||||
resp.ApplyID = mp.getApplyID()
|
||||
resp.CommittedID = mp.getCommittedID()
|
||||
|
||||
resp.RaftInfo.DownReplicas = mp.config.RaftStore.RaftServer().GetDownReplicas(mp.config.PartitionId)
|
||||
resp.RaftInfo.PendingPeers = mp.config.RaftStore.RaftServer().GetPendingReplica(mp.config.PartitionId)
|
||||
if rStatus := mp.config.RaftStore.RaftStatus(mp.config.PartitionId); rStatus != nil {
|
||||
resp.RaftInfo.RaftStatus = *rStatus
|
||||
}
|
||||
resp.RaftInfo.Hosts = mp.config.Peers
|
||||
|
||||
if err != nil {
|
||||
err = errors.Trace(err,
|
||||
"[ResponseLoadMetaPartition] check snapshot")
|
||||
|
||||
@ -23,6 +23,8 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/cubefs/cubefs/depends/tiglabs/raft"
|
||||
|
||||
"github.com/cubefs/cubefs/util"
|
||||
"github.com/cubefs/cubefs/util/log"
|
||||
)
|
||||
@ -1092,6 +1094,13 @@ type MetaPartitionLoadRequest struct {
|
||||
PartitionID uint64
|
||||
}
|
||||
|
||||
type RaftInfo struct {
|
||||
RaftStatus raft.Status
|
||||
PendingPeers []uint64
|
||||
DownReplicas []raft.DownReplica
|
||||
Hosts []Peer
|
||||
}
|
||||
|
||||
// MetaPartitionLoadResponse defines the response to the request of loading meta partition.
|
||||
type MetaPartitionLoadResponse struct {
|
||||
PartitionID uint64
|
||||
@ -1102,6 +1111,7 @@ type MetaPartitionLoadResponse struct {
|
||||
DentryCount uint64
|
||||
InodeCount uint64
|
||||
Addr string
|
||||
RaftInfo RaftInfo
|
||||
}
|
||||
|
||||
// DataPartitionResponse defines the response from a data node to the master that is related to a data partition.
|
||||
|
||||
@ -120,6 +120,7 @@ type MetaPartitionInfo struct {
|
||||
// MetaReplica defines the replica of a meta partition
|
||||
type MetaReplicaInfo struct {
|
||||
Addr string
|
||||
NodeID uint64
|
||||
DomainAddr string
|
||||
MaxInodeID uint64
|
||||
ReportTime int64
|
||||
@ -400,6 +401,7 @@ type DataPartitionDiagnosis struct {
|
||||
}
|
||||
|
||||
// meta partition diagnosis represents the inactive meta nodes, corrupt meta partitions, and meta partitions lack of replicas
|
||||
|
||||
type MetaPartitionDiagnosis struct {
|
||||
InactiveMetaNodes []string
|
||||
CorruptMetaPartitionIDs []uint64
|
||||
@ -412,6 +414,19 @@ type MetaPartitionDiagnosis struct {
|
||||
DentryCountNotEqualReplicaMetaPartitionIDs []uint64
|
||||
}
|
||||
|
||||
type MetaPartitionDiagnosisV1 struct {
|
||||
InactiveMetaNodes []string
|
||||
NoLeaderMetaPartitionIDs []uint64
|
||||
LackReplicaMetaPartitionIDs []uint64
|
||||
BadMetaPartitionIDs []BadPartitionView
|
||||
UnavailableMetaPartitionIDs []uint64
|
||||
InConsistRreplicaCntMetaPartitionIDs []uint64
|
||||
InodeCountNotEqualIDs []uint64
|
||||
MaxInodeNotEqualIDs []uint64
|
||||
DentryCountNotEqualIDs []uint64
|
||||
AbnormalRaftIDs []uint64
|
||||
}
|
||||
|
||||
type FailedDpInfo struct {
|
||||
PartitionID uint64
|
||||
ErrMsg string
|
||||
|
||||
@ -161,9 +161,11 @@ func (api *AdminAPI) DiagnoseDataPartition(ignoreDiscardDp bool) (diagnosis *pro
|
||||
return
|
||||
}
|
||||
|
||||
func (api *AdminAPI) DiagnoseMetaPartition() (diagnosis *proto.MetaPartitionDiagnosis, err error) {
|
||||
diagnosis = &proto.MetaPartitionDiagnosis{}
|
||||
err = api.mc.requestWith(diagnosis, newRequest(get, proto.AdminDiagnoseMetaPartition).Header(api.h))
|
||||
func (api *AdminAPI) DiagnoseMetaPartition() (diagnosis *proto.MetaPartitionDiagnosisV1, err error) {
|
||||
diagnosis = &proto.MetaPartitionDiagnosisV1{}
|
||||
err = api.mc.requestWith(diagnosis, newRequest(get, proto.AdminDiagnoseMetaPartition).Header(api.h).Param(
|
||||
anyParam{"v1", "true"},
|
||||
))
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user