feat(metanode): Add raft members consistent check with master hosts in metanode member check

. #1000165698

Signed-off-by: leonrayang <changliang@oppo.com>

cli/cmd/fmt.go

@@ -412,6 +412,14 @@ var (
 	partitionInfoTableHeader  = fmt.Sprintf(partitionInfoTablePattern,
 		"ID", "VOLUME", "REPLICAS", "STATUS", "MediaType", "MEMBERS")
 
+	PeerAbnormalRaftPartitionInfoTablePattern = "%-8v    %-32v    %-8v    %-12v    %-12v    %-12v   %v%v%v"
+	PeerAbnormalRaftPartitionInfoHeader       = fmt.Sprintf(PeerAbnormalRaftPartitionInfoTablePattern,
+		"ID", "VOLUME", "REPLICAS", "STATUS", "MediaType", "MEMBERS", "DIFF-RAFT-MEMBERS", "|DOWN-MEMBERS", "|PEND-MEMBERS")
+
+	badMpReplicaPartitionInfoTablePattern = "%-8v    %-32v    %-8v    %-12v    %-64v    %-24v"
+	badMpReplicaPartitionInfoTableHeader  = fmt.Sprintf(badMpReplicaPartitionInfoTablePattern,
+		"ID", "VOLUME", "REPLICAS", "STATUS", "MEMBERS", "UNAVAILABLE_REPLICAS")
+
 	badReplicaPartitionInfoTablePattern = "%-8v    %-32v    %-8v    %-12v    %-64v    %-24v"
 	badReplicaPartitionInfoTableHeader  = fmt.Sprintf(badReplicaPartitionInfoTablePattern,
 		"ID", "VOLUME", "REPLICAS", "STATUS", "MEMBERS", "UNAVAILABLE_REPLICAS")
@@ -553,6 +561,65 @@ func formatMetaPartitionInfoRow(partition *proto.MetaPartitionInfo) string {
 		formatDataPartitionStatus(partition.Status), "N/A", strings.Join(partition.Hosts, ", "))
 }
 
+func formatMetaPartitionInfoRowWithRaft(partition *proto.MetaPartitionInfo) string {
+	var (
+		info         *proto.MetaPartitionLoadResponse
+		addrArr      = make(map[uint64]string)
+		diffHosts    []string
+		downHosts    []string
+		pendingHosts []string
+	)
+	for _, peer := range partition.Peers {
+		addrArr[peer.ID] = peer.Addr
+	}
+
+	for _, info = range partition.LoadResponse {
+		if info.RaftInfo.RaftStatus.Leader == info.RaftInfo.RaftStatus.NodeID {
+			break
+		}
+	}
+	if info == nil {
+		return ""
+	}
+	for _, peer := range info.RaftInfo.Hosts {
+		if _, ok := addrArr[peer.ID]; ok {
+			continue
+		}
+		diffHosts = append(diffHosts, peer.Addr)
+		addrArr[peer.ID] = peer.Addr
+	}
+
+	for _, dr := range info.RaftInfo.DownReplicas {
+		if host, ok := addrArr[dr.NodeID]; ok {
+			downHosts = append(downHosts, host)
+		}
+	}
+	for _, pr := range info.RaftInfo.PendingPeers {
+		if host, ok := addrArr[pr]; ok {
+			pendingHosts = append(pendingHosts, host)
+		}
+	}
+	var (
+		diffInfo    = "N/A"
+		downInfo    = "N/A"
+		pendingInfo = "N/A"
+	)
+	if len(diffHosts) != 0 {
+		diffInfo = strings.Join(diffHosts, ", ")
+	}
+	if len(downHosts) != 0 {
+		downInfo = strings.Join(downHosts, ", ")
+	}
+	if len(pendingHosts) != 0 {
+		pendingInfo = strings.Join(pendingHosts, ", ")
+	}
+
+	return fmt.Sprintf(PeerAbnormalRaftPartitionInfoTablePattern,
+		partition.PartitionID, partition.VolName, partition.ReplicaNum,
+		formatDataPartitionStatus(partition.Status), "N/A", strings.Join(partition.Hosts, ", "),
+		diffInfo+"|", downInfo+"|", pendingInfo)
+}
+
 func formatBadReplicaMpInfoRow(partition *proto.MetaPartitionInfo) string {
 	sb := strings.Builder{}
 	sb.WriteString("[")

cli/cmd/metapartition.go

@@ -90,7 +90,7 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on
 "reset" command will be released in next version.`,
 		Run: func(cmd *cobra.Command, args []string) {
 			var (
-				diagnosis *proto.MetaPartitionDiagnosis
+				diagnosis *proto.MetaPartitionDiagnosisV1
 				metaNodes []*proto.MetaNodeInfo
 				err       error
 			)
@@ -120,10 +120,10 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on
 			stdout("\n")
 			stdout("[Corrupt meta partitions](no leader):\n")
 			stdout("%v\n", partitionInfoTableHeader)
-			sort.SliceStable(diagnosis.CorruptMetaPartitionIDs, func(i, j int) bool {
+			sort.SliceStable(diagnosis.NoLeaderMetaPartitionIDs, func(i, j int) bool {
-				return diagnosis.CorruptMetaPartitionIDs[i] < diagnosis.CorruptMetaPartitionIDs[j]
+				return diagnosis.NoLeaderMetaPartitionIDs[i] < diagnosis.NoLeaderMetaPartitionIDs[j]
 			})
-			for _, pid := range diagnosis.CorruptMetaPartitionIDs {
+			for _, pid := range diagnosis.NoLeaderMetaPartitionIDs {
 				var partition *proto.MetaPartitionInfo
 				if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil {
 					err = fmt.Errorf("Partition not found, err:[%v] ", err)
@@ -149,6 +149,23 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on
 				}
 			}
 
+			stdout("\n")
+			stdout("%v\n", "[Meta partition Abnormal Raft Info]:")
+			stdout("%v\n", PeerAbnormalRaftPartitionInfoHeader)
+			sort.SliceStable(diagnosis.AbnormalRaftIDs, func(i, j int) bool {
+				return diagnosis.AbnormalRaftIDs[i] < diagnosis.AbnormalRaftIDs[j]
+			})
+			for _, pid := range diagnosis.AbnormalRaftIDs {
+				var partition *proto.MetaPartitionInfo
+				if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil {
+					err = fmt.Errorf("Partition not found, err:[%v] ", err)
+					return
+				}
+				if partition != nil {
+					stdout("%v\n", formatMetaPartitionInfoRowWithRaft(partition))
+				}
+			}
+
 			stdout("\n")
 			stdout("%v\n", "[Bad meta partitions(decommission not completed)]:")
 			badPartitionTablePattern := "%-8v    %-10v\n"
@@ -164,11 +181,11 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on
 
 			stdout("\n")
 			stdout("%v\n", "[Meta Partition has unavailable replica]:")
-			stdout("%v\n", badReplicaPartitionInfoTableHeader)
+			stdout("%v\n", badMpReplicaPartitionInfoTableHeader)
-			sort.SliceStable(diagnosis.BadReplicaMetaPartitionIDs, func(i, j int) bool {
+			sort.SliceStable(diagnosis.UnavailableMetaPartitionIDs, func(i, j int) bool {
-				return diagnosis.BadReplicaMetaPartitionIDs[i] < diagnosis.BadReplicaMetaPartitionIDs[j]
+				return diagnosis.UnavailableMetaPartitionIDs[i] < diagnosis.UnavailableMetaPartitionIDs[j]
 			})
-			for _, pid := range diagnosis.BadReplicaMetaPartitionIDs {
+			for _, pid := range diagnosis.UnavailableMetaPartitionIDs {
 				var partition *proto.MetaPartitionInfo
 				if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil {
 					err = fmt.Errorf("Partition not found, err:[%v] ", err)
@@ -185,10 +202,10 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on
 			stdout("\n")
 			stdout("%v\n", "[Partition with replica inode count not equal]:")
 			stdout("%v\n", inodeCountNotEqualInfoTableHeader)
-			sort.SliceStable(diagnosis.InodeCountNotEqualReplicaMetaPartitionIDs, func(i, j int) bool {
+			sort.SliceStable(diagnosis.InodeCountNotEqualIDs, func(i, j int) bool {
-				return diagnosis.InodeCountNotEqualReplicaMetaPartitionIDs[i] < diagnosis.InodeCountNotEqualReplicaMetaPartitionIDs[j]
+				return diagnosis.InodeCountNotEqualIDs[i] < diagnosis.InodeCountNotEqualIDs[j]
 			})
-			for _, pid := range diagnosis.InodeCountNotEqualReplicaMetaPartitionIDs {
+			for _, pid := range diagnosis.InodeCountNotEqualIDs {
 				var partition *proto.MetaPartitionInfo
 				if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil {
 					err = fmt.Errorf("Partition not found, err:[%v] ", err)
@@ -202,10 +219,10 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on
 			stdout("\n")
 			stdout("%v\n", "[Partition with replica max inode not equal]:")
 			stdout("%v\n", maxInodeNotEqualInfoTableHeader)
-			sort.SliceStable(diagnosis.MaxInodeNotEqualReplicaMetaPartitionIDs, func(i, j int) bool {
+			sort.SliceStable(diagnosis.MaxInodeNotEqualIDs, func(i, j int) bool {
-				return diagnosis.MaxInodeNotEqualReplicaMetaPartitionIDs[i] < diagnosis.MaxInodeNotEqualReplicaMetaPartitionIDs[j]
+				return diagnosis.MaxInodeNotEqualIDs[i] < diagnosis.MaxInodeNotEqualIDs[j]
 			})
-			for _, pid := range diagnosis.MaxInodeNotEqualReplicaMetaPartitionIDs {
+			for _, pid := range diagnosis.MaxInodeNotEqualIDs {
 				var partition *proto.MetaPartitionInfo
 				if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil {
 					err = fmt.Errorf("Partition not found, err:[%v] ", err)
@@ -219,10 +236,10 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on
 			stdout("\n")
 			stdout("%v\n", "[Partition with replica dentry count not equal]:")
 			stdout("%v\n", dentryCountNotEqualInfoTableHeader)
-			sort.SliceStable(diagnosis.DentryCountNotEqualReplicaMetaPartitionIDs, func(i, j int) bool {
+			sort.SliceStable(diagnosis.DentryCountNotEqualIDs, func(i, j int) bool {
-				return diagnosis.DentryCountNotEqualReplicaMetaPartitionIDs[i] < diagnosis.DentryCountNotEqualReplicaMetaPartitionIDs[j]
+				return diagnosis.DentryCountNotEqualIDs[i] < diagnosis.DentryCountNotEqualIDs[j]
 			})
-			for _, pid := range diagnosis.DentryCountNotEqualReplicaMetaPartitionIDs {
+			for _, pid := range diagnosis.DentryCountNotEqualIDs {
 				var partition *proto.MetaPartitionInfo
 				if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil {
 					err = fmt.Errorf("Partition not found, err:[%v] ", err)
@@ -236,10 +253,10 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on
 			stdout("\n")
 			stdout("%v\n", "[Partition with excessive replicas]:")
 			stdout("%v\n", partitionInfoTableHeader)
-			sort.SliceStable(diagnosis.ExcessReplicaMetaPartitionIDs, func(i, j int) bool {
+			sort.SliceStable(diagnosis.InConsistRreplicaCntMetaPartitionIDs, func(i, j int) bool {
-				return diagnosis.ExcessReplicaMetaPartitionIDs[i] < diagnosis.ExcessReplicaMetaPartitionIDs[j]
+				return diagnosis.InConsistRreplicaCntMetaPartitionIDs[i] < diagnosis.InConsistRreplicaCntMetaPartitionIDs[j]
 			})
-			for _, pid := range diagnosis.ExcessReplicaMetaPartitionIDs {
+			for _, pid := range diagnosis.InConsistRreplicaCntMetaPartitionIDs {
 				var partition *proto.MetaPartitionInfo
 				if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil {
 					err = fmt.Errorf("Partition not found, err:[%v] ", err)

master/api_service.go

@@ -4583,7 +4583,7 @@ func (m *Server) getNodeInfoHandler(w http.ResponseWriter, r *http.Request) {
 	sendOkReply(w, r, newSuccessHTTPReply(resp))
 }
 
-func (m *Server) diagnoseMetaPartition(w http.ResponseWriter, r *http.Request) {
+func (m *Server) diagnoseMetaPartitionDelayDelted(w http.ResponseWriter, r *http.Request) {
 	var (
 		err                             error
 		rstMsg                          *proto.MetaPartitionDiagnosis
@@ -4608,17 +4608,14 @@ func (m *Server) diagnoseMetaPartition(w http.ResponseWriter, r *http.Request) {
 	defer func() {
 		doStatAndMetric(proto.AdminDiagnoseMetaPartition, metric, err, nil)
 	}()
-
 	corruptMpIDs = make([]uint64, 0)
 	lackReplicaMpIDs = make([]uint64, 0)
 	badReplicaMpIDs = make([]uint64, 0)
 	excessReplicaMpIDs = make([]uint64, 0)
-
 	if inactiveNodes, err = m.cluster.checkInactiveMetaNodes(); err != nil {
 		sendErrReply(w, r, newErrHTTPReply(err))
 		return
 	}
-
 	if lackReplicaMps, noLeaderMps, badReplicaMps, excessReplicaMPs,
 		inodeCountNotEqualReplicaMps, maxInodeNotEqualMPs, dentryCountNotEqualReplicaMps, err = m.cluster.checkReplicaMetaPartitions(); err != nil {
 		sendErrReply(w, r, newErrHTTPReply(err))
@@ -4636,14 +4633,12 @@ func (m *Server) diagnoseMetaPartition(w http.ResponseWriter, r *http.Request) {
 	for _, mp := range excessReplicaMPs {
 		excessReplicaMpIDs = append(excessReplicaMpIDs, mp.PartitionID)
 	}
-
 	for _, mp := range inodeCountNotEqualReplicaMps {
 		inodeCountNotEqualReplicaMpIDs = append(inodeCountNotEqualReplicaMpIDs, mp.PartitionID)
 	}
 	for _, mp := range maxInodeNotEqualMPs {
 		maxInodeNotEqualReplicaMpIDs = append(maxInodeNotEqualReplicaMpIDs, mp.PartitionID)
 	}
-
 	for _, mp := range dentryCountNotEqualReplicaMps {
 		dentryCountNotEqualReplicaMpIDs = append(dentryCountNotEqualReplicaMpIDs, mp.PartitionID)
 	}
@@ -4667,6 +4662,30 @@ func (m *Server) diagnoseMetaPartition(w http.ResponseWriter, r *http.Request) {
 	sendOkReply(w, r, newSuccessHTTPReply(rstMsg))
 }
 
+func (m *Server) diagnoseMetaPartition(w http.ResponseWriter, r *http.Request) {
+	var (
+		err       error
+		diagnosis interface{}
+	)
+	if err = r.ParseForm(); err != nil {
+		return
+	}
+	if v1 := r.FormValue("v1"); v1 == "" {
+		m.diagnoseMetaPartitionDelayDelted(w, r)
+		return
+	} else {
+		metric := exporter.NewTPCnt(apiToMetricsName(proto.AdminDiagnoseMetaPartition))
+		defer func() {
+			doStatAndMetric(proto.AdminDiagnoseMetaPartition, metric, err, nil)
+		}()
+		if diagnosis, err = m.cluster.checkReplicaMetaPartitionsV1(); err != nil {
+			sendErrReply(w, r, newErrHTTPReply(err))
+			return
+		}
+	}
+	sendOkReply(w, r, newSuccessHTTPReply(diagnosis))
+}
+
 // Decommission a disk. This will decommission all the data partitions on this disk.
 // If parameter diskDisable is true, creating data partitions on this disk will be not allowed.
 func (m *Server) decommissionDisk(w http.ResponseWriter, r *http.Request) {
@@ -5991,6 +6010,7 @@ func (m *Server) getMetaPartition(w http.ResponseWriter, r *http.Request) {
 		for i := 0; i < len(replicas); i++ {
 			replicas[i] = &proto.MetaReplicaInfo{
 				Addr:            mp.Replicas[i].Addr,
+				NodeID:          mp.Replicas[i].nodeID,
 				DomainAddr:      mp.Replicas[i].metaNode.DomainAddr,
 				MaxInodeID:      mp.Replicas[i].MaxInodeID,
 				ReportTime:      mp.Replicas[i].ReportTime,

master/cluster.go

@@ -81,6 +81,7 @@ type ClusterTopoSubItem struct {
 	inodeCountNotEqualMP  *sync.Map
 	maxInodeNotEqualMP    *sync.Map
 	dentryCountNotEqualMP *sync.Map
+	AbnormalRaftMP        *sync.Map
 
 	mnMutex sync.RWMutex // meta node mutex
 	dnMutex sync.RWMutex // data node mutex
@@ -465,6 +466,7 @@ func newCluster(name string, leaderInfo *LeaderInfo, fsm *MetadataFsm, partition
 	c.inodeCountNotEqualMP = new(sync.Map)
 	c.maxInodeNotEqualMP = new(sync.Map)
 	c.dentryCountNotEqualMP = new(sync.Map)
+	c.AbnormalRaftMP = new(sync.Map)
 	c.lcMgr = newLifecycleManager()
 	c.lcMgr.cluster = c
 	c.snapshotMgr = newSnapshotManager()

master/cluster_task.go

@@ -282,30 +282,24 @@ func (c *Cluster) checkReplicaMetaPartitions() (
 	inodeCountNotEqualMPs = make([]*MetaPartition, 0)
 	maxInodeNotEqualMPs = make([]*MetaPartition, 0)
 	dentryCountNotEqualMPs = make([]*MetaPartition, 0)
-
 	markDeleteVolNames := make(VolNameSet)
-
 	vols := c.copyVols()
 	for _, vol := range vols {
 		if vol.Status == proto.VolStatusMarkDelete {
 			markDeleteVolNames[vol.Name] = struct{}{}
 			continue
 		}
-
 		vol.mpsLock.RLock()
 		for _, mp := range vol.MetaPartitions {
 			if uint8(len(mp.Hosts)) < mp.ReplicaNum || uint8(len(mp.getActiveAddrs(defaultMetaPartitionTimeOutSec))) < mp.ReplicaNum {
 				lackReplicaMetaPartitions = append(lackReplicaMetaPartitions, mp)
 			}
-
 			if !mp.isLeaderExist() && (time.Now().Unix()-mp.LeaderReportTime > c.cfg.MpNoLeaderReportIntervalSec) {
 				noLeaderMetaPartitions = append(noLeaderMetaPartitions, mp)
 			}
-
 			if uint8(len(mp.Hosts)) > mp.ReplicaNum || uint8(len(mp.Replicas)) > mp.ReplicaNum {
 				excessReplicaMetaPartitions = append(excessReplicaMetaPartitions, mp)
 			}
-
 			for _, replica := range mp.Replicas {
 				if replica.Status == proto.Unavailable {
 					unavailableReplicaMPs = append(unavailableReplicaMPs, mp)
@@ -343,6 +337,69 @@ func (c *Cluster) checkReplicaMetaPartitions() (
 	return
 }
 
+func (c *Cluster) checkReplicaMetaPartitionsV1() (diagnosis *proto.MetaPartitionDiagnosisV1, err error) {
+	diagnosis = &proto.MetaPartitionDiagnosisV1{}
+	markDeleteVolNames := make(VolNameSet)
+	vols := c.copyVols()
+	for _, vol := range vols {
+		if vol.Status == proto.VolStatusMarkDelete {
+			markDeleteVolNames[vol.Name] = struct{}{}
+			continue
+		}
+
+		vol.mpsLock.RLock()
+		for _, mp := range vol.MetaPartitions {
+			if uint8(len(mp.Hosts)) < mp.ReplicaNum || uint8(len(mp.getActiveAddrs(defaultMetaPartitionTimeOutSec))) < mp.ReplicaNum {
+				diagnosis.LackReplicaMetaPartitionIDs = append(diagnosis.LackReplicaMetaPartitionIDs, mp.PartitionID)
+			}
+
+			if !mp.isLeaderExist() && (time.Now().Unix()-mp.LeaderReportTime > c.cfg.MpNoLeaderReportIntervalSec) {
+				diagnosis.NoLeaderMetaPartitionIDs = append(diagnosis.NoLeaderMetaPartitionIDs, mp.PartitionID)
+			}
+
+			if uint8(len(mp.Hosts)) > mp.ReplicaNum || uint8(len(mp.Replicas)) > mp.ReplicaNum {
+				diagnosis.InConsistRreplicaCntMetaPartitionIDs = append(diagnosis.InConsistRreplicaCntMetaPartitionIDs, mp.PartitionID)
+			}
+
+			for _, replica := range mp.Replicas {
+				if replica.Status == proto.Unavailable {
+					diagnosis.UnavailableMetaPartitionIDs = append(diagnosis.UnavailableMetaPartitionIDs, mp.PartitionID)
+					break
+				}
+			}
+		}
+		vol.mpsLock.RUnlock()
+	}
+
+	setAbnormalIDs := func(mpMap *sync.Map) []uint64 {
+		var resultIDs []uint64
+		mpMap.Range(func(key, value interface{}) bool {
+			mp := value.(*MetaPartition)
+			if _, ok := markDeleteVolNames[mp.volName]; !ok {
+				resultIDs = append(resultIDs, mp.PartitionID)
+			}
+			return true
+		})
+		return resultIDs
+	}
+
+	diagnosis.InodeCountNotEqualIDs = setAbnormalIDs(c.inodeCountNotEqualMP)
+	diagnosis.MaxInodeNotEqualIDs = setAbnormalIDs(c.maxInodeNotEqualMP)
+	diagnosis.DentryCountNotEqualIDs = setAbnormalIDs(c.dentryCountNotEqualMP)
+	diagnosis.AbnormalRaftIDs = setAbnormalIDs(c.AbnormalRaftMP)
+
+	log.LogInfof("clusterID[%v], lackReplicaMetaPartitions count:[%v], noLeaderMetaPartitions count[%v]"+
+		"unavailableReplicaMPs count:[%v], excessReplicaMp count:[%v], AbnormalRaftIDs count:[%v]",
+		c.Name, len(diagnosis.LackReplicaMetaPartitionIDs), len(diagnosis.NoLeaderMetaPartitionIDs),
+		len(diagnosis.UnavailableMetaPartitionIDs), len(diagnosis.InConsistRreplicaCntMetaPartitionIDs), len(diagnosis.AbnormalRaftIDs))
+
+	if diagnosis.InactiveMetaNodes, err = c.checkInactiveMetaNodes(); err != nil {
+		return
+	}
+
+	return
+}
+
 func (c *Cluster) deleteMetaReplica(partition *MetaPartition, addr string, validate bool, forceDel bool) (err error) {
 	defer func() {
 		if err != nil {
@@ -619,6 +676,7 @@ func (c *Cluster) doLoadMetaPartition(mp *MetaPartition) {
 		}(host)
 	}
 	wg.Wait()
+	mp.checkPeerDiffWithRaft(c)
 	select {
 	case err := <-errChannel:
 		msg := fmt.Sprintf("action[doLoadMetaPartition] vol[%v],mpID[%v],err[%v]", mp.volName, mp.PartitionID, err.Error())

master/meta_partition_manager.go

@@ -52,6 +52,48 @@ func (c *Cluster) checkLoadMetaPartitions() {
 	}
 }
 
+func (mp *MetaPartition) checkPeerDiffWithRaft(c *Cluster) {
+	if len(mp.LoadResponse) == 0 {
+		return
+	}
+	if !mp.doCompare() {
+		return
+	}
+	var (
+		addrArr  = make(map[uint64]string)
+		leaderID uint64
+	)
+	for _, mr := range mp.Replicas {
+		addrArr[mr.nodeID] = mr.Addr
+		if mr.IsLeader {
+			leaderID = mr.nodeID
+		}
+	}
+
+	for _, info := range mp.LoadResponse {
+		if info.RaftInfo.RaftStatus.NodeID != leaderID {
+			continue
+		}
+		for peer := range info.RaftInfo.RaftStatus.Replicas {
+			if _, ok := addrArr[peer]; !ok {
+				c.AbnormalRaftMP.Store(mp.PartitionID, mp)
+				return
+			}
+		}
+		if len(info.RaftInfo.PendingPeers) != 0 {
+			c.AbnormalRaftMP.Store(mp.PartitionID, mp)
+			return
+		}
+		if len(info.RaftInfo.DownReplicas) != 0 {
+			c.AbnormalRaftMP.Store(mp.PartitionID, mp)
+			return
+		}
+		c.AbnormalRaftMP.Delete(mp.PartitionID)
+		return
+	}
+	c.AbnormalRaftMP.Delete(mp.PartitionID)
+}
+
 func (mp *MetaPartition) checkSnapshot(c *Cluster) {
 	if len(mp.LoadResponse) == 0 {
 		return

metanode/partition.go

@@ -1431,6 +1431,14 @@ func (mp *metaPartition) ResponseLoadMetaPartition(p *Packet) (err error) {
 	resp.DentryCount = uint64(mp.GetDentryTreeLen())
 	resp.ApplyID = mp.getApplyID()
 	resp.CommittedID = mp.getCommittedID()
+
+	resp.RaftInfo.DownReplicas = mp.config.RaftStore.RaftServer().GetDownReplicas(mp.config.PartitionId)
+	resp.RaftInfo.PendingPeers = mp.config.RaftStore.RaftServer().GetPendingReplica(mp.config.PartitionId)
+	if rStatus := mp.config.RaftStore.RaftStatus(mp.config.PartitionId); rStatus != nil {
+		resp.RaftInfo.RaftStatus = *rStatus
+	}
+	resp.RaftInfo.Hosts = mp.config.Peers
+
 	if err != nil {
 		err = errors.Trace(err,
 			"[ResponseLoadMetaPartition] check snapshot")

proto/admin_proto.go

@@ -23,6 +23,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cubefs/cubefs/depends/tiglabs/raft"
+
 	"github.com/cubefs/cubefs/util"
 	"github.com/cubefs/cubefs/util/log"
 )
@@ -1092,6 +1094,13 @@ type MetaPartitionLoadRequest struct {
 	PartitionID uint64
 }
 
+type RaftInfo struct {
+	RaftStatus   raft.Status
+	PendingPeers []uint64
+	DownReplicas []raft.DownReplica
+	Hosts        []Peer
+}
+
 // MetaPartitionLoadResponse defines the response to the request of loading meta partition.
 type MetaPartitionLoadResponse struct {
 	PartitionID uint64
@@ -1102,6 +1111,7 @@ type MetaPartitionLoadResponse struct {
 	DentryCount uint64
 	InodeCount  uint64
 	Addr        string
+	RaftInfo    RaftInfo
 }
 
 // DataPartitionResponse defines the response from a data node to the master that is related to a data partition.

proto/model.go

@@ -120,6 +120,7 @@ type MetaPartitionInfo struct {
 // MetaReplica defines the replica of a meta partition
 type MetaReplicaInfo struct {
 	Addr            string
+	NodeID          uint64
 	DomainAddr      string
 	MaxInodeID      uint64
 	ReportTime      int64
@@ -400,6 +401,7 @@ type DataPartitionDiagnosis struct {
 }
 
 // meta partition diagnosis represents the inactive meta nodes, corrupt meta partitions, and meta partitions lack of replicas
+
 type MetaPartitionDiagnosis struct {
 	InactiveMetaNodes                          []string
 	CorruptMetaPartitionIDs                    []uint64
@@ -412,6 +414,19 @@ type MetaPartitionDiagnosis struct {
 	DentryCountNotEqualReplicaMetaPartitionIDs []uint64
 }
 
+type MetaPartitionDiagnosisV1 struct {
+	InactiveMetaNodes                    []string
+	NoLeaderMetaPartitionIDs             []uint64
+	LackReplicaMetaPartitionIDs          []uint64
+	BadMetaPartitionIDs                  []BadPartitionView
+	UnavailableMetaPartitionIDs          []uint64
+	InConsistRreplicaCntMetaPartitionIDs []uint64
+	InodeCountNotEqualIDs                []uint64
+	MaxInodeNotEqualIDs                  []uint64
+	DentryCountNotEqualIDs               []uint64
+	AbnormalRaftIDs                      []uint64
+}
+
 type FailedDpInfo struct {
 	PartitionID uint64
 	ErrMsg      string

sdk/master/api_admin.go

@@ -161,9 +161,11 @@ func (api *AdminAPI) DiagnoseDataPartition(ignoreDiscardDp bool) (diagnosis *pro
 	return
 }
 
-func (api *AdminAPI) DiagnoseMetaPartition() (diagnosis *proto.MetaPartitionDiagnosis, err error) {
+func (api *AdminAPI) DiagnoseMetaPartition() (diagnosis *proto.MetaPartitionDiagnosisV1, err error) {
-	diagnosis = &proto.MetaPartitionDiagnosis{}
+	diagnosis = &proto.MetaPartitionDiagnosisV1{}
-	err = api.mc.requestWith(diagnosis, newRequest(get, proto.AdminDiagnoseMetaPartition).Header(api.h))
+	err = api.mc.requestWith(diagnosis, newRequest(get, proto.AdminDiagnoseMetaPartition).Header(api.h).Param(
+		anyParam{"v1", "true"},
+	))
 	return
 }