diff --git a/cli/cmd/fmt.go b/cli/cmd/fmt.go index 12382e133..32da7181b 100644 --- a/cli/cmd/fmt.go +++ b/cli/cmd/fmt.go @@ -412,6 +412,14 @@ var ( partitionInfoTableHeader = fmt.Sprintf(partitionInfoTablePattern, "ID", "VOLUME", "REPLICAS", "STATUS", "MediaType", "MEMBERS") + PeerAbnormalRaftPartitionInfoTablePattern = "%-8v %-32v %-8v %-12v %-12v %-12v %v%v%v" + PeerAbnormalRaftPartitionInfoHeader = fmt.Sprintf(PeerAbnormalRaftPartitionInfoTablePattern, + "ID", "VOLUME", "REPLICAS", "STATUS", "MediaType", "MEMBERS", "DIFF-RAFT-MEMBERS", "|DOWN-MEMBERS", "|PEND-MEMBERS") + + badMpReplicaPartitionInfoTablePattern = "%-8v %-32v %-8v %-12v %-64v %-24v" + badMpReplicaPartitionInfoTableHeader = fmt.Sprintf(badMpReplicaPartitionInfoTablePattern, + "ID", "VOLUME", "REPLICAS", "STATUS", "MEMBERS", "UNAVAILABLE_REPLICAS") + badReplicaPartitionInfoTablePattern = "%-8v %-32v %-8v %-12v %-64v %-24v" badReplicaPartitionInfoTableHeader = fmt.Sprintf(badReplicaPartitionInfoTablePattern, "ID", "VOLUME", "REPLICAS", "STATUS", "MEMBERS", "UNAVAILABLE_REPLICAS") @@ -553,6 +561,65 @@ func formatMetaPartitionInfoRow(partition *proto.MetaPartitionInfo) string { formatDataPartitionStatus(partition.Status), "N/A", strings.Join(partition.Hosts, ", ")) } +func formatMetaPartitionInfoRowWithRaft(partition *proto.MetaPartitionInfo) string { + var ( + info *proto.MetaPartitionLoadResponse + addrArr = make(map[uint64]string) + diffHosts []string + downHosts []string + pendingHosts []string + ) + for _, peer := range partition.Peers { + addrArr[peer.ID] = peer.Addr + } + + for _, info = range partition.LoadResponse { + if info.RaftInfo.RaftStatus.Leader == info.RaftInfo.RaftStatus.NodeID { + break + } + } + if info == nil { + return "" + } + for _, peer := range info.RaftInfo.Hosts { + if _, ok := addrArr[peer.ID]; ok { + continue + } + diffHosts = append(diffHosts, peer.Addr) + addrArr[peer.ID] = peer.Addr + } + + for _, dr := range info.RaftInfo.DownReplicas { + if host, ok := addrArr[dr.NodeID]; ok { + downHosts = append(downHosts, host) + } + } + for _, pr := range info.RaftInfo.PendingPeers { + if host, ok := addrArr[pr]; ok { + pendingHosts = append(pendingHosts, host) + } + } + var ( + diffInfo = "N/A" + downInfo = "N/A" + pendingInfo = "N/A" + ) + if len(diffHosts) != 0 { + diffInfo = strings.Join(diffHosts, ", ") + } + if len(downHosts) != 0 { + downInfo = strings.Join(downHosts, ", ") + } + if len(pendingHosts) != 0 { + pendingInfo = strings.Join(pendingHosts, ", ") + } + + return fmt.Sprintf(PeerAbnormalRaftPartitionInfoTablePattern, + partition.PartitionID, partition.VolName, partition.ReplicaNum, + formatDataPartitionStatus(partition.Status), "N/A", strings.Join(partition.Hosts, ", "), + diffInfo+"|", downInfo+"|", pendingInfo) +} + func formatBadReplicaMpInfoRow(partition *proto.MetaPartitionInfo) string { sb := strings.Builder{} sb.WriteString("[") diff --git a/cli/cmd/metapartition.go b/cli/cmd/metapartition.go index e8c7641e9..cda710086 100644 --- a/cli/cmd/metapartition.go +++ b/cli/cmd/metapartition.go @@ -90,7 +90,7 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on "reset" command will be released in next version.`, Run: func(cmd *cobra.Command, args []string) { var ( - diagnosis *proto.MetaPartitionDiagnosis + diagnosis *proto.MetaPartitionDiagnosisV1 metaNodes []*proto.MetaNodeInfo err error ) @@ -120,10 +120,10 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on stdout("\n") stdout("[Corrupt meta partitions](no leader):\n") stdout("%v\n", partitionInfoTableHeader) - sort.SliceStable(diagnosis.CorruptMetaPartitionIDs, func(i, j int) bool { - return diagnosis.CorruptMetaPartitionIDs[i] < diagnosis.CorruptMetaPartitionIDs[j] + sort.SliceStable(diagnosis.NoLeaderMetaPartitionIDs, func(i, j int) bool { + return diagnosis.NoLeaderMetaPartitionIDs[i] < diagnosis.NoLeaderMetaPartitionIDs[j] }) - for _, pid := range diagnosis.CorruptMetaPartitionIDs { + for _, pid := range diagnosis.NoLeaderMetaPartitionIDs { var partition *proto.MetaPartitionInfo if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil { err = fmt.Errorf("Partition not found, err:[%v] ", err) @@ -149,6 +149,23 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on } } + stdout("\n") + stdout("%v\n", "[Meta partition Abnormal Raft Info]:") + stdout("%v\n", PeerAbnormalRaftPartitionInfoHeader) + sort.SliceStable(diagnosis.AbnormalRaftIDs, func(i, j int) bool { + return diagnosis.AbnormalRaftIDs[i] < diagnosis.AbnormalRaftIDs[j] + }) + for _, pid := range diagnosis.AbnormalRaftIDs { + var partition *proto.MetaPartitionInfo + if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil { + err = fmt.Errorf("Partition not found, err:[%v] ", err) + return + } + if partition != nil { + stdout("%v\n", formatMetaPartitionInfoRowWithRaft(partition)) + } + } + stdout("\n") stdout("%v\n", "[Bad meta partitions(decommission not completed)]:") badPartitionTablePattern := "%-8v %-10v\n" @@ -164,11 +181,11 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on stdout("\n") stdout("%v\n", "[Meta Partition has unavailable replica]:") - stdout("%v\n", badReplicaPartitionInfoTableHeader) - sort.SliceStable(diagnosis.BadReplicaMetaPartitionIDs, func(i, j int) bool { - return diagnosis.BadReplicaMetaPartitionIDs[i] < diagnosis.BadReplicaMetaPartitionIDs[j] + stdout("%v\n", badMpReplicaPartitionInfoTableHeader) + sort.SliceStable(diagnosis.UnavailableMetaPartitionIDs, func(i, j int) bool { + return diagnosis.UnavailableMetaPartitionIDs[i] < diagnosis.UnavailableMetaPartitionIDs[j] }) - for _, pid := range diagnosis.BadReplicaMetaPartitionIDs { + for _, pid := range diagnosis.UnavailableMetaPartitionIDs { var partition *proto.MetaPartitionInfo if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil { err = fmt.Errorf("Partition not found, err:[%v] ", err) @@ -185,10 +202,10 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on stdout("\n") stdout("%v\n", "[Partition with replica inode count not equal]:") stdout("%v\n", inodeCountNotEqualInfoTableHeader) - sort.SliceStable(diagnosis.InodeCountNotEqualReplicaMetaPartitionIDs, func(i, j int) bool { - return diagnosis.InodeCountNotEqualReplicaMetaPartitionIDs[i] < diagnosis.InodeCountNotEqualReplicaMetaPartitionIDs[j] + sort.SliceStable(diagnosis.InodeCountNotEqualIDs, func(i, j int) bool { + return diagnosis.InodeCountNotEqualIDs[i] < diagnosis.InodeCountNotEqualIDs[j] }) - for _, pid := range diagnosis.InodeCountNotEqualReplicaMetaPartitionIDs { + for _, pid := range diagnosis.InodeCountNotEqualIDs { var partition *proto.MetaPartitionInfo if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil { err = fmt.Errorf("Partition not found, err:[%v] ", err) @@ -202,10 +219,10 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on stdout("\n") stdout("%v\n", "[Partition with replica max inode not equal]:") stdout("%v\n", maxInodeNotEqualInfoTableHeader) - sort.SliceStable(diagnosis.MaxInodeNotEqualReplicaMetaPartitionIDs, func(i, j int) bool { - return diagnosis.MaxInodeNotEqualReplicaMetaPartitionIDs[i] < diagnosis.MaxInodeNotEqualReplicaMetaPartitionIDs[j] + sort.SliceStable(diagnosis.MaxInodeNotEqualIDs, func(i, j int) bool { + return diagnosis.MaxInodeNotEqualIDs[i] < diagnosis.MaxInodeNotEqualIDs[j] }) - for _, pid := range diagnosis.MaxInodeNotEqualReplicaMetaPartitionIDs { + for _, pid := range diagnosis.MaxInodeNotEqualIDs { var partition *proto.MetaPartitionInfo if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil { err = fmt.Errorf("Partition not found, err:[%v] ", err) @@ -219,10 +236,10 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on stdout("\n") stdout("%v\n", "[Partition with replica dentry count not equal]:") stdout("%v\n", dentryCountNotEqualInfoTableHeader) - sort.SliceStable(diagnosis.DentryCountNotEqualReplicaMetaPartitionIDs, func(i, j int) bool { - return diagnosis.DentryCountNotEqualReplicaMetaPartitionIDs[i] < diagnosis.DentryCountNotEqualReplicaMetaPartitionIDs[j] + sort.SliceStable(diagnosis.DentryCountNotEqualIDs, func(i, j int) bool { + return diagnosis.DentryCountNotEqualIDs[i] < diagnosis.DentryCountNotEqualIDs[j] }) - for _, pid := range diagnosis.DentryCountNotEqualReplicaMetaPartitionIDs { + for _, pid := range diagnosis.DentryCountNotEqualIDs { var partition *proto.MetaPartitionInfo if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil { err = fmt.Errorf("Partition not found, err:[%v] ", err) @@ -236,10 +253,10 @@ the corrupt nodes, the few remaining replicas can not reach an agreement with on stdout("\n") stdout("%v\n", "[Partition with excessive replicas]:") stdout("%v\n", partitionInfoTableHeader) - sort.SliceStable(diagnosis.ExcessReplicaMetaPartitionIDs, func(i, j int) bool { - return diagnosis.ExcessReplicaMetaPartitionIDs[i] < diagnosis.ExcessReplicaMetaPartitionIDs[j] + sort.SliceStable(diagnosis.InConsistRreplicaCntMetaPartitionIDs, func(i, j int) bool { + return diagnosis.InConsistRreplicaCntMetaPartitionIDs[i] < diagnosis.InConsistRreplicaCntMetaPartitionIDs[j] }) - for _, pid := range diagnosis.ExcessReplicaMetaPartitionIDs { + for _, pid := range diagnosis.InConsistRreplicaCntMetaPartitionIDs { var partition *proto.MetaPartitionInfo if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil { err = fmt.Errorf("Partition not found, err:[%v] ", err) diff --git a/master/api_service.go b/master/api_service.go index 4f9c00a7b..4fcc44b42 100644 --- a/master/api_service.go +++ b/master/api_service.go @@ -4583,7 +4583,7 @@ func (m *Server) getNodeInfoHandler(w http.ResponseWriter, r *http.Request) { sendOkReply(w, r, newSuccessHTTPReply(resp)) } -func (m *Server) diagnoseMetaPartition(w http.ResponseWriter, r *http.Request) { +func (m *Server) diagnoseMetaPartitionDelayDelted(w http.ResponseWriter, r *http.Request) { var ( err error rstMsg *proto.MetaPartitionDiagnosis @@ -4608,17 +4608,14 @@ func (m *Server) diagnoseMetaPartition(w http.ResponseWriter, r *http.Request) { defer func() { doStatAndMetric(proto.AdminDiagnoseMetaPartition, metric, err, nil) }() - corruptMpIDs = make([]uint64, 0) lackReplicaMpIDs = make([]uint64, 0) badReplicaMpIDs = make([]uint64, 0) excessReplicaMpIDs = make([]uint64, 0) - if inactiveNodes, err = m.cluster.checkInactiveMetaNodes(); err != nil { sendErrReply(w, r, newErrHTTPReply(err)) return } - if lackReplicaMps, noLeaderMps, badReplicaMps, excessReplicaMPs, inodeCountNotEqualReplicaMps, maxInodeNotEqualMPs, dentryCountNotEqualReplicaMps, err = m.cluster.checkReplicaMetaPartitions(); err != nil { sendErrReply(w, r, newErrHTTPReply(err)) @@ -4636,14 +4633,12 @@ func (m *Server) diagnoseMetaPartition(w http.ResponseWriter, r *http.Request) { for _, mp := range excessReplicaMPs { excessReplicaMpIDs = append(excessReplicaMpIDs, mp.PartitionID) } - for _, mp := range inodeCountNotEqualReplicaMps { inodeCountNotEqualReplicaMpIDs = append(inodeCountNotEqualReplicaMpIDs, mp.PartitionID) } for _, mp := range maxInodeNotEqualMPs { maxInodeNotEqualReplicaMpIDs = append(maxInodeNotEqualReplicaMpIDs, mp.PartitionID) } - for _, mp := range dentryCountNotEqualReplicaMps { dentryCountNotEqualReplicaMpIDs = append(dentryCountNotEqualReplicaMpIDs, mp.PartitionID) } @@ -4667,6 +4662,30 @@ func (m *Server) diagnoseMetaPartition(w http.ResponseWriter, r *http.Request) { sendOkReply(w, r, newSuccessHTTPReply(rstMsg)) } +func (m *Server) diagnoseMetaPartition(w http.ResponseWriter, r *http.Request) { + var ( + err error + diagnosis interface{} + ) + if err = r.ParseForm(); err != nil { + return + } + if v1 := r.FormValue("v1"); v1 == "" { + m.diagnoseMetaPartitionDelayDelted(w, r) + return + } else { + metric := exporter.NewTPCnt(apiToMetricsName(proto.AdminDiagnoseMetaPartition)) + defer func() { + doStatAndMetric(proto.AdminDiagnoseMetaPartition, metric, err, nil) + }() + if diagnosis, err = m.cluster.checkReplicaMetaPartitionsV1(); err != nil { + sendErrReply(w, r, newErrHTTPReply(err)) + return + } + } + sendOkReply(w, r, newSuccessHTTPReply(diagnosis)) +} + // Decommission a disk. This will decommission all the data partitions on this disk. // If parameter diskDisable is true, creating data partitions on this disk will be not allowed. func (m *Server) decommissionDisk(w http.ResponseWriter, r *http.Request) { @@ -5991,6 +6010,7 @@ func (m *Server) getMetaPartition(w http.ResponseWriter, r *http.Request) { for i := 0; i < len(replicas); i++ { replicas[i] = &proto.MetaReplicaInfo{ Addr: mp.Replicas[i].Addr, + NodeID: mp.Replicas[i].nodeID, DomainAddr: mp.Replicas[i].metaNode.DomainAddr, MaxInodeID: mp.Replicas[i].MaxInodeID, ReportTime: mp.Replicas[i].ReportTime, diff --git a/master/cluster.go b/master/cluster.go index 70d3efdb1..30c3c99d4 100644 --- a/master/cluster.go +++ b/master/cluster.go @@ -81,6 +81,7 @@ type ClusterTopoSubItem struct { inodeCountNotEqualMP *sync.Map maxInodeNotEqualMP *sync.Map dentryCountNotEqualMP *sync.Map + AbnormalRaftMP *sync.Map mnMutex sync.RWMutex // meta node mutex dnMutex sync.RWMutex // data node mutex @@ -465,6 +466,7 @@ func newCluster(name string, leaderInfo *LeaderInfo, fsm *MetadataFsm, partition c.inodeCountNotEqualMP = new(sync.Map) c.maxInodeNotEqualMP = new(sync.Map) c.dentryCountNotEqualMP = new(sync.Map) + c.AbnormalRaftMP = new(sync.Map) c.lcMgr = newLifecycleManager() c.lcMgr.cluster = c c.snapshotMgr = newSnapshotManager() diff --git a/master/cluster_task.go b/master/cluster_task.go index e583deb39..82176befc 100644 --- a/master/cluster_task.go +++ b/master/cluster_task.go @@ -282,30 +282,24 @@ func (c *Cluster) checkReplicaMetaPartitions() ( inodeCountNotEqualMPs = make([]*MetaPartition, 0) maxInodeNotEqualMPs = make([]*MetaPartition, 0) dentryCountNotEqualMPs = make([]*MetaPartition, 0) - markDeleteVolNames := make(VolNameSet) - vols := c.copyVols() for _, vol := range vols { if vol.Status == proto.VolStatusMarkDelete { markDeleteVolNames[vol.Name] = struct{}{} continue } - vol.mpsLock.RLock() for _, mp := range vol.MetaPartitions { if uint8(len(mp.Hosts)) < mp.ReplicaNum || uint8(len(mp.getActiveAddrs(defaultMetaPartitionTimeOutSec))) < mp.ReplicaNum { lackReplicaMetaPartitions = append(lackReplicaMetaPartitions, mp) } - if !mp.isLeaderExist() && (time.Now().Unix()-mp.LeaderReportTime > c.cfg.MpNoLeaderReportIntervalSec) { noLeaderMetaPartitions = append(noLeaderMetaPartitions, mp) } - if uint8(len(mp.Hosts)) > mp.ReplicaNum || uint8(len(mp.Replicas)) > mp.ReplicaNum { excessReplicaMetaPartitions = append(excessReplicaMetaPartitions, mp) } - for _, replica := range mp.Replicas { if replica.Status == proto.Unavailable { unavailableReplicaMPs = append(unavailableReplicaMPs, mp) @@ -343,6 +337,69 @@ func (c *Cluster) checkReplicaMetaPartitions() ( return } +func (c *Cluster) checkReplicaMetaPartitionsV1() (diagnosis *proto.MetaPartitionDiagnosisV1, err error) { + diagnosis = &proto.MetaPartitionDiagnosisV1{} + markDeleteVolNames := make(VolNameSet) + vols := c.copyVols() + for _, vol := range vols { + if vol.Status == proto.VolStatusMarkDelete { + markDeleteVolNames[vol.Name] = struct{}{} + continue + } + + vol.mpsLock.RLock() + for _, mp := range vol.MetaPartitions { + if uint8(len(mp.Hosts)) < mp.ReplicaNum || uint8(len(mp.getActiveAddrs(defaultMetaPartitionTimeOutSec))) < mp.ReplicaNum { + diagnosis.LackReplicaMetaPartitionIDs = append(diagnosis.LackReplicaMetaPartitionIDs, mp.PartitionID) + } + + if !mp.isLeaderExist() && (time.Now().Unix()-mp.LeaderReportTime > c.cfg.MpNoLeaderReportIntervalSec) { + diagnosis.NoLeaderMetaPartitionIDs = append(diagnosis.NoLeaderMetaPartitionIDs, mp.PartitionID) + } + + if uint8(len(mp.Hosts)) > mp.ReplicaNum || uint8(len(mp.Replicas)) > mp.ReplicaNum { + diagnosis.InConsistRreplicaCntMetaPartitionIDs = append(diagnosis.InConsistRreplicaCntMetaPartitionIDs, mp.PartitionID) + } + + for _, replica := range mp.Replicas { + if replica.Status == proto.Unavailable { + diagnosis.UnavailableMetaPartitionIDs = append(diagnosis.UnavailableMetaPartitionIDs, mp.PartitionID) + break + } + } + } + vol.mpsLock.RUnlock() + } + + setAbnormalIDs := func(mpMap *sync.Map) []uint64 { + var resultIDs []uint64 + mpMap.Range(func(key, value interface{}) bool { + mp := value.(*MetaPartition) + if _, ok := markDeleteVolNames[mp.volName]; !ok { + resultIDs = append(resultIDs, mp.PartitionID) + } + return true + }) + return resultIDs + } + + diagnosis.InodeCountNotEqualIDs = setAbnormalIDs(c.inodeCountNotEqualMP) + diagnosis.MaxInodeNotEqualIDs = setAbnormalIDs(c.maxInodeNotEqualMP) + diagnosis.DentryCountNotEqualIDs = setAbnormalIDs(c.dentryCountNotEqualMP) + diagnosis.AbnormalRaftIDs = setAbnormalIDs(c.AbnormalRaftMP) + + log.LogInfof("clusterID[%v], lackReplicaMetaPartitions count:[%v], noLeaderMetaPartitions count[%v]"+ + "unavailableReplicaMPs count:[%v], excessReplicaMp count:[%v], AbnormalRaftIDs count:[%v]", + c.Name, len(diagnosis.LackReplicaMetaPartitionIDs), len(diagnosis.NoLeaderMetaPartitionIDs), + len(diagnosis.UnavailableMetaPartitionIDs), len(diagnosis.InConsistRreplicaCntMetaPartitionIDs), len(diagnosis.AbnormalRaftIDs)) + + if diagnosis.InactiveMetaNodes, err = c.checkInactiveMetaNodes(); err != nil { + return + } + + return +} + func (c *Cluster) deleteMetaReplica(partition *MetaPartition, addr string, validate bool, forceDel bool) (err error) { defer func() { if err != nil { @@ -619,6 +676,7 @@ func (c *Cluster) doLoadMetaPartition(mp *MetaPartition) { }(host) } wg.Wait() + mp.checkPeerDiffWithRaft(c) select { case err := <-errChannel: msg := fmt.Sprintf("action[doLoadMetaPartition] vol[%v],mpID[%v],err[%v]", mp.volName, mp.PartitionID, err.Error()) diff --git a/master/meta_partition_manager.go b/master/meta_partition_manager.go index d57dca111..b83539ff2 100644 --- a/master/meta_partition_manager.go +++ b/master/meta_partition_manager.go @@ -52,6 +52,48 @@ func (c *Cluster) checkLoadMetaPartitions() { } } +func (mp *MetaPartition) checkPeerDiffWithRaft(c *Cluster) { + if len(mp.LoadResponse) == 0 { + return + } + if !mp.doCompare() { + return + } + var ( + addrArr = make(map[uint64]string) + leaderID uint64 + ) + for _, mr := range mp.Replicas { + addrArr[mr.nodeID] = mr.Addr + if mr.IsLeader { + leaderID = mr.nodeID + } + } + + for _, info := range mp.LoadResponse { + if info.RaftInfo.RaftStatus.NodeID != leaderID { + continue + } + for peer := range info.RaftInfo.RaftStatus.Replicas { + if _, ok := addrArr[peer]; !ok { + c.AbnormalRaftMP.Store(mp.PartitionID, mp) + return + } + } + if len(info.RaftInfo.PendingPeers) != 0 { + c.AbnormalRaftMP.Store(mp.PartitionID, mp) + return + } + if len(info.RaftInfo.DownReplicas) != 0 { + c.AbnormalRaftMP.Store(mp.PartitionID, mp) + return + } + c.AbnormalRaftMP.Delete(mp.PartitionID) + return + } + c.AbnormalRaftMP.Delete(mp.PartitionID) +} + func (mp *MetaPartition) checkSnapshot(c *Cluster) { if len(mp.LoadResponse) == 0 { return diff --git a/metanode/partition.go b/metanode/partition.go index 67af770d7..be62e8cc4 100644 --- a/metanode/partition.go +++ b/metanode/partition.go @@ -1431,6 +1431,14 @@ func (mp *metaPartition) ResponseLoadMetaPartition(p *Packet) (err error) { resp.DentryCount = uint64(mp.GetDentryTreeLen()) resp.ApplyID = mp.getApplyID() resp.CommittedID = mp.getCommittedID() + + resp.RaftInfo.DownReplicas = mp.config.RaftStore.RaftServer().GetDownReplicas(mp.config.PartitionId) + resp.RaftInfo.PendingPeers = mp.config.RaftStore.RaftServer().GetPendingReplica(mp.config.PartitionId) + if rStatus := mp.config.RaftStore.RaftStatus(mp.config.PartitionId); rStatus != nil { + resp.RaftInfo.RaftStatus = *rStatus + } + resp.RaftInfo.Hosts = mp.config.Peers + if err != nil { err = errors.Trace(err, "[ResponseLoadMetaPartition] check snapshot") diff --git a/proto/admin_proto.go b/proto/admin_proto.go index 65c546308..83b42ace9 100644 --- a/proto/admin_proto.go +++ b/proto/admin_proto.go @@ -23,6 +23,8 @@ import ( "strings" "time" + "github.com/cubefs/cubefs/depends/tiglabs/raft" + "github.com/cubefs/cubefs/util" "github.com/cubefs/cubefs/util/log" ) @@ -1092,6 +1094,13 @@ type MetaPartitionLoadRequest struct { PartitionID uint64 } +type RaftInfo struct { + RaftStatus raft.Status + PendingPeers []uint64 + DownReplicas []raft.DownReplica + Hosts []Peer +} + // MetaPartitionLoadResponse defines the response to the request of loading meta partition. type MetaPartitionLoadResponse struct { PartitionID uint64 @@ -1102,6 +1111,7 @@ type MetaPartitionLoadResponse struct { DentryCount uint64 InodeCount uint64 Addr string + RaftInfo RaftInfo } // DataPartitionResponse defines the response from a data node to the master that is related to a data partition. diff --git a/proto/model.go b/proto/model.go index 1b2e3cef0..718e63433 100644 --- a/proto/model.go +++ b/proto/model.go @@ -120,6 +120,7 @@ type MetaPartitionInfo struct { // MetaReplica defines the replica of a meta partition type MetaReplicaInfo struct { Addr string + NodeID uint64 DomainAddr string MaxInodeID uint64 ReportTime int64 @@ -400,6 +401,7 @@ type DataPartitionDiagnosis struct { } // meta partition diagnosis represents the inactive meta nodes, corrupt meta partitions, and meta partitions lack of replicas + type MetaPartitionDiagnosis struct { InactiveMetaNodes []string CorruptMetaPartitionIDs []uint64 @@ -412,6 +414,19 @@ type MetaPartitionDiagnosis struct { DentryCountNotEqualReplicaMetaPartitionIDs []uint64 } +type MetaPartitionDiagnosisV1 struct { + InactiveMetaNodes []string + NoLeaderMetaPartitionIDs []uint64 + LackReplicaMetaPartitionIDs []uint64 + BadMetaPartitionIDs []BadPartitionView + UnavailableMetaPartitionIDs []uint64 + InConsistRreplicaCntMetaPartitionIDs []uint64 + InodeCountNotEqualIDs []uint64 + MaxInodeNotEqualIDs []uint64 + DentryCountNotEqualIDs []uint64 + AbnormalRaftIDs []uint64 +} + type FailedDpInfo struct { PartitionID uint64 ErrMsg string diff --git a/sdk/master/api_admin.go b/sdk/master/api_admin.go index ae29dc371..d39cceed0 100644 --- a/sdk/master/api_admin.go +++ b/sdk/master/api_admin.go @@ -161,9 +161,11 @@ func (api *AdminAPI) DiagnoseDataPartition(ignoreDiscardDp bool) (diagnosis *pro return } -func (api *AdminAPI) DiagnoseMetaPartition() (diagnosis *proto.MetaPartitionDiagnosis, err error) { - diagnosis = &proto.MetaPartitionDiagnosis{} - err = api.mc.requestWith(diagnosis, newRequest(get, proto.AdminDiagnoseMetaPartition).Header(api.h)) +func (api *AdminAPI) DiagnoseMetaPartition() (diagnosis *proto.MetaPartitionDiagnosisV1, err error) { + diagnosis = &proto.MetaPartitionDiagnosisV1{} + err = api.mc.requestWith(diagnosis, newRequest(get, proto.AdminDiagnoseMetaPartition).Header(api.h).Param( + anyParam{"v1", "true"}, + )) return }