From 4b9f564cdf2bfbb2dff51849598a671a174b4eef Mon Sep 17 00:00:00 2001 From: shuqiang-zheng Date: Tue, 11 Mar 2025 10:37:57 +0800 Subject: [PATCH] feat(flashnode): enhancement of flashGroup information display. close:#23070174 Signed-off-by: shuqiang-zheng --- cli/cmd/flashgroup.go | 4 +- cli/cmd/fmt.go | 4 +- master/flash_group.go | 25 ++++- master/flash_node_topology.go | 181 ++++++++++++++-------------------- proto/distributed_cache.go | 2 + 5 files changed, 99 insertions(+), 117 deletions(-) diff --git a/cli/cmd/flashgroup.go b/cli/cmd/flashgroup.go index ce3339259..fbd812ce5 100644 --- a/cli/cmd/flashgroup.go +++ b/cli/cmd/flashgroup.go @@ -176,7 +176,7 @@ func newCmdFlashGroupRemove(client *master.MasterClient) *cobra.Command { return } } - + result, err := client.AdminAPI().RemoveFlashGroup(flashGroupID, optGradualFlag, optStep) if err != nil { return @@ -341,7 +341,7 @@ func newCmdFlashGroupList(client *master.MasterClient) *cobra.Command { slot: slot, }) } - tbl = tbl.append(arow(group.ID, group.Weight, len(group.Slots), group.Status, group.SlotStatus, group.FlashNodeCount)) + tbl = tbl.append(arow(group.ID, group.Weight, len(group.Slots), group.Status, group.SlotStatus, len(group.PendingSlots), group.Step, group.FlashNodeCount)) } stdoutln(alignTable(tbl...)) diff --git a/cli/cmd/fmt.go b/cli/cmd/fmt.go index 7ddae056a..e63b3aefe 100644 --- a/cli/cmd/fmt.go +++ b/cli/cmd/fmt.go @@ -1291,7 +1291,7 @@ var ( "STORAGE CLASS", "INODE COUNT", "USED SIZE", "QUOTA") formatFlashNodeSimpleViewTableTitle = arow("Zone", "ID", "Address", "Active", "Enable", "FlashGroupID", "ReportTime") formatFlashNodeViewTableTitle = append(formatFlashNodeSimpleViewTableTitle[:], "DataPath", "HitRate", "Evicts", "Limit", "MaxAlloc", "HasAlloc", "Num", "Status") - formatFlashGroupViewTile = arow("ID", "Weight", "Slots", "Status", "SlotStatus", "FlashNodeCount") + formatFlashGroupViewTile = arow("ID", "Weight", "Slots", "Status", "SlotStatus", "PendingSlots", "Step", "FlashNodeCount") ) func formatHybridCloudStorageTableRow(view *proto.StatOfStorageClass) (row string) { @@ -1319,5 +1319,7 @@ func formatFlashGroupView(fg *proto.FlashGroupAdminView) string { fmt.Sprintf(" Slots:%v\n", fg.Slots) + fmt.Sprintf(" Status:%v\n", fg.Status) + fmt.Sprintf(" SlotStatus:%v\n", fg.SlotStatus) + + fmt.Sprintf(" PedningSlots:%v\n", fg.PendingSlots) + + fmt.Sprintf(" Step:%v\n", fg.Step) + fmt.Sprintf(" FlashNodeCount:%v\n", fg.FlashNodeCount) } diff --git a/master/flash_group.go b/master/flash_group.go index 4e68934e2..6439f194f 100644 --- a/master/flash_group.go +++ b/master/flash_group.go @@ -70,6 +70,19 @@ func newFlashGroup(id uint64, slots []uint32, slotStatus proto.SlotStatus, pendi return fg } +func newFlashGroupFromFgv(fgv flashGroupValue) *FlashGroup { + fg := new(FlashGroup) + fg.ID = fgv.ID + fg.Slots = fgv.Slots + fg.SlotStatus = fgv.SlotStatus + fg.PendingSlots = fgv.PendingSlots + fg.Step = fgv.Step + fg.Weight = fgv.Weight + fg.Status = fgv.Status + fg.flashNodes = make(map[string]*FlashNode) + return fg +} + func (fg *FlashGroup) putFlashNode(fn *FlashNode) { fg.lock.Lock() fg.flashNodes[fn.Addr] = fn @@ -168,11 +181,13 @@ func (c *Cluster) syncPutFlashGroupInfo(opType uint32, flashGroup *FlashGroup) ( func (fg *FlashGroup) GetAdminView() (view proto.FlashGroupAdminView) { fg.lock.RLock() view = proto.FlashGroupAdminView{ - ID: fg.ID, - Slots: fg.Slots, - Weight: fg.Weight, - Status: fg.Status, - SlotStatus: fg.SlotStatus, + ID: fg.ID, + Slots: fg.Slots, + Weight: fg.Weight, + Status: fg.Status, + SlotStatus: fg.SlotStatus, + PendingSlots: fg.PendingSlots, + Step: fg.Step, } view.ZoneFlashNodes = make(map[string][]*proto.FlashNodeViewInfo) view.FlashNodeCount = len(fg.flashNodes) diff --git a/master/flash_node_topology.go b/master/flash_node_topology.go index e3375da43..f2cd2fb9d 100644 --- a/master/flash_node_topology.go +++ b/master/flash_node_topology.go @@ -261,36 +261,15 @@ func (t *flashNodeTopology) gradualRemoveFlashGroup(flashGroup *FlashGroup, c *C t.createFlashGroupLock.Lock() defer t.createFlashGroupLock.Unlock() - return t.gradualShrinkFlashGroupSlots(flashGroup, c, flashGroup.getSlots(), step) + return t.gradualExpandOrShrinkFlashGroupSlots(flashGroup, c, proto.SlotStatus_Deleting, flashGroup.getSlots(), step) } -func (t *flashNodeTopology) gradualExpandFlashGroupSlots(flashGroup *FlashGroup, c *Cluster, pendingSlots []uint32, step uint32) (err error) { //nolint:unused +func (t *flashNodeTopology) gradualExpandOrShrinkFlashGroupSlots(flashGroup *FlashGroup, c *Cluster, newSlotStatus proto.SlotStatus, pendingSlots []uint32, step uint32) (err error) { flashGroup.lock.Lock() oldSlotStatus := flashGroup.SlotStatus oldStep := flashGroup.Step oldPendingSlots := flashGroup.PendingSlots - flashGroup.SlotStatus = proto.SlotStatus_Creating - flashGroup.PendingSlots = pendingSlots - flashGroup.Step = step - if err = c.syncUpdateFlashGroup(flashGroup); err != nil { - flashGroup.SlotStatus = oldSlotStatus - flashGroup.PendingSlots = oldPendingSlots - flashGroup.Step = oldStep - flashGroup.lock.Unlock() - return - } - - flashGroup.lock.Unlock() - - return -} - -func (t *flashNodeTopology) gradualShrinkFlashGroupSlots(flashGroup *FlashGroup, c *Cluster, pendingSlots []uint32, step uint32) (err error) { - flashGroup.lock.Lock() - oldSlotStatus := flashGroup.SlotStatus - oldStep := flashGroup.Step - oldPendingSlots := flashGroup.PendingSlots - flashGroup.SlotStatus = proto.SlotStatus_Deleting + flashGroup.SlotStatus = newSlotStatus flashGroup.PendingSlots = pendingSlots flashGroup.Step = step if err = c.syncUpdateFlashGroup(flashGroup); err != nil { @@ -419,7 +398,7 @@ func (c *Cluster) loadFlashGroups() (err error) { err = fmt.Errorf("action[loadFlashGroups],value:%v,unmarshal err:%v", string(value), err) return } - flashGroup := newFlashGroup(fgv.ID, fgv.Slots, fgv.SlotStatus, fgv.PendingSlots, fgv.Step, fgv.Status, fgv.Weight) + flashGroup := newFlashGroupFromFgv(fgv) c.flashNodeTopo.flashGroupMap.Store(flashGroup.ID, flashGroup) for _, slot := range flashGroup.Slots { c.flashNodeTopo.slotsMap[slot] = flashGroup.ID @@ -501,6 +480,71 @@ func (c *Cluster) checkShrinkOrDeleteFlashGroup(flashGroup *FlashGroup) (needDel return } +func (c *Cluster) updateFlashGroupSlots(flashGroup *FlashGroup) (err error) { + var updatedSlotsNum uint32 + var newSlotStatus proto.SlotStatus + var newPendingSlots []uint32 + var needDeleteFgFlag bool + + if flashGroup.getSlotStatus() == proto.SlotStatus_Deleting { + if needDeleteFgFlag, err = c.checkShrinkOrDeleteFlashGroup(flashGroup); err != nil { + return + } + } + + flashGroup.lock.Lock() + leftPendingSlotsNum := uint32(len(flashGroup.PendingSlots)) - flashGroup.Step + oldSlots := flashGroup.Slots + oldPendingSlots := flashGroup.PendingSlots + oldSlotStatus := flashGroup.SlotStatus + oldStatus := flashGroup.Status + if leftPendingSlotsNum > 0 { // previous steps + updatedSlotsNum = flashGroup.Step + newPendingSlots = oldPendingSlots[updatedSlotsNum:] + newSlotStatus = oldSlotStatus + } else { // final step + updatedSlotsNum = uint32(len(flashGroup.PendingSlots)) + newPendingSlots = nil + newSlotStatus = proto.SlotStatus_Completed + } + newSlots := getNewSlots(flashGroup.Slots, flashGroup.PendingSlots[:updatedSlotsNum], oldSlotStatus) + flashGroup.Slots = newSlots + flashGroup.PendingSlots = newPendingSlots + flashGroup.SlotStatus = newSlotStatus + if needDeleteFgFlag { + flashGroup.Status = proto.FlashGroupStatus_Inactive + err = c.syncDeleteFlashGroup(flashGroup) + } else { + err = c.syncUpdateFlashGroup(flashGroup) + } + + if err != nil { + flashGroup.Slots = oldSlots + flashGroup.PendingSlots = oldPendingSlots + flashGroup.SlotStatus = oldSlotStatus + if needDeleteFgFlag { + flashGroup.Status = oldStatus + } + flashGroup.lock.Unlock() + return + } + + flashGroup.lock.Unlock() + + if oldSlotStatus == proto.SlotStatus_Creating { + for _, slot := range oldPendingSlots[:updatedSlotsNum] { + c.flashNodeTopo.slotsMap[slot] = flashGroup.ID + } + } else { + c.flashNodeTopo.removeSlots(oldPendingSlots[:updatedSlotsNum]) + if needDeleteFgFlag { + c.flashNodeTopo.flashGroupMap.Delete(flashGroup.ID) + } + } + + return +} + func (c *Cluster) scheduleToUpdateFlashGroupSlots() { go func() { dur := time.Minute @@ -520,91 +564,10 @@ func (c *Cluster) scheduleToUpdateFlashGroupSlots() { slotStatus := flashGroup.getSlotStatus() if slotStatus == proto.SlotStatus_Completed { return true - } else if slotStatus == proto.SlotStatus_Creating { - var addedSlotsNum uint32 - var newSlotStatus proto.SlotStatus - var newPendingSlots []uint32 - - flashGroup.lock.Lock() - leftPendingSlotsNum := uint32(len(flashGroup.PendingSlots)) - flashGroup.Step - oldSlots := flashGroup.Slots - oldPendingSlots := flashGroup.PendingSlots - oldSlotStatus := flashGroup.SlotStatus - if leftPendingSlotsNum > 0 { // previous steps - addedSlotsNum = flashGroup.Step - newPendingSlots = oldPendingSlots[addedSlotsNum:] - newSlotStatus = proto.SlotStatus_Creating - } else { // final step - addedSlotsNum = uint32(len(flashGroup.PendingSlots)) - newPendingSlots = nil - newSlotStatus = proto.SlotStatus_Completed + } else if slotStatus == proto.SlotStatus_Creating || slotStatus == proto.SlotStatus_Deleting { + if err := c.updateFlashGroupSlots(flashGroup); err == nil { + isNotUpdated = false } - newSlots := getNewSlots(flashGroup.Slots, flashGroup.PendingSlots[:addedSlotsNum], proto.SlotStatus_Creating) - flashGroup.Slots = newSlots - flashGroup.PendingSlots = newPendingSlots - flashGroup.SlotStatus = newSlotStatus - if err := c.syncUpdateFlashGroup(flashGroup); err != nil { - flashGroup.Slots = oldSlots - flashGroup.PendingSlots = oldPendingSlots - flashGroup.SlotStatus = oldSlotStatus - flashGroup.lock.Unlock() - return true - } - flashGroup.lock.Unlock() - for _, slot := range oldPendingSlots[:addedSlotsNum] { - c.flashNodeTopo.slotsMap[slot] = flashGroup.ID - } - isNotUpdated = false - return true - } else if slotStatus == proto.SlotStatus_Deleting { - var deletedSlotsNum uint32 - var newSlotStatus proto.SlotStatus - var newPendingSlots []uint32 - var needDeleteFgFlag bool - var err error - - if needDeleteFgFlag, err = c.checkShrinkOrDeleteFlashGroup(flashGroup); err != nil { - return true - } - flashGroup.lock.Lock() - leftPendingSlotsNum := uint32(len(flashGroup.PendingSlots)) - flashGroup.Step - oldSlots := flashGroup.Slots - oldPendingSlots := flashGroup.PendingSlots - oldSlotStatus := flashGroup.SlotStatus - oldStatus := flashGroup.Status - if leftPendingSlotsNum > 0 { // previous steps - deletedSlotsNum = flashGroup.Step - newPendingSlots = oldPendingSlots[deletedSlotsNum:] - newSlotStatus = proto.SlotStatus_Deleting - } else { // final step - deletedSlotsNum = uint32(len(flashGroup.PendingSlots)) - newPendingSlots = nil - newSlotStatus = proto.SlotStatus_Completed - } - newSlots := getNewSlots(flashGroup.Slots, flashGroup.PendingSlots[:deletedSlotsNum], proto.SlotStatus_Deleting) - flashGroup.Slots = newSlots - flashGroup.PendingSlots = newPendingSlots - flashGroup.SlotStatus = newSlotStatus - if needDeleteFgFlag { - flashGroup.Status = proto.FlashGroupStatus_Inactive - } - if err := c.syncUpdateFlashGroup(flashGroup); err != nil { - flashGroup.Slots = oldSlots - flashGroup.PendingSlots = oldPendingSlots - flashGroup.SlotStatus = oldSlotStatus - flashGroup.PendingSlots = oldPendingSlots - if needDeleteFgFlag { - flashGroup.Status = oldStatus - } - flashGroup.lock.Unlock() - return true - } - flashGroup.lock.Unlock() - c.flashNodeTopo.removeSlots(oldPendingSlots[:deletedSlotsNum]) - if needDeleteFgFlag { - c.flashNodeTopo.flashGroupMap.Delete(flashGroup.ID) - } - isNotUpdated = false return true } else { log.LogWarnf("scheduleToUpdateFlashGroupSlots failed, flashGroup(%v) has unknown SlotStatus(%v)", flashGroup.ID, flashGroup.SlotStatus) diff --git a/proto/distributed_cache.go b/proto/distributed_cache.go index 81c9fd782..9991422a8 100644 --- a/proto/distributed_cache.go +++ b/proto/distributed_cache.go @@ -223,6 +223,8 @@ type FlashGroupAdminView struct { Weight uint32 Status FlashGroupStatus SlotStatus SlotStatus + PendingSlots []uint32 + Step uint32 FlashNodeCount int ZoneFlashNodes map[string][]*FlashNodeViewInfo }