gian/cloudstack-mirror
1
0
Fork 0
mirror of https://github.com/apache/cloudstack synced 2026-08-01 21:25:23 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Fix consecutive-failure counting bugs flagged in PR #13743 review

Browse Source 
- Use a distinct event type (SNAPSHOT.RECURRING.FAILURE.LIMIT.REACHED)
  for the "gave up" WARN notification instead of EVENT_SNAPSHOT_CREATE,
  since that event became the latest EVENT_SNAPSHOT_CREATE entry and
  silently reset countConsecutiveFailedAttempts on the next scan.
- Filter out archived events in EventDao#listLatestEventsByResource so
  stale archived events can't be counted as the latest failure history,
  matching the archived=false filtering already used by sibling
  searches in EventDaoImpl.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Daan Hoogland 2026-07-30 08:46:33 +02:00
parent de2aee26a8
commit 1cb950b5a4
4 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/src/main/java/com/cloud/event/EventTypes.java
View File

@ -371,6 +371,7 @@ public class EventTypes {
public static final String EVENT_SNAPSHOT_REVERT = "SNAPSHOT.REVERT";
public static final String EVENT_SNAPSHOT_EXTRACT = "SNAPSHOT.EXTRACT";
public static final String EVENT_SNAPSHOT_SKIPPED = "SNAPSHOT.SKIPPED";
public static final String EVENT_SNAPSHOT_RECURRING_FAILURE_LIMIT_REACHED = "SNAPSHOT.RECURRING.FAILURE.LIMIT.REACHED";
public static final String EVENT_SNAPSHOT_POLICY_CREATE = "SNAPSHOTPOLICY.CREATE";
public static final String EVENT_SNAPSHOT_POLICY_UPDATE = "SNAPSHOTPOLICY.UPDATE";
public static final String EVENT_SNAPSHOT_POLICY_DELETE = "SNAPSHOTPOLICY.DELETE";
@ -1076,6 +1077,7 @@ public class EventTypes {
// Snapshots
entityEventDetails.put(EVENT_SNAPSHOT_CREATE, Snapshot.class);
entityEventDetails.put(EVENT_SNAPSHOT_SKIPPED, Snapshot.class);
entityEventDetails.put(EVENT_SNAPSHOT_RECURRING_FAILURE_LIMIT_REACHED, Snapshot.class);
entityEventDetails.put(EVENT_SNAPSHOT_DELETE, Snapshot.class);
entityEventDetails.put(EVENT_SNAPSHOT_EXTRACT, Snapshot.class);
entityEventDetails.put(EVENT_SNAPSHOT_ON_PRIMARY, Snapshot.class);

2
engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java
View File

@ -51,6 +51,7 @@ public class EventDaoImpl extends GenericDaoBase<EventVO, Long> implements Event
LatestEventsByResourceSearch.and("resourceId", LatestEventsByResourceSearch.entity().getResourceId(), Op.EQ);
LatestEventsByResourceSearch.and("resourceType", LatestEventsByResourceSearch.entity().getResourceType(), Op.EQ);
LatestEventsByResourceSearch.and("type", LatestEventsByResourceSearch.entity().getType(), Op.EQ);
LatestEventsByResourceSearch.and("archived", LatestEventsByResourceSearch.entity().getArchived(), Op.EQ);
LatestEventsByResourceSearch.done();
ToArchiveOrDeleteEventSearch = createSearchBuilder();
@ -118,6 +119,7 @@ public class EventDaoImpl extends GenericDaoBase<EventVO, Long> implements Event
sc.setParameters("resourceId", resourceId);
sc.setParameters("resourceType", resourceType);
sc.setParameters("type", type);
sc.setParameters("archived", false);
Filter filter = new Filter(EventVO.class, "createDate", false, 0L, (long) limit);
return listBy(sc, filter);
}

2
server/src/main/java/com/cloud/storage/snapshot/SnapshotSchedulerImpl.java
View File

@ -384,7 +384,7 @@ public class SnapshotSchedulerImpl extends ManagerBase implements SnapshotSchedu
lockedSchedule.setScheduledTimestamp(nextRegularRun);
logger.warn("Snapshot schedule [{}] for volume [{}] has failed [{}] consecutive times; it will not be retried until its next regularly scheduled run at [{}].",
snapshotToBeExecuted, volume, totalFailures, nextRegularRun);
ActionEventUtils.onCreatedActionEvent(User.UID_SYSTEM, volume.getAccountId(), EventVO.LEVEL_WARN, EventTypes.EVENT_SNAPSHOT_CREATE, true,
ActionEventUtils.onCreatedActionEvent(User.UID_SYSTEM, volume.getAccountId(), EventVO.LEVEL_WARN, EventTypes.EVENT_SNAPSHOT_RECURRING_FAILURE_LIMIT_REACHED, true,
String.format("Recurring snapshot for volume [%s] has failed %d consecutive times and will not be retried until its next regularly scheduled run.", volume, totalFailures),
volumeId, ApiCommandResourceType.Volume.toString());
} else {

7
server/src/test/java/com/cloud/storage/snapshot/SnapshotSchedulerImplTest.java
View File

@ -17,6 +17,7 @@
package com.cloud.storage.snapshot;
import com.cloud.event.ActionEventUtils;
import com.cloud.event.EventTypes;
import com.cloud.event.EventVO;
import com.cloud.event.dao.EventDao;
import com.cloud.storage.Snapshot;
@ -390,9 +391,11 @@ public class SnapshotSchedulerImplTest {
snapshotSchedulerImplSpy.handleFailedSnapshotDispatch(snapshotScheduleVoMock, volumeVoMock, snapshotScheduleVoMock, null, new Exception("boom"));
actionEventUtilsMocked.verify(() -> ActionEventUtils.onCreatedActionEvent(
Mockito.anyLong(), Mockito.anyLong(), Mockito.eq(EventVO.LEVEL_ERROR), Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyString(), Mockito.anyLong(), Mockito.anyString()));
Mockito.anyLong(), Mockito.anyLong(), Mockito.eq(EventVO.LEVEL_ERROR), Mockito.eq(EventTypes.EVENT_SNAPSHOT_CREATE), Mockito.anyBoolean(), Mockito.anyString(), Mockito.anyLong(), Mockito.anyString()));
// Must use a distinct event type from EVENT_SNAPSHOT_CREATE, otherwise it becomes the "latest event" scanned
// by countConsecutiveFailedAttempts and silently resets the consecutive-failure count on the next attempt.
actionEventUtilsMocked.verify(() -> ActionEventUtils.onCreatedActionEvent(
Mockito.anyLong(), Mockito.anyLong(), Mockito.eq(EventVO.LEVEL_WARN), Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyString(), Mockito.anyLong(), Mockito.anyString()));
Mockito.anyLong(), Mockito.anyLong(), Mockito.eq(EventVO.LEVEL_WARN), Mockito.eq(EventTypes.EVENT_SNAPSHOT_RECURRING_FAILURE_LIMIT_REACHED), Mockito.anyBoolean(), Mockito.anyString(), Mockito.anyLong(), Mockito.anyString()));
}
Mockito.verify(snapshotScheduleDaoMock).update(Mockito.anyLong(), Mockito.eq(snapshotScheduleVoMock));