KBOSS - Remove backup chain separation (#13680)
Some checks failed
CodeQL Analysis / CodeQL (actions) (push) Has been cancelled
Docker Image Build / build (push) Has been cancelled
Sonar Quality Check (Main) / Sonar JaCoCo Coverage (push) Has been cancelled

This commit is contained in:
João Jandre 2026-07-24 23:30:19 -03:00 committed by GitHub
parent e1cf0f335a
commit 4f117071c9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
29 changed files with 697 additions and 955 deletions

View File

@ -63,16 +63,6 @@ public interface BackupProvider {
*/ */
boolean removeVMFromBackupOffering(VirtualMachine vm); boolean removeVMFromBackupOffering(VirtualMachine vm);
/**
* Removes the specified backup schedule from a virtual machine.
*
* @param vm the virtual machine from which the schedule will be removed.
* @param backupSchedule the backup schedule to be removed.
* @return {@code true} if the operation was successful; {@code false} otherwise.
*/
default boolean removeVMBackupSchedule(VirtualMachine vm, BackupSchedule backupSchedule) {
return true;
}
/** /**
* Whether the provider will delete backups on removal of VM from the offering * Whether the provider will delete backups on removal of VM from the offering
@ -91,7 +81,7 @@ public interface BackupProvider {
* @param isolated * @param isolated
* @return the result and {code}Backup{code} {code}Object{code} * @return the result and {code}Backup{code} {code}Object{code}
*/ */
Pair<Boolean, Backup> takeBackup(VirtualMachine vm, Boolean quiesceVM, boolean isolated, Long backupScheduleId); Pair<Boolean, Backup> takeBackup(VirtualMachine vm, Boolean quiesceVM, boolean isolated);
/** /**
* Delete an existing backup * Delete an existing backup

View File

@ -136,7 +136,7 @@ public interface InternalBackupProvider extends BackupProvider {
default void prepareVmForSnapshotRevert(VMSnapshot vmSnapshot, VirtualMachine virtualMachine) { default void prepareVmForSnapshotRevert(VMSnapshot vmSnapshot, VirtualMachine virtualMachine) {
} }
default boolean finishBackupChains(VirtualMachine virtualMachine) { default boolean finishBackupChain(VirtualMachine virtualMachine) {
return false; return false;
} }
} }

View File

@ -18,26 +18,26 @@
*/ */
package org.apache.cloudstack.backup; package org.apache.cloudstack.backup;
import java.util.Map; import java.util.List;
import org.apache.commons.collections4.MapUtils; import org.apache.cloudstack.storage.to.VolumeObjectTO;
import org.apache.commons.collections4.CollectionUtils;
import com.cloud.agent.api.Answer; import com.cloud.agent.api.Answer;
import com.cloud.agent.api.Command; import com.cloud.agent.api.Command;
import com.cloud.utils.Pair;
public class CleanupKbossBackupErrorAnswer extends Answer { public class CleanupKbossBackupErrorAnswer extends Answer {
private Map<String, Pair<String, Boolean>> volumeIdToPathAndChainEnded; private List<VolumeObjectTO> volumeObjectTos;
private boolean vmRunning; private boolean vmRunning;
public CleanupKbossBackupErrorAnswer(Command cmd, Map<String, Pair<String, Boolean>> volumeIdToPathAndChainEnded, boolean vmRunning) { public CleanupKbossBackupErrorAnswer(Command cmd, List<VolumeObjectTO> volumeObjectTos, boolean vmRunning) {
super(cmd, MapUtils.isNotEmpty(volumeIdToPathAndChainEnded), null); super(cmd, CollectionUtils.isNotEmpty(volumeObjectTos), null);
this.volumeIdToPathAndChainEnded = volumeIdToPathAndChainEnded; this.volumeObjectTos = volumeObjectTos;
this.vmRunning = vmRunning; this.vmRunning = vmRunning;
} }
public Map<String, Pair<String, Boolean>> getVolumeIdToPathAndChainEnded() { public List<VolumeObjectTO> getVolumeObjectTos() {
return volumeIdToPathAndChainEnded; return volumeObjectTos;
} }
public boolean isVmRunning() { public boolean isVmRunning() {

View File

@ -25,40 +25,19 @@ public class CleanupKbossBackupErrorCommand extends Command {
private boolean runningVM; private boolean runningVM;
private boolean errorOnCreate;
private boolean endOfChain;
private boolean isTopDelta;
private String vmName; private String vmName;
private String imageStoreUrl; private String imageStoreUrl;
private List<KbossTO> kbossTOS; private List<KbossTO> kbossTOS;
public CleanupKbossBackupErrorCommand(boolean runningVM, boolean errorOnCreate, boolean endOfChain, boolean isTopDelta, String vmName, String imageStoreUrl, List<KbossTO> kbossTOS) { public CleanupKbossBackupErrorCommand(boolean runningVM, String vmName, String imageStoreUrl, List<KbossTO> kbossTOS) {
this.errorOnCreate = errorOnCreate;
this.runningVM = runningVM; this.runningVM = runningVM;
this.endOfChain = endOfChain;
this.isTopDelta = isTopDelta;
this.vmName = vmName; this.vmName = vmName;
this.imageStoreUrl = imageStoreUrl; this.imageStoreUrl = imageStoreUrl;
this.kbossTOS = kbossTOS; this.kbossTOS = kbossTOS;
} }
public boolean isErrorOnCreate() {
return errorOnCreate;
}
public boolean isEndOfChain() {
return endOfChain;
}
public boolean isTopDelta() {
return isTopDelta;
}
public boolean isRunningVM() { public boolean isRunningVM() {
return runningVM; return runningVM;
} }

View File

@ -16,9 +16,10 @@ package org.apache.cloudstack.storage.to;
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.stream.Collectors;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
import org.apache.commons.lang3.builder.ReflectionToStringBuilder; import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle; import org.apache.commons.lang3.builder.ToStringStyle;
@ -29,22 +30,20 @@ public class KbossTO {
private String deltaPathOnPrimary; private String deltaPathOnPrimary;
private String parentDeltaPathOnPrimary; private String parentDeltaPathOnPrimary;
private String deltaPathOnSecondary; private String deltaPathOnSecondary;
private String oldVolumePath;
private DeltaMergeTreeTO deltaMergeTreeTO; private DeltaMergeTreeTO deltaMergeTreeTO;
private List<String> deltaPaths; private List<String> vmSnapshotDeltaPaths;
public KbossTO(VolumeObjectTO volumeObjectTO, LinkedList<String> deltaPaths) { public KbossTO(VolumeObjectTO volumeObjectTO, List<SnapshotDataStoreVO> snapshotDataStoreVOs) {
this.volumeObjectTO = volumeObjectTO; this.volumeObjectTO = volumeObjectTO;
this.deltaPaths = deltaPaths; this.vmSnapshotDeltaPaths = snapshotDataStoreVOs.stream().map(SnapshotDataStoreVO::getInstallPath).collect(Collectors.toList());
} }
public KbossTO(VolumeObjectTO volumeObjectTO, String deltaPathOnPrimary, String deltaPathOnSecondary, LinkedList<String> deltaPaths) { public KbossTO(VolumeObjectTO volumeObjectTO, String deltaPathOnPrimary, String deltaPathOnSecondary) {
this.volumeObjectTO = volumeObjectTO; this.volumeObjectTO = volumeObjectTO;
this.deltaPathOnPrimary = deltaPathOnPrimary; this.deltaPathOnPrimary = deltaPathOnPrimary;
this.deltaPathOnSecondary = deltaPathOnSecondary; this.deltaPathOnSecondary = deltaPathOnSecondary;
this.deltaPaths = deltaPaths;
} }
public String getPathBackupParentOnSecondary() { public String getPathBackupParentOnSecondary() {
@ -59,8 +58,8 @@ public class KbossTO {
return deltaMergeTreeTO; return deltaMergeTreeTO;
} }
public List<String> getDeltaPaths() { public List<String> getVmSnapshotDeltaPaths() {
return deltaPaths; return vmSnapshotDeltaPaths;
} }
public String getDeltaPathOnPrimary() { public String getDeltaPathOnPrimary() {
@ -95,14 +94,6 @@ public class KbossTO {
this.deltaPathOnSecondary = deltaPathOnSecondary; this.deltaPathOnSecondary = deltaPathOnSecondary;
} }
public String getOldVolumePath() {
return oldVolumePath;
}
public void setOldVolumePath(String oldVolumePath) {
this.oldVolumePath = oldVolumePath;
}
@Override @Override
public String toString() { public String toString() {
return ReflectionToStringBuilder.toString(this, ToStringStyle.JSON_STYLE); return ReflectionToStringBuilder.toString(this, ToStringStyle.JSON_STYLE);

View File

@ -81,7 +81,6 @@ public class VolumeObjectTO extends DownloadableObjectTO implements DataTO, Seri
private String encryptFormat; private String encryptFormat;
private List<String> checkpointPaths; private List<String> checkpointPaths;
private Set<String> checkpointImageStoreUrls; private Set<String> checkpointImageStoreUrls;
private Set<String> deltasToRemove;
public VolumeObjectTO() { public VolumeObjectTO() {
@ -426,12 +425,4 @@ public class VolumeObjectTO extends DownloadableObjectTO implements DataTO, Seri
public void setCheckpointImageStoreUrls(Set<String> checkpointImageStoreUrls) { public void setCheckpointImageStoreUrls(Set<String> checkpointImageStoreUrls) {
this.checkpointImageStoreUrls = checkpointImageStoreUrls; this.checkpointImageStoreUrls = checkpointImageStoreUrls;
} }
public Set<String> getDeltasToRemove() {
return deltasToRemove;
}
public void setDeltasToRemove(Set<String> deltasToRemove) {
this.deltasToRemove = deltasToRemove;
}
} }

View File

@ -101,15 +101,6 @@ public class InternalBackupJoinVO {
@Column(name = "isolated") @Column(name = "isolated")
private Boolean isolated; private Boolean isolated;
@Column(name = "storage_pool_delta_path")
private String storagePoolDeltaPath;
@Column(name = "storage_pool_parent_path")
private String storagePoolParentPath;
@Column(name = "schedule_id")
private Long scheduleId;
public InternalBackupJoinVO() { public InternalBackupJoinVO() {
} }
@ -192,18 +183,6 @@ public class InternalBackupJoinVO {
return compressionStatus; return compressionStatus;
} }
public String getStoragePoolDeltaPath() {
return storagePoolDeltaPath;
}
public String getStoragePoolParentPath() {
return storagePoolParentPath;
}
public Long getScheduleId() {
return scheduleId;
}
@Override @Override
public String toString() { public String toString() {
return ReflectionToStringBuilder.toString(this, ToStringStyle.JSON_STYLE); return ReflectionToStringBuilder.toString(this, ToStringStyle.JSON_STYLE);

View File

@ -24,15 +24,11 @@ import java.util.List;
public interface InternalBackupJoinDao extends GenericDao<InternalBackupJoinVO, Long> { public interface InternalBackupJoinDao extends GenericDao<InternalBackupJoinVO, Long> {
List<InternalBackupJoinVO> listByBackedUpAndVmIdAndDateBeforeOrAfterOrderBy(long vmId, Long scheduleId, Date date, boolean before, boolean ascending); List<InternalBackupJoinVO> listByBackedUpAndVmIdAndDateBeforeOrAfterOrderBy(long vmId, Date date, boolean before, boolean ascending);
List<InternalBackupJoinVO> listIncludingRemovedByVmIdAndBeforeDateOrderByCreatedDesc(long vmId, Long scheduleId, Date beforeDate); List<InternalBackupJoinVO> listIncludingRemovedByVmIdAndBeforeDateOrderByCreatedDesc(long vmId, Date beforeDate);
InternalBackupJoinVO findCurrent(long vmId, Long scheduleId); InternalBackupJoinVO findCurrent(long vmId);
List<InternalBackupJoinVO> listCurrents(long vmId, boolean descending);
List<InternalBackupJoinVO> listCurrentsByVolumeIdDesc(long volumeId);
InternalBackupJoinVO findByParentId(long parentId); InternalBackupJoinVO findByParentId(long parentId);

View File

@ -39,8 +39,6 @@ public class InternalBackupJoinDaoImpl extends GenericDaoBase<InternalBackupJoin
private static final String ISOLATED = "isolated"; private static final String ISOLATED = "isolated";
private static final String PARENT_ID = "parent_id"; private static final String PARENT_ID = "parent_id";
private static final String IMAGE_STORE_ID = "image_store_id"; private static final String IMAGE_STORE_ID = "image_store_id";
private static final String SCHEDULE_ID = "schedule_id";
private static final String VOLUME_ID = "volume_id";
private SearchBuilder<InternalBackupJoinVO> backupSearch; private SearchBuilder<InternalBackupJoinVO> backupSearch;
private SearchBuilder<InternalBackupJoinVO> allBackupsSearch; private SearchBuilder<InternalBackupJoinVO> allBackupsSearch;
@ -54,7 +52,6 @@ public class InternalBackupJoinDaoImpl extends GenericDaoBase<InternalBackupJoin
backupSearch.and(CURRENT, backupSearch.entity().getCurrent(), SearchCriteria.Op.EQ); backupSearch.and(CURRENT, backupSearch.entity().getCurrent(), SearchCriteria.Op.EQ);
backupSearch.and(PARENT_ID, backupSearch.entity().getParentId(), SearchCriteria.Op.EQ); backupSearch.and(PARENT_ID, backupSearch.entity().getParentId(), SearchCriteria.Op.EQ);
backupSearch.and(ISOLATED, backupSearch.entity().getIsolated(), SearchCriteria.Op.EQ); backupSearch.and(ISOLATED, backupSearch.entity().getIsolated(), SearchCriteria.Op.EQ);
backupSearch.and(SCHEDULE_ID, backupSearch.entity().getScheduleId(), SearchCriteria.Op.EQ);
backupSearch.groupBy(backupSearch.entity().getId()); backupSearch.groupBy(backupSearch.entity().getId());
backupSearch.done(); backupSearch.done();
@ -63,14 +60,11 @@ public class InternalBackupJoinDaoImpl extends GenericDaoBase<InternalBackupJoin
allBackupsSearch.and(STATUS, allBackupsSearch.entity().getStatus(), SearchCriteria.Op.IN); allBackupsSearch.and(STATUS, allBackupsSearch.entity().getStatus(), SearchCriteria.Op.IN);
allBackupsSearch.and(PARENT_ID, allBackupsSearch.entity().getParentId(), SearchCriteria.Op.EQ); allBackupsSearch.and(PARENT_ID, allBackupsSearch.entity().getParentId(), SearchCriteria.Op.EQ);
allBackupsSearch.and(IMAGE_STORE_ID, allBackupsSearch.entity().getImageStoreId(), SearchCriteria.Op.EQ); allBackupsSearch.and(IMAGE_STORE_ID, allBackupsSearch.entity().getImageStoreId(), SearchCriteria.Op.EQ);
allBackupsSearch.and(VM_ID, allBackupsSearch.entity().getVmId(), SearchCriteria.Op.EQ);
allBackupsSearch.and(VOLUME_ID, allBackupsSearch.entity().getVolumeId(), SearchCriteria.Op.EQ);
allBackupsSearch.and(CURRENT, allBackupsSearch.entity().getCurrent(), SearchCriteria.Op.EQ);
allBackupsSearch.done(); allBackupsSearch.done();
} }
@Override @Override
public List<InternalBackupJoinVO> listByBackedUpAndVmIdAndDateBeforeOrAfterOrderBy(long vmId, Long scheduleId, Date date, boolean before, boolean ascending) { public List<InternalBackupJoinVO> listByBackedUpAndVmIdAndDateBeforeOrAfterOrderBy(long vmId, Date date, boolean before, boolean ascending) {
SearchCriteria<InternalBackupJoinVO> sc = backupSearch.create(); SearchCriteria<InternalBackupJoinVO> sc = backupSearch.create();
sc.setParameters(VM_ID, vmId); sc.setParameters(VM_ID, vmId);
sc.setParameters(STATUS, Backup.Status.BackedUp); sc.setParameters(STATUS, Backup.Status.BackedUp);
@ -80,29 +74,26 @@ public class InternalBackupJoinDaoImpl extends GenericDaoBase<InternalBackupJoin
sc.setParameters(CREATED_AFTER, date); sc.setParameters(CREATED_AFTER, date);
} }
sc.setParameters(ISOLATED, Boolean.FALSE.toString()); sc.setParameters(ISOLATED, Boolean.FALSE.toString());
sc.setParameters(SCHEDULE_ID, scheduleId);
Filter filter = new Filter(InternalBackupJoinVO.class, "date", ascending); Filter filter = new Filter(InternalBackupJoinVO.class, "date", ascending);
return new ArrayList<>(listBy(sc, filter)); return new ArrayList<>(listBy(sc, filter));
} }
@Override @Override
public List<InternalBackupJoinVO> listIncludingRemovedByVmIdAndBeforeDateOrderByCreatedDesc(long vmId, Long scheduleId, Date beforeDate) { public List<InternalBackupJoinVO> listIncludingRemovedByVmIdAndBeforeDateOrderByCreatedDesc(long vmId, Date beforeDate) {
SearchCriteria<InternalBackupJoinVO> sc = backupSearch.create(); SearchCriteria<InternalBackupJoinVO> sc = backupSearch.create();
sc.setParameters(VM_ID, vmId); sc.setParameters(VM_ID, vmId);
sc.setParameters(STATUS, Backup.Status.BackedUp, Backup.Status.Removed); sc.setParameters(STATUS, Backup.Status.BackedUp, Backup.Status.Removed);
sc.setParameters(CREATED_BEFORE, beforeDate); sc.setParameters(CREATED_BEFORE, beforeDate);
sc.setParameters(ISOLATED, Boolean.FALSE.toString()); sc.setParameters(ISOLATED, Boolean.FALSE.toString());
sc.setParameters(SCHEDULE_ID, scheduleId);
Filter filter = new Filter(InternalBackupJoinVO.class, "date", false); Filter filter = new Filter(InternalBackupJoinVO.class, "date", false);
return new ArrayList<>(listIncludingRemovedBy(sc, filter)); return new ArrayList<>(listIncludingRemovedBy(sc, filter));
} }
@Override @Override
public InternalBackupJoinVO findCurrent(long vmId, Long scheduleId) { public InternalBackupJoinVO findCurrent(long vmId) {
SearchCriteria<InternalBackupJoinVO> sc = backupSearch.create(); SearchCriteria<InternalBackupJoinVO> sc = backupSearch.create();
sc.setParameters(VM_ID, vmId); sc.setParameters(VM_ID, vmId);
sc.setParameters(CURRENT, Boolean.TRUE.toString()); sc.setParameters(CURRENT, Boolean.TRUE.toString());
sc.setParameters(SCHEDULE_ID, scheduleId);
return findOneBy(sc); return findOneBy(sc);
} }
@ -136,23 +127,4 @@ public class InternalBackupJoinDaoImpl extends GenericDaoBase<InternalBackupJoin
sc.setParameters(STATUS, Backup.Status.BackedUp); sc.setParameters(STATUS, Backup.Status.BackedUp);
return listBy(sc); return listBy(sc);
} }
@Override
public List<InternalBackupJoinVO> listCurrents(long vmId, boolean descending) {
SearchCriteria<InternalBackupJoinVO> sc = allBackupsSearch.create();
sc.setParameters(VM_ID, vmId);
sc.setParameters(CURRENT, Boolean.TRUE.toString());
Filter filter = new Filter(InternalBackupJoinVO.class, "date", !descending);
return listBy(sc, filter);
}
@Override
public List<InternalBackupJoinVO> listCurrentsByVolumeIdDesc(long volumeId) {
SearchCriteria<InternalBackupJoinVO> sc = allBackupsSearch.create();
sc.setParameters(VOLUME_ID, volumeId);
sc.setParameters(CURRENT, Boolean.TRUE.toString());
Filter filter = new Filter(InternalBackupJoinVO.class, "date", false);
return listBy(sc, filter);
}
} }

View File

@ -25,13 +25,9 @@ public interface InternalBackupStoragePoolDao extends GenericDao<InternalBackupS
List<InternalBackupStoragePoolVO> listByBackupId(long backupId); List<InternalBackupStoragePoolVO> listByBackupId(long backupId);
List<InternalBackupStoragePoolVO> listByVolumeId(long volumeId); InternalBackupStoragePoolVO findOneByVolumeId(long volumeId);
InternalBackupStoragePoolVO findOneByVolumeIdAndBackupId(long volumeId, long backupId);
void expungeByBackupId(long backupId); void expungeByBackupId(long backupId);
void expungeByVolumeId(long volumeId); void expungeByVolumeId(long volumeId);
void expungeByVolumeIdAndBackupId(long volumeId, long backupId);
} }

View File

@ -48,17 +48,9 @@ public class InternalBackupStoragePoolDaoImpl extends GenericDaoBase<InternalBac
} }
@Override @Override
public List<InternalBackupStoragePoolVO> listByVolumeId(long volumeId) { public InternalBackupStoragePoolVO findOneByVolumeId(long volumeId) {
SearchCriteria<InternalBackupStoragePoolVO> sc = backupSearch.create(); SearchCriteria<InternalBackupStoragePoolVO> sc = backupSearch.create();
sc.setParameters(VOLUME_ID, volumeId); sc.setParameters(VOLUME_ID, volumeId);
return listBy(sc);
}
@Override
public InternalBackupStoragePoolVO findOneByVolumeIdAndBackupId(long volumeId, long backupId) {
SearchCriteria<InternalBackupStoragePoolVO> sc = backupSearch.create();
sc.setParameters(VOLUME_ID, volumeId);
sc.setParameters(BACKUP_ID, backupId);
return findOneBy(sc); return findOneBy(sc);
} }
@ -75,12 +67,4 @@ public class InternalBackupStoragePoolDaoImpl extends GenericDaoBase<InternalBac
sc.setParameters(VOLUME_ID, volumeId); sc.setParameters(VOLUME_ID, volumeId);
expunge(sc); expunge(sc);
} }
@Override
public void expungeByVolumeIdAndBackupId(long volumeId, long backupId) {
SearchCriteria<InternalBackupStoragePoolVO> sc = backupSearch.create();
sc.setParameters(VOLUME_ID, volumeId);
sc.setParameters(BACKUP_ID, backupId);
expunge(sc);
}
} }

View File

@ -37,15 +37,11 @@ SELECT b.id,
MAX(CASE WHEN bd.name = 'current' THEN bd.value END) current, MAX(CASE WHEN bd.name = 'current' THEN bd.value END) current,
COALESCE(MAX(CASE WHEN bd.name = 'isolated' THEN bd.value END), 'false') isolated, COALESCE(MAX(CASE WHEN bd.name = 'isolated' THEN bd.value END), 'false') isolated,
nbpr.volume_id, nbpr.volume_id,
nbpr.backup_delta_path storage_pool_delta_path, nbsr.path image_store_path
nbpr.backup_parent_path storage_pool_parent_path,
nbsr.path image_store_path,
bs.id schedule_id
FROM backups b FROM backups b
LEFT JOIN backup_details bd ON b.id = bd.backup_id LEFT JOIN backup_details bd ON b.id = bd.backup_id
LEFT JOIN backup_offering bo ON b.backup_offering_id = bo.id LEFT JOIN backup_offering bo ON b.backup_offering_id = bo.id
LEFT JOIN internal_backup_store_ref nbsr ON b.id = nbsr.backup_id LEFT JOIN internal_backup_store_ref nbsr ON b.id = nbsr.backup_id
LEFT JOIN internal_backup_pool_ref nbpr ON nbpr.volume_id = nbsr.volume_id and nbpr.backup_id = b.id LEFT JOIN internal_backup_pool_ref nbpr ON nbpr.volume_id = nbsr.volume_id
LEFT JOIN backup_schedule bs ON bs.id = b.backup_schedule_id
WHERE bo.provider='kboss' WHERE bo.provider='kboss'
GROUP BY b.id, nbsr.volume_id; GROUP BY b.id, nbsr.volume_id;

View File

@ -18,6 +18,40 @@
*/ */
package org.apache.cloudstack.storage.vmsnapshot; package org.apache.cloudstack.storage.vmsnapshot;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.UUID;
import java.util.stream.Collectors;
import javax.inject.Inject;
import org.apache.cloudstack.backup.BackupManagerImpl;
import org.apache.cloudstack.backup.BackupOfferingVO;
import org.apache.cloudstack.backup.InternalBackupService;
import org.apache.cloudstack.backup.InternalBackupStoragePoolVO;
import org.apache.cloudstack.backup.dao.BackupOfferingDao;
import org.apache.cloudstack.backup.dao.InternalBackupStoragePoolDao;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority;
import org.apache.cloudstack.engine.subsystem.api.storage.VMSnapshotOptions;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.snapshot.SnapshotObject;
import org.apache.cloudstack.storage.to.BackupDeltaTO;
import org.apache.cloudstack.storage.to.DeltaMergeTreeTO;
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import org.apache.commons.collections.CollectionUtils;
import com.cloud.agent.api.Answer; import com.cloud.agent.api.Answer;
import com.cloud.agent.api.VMSnapshotTO; import com.cloud.agent.api.VMSnapshotTO;
import com.cloud.agent.api.storage.CreateDiskOnlyVmSnapshotAnswer; import com.cloud.agent.api.storage.CreateDiskOnlyVmSnapshotAnswer;
@ -49,40 +83,6 @@ import com.cloud.vm.VirtualMachine;
import com.cloud.vm.snapshot.VMSnapshot; import com.cloud.vm.snapshot.VMSnapshot;
import com.cloud.vm.snapshot.VMSnapshotDetailsVO; import com.cloud.vm.snapshot.VMSnapshotDetailsVO;
import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.VMSnapshotVO;
import org.apache.cloudstack.backup.BackupManagerImpl;
import org.apache.cloudstack.backup.BackupOfferingVO;
import org.apache.cloudstack.backup.InternalBackupJoinVO;
import org.apache.cloudstack.backup.InternalBackupService;
import org.apache.cloudstack.backup.InternalBackupStoragePoolVO;
import org.apache.cloudstack.backup.dao.BackupOfferingDao;
import org.apache.cloudstack.backup.dao.InternalBackupJoinDao;
import org.apache.cloudstack.backup.dao.InternalBackupStoragePoolDao;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority;
import org.apache.cloudstack.engine.subsystem.api.storage.VMSnapshotOptions;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.snapshot.SnapshotObject;
import org.apache.cloudstack.storage.to.BackupDeltaTO;
import org.apache.cloudstack.storage.to.DeltaMergeTreeTO;
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import org.apache.commons.collections.CollectionUtils;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.UUID;
import java.util.stream.Collectors;
public class KvmFileBasedStorageVmSnapshotStrategy extends StorageVMSnapshotStrategy { public class KvmFileBasedStorageVmSnapshotStrategy extends StorageVMSnapshotStrategy {
@ -106,8 +106,6 @@ public class KvmFileBasedStorageVmSnapshotStrategy extends StorageVMSnapshotStra
@Inject @Inject
private SnapshotDao snapshotDao; private SnapshotDao snapshotDao;
@Inject
private InternalBackupJoinDao internalBackupJoinDao;
@Override @Override
public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) { public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) {
@ -153,7 +151,7 @@ public class KvmFileBasedStorageVmSnapshotStrategy extends StorageVMSnapshotStra
List<SnapshotVO> volumeSnapshotVos = new ArrayList<>(); List<SnapshotVO> volumeSnapshotVos = new ArrayList<>();
if (isCurrent && numberOfChildren == 0) { if (isCurrent && numberOfChildren == 0) {
volumeSnapshotVos = mergeSucceedingDeltaOnSnapshot(vmSnapshotBeingDeleted, userVm, hostId, volumeTOs); volumeSnapshotVos = mergeCurrentDeltaOnSnapshot(vmSnapshotBeingDeleted, userVm, hostId, volumeTOs);
} else if (numberOfChildren == 0) { } else if (numberOfChildren == 0) {
logger.debug("Deleting VM snapshot [{}] as no snapshots/volumes depend on it.", vmSnapshot.getUuid()); logger.debug("Deleting VM snapshot [{}] as no snapshots/volumes depend on it.", vmSnapshot.getUuid());
volumeSnapshotVos = deleteSnapshot(vmSnapshotBeingDeleted, hostId); volumeSnapshotVos = deleteSnapshot(vmSnapshotBeingDeleted, hostId);
@ -278,7 +276,7 @@ public class KvmFileBasedStorageVmSnapshotStrategy extends StorageVMSnapshotStra
List<SnapshotVO> snapshotVos; List<SnapshotVO> snapshotVos;
if (oldParent.getCurrent()) { if (oldParent.getCurrent()) {
snapshotVos = mergeSucceedingDeltaOnSnapshot(oldParent, userVm, hostId, volumeTOs); snapshotVos = mergeCurrentDeltaOnSnapshot(oldParent, userVm, hostId, volumeTOs);
} else { } else {
List<VMSnapshotVO> oldSiblings = vmSnapshotDao.listByParentAndStateIn(oldParent.getId(), VMSnapshot.State.Ready, VMSnapshot.State.Hidden); List<VMSnapshotVO> oldSiblings = vmSnapshotDao.listByParentAndStateIn(oldParent.getId(), VMSnapshot.State.Ready, VMSnapshot.State.Hidden);
@ -426,7 +424,7 @@ public class KvmFileBasedStorageVmSnapshotStrategy extends StorageVMSnapshotStra
SnapshotObjectTO parentTO = (SnapshotObjectTO) deltaMergeTreeTO.getParent(); SnapshotObjectTO parentTO = (SnapshotObjectTO) deltaMergeTreeTO.getParent();
if (childTO instanceof BackupDeltaTO) { if (childTO instanceof BackupDeltaTO) {
InternalBackupStoragePoolVO backupDelta = internalBackupStoragePoolDao.findOneByVolumeIdAndBackupId(parentTO.getVolume().getVolumeId(), childTO.getId()); InternalBackupStoragePoolVO backupDelta = internalBackupStoragePoolDao.findOneByVolumeId(parentTO.getVolume().getVolumeId());
backupDelta.setBackupDeltaParentPath(parentTO.getPath()); backupDelta.setBackupDeltaParentPath(parentTO.getPath());
logger.debug("The child was also a KBOSS backup delta, will update the backup delta metadata. Updating backupDeltaParentPath of backupDelta [{}] to [{}].", backupDelta.getId(), parentTO.getPath()); logger.debug("The child was also a KBOSS backup delta, will update the backup delta metadata. Updating backupDeltaParentPath of backupDelta [{}] to [{}].", backupDelta.getId(), parentTO.getPath());
internalBackupStoragePoolDao.update(backupDelta.getId(), backupDelta); internalBackupStoragePoolDao.update(backupDelta.getId(), backupDelta);
@ -447,27 +445,26 @@ public class KvmFileBasedStorageVmSnapshotStrategy extends StorageVMSnapshotStra
return snapshotVOList; return snapshotVOList;
} }
private List<SnapshotVO> mergeSucceedingDeltaOnSnapshot(VMSnapshotVO vmSnapshotVo, UserVmVO userVmVO, Long hostId, List<VolumeObjectTO> volumeObjectTOS) { private List<SnapshotVO> mergeCurrentDeltaOnSnapshot(VMSnapshotVO vmSnapshotVo, UserVmVO userVmVO, Long hostId, List<VolumeObjectTO> volumeObjectTOS) {
logger.debug(String.format("Merging VM snapshot [%s] with the succeeding delta.", vmSnapshotVo.getUuid())); logger.debug("Merging VM snapshot [{}] with the current volume delta.", vmSnapshotVo.getUuid());
List<DeltaMergeTreeTO> deltaMergeTreeTOs = new ArrayList<>(); List<DeltaMergeTreeTO> deltaMergeTreeTOs = new ArrayList<>();
List<SnapshotDataStoreVO> volumeSnapshots = vmSnapshotHelper.getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(vmSnapshotVo.getId()); List<SnapshotDataStoreVO> volumeSnapshots = vmSnapshotHelper.getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(vmSnapshotVo.getId());
Map<Long, InternalBackupJoinVO> volumeIdAndSucceedingBackupMap = getVolumeIdAndSucceedingBackupMap(vmSnapshotVo);
for (VolumeObjectTO volumeObjectTO : volumeObjectTOS) { for (VolumeObjectTO volumeObjectTO : volumeObjectTOS) {
Long volumeId = volumeObjectTO.getId(); SnapshotDataStoreVO volumeParentSnapshot = volumeSnapshots.stream().filter(snapshot -> Objects.equals(snapshot.getVolumeId(), volumeObjectTO.getId()))
SnapshotDataStoreVO volumeParentSnapshot = volumeSnapshots.stream().filter(snapshot -> Objects.equals(snapshot.getVolumeId(), volumeId))
.findFirst() .findFirst()
.orElseThrow(() -> new CloudRuntimeException(String.format("Failed to find volume snapshot for volume [%s].", volumeObjectTO.getUuid()))); .orElseThrow(() -> new CloudRuntimeException(String.format("Failed to find volume snapshot for volume [%s].", volumeObjectTO.getUuid())));
DataTO parentSnapshot = snapshotDataFactory.getSnapshot(volumeParentSnapshot.getSnapshotId(), volumeParentSnapshot.getDataStoreId(), DataStoreRole.Primary).getTO(); DataTO parentSnapshot = snapshotDataFactory.getSnapshot(volumeParentSnapshot.getSnapshotId(), volumeParentSnapshot.getDataStoreId(), DataStoreRole.Primary).getTO();
if (volumeIdAndSucceedingBackupMap.containsKey(volumeId)) { InternalBackupStoragePoolVO backupDelta = internalBackupStoragePoolDao.findOneByVolumeId(volumeObjectTO.getVolumeId());
InternalBackupJoinVO succeedingBackup = volumeIdAndSucceedingBackupMap.get(volumeId);
logger.debug("The succeeding delta is also a KNIB backup delta. Will merge the snapshot delta of volume [{}] with the parent backup delta at [{}].", if (backupDelta != null && backupDelta.getBackupDeltaPath().equals(volumeObjectTO.getPath())) {
volumeObjectTO.getUuid(), succeedingBackup.getStoragePoolParentPath()); logger.debug("The current volume delta is also a KBOSS backup delta. Will merge the snapshot delta of volume [{}] with the parent backup delta at [{}].",
BackupDeltaTO childTo = new BackupDeltaTO(succeedingBackup.getId(), volumeObjectTO.getDataStore(), Hypervisor.HypervisorType.KVM, succeedingBackup.getStoragePoolParentPath()); volumeObjectTO.getUuid(), backupDelta.getBackupDeltaParentPath());
BackupDeltaTO childTo = new BackupDeltaTO(volumeObjectTO.getDataStore(), Hypervisor.HypervisorType.KVM, backupDelta.getBackupDeltaParentPath());
ArrayList<DataTO> grandChildren = new ArrayList<>(); ArrayList<DataTO> grandChildren = new ArrayList<>();
if (userVmVO.getState().equals(VirtualMachine.State.Stopped)) { if (userVmVO.getState().equals(VirtualMachine.State.Stopped)) {
grandChildren.add(new BackupDeltaTO(volumeObjectTO.getDataStore(), Hypervisor.HypervisorType.KVM, succeedingBackup.getStoragePoolDeltaPath())); grandChildren.add(new BackupDeltaTO(volumeObjectTO.getDataStore(), Hypervisor.HypervisorType.KVM, backupDelta.getBackupDeltaPath()));
} }
deltaMergeTreeTOs.add(new DeltaMergeTreeTO(volumeObjectTO, parentSnapshot, childTo, grandChildren)); deltaMergeTreeTOs.add(new DeltaMergeTreeTO(volumeObjectTO, parentSnapshot, childTo, grandChildren));
} else { } else {
@ -491,7 +488,7 @@ public class KvmFileBasedStorageVmSnapshotStrategy extends StorageVMSnapshotStra
if (dataTO instanceof BackupDeltaTO) { if (dataTO instanceof BackupDeltaTO) {
logger.debug("The child of deltaMergeTree [{}] is a backupDeltaTO, thus, we will update the backup delta metadata.", deltaMergeTreeTO); logger.debug("The child of deltaMergeTree [{}] is a backupDeltaTO, thus, we will update the backup delta metadata.", deltaMergeTreeTO);
InternalBackupStoragePoolVO backupDelta = internalBackupStoragePoolDao.findOneByVolumeIdAndBackupId(parentTO.getVolume().getVolumeId(), dataTO.getId()); InternalBackupStoragePoolVO backupDelta = internalBackupStoragePoolDao.findOneByVolumeId(parentTO.getVolume().getVolumeId());
backupDelta.setBackupDeltaParentPath(parentTO.getPath()); backupDelta.setBackupDeltaParentPath(parentTO.getPath());
internalBackupStoragePoolDao.update(backupDelta.getId(), backupDelta); internalBackupStoragePoolDao.update(backupDelta.getId(), backupDelta);
} else { } else {
@ -655,7 +652,6 @@ public class KvmFileBasedStorageVmSnapshotStrategy extends StorageVMSnapshotStra
List<SnapshotDataStoreVO> parentVolumeSnapshots = vmSnapshotHelper.getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(parent.getId()); List<SnapshotDataStoreVO> parentVolumeSnapshots = vmSnapshotHelper.getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(parent.getId());
List<SnapshotDataStoreVO> childVolumeSnapshots = vmSnapshotHelper.getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(child.getId()); List<SnapshotDataStoreVO> childVolumeSnapshots = vmSnapshotHelper.getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(child.getId());
List<SnapshotDataStoreVO> grandChildrenVolumeSnapshots = new ArrayList<>(); List<SnapshotDataStoreVO> grandChildrenVolumeSnapshots = new ArrayList<>();
Map<Long, InternalBackupJoinVO> volumeIdAndSucceedingBackupMap = getVolumeIdAndSucceedingBackupMap(parent);
for (VMSnapshotVO grandChild : grandChildren) { for (VMSnapshotVO grandChild : grandChildren) {
grandChildrenVolumeSnapshots.addAll(vmSnapshotHelper.getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(grandChild.getId())); grandChildrenVolumeSnapshots.addAll(vmSnapshotHelper.getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(grandChild.getId()));
@ -664,14 +660,14 @@ public class KvmFileBasedStorageVmSnapshotStrategy extends StorageVMSnapshotStra
for (SnapshotDataStoreVO parentSnapshotDataStoreVO : parentVolumeSnapshots) { for (SnapshotDataStoreVO parentSnapshotDataStoreVO : parentVolumeSnapshots) {
SnapshotObjectTO parentTO = (SnapshotObjectTO) snapshotDataFactory.getSnapshot(parentSnapshotDataStoreVO.getSnapshotId(), parentSnapshotDataStoreVO.getDataStoreId(), DataStoreRole.Primary).getTO(); SnapshotObjectTO parentTO = (SnapshotObjectTO) snapshotDataFactory.getSnapshot(parentSnapshotDataStoreVO.getSnapshotId(), parentSnapshotDataStoreVO.getDataStoreId(), DataStoreRole.Primary).getTO();
VolumeObjectTO volumeObjectTO = parentTO.getVolume(); VolumeObjectTO volumeObjectTO = parentTO.getVolume();
InternalBackupJoinVO succeedingBackup = volumeIdAndSucceedingBackupMap.get(volumeObjectTO.getId());
SnapshotDataStoreVO childVO = childVolumeSnapshots.stream() SnapshotDataStoreVO childVO = childVolumeSnapshots.stream()
.filter(childSnapshot -> Objects.equals(parentSnapshotDataStoreVO.getVolumeId(), childSnapshot.getVolumeId())) .filter(childSnapshot -> Objects.equals(parentSnapshotDataStoreVO.getVolumeId(), childSnapshot.getVolumeId()))
.findFirst().orElseThrow(() -> new CloudRuntimeException(String.format("Could not find child snapshot of parent [%s].", parentSnapshotDataStoreVO.getSnapshotId()))); .findFirst().orElseThrow(() -> new CloudRuntimeException(String.format("Could not find child snapshot of parent [%s].", parentSnapshotDataStoreVO.getSnapshotId())));
InternalBackupStoragePoolVO backupDelta = internalBackupStoragePoolDao.findOneByVolumeId(childVO.getVolumeId());
List<DataTO> grandChildrenTOList = new ArrayList<>(); List<DataTO> grandChildrenTOList = new ArrayList<>();
DataTO childTO = getChildAndGrandChildren(child, stoppedVm, parentSnapshotDataStoreVO, succeedingBackup, childVO, volumeObjectTO, grandChildrenTOList, DataTO childTO = getChildAndGrandChildren(child, stoppedVm, parentSnapshotDataStoreVO, backupDelta, childVO, volumeObjectTO, grandChildrenTOList,
grandChildrenVolumeSnapshots); grandChildrenVolumeSnapshots);
snapshotMergeTrees.add(new DeltaMergeTreeTO(volumeObjectTO, parentTO, childTO, grandChildrenTOList)); snapshotMergeTrees.add(new DeltaMergeTreeTO(volumeObjectTO, parentTO, childTO, grandChildrenTOList));
@ -684,16 +680,16 @@ public class KvmFileBasedStorageVmSnapshotStrategy extends StorageVMSnapshotStra
/** /**
* Gets the correct children and grandchildren, taking KBOSS backups into account. * Gets the correct children and grandchildren, taking KBOSS backups into account.
* */ * */
private DataTO getChildAndGrandChildren(VMSnapshotVO childSnapshot, boolean stoppedVm, SnapshotDataStoreVO parentSnapshotDataStoreVO, InternalBackupJoinVO childBackup, private DataTO getChildAndGrandChildren(VMSnapshotVO child, boolean stoppedVm, SnapshotDataStoreVO parentSnapshotDataStoreVO, InternalBackupStoragePoolVO backupDelta,
SnapshotDataStoreVO childVO, VolumeObjectTO volumeObjectTO, List<DataTO> grandChildrenTOList, List<SnapshotDataStoreVO> grandChildrenVolumeSnapshots) { SnapshotDataStoreVO childVO, VolumeObjectTO volumeObjectTO, List<DataTO> grandChildrenTOList, List<SnapshotDataStoreVO> grandChildrenVolumeSnapshots) {
DataTO childTO; DataTO childTO;
if (childBackup != null && childBackup.getDate().before(childSnapshot.getCreated())) { if (backupDelta != null && backupDelta.getBackupDeltaPath().equals(childVO.getInstallPath())) {
logger.debug("The child snapshot delta is also a backup delta. We will set the backup delta parent path [{}] as the child and the backup delta path [{}] " + logger.debug("The child snapshot delta is also a backup delta. We will set the backup delta parent path [{}] as the child and the backup delta path [{}] " +
"as the grand-child.", parentSnapshotDataStoreVO.getInstallPath(), childBackup.getStoragePoolDeltaPath()); "as the grand-child.", backupDelta.getBackupDeltaParentPath(), backupDelta.getBackupDeltaPath());
childTO = new BackupDeltaTO(childBackup.getId(), volumeObjectTO.getDataStore(), Hypervisor.HypervisorType.KVM, childBackup.getStoragePoolParentPath()); childTO = new BackupDeltaTO(volumeObjectTO.getDataStore(), Hypervisor.HypervisorType.KVM, backupDelta.getBackupDeltaParentPath());
if (stoppedVm) { if (!child.getCurrent() && stoppedVm) {
grandChildrenTOList.add(new BackupDeltaTO(volumeObjectTO.getDataStore(), Hypervisor.HypervisorType.KVM, childBackup.getStoragePoolDeltaPath())); grandChildrenTOList.add(new BackupDeltaTO(volumeObjectTO.getDataStore(), Hypervisor.HypervisorType.KVM, backupDelta.getBackupDeltaPath()));
} }
} else { } else {
childTO = snapshotDataFactory.getSnapshot(childVO.getSnapshotId(), childVO.getDataStoreId(), DataStoreRole.Primary).getTO(); childTO = snapshotDataFactory.getSnapshot(childVO.getSnapshotId(), childVO.getDataStoreId(), DataStoreRole.Primary).getTO();
@ -703,7 +699,7 @@ public class KvmFileBasedStorageVmSnapshotStrategy extends StorageVMSnapshotStra
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
if (childSnapshot.getCurrent() && stoppedVm && grandChildrenTOList.isEmpty()) { if (child.getCurrent() && stoppedVm) {
grandChildrenTOList.add(volumeObjectTO); grandChildrenTOList.add(volumeObjectTO);
} }
@ -762,26 +758,4 @@ public class KvmFileBasedStorageVmSnapshotStrategy extends StorageVMSnapshotStra
throw new CloudRuntimeException(msg, e); throw new CloudRuntimeException(msg, e);
} }
} }
private Map<Long, InternalBackupJoinVO> getVolumeIdAndSucceedingBackupMap(VMSnapshotVO vmSnapshotVO) {
Map<Long, InternalBackupJoinVO> volumeIdAndSucceedingBackupMap = new HashMap<>();
if (vmSnapshotVO == null) {
return volumeIdAndSucceedingBackupMap;
}
List<InternalBackupJoinVO> currents = internalBackupJoinDao.listCurrents(vmSnapshotVO.getVmId(), false)
.stream().filter(internalBackupJoinVO -> internalBackupJoinVO.getDate().after(vmSnapshotVO.getCreated())).collect(Collectors.toList());
if (currents.isEmpty()) {
logger.debug("No backups created after the VM snapshot [{}] were found, returning.", vmSnapshotVO.getUuid());
return volumeIdAndSucceedingBackupMap;
}
InternalBackupJoinVO succeedingBackup = currents.get(0);
volumeIdAndSucceedingBackupMap = currents.stream().filter(b -> succeedingBackup.getId() == b.getId())
.collect(Collectors.toMap(InternalBackupJoinVO::getVolumeId, internalBackupJoinVO -> internalBackupJoinVO));
logger.debug("Found the following backups that succeeds the VM snapshot [{}]: [{}].", vmSnapshotVO.getUuid(), volumeIdAndSucceedingBackupMap.values());
return volumeIdAndSucceedingBackupMap;
}
} }

View File

@ -154,7 +154,7 @@ public class DummyBackupProvider extends AdapterBase implements BackupProvider {
} }
@Override @Override
public Pair<Boolean, Backup> takeBackup(VirtualMachine vm, Boolean quiesceVM, boolean isolated, Long backupScheduleId) { public Pair<Boolean, Backup> takeBackup(VirtualMachine vm, Boolean quiesceVM, boolean isolated) {
logger.debug("Starting backup for VM {} on Dummy provider", vm); logger.debug("Starting backup for VM {} on Dummy provider", vm);
BackupVO backup = new BackupVO(); BackupVO backup = new BackupVO();

View File

@ -29,12 +29,10 @@ import java.io.File;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.LinkedHashSet; import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
@ -131,7 +129,6 @@ import com.cloud.uservm.UserVm;
import com.cloud.utils.DateUtil; import com.cloud.utils.DateUtil;
import com.cloud.utils.Pair; import com.cloud.utils.Pair;
import com.cloud.utils.Predicate; import com.cloud.utils.Predicate;
import com.cloud.utils.Ternary;
import com.cloud.utils.component.AdapterBase; import com.cloud.utils.component.AdapterBase;
import com.cloud.utils.db.EntityManager; import com.cloud.utils.db.EntityManager;
import com.cloud.utils.db.Transaction; import com.cloud.utils.db.Transaction;
@ -334,23 +331,11 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
logger.info("Removing VM [{}] from KBOSS backup offering.", vm.getUuid()); logger.info("Removing VM [{}] from KBOSS backup offering.", vm.getUuid());
validateVmState(vm, "remove backup offering", VirtualMachine.State.Expunging, VirtualMachine.State.Destroyed); validateVmState(vm, "remove backup offering", VirtualMachine.State.Expunging, VirtualMachine.State.Destroyed);
List<InternalBackupJoinVO> currents = internalBackupJoinDao.listCurrents(vm.getId(), true); if (endBackupChain(vm)) {
return finishAllChains(vm, currents);
}
@Override
public boolean removeVMBackupSchedule(VirtualMachine vm, BackupSchedule backupSchedule) {
logger.info("Removing VM [{}] from KBOSS backup schedule.", vm.getUuid());
if (endBackupChain(vm, backupSchedule.getId())) {
return true; return true;
} }
UserVmVO vmVO = userVmDao.findById(vm.getId()); UserVmVO vmVO = userVmDao.findById(vm.getId());
logger.error("Failed to merge deltas for VM [{}] during backup schedule removal process. Changing its state to [{}].", vm, VirtualMachine.State.BackupError); logger.error("Failed to merge deltas for VM [{}] during backup offering removal process. Changing its state to [{}].", vm, VirtualMachine.State.BackupError);
BackupVO backupVO = backupDao.findById(internalBackupJoinDao.findCurrent(vm.getId(), backupSchedule.getId()).getId());
backupVO.setStatus(Backup.Status.Error);
backupDao.update(backupVO.getId(), backupVO);
vmInstanceDetailsDao.addDetail(vm.getId(), VmDetailConstants.LAST_KNOWN_STATE, vmVO.getState().name(), false); vmInstanceDetailsDao.addDetail(vm.getId(), VmDetailConstants.LAST_KNOWN_STATE, vmVO.getState().name(), false);
vmVO.setState(VirtualMachine.State.BackupError); vmVO.setState(VirtualMachine.State.BackupError);
userVmDao.update(vmVO.getId(), vmVO); userVmDao.update(vmVO.getId(), vmVO);
@ -364,9 +349,9 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
} }
@Override @Override
public Pair<Boolean, Backup> takeBackup(VirtualMachine vm, Boolean quiesceVm, boolean isolated, Long backupScheduleId) { public Pair<Boolean, Backup> takeBackup(VirtualMachine vm, Boolean quiesceVm, boolean isolated) {
logger.debug("Queueing backup on VM [{}].", vm.getUuid()); logger.debug("Queueing backup on VM [{}].", vm.getUuid());
Outcome<?> outcome = createBackupThroughJobQueue(vm, ObjectUtils.defaultIfNull(quiesceVm, false), isolated, backupScheduleId); Outcome<?> outcome = createBackupThroughJobQueue(vm, ObjectUtils.defaultIfNull(quiesceVm, false), isolated);
try { try {
outcome.get(); outcome.get();
@ -438,19 +423,6 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
HashMap<String, InternalBackupStoragePoolVO> volumeUuidToDeltaPrimaryRef = new HashMap<>(); HashMap<String, InternalBackupStoragePoolVO> volumeUuidToDeltaPrimaryRef = new HashMap<>();
HashMap<String, InternalBackupDataStoreVO> volumeUuidToDeltaSecondaryRef = new HashMap<>(); HashMap<String, InternalBackupDataStoreVO> volumeUuidToDeltaSecondaryRef = new HashMap<>();
boolean runningVm = userVm.getState() == VirtualMachine.State.Running;
transitVmState(userVm, VirtualMachine.Event.BackupRequested, hostId);
updateBackupStatusToBackingUp(volumeTOs, backupVO);
DataStore imageStore = getImageStoreForBackup(userVm.getDataCenterId(), backupVO);
createBasicBackupDetails(imageStore.getId(), fullBackup ? 0L : parentBackup.getId(), backupVO);
List<InternalBackupJoinVO> succeedingBackupList = getSucceedingBackupList(parentBackup);
InternalBackupJoinVO succeedingBackup = succeedingBackupList.isEmpty() ? null : succeedingBackupList.get(0);
List<VMSnapshotVO> succeedingVmSnapshotList = getSucceedingVmSnapshotList(parentBackup);
VMSnapshotVO succeedingVmSnapshot = succeedingVmSnapshotList.isEmpty() ? null : succeedingVmSnapshotList.get(0);
if (!fullBackup) { if (!fullBackup) {
parentBackupDeltasOnPrimary = internalBackupStoragePoolDao.listByBackupId(parentBackup.getId()); parentBackupDeltasOnPrimary = internalBackupStoragePoolDao.listByBackupId(parentBackup.getId());
parentBackupDeltasOnSecondary = internalBackupDataStoreDao.listByBackupId(parentBackup.getId()); parentBackupDeltasOnSecondary = internalBackupDataStoreDao.listByBackupId(parentBackup.getId());
@ -458,11 +430,21 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
chainImageStoreUrls = getChainImageStoreUrls(backupChain); chainImageStoreUrls = getChainImageStoreUrls(backupChain);
} }
Map<Long, LinkedList<String>> volumeIdToSnapshotDataStoreAndBackupPathList = mapVolumesToVmSnapshotAndBackupReferences(volumeTOs, succeedingVmSnapshotList, succeedingBackupList); boolean runningVm = userVm.getState() == VirtualMachine.State.Running;
transitVmState(userVm, VirtualMachine.Event.BackupRequested, hostId);
updateBackupStatusToBackingUp(volumeTOs, backupVO);
DataStore imageStore = getImageStoreForBackup(userVm.getDataCenterId(), backupVO);
createBasicBackupDetails(imageStore.getId(), fullBackup ? 0L : parentBackup.getId(), backupVO);
List<VMSnapshotVO> succeedingVmSnapshotList = getSucceedingVmSnapshotList(parentBackup);
VMSnapshotVO succeedingVmSnapshot = succeedingVmSnapshotList.isEmpty() ? null : succeedingVmSnapshotList.get(0);
Map<Long, List<SnapshotDataStoreVO>> volumeIdToSnapshotDataStoreList = mapVolumesToVmSnapshotReferences(volumeTOs, succeedingVmSnapshotList);
for (VolumeObjectTO volumeObjectTO : volumeTOs) { for (VolumeObjectTO volumeObjectTO : volumeTOs) {
KbossTO kbossTO = new KbossTO(volumeObjectTO, volumeIdToSnapshotDataStoreAndBackupPathList.getOrDefault(volumeObjectTO.getId(), new LinkedList<>())); KbossTO kbossTO = new KbossTO(volumeObjectTO, volumeIdToSnapshotDataStoreList.getOrDefault(volumeObjectTO.getId(), new ArrayList<>()));
kbossTOs.add(kbossTO); kbossTOs.add(kbossTO);
createDeltaReferences(fullBackup, runningVm, backup, parentBackupDeltasOnSecondary, createDeltaReferences(fullBackup, !succeedingVmSnapshotList.isEmpty(), runningVm, backup, parentBackupDeltasOnSecondary,
parentBackupDeltasOnPrimary, volumeUuidToDeltaPrimaryRef, volumeUuidToDeltaSecondaryRef, succeedingVmSnapshot, kbossTO); parentBackupDeltasOnPrimary, volumeUuidToDeltaPrimaryRef, volumeUuidToDeltaSecondaryRef, succeedingVmSnapshot, kbossTO);
} }
@ -477,7 +459,7 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
} }
processBackupSuccess(runningVm, volumeTOs, volumeUuidToDeltaPrimaryRef, volumeUuidToDeltaSecondaryRef, (TakeKbossBackupAnswer)answer, parentBackupDeltasOnPrimary, processBackupSuccess(runningVm, volumeTOs, volumeUuidToDeltaPrimaryRef, volumeUuidToDeltaSecondaryRef, (TakeKbossBackupAnswer)answer, parentBackupDeltasOnPrimary,
succeedingVmSnapshot, backupVO, fullBackup, userVm, hostId, newBackupJoin.getEndOfChain(), isolated, succeedingBackup); succeedingVmSnapshotList, backupVO, fullBackup, userVm, hostId, newBackupJoin.getEndOfChain(), isolated);
if (!isolated) { if (!isolated) {
updateCurrentBackup(newBackupJoin); updateCurrentBackup(newBackupJoin);
@ -577,7 +559,7 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
List<Long> removedBackupIds = backupParentsToBeRemovedAndLastAliveBackup.first().stream().map(InternalBackupJoinVO::getId).collect(Collectors.toList()); List<Long> removedBackupIds = backupParentsToBeRemovedAndLastAliveBackup.first().stream().map(InternalBackupJoinVO::getId).collect(Collectors.toList());
removedBackupIds.add(backup.getId()); removedBackupIds.add(backup.getId());
boolean isFailedSetEmpty = processRemoveBackupFailures(forced, deleteAnswers, removedBackupIds, backupJoinVO, virtualMachine); boolean isFailedSetEmpty = processRemoveBackupFailures(forced, deleteAnswers, removedBackupIds, backupJoinVO);
processRemovedBackups(removedBackupIds); processRemovedBackups(removedBackupIds);
@ -625,10 +607,10 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
} }
InternalBackupJoinVO backupJoinVO = internalBackupJoinDao.findById(backupId); InternalBackupJoinVO backupJoinVO = internalBackupJoinDao.findById(backupId);
List<InternalBackupJoinVO> currentBackups = sameVmAsBackup ? internalBackupJoinDao.listCurrents(vm.getId(), false) : List.of(); InternalBackupJoinVO currentBackup = sameVmAsBackup ? internalBackupJoinDao.findCurrent(vm.getId()) : null;
List<InternalBackupStoragePoolVO> deltasOnPrimary = new ArrayList<>(); List<InternalBackupStoragePoolVO> deltasOnPrimary = new ArrayList<>();
for (InternalBackupJoinVO currentBackup : currentBackups) { if (currentBackup != null) {
deltasOnPrimary.addAll(0, internalBackupStoragePoolDao.listByBackupId(currentBackup.getId())); deltasOnPrimary = internalBackupStoragePoolDao.listByBackupId(currentBackup.getId());
} }
List<InternalBackupDataStoreVO> deltasOnSecondary = internalBackupDataStoreDao.listByBackupId(backupId); List<InternalBackupDataStoreVO> deltasOnSecondary = internalBackupDataStoreDao.listByBackupId(backupId);
List<VolumeObjectTO> volumeTOs = vmSnapshotHelper.getVolumeTOList(vm.getId()); List<VolumeObjectTO> volumeTOs = vmSnapshotHelper.getVolumeTOList(vm.getId());
@ -685,7 +667,7 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
updateVolumePathsAndSizeIfNeeded(vm, volumeTOs, volumeInfos, deltasToBeMerged, sameVmAsBackup); updateVolumePathsAndSizeIfNeeded(vm, volumeTOs, volumeInfos, deltasToBeMerged, sameVmAsBackup);
for (InternalBackupJoinVO currentBackup : currentBackups) { if (currentBackup != null) {
internalBackupStoragePoolDao.expungeByBackupId(currentBackup.getId()); internalBackupStoragePoolDao.expungeByBackupId(currentBackup.getId());
setEndOfChainAndRemoveCurrentForBackup(currentBackup); setEndOfChainAndRemoveCurrentForBackup(currentBackup);
} }
@ -901,17 +883,17 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
} }
@Override @Override
public boolean finishBackupChains(VirtualMachine virtualMachine) { public boolean finishBackupChain(VirtualMachine virtualMachine) {
UserVmVO vm = userVmDao.findById(virtualMachine.getId()); UserVmVO userVmVO = userVmDao.findById(virtualMachine.getId());
List<InternalBackupJoinVO> currents = internalBackupJoinDao.listCurrents(vm.getId(), true); if (allowedVmStates.contains(userVmVO.getState())) {
if (allowedVmStates.contains(vm.getState())) { return endBackupChain(userVmVO);
return finishAllChains(vm, currents);
} }
if (vm.getState() != VirtualMachine.State.BackupError) { if (userVmVO.getState() != VirtualMachine.State.BackupError) {
logger.error("VM [{}] is not in the right state to finish backup chain. It can only be in states [Running, Stopped and BackupError].", vm.getUuid()); logger.error("VM [{}] is not in the right state to finish backup chain. It can only be in states [Running, Stopped and BackupError].", userVmVO.getUuid());
return false; return false;
} }
return normalizeBackupErrorAndFinishChain(vm); return normalizeBackupErrorAndFinishChain(userVmVO);
} }
@Override @Override
@ -945,14 +927,14 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
@Override @Override
public void prepareVolumeForDetach(Volume volume, VirtualMachine virtualMachine) { public void prepareVolumeForDetach(Volume volume, VirtualMachine virtualMachine) {
logger.info("Preparing volume [{}] for detach.", volume.getUuid()); logger.info("Preparing volume [{}] for detach.", volume.getUuid());
mergeCurrentDeltasIntoVolume(volume, virtualMachine, "detach", virtualMachine.getState().equals(VirtualMachine.State.Running)); mergeCurrentDeltaIntoVolume(volume, virtualMachine, "detach", virtualMachine.getState().equals(VirtualMachine.State.Running));
} }
@Override @Override
public void prepareVolumeForMigration(Volume volume, VirtualMachine vm) { public void prepareVolumeForMigration(Volume volume, VirtualMachine vm) {
if (VirtualMachine.State.Migrating.equals(vm.getState())) { if (VirtualMachine.State.Migrating.equals(vm.getState())) {
logger.info("Preparing volume [{}] for live migration.", volume.getUuid()); logger.info("Preparing volume [{}] for live migration.", volume.getUuid());
mergeCurrentDeltasIntoVolume(volume, vm, "live migration", true); mergeCurrentDeltaIntoVolume(volume, vm, "live migration", true);
} }
} }
@ -963,37 +945,31 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
@Override @Override
public void prepareVmForSnapshotRevert(VMSnapshot vmSnapshot, VirtualMachine virtualMachine) { public void prepareVmForSnapshotRevert(VMSnapshot vmSnapshot, VirtualMachine virtualMachine) {
List<InternalBackupJoinVO> currentBackups = internalBackupJoinDao.listCurrents(virtualMachine.getId(), true); InternalBackupJoinVO currentBackup = internalBackupJoinDao.findCurrent(virtualMachine.getId());
if (currentBackups.isEmpty()) { if (currentBackup == null) {
logger.debug("There is no current backup delta, the VM [{}] is already prepared for VM snapshot revert.", virtualMachine.getUuid()); logger.debug("There is no current backup delta, the VM [{}] is already prepared for VM snapshot revert.", virtualMachine.getUuid());
return; return;
} }
currentBackups = currentBackups.stream().filter(backup -> backup.getDate().after(vmSnapshot.getCreated())).collect(Collectors.toList()); if (currentBackup.getDate().before(vmSnapshot.getCreated())) {
if (currentBackups.isEmpty()) { logger.debug("The current backup delta was taken before [{}] the VM snapshot being reverted [{}], no need to prepare the VM.", currentBackup.getDate(),
logger.debug("Existing backup deltas [{}] were created before the target VM snapshot [{}]. No preparation needed for VM [{}].", vmSnapshot.getCreated());
currentBackups, vmSnapshot.getCreated(), virtualMachine.getUuid());
return; return;
} }
logger.debug("Preparing VM [{}] for VM snapshot reversion.", virtualMachine.getUuid()); logger.debug("Preparing VM [{}] for VM snapshot reversion.", virtualMachine.getUuid());
List<VolumeObjectTO> volumeObjectTOs = vmSnapshotHelper.getVolumeTOList(virtualMachine.getId()); List<VolumeObjectTO> volumeObjectTOs = vmSnapshotHelper.getVolumeTOList(virtualMachine.getId());
VMSnapshotVO vmSnapshotSucceedingCurrentBackup = getSucceedingVmSnapshot(currentBackup);
List<DeltaMergeTreeTO> deltaMergeTreeTOList = new ArrayList<>(); List<DeltaMergeTreeTO> deltaMergeTreeTOList = new ArrayList<>();
Commands commands = new Commands(Command.OnError.Stop); Commands commands = new Commands(Command.OnError.Stop);
List<InternalBackupStoragePoolVO> deletedDeltas = new ArrayList<>(); List<InternalBackupStoragePoolVO> deletedDeltas = new ArrayList<>();
Map<InternalBackupJoinVO, VMSnapshotVO> backupVmSnapshotMap = new HashMap<>(); createDeleteCommandsAndMergeTrees(volumeObjectTOs, commands, deletedDeltas, vmSnapshotSucceedingCurrentBackup, deltaMergeTreeTOList);
for (InternalBackupJoinVO currentBackup : currentBackups) { if (!deltaMergeTreeTOList.isEmpty()) {
VMSnapshotVO vmSnapshotSucceedingCurrentBackup = getSucceedingVmSnapshot(currentBackup);
createDeleteCommandsAndMergeTrees(volumeObjectTOs, commands, deletedDeltas, vmSnapshotSucceedingCurrentBackup, deltaMergeTreeTOList, currentBackup);
backupVmSnapshotMap.put(currentBackup, vmSnapshotSucceedingCurrentBackup);
}
if (CollectionUtils.isNotEmpty(deltaMergeTreeTOList)) {
commands.addCommand(new MergeDiskOnlyVmSnapshotCommand(deltaMergeTreeTOList, false, virtualMachine.getInstanceName())); commands.addCommand(new MergeDiskOnlyVmSnapshotCommand(deltaMergeTreeTOList, false, virtualMachine.getInstanceName()));
} }
@ -1012,18 +988,12 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
throw new CloudRuntimeException(String.format("Unable to prepare VM [%s] for VM snapshot reversion.", virtualMachine.getUuid())); throw new CloudRuntimeException(String.format("Unable to prepare VM [%s] for VM snapshot reversion.", virtualMachine.getUuid()));
} }
for (Map.Entry<InternalBackupJoinVO, VMSnapshotVO> backupAndVmSnapshot : backupVmSnapshotMap.entrySet()) { List<SnapshotDataStoreVO> snapRefsSucceedingCurrentBackup = new ArrayList<>();
InternalBackupJoinVO backup = backupAndVmSnapshot.getKey(); if (vmSnapshotSucceedingCurrentBackup != null) {
VMSnapshotVO vmSnapshotSucceedingBackup = backupAndVmSnapshot.getValue(); snapRefsSucceedingCurrentBackup = vmSnapshotHelper.getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(vmSnapshotSucceedingCurrentBackup.getId());
List<SnapshotDataStoreVO> snapRefsSucceedingCurrentBackup = new ArrayList<>();
if (vmSnapshotSucceedingBackup != null) {
snapRefsSucceedingCurrentBackup = vmSnapshotHelper.getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(vmSnapshotSucceedingBackup.getId());
}
updateReferencesAfterPrepareForSnapshotRevert(deltaMergeTreeTOList, snapRefsSucceedingCurrentBackup, deletedDeltas, backup);
} }
updateReferencesAfterPrepareForSnapshotRevert(deltaMergeTreeTOList, snapRefsSucceedingCurrentBackup, deletedDeltas, currentBackup);
} }
/** /**
@ -1064,14 +1034,14 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
backupCompressionCoroutines}; backupCompressionCoroutines};
} }
protected Outcome<?> createBackupThroughJobQueue(VirtualMachine vm, boolean quiesceVm, boolean isolated, Long backupScheduleId) { protected Outcome<?> createBackupThroughJobQueue(VirtualMachine vm, boolean quiesceVm, boolean isolated) {
final CallContext context = CallContext.current(); final CallContext context = CallContext.current();
long userId = context.getCallingUser().getId(); long userId = context.getCallingUser().getId();
long accountId = context.getCallingAccount().getAccountId(); long accountId = context.getCallingAccount().getAccountId();
long vmId = vm.getId(); long vmId = vm.getId();
BackupVO backup = new BackupVO(String.format("%s-%s", vm.getHostName(), DateUtil.getDateInSystemTimeZone()), vmId, vm.getBackupOfferingId(), accountId, BackupVO backup = new BackupVO(String.format("%s-%s", vm.getHostName(), DateUtil.getDateInSystemTimeZone()), vmId, vm.getBackupOfferingId(), accountId,
vm.getDomainId(), vm.getDataCenterId(), 0, Backup.Status.Queued, backupScheduleId, vm.getDomainId(), vm.getDataCenterId(), 0, Backup.Status.Queued, null,
Backup.CompressionStatus.Uncompressed, Backup.ValidationStatus.NotValidated); Backup.CompressionStatus.Uncompressed, Backup.ValidationStatus.NotValidated);
VmWorkJobVO workJob = new VmWorkJobVO(AsyncJobExecutionContext.getOriginJobId(), userId, accountId, VmWorkTakeBackup.class.getName(), vmId, VirtualMachine.Type.Instance, VmWorkJobVO workJob = new VmWorkJobVO(AsyncJobExecutionContext.getOriginJobId(), userId, accountId, VmWorkTakeBackup.class.getName(), vmId, VirtualMachine.Type.Instance,
@ -1269,14 +1239,16 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
if (!getValidationEndChainOnFail(backupVO)) { if (!getValidationEndChainOnFail(backupVO)) {
return; return;
} }
VirtualMachine vm = userVmDao.findByIdIncludingRemoved(backupVO.getVmId());
validateVmState(vm, "end backup chain", VirtualMachine.State.Expunging, VirtualMachine.State.Destroyed);
List<InternalBackupJoinVO> backupChildren = getBackupJoinChildren(backupVO); List<InternalBackupJoinVO> backupChildren = getBackupJoinChildren(backupVO);
// Get updated record // Get updated record
InternalBackupJoinVO backupJoinVO = internalBackupJoinDao.findById(backupVO.getId()); InternalBackupJoinVO backupJoinVO = internalBackupJoinDao.findById(backupVO.getId());
if (backupJoinVO.getCurrent() || (!backupChildren.isEmpty() && backupChildren.get(backupChildren.size() - 1).getCurrent())) { if (backupJoinVO.getCurrent() || (!backupChildren.isEmpty() && backupChildren.get(backupChildren.size() - 1).getCurrent())) {
logger.info("As [{}] is true, we are ending the backup chain of schedule [{}] for VM [{}]. The next backup will be a full backup.", logger.info("As [{}] is true, we are ending the backup chain for VM [{}]. The next backup will be a full backup.",
backupVO.getBackupScheduleId(), BackupValidationServiceJobController.backupValidationEndChainOnFail.toString()); BackupValidationServiceJobController.backupValidationEndChainOnFail.toString());
endBackupChain(userVmDao.findById(backupVO.getVmId()), backupVO.getBackupScheduleId()); endBackupChain(vm);
} }
} }
@ -1292,20 +1264,12 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
boolean runningVM = detail == null || VirtualMachine.State.valueOf(detail.getValue()) == VirtualMachine.State.Running; boolean runningVM = detail == null || VirtualMachine.State.valueOf(detail.getValue()) == VirtualMachine.State.Running;
BackupVO backupVO = backupDao.findLatestByStatusAndVmId(Backup.Status.Error, userVmVO.getId()); BackupVO backupVO = backupDao.findLatestByStatusAndVmId(Backup.Status.Error, userVmVO.getId());
InternalBackupJoinVO currentOnThisChain = internalBackupJoinDao.findCurrent(userVmVO.getId(), backupVO.getBackupScheduleId()); InternalBackupJoinVO internalBackupJoinVO = internalBackupJoinDao.findById(backupVO.getId());
InternalBackupJoinVO errorBackup = internalBackupJoinDao.findById(backupVO.getId()); ImageStoreVO imageStoreVO = imageStoreDao.findById(internalBackupJoinVO.getImageStoreId());
boolean errorOnBackupCreation = currentOnThisChain == null || currentOnThisChain.getId() != errorBackup.getId();
List<InternalBackupJoinVO> succeedingBackupList = getSucceedingBackupList(currentOnThisChain);
List<VMSnapshotVO> succeedingVmSnapshotList = getSucceedingVmSnapshotList(currentOnThisChain);
List<VolumeObjectTO> volumeTOs = vmSnapshotHelper.getVolumeTOList(userVmVO.getId());
Map<Long, LinkedList<String>> volumeToDeltasAfterCurrent = mapVolumesToVmSnapshotAndBackupReferences(volumeTOs, succeedingVmSnapshotList, succeedingBackupList);
List<KbossTO> kbossTOS = new ArrayList<>(); List<KbossTO> kbossTOS = new ArrayList<>();
List<InternalBackupStoragePoolVO> deltasOnPrimary = internalBackupStoragePoolDao.listByBackupId(errorBackup.getId()); List<InternalBackupStoragePoolVO> deltasOnPrimary = internalBackupStoragePoolDao.listByBackupId(internalBackupJoinVO.getId());
InternalBackupJoinVO parent = internalBackupJoinDao.findById(errorBackup.getParentId()); InternalBackupJoinVO parent = internalBackupJoinDao.findById(internalBackupJoinVO.getParentId());
// There is a possibility that the cleanup step of the backup creation was executed, and thus we would have to merge with the old parent's parent // There is a possibility that the cleanup step of the backup creation was executed, and thus we would have to merge with the old parent's parent
List<InternalBackupStoragePoolVO> parentDeltasOnPrimary = new ArrayList<>(); List<InternalBackupStoragePoolVO> parentDeltasOnPrimary = new ArrayList<>();
@ -1313,11 +1277,9 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
parentDeltasOnPrimary = internalBackupStoragePoolDao.listByBackupId(parent.getId()); parentDeltasOnPrimary = internalBackupStoragePoolDao.listByBackupId(parent.getId());
} }
List<InternalBackupDataStoreVO> deltasOnSecondary = internalBackupDataStoreDao.listByBackupId(errorBackup.getId()); List<InternalBackupDataStoreVO> deltasOnSecondary = internalBackupDataStoreDao.listByBackupId(internalBackupJoinVO.getId());
ImageStoreVO imageStoreVO = imageStoreDao.findById(errorBackup.getImageStoreId()); configureKbossTosForCleanup(userVmVO, deltasOnPrimary, deltasOnSecondary, runningVM, parentDeltasOnPrimary, kbossTOS);
configureKbossTosForCleanup(userVmVO, deltasOnPrimary, volumeToDeltasAfterCurrent, deltasOnSecondary, parentDeltasOnPrimary, kbossTOS, errorOnBackupCreation); CleanupKbossBackupErrorCommand command = new CleanupKbossBackupErrorCommand(runningVM, userVmVO.getInstanceName(), imageStoreVO.getUrl(), kbossTOS);
CleanupKbossBackupErrorCommand command = new CleanupKbossBackupErrorCommand(runningVM, errorOnBackupCreation, errorBackup.getEndOfChain(), succeedingBackupList.isEmpty(),
userVmVO.getInstanceName(), imageStoreVO.getUrl(), kbossTOS);
long hostId = userVmVO.getHostId() != null ? userVmVO.getHostId() : vmSnapshotHelper.pickRunningHost(userVmVO.getId()); long hostId = userVmVO.getHostId() != null ? userVmVO.getHostId() : vmSnapshotHelper.pickRunningHost(userVmVO.getId());
Answer answer = sendBackupCommand(hostId, command); Answer answer = sendBackupCommand(hostId, command);
@ -1326,40 +1288,32 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
return false; return false;
} }
boolean chainAlreadyEnded = processCleanupBackupErrorAnswer(userVmVO, answer, errorBackup, currentOnThisChain, succeedingBackupList); boolean chainAlreadyEnded = processCleanupBackupErrorAnswer(userVmVO, answer);
if (!chainAlreadyEnded) { if (!chainAlreadyEnded) {
mergeCurrentBackupDeltas(errorBackup); return endBackupChain(userVmVO);
} }
InternalBackupJoinVO current = internalBackupJoinDao.findCurrent(userVmVO.getId());
internalBackupStoragePoolDao.expungeByBackupId(current.getId());
setEndOfChainAndRemoveCurrentForBackup(current);
if (currentOnThisChain != null) { return true;
internalBackupStoragePoolDao.expungeByBackupId(currentOnThisChain.getId());
setEndOfChainAndRemoveCurrentForBackup(currentOnThisChain);
}
return finishBackupChains(userVmVO);
} }
protected boolean processCleanupBackupErrorAnswer(UserVmVO userVmVO, Answer answer, InternalBackupJoinVO errorBackup, InternalBackupJoinVO currentBackup, protected boolean processCleanupBackupErrorAnswer(UserVmVO userVmVO, Answer answer) {
List<InternalBackupJoinVO> succeedingBackups) {
boolean runningVM; boolean runningVM;
CleanupKbossBackupErrorAnswer cleanAnswer = (CleanupKbossBackupErrorAnswer) answer; CleanupKbossBackupErrorAnswer cleanAnswer = (CleanupKbossBackupErrorAnswer) answer;
logger.info("Successfully finished chain for VM [{}] and normalizing the BackupError state. Cleaning up metadata.", userVmVO.getUuid()); logger.info("Successfully finished chain for VM [{}] and normalizing the BackupError state. Cleaning up metadata.", userVmVO.getUuid());
boolean chainAlreadyEnded = true; boolean chainAlreadyEnded = false;
for (Map.Entry<String, Pair<String,Boolean>> entry : cleanAnswer.getVolumeIdToPathAndChainEnded().entrySet()) { for (VolumeObjectTO volumeObjectTO : cleanAnswer.getVolumeObjectTos()) {
VolumeVO volumeVO = volumeDao.findByUuid(entry.getKey()); VolumeVO volumeVO = volumeDao.findById(volumeObjectTO.getId());
if (!entry.getValue().first().equals(volumeVO.getPath())) { if (!volumeObjectTO.getPath().equals(volumeVO.getPath())) {
volumeVO.setPath(entry.getValue().first()); volumeVO.setPath(volumeObjectTO.getPath());
volumeDao.update(volumeVO.getId(), volumeVO); volumeDao.update(volumeVO.getId(), volumeVO);
if (!entry.getValue().second()) { chainAlreadyEnded = true;
chainAlreadyEnded = false;
continue;
}
internalBackupStoragePoolDao.expungeByVolumeIdAndBackupId(volumeVO.getId(), errorBackup.getId());
} }
} }
updateSucceedingBackupIfNeeded(currentBackup, succeedingBackups);
runningVM = cleanAnswer.isVmRunning(); runningVM = cleanAnswer.isVmRunning();
userVmVO.setState(runningVM ? VirtualMachine.State.Running : VirtualMachine.State.Stopped); userVmVO.setState(runningVM ? VirtualMachine.State.Running : VirtualMachine.State.Stopped);
@ -1368,20 +1322,6 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
return chainAlreadyEnded; return chainAlreadyEnded;
} }
private void updateSucceedingBackupIfNeeded(InternalBackupJoinVO currentBackup, List<InternalBackupJoinVO> succeedingBackups) {
if (currentBackup == null || succeedingBackups.isEmpty()) {
return;
}
InternalBackupJoinVO succeedingBackup = succeedingBackups.get(0);
for (InternalBackupStoragePoolVO deltaRef : internalBackupStoragePoolDao.listByBackupId(currentBackup.getId())) {
InternalBackupStoragePoolVO succeedingDelta = internalBackupStoragePoolDao.findOneByVolumeIdAndBackupId(deltaRef.getVolumeId(), succeedingBackup.getId());
if (succeedingDelta != null) {
succeedingDelta.setBackupDeltaParentPath(deltaRef.getBackupDeltaParentPath());
internalBackupStoragePoolDao.update(succeedingDelta.getId(), succeedingDelta);
}
}
}
protected void calculateAndSaveHash(Set<Pair<BackupDeltaTO, VolumeObjectTO>> backupDeltaAndVolumePairs, BackupVO backupVO, long hostId) { protected void calculateAndSaveHash(Set<Pair<BackupDeltaTO, VolumeObjectTO>> backupDeltaAndVolumePairs, BackupVO backupVO, long hostId) {
TakeBackupHashCommand cmd = new TakeBackupHashCommand(backupDeltaAndVolumePairs.stream().map(Pair::first).collect(Collectors.toList()), backupVO.getUuid()); TakeBackupHashCommand cmd = new TakeBackupHashCommand(backupDeltaAndVolumePairs.stream().map(Pair::first).collect(Collectors.toList()), backupVO.getUuid());
Answer answer = sendBackupCommand(hostId, cmd); Answer answer = sendBackupCommand(hostId, cmd);
@ -1585,10 +1525,51 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
return true; return true;
} }
/**
* Merges the current delta on primary storage, if any, into the given volume. If the backup has no more deltas on primary storage, will set the backup as end_of_chain.
* */
protected void mergeCurrentDeltaIntoVolume(Volume volume, VirtualMachine virtualMachine, String operation, boolean isVmRunning) {
InternalBackupStoragePoolVO delta = internalBackupStoragePoolDao.findOneByVolumeId(volume.getId());
if (delta == null) {
logger.debug("Volume [{}] has no deltas to merge, doing nothing.", volume.getUuid());
return;
}
InternalBackupJoinVO internalBackupJoinVO = internalBackupJoinDao.findById(delta.getBackupId());
VMSnapshotVO succeedingVmSnapshotVO = getSucceedingVmSnapshot(internalBackupJoinVO);
DataStore store = dataStoreManager.getDataStore(volume.getPoolId(), DataStoreRole.Primary);
VolumeObject volumeObject = VolumeObject.getVolumeObject(store, (VolumeVO)volume);
DeltaMergeTreeTO deltaMergeTreeTO = createDeltaMergeTree(succeedingVmSnapshotVO == null, isVmRunning, delta, (VolumeObjectTO)volumeObject.getTO(), succeedingVmSnapshotVO);
MergeDiskOnlyVmSnapshotCommand cmd = new MergeDiskOnlyVmSnapshotCommand(List.of(deltaMergeTreeTO), isVmRunning, virtualMachine.getInstanceName());
Answer answer = sendBackupCommand(vmSnapshotHelper.pickRunningHost(virtualMachine.getId()), cmd);
if (answer == null || !answer.getResult()) {
logger.error("Error while trying to prepare volume [{}] for {}. Got [{}] as answer from host.", volume.getUuid(), operation, answer != null ? answer.getDetails() : null);
throw new CloudRuntimeException(String.format("Unable to prepare volume [%s] for [%s].", volume.getUuid(), operation));
}
if (succeedingVmSnapshotVO == null) {
VolumeVO volumeVO = volumeDao.findById(volumeObject.getId());
volumeVO.setPath(deltaMergeTreeTO.getParent().getPath());
volumeDao.update(volumeVO.getId(), volumeVO);
}
expungeOldDeltasAndUpdateVmSnapshotIfNeeded(List.of(delta), succeedingVmSnapshotVO);
List<InternalBackupStoragePoolVO> backupDeltas = internalBackupStoragePoolDao.listByBackupId(delta.getBackupId());
if (backupDeltas.isEmpty()) {
logger.debug("Backup [{}] has no more deltas on primary storage due to prepare volume [{}] for {} operation. Will set it as end of chain and not current.",
internalBackupJoinVO.getUuid(), volume.getUuid(), operation);
setEndOfChainAndRemoveCurrentForBackup(internalBackupJoinVO);
}
}
/** /**
* Creates the necessary delta references on both primary and secondary storage. Also maps the volume to the parent delta backup and create the delta merge tree. * Creates the necessary delta references on both primary and secondary storage. Also maps the volume to the parent delta backup and create the delta merge tree.
* */ * */
protected void createDeltaReferences(boolean fullBackup, boolean runningVm, Backup backup, protected void createDeltaReferences(boolean fullBackup, boolean hasVmSnapshotSucceedingLastBackup, boolean runningVm, Backup backup,
List<InternalBackupDataStoreVO> parentBackupDeltasOnSecondary, List<InternalBackupStoragePoolVO> parentBackupDeltasOnPrimary, List<InternalBackupDataStoreVO> parentBackupDeltasOnSecondary, List<InternalBackupStoragePoolVO> parentBackupDeltasOnPrimary,
HashMap<String, InternalBackupStoragePoolVO> volumeUuidToDeltaPrimaryRef, HashMap<String, InternalBackupDataStoreVO> volumeUuidToDeltaSecondaryRef, HashMap<String, InternalBackupStoragePoolVO> volumeUuidToDeltaPrimaryRef, HashMap<String, InternalBackupDataStoreVO> volumeUuidToDeltaSecondaryRef,
VMSnapshotVO succeedingVmSnapshot, KbossTO kbossTO) { VMSnapshotVO succeedingVmSnapshot, KbossTO kbossTO) {
@ -1603,8 +1584,7 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
InternalBackupDataStoreVO deltaSecondaryRef = new InternalBackupDataStoreVO(backup.getId(), volumeObjectTO.getVolumeId(), volumeObjectTO.getDeviceId(), relativePathOnSecondary); InternalBackupDataStoreVO deltaSecondaryRef = new InternalBackupDataStoreVO(backup.getId(), volumeObjectTO.getVolumeId(), volumeObjectTO.getDeviceId(), relativePathOnSecondary);
if (!fullBackup) { if (!fullBackup) {
InternalBackupStoragePoolVO parentDeltaOnPrimary = createDeltaMergeTreeForVolume(false, runningVm, parentBackupDeltasOnPrimary, succeedingVmSnapshot, kbossTO, InternalBackupStoragePoolVO parentDeltaOnPrimary = createDeltaMergeTreeForVolume(false, runningVm, parentBackupDeltasOnPrimary, succeedingVmSnapshot, kbossTO);
new ArrayList<>());
findAndSetParentBackupPath(parentBackupDeltasOnSecondary, parentDeltaOnPrimary, kbossTO); findAndSetParentBackupPath(parentBackupDeltasOnSecondary, parentDeltaOnPrimary, kbossTO);
} }
@ -1615,8 +1595,10 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
InternalBackupStoragePoolVO deltaPrimaryRef = new InternalBackupStoragePoolVO(backup.getId(), volumeObjectTO.getPoolId(), volumeObjectTO.getVolumeId(), filename, InternalBackupStoragePoolVO deltaPrimaryRef = new InternalBackupStoragePoolVO(backup.getId(), volumeObjectTO.getPoolId(), volumeObjectTO.getVolumeId(), filename,
volumeObjectTO.getPath()); volumeObjectTO.getPath());
if (kbossTO.getDeltaMergeTreeTO() != null && CollectionUtils.isEmpty(kbossTO.getDeltaPaths())) { if (kbossTO.getDeltaMergeTreeTO() != null && !hasVmSnapshotSucceedingLastBackup) {
deltaPrimaryRef.setBackupDeltaParentPath(kbossTO.getDeltaMergeTreeTO().getParent().getPath()); deltaPrimaryRef.setBackupDeltaParentPath(kbossTO.getDeltaMergeTreeTO().getParent().getPath());
} else if (hasVmSnapshotSucceedingLastBackup) {
deltaPrimaryRef.setBackupDeltaParentPath(volumeObjectTO.getPath());
} }
InternalBackupStoragePoolVO referenceOnPrimary = internalBackupStoragePoolDao.persist(deltaPrimaryRef); InternalBackupStoragePoolVO referenceOnPrimary = internalBackupStoragePoolDao.persist(deltaPrimaryRef);
@ -1624,49 +1606,6 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
volumeUuidToDeltaPrimaryRef.put(volumeObjectTO.getUuid(), referenceOnPrimary); volumeUuidToDeltaPrimaryRef.put(volumeObjectTO.getUuid(), referenceOnPrimary);
} }
/**
* Merges the current delta on primary storage, if any, into the given volume. If the backup has no more deltas on primary storage, will set the backup as end_of_chain.
* */
protected void mergeCurrentDeltasIntoVolume(Volume volume, VirtualMachine virtualMachine, String operation, boolean isVmRunning) {
List<InternalBackupJoinVO> currents = internalBackupJoinDao.listCurrentsByVolumeIdDesc(volume.getId());
if (currents.isEmpty()) {
logger.debug("Volume [{}] has no deltas to merge, doing nothing.", volume.getUuid());
return;
}
for (InternalBackupJoinVO current : currents) {
InternalBackupStoragePoolVO delta = internalBackupStoragePoolDao.findOneByVolumeIdAndBackupId(volume.getId(), current.getId());
DataStore store = dataStoreManager.getDataStore(volume.getPoolId(), DataStoreRole.Primary);
VolumeObject volumeObject = VolumeObject.getVolumeObject(store, (VolumeVO)volume);
DeltaMergeTreeTO deltaMergeTreeTO = createDeltaMergeTree(true, isVmRunning, delta, (VolumeObjectTO)volumeObject.getTO(), null, new ArrayList<>());
MergeDiskOnlyVmSnapshotCommand cmd = new MergeDiskOnlyVmSnapshotCommand(List.of(deltaMergeTreeTO), isVmRunning, virtualMachine.getInstanceName());
Answer answer = sendBackupCommand(vmSnapshotHelper.pickRunningHost(virtualMachine.getId()), cmd);
if (answer == null || !answer.getResult()) {
logger.error("Error while trying to prepare volume [{}] for {}. Got [{}] as answer from host.", volume.getUuid(), operation, answer != null ? answer.getDetails() : null);
throw new CloudRuntimeException(String.format("Unable to prepare volume [%s] for [%s].", volume.getUuid(), operation));
}
VolumeVO volumeVO = volumeDao.findById(volumeObject.getId());
volumeVO.setPath(deltaMergeTreeTO.getParent().getPath());
volumeDao.update(volumeVO.getId(), volumeVO);
volume = volumeVO;
List<InternalBackupStoragePoolVO> deltaOnPrimary = List.of(delta);
expungeOldDeltasAndUpdateVmSnapshotOrBackup(deltaOnPrimary, null, null);
List<InternalBackupStoragePoolVO> backupDeltas = internalBackupStoragePoolDao.listByBackupId(delta.getBackupId());
if (backupDeltas.isEmpty()) {
logger.debug("Backup [{}] has no more deltas on primary storage due to prepare volume [{}] for {} operation. Will set it as end of chain and not current.",
current.getUuid(), volume.getUuid(), operation);
setEndOfChainAndRemoveCurrentForBackup(current);
}
}
}
protected HostVO getHostToRestore(VirtualMachine vm, boolean quickRestore, Long hostId) throws AgentUnavailableException { protected HostVO getHostToRestore(VirtualMachine vm, boolean quickRestore, Long hostId) throws AgentUnavailableException {
HostVO host; HostVO host;
if (quickRestore) { if (quickRestore) {
@ -1732,59 +1671,54 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
} }
/** /**
* Returns ordered list of backups taken after the last backup. The list is ordered from oldest to newest. * Given a VM snapshot, returns a map of volume id to list of snapshot references of the children of the VM snapshot.
* */ * */
protected List<InternalBackupJoinVO> getSucceedingBackupList(InternalBackupJoinVO backup) { protected Map<Long, List<SnapshotDataStoreVO>> gatherSnapshotReferencesOfChildrenSnapshot(List<VolumeObjectTO> volumeObjectTOs, VMSnapshot vmSnapshotVO) {
List<InternalBackupJoinVO> internalBackupJoinVOS = new ArrayList<>(); Map<Long, List<SnapshotDataStoreVO>> volumeToSnapshotRefs = new HashMap<>();
if (backup == null) { if (vmSnapshotVO == null) {
return internalBackupJoinVOS; return volumeToSnapshotRefs;
} }
List<InternalBackupJoinVO> currentBackups = internalBackupJoinDao.listCurrents(backup.getVmId(), false); List<VMSnapshotVO> snapshotChildren = vmSnapshotDao.listByParent(vmSnapshotVO.getId());
if (currentBackups.isEmpty()) { if (CollectionUtils.isEmpty(snapshotChildren)) {
return internalBackupJoinVOS; return volumeToSnapshotRefs;
} }
internalBackupJoinVOS = currentBackups.stream().filter(internalBackupJoinVO -> internalBackupJoinVO.getDate().after(backup.getDate())).collect(Collectors.toList()); List<SnapshotDataStoreVO> snapshotDataStoreVOS = new ArrayList<>();
logger.debug("Found the following backups that succeed the backup [{}]: [{}].", backup.getUuid(), internalBackupJoinVOS); snapshotChildren.stream()
.map(snapshotVo -> vmSnapshotHelper.getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(snapshotVo.getId()))
.forEach(snapshotDataStoreVOS::addAll);
mapVolumesToSnapshotReferences(volumeObjectTOs, snapshotDataStoreVOS, volumeToSnapshotRefs);
if (logger.isDebugEnabled()) {
StringBuilder log = new StringBuilder(String.format("Found the following snapshot references that succeed the VM snapshot [%s].", vmSnapshotVO.getUuid()));
for (VolumeObjectTO volumeObjectTO : volumeObjectTOs) {
log.append(String.format(" Volume [%s]; Snapshot references [%s].", volumeObjectTO.getUuid(), volumeToSnapshotRefs.get(volumeObjectTO.getId())));
}
logger.debug(log.toString());
}
return internalBackupJoinVOS; return volumeToSnapshotRefs;
} }
/** /**
* Given a list of volumes and VM snapshots/backups, maps the volumes to the delta references of the VM snapshots/backups. * Given a list of volumes and VM snapshots, maps the volumes to the snapshot references of the VM snapshots.
* */ * */
protected Map<Long, LinkedList<String>> mapVolumesToVmSnapshotAndBackupReferences(List<VolumeObjectTO> volumeObjectTOs, List<VMSnapshotVO> vmSnapshotVOList, List<InternalBackupJoinVO> internalBackupJoinVOList) { protected Map<Long, List<SnapshotDataStoreVO>> mapVolumesToVmSnapshotReferences(List<VolumeObjectTO> volumeObjectTOs, List<VMSnapshotVO> vmSnapshotVOList) {
Map<Long, LinkedList<String>> volumeToSnapshotAndBackupRefs = new HashMap<>(); Map<Long, List<SnapshotDataStoreVO>> volumeToSnapshotRefs = new HashMap<>();
if (vmSnapshotVOList.isEmpty() && internalBackupJoinVOList.isEmpty()) { if (vmSnapshotVOList.isEmpty()) {
logger.trace("No VM snapshot nor backup to map to any volume, returning."); logger.trace("No VM snapshot to map to any volume, returning.");
return volumeToSnapshotAndBackupRefs; return volumeToSnapshotRefs;
}
List<Ternary<Long, String, Date>> volumeIdAndResourcePathAndCreatedDateList = new ArrayList<>();
for (InternalBackupJoinVO internalBackupJoinVO : internalBackupJoinVOList) {
volumeIdAndResourcePathAndCreatedDateList.add(new Ternary<>(internalBackupJoinVO.getVolumeId(), internalBackupJoinVO.getStoragePoolDeltaPath(), internalBackupJoinVO.getDate()));
} }
ArrayList<SnapshotDataStoreVO> allRefs = new ArrayList<>();
for (VMSnapshotVO vmSnapshotVO : vmSnapshotVOList) { for (VMSnapshotVO vmSnapshotVO : vmSnapshotVOList) {
vmSnapshotHelper.getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(vmSnapshotVO.getId()) allRefs.addAll(vmSnapshotHelper.getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(vmSnapshotVO.getId()));
.forEach(snapshotDataStoreVO -> volumeIdAndResourcePathAndCreatedDateList.add(new Ternary<>(snapshotDataStoreVO.getVolumeId(), snapshotDataStoreVO.getInstallPath(), snapshotDataStoreVO.getCreated())));
} }
mapVolumesToSnapshotReferences(volumeObjectTOs, allRefs, volumeToSnapshotRefs);
volumeIdAndResourcePathAndCreatedDateList.sort(Comparator.comparing(Ternary::third)); logger.trace("Given volume objects [{}] and VM snapshots [{}], created the following map [{}].", volumeObjectTOs, vmSnapshotVOList, volumeToSnapshotRefs);
return volumeToSnapshotRefs;
for (Ternary<Long, String, Date> volumeIdAndResourcePathAndCreatedDate : volumeIdAndResourcePathAndCreatedDateList) {
long volumeId = volumeIdAndResourcePathAndCreatedDate.first();
String resourcePath = volumeIdAndResourcePathAndCreatedDate.second();
volumeToSnapshotAndBackupRefs.computeIfAbsent(volumeId, k -> new LinkedList<>()).addLast(resourcePath);
}
logger.trace("Given volume objects [{}], VM snapshots [{}] and backups [{}], created the following map [{}].", volumeObjectTOs, vmSnapshotVOList, internalBackupJoinVOList, volumeToSnapshotAndBackupRefs);
return volumeToSnapshotAndBackupRefs;
} }
protected void mapVolumesToSnapshotReferences(List<VolumeObjectTO> volumeObjectTOs, List<SnapshotDataStoreVO> snapshotDataStoreVOS, Map<Long, List<SnapshotDataStoreVO>> volumeToSnapshotRefs) { protected void mapVolumesToSnapshotReferences(List<VolumeObjectTO> volumeObjectTOs, List<SnapshotDataStoreVO> snapshotDataStoreVOS, Map<Long, List<SnapshotDataStoreVO>> volumeToSnapshotRefs) {
for (VolumeObjectTO volumeObjectTO : volumeObjectTOs) { for (VolumeObjectTO volumeObjectTO : volumeObjectTOs) {
List<SnapshotDataStoreVO> associatedSnapshots = snapshotDataStoreVOS.stream() List<SnapshotDataStoreVO> associatedSnapshots = snapshotDataStoreVOS.stream()
@ -1827,68 +1761,30 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
} }
/** /**
* Expunge the old backup deltas and if there were disk-only VM snapshot or backup deltas after the last backup, update their paths. * Expunge the old backup deltas and if there were disk-only VM snapshot deltas after the last backup, update their paths.
* */ * */
protected void expungeOldDeltasAndUpdateVmSnapshotOrBackupIfNeeded(List<InternalBackupStoragePoolVO> oldDeltasOnPrimary, VMSnapshot vmSnapshot, protected void expungeOldDeltasAndUpdateVmSnapshotIfNeeded(List<InternalBackupStoragePoolVO> oldDeltasOnPrimary, VMSnapshot vmSnapshot) {
InternalBackupJoinVO lastBackup) {
List<SnapshotDataStoreVO> snapshotRefs = vmSnapshot == null ? List.of() : vmSnapshotHelper.getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(vmSnapshot.getId()); List<SnapshotDataStoreVO> snapshotRefs = vmSnapshot == null ? List.of() : vmSnapshotHelper.getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(vmSnapshot.getId());
List<InternalBackupStoragePoolVO> newBackupDeltas = new ArrayList<>();
Map<Long, InternalBackupStoragePoolVO> volumeIdNewBackupDeltaMap = new HashMap<>();
if (lastBackup != null) {
newBackupDeltas = internalBackupStoragePoolDao.listByBackupId(lastBackup.getId());
volumeIdNewBackupDeltaMap = newBackupDeltas.stream().collect(Collectors.toMap(InternalBackupStoragePoolVO::getVolumeId, nbsp -> nbsp));
}
for (InternalBackupStoragePoolVO oldBackupDelta : oldDeltasOnPrimary) { for (InternalBackupStoragePoolVO oldBackupDelta : oldDeltasOnPrimary) {
logger.trace("Expunging old backup delta [{}].", oldBackupDelta); logger.trace("Expunging old backup delta [{}].", oldBackupDelta);
internalBackupStoragePoolDao.expunge(oldBackupDelta.getId()); internalBackupStoragePoolDao.expunge(oldBackupDelta.getId());
SnapshotDataStoreVO snapshotDataStoreVO = snapshotRefs.stream().filter(ref -> ref.getVolumeId() == oldBackupDelta.getVolumeId()).findFirst().orElse(null); SnapshotDataStoreVO snapshotDataStoreVO = snapshotRefs.stream().filter(ref -> ref.getVolumeId() == oldBackupDelta.getVolumeId()).findFirst().orElse(null);
if (snapshotDataStoreVO != null) { if (snapshotDataStoreVO == null) {
snapshotDataStoreVO.setInstallPath(oldBackupDelta.getBackupDeltaParentPath());
logger.debug("Updating snapshot delta [{}] path to [{}].", snapshotDataStoreVO.getId(), oldBackupDelta.getBackupDeltaParentPath());
snapshotDataStoreDao.update(snapshotDataStoreVO.getId(), snapshotDataStoreVO);
continue; continue;
} }
if (lastBackup != null) { snapshotDataStoreVO.setInstallPath(oldBackupDelta.getBackupDeltaParentPath());
InternalBackupStoragePoolVO newBackupDelta = volumeIdNewBackupDeltaMap.get(oldBackupDelta.getVolumeId()); logger.debug("Updating snapshot delta [{}] path to [{}].", snapshotDataStoreVO.getId(), oldBackupDelta.getBackupDeltaParentPath());
newBackupDelta.setBackupDeltaParentPath(oldBackupDelta.getBackupDeltaParentPath()); snapshotDataStoreDao.update(snapshotDataStoreVO.getId(), snapshotDataStoreVO);
logger.debug("Updating backup delta [{}] path to [{}].", newBackupDelta.getId(), oldBackupDelta.getBackupDeltaParentPath());
internalBackupStoragePoolDao.update(newBackupDelta.getId(), newBackupDelta);
}
} }
} }
/**
* Expunges old deltas on primary storage and updates the metadata for either
* the succeeding VM snapshot or the succeeding backup based on their chronological order.
* If only one (or neither) is provided, it proceeds with the available entities.
*
* @param oldDeltasOnPrimary The list of delta references on the primary storage to be removed;
* @param succeedingVmSnapshotVO The VM snapshot that follows the deltas being expunged;
* @param succeedingBackup The backup entity that follows the deltas being expunged.
*/
protected void expungeOldDeltasAndUpdateVmSnapshotOrBackup(List<InternalBackupStoragePoolVO> oldDeltasOnPrimary, VMSnapshot succeedingVmSnapshotVO,
InternalBackupJoinVO succeedingBackup) {
if (ObjectUtils.allNotNull(succeedingVmSnapshotVO, succeedingBackup)) {
if (succeedingVmSnapshotVO.getCreated().before(succeedingBackup.getDate())) {
expungeOldDeltasAndUpdateVmSnapshotOrBackupIfNeeded(oldDeltasOnPrimary, succeedingVmSnapshotVO, null);
} else {
expungeOldDeltasAndUpdateVmSnapshotOrBackupIfNeeded(oldDeltasOnPrimary, null, succeedingBackup);
}
} else {
expungeOldDeltasAndUpdateVmSnapshotOrBackupIfNeeded(oldDeltasOnPrimary, succeedingVmSnapshotVO, succeedingBackup);
}
}
/** /**
* Create a {@link DeltaMergeTreeTO} for the volume if it has a delta on primary and add it to the list. * Create a {@link DeltaMergeTreeTO} for the volume if it has a delta on primary and add it to the list.
* *
* @return the delta on primary of the volume. Null if no delta. * @return the delta on primary of the volume. Null if no delta.
* */ * */
protected InternalBackupStoragePoolVO createDeltaMergeTreeForVolume(boolean childIsVolume, boolean runningVm, List<InternalBackupStoragePoolVO> deltasOnPrimary, VMSnapshotVO succeedingVmSnapshot, protected InternalBackupStoragePoolVO createDeltaMergeTreeForVolume(boolean childIsVolume, boolean runningVm, List<InternalBackupStoragePoolVO> deltasOnPrimary, VMSnapshotVO succeedingVmSnapshot,
KbossTO kbossTO, List<InternalBackupJoinVO> succeedingBackupList) { KbossTO kbossTO) {
VolumeObjectTO volumeObjectTO = kbossTO.getVolumeObjectTO(); VolumeObjectTO volumeObjectTO = kbossTO.getVolumeObjectTO();
InternalBackupStoragePoolVO deltaOnPrimary = deltasOnPrimary.stream() InternalBackupStoragePoolVO deltaOnPrimary = deltasOnPrimary.stream()
@ -1902,12 +1798,12 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
logger.debug("Volume [{}] has a backup delta on primary storage [{}].", volumeObjectTO.getUuid(), deltaOnPrimary); logger.debug("Volume [{}] has a backup delta on primary storage [{}].", volumeObjectTO.getUuid(), deltaOnPrimary);
kbossTO.setDeltaMergeTreeTO(createDeltaMergeTree(childIsVolume, runningVm, deltaOnPrimary, volumeObjectTO, succeedingVmSnapshot, succeedingBackupList)); kbossTO.setDeltaMergeTreeTO(createDeltaMergeTree(childIsVolume, runningVm, deltaOnPrimary, volumeObjectTO, succeedingVmSnapshot));
return deltaOnPrimary; return deltaOnPrimary;
} }
protected DeltaMergeTreeTO createDeltaMergeTree(boolean childIsVolume, boolean runningVm, InternalBackupStoragePoolVO deltaOnPrimary, protected DeltaMergeTreeTO createDeltaMergeTree(boolean childIsVolume, boolean runningVm, InternalBackupStoragePoolVO deltaOnPrimary,
VolumeObjectTO volumeObjectTO, VMSnapshotVO succeedingVmSnapshot, List<InternalBackupJoinVO> succeedingBackupsList) { VolumeObjectTO volumeObjectTO, VMSnapshotVO succeedingVmSnapshot) {
DataStore store = dataStoreManager.getDataStore(deltaOnPrimary.getStoragePoolId(), DataStoreRole.Primary); DataStore store = dataStoreManager.getDataStore(deltaOnPrimary.getStoragePoolId(), DataStoreRole.Primary);
DataTO deltaChild; DataTO deltaChild;
if (childIsVolume) { if (childIsVolume) {
@ -1917,12 +1813,11 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
} }
BackupDeltaTO deltaParent = new BackupDeltaTO(store.getTO(), Hypervisor.HypervisorType.KVM, deltaOnPrimary.getBackupDeltaParentPath()); BackupDeltaTO deltaParent = new BackupDeltaTO(store.getTO(), Hypervisor.HypervisorType.KVM, deltaOnPrimary.getBackupDeltaParentPath());
List<VMSnapshotVO> succeedingSnapshotList = succeedingVmSnapshot != null ? vmSnapshotDao.listByParent(succeedingVmSnapshot.getId()) : new ArrayList<>();
List<String> succeedingDeltaPaths = new ArrayList<>(); List<String> succeedingDeltaPaths = new ArrayList<>();
if (succeedingVmSnapshot != null || CollectionUtils.isNotEmpty(succeedingBackupsList)) { if (succeedingVmSnapshot != null) {
succeedingDeltaPaths = mapVolumesToVmSnapshotAndBackupReferences(List.of(volumeObjectTO), succeedingSnapshotList, succeedingBackupsList) succeedingDeltaPaths = gatherSnapshotReferencesOfChildrenSnapshot(List.of(volumeObjectTO), succeedingVmSnapshot).getOrDefault(volumeObjectTO.getVolumeId(), List.of())
.getOrDefault(volumeObjectTO.getVolumeId(), new LinkedList<>()); .stream().map(SnapshotDataStoreVO::getInstallPath).collect(Collectors.toList());
if (!childIsVolume && !runningVm && succeedingDeltaPaths.isEmpty()) { if (!childIsVolume && !runningVm && succeedingDeltaPaths.isEmpty()) {
succeedingDeltaPaths = List.of(volumeObjectTO.getPath()); succeedingDeltaPaths = List.of(volumeObjectTO.getPath());
@ -2071,7 +1966,7 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
VolumeObjectTO volumeObjectTO = optional.get(); VolumeObjectTO volumeObjectTO = optional.get();
if (volumesNotPartOfTheBackupBeingRestored.contains(volumeObjectTO)) { if (volumesNotPartOfTheBackupBeingRestored.contains(volumeObjectTO)) {
deltasToBeMerged.add(createDeltaMergeTree(true, false, deltaOnPrimary, volumeObjectTO, null, new ArrayList<>())); deltasToBeMerged.add(createDeltaMergeTree(true, false, deltaOnPrimary, volumeObjectTO, null));
continue; continue;
} }
@ -2189,8 +2084,7 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
protected void processBackupSuccess(boolean runningVm, List<VolumeObjectTO> volumeTOs, HashMap<String, InternalBackupStoragePoolVO> volumeUuidToDeltaPrimaryRef, protected void processBackupSuccess(boolean runningVm, List<VolumeObjectTO> volumeTOs, HashMap<String, InternalBackupStoragePoolVO> volumeUuidToDeltaPrimaryRef,
HashMap<String, InternalBackupDataStoreVO> volumeUuidToDeltaSecondaryRef, TakeKbossBackupAnswer answer, List<InternalBackupStoragePoolVO> parentBackupDeltasOnPrimary, HashMap<String, InternalBackupDataStoreVO> volumeUuidToDeltaSecondaryRef, TakeKbossBackupAnswer answer, List<InternalBackupStoragePoolVO> parentBackupDeltasOnPrimary,
VMSnapshotVO succeedingVmSnapshot, BackupVO backupVO, boolean fullBackup, VirtualMachine userVm, Long hostId, boolean endChain, boolean isolated, List<VMSnapshotVO> succeedingVmSnapshots, BackupVO backupVO, boolean fullBackup, VirtualMachine userVm, Long hostId, boolean endChain, boolean isolated) {
InternalBackupJoinVO succeedingBackup) {
long physicalBackupSize = 0; long physicalBackupSize = 0;
logger.debug("Processing backup [{}] success.", backupVO.getUuid()); logger.debug("Processing backup [{}] success.", backupVO.getUuid());
for (VolumeObjectTO volumeObjectTO : volumeTOs) { for (VolumeObjectTO volumeObjectTO : volumeTOs) {
@ -2198,7 +2092,7 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
physicalBackupSize, endChain, isolated, backupVO); physicalBackupSize, endChain, isolated, backupVO);
} }
expungeOldDeltasAndUpdateVmSnapshotOrBackup(parentBackupDeltasOnPrimary, succeedingVmSnapshot, succeedingBackup); expungeOldDeltasAndUpdateVmSnapshotIfNeeded(parentBackupDeltasOnPrimary, succeedingVmSnapshots.isEmpty() ? null : succeedingVmSnapshots.get(0));
backupVO.setSize(physicalBackupSize); backupVO.setSize(physicalBackupSize);
backupVO.setStatus(Backup.Status.BackedUp); backupVO.setStatus(Backup.Status.BackedUp);
@ -2241,7 +2135,7 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
* For every backup, except for the one which the command was issued, will set them as Expunged regardless and hope operators will look * For every backup, except for the one which the command was issued, will set them as Expunged regardless and hope operators will look
* at the logs. For the current one, if forced=false, will set it as error, otherwise, will set it as Expunged as well. * at the logs. For the current one, if forced=false, will set it as error, otherwise, will set it as Expunged as well.
* */ * */
protected boolean processRemoveBackupFailures(boolean forced, Answer[] deleteAnswers, List<Long> removedBackupIds, InternalBackupJoinVO backupJoinVO, VirtualMachine vm) { protected boolean processRemoveBackupFailures(boolean forced, Answer[] deleteAnswers, List<Long> removedBackupIds, InternalBackupJoinVO backupJoinVO) {
List<Answer> failures = Arrays.stream(deleteAnswers).filter(answer -> !answer.getResult()).collect(Collectors.toList()); List<Answer> failures = Arrays.stream(deleteAnswers).filter(answer -> !answer.getResult()).collect(Collectors.toList());
Set<Long> failedToRemoveBackupIdSet = new HashSet<>(); Set<Long> failedToRemoveBackupIdSet = new HashSet<>();
if (CollectionUtils.isNotEmpty(failures)) { if (CollectionUtils.isNotEmpty(failures)) {
@ -2262,7 +2156,6 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
logger.info("Since backup delete command was not forced, will not set the main backup [{}] as Expunged, will set it as error instead.", failedVO.getUuid()); logger.info("Since backup delete command was not forced, will not set the main backup [{}] as Expunged, will set it as error instead.", failedVO.getUuid());
failedVO.setStatus(Backup.Status.Error); failedVO.setStatus(Backup.Status.Error);
backupDao.update(failedVO.getId(), failedVO); backupDao.update(failedVO.getId(), failedVO);
vmInstanceDetailsDao.addDetail(vm.getId(), VmDetailConstants.LAST_KNOWN_STATE, vm.getState().name(), false);
} }
for (Long failedToRemove : failedToRemoveBackupIdSet) { for (Long failedToRemove : failedToRemoveBackupIdSet) {
@ -2369,41 +2262,17 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
} else if (jobResult instanceof BackupProviderException) { } else if (jobResult instanceof BackupProviderException) {
throw (BackupProviderException) jobResult; throw (BackupProviderException) jobResult;
} }
throw new CloudRuntimeException(String.format("Exception while restoring KVM internal incremental backup [%s]. Check the logs for more information.", backup.getUuid()), ((Throwable)jobResult).getCause()); throw new CloudRuntimeException(String.format("Exception while restoring KVM internal incremental backup [%s]. Check the logs for more information.", backup.getUuid()),
((Throwable)jobResult).getCause());
} }
protected boolean finishAllChains(VirtualMachine vm, List<InternalBackupJoinVO> currents) { protected boolean endBackupChain(VirtualMachine vm) {
if (currents.isEmpty()) { InternalBackupJoinVO current = internalBackupJoinDao.findCurrent(vm.getId());
logger.debug("There is no current active chain, no need to do anything.");
return true;
}
for (InternalBackupJoinVO current : currents) {
if (!mergeCurrentBackupDeltas(current)) {
UserVmVO vmVO = userVmDao.findById(vm.getId());
logger.error("Failed to merge deltas for VM [{}] during backup offering removal process. Changing its state to [{}].", vm, VirtualMachine.State.BackupError);
BackupVO backupVO = backupDao.findById(current.getId());
backupVO.setStatus(Backup.Status.Error);
backupDao.update(backupVO.getId(), backupVO);
vmVO.setState(VirtualMachine.State.BackupError);
userVmDao.update(vmVO.getId(), vmVO);
return false;
}
setEndOfChainAndRemoveCurrentForBackup(current);
}
return true;
}
protected boolean endBackupChain(VirtualMachine vm, Long backupScheduleId) {
InternalBackupJoinVO current = internalBackupJoinDao.findCurrent(vm.getId(), backupScheduleId);
if (current == null) { if (current == null) {
logger.debug("There is no current active chain, no need to do anything."); logger.debug("There is no current active chain, no need to do anything.");
return true; return true;
} }
validateVmState(vm, "end backup chain");
if (mergeCurrentBackupDeltas(current)) { if (mergeCurrentBackupDeltas(current)) {
setEndOfChainAndRemoveCurrentForBackup(current); setEndOfChainAndRemoveCurrentForBackup(current);
return true; return true;
@ -2418,11 +2287,8 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
* */ * */
protected boolean mergeCurrentBackupDeltas(InternalBackupJoinVO backupJoinVO) { protected boolean mergeCurrentBackupDeltas(InternalBackupJoinVO backupJoinVO) {
VirtualMachine userVm = userVmDao.findById(backupJoinVO.getVmId()); VirtualMachine userVm = userVmDao.findById(backupJoinVO.getVmId());
List<InternalBackupJoinVO> succeedingBackupList = getSucceedingBackupList(backupJoinVO);
InternalBackupJoinVO succeedingBackup = succeedingBackupList.isEmpty() ? null : succeedingBackupList.get(0);
VMSnapshotVO succeedingVmSnapshot = getSucceedingVmSnapshot(backupJoinVO); VMSnapshotVO succeedingVmSnapshot = getSucceedingVmSnapshot(backupJoinVO);
MergeDiskOnlyVmSnapshotCommand cmd = buildMergeDiskOnlyVmSnapshotCommandForCurrentBackup(backupJoinVO, userVm, succeedingVmSnapshot, succeedingBackupList); MergeDiskOnlyVmSnapshotCommand cmd = buildMergeDiskOnlyVmSnapshotCommandForCurrentBackup(backupJoinVO, userVm, succeedingVmSnapshot);
Long hostId = vmSnapshotHelper.pickRunningHost(backupJoinVO.getVmId()); Long hostId = vmSnapshotHelper.pickRunningHost(backupJoinVO.getVmId());
Answer answer = sendBackupCommand(hostId, cmd); Answer answer = sendBackupCommand(hostId, cmd);
@ -2432,10 +2298,9 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
return false; return false;
} }
List<InternalBackupStoragePoolVO> deltasOnPrimary = internalBackupStoragePoolDao.listByBackupId(backupJoinVO.getId()); expungeOldDeltasAndUpdateVmSnapshotIfNeeded(internalBackupStoragePoolDao.listByBackupId(backupJoinVO.getId()), succeedingVmSnapshot);
expungeOldDeltasAndUpdateVmSnapshotOrBackup(deltasOnPrimary, succeedingVmSnapshot, succeedingBackup);
if (ObjectUtils.anyNotNull(succeedingVmSnapshot, succeedingBackup)) { if (succeedingVmSnapshot != null) {
return true; return true;
} }
@ -2450,18 +2315,18 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
} }
protected void createDeleteCommandsAndMergeTrees(List<VolumeObjectTO> volumeObjectTOs, Commands commands, List<InternalBackupStoragePoolVO> deletedDeltas, protected void createDeleteCommandsAndMergeTrees(List<VolumeObjectTO> volumeObjectTOs, Commands commands, List<InternalBackupStoragePoolVO> deletedDeltas,
VMSnapshotVO vmSnapshotSucceedingCurrentBackup, List<DeltaMergeTreeTO> deltaMergeTreeTOList, InternalBackupJoinVO currentBackup) { VMSnapshotVO vmSnapshotSucceedingCurrentBackup, List<DeltaMergeTreeTO> deltaMergeTreeTOList) {
for (VolumeObjectTO volumeObjectTO : volumeObjectTOs) { for (VolumeObjectTO volumeObjectTO : volumeObjectTOs) {
InternalBackupStoragePoolVO delta = internalBackupStoragePoolDao.findOneByVolumeIdAndBackupId(volumeObjectTO.getVolumeId(), currentBackup.getId()); InternalBackupStoragePoolVO delta = internalBackupStoragePoolDao.findOneByVolumeId(volumeObjectTO.getVolumeId());
if (delta == null) { if (delta == null) {
continue; continue;
} }
if (vmSnapshotSucceedingCurrentBackup == null) { if (delta.getBackupDeltaPath().equals(volumeObjectTO.getPath())) {
commands.addCommand(new DeleteCommand(new BackupDeltaTO(volumeObjectTO.getDataStore(), Hypervisor.HypervisorType.KVM, delta.getBackupDeltaParentPath()))); commands.addCommand(new DeleteCommand(new BackupDeltaTO(volumeObjectTO.getDataStore(), Hypervisor.HypervisorType.KVM, delta.getBackupDeltaParentPath())));
deletedDeltas.add(delta); deletedDeltas.add(delta);
logger.debug("Volume [{}] has a backup delta that will be deleted as part of the preparation to revert a VM snapshot.", volumeObjectTO.getUuid()); logger.debug("Volume [{}] has a backup delta that will be deleted as part of the preparation to revert a VM snapshot.", volumeObjectTO.getUuid());
} else { } else {
deltaMergeTreeTOList.add(createDeltaMergeTree(false, false, delta, volumeObjectTO, vmSnapshotSucceedingCurrentBackup, new ArrayList<>())); deltaMergeTreeTOList.add(createDeltaMergeTree(false, false, delta, volumeObjectTO, vmSnapshotSucceedingCurrentBackup));
} }
} }
} }
@ -2495,17 +2360,16 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
return new Pair<>(backupParentsToBeExpunged, lastAliveBackup); return new Pair<>(backupParentsToBeExpunged, lastAliveBackup);
} }
private MergeDiskOnlyVmSnapshotCommand buildMergeDiskOnlyVmSnapshotCommandForCurrentBackup(InternalBackupJoinVO backupJoinVO, VirtualMachine userVm, VMSnapshotVO vmSnapshot, protected MergeDiskOnlyVmSnapshotCommand buildMergeDiskOnlyVmSnapshotCommandForCurrentBackup(InternalBackupJoinVO backupJoinVO, VirtualMachine userVm, VMSnapshotVO vmSnapshot) {
List<InternalBackupJoinVO> succeedingBackupList) {
List<DeltaMergeTreeTO> deltaMergeTreeTOs = new ArrayList<>(); List<DeltaMergeTreeTO> deltaMergeTreeTOs = new ArrayList<>();
List<VolumeObjectTO> volumeTOs = vmSnapshotHelper.getVolumeTOList(backupJoinVO.getVmId()); List<VolumeObjectTO> volumeTOs = vmSnapshotHelper.getVolumeTOList(backupJoinVO.getVmId());
Map<Long, List<SnapshotDataStoreVO>> volumeIdToSnapshotDataStoreList = gatherSnapshotReferencesOfChildrenSnapshot(volumeTOs, vmSnapshot);
List<InternalBackupStoragePoolVO> deltasOnPrimary = internalBackupStoragePoolDao.listByBackupId(backupJoinVO.getId()); List<InternalBackupStoragePoolVO> deltasOnPrimary = internalBackupStoragePoolDao.listByBackupId(backupJoinVO.getId());
for (VolumeObjectTO volumeObjectTO : volumeTOs) { for (VolumeObjectTO volumeObjectTO : volumeTOs) {
KbossTO kbossTO = new KbossTO(volumeObjectTO, new LinkedList<>()); KbossTO kbossTO = new KbossTO(volumeObjectTO, volumeIdToSnapshotDataStoreList.getOrDefault(volumeObjectTO.getId(), new ArrayList<>()));
boolean childIsVolume = vmSnapshot == null && succeedingBackupList.isEmpty(); createDeltaMergeTreeForVolume(vmSnapshot == null, userVm.getState() == VirtualMachine.State.Running, deltasOnPrimary, vmSnapshot, kbossTO);
createDeltaMergeTreeForVolume(childIsVolume, userVm.getState() == VirtualMachine.State.Running, deltasOnPrimary, vmSnapshot, kbossTO, succeedingBackupList);
if (kbossTO.getDeltaMergeTreeTO() != null) { if (kbossTO.getDeltaMergeTreeTO() != null) {
deltaMergeTreeTOs.add(kbossTO.getDeltaMergeTreeTO()); deltaMergeTreeTOs.add(kbossTO.getDeltaMergeTreeTO());
} else { } else {
@ -2558,11 +2422,9 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
List<InternalBackupJoinVO> ancestorBackups; List<InternalBackupJoinVO> ancestorBackups;
if (includeRemoved) { if (includeRemoved) {
ancestorBackups = internalBackupJoinDao.listIncludingRemovedByVmIdAndBeforeDateOrderByCreatedDesc(backupVO.getVmId(), backupVO.getBackupScheduleId(), ancestorBackups = internalBackupJoinDao.listIncludingRemovedByVmIdAndBeforeDateOrderByCreatedDesc(backupVO.getVmId(), backupVO.getDate());
backupVO.getDate());
} else { } else {
ancestorBackups = internalBackupJoinDao.listByBackedUpAndVmIdAndDateBeforeOrAfterOrderBy(backupVO.getVmId(), backupVO.getBackupScheduleId(), backupVO.getDate(), true, ancestorBackups = internalBackupJoinDao.listByBackedUpAndVmIdAndDateBeforeOrAfterOrderBy(backupVO.getVmId(), backupVO.getDate(), true, false);
false);
} }
for (int i = 0; i < ancestorBackups.size(); i++) { for (int i = 0; i < ancestorBackups.size(); i++) {
@ -2589,8 +2451,7 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
* @return list of children, or and empty list if no children found. * @return list of children, or and empty list if no children found.
* */ * */
protected List<InternalBackupJoinVO> getBackupJoinChildren(BackupVO backupVO) { protected List<InternalBackupJoinVO> getBackupJoinChildren(BackupVO backupVO) {
List<InternalBackupJoinVO> children = internalBackupJoinDao.listByBackedUpAndVmIdAndDateBeforeOrAfterOrderBy(backupVO.getVmId(), backupVO.getBackupScheduleId(), List<InternalBackupJoinVO> children = internalBackupJoinDao.listByBackedUpAndVmIdAndDateBeforeOrAfterOrderBy(backupVO.getVmId(), backupVO.getDate(), false, true);
backupVO.getDate(), false, true);
long parentId = backupVO.getId(); long parentId = backupVO.getId();
for (int i = 0; i < children.size(); i++) { for (int i = 0; i < children.size(); i++) {
@ -2632,7 +2493,7 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
* Retrieves the current backup and removes the CURRENT detail. If the informed backup is not the end of chain, sets it as the new CURRENT * Retrieves the current backup and removes the CURRENT detail. If the informed backup is not the end of chain, sets it as the new CURRENT
* */ * */
protected void updateCurrentBackup(InternalBackupJoinVO backup) { protected void updateCurrentBackup(InternalBackupJoinVO backup) {
InternalBackupJoinVO current = internalBackupJoinDao.findCurrent(backup.getVmId(), backup.getScheduleId()); InternalBackupJoinVO current = internalBackupJoinDao.findCurrent(backup.getVmId());
if (current != null) { if (current != null) {
backupDetailDao.removeDetail(current.getId(), CURRENT); backupDetailDao.removeDetail(current.getId(), CURRENT);
@ -2673,29 +2534,20 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
backupVO.getName()), msg); backupVO.getName()), msg);
} }
protected void configureKbossTosForCleanup(UserVmVO userVmVO, List<InternalBackupStoragePoolVO> deltasOnPrimary, Map<Long, LinkedList<String>> volumeIdToDeltasAfterCurrent, protected void configureKbossTosForCleanup(UserVmVO userVmVO, List<InternalBackupStoragePoolVO> deltasOnPrimary, List<InternalBackupDataStoreVO> deltasOnSecondary, boolean runningVM,
List<InternalBackupDataStoreVO> deltasOnSecondary, List<InternalBackupStoragePoolVO> parentDeltasOnPrimary, List<KbossTO> kbossTOS, boolean errorOnCreation) { List<InternalBackupStoragePoolVO> parentDeltasOnPrimary, List<KbossTO> kbossTOS) {
for (VolumeObjectTO volumeObjectTO : vmSnapshotHelper.getVolumeTOList(userVmVO.getId())) { for (VolumeObjectTO volumeObjectTO : vmSnapshotHelper.getVolumeTOList(userVmVO.getId())) {
InternalBackupStoragePoolVO deltaOnPrimary = deltasOnPrimary.stream() InternalBackupStoragePoolVO deltaOnPrimary = deltasOnPrimary.stream()
.filter(delta -> delta.getVolumeId() == volumeObjectTO.getVolumeId()).findFirst().orElseThrow(); .filter(delta -> delta.getVolumeId() == volumeObjectTO.getVolumeId()).findFirst().orElseThrow();
volumeObjectTO.setPath(deltaOnPrimary.getBackupDeltaPath());
InternalBackupDataStoreVO deltaOnSecondary = InternalBackupDataStoreVO deltaOnSecondary =
deltasOnSecondary.stream().filter(delta -> delta.getVolumeId() == volumeObjectTO.getVolumeId()).findFirst().orElseThrow(); deltasOnSecondary.stream().filter(delta -> delta.getVolumeId() == volumeObjectTO.getVolumeId()).findFirst().orElseThrow();
KbossTO kbossTO; KbossTO kbossTO = new KbossTO(volumeObjectTO, deltaOnPrimary.getBackupDeltaParentPath(), deltaOnSecondary.getBackupPath());
if (errorOnCreation) {
InternalBackupStoragePoolVO parent = parentDeltasOnPrimary.stream().filter(delta -> delta.getVolumeId() == volumeObjectTO.getVolumeId()).findFirst().orElse(null);
kbossTO = new KbossTO(volumeObjectTO, parent == null ? deltaOnPrimary.getBackupDeltaParentPath() : parent.getBackupDeltaPath(), deltaOnSecondary.getBackupPath(),
volumeIdToDeltasAfterCurrent.get(volumeObjectTO.getId()));
if (parent != null) {
kbossTO.setParentDeltaPathOnPrimary(parent.getBackupDeltaParentPath());
}
kbossTO.setOldVolumePath(volumeObjectTO.getPath());
volumeObjectTO.setPath(deltaOnPrimary.getBackupDeltaPath());
} else {
kbossTO = new KbossTO(volumeObjectTO, deltaOnPrimary.getBackupDeltaPath(), deltaOnSecondary.getBackupPath(),
volumeIdToDeltasAfterCurrent.get(volumeObjectTO.getId()));
kbossTO.setParentDeltaPathOnPrimary(deltaOnPrimary.getBackupDeltaParentPath());
}
parentDeltasOnPrimary.stream()
.filter(delta -> delta.getVolumeId() == volumeObjectTO.getVolumeId()).findFirst()
.ifPresent(parentDelta -> kbossTO.setParentDeltaPathOnPrimary(parentDelta.getBackupDeltaParentPath()));
kbossTOS.add(kbossTO); kbossTOS.add(kbossTO);
} }
} }
@ -2761,11 +2613,11 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr
logger.debug("Updating snapshot reference [{}] path to [{}] as part of the preparation to restore a VM snapshot.", snapshotRef.getId(), snapshotRef.getInstallPath()); logger.debug("Updating snapshot reference [{}] path to [{}] as part of the preparation to restore a VM snapshot.", snapshotRef.getId(), snapshotRef.getInstallPath());
snapshotDataStoreDao.update(snapshotRef.getId(), snapshotRef); snapshotDataStoreDao.update(snapshotRef.getId(), snapshotRef);
} }
internalBackupStoragePoolDao.expungeByVolumeIdAndBackupId(deltaMergeTreeTO.getVolumeObjectTO().getVolumeId(), backupVO.getId()); internalBackupStoragePoolDao.expungeByVolumeId(deltaMergeTreeTO.getVolumeObjectTO().getVolumeId());
} }
for (InternalBackupStoragePoolVO delta : deletedDeltas) { for (InternalBackupStoragePoolVO delta : deletedDeltas) {
internalBackupStoragePoolDao.expungeByVolumeIdAndBackupId(delta.getVolumeId(), delta.getBackupId()); internalBackupStoragePoolDao.expungeByVolumeId(delta.getVolumeId());
} }
setEndOfChainAndRemoveCurrentForBackup(backupVO); setEndOfChainAndRemoveCurrentForBackup(backupVO);

View File

@ -25,6 +25,7 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyMap;
import static org.mockito.ArgumentMatchers.contains; import static org.mockito.ArgumentMatchers.contains;
import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doNothing;
@ -40,8 +41,8 @@ import java.time.Instant;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.Set; import java.util.Set;
import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiConstants;
@ -61,10 +62,12 @@ import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.secstorage.heuristics.HeuristicType;
import org.apache.cloudstack.storage.command.BackupDeleteAnswer; import org.apache.cloudstack.storage.command.BackupDeleteAnswer;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper;
import org.apache.cloudstack.storage.to.BackupDeltaTO; import org.apache.cloudstack.storage.to.BackupDeltaTO;
@ -142,6 +145,12 @@ public class KbossBackupProviderTest {
@Mock @Mock
private VolumeVO volumeVoMock; private VolumeVO volumeVoMock;
@Mock
private SnapshotDataStoreDao snapshotDataStoreDaoMock;
@Mock
private SnapshotDataStoreVO snapshotDataStoreVoMock;
@Mock @Mock
private VMSnapshotDao vmSnapshotDaoMock; private VMSnapshotDao vmSnapshotDaoMock;
@ -178,6 +187,9 @@ public class KbossBackupProviderTest {
@Mock @Mock
private BackupDetailVO backupDetailVoMock; private BackupDetailVO backupDetailVoMock;
@Mock
private ConfigKey<Integer> backupChainSize;
@Mock @Mock
private DataStoreManager dataStoreManagerMock; private DataStoreManager dataStoreManagerMock;
@ -287,7 +299,7 @@ public class KbossBackupProviderTest {
private long vmId = 319832; private long vmId = 319832;
private long volumeId = 41; private long volumeId = 41;
private Long backupId = 312L; Long backupId = 312L;
@Before @Before
public void setup() { public void setup() {
@ -359,7 +371,7 @@ public class KbossBackupProviderTest {
@Test @Test
public void removeVMFromBackupOfferingTestWithActiveChain() { public void removeVMFromBackupOfferingTestWithActiveChain() {
doReturn(List.of(internalBackupJoinVoMock)).when(internalBackupJoinDaoMock).listCurrents(vmId, true); doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findCurrent(vmId);
doReturn(true).when(kbossBackupProviderSpy).mergeCurrentBackupDeltas(any()); doReturn(true).when(kbossBackupProviderSpy).mergeCurrentBackupDeltas(any());
doReturn(VirtualMachine.State.Stopped).when(virtualMachineMock).getState(); doReturn(VirtualMachine.State.Stopped).when(virtualMachineMock).getState();
@ -369,12 +381,25 @@ public class KbossBackupProviderTest {
assertTrue(result); assertTrue(result);
} }
@Test
public void removeVMFromBackupOfferingTestFailedToEndChain() {
doReturn(VirtualMachine.State.Stopped).when(userVmVOMock).getState();
doReturn(false).when(kbossBackupProviderSpy).endBackupChain(any());
doReturn(userVmVOMock).when(userVmDaoMock).findById(any());
doNothing().when(vmInstanceDetailsDaoMock).addDetail(Mockito.anyLong(), any(), any(), Mockito.anyBoolean());
boolean result = kbossBackupProviderSpy.removeVMFromBackupOffering(userVmVOMock);
verify(vmInstanceDetailsDaoMock, Mockito.times(1)).addDetail(Mockito.anyLong(), any(), any(), Mockito.anyBoolean());
verify(userVmDaoMock, Mockito.times(1)).update(Mockito.anyLong(), any());
assertFalse(result);
}
@Test @Test
public void getBackupJoinParentsTestIncludeRemovedEmptyList() { public void getBackupJoinParentsTestIncludeRemovedEmptyList() {
Date date = DateUtil.now(); Date date = DateUtil.now();
doReturn(date).when(backupVoMock).getDate(); doReturn(date).when(backupVoMock).getDate();
doReturn(null).when(backupVoMock).getBackupScheduleId(); doReturn(new ArrayList<>()).when(internalBackupJoinDaoMock).listIncludingRemovedByVmIdAndBeforeDateOrderByCreatedDesc(vmId, date);
doReturn(new ArrayList<>()).when(internalBackupJoinDaoMock).listIncludingRemovedByVmIdAndBeforeDateOrderByCreatedDesc(vmId, null, date);
List<InternalBackupJoinVO> result = kbossBackupProviderSpy.getBackupJoinParents(backupVoMock, true); List<InternalBackupJoinVO> result = kbossBackupProviderSpy.getBackupJoinParents(backupVoMock, true);
@ -385,9 +410,8 @@ public class KbossBackupProviderTest {
public void getBackupJoinParentsTestIncludeRemovedAncestorIsEndOfChain() { public void getBackupJoinParentsTestIncludeRemovedAncestorIsEndOfChain() {
Date date = DateUtil.now(); Date date = DateUtil.now();
doReturn(date).when(backupVoMock).getDate(); doReturn(date).when(backupVoMock).getDate();
doReturn(null).when(backupVoMock).getBackupScheduleId();
doReturn(true).when(internalBackupJoinVoMock).getEndOfChain(); doReturn(true).when(internalBackupJoinVoMock).getEndOfChain();
doReturn(List.of(internalBackupJoinVoMock)).when(internalBackupJoinDaoMock).listIncludingRemovedByVmIdAndBeforeDateOrderByCreatedDesc(vmId, null, date); doReturn(List.of(internalBackupJoinVoMock)).when(internalBackupJoinDaoMock).listIncludingRemovedByVmIdAndBeforeDateOrderByCreatedDesc(vmId, date);
List<InternalBackupJoinVO> result = kbossBackupProviderSpy.getBackupJoinParents(backupVoMock, true); List<InternalBackupJoinVO> result = kbossBackupProviderSpy.getBackupJoinParents(backupVoMock, true);
@ -398,13 +422,12 @@ public class KbossBackupProviderTest {
public void getBackupJoinParentsTestIncludeRemovedAncestorMultipleAncestors() { public void getBackupJoinParentsTestIncludeRemovedAncestorMultipleAncestors() {
Date date = DateUtil.now(); Date date = DateUtil.now();
doReturn(date).when(backupVoMock).getDate(); doReturn(date).when(backupVoMock).getDate();
doReturn(null).when(backupVoMock).getBackupScheduleId();
InternalBackupJoinVO internalBackupJoinVoMock1 = Mockito.mock(InternalBackupJoinVO.class); InternalBackupJoinVO internalBackupJoinVoMock1 = Mockito.mock(InternalBackupJoinVO.class);
doReturn(false).when(internalBackupJoinVoMock1).getEndOfChain(); doReturn(false).when(internalBackupJoinVoMock1).getEndOfChain();
InternalBackupJoinVO internalBackupJoinVoMock2 = Mockito.mock(InternalBackupJoinVO.class); InternalBackupJoinVO internalBackupJoinVoMock2 = Mockito.mock(InternalBackupJoinVO.class);
doReturn(false).when(internalBackupJoinVoMock2).getEndOfChain(); doReturn(false).when(internalBackupJoinVoMock2).getEndOfChain();
doReturn(true).when(internalBackupJoinVoMock).getEndOfChain(); doReturn(true).when(internalBackupJoinVoMock).getEndOfChain();
doReturn(List.of(internalBackupJoinVoMock1, internalBackupJoinVoMock2, internalBackupJoinVoMock)).when(internalBackupJoinDaoMock).listIncludingRemovedByVmIdAndBeforeDateOrderByCreatedDesc(vmId, null, date); doReturn(List.of(internalBackupJoinVoMock1, internalBackupJoinVoMock2, internalBackupJoinVoMock)).when(internalBackupJoinDaoMock).listIncludingRemovedByVmIdAndBeforeDateOrderByCreatedDesc(vmId, date);
List<InternalBackupJoinVO> result = kbossBackupProviderSpy.getBackupJoinParents(backupVoMock, true); List<InternalBackupJoinVO> result = kbossBackupProviderSpy.getBackupJoinParents(backupVoMock, true);
@ -415,13 +438,12 @@ public class KbossBackupProviderTest {
public void getBackupJoinParentsTestIncludeRemovedAncestorMultipleAncestorsNoEndOfChain() { public void getBackupJoinParentsTestIncludeRemovedAncestorMultipleAncestorsNoEndOfChain() {
Date date = DateUtil.now(); Date date = DateUtil.now();
doReturn(date).when(backupVoMock).getDate(); doReturn(date).when(backupVoMock).getDate();
doReturn(null).when(backupVoMock).getBackupScheduleId();
InternalBackupJoinVO internalBackupJoinVoMock1 = Mockito.mock(InternalBackupJoinVO.class); InternalBackupJoinVO internalBackupJoinVoMock1 = Mockito.mock(InternalBackupJoinVO.class);
doReturn(false).when(internalBackupJoinVoMock1).getEndOfChain(); doReturn(false).when(internalBackupJoinVoMock1).getEndOfChain();
InternalBackupJoinVO internalBackupJoinVoMock2 = Mockito.mock(InternalBackupJoinVO.class); InternalBackupJoinVO internalBackupJoinVoMock2 = Mockito.mock(InternalBackupJoinVO.class);
doReturn(false).when(internalBackupJoinVoMock2).getEndOfChain(); doReturn(false).when(internalBackupJoinVoMock2).getEndOfChain();
doReturn(false).when(internalBackupJoinVoMock).getEndOfChain(); doReturn(false).when(internalBackupJoinVoMock).getEndOfChain();
doReturn(List.of(internalBackupJoinVoMock1, internalBackupJoinVoMock2, internalBackupJoinVoMock)).when(internalBackupJoinDaoMock).listIncludingRemovedByVmIdAndBeforeDateOrderByCreatedDesc(vmId, null, date); doReturn(List.of(internalBackupJoinVoMock1, internalBackupJoinVoMock2, internalBackupJoinVoMock)).when(internalBackupJoinDaoMock).listIncludingRemovedByVmIdAndBeforeDateOrderByCreatedDesc(vmId, date);
List<InternalBackupJoinVO> result = kbossBackupProviderSpy.getBackupJoinParents(backupVoMock, true); List<InternalBackupJoinVO> result = kbossBackupProviderSpy.getBackupJoinParents(backupVoMock, true);
@ -432,13 +454,12 @@ public class KbossBackupProviderTest {
public void getBackupJoinParentsTestNoRemovedAncestorMultipleAncestorsNoEndOfChain() { public void getBackupJoinParentsTestNoRemovedAncestorMultipleAncestorsNoEndOfChain() {
Date date = DateUtil.now(); Date date = DateUtil.now();
doReturn(date).when(backupVoMock).getDate(); doReturn(date).when(backupVoMock).getDate();
doReturn(null).when(backupVoMock).getBackupScheduleId();
InternalBackupJoinVO internalBackupJoinVoMock1 = Mockito.mock(InternalBackupJoinVO.class); InternalBackupJoinVO internalBackupJoinVoMock1 = Mockito.mock(InternalBackupJoinVO.class);
doReturn(false).when(internalBackupJoinVoMock1).getEndOfChain(); doReturn(false).when(internalBackupJoinVoMock1).getEndOfChain();
InternalBackupJoinVO internalBackupJoinVoMock2 = Mockito.mock(InternalBackupJoinVO.class); InternalBackupJoinVO internalBackupJoinVoMock2 = Mockito.mock(InternalBackupJoinVO.class);
doReturn(false).when(internalBackupJoinVoMock2).getEndOfChain(); doReturn(false).when(internalBackupJoinVoMock2).getEndOfChain();
doReturn(false).when(internalBackupJoinVoMock).getEndOfChain(); doReturn(false).when(internalBackupJoinVoMock).getEndOfChain();
doReturn(List.of(internalBackupJoinVoMock1, internalBackupJoinVoMock2, internalBackupJoinVoMock)).when(internalBackupJoinDaoMock).listByBackedUpAndVmIdAndDateBeforeOrAfterOrderBy(vmId, null, date, true, doReturn(List.of(internalBackupJoinVoMock1, internalBackupJoinVoMock2, internalBackupJoinVoMock)).when(internalBackupJoinDaoMock).listByBackedUpAndVmIdAndDateBeforeOrAfterOrderBy(vmId, date, true,
false); false);
List<InternalBackupJoinVO> result = kbossBackupProviderSpy.getBackupJoinParents(backupVoMock, false); List<InternalBackupJoinVO> result = kbossBackupProviderSpy.getBackupJoinParents(backupVoMock, false);
@ -647,30 +668,32 @@ public class KbossBackupProviderTest {
} }
@Test @Test
public void mapVolumesToVmSnapshotReferencesTestVmSnapshotAndBackupVOListIsEmpty() { public void mapVolumesToVmSnapshotReferencesTestVmSnapshotVOListIsEmpty() {
kbossBackupProviderSpy.mapVolumesToVmSnapshotAndBackupReferences(List.of(), List.of(), List.of()); kbossBackupProviderSpy.mapVolumesToVmSnapshotReferences(List.of(), List.of());
verify(vmSnapshotHelperMock, Mockito.never()).getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(1); verify(vmSnapshotHelperMock, Mockito.never()).getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(1);
} }
@Test @Test
public void mapVolumesToVmSnapshotAndBackupReferencesTestVmSnapshotAndBackupVOListHasTwoElements() { public void mapVolumesToVmSnapshotReferencesTestVmSnapshotVOListHasTwoElements() {
VMSnapshotVO vmSnapshotVoMock1 = Mockito.mock(VMSnapshotVO.class); VMSnapshotVO vmSnapshotVoMock1 = Mockito.mock(VMSnapshotVO.class);
doReturn(1L).when(vmSnapshotVoMock).getId(); doReturn(1L).when(vmSnapshotVoMock).getId();
doReturn(2L).when(vmSnapshotVoMock1).getId(); doReturn(2L).when(vmSnapshotVoMock1).getId();
doNothing().when(kbossBackupProviderSpy).mapVolumesToSnapshotReferences(Mockito.anyList(), Mockito.anyList(), anyMap());
kbossBackupProviderSpy.mapVolumesToVmSnapshotAndBackupReferences(List.of(), List.of(vmSnapshotVoMock, vmSnapshotVoMock1), List.of()); kbossBackupProviderSpy.mapVolumesToVmSnapshotReferences(List.of(), List.of(vmSnapshotVoMock, vmSnapshotVoMock1));
verify(vmSnapshotHelperMock, times(1)).getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(1); verify(vmSnapshotHelperMock, times(1)).getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(1);
verify(vmSnapshotHelperMock, times(1)).getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(2); verify(vmSnapshotHelperMock, times(1)).getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(2);
verify(kbossBackupProviderSpy, times(1)).mapVolumesToSnapshotReferences(Mockito.anyList(), Mockito.anyList(), anyMap());
} }
@Test @Test
public void createDeltaReferencesTestFullBackupEndOfChain() { public void createDeltaReferencesTestFullBackupEndOfChain() {
doReturn(internalBackupDataStoreVoMock).when(internalBackupDataStoreDaoMock).persist(any()); doReturn(internalBackupDataStoreVoMock).when(internalBackupDataStoreDaoMock).persist(any());
kbossBackupProviderSpy.createDeltaReferences(true, true, backupVoMock, List.of(), List.of(), new HashMap<>(), new HashMap<>(), null, new KbossTO(volumeObjectToMock, kbossBackupProviderSpy.createDeltaReferences(true,
new LinkedList<>())); true, true, backupVoMock, List.of(), List.of(), new HashMap<>(), new HashMap<>(), null, new KbossTO(volumeObjectToMock, List.of()));
verify(internalBackupDataStoreDaoMock, Mockito.times(1)).persist(any()); verify(internalBackupDataStoreDaoMock, Mockito.times(1)).persist(any());
} }
@ -679,8 +702,8 @@ public class KbossBackupProviderTest {
public void createDeltaReferencesTestIsolatedBackup() { public void createDeltaReferencesTestIsolatedBackup() {
doReturn(internalBackupDataStoreVoMock).when(internalBackupDataStoreDaoMock).persist(any()); doReturn(internalBackupDataStoreVoMock).when(internalBackupDataStoreDaoMock).persist(any());
kbossBackupProviderSpy.createDeltaReferences(true, true, backupVoMock, List.of(), List.of(), new HashMap<>(), new HashMap<>(), null, new KbossTO(volumeObjectToMock, kbossBackupProviderSpy.createDeltaReferences(true,
new LinkedList<>())); true, true, backupVoMock, List.of(), List.of(), new HashMap<>(), new HashMap<>(), null, new KbossTO(volumeObjectToMock, List.of()));
verify(internalBackupDataStoreDaoMock, Mockito.times(1)).persist(any()); verify(internalBackupDataStoreDaoMock, Mockito.times(1)).persist(any());
verify(kbossBackupProviderSpy, Mockito.times(0)).findAndSetParentBackupPath(any(), any(), any()); verify(kbossBackupProviderSpy, Mockito.times(0)).findAndSetParentBackupPath(any(), any(), any());
@ -691,11 +714,11 @@ public class KbossBackupProviderTest {
@Test @Test
public void createDeltaReferencesTestNotFullBackupEndOfChain() { public void createDeltaReferencesTestNotFullBackupEndOfChain() {
doReturn(internalBackupDataStoreVoMock).when(internalBackupDataStoreDaoMock).persist(any()); doReturn(internalBackupDataStoreVoMock).when(internalBackupDataStoreDaoMock).persist(any());
KbossTO kbossTO = new KbossTO(volumeObjectToMock, new LinkedList<>()); KbossTO kbossTO = new KbossTO(volumeObjectToMock, List.of());
doReturn(null).when(kbossBackupProviderSpy).createDeltaMergeTreeForVolume(false, true, List.of(), null, kbossTO, List.of()); doReturn(null).when(kbossBackupProviderSpy).createDeltaMergeTreeForVolume(false, true, List.of(), null, kbossTO);
doNothing().when(kbossBackupProviderSpy).findAndSetParentBackupPath(List.of(), null, kbossTO); doNothing().when(kbossBackupProviderSpy).findAndSetParentBackupPath(List.of(), null, kbossTO);
kbossBackupProviderSpy.createDeltaReferences(false, true, backupVoMock, List.of(), List.of(), new HashMap<>(), new HashMap<>(), null, kbossTO); kbossBackupProviderSpy.createDeltaReferences(false, true, true, backupVoMock, List.of(), List.of(), new HashMap<>(), new HashMap<>(), null, kbossTO);
verify(internalBackupDataStoreDaoMock, Mockito.times(1)).persist(any()); verify(internalBackupDataStoreDaoMock, Mockito.times(1)).persist(any());
verify(kbossBackupProviderSpy, Mockito.times(1)).findAndSetParentBackupPath(List.of(), null, kbossTO); verify(kbossBackupProviderSpy, Mockito.times(1)).findAndSetParentBackupPath(List.of(), null, kbossTO);
@ -705,8 +728,8 @@ public class KbossBackupProviderTest {
public void createDeltaReferencesTestFullBackupNotEndOfChainDoesNotHaveVmSnapshotSucceedingLastBackup() { public void createDeltaReferencesTestFullBackupNotEndOfChainDoesNotHaveVmSnapshotSucceedingLastBackup() {
doReturn(internalBackupDataStoreVoMock).when(internalBackupDataStoreDaoMock).persist(any()); doReturn(internalBackupDataStoreVoMock).when(internalBackupDataStoreDaoMock).persist(any());
kbossBackupProviderSpy.createDeltaReferences(true, true, backupVoMock, List.of(), List.of(), new HashMap<>(), new HashMap<>(), null, new KbossTO(volumeObjectToMock, kbossBackupProviderSpy.createDeltaReferences(true,
new LinkedList<>())); false, true, backupVoMock, List.of(), List.of(), new HashMap<>(), new HashMap<>(), null, new KbossTO(volumeObjectToMock, List.of()));
verify(internalBackupDataStoreDaoMock, Mockito.times(1)).persist(any()); verify(internalBackupDataStoreDaoMock, Mockito.times(1)).persist(any());
} }
@ -769,7 +792,7 @@ public class KbossBackupProviderTest {
assertFalse(result.first()); assertFalse(result.first());
assertNull(result.second()); assertNull(result.second());
verify(kbossBackupProviderSpy, Mockito.times(1)).setBackupAsIsolated(backupVoMock); verify(kbossBackupProviderSpy, Mockito.times(1)).setBackupAsIsolated(backupVoMock);
verify(kbossBackupProviderSpy, Mockito.times(2)).createDeltaReferences(Mockito.anyBoolean(), Mockito.anyBoolean(), any(), any(), any(), any(), any(), any(), any()); verify(kbossBackupProviderSpy, Mockito.times(2)).createDeltaReferences(Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean(), any(), any(), any(), any(), any(), any(), any());
verify(kbossBackupProviderSpy, Mockito.times(1)).processBackupFailure(any(), any(), Mockito.anyLong(), Mockito.anyBoolean(), any()); verify(kbossBackupProviderSpy, Mockito.times(1)).processBackupFailure(any(), any(), Mockito.anyLong(), Mockito.anyBoolean(), any());
} }
@ -792,7 +815,7 @@ public class KbossBackupProviderTest {
doReturn(takeKbossBackupAnswerMock).when(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any()); doReturn(takeKbossBackupAnswerMock).when(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any());
doReturn(true).when(takeKbossBackupAnswerMock).getResult(); doReturn(true).when(takeKbossBackupAnswerMock).getResult();
doNothing().when(kbossBackupProviderSpy).processBackupSuccess(anyBoolean(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), any(), doNothing().when(kbossBackupProviderSpy).processBackupSuccess(anyBoolean(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), any(),
anyLong(), anyBoolean(), anyBoolean(), any()); anyLong(), anyBoolean(), anyBoolean());
doReturn(true).when(kbossBackupProviderSpy).offeringSupportsCompression(internalBackupJoinVoMock); doReturn(true).when(kbossBackupProviderSpy).offeringSupportsCompression(internalBackupJoinVoMock);
doNothing().when(kbossBackupProviderSpy).compressBackupAsync(internalBackupJoinVoMock, 0, 0); doNothing().when(kbossBackupProviderSpy).compressBackupAsync(internalBackupJoinVoMock, 0, 0);
@ -800,9 +823,9 @@ public class KbossBackupProviderTest {
assertTrue(result.first()); assertTrue(result.first());
assertEquals(backupId, result.second()); assertEquals(backupId, result.second());
verify(kbossBackupProviderSpy, Mockito.times(1)).setBackupAsIsolated(backupVoMock); verify(kbossBackupProviderSpy, Mockito.times(1)).setBackupAsIsolated(backupVoMock);
verify(kbossBackupProviderSpy, Mockito.times(2)).createDeltaReferences(Mockito.anyBoolean(), Mockito.anyBoolean(), any(), any(), any(), any(), any(), any(), any()); verify(kbossBackupProviderSpy, Mockito.times(2)).createDeltaReferences(Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean(), any(), any(), any(), any(), any(), any(), any());
verify(kbossBackupProviderSpy, Mockito.times(1)).processBackupSuccess(anyBoolean(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), any(), verify(kbossBackupProviderSpy, Mockito.times(1)).processBackupSuccess(anyBoolean(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), any(),
anyLong(), anyBoolean(), anyBoolean(), any()); anyLong(), anyBoolean(), anyBoolean());
verify(kbossBackupProviderSpy, Mockito.times(1)).compressBackupAsync(internalBackupJoinVoMock, 0, 0); verify(kbossBackupProviderSpy, Mockito.times(1)).compressBackupAsync(internalBackupJoinVoMock, 0, 0);
} }
@ -827,7 +850,7 @@ public class KbossBackupProviderTest {
doReturn(takeKbossBackupAnswerMock).when(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any()); doReturn(takeKbossBackupAnswerMock).when(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any());
doReturn(true).when(takeKbossBackupAnswerMock).getResult(); doReturn(true).when(takeKbossBackupAnswerMock).getResult();
doNothing().when(kbossBackupProviderSpy).processBackupSuccess(anyBoolean(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), any(), doNothing().when(kbossBackupProviderSpy).processBackupSuccess(anyBoolean(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), any(),
anyLong(), anyBoolean(), anyBoolean(), any()); anyLong(), anyBoolean(), anyBoolean());
doReturn(false).when(kbossBackupProviderSpy).offeringSupportsCompression(internalBackupJoinVoMock); doReturn(false).when(kbossBackupProviderSpy).offeringSupportsCompression(internalBackupJoinVoMock);
doNothing().when(kbossBackupProviderSpy).validateBackupAsyncIfHasOfferingSupport(any(), anyLong(), anyLong()); doNothing().when(kbossBackupProviderSpy).validateBackupAsyncIfHasOfferingSupport(any(), anyLong(), anyLong());
@ -836,9 +859,9 @@ public class KbossBackupProviderTest {
assertEquals(backupId, result.second()); assertEquals(backupId, result.second());
verify(internalBackupStoragePoolDaoMock).listByBackupId(0); verify(internalBackupStoragePoolDaoMock).listByBackupId(0);
verify(internalBackupDataStoreDaoMock).listByBackupId(0); verify(internalBackupDataStoreDaoMock).listByBackupId(0);
verify(kbossBackupProviderSpy, Mockito.times(2)).createDeltaReferences(Mockito.anyBoolean(), Mockito.anyBoolean(), any(), any(), any(), any(), any(), any(), any()); verify(kbossBackupProviderSpy, Mockito.times(2)).createDeltaReferences(Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean(), any(), any(), any(), any(), any(), any(), any());
verify(kbossBackupProviderSpy, Mockito.times(1)).processBackupSuccess(anyBoolean(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), any(), verify(kbossBackupProviderSpy, Mockito.times(1)).processBackupSuccess(anyBoolean(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), any(),
anyLong(), anyBoolean(), anyBoolean(), any()); anyLong(), anyBoolean(), anyBoolean());
verify(kbossBackupProviderSpy, Mockito.times(1)).validateBackupAsyncIfHasOfferingSupport(internalBackupJoinVoMock, 0, 0); verify(kbossBackupProviderSpy, Mockito.times(1)).validateBackupAsyncIfHasOfferingSupport(internalBackupJoinVoMock, 0, 0);
} }
@ -975,7 +998,7 @@ public class KbossBackupProviderTest {
doReturn(new Pair<>(List.of(), parentVo)).when(kbossBackupProviderSpy).getParentsToBeExpungedWithBackupAndAddThemToListOfDeleteCommands(any(), any()); doReturn(new Pair<>(List.of(), parentVo)).when(kbossBackupProviderSpy).getParentsToBeExpungedWithBackupAndAddThemToListOfDeleteCommands(any(), any());
doReturn(endPointMock).when(endPointSelectorMock).select((DataStore)null); doReturn(endPointMock).when(endPointSelectorMock).select((DataStore)null);
doReturn(null).when(kbossBackupProviderSpy).sendBackupCommands(anyLong(), any()); doReturn(null).when(kbossBackupProviderSpy).sendBackupCommands(anyLong(), any());
doReturn(false).when(kbossBackupProviderSpy).processRemoveBackupFailures(anyBoolean(), any(), any(), any(), any()); doReturn(false).when(kbossBackupProviderSpy).processRemoveBackupFailures(anyBoolean(), any(), any(), any());
doNothing().when(kbossBackupProviderSpy).processRemovedBackups(any()); doNothing().when(kbossBackupProviderSpy).processRemovedBackups(any());
@ -1005,7 +1028,7 @@ public class KbossBackupProviderTest {
doReturn(new Pair<>(List.of(), parentVo)).when(kbossBackupProviderSpy).getParentsToBeExpungedWithBackupAndAddThemToListOfDeleteCommands(any(), any()); doReturn(new Pair<>(List.of(), parentVo)).when(kbossBackupProviderSpy).getParentsToBeExpungedWithBackupAndAddThemToListOfDeleteCommands(any(), any());
doReturn(endPointMock).when(endPointSelectorMock).select((DataStore)null); doReturn(endPointMock).when(endPointSelectorMock).select((DataStore)null);
doReturn(null).when(kbossBackupProviderSpy).sendBackupCommands(anyLong(), any()); doReturn(null).when(kbossBackupProviderSpy).sendBackupCommands(anyLong(), any());
doReturn(true).when(kbossBackupProviderSpy).processRemoveBackupFailures(anyBoolean(), any(), any(), any(), any()); doReturn(true).when(kbossBackupProviderSpy).processRemoveBackupFailures(anyBoolean(), any(), any(), any());
doNothing().when(kbossBackupProviderSpy).processRemovedBackups(any()); doNothing().when(kbossBackupProviderSpy).processRemovedBackups(any());
@ -1048,6 +1071,8 @@ public class KbossBackupProviderTest {
doReturn(new Pair<>(true, backupVoMock)).when(kbossBackupProviderSpy).validateCompressionStateForRestoreAndGetBackup(backupId); doReturn(new Pair<>(true, backupVoMock)).when(kbossBackupProviderSpy).validateCompressionStateForRestoreAndGetBackup(backupId);
long currentBackupId = 39; long currentBackupId = 39;
InternalBackupJoinVO currentBackup = Mockito.mock(InternalBackupJoinVO.class); InternalBackupJoinVO currentBackup = Mockito.mock(InternalBackupJoinVO.class);
doReturn(currentBackupId).when(currentBackup).getId();
doReturn(currentBackup).when(internalBackupJoinDaoMock).findCurrent(vmId);
doReturn(hostVOMock).when(kbossBackupProviderSpy).getHostToRestore(virtualMachineMock, false, null); doReturn(hostVOMock).when(kbossBackupProviderSpy).getHostToRestore(virtualMachineMock, false, null);
doNothing().when(kbossBackupProviderSpy).createAndAttachVolumes(any(), any(), any(), any()); doNothing().when(kbossBackupProviderSpy).createAndAttachVolumes(any(), any(), any(), any());
doReturn(Set.of()).when(kbossBackupProviderSpy).generateBackupAndVolumePairsToRestore(any(), any(), any(), anyBoolean()); doReturn(Set.of()).when(kbossBackupProviderSpy).generateBackupAndVolumePairsToRestore(any(), any(), any(), anyBoolean());
@ -1070,6 +1095,8 @@ public class KbossBackupProviderTest {
doReturn(new Pair<>(true, backupVoMock)).when(kbossBackupProviderSpy).validateCompressionStateForRestoreAndGetBackup(backupId); doReturn(new Pair<>(true, backupVoMock)).when(kbossBackupProviderSpy).validateCompressionStateForRestoreAndGetBackup(backupId);
long currentBackupId = 39; long currentBackupId = 39;
InternalBackupJoinVO currentBackup = Mockito.mock(InternalBackupJoinVO.class); InternalBackupJoinVO currentBackup = Mockito.mock(InternalBackupJoinVO.class);
doReturn(currentBackupId).when(currentBackup).getId();
doReturn(currentBackup).when(internalBackupJoinDaoMock).findCurrent(vmId);
doReturn(hostVOMock).when(kbossBackupProviderSpy).getHostToRestore(virtualMachineMock, false, null); doReturn(hostVOMock).when(kbossBackupProviderSpy).getHostToRestore(virtualMachineMock, false, null);
doNothing().when(kbossBackupProviderSpy).createAndAttachVolumes(any(), any(), any(), any()); doNothing().when(kbossBackupProviderSpy).createAndAttachVolumes(any(), any(), any(), any());
doReturn(Set.of()).when(kbossBackupProviderSpy).generateBackupAndVolumePairsToRestore(any(), any(), any(), anyBoolean()); doReturn(Set.of()).when(kbossBackupProviderSpy).generateBackupAndVolumePairsToRestore(any(), any(), any(), anyBoolean());
@ -1080,6 +1107,7 @@ public class KbossBackupProviderTest {
boolean result = kbossBackupProviderSpy.orchestrateRestoreVMFromBackup(backupVoMock, virtualMachineMock, false, null, true); boolean result = kbossBackupProviderSpy.orchestrateRestoreVMFromBackup(backupVoMock, virtualMachineMock, false, null, true);
verify(internalBackupStoragePoolDaoMock).listByBackupId(currentBackupId);
verify(kbossBackupProviderSpy).createAndAttachVolumes(any(), any(), any(), any()); verify(kbossBackupProviderSpy).createAndAttachVolumes(any(), any(), any(), any());
verify(kbossBackupProviderSpy).populateDeltasToRemoveAndToMergeAndUpdateVolumePaths(any(), any(), any(), any(), any()); verify(kbossBackupProviderSpy).populateDeltasToRemoveAndToMergeAndUpdateVolumePaths(any(), any(), any(), any(), any());
assertFalse(result); assertFalse(result);
@ -1092,6 +1120,8 @@ public class KbossBackupProviderTest {
doReturn(new Pair<>(true, backupVoMock)).when(kbossBackupProviderSpy).validateCompressionStateForRestoreAndGetBackup(backupId); doReturn(new Pair<>(true, backupVoMock)).when(kbossBackupProviderSpy).validateCompressionStateForRestoreAndGetBackup(backupId);
long currentBackupId = 39; long currentBackupId = 39;
InternalBackupJoinVO currentBackup = Mockito.mock(InternalBackupJoinVO.class); InternalBackupJoinVO currentBackup = Mockito.mock(InternalBackupJoinVO.class);
doReturn(currentBackupId).when(currentBackup).getId();
doReturn(currentBackup).when(internalBackupJoinDaoMock).findCurrent(vmId);
doReturn(hostVOMock).when(kbossBackupProviderSpy).getHostToRestore(virtualMachineMock, false, null); doReturn(hostVOMock).when(kbossBackupProviderSpy).getHostToRestore(virtualMachineMock, false, null);
doNothing().when(kbossBackupProviderSpy).createAndAttachVolumes(any(), any(), any(), any()); doNothing().when(kbossBackupProviderSpy).createAndAttachVolumes(any(), any(), any(), any());
doReturn(Set.of()).when(kbossBackupProviderSpy).generateBackupAndVolumePairsToRestore(any(), any(), any(), anyBoolean()); doReturn(Set.of()).when(kbossBackupProviderSpy).generateBackupAndVolumePairsToRestore(any(), any(), any(), anyBoolean());
@ -1103,6 +1133,7 @@ public class KbossBackupProviderTest {
boolean result = kbossBackupProviderSpy.orchestrateRestoreVMFromBackup(backupVoMock, virtualMachineMock, false, null, true); boolean result = kbossBackupProviderSpy.orchestrateRestoreVMFromBackup(backupVoMock, virtualMachineMock, false, null, true);
verify(internalBackupStoragePoolDaoMock).listByBackupId(currentBackupId);
verify(kbossBackupProviderSpy).createAndAttachVolumes(any(), any(), any(), any()); verify(kbossBackupProviderSpy).createAndAttachVolumes(any(), any(), any(), any());
verify(kbossBackupProviderSpy).populateDeltasToRemoveAndToMergeAndUpdateVolumePaths(any(), any(), any(), any(), any()); verify(kbossBackupProviderSpy).populateDeltasToRemoveAndToMergeAndUpdateVolumePaths(any(), any(), any(), any(), any());
assertFalse(result); assertFalse(result);
@ -1115,6 +1146,8 @@ public class KbossBackupProviderTest {
doReturn(new Pair<>(true, backupVoMock)).when(kbossBackupProviderSpy).validateCompressionStateForRestoreAndGetBackup(backupId); doReturn(new Pair<>(true, backupVoMock)).when(kbossBackupProviderSpy).validateCompressionStateForRestoreAndGetBackup(backupId);
long currentBackupId = 39; long currentBackupId = 39;
InternalBackupJoinVO currentBackup = Mockito.mock(InternalBackupJoinVO.class); InternalBackupJoinVO currentBackup = Mockito.mock(InternalBackupJoinVO.class);
doReturn(currentBackupId).when(currentBackup).getId();
doReturn(currentBackup).when(internalBackupJoinDaoMock).findCurrent(vmId);
doReturn(hostVOMock).when(kbossBackupProviderSpy).getHostToRestore(virtualMachineMock, true, null); doReturn(hostVOMock).when(kbossBackupProviderSpy).getHostToRestore(virtualMachineMock, true, null);
doNothing().when(kbossBackupProviderSpy).createAndAttachVolumes(any(), any(), any(), any()); doNothing().when(kbossBackupProviderSpy).createAndAttachVolumes(any(), any(), any(), any());
doReturn(Set.of()).when(kbossBackupProviderSpy).generateBackupAndVolumePairsToRestore(any(), any(), any(), anyBoolean()); doReturn(Set.of()).when(kbossBackupProviderSpy).generateBackupAndVolumePairsToRestore(any(), any(), any(), anyBoolean());
@ -1123,14 +1156,18 @@ public class KbossBackupProviderTest {
doReturn(VirtualMachine.State.Stopped).when(virtualMachineMock).getState(); doReturn(VirtualMachine.State.Stopped).when(virtualMachineMock).getState();
doReturn(new Answer[]{Mockito.mock(Answer.class)}).when(kbossBackupProviderSpy).sendBackupCommands(anyLong(), any()); doReturn(new Answer[]{Mockito.mock(Answer.class)}).when(kbossBackupProviderSpy).sendBackupCommands(anyLong(), any());
doReturn(true).when(kbossBackupProviderSpy).processRestoreAnswers(any(), any(), anyBoolean()); doReturn(true).when(kbossBackupProviderSpy).processRestoreAnswers(any(), any(), anyBoolean());
doNothing().when(kbossBackupProviderSpy).setEndOfChainAndRemoveCurrentForBackup(currentBackup);
doReturn(List.of()).when(kbossBackupProviderSpy).getVolumesToConsolidate(any(), any(), any(), anyLong(), anyBoolean()); doReturn(List.of()).when(kbossBackupProviderSpy).getVolumesToConsolidate(any(), any(), any(), anyLong(), anyBoolean());
doReturn(true).when(kbossBackupProviderSpy).finalizeQuickRestore(any(), anyList(), anyLong()); doReturn(true).when(kbossBackupProviderSpy).finalizeQuickRestore(any(), anyList(), anyLong());
boolean result = kbossBackupProviderSpy.orchestrateRestoreVMFromBackup(backupVoMock, virtualMachineMock, true, null, true); boolean result = kbossBackupProviderSpy.orchestrateRestoreVMFromBackup(backupVoMock, virtualMachineMock, true, null, true);
verify(internalBackupStoragePoolDaoMock).listByBackupId(currentBackupId);
verify(kbossBackupProviderSpy).createAndAttachVolumes(any(), any(), any(), any()); verify(kbossBackupProviderSpy).createAndAttachVolumes(any(), any(), any(), any());
verify(kbossBackupProviderSpy).populateDeltasToRemoveAndToMergeAndUpdateVolumePaths(any(), any(), any(), any(), any()); verify(kbossBackupProviderSpy).populateDeltasToRemoveAndToMergeAndUpdateVolumePaths(any(), any(), any(), any(), any());
verify(kbossBackupProviderSpy).updateVolumePathsAndSizeIfNeeded(any(), any(), anyList(), anyList(), anyBoolean()); verify(kbossBackupProviderSpy).updateVolumePathsAndSizeIfNeeded(any(), any(), anyList(), anyList(), anyBoolean());
verify(internalBackupStoragePoolDaoMock).expungeByBackupId(currentBackupId);
verify(kbossBackupProviderSpy).setEndOfChainAndRemoveCurrentForBackup(currentBackup);
verify(kbossBackupProviderSpy).finalizeQuickRestore(any(), anyList(), anyLong()); verify(kbossBackupProviderSpy).finalizeQuickRestore(any(), anyList(), anyLong());
assertTrue(result); assertTrue(result);
} }
@ -1340,25 +1377,25 @@ public class KbossBackupProviderTest {
} }
@Test @Test
public void finishBackupChainsTestInvalidState() { public void finishBackupChainTestInvalidState() {
doReturn(userVmVOMock).when(userVmDaoMock).findById(vmId); doReturn(userVmVOMock).when(userVmDaoMock).findById(vmId);
doReturn(VirtualMachine.State.Migrating).when(userVmVOMock).getState(); doReturn(VirtualMachine.State.Migrating).when(userVmVOMock).getState();
boolean result = kbossBackupProviderSpy.finishBackupChains(virtualMachineMock); boolean result = kbossBackupProviderSpy.finishBackupChain(virtualMachineMock);
assertFalse(result); assertFalse(result);
} }
@Test @Test
public void finishBackupChainsTestRunningVm() { public void finishBackupChainTestRunningVm() {
doReturn(userVmVOMock).when(userVmDaoMock).findById(vmId); doReturn(userVmVOMock).when(userVmDaoMock).findById(vmId);
doReturn(VirtualMachine.State.Running).when(userVmVOMock).getState(); doReturn(VirtualMachine.State.Running).when(userVmVOMock).getState();
doReturn(true).when(kbossBackupProviderSpy).finishAllChains(eq(userVmVOMock), any()); doReturn(true).when(kbossBackupProviderSpy).endBackupChain(userVmVOMock);
boolean result = kbossBackupProviderSpy.finishBackupChains(virtualMachineMock); boolean result = kbossBackupProviderSpy.finishBackupChain(virtualMachineMock);
assertTrue(result); assertTrue(result);
verify(kbossBackupProviderSpy).finishAllChains(eq(userVmVOMock), any()); verify(kbossBackupProviderSpy).endBackupChain(userVmVOMock);
} }
@Test @Test
@ -1367,7 +1404,7 @@ public class KbossBackupProviderTest {
doReturn(VirtualMachine.State.BackupError).when(userVmVOMock).getState(); doReturn(VirtualMachine.State.BackupError).when(userVmVOMock).getState();
doReturn(true).when(kbossBackupProviderSpy).normalizeBackupErrorAndFinishChain(userVmVOMock); doReturn(true).when(kbossBackupProviderSpy).normalizeBackupErrorAndFinishChain(userVmVOMock);
boolean result = kbossBackupProviderSpy.finishBackupChains(virtualMachineMock); boolean result = kbossBackupProviderSpy.finishBackupChain(virtualMachineMock);
assertTrue(result); assertTrue(result);
verify(kbossBackupProviderSpy).normalizeBackupErrorAndFinishChain(userVmVOMock); verify(kbossBackupProviderSpy).normalizeBackupErrorAndFinishChain(userVmVOMock);
@ -1375,6 +1412,8 @@ public class KbossBackupProviderTest {
@Test @Test
public void prepareVmForSnapshotRevertTestNoCurrentBackup() { public void prepareVmForSnapshotRevertTestNoCurrentBackup() {
doReturn(null).when(internalBackupJoinDaoMock).findCurrent(vmId);
kbossBackupProviderSpy.prepareVmForSnapshotRevert(vmSnapshotVoMock, virtualMachineMock); kbossBackupProviderSpy.prepareVmForSnapshotRevert(vmSnapshotVoMock, virtualMachineMock);
verify(kbossBackupProviderSpy, never()).getSucceedingVmSnapshot(any()); verify(kbossBackupProviderSpy, never()).getSucceedingVmSnapshot(any());
@ -1382,7 +1421,7 @@ public class KbossBackupProviderTest {
@Test @Test
public void prepareVmForSnapshotRevertTestCurrentBackupBeforeVmSnapshot() { public void prepareVmForSnapshotRevertTestCurrentBackupBeforeVmSnapshot() {
doReturn(List.of(internalBackupJoinVoMock)).when(internalBackupJoinDaoMock).listCurrents(anyLong(), anyBoolean()); doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findCurrent(vmId);
doReturn(Date.from(Instant.EPOCH)).when(internalBackupJoinVoMock).getDate(); doReturn(Date.from(Instant.EPOCH)).when(internalBackupJoinVoMock).getDate();
doReturn(Date.from(Instant.now())).when(vmSnapshotVoMock).getCreated(); doReturn(Date.from(Instant.now())).when(vmSnapshotVoMock).getCreated();
@ -1393,12 +1432,12 @@ public class KbossBackupProviderTest {
@Test (expected = CloudRuntimeException.class) @Test (expected = CloudRuntimeException.class)
public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotTimeout() throws OperationTimedoutException, AgentUnavailableException { public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotTimeout() throws OperationTimedoutException, AgentUnavailableException {
doReturn(List.of(internalBackupJoinVoMock)).when(internalBackupJoinDaoMock).listCurrents(anyLong(), anyBoolean()); doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findCurrent(vmId);
doReturn(Date.from(Instant.now())).when(internalBackupJoinVoMock).getDate(); doReturn(Date.from(Instant.now())).when(internalBackupJoinVoMock).getDate();
doReturn(Date.from(Instant.EPOCH)).when(vmSnapshotVoMock).getCreated(); doReturn(Date.from(Instant.EPOCH)).when(vmSnapshotVoMock).getCreated();
doReturn(List.of()).when(vmSnapshotHelperMock).getVolumeTOList(vmId); doReturn(List.of()).when(vmSnapshotHelperMock).getVolumeTOList(vmId);
doReturn(vmSnapshotVoMock).when(kbossBackupProviderSpy).getSucceedingVmSnapshot(internalBackupJoinVoMock); doReturn(vmSnapshotVoMock).when(kbossBackupProviderSpy).getSucceedingVmSnapshot(internalBackupJoinVoMock);
doNothing().when(kbossBackupProviderSpy).createDeleteCommandsAndMergeTrees(any(), any(), any(), any(), anyList(), any()); doNothing().when(kbossBackupProviderSpy).createDeleteCommandsAndMergeTrees(any(), any(), any(), any(), anyList());
doThrow(OperationTimedoutException.class).when(kbossBackupProviderSpy).sendBackupCommands(any(), any()); doThrow(OperationTimedoutException.class).when(kbossBackupProviderSpy).sendBackupCommands(any(), any());
kbossBackupProviderSpy.prepareVmForSnapshotRevert(vmSnapshotVoMock, virtualMachineMock); kbossBackupProviderSpy.prepareVmForSnapshotRevert(vmSnapshotVoMock, virtualMachineMock);
@ -1408,12 +1447,12 @@ public class KbossBackupProviderTest {
@Test (expected = CloudRuntimeException.class) @Test (expected = CloudRuntimeException.class)
public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotNullAnswer() throws OperationTimedoutException, AgentUnavailableException { public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotNullAnswer() throws OperationTimedoutException, AgentUnavailableException {
doReturn(List.of(internalBackupJoinVoMock)).when(internalBackupJoinDaoMock).listCurrents(anyLong(), anyBoolean()); doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findCurrent(vmId);
doReturn(Date.from(Instant.now())).when(internalBackupJoinVoMock).getDate(); doReturn(Date.from(Instant.now())).when(internalBackupJoinVoMock).getDate();
doReturn(Date.from(Instant.EPOCH)).when(vmSnapshotVoMock).getCreated(); doReturn(Date.from(Instant.EPOCH)).when(vmSnapshotVoMock).getCreated();
doReturn(List.of()).when(vmSnapshotHelperMock).getVolumeTOList(vmId); doReturn(List.of()).when(vmSnapshotHelperMock).getVolumeTOList(vmId);
doReturn(vmSnapshotVoMock).when(kbossBackupProviderSpy).getSucceedingVmSnapshot(internalBackupJoinVoMock); doReturn(vmSnapshotVoMock).when(kbossBackupProviderSpy).getSucceedingVmSnapshot(internalBackupJoinVoMock);
doNothing().when(kbossBackupProviderSpy).createDeleteCommandsAndMergeTrees(any(), any(), any(), any(), anyList(), any()); doNothing().when(kbossBackupProviderSpy).createDeleteCommandsAndMergeTrees(any(), any(), any(), any(), anyList());
doReturn(null).when(kbossBackupProviderSpy).sendBackupCommands(any(), any()); doReturn(null).when(kbossBackupProviderSpy).sendBackupCommands(any(), any());
kbossBackupProviderSpy.prepareVmForSnapshotRevert(vmSnapshotVoMock, virtualMachineMock); kbossBackupProviderSpy.prepareVmForSnapshotRevert(vmSnapshotVoMock, virtualMachineMock);
@ -1423,12 +1462,12 @@ public class KbossBackupProviderTest {
@Test @Test
public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotSuccess() throws OperationTimedoutException, AgentUnavailableException { public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotSuccess() throws OperationTimedoutException, AgentUnavailableException {
doReturn(List.of(internalBackupJoinVoMock)).when(internalBackupJoinDaoMock).listCurrents(anyLong(), anyBoolean()); doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findCurrent(vmId);
doReturn(Date.from(Instant.now())).when(internalBackupJoinVoMock).getDate(); doReturn(Date.from(Instant.now())).when(internalBackupJoinVoMock).getDate();
doReturn(Date.from(Instant.EPOCH)).when(vmSnapshotVoMock).getCreated(); doReturn(Date.from(Instant.EPOCH)).when(vmSnapshotVoMock).getCreated();
doReturn(List.of()).when(vmSnapshotHelperMock).getVolumeTOList(vmId); doReturn(List.of()).when(vmSnapshotHelperMock).getVolumeTOList(vmId);
doReturn(vmSnapshotVoMock).when(kbossBackupProviderSpy).getSucceedingVmSnapshot(internalBackupJoinVoMock); doReturn(vmSnapshotVoMock).when(kbossBackupProviderSpy).getSucceedingVmSnapshot(internalBackupJoinVoMock);
doNothing().when(kbossBackupProviderSpy).createDeleteCommandsAndMergeTrees(any(), any(), any(), any(), anyList(), any()); doNothing().when(kbossBackupProviderSpy).createDeleteCommandsAndMergeTrees(any(), any(), any(), any(), anyList());
doReturn(new Answer[]{}).when(kbossBackupProviderSpy).sendBackupCommands(any(), any()); doReturn(new Answer[]{}).when(kbossBackupProviderSpy).sendBackupCommands(any(), any());
doNothing().when(kbossBackupProviderSpy).updateReferencesAfterPrepareForSnapshotRevert(any(), any(), any(), any()); doNothing().when(kbossBackupProviderSpy).updateReferencesAfterPrepareForSnapshotRevert(any(), any(), any(), any());
@ -1565,6 +1604,7 @@ public class KbossBackupProviderTest {
doReturn(virtualMachineToMock).when(hypervisorGuruMock).implement(any()); doReturn(virtualMachineToMock).when(hypervisorGuruMock).implement(any());
doReturn(false).when(kbossBackupProviderSpy).validateBackup(anyLong(), any(), any(), any(), any(), any()); doReturn(false).when(kbossBackupProviderSpy).validateBackup(anyLong(), any(), any(), any(), any(), any());
doNothing().when(kbossBackupProviderSpy).sendCleanupFailedEmail(any(), any()); doNothing().when(kbossBackupProviderSpy).sendCleanupFailedEmail(any(), any());
doNothing().when(kbossBackupProviderSpy).validateVmState(any(), any(), any(), any());
boolean result = kbossBackupProviderSpy.validateWithValidationVm(backupId, 2L, backupVoMock); boolean result = kbossBackupProviderSpy.validateWithValidationVm(backupId, 2L, backupVoMock);
@ -1628,7 +1668,7 @@ public class KbossBackupProviderTest {
kbossBackupProviderSpy.endBackupChainIfConfigured(backupVoMock); kbossBackupProviderSpy.endBackupChainIfConfigured(backupVoMock);
verify(kbossBackupProviderSpy, never()).endBackupChain(any(), any()); verify(kbossBackupProviderSpy, never()).endBackupChain(any());
} }
@Test @Test
@ -1639,10 +1679,11 @@ public class KbossBackupProviderTest {
InternalBackupJoinVO child = mock(InternalBackupJoinVO.class); InternalBackupJoinVO child = mock(InternalBackupJoinVO.class);
doReturn(false).when(child).getCurrent(); doReturn(false).when(child).getCurrent();
doReturn(List.of(child)).when(kbossBackupProviderSpy).getBackupJoinChildren(any()); doReturn(List.of(child)).when(kbossBackupProviderSpy).getBackupJoinChildren(any());
doNothing().when(kbossBackupProviderSpy).validateVmState(any(), any(), any(), any());
kbossBackupProviderSpy.endBackupChainIfConfigured(backupVoMock); kbossBackupProviderSpy.endBackupChainIfConfigured(backupVoMock);
verify(kbossBackupProviderSpy, never()).endBackupChain(any(), any()); verify(kbossBackupProviderSpy, never()).endBackupChain(any());
} }
@Test @Test
@ -1651,12 +1692,13 @@ public class KbossBackupProviderTest {
doReturn(true).when(internalBackupJoinVoMock).getCurrent(); doReturn(true).when(internalBackupJoinVoMock).getCurrent();
doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findById(anyLong()); doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findById(anyLong());
doReturn(List.of()).when(kbossBackupProviderSpy).getBackupJoinChildren(any()); doReturn(List.of()).when(kbossBackupProviderSpy).getBackupJoinChildren(any());
doReturn(userVmVOMock).when(userVmDaoMock).findById(anyLong()); doReturn(userVmVOMock).when(userVmDaoMock).findByIdIncludingRemoved(anyLong());
doReturn(true).when(kbossBackupProviderSpy).endBackupChain(any(), any()); doReturn(true).when(kbossBackupProviderSpy).endBackupChain(any());
doNothing().when(kbossBackupProviderSpy).validateVmState(any(), any(), any(), any());
kbossBackupProviderSpy.endBackupChainIfConfigured(backupVoMock); kbossBackupProviderSpy.endBackupChainIfConfigured(backupVoMock);
verify(kbossBackupProviderSpy, times(1)).endBackupChain(eq(userVmVOMock), anyLong()); verify(kbossBackupProviderSpy, times(1)).endBackupChain(userVmVOMock);
} }
@Test @Test
@ -1667,12 +1709,13 @@ public class KbossBackupProviderTest {
InternalBackupJoinVO child = mock(InternalBackupJoinVO.class); InternalBackupJoinVO child = mock(InternalBackupJoinVO.class);
doReturn(true).when(child).getCurrent(); doReturn(true).when(child).getCurrent();
doReturn(List.of(child)).when(kbossBackupProviderSpy).getBackupJoinChildren(any()); doReturn(List.of(child)).when(kbossBackupProviderSpy).getBackupJoinChildren(any());
doReturn(userVmVOMock).when(userVmDaoMock).findById(anyLong()); doReturn(userVmVOMock).when(userVmDaoMock).findByIdIncludingRemoved(anyLong());
doReturn(true).when(kbossBackupProviderSpy).endBackupChain(any(), any()); doReturn(true).when(kbossBackupProviderSpy).endBackupChain(any());
doNothing().when(kbossBackupProviderSpy).validateVmState(any(), any(), any(), any());
kbossBackupProviderSpy.endBackupChainIfConfigured(backupVoMock); kbossBackupProviderSpy.endBackupChainIfConfigured(backupVoMock);
verify(kbossBackupProviderSpy, times(1)).endBackupChain(eq(userVmVOMock), anyLong()); verify(kbossBackupProviderSpy, times(1)).endBackupChain(userVmVOMock);
} }
@ -1687,7 +1730,7 @@ public class KbossBackupProviderTest {
doReturn(parentId).when(internalBackupJoinVoMock).getParentId(); doReturn(parentId).when(internalBackupJoinVoMock).getParentId();
doReturn(null).when(internalBackupJoinDaoMock).findById(parentId); doReturn(null).when(internalBackupJoinDaoMock).findById(parentId);
doReturn(List.of()).when(internalBackupDataStoreDaoMock).listByBackupId(anyLong()); doReturn(List.of()).when(internalBackupDataStoreDaoMock).listByBackupId(anyLong());
doNothing().when(kbossBackupProviderSpy).configureKbossTosForCleanup(any(), any(), any(), any(), any(), any(),anyBoolean()); doNothing().when(kbossBackupProviderSpy).configureKbossTosForCleanup(any(), any(), any(), anyBoolean(), any(), any());
doReturn(null).when(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any()); doReturn(null).when(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any());
boolean result = kbossBackupProviderSpy.normalizeBackupErrorAndFinishChain(userVmVOMock); boolean result = kbossBackupProviderSpy.normalizeBackupErrorAndFinishChain(userVmVOMock);
@ -1706,7 +1749,7 @@ public class KbossBackupProviderTest {
doReturn(parentId).when(internalBackupJoinVoMock).getParentId(); doReturn(parentId).when(internalBackupJoinVoMock).getParentId();
doReturn(null).when(internalBackupJoinDaoMock).findById(parentId); doReturn(null).when(internalBackupJoinDaoMock).findById(parentId);
doReturn(List.of()).when(internalBackupDataStoreDaoMock).listByBackupId(anyLong()); doReturn(List.of()).when(internalBackupDataStoreDaoMock).listByBackupId(anyLong());
doNothing().when(kbossBackupProviderSpy).configureKbossTosForCleanup(any(), any(), any(), any(), any(), any(), anyBoolean()); doNothing().when(kbossBackupProviderSpy).configureKbossTosForCleanup(any(), any(), any(), anyBoolean(), any(), any());
doReturn(answerMock).when(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any()); doReturn(answerMock).when(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any());
doReturn(false).when(answerMock).getResult(); doReturn(false).when(answerMock).getResult();
@ -1717,8 +1760,6 @@ public class KbossBackupProviderTest {
@Test @Test
public void normalizeBackupErrorAndFinishChainTestSuccessCallsEndChain() { public void normalizeBackupErrorAndFinishChainTestSuccessCallsEndChain() {
doReturn(userVmVOMock).when(userVmDaoMock).findById(any());
doReturn(VirtualMachine.State.Running).when(userVmVOMock).getState();
doReturn(null).when(vmInstanceDetailsDaoMock).findDetail(anyLong(), any()); doReturn(null).when(vmInstanceDetailsDaoMock).findDetail(anyLong(), any());
doReturn(backupVoMock).when(backupDaoMock).findLatestByStatusAndVmId(any(), anyLong()); doReturn(backupVoMock).when(backupDaoMock).findLatestByStatusAndVmId(any(), anyLong());
doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findById(anyLong()); doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findById(anyLong());
@ -1728,23 +1769,21 @@ public class KbossBackupProviderTest {
doReturn(parentId).when(internalBackupJoinVoMock).getParentId(); doReturn(parentId).when(internalBackupJoinVoMock).getParentId();
doReturn(null).when(internalBackupJoinDaoMock).findById(parentId); doReturn(null).when(internalBackupJoinDaoMock).findById(parentId);
doReturn(List.of()).when(internalBackupDataStoreDaoMock).listByBackupId(anyLong()); doReturn(List.of()).when(internalBackupDataStoreDaoMock).listByBackupId(anyLong());
doNothing().when(kbossBackupProviderSpy).configureKbossTosForCleanup(any(), any(), any(), any(), any(), any(), anyBoolean()); doNothing().when(kbossBackupProviderSpy).configureKbossTosForCleanup(any(), any(), any(), anyBoolean(), any(), any());
doReturn(answerMock).when(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any()); doReturn(answerMock).when(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any());
doReturn(true).when(answerMock).getResult(); doReturn(true).when(answerMock).getResult();
doReturn(false).when(kbossBackupProviderSpy).processCleanupBackupErrorAnswer(any(), any(), any(), any(), any()); doReturn(false).when(kbossBackupProviderSpy).processCleanupBackupErrorAnswer(any(), any());
doReturn(true).when(kbossBackupProviderSpy).endBackupChain(any());
boolean result = kbossBackupProviderSpy.normalizeBackupErrorAndFinishChain(userVmVOMock); boolean result = kbossBackupProviderSpy.normalizeBackupErrorAndFinishChain(userVmVOMock);
assertTrue(result); assertTrue(result);
verify(kbossBackupProviderSpy).mergeCurrentBackupDeltas(internalBackupJoinVoMock); verify(kbossBackupProviderSpy).endBackupChain(userVmVOMock);
verify(kbossBackupProviderSpy).finishBackupChains(userVmVOMock);
} }
@Test @Test
public void normalizeBackupErrorAndFinishChainTestChainAlreadyEnded() { public void normalizeBackupErrorAndFinishChainTestChainAlreadyEnded() {
doReturn(userVmVOMock).when(userVmDaoMock).findById(any());
doReturn(VirtualMachine.State.Running).when(userVmVOMock).getState();
doReturn(null).when(vmInstanceDetailsDaoMock).findDetail(anyLong(), any()); doReturn(null).when(vmInstanceDetailsDaoMock).findDetail(anyLong(), any());
doReturn(backupVoMock).when(backupDaoMock).findLatestByStatusAndVmId(any(), anyLong()); doReturn(backupVoMock).when(backupDaoMock).findLatestByStatusAndVmId(any(), anyLong());
doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findById(anyLong()); doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findById(anyLong());
@ -1754,12 +1793,12 @@ public class KbossBackupProviderTest {
doReturn(parentId).when(internalBackupJoinVoMock).getParentId(); doReturn(parentId).when(internalBackupJoinVoMock).getParentId();
doReturn(null).when(internalBackupJoinDaoMock).findById(parentId); doReturn(null).when(internalBackupJoinDaoMock).findById(parentId);
doReturn(List.of()).when(internalBackupDataStoreDaoMock).listByBackupId(anyLong()); doReturn(List.of()).when(internalBackupDataStoreDaoMock).listByBackupId(anyLong());
doNothing().when(kbossBackupProviderSpy).configureKbossTosForCleanup(any(), any(), any(), any(), any(), any(), anyBoolean()); doNothing().when(kbossBackupProviderSpy).configureKbossTosForCleanup(any(), any(), any(), anyBoolean(), any(), any());
doReturn(answerMock).when(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any()); doReturn(answerMock).when(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any());
doReturn(true).when(answerMock).getResult(); doReturn(true).when(answerMock).getResult();
doReturn(true).when(kbossBackupProviderSpy).processCleanupBackupErrorAnswer(any(), any(), any(), any(), any()); doReturn(true).when(kbossBackupProviderSpy).processCleanupBackupErrorAnswer(any(), any());
InternalBackupJoinVO current = mock(InternalBackupJoinVO.class); InternalBackupJoinVO current = mock(InternalBackupJoinVO.class);
doReturn(current).when(internalBackupJoinDaoMock).findCurrent(anyLong(), any()); doReturn(current).when(internalBackupJoinDaoMock).findCurrent(anyLong());
doNothing().when(internalBackupStoragePoolDaoMock).expungeByBackupId(anyLong()); doNothing().when(internalBackupStoragePoolDaoMock).expungeByBackupId(anyLong());
doNothing().when(kbossBackupProviderSpy).setEndOfChainAndRemoveCurrentForBackup(any()); doNothing().when(kbossBackupProviderSpy).setEndOfChainAndRemoveCurrentForBackup(any());
@ -1992,53 +2031,57 @@ public class KbossBackupProviderTest {
@Test @Test
public void mergeCurrentDeltaIntoVolumeTestNoDeltaDoesNothing() { public void mergeCurrentDeltaIntoVolumeTestNoDeltaDoesNothing() {
doReturn(volumeId).when(volumeVoMock).getId(); doReturn(volumeId).when(volumeVoMock).getId();
doReturn(List.of()).when(internalBackupJoinDaoMock).listCurrentsByVolumeIdDesc(volumeId); doReturn(null).when(internalBackupStoragePoolDaoMock).findOneByVolumeId(volumeId);
kbossBackupProviderSpy.mergeCurrentDeltasIntoVolume(volumeVoMock, virtualMachineMock, "detach", true); kbossBackupProviderSpy.mergeCurrentDeltaIntoVolume(volumeVoMock, virtualMachineMock, "detach", true);
verify(internalBackupJoinDaoMock, times(1)).listCurrentsByVolumeIdDesc(volumeId); verify(internalBackupStoragePoolDaoMock, times(1)).findOneByVolumeId(volumeId);
verify(internalBackupJoinDaoMock, never()).findById(anyLong()); verify(internalBackupJoinDaoMock, never()).findById(anyLong());
} }
@Test (expected = CloudRuntimeException.class) @Test (expected = CloudRuntimeException.class)
public void mergeCurrentDeltaIntoVolumeTestNullAnswer() { public void mergeCurrentDeltaIntoVolumeTestNullAnswer() {
doReturn(volumeId).when(volumeVoMock).getId(); doReturn(volumeId).when(volumeVoMock).getId();
doReturn(List.of(internalBackupJoinVoMock)).when(internalBackupJoinDaoMock).listCurrentsByVolumeIdDesc(volumeId); doReturn(internalBackupStoragePoolVoMock).when(internalBackupStoragePoolDaoMock).findOneByVolumeId(volumeId);
doReturn(deltaMergeTreeToMock).when(kbossBackupProviderSpy).createDeltaMergeTree(anyBoolean(), anyBoolean(), any(), any(), any(), any()); doReturn(backupId).when(internalBackupStoragePoolVoMock).getBackupId();
doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findById(backupId);
doReturn(null).when(kbossBackupProviderSpy).getSucceedingVmSnapshot(internalBackupJoinVoMock);
doReturn(deltaMergeTreeToMock).when(kbossBackupProviderSpy).createDeltaMergeTree(anyBoolean(), anyBoolean(), any(), any(), any());
doReturn(null).when(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any()); doReturn(null).when(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any());
try (MockedStatic<VolumeObject> volumeObjectMockedStatic = Mockito.mockStatic(VolumeObject.class)) { try (MockedStatic<VolumeObject> volumeObjectMockedStatic = Mockito.mockStatic(VolumeObject.class)) {
when(VolumeObject.getVolumeObject(any(), any())).thenReturn(volumeObjectMock); when(VolumeObject.getVolumeObject(any(), any())).thenReturn(volumeObjectMock);
kbossBackupProviderSpy.mergeCurrentDeltasIntoVolume(volumeVoMock, virtualMachineMock, "detach", true); kbossBackupProviderSpy.mergeCurrentDeltaIntoVolume(volumeVoMock, virtualMachineMock, "detach", true);
verify(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any()); verify(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any());
verify(kbossBackupProviderSpy, never()).expungeOldDeltasAndUpdateVmSnapshotOrBackup(anyList(), any(), any()); verify(kbossBackupProviderSpy, never()).expungeOldDeltasAndUpdateVmSnapshotIfNeeded(anyList(), any());
} }
} }
@Test @Test
public void mergeCurrentDeltaIntoVolumeTestNoSucceedingSnapshot() { public void mergeCurrentDeltaIntoVolumeTestNoSucceedingSnapshot() {
doReturn(volumeId).when(volumeVoMock).getId(); doReturn(volumeId).when(volumeVoMock).getId();
doReturn(List.of(internalBackupJoinVoMock)).when(internalBackupJoinDaoMock).listCurrentsByVolumeIdDesc(volumeId); doReturn(internalBackupStoragePoolVoMock).when(internalBackupStoragePoolDaoMock).findOneByVolumeId(volumeId);
doReturn(backupId).when(internalBackupStoragePoolVoMock).getBackupId(); doReturn(backupId).when(internalBackupStoragePoolVoMock).getBackupId();
doReturn(deltaMergeTreeToMock).when(kbossBackupProviderSpy).createDeltaMergeTree(anyBoolean(), anyBoolean(), any(), any(), any(), any()); doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findById(backupId);
doReturn(null).when(kbossBackupProviderSpy).getSucceedingVmSnapshot(internalBackupJoinVoMock);
doReturn(deltaMergeTreeToMock).when(kbossBackupProviderSpy).createDeltaMergeTree(anyBoolean(), anyBoolean(), any(), any(), any());
doReturn(answerMock).when(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any()); doReturn(answerMock).when(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any());
doReturn(true).when(answerMock).getResult(); doReturn(true).when(answerMock).getResult();
doReturn(volumeVoMock).when(volumeDaoMock).findById(anyLong()); doReturn(volumeVoMock).when(volumeDaoMock).findById(anyLong());
doReturn(backupDeltaToMock).when(deltaMergeTreeToMock).getParent(); doReturn(backupDeltaToMock).when(deltaMergeTreeToMock).getParent();
doNothing().when(kbossBackupProviderSpy).expungeOldDeltasAndUpdateVmSnapshotOrBackup(anyList(), any(), any()); doNothing().when(kbossBackupProviderSpy).expungeOldDeltasAndUpdateVmSnapshotIfNeeded(anyList(), any());
doReturn(List.of()).when(internalBackupStoragePoolDaoMock).listByBackupId(backupId); doReturn(List.of()).when(internalBackupStoragePoolDaoMock).listByBackupId(backupId);
doNothing().when(kbossBackupProviderSpy).setEndOfChainAndRemoveCurrentForBackup(any()); doNothing().when(kbossBackupProviderSpy).setEndOfChainAndRemoveCurrentForBackup(any());
doReturn(internalBackupStoragePoolVoMock).when(internalBackupStoragePoolDaoMock).findOneByVolumeIdAndBackupId(anyLong(), anyLong());
try (MockedStatic<VolumeObject> volumeObjectMockedStatic = Mockito.mockStatic(VolumeObject.class)) { try (MockedStatic<VolumeObject> volumeObjectMockedStatic = Mockito.mockStatic(VolumeObject.class)) {
when(VolumeObject.getVolumeObject(any(), any())).thenReturn(volumeObjectMock); when(VolumeObject.getVolumeObject(any(), any())).thenReturn(volumeObjectMock);
kbossBackupProviderSpy.mergeCurrentDeltasIntoVolume(volumeVoMock, virtualMachineMock, "detach", true); kbossBackupProviderSpy.mergeCurrentDeltaIntoVolume(volumeVoMock, virtualMachineMock, "detach", true);
verify(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any()); verify(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any());
verify(volumeDaoMock).update(volumeId, volumeVoMock); verify(volumeDaoMock).update(volumeId, volumeVoMock);
verify(kbossBackupProviderSpy).expungeOldDeltasAndUpdateVmSnapshotOrBackup(anyList(), any(), any()); verify(kbossBackupProviderSpy).expungeOldDeltasAndUpdateVmSnapshotIfNeeded(anyList(), any());
verify(kbossBackupProviderSpy).setEndOfChainAndRemoveCurrentForBackup(any()); verify(kbossBackupProviderSpy).setEndOfChainAndRemoveCurrentForBackup(any());
} }
} }
@ -2046,24 +2089,24 @@ public class KbossBackupProviderTest {
@Test @Test
public void mergeCurrentDeltaIntoVolumeTestWithSucceedingSnapshotWithMoreDeltas() { public void mergeCurrentDeltaIntoVolumeTestWithSucceedingSnapshotWithMoreDeltas() {
doReturn(volumeId).when(volumeVoMock).getId(); doReturn(volumeId).when(volumeVoMock).getId();
doReturn(List.of(internalBackupJoinVoMock)).when(internalBackupJoinDaoMock).listCurrentsByVolumeIdDesc(volumeId); doReturn(internalBackupStoragePoolVoMock).when(internalBackupStoragePoolDaoMock).findOneByVolumeId(volumeId);
doReturn(backupId).when(internalBackupStoragePoolVoMock).getBackupId(); doReturn(backupId).when(internalBackupStoragePoolVoMock).getBackupId();
doReturn(deltaMergeTreeToMock).when(kbossBackupProviderSpy).createDeltaMergeTree(anyBoolean(), anyBoolean(), any(), any(), any(), any()); doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findById(backupId);
doReturn(backupDeltaToMock).when(deltaMergeTreeToMock).getParent(); doReturn(vmSnapshotVoMock).when(kbossBackupProviderSpy).getSucceedingVmSnapshot(internalBackupJoinVoMock);
doReturn(deltaMergeTreeToMock).when(kbossBackupProviderSpy).createDeltaMergeTree(anyBoolean(), anyBoolean(), any(), any(), any());
doReturn(answerMock).when(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any()); doReturn(answerMock).when(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any());
doReturn(true).when(answerMock).getResult(); doReturn(true).when(answerMock).getResult();
doNothing().when(kbossBackupProviderSpy).expungeOldDeltasAndUpdateVmSnapshotOrBackup(anyList(), any(), any()); doNothing().when(kbossBackupProviderSpy).expungeOldDeltasAndUpdateVmSnapshotIfNeeded(anyList(), any());
doReturn(List.of(internalBackupStoragePoolVoMock)).when(internalBackupStoragePoolDaoMock).listByBackupId(backupId); doReturn(List.of(internalBackupStoragePoolVoMock)).when(internalBackupStoragePoolDaoMock).listByBackupId(backupId);
doReturn(volumeVoMock).when(volumeDaoMock).findById(anyLong());
doReturn(internalBackupStoragePoolVoMock).when(internalBackupStoragePoolDaoMock).findOneByVolumeIdAndBackupId(anyLong(), anyLong());
try (MockedStatic<VolumeObject> volumeObjectMockedStatic = Mockito.mockStatic(VolumeObject.class)) { try (MockedStatic<VolumeObject> volumeObjectMockedStatic = Mockito.mockStatic(VolumeObject.class)) {
when(VolumeObject.getVolumeObject(any(), any())).thenReturn(volumeObjectMock); when(VolumeObject.getVolumeObject(any(), any())).thenReturn(volumeObjectMock);
kbossBackupProviderSpy.mergeCurrentDeltasIntoVolume(volumeVoMock, virtualMachineMock, "detach", true); kbossBackupProviderSpy.mergeCurrentDeltaIntoVolume(volumeVoMock, virtualMachineMock, "detach", true);
verify(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any()); verify(kbossBackupProviderSpy).sendBackupCommand(anyLong(), any());
verify(kbossBackupProviderSpy).expungeOldDeltasAndUpdateVmSnapshotOrBackup(anyList(), any(), any()); verify(volumeDaoMock, never()).update(volumeId, volumeVoMock);
verify(kbossBackupProviderSpy).expungeOldDeltasAndUpdateVmSnapshotIfNeeded(anyList(), any());
verify(kbossBackupProviderSpy, never()).setEndOfChainAndRemoveCurrentForBackup(any()); verify(kbossBackupProviderSpy, never()).setEndOfChainAndRemoveCurrentForBackup(any());
} }
} }
@ -2143,13 +2186,90 @@ public class KbossBackupProviderTest {
kbossBackupProviderSpy.getHostToRestore(virtualMachineMock, true, 55L); kbossBackupProviderSpy.getHostToRestore(virtualMachineMock, true, 55L);
} }
@Test
public void gatherSnapshotReferencesOfChildrenSnapshotTestVmSnapshotIsNull() {
List<VolumeObjectTO> volumeObjectTOs = List.of(volumeObjectToMock);
Map<Long, List<SnapshotDataStoreVO>> result = kbossBackupProviderSpy.gatherSnapshotReferencesOfChildrenSnapshot(volumeObjectTOs, null);
assertTrue(result.isEmpty());
verify(vmSnapshotDaoMock, never()).listByParent(anyLong());
verify(vmSnapshotHelperMock, never()).getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(anyLong());
verify(kbossBackupProviderSpy, never()).mapVolumesToSnapshotReferences(anyList(), anyList(), anyMap());
}
@Test
public void gatherSnapshotReferencesOfChildrenSnapshotTestChildrenListIsEmpty() {
doReturn(100L).when(vmSnapshotVoMock).getId();
doReturn(List.of()).when(vmSnapshotDaoMock).listByParent(100L);
List<VolumeObjectTO> volumeObjectTOs = List.of(volumeObjectToMock);
Map<Long, List<SnapshotDataStoreVO>> result = kbossBackupProviderSpy.gatherSnapshotReferencesOfChildrenSnapshot(volumeObjectTOs, vmSnapshotVoMock);
assertTrue(result.isEmpty());
verify(vmSnapshotDaoMock, times(1)).listByParent(100L);
verify(vmSnapshotHelperMock, never()).getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(anyLong());
verify(kbossBackupProviderSpy, never()).mapVolumesToSnapshotReferences(anyList(), anyList(), anyMap());
}
@Test
public void gatherSnapshotReferencesOfChildrenSnapshotTestSingleChildWithSingleSnapshotReference() {
VMSnapshotVO childSnapshot = Mockito.mock(VMSnapshotVO.class);
SnapshotDataStoreVO snapshotDataStoreVO = Mockito.mock(SnapshotDataStoreVO.class);
doReturn(100L).when(vmSnapshotVoMock).getId();
doReturn(List.of(childSnapshot)).when(vmSnapshotDaoMock).listByParent(100L);
doReturn(200L).when(childSnapshot).getId();
doReturn(List.of(snapshotDataStoreVO)).when(vmSnapshotHelperMock).getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(200L);
doNothing().when(kbossBackupProviderSpy).mapVolumesToSnapshotReferences(anyList(), anyList(), anyMap());
List<VolumeObjectTO> volumeObjectTOs = List.of(volumeObjectToMock);
Map<Long, List<SnapshotDataStoreVO>> result =
kbossBackupProviderSpy.gatherSnapshotReferencesOfChildrenSnapshot(volumeObjectTOs, vmSnapshotVoMock);
assertTrue(result.isEmpty());
verify(vmSnapshotDaoMock, times(1)).listByParent(100L);
verify(vmSnapshotHelperMock, times(1)).getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(200L);
verify(kbossBackupProviderSpy, times(1)).mapVolumesToSnapshotReferences(eq(volumeObjectTOs), anyList(), anyMap());
}
@Test
public void gatherSnapshotReferencesOfChildrenSnapshotTestMultipleChildrenAggregatesSnapshotReferences() {
VMSnapshotVO childSnapshot1 = Mockito.mock(VMSnapshotVO.class);
VMSnapshotVO childSnapshot2 = Mockito.mock(VMSnapshotVO.class);
SnapshotDataStoreVO snapshotDataStoreVO1 = Mockito.mock(SnapshotDataStoreVO.class);
SnapshotDataStoreVO snapshotDataStoreVO2 = Mockito.mock(SnapshotDataStoreVO.class);
doReturn(100L).when(vmSnapshotVoMock).getId();
doReturn(List.of(childSnapshot1, childSnapshot2)).when(vmSnapshotDaoMock).listByParent(100L);
doReturn(201L).when(childSnapshot1).getId();
doReturn(202L).when(childSnapshot2).getId();
doReturn(List.of(snapshotDataStoreVO1)).when(vmSnapshotHelperMock)
.getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(201L);
doReturn(List.of(snapshotDataStoreVO2)).when(vmSnapshotHelperMock)
.getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(202L);
doNothing().when(kbossBackupProviderSpy).mapVolumesToSnapshotReferences(anyList(), anyList(), anyMap());
List<VolumeObjectTO> volumeObjectTOs = List.of(volumeObjectToMock);
Map<Long, List<SnapshotDataStoreVO>> result =
kbossBackupProviderSpy.gatherSnapshotReferencesOfChildrenSnapshot(volumeObjectTOs, vmSnapshotVoMock);
assertTrue(result.isEmpty());
verify(vmSnapshotHelperMock, times(1)).getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(201L);
verify(vmSnapshotHelperMock, times(1)).getVolumeSnapshotsAssociatedWithKvmDiskOnlyVmSnapshot(202L);
verify(kbossBackupProviderSpy, times(1)).mapVolumesToSnapshotReferences(eq(volumeObjectTOs), anyList(), anyMap());
}
@Test @Test
public void createDeltaMergeTreeTestChildIsVolumeWithoutSucceedingSnapshot() { public void createDeltaMergeTreeTestChildIsVolumeWithoutSucceedingSnapshot() {
doReturn(dataStoreMock).when(dataStoreManagerMock).getDataStore(anyLong(), eq(DataStoreRole.Primary)); doReturn(dataStoreMock).when(dataStoreManagerMock).getDataStore(anyLong(), eq(DataStoreRole.Primary));
doReturn("parent-path").when(internalBackupStoragePoolVoMock).getBackupDeltaParentPath(); doReturn("parent-path").when(internalBackupStoragePoolVoMock).getBackupDeltaParentPath();
DeltaMergeTreeTO result = kbossBackupProviderSpy.createDeltaMergeTree(true, true, internalBackupStoragePoolVoMock, DeltaMergeTreeTO result = kbossBackupProviderSpy.createDeltaMergeTree(true, true, internalBackupStoragePoolVoMock,
volumeObjectToMock, null, null); volumeObjectToMock, null);
assertEquals(volumeObjectToMock, result.getVolumeObjectTO()); assertEquals(volumeObjectToMock, result.getVolumeObjectTO());
assertTrue(result.getGrandChildren().isEmpty()); assertTrue(result.getGrandChildren().isEmpty());
@ -2164,7 +2284,7 @@ public class KbossBackupProviderTest {
doReturn("child-path").when(internalBackupStoragePoolVoMock).getBackupDeltaPath(); doReturn("child-path").when(internalBackupStoragePoolVoMock).getBackupDeltaPath();
DeltaMergeTreeTO result = kbossBackupProviderSpy.createDeltaMergeTree(false, true, internalBackupStoragePoolVoMock, DeltaMergeTreeTO result = kbossBackupProviderSpy.createDeltaMergeTree(false, true, internalBackupStoragePoolVoMock,
volumeObjectToMock, null, null); volumeObjectToMock, null);
assertEquals(volumeObjectToMock, result.getVolumeObjectTO()); assertEquals(volumeObjectToMock, result.getVolumeObjectTO());
assertEquals("parent-path", result.getParent().getPath()); assertEquals("parent-path", result.getParent().getPath());
@ -2180,14 +2300,16 @@ public class KbossBackupProviderTest {
doReturn("parent-path").when(internalBackupStoragePoolVoMock).getBackupDeltaParentPath(); doReturn("parent-path").when(internalBackupStoragePoolVoMock).getBackupDeltaParentPath();
doReturn("child-path").when(internalBackupStoragePoolVoMock).getBackupDeltaPath(); doReturn("child-path").when(internalBackupStoragePoolVoMock).getBackupDeltaPath();
doReturn(volumeId).when(volumeObjectToMock).getVolumeId(); doReturn(volumeId).when(volumeObjectToMock).getVolumeId();
doReturn("path").when(volumeObjectToMock).getPath(); doReturn("snapshot-grandchild").when(snapshotRefMock).getInstallPath();
DeltaMergeTreeTO result = kbossBackupProviderSpy.createDeltaMergeTree(false, false, internalBackupStoragePoolVoMock, doReturn(Map.of(volumeId, List.of(snapshotRefMock))).when(kbossBackupProviderSpy).gatherSnapshotReferencesOfChildrenSnapshot(List.of(volumeObjectToMock), vmSnapshotVoMock);
volumeObjectToMock, vmSnapshotVoMock, List.of());
DeltaMergeTreeTO result = kbossBackupProviderSpy.createDeltaMergeTree(false, true, internalBackupStoragePoolVoMock,
volumeObjectToMock, vmSnapshotVoMock);
assertEquals("child-path", result.getChild().getPath()); assertEquals("child-path", result.getChild().getPath());
assertEquals(1, result.getGrandChildren().size()); assertEquals(1, result.getGrandChildren().size());
assertEquals("path", result.getGrandChildren().get(0).getPath()); assertEquals("snapshot-grandchild", result.getGrandChildren().get(0).getPath());
} }
@Test @Test
@ -2197,8 +2319,11 @@ public class KbossBackupProviderTest {
doReturn("child-path").when(internalBackupStoragePoolVoMock).getBackupDeltaPath(); doReturn("child-path").when(internalBackupStoragePoolVoMock).getBackupDeltaPath();
doReturn("/volume/path").when(volumeObjectToMock).getPath(); doReturn("/volume/path").when(volumeObjectToMock).getPath();
doReturn(Map.of()).when(kbossBackupProviderSpy)
.gatherSnapshotReferencesOfChildrenSnapshot(List.of(volumeObjectToMock), vmSnapshotVoMock);
DeltaMergeTreeTO result = kbossBackupProviderSpy.createDeltaMergeTree(false, false, internalBackupStoragePoolVoMock, DeltaMergeTreeTO result = kbossBackupProviderSpy.createDeltaMergeTree(false, false, internalBackupStoragePoolVoMock,
volumeObjectToMock, vmSnapshotVoMock, List.of()); volumeObjectToMock, vmSnapshotVoMock);
assertEquals(1, result.getGrandChildren().size()); assertEquals(1, result.getGrandChildren().size());
assertEquals("/volume/path", result.getGrandChildren().get(0).getPath()); assertEquals("/volume/path", result.getGrandChildren().get(0).getPath());
@ -2275,16 +2400,14 @@ public class KbossBackupProviderTest {
Set<BackupDeltaTO> deltasToRemove = new java.util.HashSet<>(); Set<BackupDeltaTO> deltasToRemove = new java.util.HashSet<>();
doReturn(deltaMergeTreeToMock).when(kbossBackupProviderSpy).createDeltaMergeTree(eq(true), eq(false), eq(internalBackupStoragePoolVoMock), eq(volumeObjectToMock), eq(null), doReturn(deltaMergeTreeToMock).when(kbossBackupProviderSpy).createDeltaMergeTree(true, false, internalBackupStoragePoolVoMock, volumeObjectToMock, null);
eq(new ArrayList<>()));
List<DeltaMergeTreeTO> result = kbossBackupProviderSpy.populateDeltasToRemoveAndToMergeAndUpdateVolumePaths(List.of(internalBackupStoragePoolVoMock), deltasToRemove, List<DeltaMergeTreeTO> result = kbossBackupProviderSpy.populateDeltasToRemoveAndToMergeAndUpdateVolumePaths(List.of(internalBackupStoragePoolVoMock), deltasToRemove,
List.of(volumeObjectToMock), List.of(volumeObjectToMock), "vm-uuid"); List.of(volumeObjectToMock), List.of(volumeObjectToMock), "vm-uuid");
assertEquals(List.of(deltaMergeTreeToMock), result); assertEquals(List.of(deltaMergeTreeToMock), result);
assertTrue(deltasToRemove.isEmpty()); assertTrue(deltasToRemove.isEmpty());
verify(kbossBackupProviderSpy, times(1)).createDeltaMergeTree(eq(true), eq(false), eq(internalBackupStoragePoolVoMock), eq(volumeObjectToMock), eq(null), verify(kbossBackupProviderSpy, times(1)).createDeltaMergeTree(true, false, internalBackupStoragePoolVoMock, volumeObjectToMock, null);
eq(new ArrayList<>()));
verify(dataStoreManagerMock, never()).getDataStore(anyLong(), eq(DataStoreRole.Primary)); verify(dataStoreManagerMock, never()).getDataStore(anyLong(), eq(DataStoreRole.Primary));
} }
@ -2384,7 +2507,7 @@ public class KbossBackupProviderTest {
List<Long> removedBackupIds = new ArrayList<>(List.of(backupId, 200L)); List<Long> removedBackupIds = new ArrayList<>(List.of(backupId, 200L));
boolean result = kbossBackupProviderSpy.processRemoveBackupFailures(false, deleteAnswers, removedBackupIds, internalBackupJoinVoMock, virtualMachineMock); boolean result = kbossBackupProviderSpy.processRemoveBackupFailures(false, deleteAnswers, removedBackupIds, internalBackupJoinVoMock);
assertTrue(result); assertTrue(result);
assertEquals(List.of(backupId, 200L), removedBackupIds); assertEquals(List.of(backupId, 200L), removedBackupIds);
@ -2403,11 +2526,10 @@ public class KbossBackupProviderTest {
doReturn(backupId).when(backupVoMock).getId(); doReturn(backupId).when(backupVoMock).getId();
doReturn(backupVoMock).when(backupDaoMock).findByIdIncludingRemoved(backupId); doReturn(backupVoMock).when(backupDaoMock).findByIdIncludingRemoved(backupId);
doReturn(VirtualMachine.State.Stopped).when(virtualMachineMock).getState();
List<Long> removedBackupIds = new ArrayList<>(List.of(backupId, 200L)); List<Long> removedBackupIds = new ArrayList<>(List.of(backupId, 200L));
boolean result = kbossBackupProviderSpy.processRemoveBackupFailures(false, new Answer[]{failedCurrentBackupAnswer}, removedBackupIds, internalBackupJoinVoMock, virtualMachineMock); boolean result = kbossBackupProviderSpy.processRemoveBackupFailures(false, new Answer[]{failedCurrentBackupAnswer}, removedBackupIds, internalBackupJoinVoMock);
assertFalse(result); assertFalse(result);
assertEquals(List.of(200L), removedBackupIds); assertEquals(List.of(200L), removedBackupIds);
@ -2426,7 +2548,7 @@ public class KbossBackupProviderTest {
List<Long> removedBackupIds = new ArrayList<>(List.of(backupId, 200L)); List<Long> removedBackupIds = new ArrayList<>(List.of(backupId, 200L));
boolean result = kbossBackupProviderSpy.processRemoveBackupFailures(true, new Answer[]{failedCurrentBackupAnswer}, removedBackupIds, internalBackupJoinVoMock, virtualMachineMock); boolean result = kbossBackupProviderSpy.processRemoveBackupFailures(true, new Answer[]{failedCurrentBackupAnswer}, removedBackupIds, internalBackupJoinVoMock);
assertFalse(result); assertFalse(result);
assertEquals(List.of(200L), removedBackupIds); assertEquals(List.of(200L), removedBackupIds);
@ -2449,7 +2571,7 @@ public class KbossBackupProviderTest {
List<Long> removedBackupIds = new ArrayList<>(List.of(backupId, 200L)); List<Long> removedBackupIds = new ArrayList<>(List.of(backupId, 200L));
boolean result = kbossBackupProviderSpy.processRemoveBackupFailures(false, new Answer[]{failedOtherBackupAnswer}, removedBackupIds, internalBackupJoinVoMock, virtualMachineMock); boolean result = kbossBackupProviderSpy.processRemoveBackupFailures(false, new Answer[]{failedOtherBackupAnswer}, removedBackupIds, internalBackupJoinVoMock);
assertFalse(result); assertFalse(result);
assertEquals(List.of(backupId), removedBackupIds); assertEquals(List.of(backupId), removedBackupIds);

View File

@ -550,7 +550,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co
} }
@Override @Override
public Pair<Boolean, Backup> takeBackup(final VirtualMachine vm, Boolean quiesceVM, boolean isolated, Long scheduleId) { public Pair<Boolean, Backup> takeBackup(final VirtualMachine vm, Boolean quiesceVM, boolean isolated) {
final Host host = getVMHypervisorHostForBackup(vm); final Host host = getVMHypervisorHostForBackup(vm);
final BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(vm.getBackupOfferingId()); final BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(vm.getBackupOfferingId());

View File

@ -251,7 +251,7 @@ public class NASBackupProviderTest {
Mockito.when(backupDao.persist(Mockito.any(BackupVO.class))).thenAnswer(invocation -> invocation.getArgument(0)); Mockito.when(backupDao.persist(Mockito.any(BackupVO.class))).thenAnswer(invocation -> invocation.getArgument(0));
Mockito.when(backupDao.update(Mockito.anyLong(), Mockito.any(BackupVO.class))).thenReturn(true); Mockito.when(backupDao.update(Mockito.anyLong(), Mockito.any(BackupVO.class))).thenReturn(true);
Pair<Boolean, Backup> result = nasBackupProvider.takeBackup(vm, false, false, null); Pair<Boolean, Backup> result = nasBackupProvider.takeBackup(vm, false, false);
Assert.assertTrue(result.first()); Assert.assertTrue(result.first());
Assert.assertNotNull(result.second()); Assert.assertNotNull(result.second());

View File

@ -492,7 +492,7 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid
} }
@Override @Override
public Pair<Boolean, Backup> takeBackup(VirtualMachine vm, Boolean quiesceVM, boolean isolated, Long scheduleId) { public Pair<Boolean, Backup> takeBackup(VirtualMachine vm, Boolean quiesceVM, boolean isolated) {
String networkerServer; String networkerServer;
String clusterName; String clusterName;

View File

@ -219,7 +219,7 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider,
} }
@Override @Override
public Pair<Boolean, Backup> takeBackup(final VirtualMachine vm, Boolean quiesceVM, boolean isolated, Long scheduleId) { public Pair<Boolean, Backup> takeBackup(final VirtualMachine vm, Boolean quiesceVM, boolean isolated) {
final VeeamClient client = getClient(vm.getDataCenterId()); final VeeamClient client = getClient(vm.getDataCenterId());
Boolean result = client.startBackupJob(vm.getBackupExternalId()); Boolean result = client.startBackupJob(vm.getBackupExternalId());
return new Pair<>(result, null); return new Pair<>(result, null);

View File

@ -16,16 +16,17 @@
// under the License. // under the License.
package com.cloud.hypervisor.kvm.resource.wrapper; package com.cloud.hypervisor.kvm.resource.wrapper;
import java.io.File; import com.cloud.agent.api.Answer;
import java.io.IOException; import com.cloud.hypervisor.Hypervisor;
import java.nio.file.Files; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
import java.nio.file.Path; import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser;
import java.util.ArrayList; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef;
import java.util.Arrays; import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk;
import java.util.HashMap; import com.cloud.hypervisor.kvm.storage.KVMStoragePool;
import java.util.List; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
import java.util.Map; import com.cloud.resource.CommandWrapper;
import com.cloud.resource.ResourceWrapper;
import com.cloud.utils.Pair;
import org.apache.cloudstack.backup.CleanupKbossBackupErrorAnswer; import org.apache.cloudstack.backup.CleanupKbossBackupErrorAnswer;
import org.apache.cloudstack.backup.CleanupKbossBackupErrorCommand; import org.apache.cloudstack.backup.CleanupKbossBackupErrorCommand;
import org.apache.cloudstack.storage.to.BackupDeltaTO; import org.apache.cloudstack.storage.to.BackupDeltaTO;
@ -39,19 +40,12 @@ import org.libvirt.Domain;
import org.libvirt.Error; import org.libvirt.Error;
import org.libvirt.LibvirtException; import org.libvirt.LibvirtException;
import com.cloud.agent.api.Answer; import java.io.File;
import com.cloud.agent.api.to.DataTO; import java.io.IOException;
import com.cloud.hypervisor.Hypervisor; import java.nio.file.Files;
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import java.nio.file.Path;
import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; import java.util.ArrayList;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; import java.util.List;
import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk;
import com.cloud.hypervisor.kvm.storage.KVMStoragePool;
import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
import com.cloud.resource.CommandWrapper;
import com.cloud.resource.ResourceWrapper;
import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
@ResourceWrapper(handles = CleanupKbossBackupErrorCommand.class) @ResourceWrapper(handles = CleanupKbossBackupErrorCommand.class)
public class LibvirtCleanupKbossVmBackupCommandWrapper extends CommandWrapper<CleanupKbossBackupErrorCommand, Answer, LibvirtComputingResource> { public class LibvirtCleanupKbossVmBackupCommandWrapper extends CommandWrapper<CleanupKbossBackupErrorCommand, Answer, LibvirtComputingResource> {
@ -64,14 +58,14 @@ public class LibvirtCleanupKbossVmBackupCommandWrapper extends CommandWrapper<Cl
cleanupBackupDeltasOnSecondary(command, storagePoolManager, kbossTOS); cleanupBackupDeltasOnSecondary(command, storagePoolManager, kbossTOS);
if (command.isRunningVM()) { if (command.isRunningVM()) {
Pair<Map<String, Pair<String,Boolean>>, Boolean> volumeTosAndIsVmRunning = cleanupRunningVm(command, serverResource); Pair<List<VolumeObjectTO>, Boolean> volumeTosAndIsVmRunning = cleanupRunningVm(command, serverResource);
return new CleanupKbossBackupErrorAnswer(command, volumeTosAndIsVmRunning.first(), volumeTosAndIsVmRunning.second()); return new CleanupKbossBackupErrorAnswer(command, volumeTosAndIsVmRunning.first(), volumeTosAndIsVmRunning.second());
} }
return new CleanupKbossBackupErrorAnswer(command, mergeDeltasForStoppedVmIfNeeded(command, serverResource), false); return new CleanupKbossBackupErrorAnswer(command, mergeDeltasForStoppedVmIfNeeded(command, serverResource), false);
} }
private Pair<Map<String, Pair<String,Boolean>>, Boolean> cleanupRunningVm(CleanupKbossBackupErrorCommand command, LibvirtComputingResource serverResource) { private Pair<List<VolumeObjectTO>, Boolean> cleanupRunningVm(CleanupKbossBackupErrorCommand command, LibvirtComputingResource serverResource) {
Domain dm = null; Domain dm = null;
try { try {
dm = serverResource.getDomain(serverResource.getLibvirtUtilitiesHelper().getConnection(), command.getVmName()); dm = serverResource.getDomain(serverResource.getLibvirtUtilitiesHelper().getConnection(), command.getVmName());
@ -81,7 +75,7 @@ public class LibvirtCleanupKbossVmBackupCommandWrapper extends CommandWrapper<Cl
return new Pair<>(mergeDeltasForStoppedVmIfNeeded(command, serverResource), false); return new Pair<>(mergeDeltasForStoppedVmIfNeeded(command, serverResource), false);
} }
logger.error("Error while trying to get VM [{}]. Aborting the process.", command.getVmName(), e); logger.error("Error while trying to get VM [{}]. Aborting the process.", command.getVmName(), e);
return new Pair<>(Map.of(), false); return new Pair<>(List.of(), false);
} finally { } finally {
if (dm != null) { if (dm != null) {
try { try {
@ -93,140 +87,87 @@ public class LibvirtCleanupKbossVmBackupCommandWrapper extends CommandWrapper<Cl
} }
} }
private Map<String, Pair<String,Boolean>> mergeDeltasForStoppedVmIfNeeded(CleanupKbossBackupErrorCommand command, LibvirtComputingResource serverResource) { private List<VolumeObjectTO> mergeDeltasForStoppedVmIfNeeded(CleanupKbossBackupErrorCommand command, LibvirtComputingResource serverResource) {
HashMap<String, Pair<String,Boolean>> volumeToChainEnded = new HashMap<>(); List<VolumeObjectTO> volumeObjectTOList = new ArrayList<>();
for (KbossTO kbossTO : command.getKbossTOs()) { for (KbossTO kbossTO : command.getKbossTOs()) {
VolumeObjectTO volumeObjectTO = kbossTO.getVolumeObjectTO(); VolumeObjectTO volumeObjectTO = kbossTO.getVolumeObjectTO();
PrimaryDataStoreTO primaryDataStoreTO = (PrimaryDataStoreTO)volumeObjectTO.getDataStore(); PrimaryDataStoreTO primaryDataStoreTO = (PrimaryDataStoreTO)volumeObjectTO.getDataStore();
KVMStoragePool kvmStoragePool = serverResource.getStoragePoolMgr().getStoragePool(primaryDataStoreTO.getPoolType(), primaryDataStoreTO.getUuid()); KVMStoragePool kvmStoragePool = serverResource.getStoragePoolMgr().getStoragePool(primaryDataStoreTO.getPoolType(), primaryDataStoreTO.getUuid());
boolean volumePathMissing = !Files.exists(Path.of(kvmStoragePool.getLocalPathFor(volumeObjectTO.getPath()))); boolean backupErrorDeltaExists = Files.exists(Path.of(kvmStoragePool.getLocalPathFor(volumeObjectTO.getPath())));
boolean deltaPathMissing = !Files.exists(Path.of(kvmStoragePool.getLocalPathFor(kbossTO.getDeltaPathOnPrimary()))); boolean parentBackupDeltaExists = Files.exists(Path.of(kvmStoragePool.getLocalPathFor(kbossTO.getDeltaPathOnPrimary())));
boolean basePathMissing = kbossTO.getParentDeltaPathOnPrimary() != null && !Files.exists(Path.of(kvmStoragePool.getLocalPathFor(kbossTO.getParentDeltaPathOnPrimary()))); boolean shouldBaseDeltaExist = kbossTO.getParentDeltaPathOnPrimary() != null;
List<DataTO> grandchildren = kbossTO.getDeltaPaths().isEmpty() ? List.of() : List.of(new BackupDeltaTO(volumeObjectTO.getDataStore(), boolean baseDeltaExists = shouldBaseDeltaExist && Files.exists(Path.of(kvmStoragePool.getLocalPathFor(kbossTO.getParentDeltaPathOnPrimary())));
Hypervisor.HypervisorType.KVM, kbossTO.getDeltaPaths().get(0)));
Boolean chainEnded = mergeDeltaIfNeeded(serverResource, kbossTO, volumeObjectTO, grandchildren, volumePathMissing, deltaPathMissing, basePathMissing, if(!mergeDeltaIfNeeded(serverResource, kbossTO, backupErrorDeltaExists, parentBackupDeltaExists, shouldBaseDeltaExist, baseDeltaExists,
command.isErrorOnCreate(), false, command.isTopDelta(), command.isEndOfChain()); false, volumeObjectTO, volumeObjectTOList)) {
volumeToChainEnded.put(volumeObjectTO.getUuid(), new Pair<>(volumeObjectTO.getPath(), chainEnded)); return List.of();
}
} }
return volumeToChainEnded; return volumeObjectTOList;
} }
private Map<String, Pair<String, Boolean>> mergeDeltasForRunningVmIfNeeded(CleanupKbossBackupErrorCommand command, LibvirtComputingResource serverResource, Domain dm) throws LibvirtException { private List<VolumeObjectTO> mergeDeltasForRunningVmIfNeeded(CleanupKbossBackupErrorCommand command, LibvirtComputingResource serverResource, Domain dm) throws LibvirtException {
HashMap<String, Pair<String, Boolean>> volumeIdToPathAndChainEnded = new HashMap<>();
String xmlDesc = dm.getXMLDesc(0); String xmlDesc = dm.getXMLDesc(0);
LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser(); LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser();
parser.parseDomainXML(xmlDesc); parser.parseDomainXML(xmlDesc);
List<VolumeObjectTO> volumeObjectTOList = new ArrayList<>();
for (KbossTO kbossTO : command.getKbossTOs()) { for (KbossTO kbossTO : command.getKbossTOs()) {
VolumeObjectTO volumeObjectTO = kbossTO.getVolumeObjectTO();
String volumePath = volumeObjectTO.getPath();
LibvirtVMDef.DiskDef diskDef = parser.getDisks().stream() LibvirtVMDef.DiskDef diskDef = parser.getDisks().stream()
.filter(disk -> hasPath(disk, volumePath, kbossTO.getDeltaPathOnPrimary(), kbossTO.getParentDeltaPathOnPrimary())) .filter(disk -> StringUtils.contains(disk.getDiskPath(), kbossTO.getVolumeObjectTO().getPath()) ||
.findFirst().orElse(null); StringUtils.contains(disk.getDiskPath(), kbossTO.getDeltaPathOnPrimary()) ||
StringUtils.contains(disk.getDiskPath(), kbossTO.getParentDeltaPathOnPrimary())).findFirst().orElse(null);
if (diskDef == null) { if (diskDef == null) {
logger.warn("Volume [{}] does not match any record we have. This must be manually normalized.", volumeObjectTO.getUuid()); logger.warn("Volume [{}] does not match any record we have. This must be manually normalized.", kbossTO.getVolumeObjectTO().getUuid());
return Map.of(); return List.of();
} }
List<String> backingStoreList = diskDef.getBackingStoreList(); boolean backupErrorDeltaExists = diskDef.getDiskPath().contains(kbossTO.getVolumeObjectTO().getPath());
backingStoreList.add(0, diskDef.getDiskPath()); boolean parentBackupDeltaExists = diskDef.getDiskPath().contains(kbossTO.getDeltaPathOnPrimary()) ||
diskDef.getBackingStoreList().stream().anyMatch(path -> path.contains(kbossTO.getDeltaPathOnPrimary()));
boolean shouldBaseDeltaExist = kbossTO.getParentDeltaPathOnPrimary() != null;
boolean baseDeltaExists = shouldBaseDeltaExist && StringUtils.contains(diskDef.getDiskPath(), kbossTO.getParentDeltaPathOnPrimary()) ||
diskDef.getBackingStoreList().stream().anyMatch(path -> StringUtils.contains(path, kbossTO.getParentDeltaPathOnPrimary()));
boolean volumePathMissing = true; mergeDeltaIfNeeded(serverResource, kbossTO, backupErrorDeltaExists, parentBackupDeltaExists, shouldBaseDeltaExist, baseDeltaExists, true,
boolean deltaPathMissing = true; kbossTO.getVolumeObjectTO(), volumeObjectTOList);
boolean basePathMissing = kbossTO.getParentDeltaPathOnPrimary() != null;
for (String delta : backingStoreList) {
if (StringUtils.contains(delta, volumePath)) {
volumePathMissing = false;
}
if (StringUtils.contains(delta, kbossTO.getDeltaPathOnPrimary())) {
deltaPathMissing = false;
}
if (StringUtils.contains(delta, kbossTO.getParentDeltaPathOnPrimary())) {
basePathMissing = false;
}
}
Boolean chainEnded = mergeDeltaIfNeeded(serverResource, kbossTO, volumeObjectTO, List.of(), volumePathMissing, deltaPathMissing, basePathMissing,
command.isErrorOnCreate(), true, command.isTopDelta(), command.isEndOfChain());
volumeIdToPathAndChainEnded.put(volumeObjectTO.getUuid(), new Pair<>(volumeObjectTO.getPath(), chainEnded));
} }
return volumeIdToPathAndChainEnded; return volumeObjectTOList;
} }
private boolean hasPath(LibvirtVMDef.DiskDef diskDef, String... paths) { private boolean mergeDeltaIfNeeded(LibvirtComputingResource serverResource, KbossTO kbossTO, boolean backupErrorDeltaExists,
List<String> chain = diskDef.getBackingStoreList(); boolean parentBackupDeltaExists, boolean shouldBaseDeltaExist, boolean baseDeltaExists, boolean runningVm, VolumeObjectTO volumeObjectTO,
chain = chain != null ? chain : new ArrayList<>(); List<VolumeObjectTO> volumeObjectTOList) {
chain.add(diskDef.getDiskPath());
for (String delta : chain) {
if (Arrays.stream(paths).anyMatch(path -> StringUtils.contains(delta, path))) {
return true;
}
}
return false;
}
/**
* @return True if error chain is already ended, false otherwise.
* */
private boolean mergeDeltaIfNeeded(LibvirtComputingResource serverResource, KbossTO kbossTO, VolumeObjectTO volumeObjectTO, List<DataTO> grandChildren,
boolean volumePathMissing, boolean deltaPathMissing, boolean basePathMissing, boolean errorOnCreate, boolean runningVm, boolean isTopDelta, boolean isEndOfChain) {
String errorMessage = String.format("Volume [%s] is inconsistent in an anomalous way. We cannot normalize it automatically.", volumeObjectTO.getUuid());
if (!errorOnCreate) {
// Base should never be missing if it is not an error from creation. If the volume path is missing and it is not the delta that was being removed, it is an anomaly as well.
if (basePathMissing || (volumePathMissing && !isTopDelta)) {
logger.warn(errorMessage);
throw new CloudRuntimeException(String.format ("Unable to find the base delta or the volume path was not found. We cannot normalize it automatically. At least " +
"one of these should exist: volume [%s]; base path [%s].", volumeObjectTO.getPath(), kbossTO.getParentDeltaPathOnPrimary()));
}
// This means that the delta merge likely succeeded but the host was unable to reply to the Management Server
if (deltaPathMissing) {
// This is if the delta being merged was the top delta. Then we must update its path.
if (volumePathMissing) {
volumeObjectTO.setPath(kbossTO.getParentDeltaPathOnPrimary());
}
logger.debug("Volume [{}] is already consistent. Its path is [{}].", volumeObjectTO.getUuid(), volumeObjectTO.getPath());
return true;
}
return false;
}
DeltaMergeTreeTO deltaMergeTreeTO; DeltaMergeTreeTO deltaMergeTreeTO;
boolean errorChainFinished; if (!backupErrorDeltaExists) {
if (volumePathMissing && !deltaPathMissing) { // The process was not started for this volume if (parentBackupDeltaExists && (!shouldBaseDeltaExist || baseDeltaExists)) {
DataTO child;
// If it is the top delta, we should set the volume path as the delta path on primary, as it is the real path. This will get updated later after being merged.
if (isTopDelta) {
volumeObjectTO.setPath(kbossTO.getDeltaPathOnPrimary()); volumeObjectTO.setPath(kbossTO.getDeltaPathOnPrimary());
child = volumeObjectTO; logger.debug("Volume [{}] is already consistent. Its path is [{}].", volumeObjectTO.getUuid(), volumeObjectTO.getPath());
} else { // Otherwise, we set it as the old path of the volume. In this case, this will be its final path. volumeObjectTOList.add(volumeObjectTO);
volumeObjectTO.setPath(kbossTO.getOldVolumePath()); return true;
child = new BackupDeltaTO(volumeObjectTO.getDataStore(), Hypervisor.HypervisorType.KVM, kbossTO.getDeltaPathOnPrimary()); } else if (baseDeltaExists) {
volumeObjectTO.setPath(kbossTO.getParentDeltaPathOnPrimary());
logger.debug("Volume [{}] is already consistent. Its path is [{}].", volumeObjectTO.getUuid(), volumeObjectTO.getPath());
volumeObjectTOList.add(volumeObjectTO);
return true;
} else {
logger.warn("Volume [{}] is inconsistent in an anomalous way. We cannot normalize it automatically.", volumeObjectTO.getUuid());
return false;
} }
logger.debug("Volume [{}] is consistent, the backup process for it was not started. Its current path is [{}]. We will merge the old backup chain.", } else if (parentBackupDeltaExists) {
volumeObjectTO.getUuid(), volumeObjectTO.getPath()); logger.debug("Volume [{}] is inconsistent, but we can normalize it. We will merge the delta created by this backup with the delta created by the previous " +
"backup.", volumeObjectTO.getUuid());
deltaMergeTreeTO = new DeltaMergeTreeTO(volumeObjectTO, new BackupDeltaTO(volumeObjectTO.getDataStore(), Hypervisor.HypervisorType.KVM, deltaMergeTreeTO = new DeltaMergeTreeTO(volumeObjectTO, new BackupDeltaTO(volumeObjectTO.getDataStore(), Hypervisor.HypervisorType.KVM,
kbossTO.getParentDeltaPathOnPrimary()), child, grandChildren); kbossTO.getDeltaPathOnPrimary()), volumeObjectTO, List.of());
errorChainFinished = true; } else if (baseDeltaExists) {
} else if (!isEndOfChain && !volumePathMissing && (deltaPathMissing || kbossTO.getParentDeltaPathOnPrimary() == null)) { // The process was completed for this volume logger.debug("Volume [{}] is inconsistent, but we can normalize it. We will merge the delta created by this backup with the base volume.",
logger.debug("Volume [{}] is consistent, the backup process was completed for it. Its current path is [{}].", volumeObjectTO.getUuid(), volumeObjectTO.getPath());
return false;
} else if (isEndOfChain && volumePathMissing && !basePathMissing) { // The process was completed for this volume
volumeObjectTO.setPath(kbossTO.getParentDeltaPathOnPrimary());
logger.debug("Volume [{}] is consistent, the backup process was completed for it. Its current path is [{}].", volumeObjectTO.getUuid(), volumeObjectTO.getPath());
return true;
} else if (!volumePathMissing && !deltaPathMissing) { // The process stopped midway
logger.debug("Volume [{}] is inconsistent, but we can normalize it. We will merge the delta created by the last backup with the base volume.",
volumeObjectTO.getUuid()); volumeObjectTO.getUuid());
deltaMergeTreeTO = new DeltaMergeTreeTO(volumeObjectTO, new BackupDeltaTO(volumeObjectTO.getDataStore(), Hypervisor.HypervisorType.KVM, deltaMergeTreeTO = new DeltaMergeTreeTO(volumeObjectTO, new BackupDeltaTO(volumeObjectTO.getDataStore(), Hypervisor.HypervisorType.KVM,
kbossTO.getParentDeltaPathOnPrimary()), new BackupDeltaTO(volumeObjectTO.getDataStore(), Hypervisor.HypervisorType.KVM, kbossTO.getDeltaPathOnPrimary()), kbossTO.getParentDeltaPathOnPrimary()), volumeObjectTO, List.of());
grandChildren);
errorChainFinished = false;
isTopDelta = false;
} else { } else {
logger.warn(errorMessage); logger.warn("Volume [{}] is inconsistent in an anomalous way. We cannot normalize it automatically.", volumeObjectTO.getUuid());
throw new CloudRuntimeException(errorMessage + " Maybe it is a good idea to open an issue to get help on this."); return false;
} }
try { try {
@ -235,19 +176,15 @@ public class LibvirtCleanupKbossVmBackupCommandWrapper extends CommandWrapper<Cl
} else { } else {
serverResource.mergeDeltaForStoppedVm(deltaMergeTreeTO); serverResource.mergeDeltaForStoppedVm(deltaMergeTreeTO);
} }
if (isTopDelta) { volumeObjectTO.setPath(deltaMergeTreeTO.getParent().getPath());
volumeObjectTO.setPath(deltaMergeTreeTO.getParent().getPath()); volumeObjectTOList.add(volumeObjectTO);
} return true;
return errorChainFinished;
} catch (QemuImgException | IOException | LibvirtException ex) { } catch (QemuImgException | IOException | LibvirtException ex) {
logger.error("Got an exception while trying to merge delta for volume [{}].", volumeObjectTO.getUuid(), ex); logger.error("Got an exception while trying to merge delta for volume [{}].", volumeObjectTO.getUuid(), ex);
throw new CloudRuntimeException(ex); return false;
} }
} }
/**
* Checks if the VM is really stopped by checking if its root volume has had any writes on the last 30 seconds.
* */
private boolean isVmReallyStopped(CleanupKbossBackupErrorCommand command, LibvirtComputingResource serverResource) { private boolean isVmReallyStopped(CleanupKbossBackupErrorCommand command, LibvirtComputingResource serverResource) {
VolumeObjectTO volume = command.getKbossTOs().stream() VolumeObjectTO volume = command.getKbossTOs().stream()
.filter(kbossTO -> kbossTO.getVolumeObjectTO().getDeviceId() == 0) .filter(kbossTO -> kbossTO.getVolumeObjectTO().getDeviceId() == 0)

View File

@ -57,7 +57,6 @@ import com.cloud.utils.script.Script;
import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.cloudstack.utils.qemu.QemuImg;
import org.apache.cloudstack.utils.qemu.QemuImgException; import org.apache.cloudstack.utils.qemu.QemuImgException;
import org.apache.cloudstack.utils.qemu.QemuImgFile; import org.apache.cloudstack.utils.qemu.QemuImgFile;
import org.apache.commons.collections4.CollectionUtils;
import org.libvirt.LibvirtException; import org.libvirt.LibvirtException;
import static com.cloud.hypervisor.kvm.storage.KVMStorageProcessor.poolTypesToDeleteChainInfo; import static com.cloud.hypervisor.kvm.storage.KVMStorageProcessor.poolTypesToDeleteChainInfo;
@ -182,12 +181,10 @@ public class LibvirtRevertSnapshotCommandWrapper extends CommandWrapper<RevertSn
try { try {
replaceVolumeWithSnapshot(volumePath, snapshotPath); replaceVolumeWithSnapshot(volumePath, snapshotPath);
if (CollectionUtils.isNotEmpty(volumeObjectTo.getDeltasToRemove()) && poolTypesToDeleteChainInfo.contains(kvmStoragePoolPrimary.getType()) && if (volumeObjectTo.getChainInfo() != null && poolTypesToDeleteChainInfo.contains(kvmStoragePoolPrimary.getType()) &&
volumeObjectTo.getFormat() == Storage.ImageFormat.QCOW2 && deleteChain) { volumeObjectTo.getFormat() == Storage.ImageFormat.QCOW2 && deleteChain) {
for (String deltaPath : volumeObjectTo.getDeltasToRemove()) { logger.debug("Deleting leftover backup delta at [{}].", volumeObjectTo.getChainInfo());
logger.debug("Deleting leftover backup delta at [{}].", deltaPath); kvmStoragePoolPrimary.deletePhysicalDisk(volumeObjectTo.getChainInfo(), volumeObjectTo.getFormat());
kvmStoragePoolPrimary.deletePhysicalDisk(deltaPath, volumeObjectTo.getFormat());
}
} }
logger.debug(String.format("Successfully reverted volume [%s] to snapshot [%s].", volumeObjectTo, snapshotToPrint)); logger.debug(String.format("Successfully reverted volume [%s] to snapshot [%s].", volumeObjectTo, snapshotToPrint));
} catch (LibvirtException | QemuImgException ex) { } catch (LibvirtException | QemuImgException ex) {

View File

@ -40,7 +40,6 @@ import org.apache.cloudstack.utils.qemu.QemuImg;
import org.apache.cloudstack.utils.qemu.QemuImgException; import org.apache.cloudstack.utils.qemu.QemuImgException;
import org.apache.cloudstack.utils.qemu.QemuImgFile; import org.apache.cloudstack.utils.qemu.QemuImgFile;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.libvirt.LibvirtException; import org.libvirt.LibvirtException;
import java.io.File; import java.io.File;
@ -144,8 +143,8 @@ public class LibvirtTakeKbossBackupCommandWrapper extends CommandWrapper<TakeKbo
volumeObjectTO.setPath(kbossTO.getDeltaPathOnPrimary()); volumeObjectTO.setPath(kbossTO.getDeltaPathOnPrimary());
if (deltaMergeTreeTO != null) { if (deltaMergeTreeTO != null) {
List<String> snapshotDataStoreVos = kbossTO.getDeltaPaths(); List<String> snapshotDataStoreVos = kbossTO.getVmSnapshotDeltaPaths();
mergeBackupDelta(resource, deltaMergeTreeTO, volumeObjectTO, vmName, runningVM, volumeUuid, CollectionUtils.isEmpty(snapshotDataStoreVos)); mergeBackupDelta(resource, deltaMergeTreeTO, volumeObjectTO, vmName, runningVM, volumeUuid, snapshotDataStoreVos.isEmpty());
} }
if (command.isEndChain() || command.isIsolated()) { if (command.isEndChain() || command.isIsolated()) {
@ -169,7 +168,7 @@ public class LibvirtTakeKbossBackupCommandWrapper extends CommandWrapper<TakeKbo
int waitInMillis) { int waitInMillis) {
VolumeObjectTO delta = kbossTO.getVolumeObjectTO(); VolumeObjectTO delta = kbossTO.getVolumeObjectTO();
String parentDeltaPathOnSecondary = kbossTO.getPathBackupParentOnSecondary(); String parentDeltaPathOnSecondary = kbossTO.getPathBackupParentOnSecondary();
List<String> deltaPathsToCopy = ObjectUtils.defaultIfNull(kbossTO.getDeltaPaths(), new ArrayList<>()); List<String> deltaPathsToCopy = CollectionUtils.isEmpty(kbossTO.getVmSnapshotDeltaPaths()) ? new ArrayList<>() : new ArrayList<>(kbossTO.getVmSnapshotDeltaPaths());
deltaPathsToCopy.add(delta.getPath()); deltaPathsToCopy.add(delta.getPath());
KVMStoragePool parentImagePool = null; KVMStoragePool parentImagePool = null;

View File

@ -99,7 +99,6 @@ import org.apache.cloudstack.utils.qemu.QemuObject;
import org.apache.cloudstack.utils.qemu.QemuObject.EncryptFormat; import org.apache.cloudstack.utils.qemu.QemuObject.EncryptFormat;
import org.apache.cloudstack.utils.security.ParserUtils; import org.apache.cloudstack.utils.security.ParserUtils;
import org.apache.commons.collections.MapUtils; import org.apache.commons.collections.MapUtils;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.BooleanUtils;
@ -2907,11 +2906,9 @@ public class KVMStorageProcessor implements StorageProcessor {
} }
} }
pool.deletePhysicalDisk(vol.getPath(), vol.getFormat()); pool.deletePhysicalDisk(vol.getPath(), vol.getFormat());
if (CollectionUtils.isNotEmpty(vol.getDeltasToRemove()) && poolTypesToDeleteChainInfo.contains(pool.getType()) && vol.getFormat() == ImageFormat.QCOW2 && cmd.isDeleteChain()) { if (vol.getChainInfo() != null && poolTypesToDeleteChainInfo.contains(pool.getType()) && vol.getFormat() == ImageFormat.QCOW2 && cmd.isDeleteChain()) {
for (String deltaPath : vol.getDeltasToRemove()) { logger.debug("Deleting leftover backup delta at [{}].", vol.getChainInfo());
logger.debug("Deleting leftover backup delta at [{}].", deltaPath); pool.deletePhysicalDisk(vol.getChainInfo(), vol.getFormat());
pool.deletePhysicalDisk(deltaPath, vol.getFormat());
}
} }
return new Answer(null); return new Answer(null);
} catch (final CloudRuntimeException e) { } catch (final CloudRuntimeException e) {

View File

@ -266,7 +266,7 @@ public class LibvirTakeKbossBackupCommandWrapperTest {
doReturn(volumePath).when(volumeObjectToMock1).getPath(); doReturn(volumePath).when(volumeObjectToMock1).getPath();
doReturn(volUuid1).when(volumeObjectToMock1).getUuid(); doReturn(volUuid1).when(volumeObjectToMock1).getUuid();
doReturn(parentPath).when(kbossTO1).getPathBackupParentOnSecondary(); doReturn(parentPath).when(kbossTO1).getPathBackupParentOnSecondary();
doReturn(new ArrayList<>(List.of(deltaPath2))).when(kbossTO1).getDeltaPaths(); doReturn(new ArrayList<>(List.of(deltaPath2))).when(kbossTO1).getVmSnapshotDeltaPaths();
doReturn(deltaPath1).when(kbossTO1).getDeltaPathOnSecondary(); doReturn(deltaPath1).when(kbossTO1).getDeltaPathOnSecondary();
doReturn(kvmStoragePool1).when(kvmStoragePoolManagerMock).getStoragePoolByURI(secondaryUrl); doReturn(kvmStoragePool1).when(kvmStoragePoolManagerMock).getStoragePoolByURI(secondaryUrl);
doReturn(kvmStoragePool2).when(kvmStoragePoolManagerMock).getStoragePoolByURI(secondaryUrl2); doReturn(kvmStoragePool2).when(kvmStoragePoolManagerMock).getStoragePoolByURI(secondaryUrl2);

View File

@ -971,7 +971,6 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager {
throw new InvalidParameterValueException("Could not find the requested backup schedule."); throw new InvalidParameterValueException("Could not find the requested backup schedule.");
} }
checkCallerAccessToBackupScheduleVm(schedule.getVmId()); checkCallerAccessToBackupScheduleVm(schedule.getVmId());
finalizeBackupScheduleIfNeeded(schedule);
return backupScheduleDao.remove(schedule.getId()); return backupScheduleDao.remove(schedule.getId());
} }
@ -979,33 +978,6 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager {
return deleteAllVmBackupSchedules(vmId); return deleteAllVmBackupSchedules(vmId);
} }
/**
* Terminates the backup schedule if necessary.
*
* @param backupSchedule the backup schedule to be processed for termination.
* @throws CloudRuntimeException if the backup offering associated with the
* virtual machine was not found or if the backup provider could not finalize
* the backup schedule.
*/
protected void finalizeBackupScheduleIfNeeded(BackupSchedule backupSchedule) {
VMInstanceVO vm = findVmById(backupSchedule.getVmId());
if (vm.getBackupOfferingId() == null) {
logger.debug("The virtual machine {} backup offering has already been removed; therefore, it is not necessary to finalize the backup schedule.", vm.getUuid());
return;
}
BackupOfferingVO backupOffering = backupOfferingDao.findById(vm.getBackupOfferingId());
if (backupOffering == null) {
throw new CloudRuntimeException("Could not find the backup offering of the backup schedule virtual machine.");
}
BackupProvider backupProvider = getBackupProvider(backupOffering.getProvider());
if (!backupProvider.removeVMBackupSchedule(vm, backupSchedule)) {
throw new CloudRuntimeException(String.format("Failed to finalize VM backup schedule with ID [%s].", backupSchedule.getUuid()));
}
}
/** /**
* Checks if the backup framework is enabled for the zone in which the VM with specified ID is allocated and * Checks if the backup framework is enabled for the zone in which the VM with specified ID is allocated and
* if the caller has access to the VM. * if the caller has access to the VM.
@ -1030,7 +1002,6 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager {
List<BackupScheduleVO> vmBackupSchedules = backupScheduleDao.listByVM(vmId); List<BackupScheduleVO> vmBackupSchedules = backupScheduleDao.listByVM(vmId);
boolean success = true; boolean success = true;
for (BackupScheduleVO vmBackupSchedule : vmBackupSchedules) { for (BackupScheduleVO vmBackupSchedule : vmBackupSchedules) {
finalizeBackupScheduleIfNeeded(vmBackupSchedule);
success = success && backupScheduleDao.remove(vmBackupSchedule.getId()); success = success && backupScheduleDao.remove(vmBackupSchedule.getId());
} }
return success; return success;
@ -1096,7 +1067,7 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager {
CheckedReservation backupStorageReservation = new CheckedReservation(owner, CheckedReservation backupStorageReservation = new CheckedReservation(owner,
Resource.ResourceType.backup_storage, backupSize, reservationDao, resourceLimitMgr)) { Resource.ResourceType.backup_storage, backupSize, reservationDao, resourceLimitMgr)) {
Pair<Boolean, Backup> result = backupProvider.takeBackup(vm, cmd.getQuiesceVM(), cmd.isIsolated(), backupScheduleId); Pair<Boolean, Backup> result = backupProvider.takeBackup(vm, cmd.getQuiesceVM(), cmd.isIsolated());
if (!result.first()) { if (!result.first()) {
throw new CloudRuntimeException("Failed to create Instance Backup"); throw new CloudRuntimeException("Failed to create Instance Backup");
} }

View File

@ -70,7 +70,6 @@ import javax.inject.Inject;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors;
public class InternalBackupServiceImpl extends ComponentLifecycleBase implements InternalBackupService, VmWorkJobHandler { public class InternalBackupServiceImpl extends ComponentLifecycleBase implements InternalBackupService, VmWorkJobHandler {
protected Logger logger = LogManager.getLogger(getClass()); protected Logger logger = LogManager.getLogger(getClass());
@ -127,17 +126,16 @@ public class InternalBackupServiceImpl extends ComponentLifecycleBase implements
return; return;
} }
VolumeObjectTO volumeObjectTO = (VolumeObjectTO) volumeTo; VolumeObjectTO volumeObjectTO = (VolumeObjectTO) volumeTo;
List<InternalBackupStoragePoolVO> backupDeltas = internalBackupStoragePoolDao.listByVolumeId(volumeObjectTO.getVolumeId()); InternalBackupStoragePoolVO backupDelta = internalBackupStoragePoolDao.findOneByVolumeId(volumeObjectTO.getVolumeId());
if (backupDeltas.isEmpty()) { if (backupDelta == null) {
return; return;
} }
volumeObjectTO.setDeltasToRemove(backupDeltas.stream().map(InternalBackupStoragePoolVO::getBackupDeltaParentPath).collect(Collectors.toSet())); volumeObjectTO.setChainInfo(backupDelta.getBackupDeltaParentPath());
if (cmd instanceof DeleteCommand) { if (cmd instanceof DeleteCommand) {
((DeleteCommand) cmd).setDeleteChain(true); ((DeleteCommand) cmd).setDeleteChain(true);
} else if (cmd instanceof RevertSnapshotCommand) { }
if (cmd instanceof RevertSnapshotCommand) {
((RevertSnapshotCommand) cmd).setDeleteChain(true); ((RevertSnapshotCommand) cmd).setDeleteChain(true);
} else {
return;
} }
logger.debug("Configured chain info for volume [{}]. Set it as [{}].", volumeObjectTO.getUuid(), volumeObjectTO.getChainInfo()); logger.debug("Configured chain info for volume [{}]. Set it as [{}].", volumeObjectTO.getUuid(), volumeObjectTO.getChainInfo());
} }
@ -145,23 +143,20 @@ public class InternalBackupServiceImpl extends ComponentLifecycleBase implements
@Override @Override
public void cleanupBackupMetadata(long volumeId) { public void cleanupBackupMetadata(long volumeId) {
logger.debug("Cleaning up backup metadata for volume [{}].", volumeId); logger.debug("Cleaning up backup metadata for volume [{}].", volumeId);
List<InternalBackupJoinVO> currents = internalBackupJoinDao.listCurrentsByVolumeIdDesc(volumeId); InternalBackupStoragePoolVO delta = internalBackupStoragePoolDao.findOneByVolumeId(volumeId);
if (currents.isEmpty()) { if (delta == null) {
return; return;
} }
internalBackupStoragePoolDao.expungeByVolumeId(volumeId); internalBackupStoragePoolDao.expungeByVolumeId(volumeId);
for (InternalBackupJoinVO current : currents) { if (CollectionUtils.isNotEmpty(internalBackupStoragePoolDao.listByBackupId(delta.getBackupId()))) {
if (CollectionUtils.isNotEmpty(internalBackupStoragePoolDao.listByBackupId(current.getId()))) { return;
continue; }
} InternalBackupJoinVO joinVO = internalBackupJoinDao.findById(delta.getBackupId());
logger.debug("Volume [{}] was the last volume with deltas in backup [{}]. Setting the backup as not current and not END_OF_CHAIN.", volumeId, joinVO.getUuid());
logger.debug("Volume [{}] was the last volume with deltas in backup [{}]. Setting the backup as END_OF_CHAIN and not current.", volumeId, current.getUuid()); backupDetailDao.removeDetail(joinVO.getId(), BackupDetailsDao.CURRENT);
backupDetailDao.removeDetail(current.getId(), BackupDetailsDao.CURRENT); if (!joinVO.getEndOfChain()) {
if (!current.getEndOfChain()) { backupDetailDao.persist(new BackupDetailVO(joinVO.getId(), BackupDetailsDao.END_OF_CHAIN, Boolean.TRUE.toString(), true));
backupDetailDao.persist(new BackupDetailVO(current.getId(), BackupDetailsDao.END_OF_CHAIN, Boolean.TRUE.toString(), true));
}
} }
} }
@ -307,7 +302,7 @@ public class InternalBackupServiceImpl extends ComponentLifecycleBase implements
if (internalBackupProvider == null) { if (internalBackupProvider == null) {
return false; return false;
} }
return internalBackupProvider.finishBackupChains(vm); return internalBackupProvider.finishBackupChain(vm);
} }
@Override @Override

View File

@ -725,7 +725,7 @@ public class BackupManagerTest {
when(backup.getId()).thenReturn(backupId); when(backup.getId()).thenReturn(backupId);
when(backup.getSize()).thenReturn(newBackupSize); when(backup.getSize()).thenReturn(newBackupSize);
when(backupProvider.getName()).thenReturn("testbackupprovider"); when(backupProvider.getName()).thenReturn("testbackupprovider");
when(backupProvider.takeBackup(vmInstanceVOMock, null, false, scheduleId)).thenReturn(new Pair<>(true, backup)); when(backupProvider.takeBackup(vmInstanceVOMock, null, false)).thenReturn(new Pair<>(true, backup));
Map<String, BackupProvider> backupProvidersMap = new HashMap<>(); Map<String, BackupProvider> backupProvidersMap = new HashMap<>();
backupProvidersMap.put(backupProvider.getName().toLowerCase(), backupProvider); backupProvidersMap.put(backupProvider.getName().toLowerCase(), backupProvider);
ReflectionTestUtils.setField(backupManager, "backupProvidersMap", backupProvidersMap); ReflectionTestUtils.setField(backupManager, "backupProvidersMap", backupProvidersMap);
@ -955,7 +955,6 @@ public class BackupManagerTest {
Mockito.when(backupSchedules.get(0).getId()).thenReturn(2L); Mockito.when(backupSchedules.get(0).getId()).thenReturn(2L);
Mockito.when(backupSchedules.get(1).getId()).thenReturn(3L); Mockito.when(backupSchedules.get(1).getId()).thenReturn(3L);
Mockito.when(backupScheduleDao.remove(Mockito.anyLong())).thenReturn(true); Mockito.when(backupScheduleDao.remove(Mockito.anyLong())).thenReturn(true);
Mockito.doNothing().when(backupManager).finalizeBackupScheduleIfNeeded(Mockito.any());
boolean success = backupManager.deleteAllVmBackupSchedules(vmId); boolean success = backupManager.deleteAllVmBackupSchedules(vmId);
assertTrue(success); assertTrue(success);
@ -971,7 +970,6 @@ public class BackupManagerTest {
Mockito.when(backupSchedules.get(1).getId()).thenReturn(3L); Mockito.when(backupSchedules.get(1).getId()).thenReturn(3L);
Mockito.when(backupScheduleDao.remove(2L)).thenReturn(true); Mockito.when(backupScheduleDao.remove(2L)).thenReturn(true);
Mockito.when(backupScheduleDao.remove(3L)).thenReturn(false); Mockito.when(backupScheduleDao.remove(3L)).thenReturn(false);
Mockito.doNothing().when(backupManager).finalizeBackupScheduleIfNeeded(Mockito.any());
boolean success = backupManager.deleteAllVmBackupSchedules(vmId); boolean success = backupManager.deleteAllVmBackupSchedules(vmId);
assertFalse(success); assertFalse(success);
@ -1017,7 +1015,6 @@ public class BackupManagerTest {
Mockito.doNothing().when(backupManager).checkCallerAccessToBackupScheduleVm(vmId); Mockito.doNothing().when(backupManager).checkCallerAccessToBackupScheduleVm(vmId);
when(backupScheduleVOMock.getId()).thenReturn(id); when(backupScheduleVOMock.getId()).thenReturn(id);
when(backupScheduleDao.remove(id)).thenReturn(true); when(backupScheduleDao.remove(id)).thenReturn(true);
Mockito.doNothing().when(backupManager).finalizeBackupScheduleIfNeeded(Mockito.any());
boolean success = backupManager.deleteBackupSchedule(deleteBackupScheduleCmdMock); boolean success = backupManager.deleteBackupSchedule(deleteBackupScheduleCmdMock);
assertTrue(success); assertTrue(success);
@ -1332,7 +1329,6 @@ public class BackupManagerTest {
when(schedule.getId()).thenReturn(scheduleId); when(schedule.getId()).thenReturn(scheduleId);
when(backupScheduleDao.listByVM(vmId)).thenReturn(List.of(schedule)); when(backupScheduleDao.listByVM(vmId)).thenReturn(List.of(schedule));
when(backupScheduleDao.remove(scheduleId)).thenReturn(true); when(backupScheduleDao.remove(scheduleId)).thenReturn(true);
doNothing().when(backupManager).finalizeBackupScheduleIfNeeded(any());
boolean result = backupManager.deleteBackupSchedule(cmd); boolean result = backupManager.deleteBackupSchedule(cmd);
assertTrue(result); assertTrue(result);
@ -2878,45 +2874,4 @@ public class BackupManagerTest {
verify(backupOfferingDao).persist(any()); verify(backupOfferingDao).persist(any());
verify(backupOfferingDetailsDao, never()).saveDetails(any()); verify(backupOfferingDetailsDao, never()).saveDetails(any());
} }
@Test(expected = CloudRuntimeException.class)
public void endScheduleBackupChainIfNeededTestInvalidVirtualMachineThrowCloudRuntimeException() {
Mockito.doReturn(1L).when(backupScheduleVOMock).getVmId();
backupManager.finalizeBackupScheduleIfNeeded(backupScheduleVOMock);
}
@Test(expected = CloudRuntimeException.class)
public void endScheduleBackupChainIfNeededTestInvalidBackupOfferingThrowCloudRuntimeException() {
Mockito.doReturn(1L).when(backupScheduleVOMock).getVmId();
Mockito.doReturn(vmInstanceVOMock).when(vmInstanceDao).findById(1L);
backupManager.finalizeBackupScheduleIfNeeded(backupScheduleVOMock);
}
@Test
public void endScheduleBackupChainIfNeededTestBackupProviderSuccessDoesNotThrowException() {
Mockito.doReturn(1L).when(backupScheduleVOMock).getVmId();
Mockito.doReturn(vmInstanceVOMock).when(vmInstanceDao).findById(1L);
Mockito.doReturn(2L).when(vmInstanceVOMock).getBackupOfferingId();
Mockito.doReturn(backupOfferingVOMock).when(backupOfferingDao).findById(2L);
Mockito.doReturn(BackupManagerImpl.KBOSS_BACKUP_PROVIDER).when(backupOfferingVOMock).getProvider();
Mockito.doReturn(backupProvider).when(backupManager).getBackupProvider(BackupManagerImpl.KBOSS_BACKUP_PROVIDER);
Mockito.doReturn(true).when(backupProvider).removeVMBackupSchedule(vmInstanceVOMock, backupScheduleVOMock);
backupManager.finalizeBackupScheduleIfNeeded(backupScheduleVOMock);
}
@Test(expected = CloudRuntimeException.class)
public void endScheduleBackupChainIfNeededTestBackupProviderFailThrowCloudRuntimeException() {
Mockito.doReturn(1L).when(backupScheduleVOMock).getVmId();
Mockito.doReturn(vmInstanceVOMock).when(vmInstanceDao).findById(1L);
Mockito.doReturn(2L).when(vmInstanceVOMock).getBackupOfferingId();
Mockito.doReturn(backupOfferingVOMock).when(backupOfferingDao).findById(2L);
Mockito.doReturn(BackupManagerImpl.KBOSS_BACKUP_PROVIDER).when(backupOfferingVOMock).getProvider();
Mockito.doReturn(backupProvider).when(backupManager).getBackupProvider(BackupManagerImpl.KBOSS_BACKUP_PROVIDER);
Mockito.doReturn(false).when(backupProvider).removeVMBackupSchedule(vmInstanceVOMock, backupScheduleVOMock);
backupManager.finalizeBackupScheduleIfNeeded(backupScheduleVOMock);
}
} }

View File

@ -17,7 +17,9 @@
package org.apache.cloudstack.backup; package org.apache.cloudstack.backup;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.any; import static org.mockito.Mockito.any;
import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doNothing;
@ -33,6 +35,8 @@ import org.apache.cloudstack.backup.dao.BackupDetailsDao;
import org.apache.cloudstack.backup.dao.InternalBackupJoinDao; import org.apache.cloudstack.backup.dao.InternalBackupJoinDao;
import org.apache.cloudstack.backup.dao.InternalBackupStoragePoolDao; import org.apache.cloudstack.backup.dao.InternalBackupStoragePoolDao;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.storage.command.DeleteCommand;
import org.apache.cloudstack.storage.command.RevertSnapshotCommand;
import org.apache.cloudstack.storage.datastore.db.ImageStoreObjectDownloadDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreObjectDownloadDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreObjectDownloadVO; import org.apache.cloudstack.storage.datastore.db.ImageStoreObjectDownloadVO;
import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
@ -140,26 +144,83 @@ public class InternalBackupServiceImplTest {
internalBackupServiceImplSpy.configureChainInfo(dataToMock, cmdMock); internalBackupServiceImplSpy.configureChainInfo(dataToMock, cmdMock);
verify(internalBackupStoragePoolDaoMock, never()).listByVolumeId(anyLong()); verify(internalBackupStoragePoolDaoMock, never()).findOneByVolumeId(anyLong());
}
@Test
public void configureChainInfoTestVolumeWithoutBackupDeltaReturnsImmediately() {
doReturn(VOLUME_ID).when(volumeObjectToMock).getVolumeId();
doReturn(null).when(internalBackupStoragePoolDaoMock).findOneByVolumeId(VOLUME_ID);
internalBackupServiceImplSpy.configureChainInfo(volumeObjectToMock, mock(Command.class));
verify(internalBackupStoragePoolDaoMock).findOneByVolumeId(VOLUME_ID);
verify(volumeObjectToMock, never()).setChainInfo(anyString());
}
@Test
public void configureChainInfoTestSetsChainInfoForGenericCommand() {
doReturn(VOLUME_ID).when(volumeObjectToMock).getVolumeId();
doReturn(internalBackupStoragePoolVoMock).when(internalBackupStoragePoolDaoMock).findOneByVolumeId(VOLUME_ID);
doReturn("/path/to/parent").when(internalBackupStoragePoolVoMock).getBackupDeltaParentPath();
Command cmdMock = mock(Command.class);
internalBackupServiceImplSpy.configureChainInfo(volumeObjectToMock, cmdMock);
verify(internalBackupStoragePoolDaoMock).findOneByVolumeId(VOLUME_ID);
verify(volumeObjectToMock).setChainInfo("/path/to/parent");
}
@Test
public void configureChainInfoTestSetsDeleteChainForDeleteCommand() {
doReturn(VOLUME_ID).when(volumeObjectToMock).getVolumeId();
doReturn(internalBackupStoragePoolVoMock).when(internalBackupStoragePoolDaoMock).findOneByVolumeId(VOLUME_ID);
doReturn("/path/to/parent").when(internalBackupStoragePoolVoMock).getBackupDeltaParentPath();
DeleteCommand deleteCommand = new DeleteCommand(volumeObjectToMock);
internalBackupServiceImplSpy.configureChainInfo(volumeObjectToMock, deleteCommand);
verify(volumeObjectToMock).setChainInfo("/path/to/parent");
assertTrue(deleteCommand.isDeleteChain());
}
@Test
public void configureChainInfoTestSetsDeleteChainForRevertSnapshotCommand() {
doReturn(VOLUME_ID).when(volumeObjectToMock).getVolumeId();
doReturn(internalBackupStoragePoolVoMock).when(internalBackupStoragePoolDaoMock).findOneByVolumeId(VOLUME_ID);
doReturn("/path/to/parent").when(internalBackupStoragePoolVoMock).getBackupDeltaParentPath();
RevertSnapshotCommand revertSnapshotCommand = new RevertSnapshotCommand(snapshotObjectToMock, snapshotObjectToMock);
internalBackupServiceImplSpy.configureChainInfo(volumeObjectToMock, revertSnapshotCommand);
verify(volumeObjectToMock).setChainInfo("/path/to/parent");
assertTrue(revertSnapshotCommand.isDeleteChain());
} }
@Test @Test
public void cleanupBackupMetadataTestNoDeltaReturnsImmediately() { public void cleanupBackupMetadataTestNoDeltaReturnsImmediately() {
doReturn(null).when(internalBackupStoragePoolDaoMock).findOneByVolumeId(VOLUME_ID);
internalBackupServiceImplSpy.cleanupBackupMetadata(VOLUME_ID); internalBackupServiceImplSpy.cleanupBackupMetadata(VOLUME_ID);
verify(internalBackupStoragePoolDaoMock).findOneByVolumeId(VOLUME_ID);
verify(internalBackupStoragePoolDaoMock, never()).expungeByVolumeId(VOLUME_ID); verify(internalBackupStoragePoolDaoMock, never()).expungeByVolumeId(VOLUME_ID);
verify(internalBackupJoinDaoMock, never()).findById(anyLong()); verify(internalBackupJoinDaoMock, never()).findById(anyLong());
} }
@Test @Test
public void cleanupBackupMetadataTestDeltaExistsButOtherDeltasRemainReturnsImmediately() { public void cleanupBackupMetadataTestDeltaExistsButOtherDeltasRemainReturnsImmediately() {
doReturn(BACKUP_ID).when(internalBackupJoinVoMock).getId(); doReturn(BACKUP_ID).when(internalBackupStoragePoolVoMock).getBackupId();
doReturn(List.of(internalBackupJoinVoMock)).when(internalBackupJoinDaoMock).listCurrentsByVolumeIdDesc(VOLUME_ID); doReturn(internalBackupStoragePoolVoMock).when(internalBackupStoragePoolDaoMock).findOneByVolumeId(VOLUME_ID);
doReturn(List.of(internalBackupStoragePoolVoMock, mock(InternalBackupStoragePoolVO.class))) doReturn(List.of(internalBackupStoragePoolVoMock, mock(InternalBackupStoragePoolVO.class)))
.when(internalBackupStoragePoolDaoMock).listByBackupId(BACKUP_ID); .when(internalBackupStoragePoolDaoMock).listByBackupId(BACKUP_ID);
internalBackupServiceImplSpy.cleanupBackupMetadata(VOLUME_ID); internalBackupServiceImplSpy.cleanupBackupMetadata(VOLUME_ID);
verify(internalBackupStoragePoolDaoMock).findOneByVolumeId(VOLUME_ID);
verify(internalBackupStoragePoolDaoMock).expungeByVolumeId(VOLUME_ID); verify(internalBackupStoragePoolDaoMock).expungeByVolumeId(VOLUME_ID);
verify(internalBackupStoragePoolDaoMock).listByBackupId(BACKUP_ID); verify(internalBackupStoragePoolDaoMock).listByBackupId(BACKUP_ID);
verify(internalBackupJoinDaoMock, never()).findById(anyLong()); verify(internalBackupJoinDaoMock, never()).findById(anyLong());
@ -167,30 +228,38 @@ public class InternalBackupServiceImplTest {
@Test @Test
public void cleanupBackupMetadataTestLastDeltaAndEndOfChainTrue() { public void cleanupBackupMetadataTestLastDeltaAndEndOfChainTrue() {
doReturn(List.of(internalBackupJoinVoMock)).when(internalBackupJoinDaoMock).listCurrentsByVolumeIdDesc(VOLUME_ID); doReturn(BACKUP_ID).when(internalBackupStoragePoolVoMock).getBackupId();
doReturn(internalBackupStoragePoolVoMock).when(internalBackupStoragePoolDaoMock).findOneByVolumeId(VOLUME_ID);
doReturn(List.of()).when(internalBackupStoragePoolDaoMock).listByBackupId(BACKUP_ID); doReturn(List.of()).when(internalBackupStoragePoolDaoMock).listByBackupId(BACKUP_ID);
doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findById(BACKUP_ID);
doReturn(BACKUP_ID).when(internalBackupJoinVoMock).getId(); doReturn(BACKUP_ID).when(internalBackupJoinVoMock).getId();
doReturn(true).when(internalBackupJoinVoMock).getEndOfChain(); doReturn(true).when(internalBackupJoinVoMock).getEndOfChain();
internalBackupServiceImplSpy.cleanupBackupMetadata(VOLUME_ID); internalBackupServiceImplSpy.cleanupBackupMetadata(VOLUME_ID);
verify(internalBackupStoragePoolDaoMock).findOneByVolumeId(VOLUME_ID);
verify(internalBackupStoragePoolDaoMock).expungeByVolumeId(VOLUME_ID); verify(internalBackupStoragePoolDaoMock).expungeByVolumeId(VOLUME_ID);
verify(internalBackupStoragePoolDaoMock).listByBackupId(BACKUP_ID); verify(internalBackupStoragePoolDaoMock).listByBackupId(BACKUP_ID);
verify(internalBackupJoinDaoMock).findById(BACKUP_ID);
verify(backupDetailDaoMock).removeDetail(BACKUP_ID, BackupDetailsDao.CURRENT); verify(backupDetailDaoMock).removeDetail(BACKUP_ID, BackupDetailsDao.CURRENT);
verify(backupDetailDaoMock, never()).persist(any()); verify(backupDetailDaoMock, never()).persist(any());
} }
@Test @Test
public void cleanupBackupMetadataTestLastDeltaAndEndOfChainFalse() { public void cleanupBackupMetadataTestLastDeltaAndEndOfChainFalse() {
doReturn(List.of(internalBackupJoinVoMock)).when(internalBackupJoinDaoMock).listCurrentsByVolumeIdDesc(VOLUME_ID); doReturn(BACKUP_ID).when(internalBackupStoragePoolVoMock).getBackupId();
doReturn(internalBackupStoragePoolVoMock).when(internalBackupStoragePoolDaoMock).findOneByVolumeId(VOLUME_ID);
doReturn(List.of()).when(internalBackupStoragePoolDaoMock).listByBackupId(BACKUP_ID); doReturn(List.of()).when(internalBackupStoragePoolDaoMock).listByBackupId(BACKUP_ID);
doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findById(BACKUP_ID);
doReturn(BACKUP_ID).when(internalBackupJoinVoMock).getId(); doReturn(BACKUP_ID).when(internalBackupJoinVoMock).getId();
doReturn(false).when(internalBackupJoinVoMock).getEndOfChain(); doReturn(false).when(internalBackupJoinVoMock).getEndOfChain();
internalBackupServiceImplSpy.cleanupBackupMetadata(VOLUME_ID); internalBackupServiceImplSpy.cleanupBackupMetadata(VOLUME_ID);
verify(internalBackupStoragePoolDaoMock).findOneByVolumeId(VOLUME_ID);
verify(internalBackupStoragePoolDaoMock).expungeByVolumeId(VOLUME_ID); verify(internalBackupStoragePoolDaoMock).expungeByVolumeId(VOLUME_ID);
verify(internalBackupStoragePoolDaoMock).listByBackupId(BACKUP_ID); verify(internalBackupStoragePoolDaoMock).listByBackupId(BACKUP_ID);
verify(internalBackupJoinDaoMock).findById(BACKUP_ID);
verify(backupDetailDaoMock).removeDetail(BACKUP_ID, BackupDetailsDao.CURRENT); verify(backupDetailDaoMock).removeDetail(BACKUP_ID, BackupDetailsDao.CURRENT);
verify(backupDetailDaoMock).persist(any(BackupDetailVO.class)); verify(backupDetailDaoMock).persist(any(BackupDetailVO.class));
} }