Merge branch '4.22' of https://github.com/apache/cloudstack into fix-stg-access-group-validation

debian/rules

@@ -95,6 +95,7 @@ override_dh_auto_install:
 	# nast hack for a couple of configuration files
 	mv $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)/server/cloudstack-limits.conf $(DESTDIR)/$(SYSCONFDIR)/security/limits.d/
 	mv $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)/server/cloudstack-sudoers $(DESTDIR)/$(SYSCONFDIR)/sudoers.d/$(PACKAGE)
+	sed -i '/requiretty/d' $(DESTDIR)/$(SYSCONFDIR)/sudoers.d/$(PACKAGE)
 	chmod 0440 $(DESTDIR)/$(SYSCONFDIR)/sudoers.d/$(PACKAGE)
 
 	install -D client/target/utilities/bin/cloud-update-xenserver-licenses $(DESTDIR)/usr/bin/cloudstack-update-xenserver-licenses

engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java

@@ -131,6 +131,9 @@ public interface NetworkOrchestrationService {
                     true,
                     Scope.Global);
 
+    ConfigKey<Integer> VmNetworkThrottlingRate = new ConfigKey<Integer>("Network", Integer.class, "vm.network.throttling.rate", "200",
+            "Default data transfer rate in megabits per second allowed in User vm's default network.", true, ConfigKey.Scope.Zone);
+
     List<? extends Network> setupNetwork(Account owner, NetworkOffering offering, DeploymentPlan plan, String name, String displayText, boolean isDefault)
         throws ConcurrentOperationException;
 

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java

@@ -4937,7 +4937,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
     @Override
     public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[]{NetworkGcWait, NetworkGcInterval, NetworkLockTimeout, DeniedRoutes,
-                GuestDomainSuffix, NetworkThrottlingRate, MinVRVersion,
+                GuestDomainSuffix, NetworkThrottlingRate, VmNetworkThrottlingRate, MinVRVersion,
                 PromiscuousMode, MacAddressChanges, ForgedTransmits, MacLearning, RollingRestartEnabled,
                 TUNGSTEN_ENABLED, NSX_ENABLED, NETRIS_ENABLED, NETWORK_LB_HAPROXY_MAX_CONN,
                 NETWORK_LB_HAPROXY_IDLE_TIMEOUT};

engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java

@@ -89,7 +89,9 @@ import com.cloud.upgrade.dao.Upgrade41810to41900;
 import com.cloud.upgrade.dao.Upgrade41900to41910;
 import com.cloud.upgrade.dao.Upgrade41910to42000;
 import com.cloud.upgrade.dao.Upgrade42000to42010;
-import com.cloud.upgrade.dao.Upgrade42010to42100;
+import com.cloud.upgrade.dao.Upgrade42020to42030;
+import com.cloud.upgrade.dao.Upgrade42030to42040;
+import com.cloud.upgrade.dao.Upgrade42040to42100;
 import com.cloud.upgrade.dao.Upgrade42100to42200;
 import com.cloud.upgrade.dao.Upgrade42200to42210;
 import com.cloud.upgrade.dao.Upgrade420to421;
@@ -239,7 +241,9 @@ public class DatabaseUpgradeChecker implements SystemIntegrityChecker {
                 .next("4.19.0.0", new Upgrade41900to41910())
                 .next("4.19.1.0", new Upgrade41910to42000())
                 .next("4.20.0.0", new Upgrade42000to42010())
-                .next("4.20.1.0", new Upgrade42010to42100())
+                .next("4.20.2.0", new Upgrade42020to42030())
+                .next("4.20.3.0", new Upgrade42030to42040())
+                .next("4.20.4.0", new Upgrade42040to42100())
                 .next("4.21.0.0", new Upgrade42100to42200())
                 .next("4.22.0.0", new Upgrade42200to42210())
                 .build();

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42030to42040.java

@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.upgrade.dao;
+
+import java.io.InputStream;
+import java.sql.Connection;
+import java.util.ArrayList;
+import java.util.List;
+
+public class Upgrade42030to42040 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate {
+
+    @Override
+    public String[] getUpgradableVersionRange() {
+        return new String[]{"4.20.3.0", "4.20.4.0"};
+    }
+
+    @Override
+    public String getUpgradedVersion() {
+        return "4.20.4.0";
+    }
+
+    @Override
+    public boolean supportsRollingUpgrade() {
+        return false;
+    }
+
+    @Override
+    public InputStream[] getPrepareScripts() {
+        return null;
+    }
+
+    @Override
+    public void performDataMigration(Connection conn) {
+        final List<String> indexList = new ArrayList<String>();
+        logger.debug("Dropping index vm_instance_id from usage_vm_instance table if it exists");
+        indexList.add("vm_instance_id");
+        DbUpgradeUtils.dropKeysIfExist(conn, "cloud_usage.usage_vm_instance", indexList, false);
+    }
+
+    @Override
+    public InputStream[] getCleanupScripts() {
+        return null;
+    }
+}

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42040to42100.java

@@ -32,12 +32,12 @@ import com.cloud.upgrade.SystemVmTemplateRegistration;
 import com.cloud.utils.db.TransactionLegacy;
 import com.cloud.utils.exception.CloudRuntimeException;
 
-public class Upgrade42010to42100 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate {
+public class Upgrade42040to42100 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate {
     private SystemVmTemplateRegistration systemVmTemplateRegistration;
 
     @Override
     public String[] getUpgradableVersionRange() {
-        return new String[] {"4.20.1.0", "4.21.0.0"};
+        return new String[] {"4.20.4.0", "4.21.0.0"};
     }
 
     @Override
@@ -52,7 +52,7 @@ public class Upgrade42010to42100 extends DbUpgradeAbstractImpl implements DbUpgr
 
     @Override
     public InputStream[] getPrepareScripts() {
-        final String scriptFile = "META-INF/db/schema-42010to42100.sql";
+        final String scriptFile = "META-INF/db/schema-42040to42100.sql";
         final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile);
         if (script == null) {
             throw new CloudRuntimeException("Unable to find " + scriptFile);
@@ -69,7 +69,7 @@ public class Upgrade42010to42100 extends DbUpgradeAbstractImpl implements DbUpgr
 
     @Override
     public InputStream[] getCleanupScripts() {
-        final String scriptFile = "META-INF/db/schema-42010to42100-cleanup.sql";
+        final String scriptFile = "META-INF/db/schema-42040to42100-cleanup.sql";
         final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile);
         if (script == null) {
             throw new CloudRuntimeException("Unable to find " + scriptFile);

engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDao.java

@@ -113,4 +113,6 @@ public interface ResourceDetailsDao<R extends ResourceDetail> extends GenericDao
     long batchExpungeForResources(List<Long> ids, Long batchSize);
 
     String getActualValue(ResourceDetail resourceDetail);
+
+    List<R> listDetailsForResourceIdsAndKey(List<Long> resourceIds, String key);
 }

engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDaoBase.java

@@ -16,11 +16,13 @@
 // under the License.
 package org.apache.cloudstack.resourcedetail;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import com.cloud.utils.StringUtils;
 import org.apache.commons.collections.CollectionUtils;
 
 import com.cloud.utils.Pair;
@@ -48,6 +50,7 @@ public abstract class ResourceDetailsDaoBase<R extends ResourceDetail> extends G
     public ResourceDetailsDaoBase() {
         AllFieldsSearch = createSearchBuilder();
         AllFieldsSearch.and("resourceId", AllFieldsSearch.entity().getResourceId(), SearchCriteria.Op.EQ);
+        AllFieldsSearch.and("resourceIdIn", AllFieldsSearch.entity().getResourceId(), SearchCriteria.Op.IN);
         AllFieldsSearch.and("name", AllFieldsSearch.entity().getName(), SearchCriteria.Op.EQ);
         AllFieldsSearch.and("value", AllFieldsSearch.entity().getValue(), SearchCriteria.Op.EQ);
         // FIXME SnapshotDetailsVO doesn't have a display field
@@ -266,4 +269,15 @@ public abstract class ResourceDetailsDaoBase<R extends ResourceDetail> extends G
         }
         return resourceDetail.getValue();
     }
+
+    @Override
+    public List<R> listDetailsForResourceIdsAndKey(List<Long> resourceIds, String key) {
+        if (CollectionUtils.isEmpty(resourceIds) || StringUtils.isBlank(key)) {
+            return Collections.emptyList();
+        }
+        SearchCriteria<R> sc = AllFieldsSearch.create();
+        sc.setParameters("name", key);
+        sc.setParameters("resourceIdIn", resourceIds.toArray());
+        return search(sc, null);
+    }
 }

engine/schema/src/main/resources/META-INF/db/schema-42040to42100-cleanup.sql

@@ -16,5 +16,5 @@
 -- under the License.
 
 --;
--- Schema upgrade cleanup from 4.20.1.0 to 4.21.0.0
+-- Schema upgrade cleanup from 4.20.4.0 to 4.21.0.0
 --;

engine/schema/src/main/resources/META-INF/db/schema-42040to42100.sql

@@ -16,7 +16,7 @@
 -- under the License.
 
 --;
--- Schema upgrade from 4.20.1.0 to 4.21.0.0
+-- Schema upgrade from 4.20.4.0 to 4.21.0.0
 --;
 
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backup_schedule', 'max_backups', 'INT(8) UNSIGNED NOT NULL DEFAULT 0 COMMENT ''Maximum number of backups to be retained''');

engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java

@@ -44,7 +44,9 @@ import com.cloud.upgrade.dao.Upgrade41120to41130;
 import com.cloud.upgrade.dao.Upgrade41120to41200;
 import com.cloud.upgrade.dao.Upgrade41510to41520;
 import com.cloud.upgrade.dao.Upgrade41610to41700;
-import com.cloud.upgrade.dao.Upgrade42010to42100;
+import com.cloud.upgrade.dao.Upgrade42020to42030;
+import com.cloud.upgrade.dao.Upgrade42030to42040;
+import com.cloud.upgrade.dao.Upgrade42040to42100;
 import com.cloud.upgrade.dao.Upgrade452to453;
 import com.cloud.upgrade.dao.Upgrade453to460;
 import com.cloud.upgrade.dao.Upgrade460to461;
@@ -381,6 +383,26 @@ public class DatabaseUpgradeCheckerTest {
         assertFalse("DatabaseUpgradeChecker should not be a standalone component", checker.isStandalone());
     }
 
+    @Test
+    public void testCalculateUpgradePath42010to42030() {
+
+        final CloudStackVersion dbVersion = CloudStackVersion.parse("4.20.1.0");
+        assertNotNull(dbVersion);
+
+        final CloudStackVersion currentVersion = CloudStackVersion.parse("4.20.3.0");
+        assertNotNull(currentVersion);
+
+        final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker();
+        final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion);
+
+        assertNotNull(upgrades);
+        assertEquals(1, upgrades.length);
+        assertTrue(upgrades[0] instanceof Upgrade42020to42030);
+
+        assertArrayEquals(new String[]{"4.20.2.0", "4.20.3.0"}, upgrades[0].getUpgradableVersionRange());
+        assertEquals(currentVersion.toString(), upgrades[0].getUpgradedVersion());
+    }
+
     @Test
     public void testCalculateUpgradePath42010to42100() {
 
@@ -394,10 +416,10 @@ public class DatabaseUpgradeCheckerTest {
         final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion);
 
         assertNotNull(upgrades);
-        assertEquals(1, upgrades.length);
-        assertTrue(upgrades[0] instanceof Upgrade42010to42100);
-
-        assertArrayEquals(new String[]{"4.20.1.0", "4.21.0.0"}, upgrades[0].getUpgradableVersionRange());
-        assertEquals(currentVersion.toString(), upgrades[0].getUpgradedVersion());
+        assertEquals(3, upgrades.length);
+        assertTrue(upgrades[0] instanceof Upgrade42020to42030);
+        assertTrue(upgrades[1] instanceof Upgrade42030to42040);
+        assertTrue(upgrades[2] instanceof Upgrade42040to42100);
+        assertEquals(currentVersion.toString(), upgrades[2].getUpgradedVersion());
     }
 }

engine/schema/src/test/java/com/cloud/upgrade/dao/Upgrade42040to42100Test.java

@@ -35,9 +35,9 @@ import org.mockito.junit.MockitoJUnitRunner;
 import com.cloud.utils.db.TransactionLegacy;
 
 @RunWith(MockitoJUnitRunner.class)
-public class Upgrade42010to42100Test {
+public class Upgrade42040to42100Test {
     @Spy
-    Upgrade42010to42100 upgrade;
+    Upgrade42040to42100 upgrade;
 
     @Mock
     private Connection conn;

framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java

@@ -31,6 +31,8 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
@@ -116,6 +118,8 @@ import com.cloud.vm.dao.VMInstanceDao;
 import org.apache.logging.log4j.ThreadContext;
 
 public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, ClusterManagerListener, Configurable {
+    private static final Pattern PASSWORD_FIELD_PATTERN = Pattern.compile("\\\"password\\\":\\\"([^\\\"]*)\\\"+");
+
     // Advanced
     public static final ConfigKey<Long> JobExpireMinutes = new ConfigKey<Long>("Advanced", Long.class, "job.expire.minutes", "1440",
         "Time (in minutes) for async-jobs to be kept in system", true, ConfigKey.Scope.Global);
@@ -555,22 +559,26 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager,
     }
 
     public String obfuscatePassword(String result, boolean hidePassword) {
-        if (hidePassword) {
-            String pattern = "\"password\":";
-            if (result != null) {
-                if (result.contains(pattern)) {
-                    String[] resp = result.split(pattern);
-                    String psswd = resp[1].toString().split(",")[0];
-                    if (psswd.endsWith("}")) {
-                        psswd = psswd.substring(0, psswd.length() - 1);
-                        result = resp[0] + pattern + psswd.replace(psswd.substring(2, psswd.length() - 1), "*****") + "}," + resp[1].split(",", 2)[1];
-                    } else {
-                        result = resp[0] + pattern + psswd.replace(psswd.substring(2, psswd.length() - 1), "*****") + "," + resp[1].split(",", 2)[1];
-                    }
-                }
-            }
+        if (!hidePassword || StringUtils.isBlank(result)) {
+            return result;
         }
-        return result;
+
+        Matcher matcher = PASSWORD_FIELD_PATTERN.matcher(result);
+        StringBuilder obfuscatedResult = new StringBuilder();
+        while (matcher.find()) {
+            String password = matcher.group(1);
+            String replacement = "\"password\":\"" + obfuscatePasswordValue(password) + "\"";
+            matcher.appendReplacement(obfuscatedResult, Matcher.quoteReplacement(replacement));
+        }
+        matcher.appendTail(obfuscatedResult);
+        return obfuscatedResult.toString();
+    }
+
+    private String obfuscatePasswordValue(String password) {
+        if (StringUtils.isEmpty(password)) {
+            return password;
+        }
+        return password.charAt(0) + "*****";
     }
 
     private void scheduleExecution(final AsyncJobVO job) {

framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/AsyncJobManagerTest.java

@@ -17,12 +17,15 @@
 package org.apache.cloudstack.framework.jobs;
 
 import org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl;
+import org.apache.commons.lang3.StringUtils;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.Spy;
 import org.mockito.junit.MockitoJUnitRunner;
 
+import com.cloud.utils.HumanReadableJson;
+
 @RunWith (MockitoJUnitRunner.class)
 public class AsyncJobManagerTest {
 
@@ -37,6 +40,12 @@ public class AsyncJobManagerTest {
     String inputNoBraces = "\"password\":\"password\"\",\"action\":\"OFF\"";
     String expectedNoBraces = "\"password\":\"p*****\",\"action\":\"OFF\"";
 
+    String realUserVmResponseWithPasswordInput = "{\"id\":\"f75b0990-5801-4b78-bcb0-58a503afa49c\",\"name\":\"pw-vm\"," +
+            "\"displayname\":\"pw-vm\",\"account\":\"admin\",\"password\":\"67wSK5\",\"instancename\":\"i-2-17-VM\"," +
+            "\"details\":{\"password\":\"3WTVryPJZJwMZGcJJ+OOYf84+uixk/1FraomPG9N6/Uvng\\u003d\\u003d\"," +
+            "\"Message.ReservedCapacityFreed.Flag\":\"true\",\"rootDiskController\":\"osdefault\"}," +
+            "\"arch\":\"x86_64\",\"jobid\":\"c13865d3-61ec-4269-979a-3d799181d5fe\",\"jobstatus\":0}";
+
     @Test
     public void obfuscatePasswordTest() {
         String result = asyncJobManager.obfuscatePassword(input, true);
@@ -79,4 +88,15 @@ public class AsyncJobManagerTest {
         Assert.assertEquals(noPassword, result);
     }
 
+    @Test
+    public void obfuscatePasswordTestHidePasswordRealInput() {
+        String result = asyncJobManager.obfuscatePassword(realUserVmResponseWithPasswordInput, true);
+
+        Assert.assertNotNull(result);
+        Assert.assertFalse(result.contains("\"password\":\"3WTVryPJZJwMZGcJJ+OOYf84+uixk\""));
+        String jsonObject = HumanReadableJson.getHumanReadableBytesJson(result);
+        Assert.assertTrue(StringUtils.isNotEmpty(jsonObject));
+        Assert.assertTrue(jsonObject.contains("\"password\":\"3*****\""));
+    }
+
 }

packaging/systemd/cloudstack-management.default

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-JAVA_OPTS="-Djava.security.properties=/etc/cloudstack/management/java.security.ciphers -Djava.awt.headless=true -Xmx2G -XX:+UseParallelGC -XX:MaxGCPauseMillis=500 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/cloudstack/management/ -XX:ErrorFile=/var/log/cloudstack/management/cloudstack-management.err --add-opens=java.base/java.lang=ALL-UNNAMED --add-exports=java.base/sun.security.x509=ALL-UNNAMED"
+JAVA_OPTS="-Djava.security.properties=/etc/cloudstack/management/java.security.ciphers -Djava.awt.headless=true -Xmx2G -XX:+UseParallelGC -XX:MaxGCPauseMillis=500 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/cloudstack/management/ -XX:ErrorFile=/var/log/cloudstack/management/cloudstack-management.err --add-opens=java.base/java.lang=ALL-UNNAMED --add-exports=java.base/sun.security.x509=ALL-UNNAMED -Djava.io.tmpdir=/var/tmp"
 
 CLASSPATH="/usr/share/cloudstack-management/lib/*:/etc/cloudstack/management:/usr/share/cloudstack-common:/usr/share/cloudstack-management/setup:/usr/share/cloudstack-management:/usr/share/cloudstack-mysql-ha/lib/*"
 

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java

@@ -2092,23 +2092,24 @@ public class KVMStorageProcessor implements StorageProcessor {
 
         logger.debug("Rebasing snapshot [{}] with parent [{}].", snapshotName, parentSnapshotPath);
 
+        long snapshotTimeoutInMillis = wait * 1000L;
         try {
-            QemuImg qemuImg = new QemuImg(wait);
+            QemuImg qemuImg = new QemuImg(snapshotTimeoutInMillis);
             qemuImg.rebase(snapshotFile, parentSnapshotFile, PhysicalDiskFormat.QCOW2.toString(), false);
         } catch (LibvirtException | QemuImgException e) {
             if (!StringUtils.contains(e.getMessage(), "Is another process using the image")) {
                 logger.error("Exception while rebasing incremental snapshot [{}] due to: [{}].", snapshotName, e.getMessage(), e);
                 throw new CloudRuntimeException(e);
             }
-            retryRebase(snapshotName, wait, e, snapshotFile, parentSnapshotFile);
+            retryRebase(snapshotName, snapshotTimeoutInMillis, e, snapshotFile, parentSnapshotFile);
         }
     }
 
-    private void retryRebase(String snapshotName, int wait, Exception e, QemuImgFile snapshotFile, QemuImgFile parentSnapshotFile) {
+    private void retryRebase(String snapshotName, long waitInMilliseconds, Exception e, QemuImgFile snapshotFile, QemuImgFile parentSnapshotFile) {
         logger.warn("Libvirt still has not released the lock, will wait [{}] milliseconds and try again later.", incrementalSnapshotRetryRebaseWait);
         try {
             Thread.sleep(incrementalSnapshotRetryRebaseWait);
-            QemuImg qemuImg = new QemuImg(wait);
+            QemuImg qemuImg = new QemuImg(waitInMilliseconds);
             qemuImg.rebase(snapshotFile, parentSnapshotFile, PhysicalDiskFormat.QCOW2.toString(), false);
         } catch (LibvirtException | QemuImgException | InterruptedException ex) {
             logger.error("Unable to rebase snapshot [{}].", snapshotName, ex);
@@ -2536,7 +2537,7 @@ public class KVMStorageProcessor implements StorageProcessor {
         QemuImgFile destFile = new QemuImgFile(snapshotPath);
         destFile.setFormat(PhysicalDiskFormat.QCOW2);
 
-        QemuImg q = new QemuImg(wait);
+        QemuImg q = new QemuImg(wait * 1000L);
         q.convert(srcFile, destFile, options, qemuObjects, qemuImageOpts, null, true);
     }
 

plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java

@@ -61,7 +61,7 @@ public class QemuImg {
     /* The qemu-img binary. We expect this to be in $PATH */
     public String _qemuImgPath = "qemu-img";
     private String cloudQemuImgPath = "cloud-qemu-img";
-    private int timeout;
+    private long timeout;
     private boolean skipZero = false;
     private boolean skipTargetVolumeCreation = false;
     private boolean noCache = false;
@@ -129,7 +129,7 @@ public class QemuImg {
      * @param skipZeroIfSupported Don't write zeroes to target device during convert, if supported by qemu-img
      * @param noCache Ensure we flush writes to target disk (useful for block device targets)
      */
-    public QemuImg(final int timeout, final boolean skipZeroIfSupported, final boolean noCache) throws LibvirtException {
+    public QemuImg(final long timeout, final boolean skipZeroIfSupported, final boolean noCache) throws LibvirtException {
         if (skipZeroIfSupported) {
             final Script s = new Script(_qemuImgPath, timeout);
             s.add("--help");
@@ -159,7 +159,7 @@ public class QemuImg {
      * @param timeout
      *            The timeout of scripts executed by this QemuImg object.
      */
-    public QemuImg(final int timeout) throws LibvirtException, QemuImgException {
+    public QemuImg(final long timeout) throws LibvirtException, QemuImgException {
         this(timeout, false, false);
     }
 
@@ -927,7 +927,10 @@ public class QemuImg {
     }
 
     protected static boolean helpSupportsImageFormat(String text, QemuImg.PhysicalDiskFormat format) {
-        Pattern pattern = Pattern.compile("Supported\\sformats:[a-zA-Z0-9-_\\s]*?\\b" + format + "\\b", CASE_INSENSITIVE);
+        // QEMU >= 10.1.0 changed the qemu-img --help header from
+        // "Supported formats:" to "Supported image formats:", so the word
+        // "image" must be treated as optional here.
+        Pattern pattern = Pattern.compile("Supported\\s(image\\s)?formats:[a-zA-Z0-9-_\\s]*?\\b" + format + "\\b", CASE_INSENSITIVE);
         return pattern.matcher(text).find();
     }
 

plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/qemu/QemuImgTest.java

@@ -390,6 +390,21 @@ public class QemuImgTest {
         Assert.assertFalse("should not support http", QemuImg.helpSupportsImageFormat(partialHelp, PhysicalDiskFormat.SHEEPDOG));
     }
 
+    @Test
+    public void testHelpSupportsImageFormatQemu101Header() throws QemuImgException, LibvirtException {
+        // qemu-img 10.1.0 (e.g. RHEL 9.8: qemu-kvm-10.1.0-17.el9_8.3) changed the
+        // help header from "Supported formats:" to "Supported image formats:"
+        String help = "Supported image formats:\n" +
+                "  blkdebug blklogwrites blkverify compress copy-before-write copy-on-read\n" +
+                "  file ftp ftps host_cdrom host_device http https io_uring luks nbd null-aio\n" +
+                "  null-co nvme nvme-io_uring preallocate qcow2 quorum raw rbd\n" +
+                "  snapshot-access throttle vdi vhdx virtio-blk-vfio-pci\n" +
+                "  virtio-blk-vhost-user virtio-blk-vhost-vdpa vmdk vpc\n";
+        Assert.assertTrue("should support luks", QemuImg.helpSupportsImageFormat(help, PhysicalDiskFormat.LUKS));
+        Assert.assertTrue("should support qcow2", QemuImg.helpSupportsImageFormat(help, PhysicalDiskFormat.QCOW2));
+        Assert.assertFalse("should not support sheepdog", QemuImg.helpSupportsImageFormat(help, PhysicalDiskFormat.SHEEPDOG));
+    }
+
     @Test
     public void testCheckAndRepair() throws LibvirtException {
         String filename = "/tmp/" + UUID.randomUUID() + ".qcow2";

plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayAdapter.java

@@ -28,10 +28,13 @@ import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 
 import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.SSLContext;
 
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.http.Header;
 import org.apache.http.NameValuePair;
 import org.apache.cloudstack.storage.datastore.adapter.ProviderAdapter;
@@ -62,6 +65,7 @@ import org.apache.http.ssl.SSLContextBuilder;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -86,6 +90,10 @@ public class FlashArrayAdapter implements ProviderAdapter {
     private static final String API_LOGIN_VERSION_DEFAULT = "1.19";
     private static final String API_VERSION_DEFAULT = "2.23";
 
+    // URLs for which the legacy-auth deprecation WARN has already been emitted,
+    // so we don't spam the logs once per refresh per pool while it's still configured.
+    private static final Set<String> WARNED_LEGACY_URLS = ConcurrentHashMap.newKeySet();
+
     static final ObjectMapper mapper = new ObjectMapper();
     public String pod = null;
     public String hostgroup = null;
@@ -492,18 +500,49 @@ public class FlashArrayAdapter implements ProviderAdapter {
     @Override
     public ProviderVolumeStorageStats getManagedStorageStats() {
         FlashArrayPod pod = getVolumeNamespace(this.pod);
-        // just in case
-        if (pod == null || pod.getFootprint() == 0) {
+        if (pod == null) {
             return null;
         }
         Long capacityBytes = pod.getQuotaLimit();
-        Long usedBytes = pod.getQuotaLimit() - (pod.getQuotaLimit() - pod.getFootprint());
+        if (capacityBytes == null || capacityBytes == 0) {
+            // Pod has no explicit quota set; report the array total physical
+            // capacity so the CloudStack allocator has a real ceiling to plan
+            // against rather than bailing out with a zero-capacity pool.
+            capacityBytes = getArrayTotalCapacity();
+        }
+        if (capacityBytes == null || capacityBytes == 0) {
+            return null;
+        }
+        Long usedBytes = pod.getFootprint();
+        if (usedBytes == null) {
+            usedBytes = 0L;
+        }
         ProviderVolumeStorageStats stats = new ProviderVolumeStorageStats();
         stats.setCapacityInBytes(capacityBytes);
         stats.setActualUsedInBytes(usedBytes);
         return stats;
     }
 
+    private Long getArrayTotalCapacity() {
+        try {
+            FlashArrayList<Map<String, Object>> list = GET("/arrays?space=true",
+                    new TypeReference<FlashArrayList<Map<String, Object>>>() {
+                    });
+            if (list != null && CollectionUtils.isNotEmpty(list.getItems())) {
+                Object cap = list.getItems().get(0).get("capacity");
+                if (cap instanceof Number) {
+                    return ((Number) cap).longValue();
+                }
+            }
+        } catch (Exception e) {
+            logger.warn("Could not retrieve total capacity for FlashArray [{}] (pod [{}]): {}",
+                    this.url, this.pod, e.getMessage());
+            logger.debug("Stack trace for array total capacity lookup failure on FlashArray [{}] (pod [{}])",
+                    this.url, this.pod, e);
+        }
+        return null;
+    }
+
     @Override
     public ProviderVolumeStats getVolumeStats(ProviderAdapterContext context, ProviderAdapterDataObject dataObject) {
         ProviderVolume vol = getVolume(dataObject.getExternalName());
@@ -557,6 +596,91 @@ public class FlashArrayAdapter implements ProviderAdapter {
         return accessToken;
     }
 
+    /**
+     * Discover the latest supported Purity REST API version by hitting the unauthenticated
+     * {@code /api/api_version} endpoint (returns {@code {"version":["1.0",...,"2.36"]}}).
+     * The discovered version is stored on {@link #apiVersion}; on failure the caller-configured
+     * default remains in place.
+     */
+    private void fetchApiVersionFromPurity(CloseableHttpClient client) {
+        HttpGet vReq = new HttpGet(url + "/api_version");
+        CloseableHttpResponse vResp = null;
+        try {
+            vResp = client.execute(vReq);
+            if (vResp.getStatusLine().getStatusCode() == 200) {
+                JsonNode root = mapper.readTree(vResp.getEntity().getContent());
+                JsonNode versions = root.get("version");
+                if (versions != null && versions.isArray() && versions.size() > 0) {
+                    apiVersion = versions.get(versions.size() - 1).asText();
+                }
+            } else {
+                logger.warn("Unexpected HTTP " + vResp.getStatusLine().getStatusCode()
+                        + " from FlashArray [" + url + "] /api_version, falling back to default "
+                        + API_VERSION_DEFAULT);
+            }
+        } catch (Exception e) {
+            logger.warn("Failed to discover Purity REST API version from " + url
+                    + "/api_version, falling back to default " + API_VERSION_DEFAULT, e);
+        } finally {
+            if (vResp != null) {
+                try {
+                    vResp.close();
+                } catch (IOException e) {
+                    logger.debug("Error closing /api_version response from FlashArray [" + url + "]", e);
+                }
+            }
+        }
+    }
+
+    /**
+     * Exchange the operator-configured username/password for a long-lived Purity api-token
+     * via REST 1.x {@code /auth/apitoken}. Emits the once-per-URL deprecation WARN.
+     * @return the api-token to feed into the REST 2.x /login exchange.
+     */
+    private String getApiTokenUsingUserPass(CloseableHttpClient client) throws IOException {
+        if (WARNED_LEGACY_URLS.add(url)) {
+            logger.warn("FlashArray adapter at [" + url + "] is using deprecated username/password "
+                    + "login against Purity REST 1.x. Replace with a pre-minted "
+                    + ProviderAdapter.API_TOKEN_KEY + " detail; the username/password code path will be "
+                    + "removed in a future release.");
+        }
+        HttpPost request = new HttpPost(url + "/" + apiLoginVersion + "/auth/apitoken");
+        ArrayList<NameValuePair> postParms = new ArrayList<NameValuePair>();
+        postParms.add(new BasicNameValuePair("username", username));
+        postParms.add(new BasicNameValuePair("password", password));
+        request.setEntity(new UrlEncodedFormEntity(postParms, "UTF-8"));
+        CloseableHttpResponse response = null;
+        try {
+            response = client.execute(request);
+            int statusCode = response.getStatusLine().getStatusCode();
+            if (statusCode == 200 || statusCode == 201) {
+                FlashArrayApiToken legacyToken = mapper.readValue(response.getEntity().getContent(),
+                        FlashArrayApiToken.class);
+                if (legacyToken == null || legacyToken.getApiToken() == null) {
+                    throw new CloudRuntimeException(
+                            "Authentication responded successfully but no api token was returned");
+                }
+                return legacyToken.getApiToken();
+            } else if (statusCode == 401 || statusCode == 403) {
+                throw new CloudRuntimeException(
+                        "Authentication or Authorization to FlashArray [" + url + "] with user [" + username
+                                + "] failed, unable to retrieve session token");
+            } else {
+                throw new CloudRuntimeException(
+                        "Unexpected HTTP response code from FlashArray [" + url + "] - [" + statusCode
+                                + "] - " + response.getStatusLine().getReasonPhrase());
+            }
+        } finally {
+            if (response != null) {
+                try {
+                    response.close();
+                } catch (IOException e) {
+                    logger.debug("Error closing legacy auth/apitoken response from FlashArray [" + url + "]", e);
+                }
+            }
+        }
+    }
+
     private synchronized void refreshSession(boolean force) {
         try {
             if (force || keyExpiration < System.currentTimeMillis()) {
@@ -631,9 +755,11 @@ public class FlashArrayAdapter implements ProviderAdapter {
         }
 
         apiVersion = connectionDetails.get(FlashArrayAdapter.API_VERSION);
-        if (apiVersion == null) {
+        boolean apiVersionExplicit = apiVersion != null;
+        if (!apiVersionExplicit) {
             apiVersion = queryParms.get(FlashArrayAdapter.API_VERSION);
-            if (apiVersion == null) {
+            apiVersionExplicit = apiVersion != null;
+            if (!apiVersionExplicit) {
                 apiVersion = API_VERSION_DEFAULT;
             }
         }
@@ -700,72 +826,66 @@ public class FlashArrayAdapter implements ProviderAdapter {
             skipTlsValidation = true;
         }
 
+        // Resolve the long-lived API token. Prefer a pre-minted api_token (Purity REST 2.x flow);
+        // fall back to legacy username/password auth via Purity REST 1.x for backward compatibility.
+        String apiToken = connectionDetails.get(ProviderAdapter.API_TOKEN_KEY);
+        if (apiToken != null && apiToken.isEmpty()) {
+            apiToken = null;
+        }
+        boolean usingLegacyUserPass = apiToken == null;
+        if (usingLegacyUserPass && (username == null || password == null)) {
+            throw new CloudRuntimeException("FlashArray adapter requires either " + ProviderAdapter.API_TOKEN_KEY
+                    + " (preferred) or both " + ProviderAdapter.API_USERNAME_KEY + " and "
+                    + ProviderAdapter.API_PASSWORD_KEY + " in the connection details");
+        }
+
+        CloseableHttpClient client = getClient();
         CloseableHttpResponse response = null;
         try {
-            HttpPost request = new HttpPost(url + "/" + apiLoginVersion + "/auth/apitoken");
-            // request.addHeader("Content-Type", "application/json");
-            // request.addHeader("Accept", "application/json");
-            ArrayList<NameValuePair> postParms = new ArrayList<NameValuePair>();
-            postParms.add(new BasicNameValuePair("username", username));
-            postParms.add(new BasicNameValuePair("password", password));
-            request.setEntity(new UrlEncodedFormEntity(postParms, "UTF-8"));
-            CloseableHttpClient client = getClient();
-            response = (CloseableHttpResponse) client.execute(request);
-
-            int statusCode = response.getStatusLine().getStatusCode();
-            FlashArrayApiToken apitoken = null;
-            if (statusCode == 200 | statusCode == 201) {
-                apitoken = mapper.readValue(response.getEntity().getContent(), FlashArrayApiToken.class);
-                if (apitoken == null) {
-                    throw new CloudRuntimeException(
-                            "Authentication responded successfully but no api token was returned");
-                }
-            } else if (statusCode == 401 || statusCode == 403) {
-                throw new CloudRuntimeException(
-                        "Authentication or Authorization to FlashArray [" + url + "] with user [" + username
-                                + "] failed, unable to retrieve session token");
-            } else {
-                throw new CloudRuntimeException(
-                        "Unexpected HTTP response code from FlashArray [" + url + "] - [" + statusCode
-                                + "] - " + response.getStatusLine().getReasonPhrase());
+            // Discover the latest supported API version from the array unless one was explicitly configured.
+            // GET /api/api_version is unauthenticated and returns {"version":["1.0",...,"2.36"]}.
+            if (!apiVersionExplicit) {
+                fetchApiVersionFromPurity(client);
             }
 
-            // now we need to get the access token
-            request = new HttpPost(url + "/" + apiVersion + "/login");
-            request.addHeader("api-token", apitoken.getApiToken());
-            response = (CloseableHttpResponse) client.execute(request);
+            if (usingLegacyUserPass) {
+                apiToken = getApiTokenUsingUserPass(client);
+            }
 
-            statusCode = response.getStatusLine().getStatusCode();
-            if (statusCode == 200 | statusCode == 201) {
+            // Exchange the long-lived api-token for a short-lived x-auth-token (REST 2.x).
+            HttpPost request = new HttpPost(url + "/" + apiVersion + "/login");
+            request.addHeader("api-token", apiToken);
+            response = client.execute(request);
+            int statusCode = response.getStatusLine().getStatusCode();
+            if (statusCode == 200 || statusCode == 201) {
                 Header[] headers = response.getHeaders("x-auth-token");
                 if (headers == null || headers.length == 0) {
                     throw new CloudRuntimeException(
-                            "Getting access token responded successfully but access token was not available");
+                            "FlashArray /login responded successfully but no x-auth-token header was returned");
                 }
                 accessToken = headers[0].getValue();
             } else if (statusCode == 401 || statusCode == 403) {
                 throw new CloudRuntimeException(
-                        "Authentication or Authorization to FlashArray [" + url + "] with user [" + username
-                                + "] failed, unable to retrieve session token");
+                        "FlashArray [" + url + "] rejected the api-token at /" + apiVersion + "/login");
             } else {
                 throw new CloudRuntimeException(
-                        "Unexpected HTTP response code from FlashArray [" + url + "] - [" + statusCode
-                                + "] - " + response.getStatusLine().getReasonPhrase());
+                        "Unexpected HTTP response code from FlashArray [" + url + "] /" + apiVersion
+                                + "/login - [" + statusCode + "] - "
+                                + response.getStatusLine().getReasonPhrase());
             }
-
         } catch (UnsupportedEncodingException e) {
-            throw new CloudRuntimeException("Error creating input for login, check username/password encoding");
+            throw new CloudRuntimeException("Error encoding login form for FlashArray [" + url + "]", e);
         } catch (UnsupportedOperationException e) {
             throw new CloudRuntimeException("Error processing login response from FlashArray [" + url + "]", e);
         } catch (IOException e) {
             throw new CloudRuntimeException("Error sending login request to FlashArray [" + url + "]", e);
         } finally {
-            try {
-                if (response != null) {
+            if (response != null) {
+                try {
                     response.close();
+                } catch (IOException e) {
+                    logger.debug("Error closing response from login attempt to FlashArray", e);
                 }
-            } catch (IOException e) {
-                logger.debug("Error closing response from login attempt to FlashArray", e);
             }
         }
     }
@@ -933,7 +1053,7 @@ public class FlashArrayAdapter implements ProviderAdapter {
             request.setEntity(new StringEntity(data));
 
             CloseableHttpClient client = getClient();
-            response = (CloseableHttpResponse) client.execute(request);
+            response = client.execute(request);
 
             final int statusCode = response.getStatusLine().getStatusCode();
             if (statusCode == 200 || statusCode == 201) {
@@ -988,7 +1108,7 @@ public class FlashArrayAdapter implements ProviderAdapter {
             request.addHeader("X-auth-token", getAccessToken());
 
             CloseableHttpClient client = getClient();
-            response = (CloseableHttpResponse) client.execute(request);
+            response = client.execute(request);
             final int statusCode = response.getStatusLine().getStatusCode();
             if (statusCode == 200) {
                 try {
@@ -1030,7 +1150,7 @@ public class FlashArrayAdapter implements ProviderAdapter {
             request.addHeader("X-auth-token", getAccessToken());
 
             CloseableHttpClient client = getClient();
-            response = (CloseableHttpResponse) client.execute(request);
+            response = client.execute(request);
             final int statusCode = response.getStatusLine().getStatusCode();
             if (statusCode == 200 || statusCode == 404 || statusCode == 400) {
                 // this means the volume was deleted successfully, or doesn't exist (effective

plugins/storage/volume/linstor/CHANGELOG.md

@@ -24,6 +24,14 @@ All notable changes to Linstor CloudStack plugin will be documented in this file
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2026-06-24]
+
+### Fixed
+
+- Restore of encrypted volume snapshots: snapshots of encrypted volumes are now
+  stored as LUKS-encrypted qcow2 files and decrypted on revert (previously the
+  restored data was corrupted and the root device unbootable).
+
 ## [2026-06-03]
 
 ### Added

plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java

@@ -18,6 +18,10 @@ package com.cloud.hypervisor.kvm.resource.wrapper;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 import com.cloud.agent.api.to.DataStoreTO;
 import com.cloud.agent.api.to.NfsTO;
@@ -31,9 +35,11 @@ import com.cloud.storage.Storage;
 import com.cloud.utils.script.Script;
 import org.apache.cloudstack.storage.command.CopyCmdAnswer;
 import org.apache.cloudstack.storage.to.SnapshotObjectTO;
+import org.apache.cloudstack.utils.cryptsetup.KeyFile;
 import org.apache.cloudstack.utils.qemu.QemuImg;
 import org.apache.cloudstack.utils.qemu.QemuImgException;
 import org.apache.cloudstack.utils.qemu.QemuImgFile;
+import org.apache.cloudstack.utils.qemu.QemuObject;
 import org.apache.commons.io.FileUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -83,6 +89,7 @@ public final class LinstorBackupSnapshotCommandWrapper
         final String srcPath,
         final SnapshotObjectTO dst,
         final KVMStoragePool secondaryPool,
+        final byte[] passphrase,
         int waitMilliSeconds
         )
         throws LibvirtException, QemuImgException, IOException
@@ -94,9 +101,22 @@ public final class LinstorBackupSnapshotCommandWrapper
         final QemuImgFile srcFile = new QemuImgFile(srcPath, QemuImg.PhysicalDiskFormat.RAW);
         final QemuImgFile dstFile = new QemuImgFile(dstPath, QemuImg.PhysicalDiskFormat.QCOW2);
 
-        // NOTE: the qemu img will also contain the drbd metadata at the end
         final QemuImg qemu = new QemuImg(waitMilliSeconds);
-        qemu.convert(srcFile, dstFile);
+        if (passphrase != null && passphrase.length > 0) {
+            // Encrypted volumes are backed up from their decrypted DRBD device, so the snapshot
+            // data here is plaintext. Encrypt the destination qcow2 with the volume's passphrase
+            // (LUKS), so the snapshot is not stored in clear text on secondary storage.
+            try (KeyFile keyFile = new KeyFile(passphrase)) {
+                final Map<String, String> options = new HashMap<>();
+                final List<QemuObject> qemuObjects = new ArrayList<>();
+                qemuObjects.add(QemuObject.prepareSecretForQemuImg(QemuImg.PhysicalDiskFormat.QCOW2,
+                    QemuObject.EncryptFormat.LUKS, keyFile.toString(), "sec0", options));
+                qemu.convert(srcFile, dstFile, options, qemuObjects, null, true);
+            }
+        } else {
+            // NOTE: the qemu img will also contain the drbd metadata at the end
+            qemu.convert(srcFile, dstFile);
+        }
         LOGGER.info("Backup snapshot '{}' to '{}'", srcPath, dstPath);
         return dstPath;
     }
@@ -153,14 +173,21 @@ public final class LinstorBackupSnapshotCommandWrapper
 
             secondaryPool = storagePoolMgr.getStoragePoolByURI(dstDataStore.getUrl());
 
-            String dstPath = convertImageToQCow2(srcPath, dst, secondaryPool, cmd.getWaitInMillSeconds());
+            final byte[] passphrase = src.getVolume() != null ? src.getVolume().getPassphrase() : null;
+            final boolean encrypted = passphrase != null && passphrase.length > 0;
 
-            // resize to real volume size, cutting of drbd metadata
-            String result = qemuShrink(dstPath, src.getVolume().getSize(), cmd.getWaitInMillSeconds());
-            if (result != null) {
-                return new CopyCmdAnswer("qemu-img shrink failed: " + result);
+            String dstPath = convertImageToQCow2(srcPath, dst, secondaryPool, passphrase, cmd.getWaitInMillSeconds());
+
+            if (!encrypted) {
+                // resize to real volume size, cutting of drbd metadata
+                // For encrypted volumes the source is the decrypted DRBD device (already net-sized,
+                // no drbd metadata to cut); shrinking an encrypted qcow2 would also need the secret.
+                String result = qemuShrink(dstPath, src.getVolume().getSize(), cmd.getWaitInMillSeconds());
+                if (result != null) {
+                    return new CopyCmdAnswer("qemu-img shrink failed: " + result);
+                }
+                LOGGER.info("Backup shrunk " + dstPath + " to actual size " + src.getVolume().getSize());
             }
-            LOGGER.info("Backup shrunk " + dstPath + " to actual size " + src.getVolume().getSize());
 
             SnapshotObjectTO snapshot = setCorrectSnapshotSize(dst, dstPath);
             LOGGER.info("Actual file size for '{}' is {}", dstPath, snapshot.getPhysicalSize());
@@ -171,6 +198,9 @@ public final class LinstorBackupSnapshotCommandWrapper
             LOGGER.error(error);
             return new CopyCmdAnswer(cmd, e);
         } finally {
+            if (src.getVolume() != null) {
+                src.getVolume().clearPassphrase();
+            }
             cleanupSecondaryPool(secondaryPool);
             if (zfsHidden) {
                 zfsSnapdev(true, src.getPath());

plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorRevertBackupSnapshotCommandWrapper.java

@@ -17,6 +17,7 @@
 package com.cloud.hypervisor.kvm.resource.wrapper;
 
 import java.io.File;
+import java.util.Collections;
 
 import com.cloud.agent.api.to.DataStoreTO;
 import com.cloud.api.storage.LinstorRevertBackupSnapshotCommand;
@@ -31,9 +32,12 @@ import org.apache.cloudstack.storage.command.CopyCmdAnswer;
 import org.apache.cloudstack.storage.datastore.util.LinstorUtil;
 import org.apache.cloudstack.storage.to.SnapshotObjectTO;
 import org.apache.cloudstack.storage.to.VolumeObjectTO;
+import org.apache.cloudstack.utils.cryptsetup.KeyFile;
+import org.apache.cloudstack.utils.qemu.QemuImageOptions;
 import org.apache.cloudstack.utils.qemu.QemuImg;
 import org.apache.cloudstack.utils.qemu.QemuImgException;
 import org.apache.cloudstack.utils.qemu.QemuImgFile;
+import org.apache.cloudstack.utils.qemu.QemuObject;
 import org.joda.time.Duration;
 import org.libvirt.LibvirtException;
 
@@ -43,8 +47,9 @@ public final class LinstorRevertBackupSnapshotCommandWrapper
 {
 
     private void convertQCow2ToRAW(
-            KVMStoragePool pool, final String srcPath, final String dstUuid, int waitMilliSeconds)
-        throws LibvirtException, QemuImgException
+            KVMStoragePool pool, final String srcPath, final String dstUuid, final byte[] passphrase,
+            int waitMilliSeconds)
+        throws LibvirtException, QemuImgException, java.io.IOException
     {
         final String dstPath = pool.getPhysicalDisk(dstUuid).getPath();
         final QemuImgFile srcQemuFile = new QemuImgFile(
@@ -60,7 +65,20 @@ public final class LinstorRevertBackupSnapshotCommandWrapper
         }
         final QemuImg qemu = new QemuImg(waitMilliSeconds, zeroedDevice, true);
         final QemuImgFile dstFile = new QemuImgFile(dstPath, QemuImg.PhysicalDiskFormat.RAW);
-        qemu.convert(srcQemuFile, dstFile);
+        if (passphrase != null && passphrase.length > 0) {
+            // The backed-up qcow2 is LUKS-encrypted with the volume's passphrase. Decrypt it while
+            // writing plaintext to the (decrypted) DRBD device; the Linstor LUKS layer re-encrypts it,
+            // so no qemu encryption must be applied to the destination.
+            try (KeyFile keyFile = new KeyFile(passphrase)) {
+                final QemuObject srcSecret = QemuObject.prepareSecretForQemuImg(
+                    QemuImg.PhysicalDiskFormat.QCOW2, QemuObject.EncryptFormat.LUKS, keyFile.toString(), "sec0", null);
+                final QemuImageOptions srcImageOpts = new QemuImageOptions(
+                    QemuImg.PhysicalDiskFormat.QCOW2, srcPath, "sec0");
+                qemu.convert(srcQemuFile, dstFile, null, Collections.singletonList(srcSecret), srcImageOpts, null, false);
+            }
+        } else {
+            qemu.convert(srcQemuFile, dstFile);
+        }
     }
 
     @Override
@@ -84,10 +102,13 @@ public final class LinstorRevertBackupSnapshotCommandWrapper
             secondaryPool = storagePoolMgr.getStoragePoolByURI(
                 srcDataStore.getUrl() + File.separator + srcFile.getParent());
 
+            // The destination volume is the (same) original volume, whose passphrase the backed-up
+            // qcow2 was encrypted with; use it to decrypt while restoring.
             convertQCow2ToRAW(
                 linstorPool,
                 secondaryPool.getLocalPath() + File.separator + srcFile.getName(),
                 dst.getPath(),
+                dst.getPassphrase(),
                 cmd.getWaitInMillSeconds());
 
             final VolumeObjectTO dstVolume = new VolumeObjectTO();
@@ -99,6 +120,7 @@ public final class LinstorRevertBackupSnapshotCommandWrapper
             logger.error(error);
             return new CopyCmdAnswer(cmd, e);
         } finally {
+            dst.clearPassphrase();
             LinstorBackupSnapshotCommandWrapper.cleanupSecondaryPool(secondaryPool);
         }
     }

plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java

@@ -1095,12 +1095,22 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
                 VirtualMachineManager.ExecuteInSequence.value());
             cmd.setOptions(options);
 
-            Optional<RemoteHostEndPoint> optEP = getDiskfullEP(api, pool, rscName);
+            // For encrypted volumes Linstor adds a LUKS layer (DRBD -> LUKS -> STORAGE). The storage
+            // layer snapshot device (getSnapshotPath) therefore only exposes the raw LUKS ciphertext,
+            // while restore writes onto the decrypted DRBD device (/dev/drbd/by-res/.../0). Backing up
+            // the ciphertext and writing it back to the decrypted layer corrupts the volume (and the
+            // shrink to the net volume size would even truncate the ciphertext). So for encrypted
+            // volumes we never read the storage snapshot directly: restore the snapshot into a temporary
+            // resource and back up its decrypted DRBD device instead, symmetric to the restore path.
+            final boolean encrypted = snapshotObject.getBaseVolume().getPassphraseId() != null;
+            Optional<RemoteHostEndPoint> optEP = encrypted ?
+                Optional.empty() : getDiskfullEP(api, pool, rscName);
             Answer answer;
             if (optEP.isPresent()) {
                 answer = optEP.get().sendMessage(cmd);
             } else {
-                logger.debug("No diskfull endpoint found to copy image, creating diskless endpoint");
+                logger.debug("No diskfull endpoint used to copy image (encrypted={}), using temporary resource",
+                    encrypted);
                 answer = copyFromTemporaryResource(api, pool, rscName, snapshotName, snapshotObject, cmd);
             }
             return answer;

scripts/storage/secondary/createtmplt.sh

@@ -218,7 +218,7 @@ imgsize=$(ls -l $tmpltimg2| awk -F" " '{print $5}')
 if [ "$cloud" == "true" ]
 then
     create_from_file_user $tmpltfs $tmpltimg2 $tmpltname
-    tmpltfs=/tmp/cloud/templates/
+    tmpltfs=/var/tmp/cloud/templates/
 else
     create_from_file $tmpltfs $tmpltimg2 $tmpltname
 fi

scripts/storage/secondary/setup-sysvm-tmplt

@@ -105,7 +105,7 @@ if [[ "$destfiles" != "" ]]; then
   failed 2 "Data already exists at destination $destdir"
 fi
 
-tmpfolder=/tmp/cloud/templates/
+tmpfolder=/var/tmp/cloud/templates/
 mkdir -p $tmpfolder
 tmplfile=$tmpfolder/$localfile
 

server/src/main/java/com/cloud/api/ApiServer.java

@@ -283,11 +283,11 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer
             , "Do URL encoding for the api response, false by default"
             , false
             , ConfigKey.Scope.Global);
-    static final ConfigKey<String> JSONcontentType = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED
+    static final ConfigKey<String> JSONContentType = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED
             , String.class
             , "json.content.type"
             , "application/json; charset=UTF-8"
-            , "Http response content type for .js files (default is text/javascript)"
+            , "Http response content type for JSON"
             , false
             , ConfigKey.Scope.Global);
     static final ConfigKey<Boolean> EnableSecureSessionCookie = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED
@@ -1490,7 +1490,7 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer
             final BasicHttpEntity body = new BasicHttpEntity();
             if (HttpUtils.RESPONSE_TYPE_JSON.equalsIgnoreCase(responseType)) {
                 // JSON response
-                body.setContentType(JSONcontentType.value());
+                body.setContentType(JSONContentType.value());
                 if (responseText == null) {
                     body.setContent(new ByteArrayInputStream("{ \"error\" : { \"description\" : \"Internal Server Error\" } }".getBytes(HttpUtils.UTF_8)));
                 }
@@ -1728,7 +1728,7 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer
                 ConcurrentSnapshotsThresholdPerHost,
                 EncodeApiResponse,
                 EnableSecureSessionCookie,
-                JSONDefaultContentType,
+                JSONContentType,
                 proxyForwardList,
                 useForwardHeader,
                 listOfForwardHeaders,

server/src/main/java/com/cloud/api/ApiServlet.java

@@ -216,7 +216,7 @@ public class ApiServlet extends HttpServlet {
                     "UnknownHostException when trying to lookup remote IP-Address", null,
                     HttpUtils.RESPONSE_TYPE_XML);
             HttpUtils.writeHttpResponse(resp, response, HttpServletResponse.SC_INTERNAL_SERVER_ERROR,
-                    HttpUtils.RESPONSE_TYPE_XML, ApiServer.JSONcontentType.value());
+                    HttpUtils.RESPONSE_TYPE_XML, ApiServer.JSONContentType.value());
             return;
         }
 
@@ -342,7 +342,7 @@ public class ApiServlet extends HttpServlet {
                             }
                         }
                     }
-                    HttpUtils.writeHttpResponse(resp, responseString, httpResponseCode, responseType, ApiServer.JSONcontentType.value());
+                    HttpUtils.writeHttpResponse(resp, responseString, httpResponseCode, responseType, ApiServer.JSONContentType.value());
                     return;
                 }
             } else {
@@ -377,7 +377,7 @@ public class ApiServlet extends HttpServlet {
                 final String serializedResponse =
                         apiServer.getSerializedApiError(new ServerApiException(ApiErrorCode.BAD_REQUEST, errorText), params,
                                 responseType);
-                HttpUtils.writeHttpResponse(resp, serializedResponse, HttpServletResponse.SC_BAD_REQUEST, responseType, ApiServer.JSONcontentType.value());
+                HttpUtils.writeHttpResponse(resp, serializedResponse, HttpServletResponse.SC_BAD_REQUEST, responseType, ApiServer.JSONContentType.value());
                 return;
             }
 
@@ -415,7 +415,7 @@ public class ApiServlet extends HttpServlet {
                 setProjectContext(params);
                 setClientAddressForConsoleEndpointAccess(command, params, req);
                 final String response = apiServer.handleRequest(params, responseType, auditTrailSb);
-                HttpUtils.writeHttpResponse(resp, response != null ? response : "", HttpServletResponse.SC_OK, responseType, ApiServer.JSONcontentType.value());
+                HttpUtils.writeHttpResponse(resp, response != null ? response : "", HttpServletResponse.SC_OK, responseType, ApiServer.JSONContentType.value());
             } else {
                 if (session != null) {
                     invalidateHttpSession(session, String.format("request verification failed for %s from %s", userId, remoteAddress.getHostAddress()));
@@ -425,12 +425,12 @@ public class ApiServlet extends HttpServlet {
                 final String serializedResponse =
                         apiServer.getSerializedApiError(HttpServletResponse.SC_UNAUTHORIZED, "unable to verify user credentials and/or request signature", params,
                                 responseType);
-                HttpUtils.writeHttpResponse(resp, serializedResponse, HttpServletResponse.SC_UNAUTHORIZED, responseType, ApiServer.JSONcontentType.value());
+                HttpUtils.writeHttpResponse(resp, serializedResponse, HttpServletResponse.SC_UNAUTHORIZED, responseType, ApiServer.JSONContentType.value());
             }
         } catch (final ServerApiException se) {
             final String serializedResponseText = apiServer.getSerializedApiError(se, params, responseType);
             resp.setHeader("X-Description", se.getDescription());
-            HttpUtils.writeHttpResponse(resp, serializedResponseText, se.getErrorCode().getHttpCode(), responseType, ApiServer.JSONcontentType.value());
+            HttpUtils.writeHttpResponse(resp, serializedResponseText, se.getErrorCode().getHttpCode(), responseType, ApiServer.JSONContentType.value());
             auditTrailSb.append(" " + se.getErrorCode() + " " + se.getDescription());
         } catch (final Exception ex) {
             LOGGER.error("unknown exception writing api response", ex);
@@ -539,7 +539,7 @@ public class ApiServlet extends HttpServlet {
             if (apiAuthenticator != null) {
                 String responseString = apiAuthenticator.authenticate(command, params, session, remoteAddress, responseType, auditTrailSb, req, resp);
                 session.setAttribute(ApiConstants.IS_2FA_VERIFIED, true);
-                HttpUtils.writeHttpResponse(resp, responseString, HttpServletResponse.SC_OK, responseType, ApiServer.JSONcontentType.value());
+                HttpUtils.writeHttpResponse(resp, responseString, HttpServletResponse.SC_OK, responseType, ApiServer.JSONContentType.value());
                 verify2FA = true;
             } else {
                 LOGGER.error("Cannot find API authenticator while verifying 2FA");
@@ -571,7 +571,7 @@ public class ApiServlet extends HttpServlet {
             invalidateHttpSession(session, String.format("Unable to process the API request for %s from %s due to %s", userId, remoteAddress.getHostAddress(), errorMsg));
             auditTrailSb.append(" " + ApiErrorCode.UNAUTHORIZED2FA + " " + errorMsg);
             final String serializedResponse = apiServer.getSerializedApiError(ApiErrorCode.UNAUTHORIZED2FA.getHttpCode(), "Unable to process the API request due to :" + errorMsg, params, responseType);
-            HttpUtils.writeHttpResponse(resp, serializedResponse, ApiErrorCode.UNAUTHORIZED2FA.getHttpCode(), responseType, ApiServer.JSONcontentType.value());
+            HttpUtils.writeHttpResponse(resp, serializedResponse, ApiErrorCode.UNAUTHORIZED2FA.getHttpCode(), responseType, ApiServer.JSONContentType.value());
             verify2FA = false;
         }
 
@@ -600,7 +600,7 @@ public class ApiServlet extends HttpServlet {
                 LOGGER.info("missing command, ignoring request...");
                 auditTrailSb.append(" " + HttpServletResponse.SC_BAD_REQUEST + " " + "no command specified");
                 final String serializedResponse = apiServer.getSerializedApiError(HttpServletResponse.SC_BAD_REQUEST, "no command specified", params, responseType);
-                HttpUtils.writeHttpResponse(resp, serializedResponse, HttpServletResponse.SC_BAD_REQUEST, responseType, ApiServer.JSONcontentType.value());
+                HttpUtils.writeHttpResponse(resp, serializedResponse, HttpServletResponse.SC_BAD_REQUEST, responseType, ApiServer.JSONContentType.value());
                 return true;
             }
             final User user = entityMgr.findById(User.class, userId);
@@ -611,7 +611,7 @@ public class ApiServlet extends HttpServlet {
             auditTrailSb.append(" " + HttpServletResponse.SC_UNAUTHORIZED + " " + "unable to verify user credentials");
             final String serializedResponse =
                     apiServer.getSerializedApiError(HttpServletResponse.SC_UNAUTHORIZED, "unable to verify user credentials", params, responseType);
-            HttpUtils.writeHttpResponse(resp, serializedResponse, HttpServletResponse.SC_UNAUTHORIZED, responseType, ApiServer.JSONcontentType.value());
+            HttpUtils.writeHttpResponse(resp, serializedResponse, HttpServletResponse.SC_UNAUTHORIZED, responseType, ApiServer.JSONContentType.value());
             return false;
         }
         return true;
@@ -626,7 +626,7 @@ public class ApiServlet extends HttpServlet {
             auditTrailSb.append(" " + HttpServletResponse.SC_UNAUTHORIZED + " " + "unable to verify user credentials");
             final String serializedResponse =
                     apiServer.getSerializedApiError(HttpServletResponse.SC_UNAUTHORIZED, "unable to verify user credentials", params, responseType);
-            HttpUtils.writeHttpResponse(resp, serializedResponse, HttpServletResponse.SC_UNAUTHORIZED, responseType, ApiServer.JSONcontentType.value());
+            HttpUtils.writeHttpResponse(resp, serializedResponse, HttpServletResponse.SC_UNAUTHORIZED, responseType, ApiServer.JSONContentType.value());
             return true;
         }
         return false;

server/src/main/java/com/cloud/configuration/Config.java

@@ -289,15 +289,6 @@ public enum Config {
             "cloud-public",
             "Default network label to be used when fetching interface for GRE endpoints",
             null),
-    VmNetworkThrottlingRate(
-            "Network",
-            ManagementServer.class,
-            Integer.class,
-            "vm.network.throttling.rate",
-            "200",
-            "Default data transfer rate in megabits per second allowed in User vm's default network.",
-            null),
-
     SecurityGroupWorkCleanupInterval(
             "Network",
             ManagementServer.class,

server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -8269,7 +8269,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
             if (offering.getVmType() != null && offering.getVmType().equalsIgnoreCase(VirtualMachine.Type.DomainRouter.toString())) {
                 networkRate = NetworkOrchestrationService.NetworkThrottlingRate.valueIn(dataCenterId);
             } else {
-                networkRate = Integer.parseInt(_configDao.getValue(Config.VmNetworkThrottlingRate.key()));
+                networkRate = NetworkOrchestrationService.VmNetworkThrottlingRate.valueIn(dataCenterId);
             }
         }
 

server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java

@@ -1486,12 +1486,9 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy
             return false;
         }
 
-        List<ConsoleProxyVO> l = consoleProxyDao.getProxyListInStates(State.Starting, State.Stopping);
-        if (l.size() > 0) {
-            if (logger.isDebugEnabled()) {
-                logger.debug("Zone {} has {} console proxy VM(s) in transition state", zone, l.size());
-            }
-
+        List<ConsoleProxyVO> consoleProxiesInTransitionStates = consoleProxyDao.getProxyListInStates(dataCenterId, State.Starting, State.Stopping);
+        if (!consoleProxiesInTransitionStates.isEmpty()) {
+            logger.debug("Zone {} has {} console proxy VM(s) in transition state.", zone, consoleProxiesInTransitionStates.size());
             return false;
         }
 

server/src/main/java/com/cloud/network/element/VirtualRouterElement.java

@@ -24,28 +24,21 @@ import java.util.Set;
 
 import javax.inject.Inject;
 
-import org.apache.cloudstack.network.BgpPeer;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.ObjectUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-
-import com.cloud.storage.dao.VMTemplateDao;
-import com.cloud.vm.VirtualMachineProfileImpl;
-import com.cloud.vm.VmDetailConstants;
-import com.cloud.vm.dao.NicDao;
-import com.google.gson.Gson;
-
 import org.apache.cloudstack.api.command.admin.router.ConfigureOvsElementCmd;
 import org.apache.cloudstack.api.command.admin.router.ConfigureVirtualRouterElementCmd;
 import org.apache.cloudstack.api.command.admin.router.CreateVirtualRouterElementCmd;
 import org.apache.cloudstack.api.command.admin.router.ListOvsElementsCmd;
 import org.apache.cloudstack.api.command.admin.router.ListVirtualRouterElementsCmd;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.cloudstack.network.BgpPeer;
 import org.apache.cloudstack.network.router.deployment.RouterDeploymentDefinition;
 import org.apache.cloudstack.network.router.deployment.RouterDeploymentDefinitionBuilder;
 import org.apache.cloudstack.network.topology.NetworkTopology;
 import org.apache.cloudstack.network.topology.NetworkTopologyContext;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.ObjectUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 
 import com.cloud.agent.api.to.LoadBalancerTO;
 import com.cloud.configuration.ConfigurationManager;
@@ -101,6 +94,7 @@ import com.cloud.network.vpc.Vpc;
 import com.cloud.offering.NetworkOffering;
 import com.cloud.offerings.NetworkOfferingVO;
 import com.cloud.offerings.dao.NetworkOfferingDao;
+import com.cloud.storage.dao.VMTemplateDao;
 import com.cloud.user.Account;
 import com.cloud.user.AccountManager;
 import com.cloud.utils.component.AdapterBase;
@@ -117,8 +111,12 @@ import com.cloud.vm.UserVmVO;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachine.State;
 import com.cloud.vm.VirtualMachineProfile;
+import com.cloud.vm.VirtualMachineProfileImpl;
+import com.cloud.vm.VmDetailConstants;
 import com.cloud.vm.dao.DomainRouterDao;
+import com.cloud.vm.dao.NicDao;
 import com.cloud.vm.dao.UserVmDao;
+import com.google.gson.Gson;
 
 public class VirtualRouterElement extends AdapterBase implements VirtualRouterElementService, DhcpServiceProvider, UserDataServiceProvider, SourceNatServiceProvider,
 StaticNatServiceProvider, FirewallServiceProvider, LoadBalancingServiceProvider, PortForwardingServiceProvider, RemoteAccessVPNServiceProvider, IpDeployer,
@@ -738,7 +736,7 @@ NetworkMigrationResponder, AggregatedCommandExecutor, RedundantResource, DnsServ
 
         _userVmDao.loadDetails(userVmVO);
         userVmVO.setDetail(VmDetailConstants.PASSWORD, password_encrypted);
-        _userVmDao.saveDetails(userVmVO);
+        _userVmDao.saveDetails(userVmVO, List.of(VmDetailConstants.PASSWORD));
 
         userVmVO.setUpdateParameters(true);
         _userVmDao.update(userVmVO.getId(), userVmVO);

server/src/main/java/com/cloud/resource/ResourceManagerImpl.java

@@ -1610,7 +1610,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
      */
     private void migrateAwayVmWithVolumes(HostVO host, VMInstanceVO vm) {
         final DataCenterDeployment plan = new DataCenterDeployment(host.getDataCenterId(), host.getPodId(), host.getClusterId(), null, null, null);
-        ServiceOfferingVO offeringVO = serviceOfferingDao.findById(vm.getServiceOfferingId());
+        ServiceOfferingVO offeringVO = serviceOfferingDao.findById(vm.getId(), vm.getServiceOfferingId());
         final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm, null, offeringVO, null, null);
         plan.setMigrationPlan(true);
         DeployDestination dest = getDeployDestination(vm, profile, plan, host);
@@ -2465,7 +2465,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
             List<HostVO> hostsInZone = _hostDao.findByDataCenterId(zoneId);
             Set<Long> hostIdsInUseSet = hostIdsUsingStorageAccessGroups.stream().collect(Collectors.toSet());
 
-            boolean allInUseZone = hostsInZone.stream()
+            // allMatch returns true on empty stream, need to check whether collection is not empty first
+            boolean allInUseZone = !hostsInZone.isEmpty() && hostsInZone.stream()
                     .map(HostVO::getId)
                     .allMatch(hostIdsInUseSet::contains);
 
@@ -2479,7 +2480,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
             List<HostVO> hostsInCluster = _hostDao.findByClusterId(clusterId, Type.Routing);
             Set<Long> hostIdsInUseSet = hostIdsUsingStorageAccessGroups.stream().collect(Collectors.toSet());
 
-            boolean allInUseCluster = hostsInCluster.stream()
+            // allMatch returns true on empty stream, need to check whether collection is not empty first
+            boolean allInUseCluster = !hostsInCluster.isEmpty() && hostsInCluster.stream()
                     .map(HostVO::getId)
                     .allMatch(hostIdsInUseSet::contains);
 
@@ -2493,7 +2495,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
             List<HostVO> hostsInPod = _hostDao.findByPodId(podId, Type.Routing);
             Set<Long> hostIdsInUseSet = hostIdsUsingStorageAccessGroups.stream().collect(Collectors.toSet());
 
-            boolean allInUsePod = hostsInPod.stream()
+            // allMatch returns true on empty stream, need to check whether collection is not empty first
+            boolean allInUsePod = !hostsInPod.isEmpty() && hostsInPod.stream()
                     .map(HostVO::getId)
                     .allMatch(hostIdsInUseSet::contains);
 

server/src/main/java/com/cloud/server/ManagementServerImpl.java

@@ -1500,6 +1500,7 @@ public class ManagementServerImpl extends MutualExclusiveIdsManagerBase implemen
      */
     Ternary<Pair<List<? extends Host>, Integer>, List<? extends Host>, Map<Host, Boolean>> getTechnicallyCompatibleHosts(
             final VirtualMachine vm,
+            final Host srcHost,
             final Long startIndex,
             final Long pageSize,
             final String keyword) {
@@ -1510,31 +1511,6 @@ public class ManagementServerImpl extends MutualExclusiveIdsManagerBase implemen
             return new Ternary<>(new Pair<>(new ArrayList<>(), 0), new ArrayList<>(), new HashMap<>());
         }
 
-        final long srcHostId = vm.getHostId();
-        final Host srcHost = _hostDao.findById(srcHostId);
-        if (srcHost == null) {
-            if (logger.isDebugEnabled()) {
-                logger.debug("Unable to find the host with ID: " + srcHostId + " of this Instance: " + vm);
-            }
-            final InvalidParameterValueException ex = new InvalidParameterValueException("Unable to find the host (with specified ID) of instance with specified ID");
-            ex.addProxyObject(String.valueOf(srcHostId), "hostId");
-            ex.addProxyObject(vm.getUuid(), "vmId");
-            throw ex;
-        }
-
-        String srcHostVersion = srcHost.getHypervisorVersion();
-        if (HypervisorType.KVM.equals(srcHost.getHypervisorType()) && srcHostVersion == null) {
-            srcHostVersion = "";
-        }
-
-        // Check if the vm can be migrated with storage.
-        boolean canMigrateWithStorage = false;
-
-        List<HypervisorType> hypervisorTypes = Arrays.asList(new HypervisorType[]{HypervisorType.VMware, HypervisorType.KVM});
-        if (VirtualMachine.Type.User.equals(vm.getType()) || hypervisorTypes.contains(vm.getHypervisorType())) {
-            canMigrateWithStorage = _hypervisorCapabilitiesDao.isStorageMotionSupported(srcHost.getHypervisorType(), srcHostVersion);
-        }
-
         // Check if the vm is using any disks on local storage.
         final VirtualMachineProfile vmProfile = new VirtualMachineProfileImpl(vm, null, _offeringDao.findById(vm.getId(), vm.getServiceOfferingId()), null, null);
         final List<VolumeVO> volumes = _volumeDao.findCreatedByInstance(vmProfile.getId());
@@ -1548,12 +1524,14 @@ public class ManagementServerImpl extends MutualExclusiveIdsManagerBase implemen
             }
         }
 
+        boolean canMigrateWithStorage = isStorageMigrationSupported(vm, srcHost);
         if (!canMigrateWithStorage && usesLocal) {
             throw new InvalidParameterValueException("Unsupported operation, instance uses Local storage, cannot migrate");
         }
 
         validateVgpuProfileForVmMigration(vmProfile);
 
+        final String srcHostVersion = getHypervisorVersionOfHost(srcHost);
         final Type hostType = srcHost.getType();
         Pair<List<HostVO>, Integer> allHostsPair;
         List<HostVO> allHosts;
@@ -1629,6 +1607,23 @@ public class ManagementServerImpl extends MutualExclusiveIdsManagerBase implemen
         return new Ternary<>(allHostsPairResult, filteredHosts, requiresStorageMotion);
     }
 
+    protected boolean isStorageMigrationSupported(final VirtualMachine vm, final Host srcHost) {
+        final List<HypervisorType> hypervisorTypes = Arrays.asList(HypervisorType.VMware, HypervisorType.KVM);
+        if (VirtualMachine.Type.User.equals(vm.getType()) || hypervisorTypes.contains(vm.getHypervisorType())) {
+            final String srcHostVersion = getHypervisorVersionOfHost(srcHost);
+            return _hypervisorCapabilitiesDao.isStorageMotionSupported(srcHost.getHypervisorType(), srcHostVersion);
+        }
+        return false;
+    }
+
+    protected String getHypervisorVersionOfHost(final Host host) {
+        final String version = host.getHypervisorVersion();
+        if (version == null && HypervisorType.KVM.equals(host.getHypervisorType())) {
+            return "";
+        }
+        return version;
+    }
+
     /**
      * Apply affinity group constraints and other exclusion rules for VM migration.
      * This builds an ExcludeList based on affinity groups, DPDK requirements, and dedicated resources.
@@ -1725,9 +1720,19 @@ public class ManagementServerImpl extends MutualExclusiveIdsManagerBase implemen
 
         validateVmForHostMigration(vm);
 
+        final long srcHostId = vm.getHostId();
+        final Host srcHost = _hostDao.findById(srcHostId);
+        if (srcHost == null) {
+            logger.debug("Unable to find the host with ID: {} of this Instance: {}", srcHostId, vm);
+            final InvalidParameterValueException ex = new InvalidParameterValueException("Unable to find the host (with specified ID) of instance with specified ID");
+            ex.addProxyObject(String.valueOf(srcHostId), "hostId");
+            ex.addProxyObject(vm.getUuid(), "vmId");
+            throw ex;
+        }
+
         // Get technically compatible hosts (storage, hypervisor, UEFI)
         Ternary<Pair<List<? extends Host>, Integer>, List<? extends Host>, Map<Host, Boolean>> compatibilityResult =
-            getTechnicallyCompatibleHosts(vm, startIndex, pageSize, keyword);
+            getTechnicallyCompatibleHosts(vm, srcHost, startIndex, pageSize, keyword);
 
         Pair<List<? extends Host>, Integer> allHostsPair = compatibilityResult.first();
         List<? extends Host> filteredHosts = compatibilityResult.second();
@@ -1740,9 +1745,7 @@ public class ManagementServerImpl extends MutualExclusiveIdsManagerBase implemen
         }
 
         // Create deployment plan and VM profile
-        final Host srcHost = _hostDao.findById(vm.getHostId());
-        final DataCenterDeployment plan = new DataCenterDeployment(
-            srcHost.getDataCenterId(), srcHost.getPodId(), srcHost.getClusterId(), null, null, null);
+        final DataCenterDeployment plan = createDeploymentPlanForMigrationListing(vm, srcHost);
         final VirtualMachineProfile vmProfile = new VirtualMachineProfileImpl(
             vm, null, _offeringDao.findById(vm.getId(), vm.getServiceOfferingId()), null, null);
 
@@ -1767,6 +1770,14 @@ public class ManagementServerImpl extends MutualExclusiveIdsManagerBase implemen
         }
     }
 
+    protected DataCenterDeployment createDeploymentPlanForMigrationListing(final VirtualMachine vm, final Host srcHost) {
+        final boolean canMigrateWithStorage = isStorageMigrationSupported(vm, srcHost);
+        if (canMigrateWithStorage) {
+            return new DataCenterDeployment(srcHost.getDataCenterId(), srcHost.getPodId(), null, null, null, null);
+        }
+        return new DataCenterDeployment(srcHost.getDataCenterId(), srcHost.getPodId(), srcHost.getClusterId(), null, null, null);
+    }
+
     /**
      * Add non DPDK enabled hosts to the avoid list
      */

server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java

@@ -1435,7 +1435,7 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager {
         accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm);
 
         if (vm.getBackupOfferingId() != null && !BackupEnableAttachDetachVolumes.value()) {
-            throw new CloudRuntimeException("The selected VM has backups, cannot restore and attach volume to the VM.");
+            throw new CloudRuntimeException("The selected VM is attached to a backup offering and, thus, it is not possible to restore and attach volumes from backups to the instance.");
         }
 
         if (backup.getZoneId() != vm.getDataCenterId()) {

server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java

@@ -51,11 +51,13 @@ import com.cloud.utils.db.GlobalLock;
 import com.cloud.utils.db.Transaction;
 import com.cloud.utils.db.TransactionCallback;
 import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.vm.VMInstanceDetailVO;
 import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachineProfile;
 import com.cloud.vm.VirtualMachineProfileImpl;
 import com.cloud.vm.VmDetailConstants;
+import com.cloud.vm.dao.VMInstanceDetailsDao;
 import com.cloud.vm.dao.VMInstanceDao;
 import org.apache.cloudstack.api.ApiCommandResourceType;
 import org.apache.cloudstack.api.ApiConstants;
@@ -78,7 +80,6 @@ import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO;
 import org.apache.cloudstack.jobs.JobInfo;
 import org.apache.cloudstack.managed.context.ManagedContextTimerTask;
 import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang3.time.DateUtils;
 
 import javax.inject.Inject;
@@ -134,6 +135,9 @@ public class ClusterDrsServiceImpl extends ManagerBase implements ClusterDrsServ
     @Inject
     ServiceOfferingDao serviceOfferingDao;
 
+    @Inject
+    VMInstanceDetailsDao vmInstanceDetailsDao;
+
     @Inject
     ManagementServer managementServer;
 
@@ -475,12 +479,16 @@ public class ClusterDrsServiceImpl extends ManagerBase implements ClusterDrsServ
         Map<Long, List<? extends Host>> vmToCompatibleHostsCache = new HashMap<>();
         Map<Long, Map<Host, Boolean>> vmToStorageMotionCache = new HashMap<>();
 
+        List<Long> vmIds = vmList.stream().map(VirtualMachine::getId).collect(Collectors.toList());
+        Set<Long> skipDrsVmIds = vmInstanceDetailsDao.listDetailsForResourceIdsAndKey(vmIds, VmDetailConstants.SKIP_DRS)
+                .stream().filter(d -> "true".equalsIgnoreCase(d.getValue()))
+                .map(VMInstanceDetailVO::getResourceId)
+                .collect(Collectors.toSet());
+
         for (VirtualMachine vm : vmList) {
             // Skip ineligible VMs
-            if (vm.getType().isUsedBySystem() ||
-                vm.getState() != VirtualMachine.State.Running ||
-                (MapUtils.isNotEmpty(vm.getDetails()) &&
-                 "true".equalsIgnoreCase(vm.getDetails().get(VmDetailConstants.SKIP_DRS)))) {
+            if (shouldSkipVMForDRS(vm, skipDrsVmIds)) {
+                logger.debug("Skipping VM {} for DRS as it is ineligible.", vm);
                 continue;
             }
 
@@ -607,7 +615,7 @@ public class ClusterDrsServiceImpl extends ManagerBase implements ClusterDrsServ
             ExcludeList excludes = vmToExcludesMap.get(vm.getId());
 
             ServiceOffering serviceOffering = vmIdServiceOfferingMap.get(vm.getId());
-            if (skipDrs(vm, compatibleHosts, serviceOffering)) {
+            if (CollectionUtils.isEmpty(compatibleHosts) || serviceOffering == null) {
                 continue;
             }
 
@@ -633,21 +641,11 @@ public class ClusterDrsServiceImpl extends ManagerBase implements ClusterDrsServ
         return bestMigration;
     }
 
-    private boolean skipDrs(VirtualMachine vm, List<? extends Host> compatibleHosts, ServiceOffering serviceOffering) {
+    private boolean shouldSkipVMForDRS(VirtualMachine vm, Set<Long> skipDrsVmIds) {
         if (vm.getType().isUsedBySystem() || vm.getState() != VirtualMachine.State.Running) {
             return true;
         }
-        if (MapUtils.isNotEmpty(vm.getDetails()) &&
-            "true".equalsIgnoreCase(vm.getDetails().get(VmDetailConstants.SKIP_DRS))) {
-            return true;
-        }
-        if (CollectionUtils.isEmpty(compatibleHosts)) {
-            return true;
-        }
-        if (serviceOffering == null) {
-            return true;
-        }
-        return false;
+        return skipDrsVmIds.contains(vm.getId());
     }
 
     private Pair<double[], Map<Long, Integer>> getBaseMetricsArrayAndHostIdIndexMap(

server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java

@@ -1211,4 +1211,96 @@ public class ResourceManagerImplTest {
 
         Mockito.verify(resourceManager).doDeleteHost(hostId, false, false);
     }
+
+    @Test
+    public void testUpdateClusterStorageAccessGroupsWithEmptyHostsInCluster() {
+        Long clusterId = 1L;
+        List<String> newStorageAccessGroups = Arrays.asList("sag1", "sag2");
+
+        ClusterVO cluster = Mockito.mock(ClusterVO.class);
+        Mockito.when(cluster.getId()).thenReturn(clusterId);
+        Mockito.when(cluster.getStorageAccessGroups()).thenReturn("sag3,sag4"); // existing SAGs
+        Mockito.when(resourceManager.getCluster(clusterId)).thenReturn(cluster);
+        List<HostVO> emptyHostsList = new ArrayList<>();
+        Mockito.when(hostDao.findHypervisorHostInCluster(clusterId)).thenReturn(emptyHostsList);
+        Mockito.when(hostDao.findByClusterId(clusterId, Host.Type.Routing)).thenReturn(emptyHostsList);
+        List<Long> emptyHostIdsList = new ArrayList<>();
+        Mockito.doReturn(emptyHostIdsList).when(resourceManager)
+                .listOfHostIdsUsingTheStorageAccessGroups(Mockito.anyList(), eq(clusterId), eq(null), eq(null));
+        try {
+            resourceManager.updateClusterStorageAccessGroups(clusterId, newStorageAccessGroups);
+        } catch (CloudRuntimeException e) {
+            Assert.fail("updateClusterStorageAccessGroups should not throw CloudRuntimeException when cluster has no hosts. Error: " + e.getMessage());
+        }
+        Mockito.verify(resourceManager).checkIfAllHostsInUse(Mockito.anyList(), eq(clusterId), eq(null), eq(null));
+    }
+
+    @Test
+    public void testUpdateClusterStorageAccessGroupsWithEmptyHostsInZone() {
+        List<String> sagsToDelete = Arrays.asList("tag1", "tag2");
+        Long clusterId = null;
+        Long podId = null;
+        Long zoneId = 3L;
+
+        List<Long> emptyHostIdsList = new ArrayList<>();
+        Mockito.doReturn(emptyHostIdsList).when(resourceManager)
+                .listOfHostIdsUsingTheStorageAccessGroups(sagsToDelete, clusterId, podId, zoneId);
+        List<HostVO> emptyHostsInZone = new ArrayList<>();
+        Mockito.doReturn(emptyHostsInZone).when(hostDao).findByDataCenterId(zoneId);
+
+        try {
+            resourceManager.checkIfAllHostsInUse(sagsToDelete, clusterId, podId, zoneId);
+        } catch (CloudRuntimeException e) {
+            Assert.fail("checkIfAllHostsInUse should not throw CloudRuntimeException when zone has no hosts. Error: " + e.getMessage());
+        }
+        Mockito.verify(resourceManager).checkIfAllHostsInUse(Mockito.anyList(), eq(null), eq(null), eq(zoneId));
+    }
+
+    @Test
+    public void testUpdateClusterStorageAccessGroupsWithEmptyHostsInPod() {
+        List<String> sagsToDelete = Arrays.asList("tag1", "tag2");
+        Long clusterId = null;
+        Long podId = 2L;
+        Long zoneId = null;
+
+        List<Long> emptyHostIdsList = new ArrayList<>();
+        Mockito.doReturn(emptyHostIdsList).when(resourceManager)
+                .listOfHostIdsUsingTheStorageAccessGroups(sagsToDelete, clusterId, podId, zoneId);
+        List<HostVO> emptyHostsInPod = new ArrayList<>();
+        Mockito.doReturn(emptyHostsInPod).when(hostDao).findByPodId(podId, Host.Type.Routing);
+
+        try {
+            resourceManager.checkIfAllHostsInUse(sagsToDelete, clusterId, podId, zoneId);
+        } catch (CloudRuntimeException e) {
+            Assert.fail("checkIfAllHostsInUse should not throw CloudRuntimeException when pod has no hosts. Error: " + e.getMessage());
+        }
+        Mockito.verify(resourceManager).checkIfAllHostsInUse(Mockito.anyList(), eq(null), eq(podId), eq(null));
+    }
+
+    @Test
+    public void testCheckIfAllHostsInUseWithEmptyHostsInMultipleLevels() {
+        List<String> sagsToDelete = Arrays.asList("tag1", "tag2");
+        Long clusterId = 1L;
+        Long podId = 2L;
+        Long zoneId = 3L;
+
+        List<Long> emptyHostIdsList = new ArrayList<>();
+        Mockito.doReturn(emptyHostIdsList).when(resourceManager)
+                .listOfHostIdsUsingTheStorageAccessGroups(sagsToDelete, clusterId, podId, zoneId);
+        List<HostVO> emptyHostsInZone = new ArrayList<>();
+        List<HostVO> emptyHostsInCluster = new ArrayList<>();
+        List<HostVO> emptyHostsInPod = new ArrayList<>();
+        Mockito.doReturn(emptyHostsInZone).when(hostDao).findByDataCenterId(zoneId);
+        Mockito.doReturn(emptyHostsInCluster).when(hostDao).findByClusterId(clusterId, Host.Type.Routing);
+        Mockito.doReturn(emptyHostsInPod).when(hostDao).findByPodId(podId, Host.Type.Routing);
+
+        try {
+            resourceManager.checkIfAllHostsInUse(sagsToDelete, clusterId, podId, zoneId);
+        } catch (CloudRuntimeException e) {
+            Assert.fail("checkIfAllHostsInUse should not throw CloudRuntimeException when all levels have no hosts. Error: " + e.getMessage());
+        }
+        Mockito.verify(hostDao).findByDataCenterId(zoneId);
+        Mockito.verify(hostDao).findByClusterId(clusterId, Host.Type.Routing);
+        Mockito.verify(hostDao).findByPodId(podId, Host.Type.Routing);
+    }
 }

server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java

@@ -40,9 +40,11 @@ import com.cloud.utils.Pair;
 import com.cloud.utils.Ternary;
 import com.cloud.utils.db.GlobalLock;
 import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.vm.VMInstanceDetailVO;
 import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VmDetailConstants;
+import com.cloud.vm.dao.VMInstanceDetailsDao;
 import com.cloud.vm.dao.VMInstanceDao;
 import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
 import org.apache.cloudstack.api.command.admin.cluster.GenerateClusterDrsPlanCmd;
@@ -121,6 +123,9 @@ public class ClusterDrsServiceImplTest {
     @Mock
     private AffinityGroupVMMapDao affinityGroupVMMapDao;
 
+    @Mock
+    private VMInstanceDetailsDao vmInstanceDetailsDao;
+
     @Spy
     @InjectMocks
     private ClusterDrsServiceImpl clusterDrsService = new ClusterDrsServiceImpl();
@@ -294,6 +299,8 @@ public class ClusterDrsServiceImplTest {
 
         List<Ternary<VirtualMachine, Host, Host>> result = clusterDrsService.getDrsPlan(cluster, 5);
         assertEquals(0, result.size());
+        Mockito.verify(managementServer, Mockito.never()).listHostsForMigrationOfVM(
+                Mockito.eq(systemVm), Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.anyList());
     }
 
     @Test
@@ -334,6 +341,8 @@ public class ClusterDrsServiceImplTest {
 
         List<Ternary<VirtualMachine, Host, Host>> result = clusterDrsService.getDrsPlan(cluster, 5);
         assertEquals(0, result.size());
+        Mockito.verify(managementServer, Mockito.never()).listHostsForMigrationOfVM(
+                Mockito.eq(stoppedVm), Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.anyList());
     }
 
     @Test
@@ -350,9 +359,6 @@ public class ClusterDrsServiceImplTest {
         Mockito.when(skippedVm.getHostId()).thenReturn(1L);
         Mockito.when(skippedVm.getType()).thenReturn(VirtualMachine.Type.User);
         Mockito.when(skippedVm.getState()).thenReturn(VirtualMachine.State.Running);
-        Map<String, String> details = new HashMap<>();
-        details.put(VmDetailConstants.SKIP_DRS, "true");
-        Mockito.when(skippedVm.getDetails()).thenReturn(details);
 
         List<HostVO> hostList = new ArrayList<>();
         hostList.add(host1);
@@ -370,6 +376,11 @@ public class ClusterDrsServiceImplTest {
         Mockito.when(hostJoin1.getMemReservedCapacity()).thenReturn(0L);
         Mockito.when(hostJoin1.getTotalMemory()).thenReturn(8192L);
 
+        // Return the SKIP_DRS detail for skippedVm so the flag is actually honoured
+        VMInstanceDetailVO skipDrsDetail = new VMInstanceDetailVO(1L, VmDetailConstants.SKIP_DRS, "true", true);
+        Mockito.when(vmInstanceDetailsDao.listDetailsForResourceIdsAndKey(Mockito.anyList(),
+                Mockito.eq(VmDetailConstants.SKIP_DRS))).thenReturn(List.of(skipDrsDetail));
+
         Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList);
         Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList);
         Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList())).thenReturn(true);
@@ -377,6 +388,9 @@ public class ClusterDrsServiceImplTest {
 
         List<Ternary<VirtualMachine, Host, Host>> result = clusterDrsService.getDrsPlan(cluster, 5);
         assertEquals(0, result.size());
+        // Verify the VM was skipped before any host-compatibility lookup was attempted
+        Mockito.verify(managementServer, Mockito.never()).listHostsForMigrationOfVM(
+                Mockito.eq(skippedVm), Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.anyList());
     }
 
     @Test
@@ -393,7 +407,6 @@ public class ClusterDrsServiceImplTest {
         Mockito.when(vm1.getHostId()).thenReturn(1L);
         Mockito.when(vm1.getType()).thenReturn(VirtualMachine.Type.User);
         Mockito.when(vm1.getState()).thenReturn(VirtualMachine.State.Running);
-        Mockito.when(vm1.getDetails()).thenReturn(Collections.emptyMap());
 
         List<HostVO> hostList = new ArrayList<>();
         hostList.add(host1);
@@ -418,6 +431,10 @@ public class ClusterDrsServiceImplTest {
         Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList())).thenReturn(true);
         Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(Mockito.anyLong(), Mockito.anyLong())).thenReturn(serviceOffering);
         Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1));
+        // Return a Ternary with an empty suitable-hosts list to exercise the "no compatible hosts" path
+        Mockito.when(managementServer.listHostsForMigrationOfVM(Mockito.eq(vm1), Mockito.anyLong(),
+                Mockito.anyLong(), Mockito.any(), Mockito.anyList()))
+                .thenReturn(new Ternary<>(new Pair<>(Collections.emptyList(), 0), Collections.emptyList(), Collections.emptyMap()));
 
         List<Ternary<VirtualMachine, Host, Host>> result = clusterDrsService.getDrsPlan(cluster, 5);
         assertEquals(0, result.size());
@@ -438,7 +455,6 @@ public class ClusterDrsServiceImplTest {
         Mockito.when(vm1.getHostId()).thenReturn(1L);
         Mockito.when(vm1.getType()).thenReturn(VirtualMachine.Type.User);
         Mockito.when(vm1.getState()).thenReturn(VirtualMachine.State.Running);
-        Mockito.when(vm1.getDetails()).thenReturn(Collections.emptyMap());
 
         List<HostVO> hostList = new ArrayList<>();
         hostList.add(host1);
@@ -463,6 +479,10 @@ public class ClusterDrsServiceImplTest {
         Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList())).thenReturn(true);
         Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(Mockito.anyLong(), Mockito.anyLong())).thenReturn(serviceOffering);
         Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1));
+        // Throw an explicit exception so the catch-and-log path is exercised intentionally
+        Mockito.when(managementServer.listHostsForMigrationOfVM(Mockito.eq(vm1), Mockito.anyLong(),
+                Mockito.anyLong(), Mockito.any(), Mockito.anyList()))
+                .thenThrow(new RuntimeException("Simulated host compatibility check failure"));
 
         List<Ternary<VirtualMachine, Host, Host>> result = clusterDrsService.getDrsPlan(cluster, 5);
         assertEquals(0, result.size());
@@ -484,7 +504,6 @@ public class ClusterDrsServiceImplTest {
         Mockito.when(vm1.getHostId()).thenReturn(1L);
         Mockito.when(vm1.getType()).thenReturn(VirtualMachine.Type.User);
         Mockito.when(vm1.getState()).thenReturn(VirtualMachine.State.Running);
-        Mockito.when(vm1.getDetails()).thenReturn(Collections.emptyMap());
 
         List<HostVO> hostList = new ArrayList<>();
         hostList.add(host1);
@@ -539,14 +558,12 @@ public class ClusterDrsServiceImplTest {
         Mockito.when(vm1.getHostId()).thenReturn(1L);
         Mockito.when(vm1.getType()).thenReturn(VirtualMachine.Type.User);
         Mockito.when(vm1.getState()).thenReturn(VirtualMachine.State.Running);
-        Mockito.when(vm1.getDetails()).thenReturn(Collections.emptyMap());
 
         VMInstanceVO vm2 = Mockito.mock(VMInstanceVO.class);
         Mockito.when(vm2.getId()).thenReturn(2L);
         Mockito.when(vm2.getHostId()).thenReturn(1L);
         Mockito.when(vm2.getType()).thenReturn(VirtualMachine.Type.User);
         Mockito.when(vm2.getState()).thenReturn(VirtualMachine.State.Running);
-        Mockito.when(vm2.getDetails()).thenReturn(Collections.emptyMap());
 
         List<HostVO> hostList = new ArrayList<>();
         hostList.add(host1);
@@ -617,7 +634,6 @@ public class ClusterDrsServiceImplTest {
         Mockito.when(vm1.getHostId()).thenReturn(1L);
         Mockito.when(vm1.getType()).thenReturn(VirtualMachine.Type.User);
         Mockito.when(vm1.getState()).thenReturn(VirtualMachine.State.Running);
-        Mockito.when(vm1.getDetails()).thenReturn(Collections.emptyMap());
 
         List<HostVO> hostList = new ArrayList<>();
         hostList.add(host1);
@@ -786,15 +802,9 @@ public class ClusterDrsServiceImplTest {
 
         VMInstanceVO vm1 = Mockito.mock(VMInstanceVO.class);
         Mockito.when(vm1.getId()).thenReturn(1L);
-        Mockito.when(vm1.getType()).thenReturn(VirtualMachine.Type.User);
-        Mockito.when(vm1.getState()).thenReturn(VirtualMachine.State.Running);
-        Mockito.when(vm1.getDetails()).thenReturn(Collections.emptyMap());
 
         VMInstanceVO vm2 = Mockito.mock(VMInstanceVO.class);
         Mockito.when(vm2.getId()).thenReturn(2L);
-        Mockito.when(vm2.getType()).thenReturn(VirtualMachine.Type.User);
-        Mockito.when(vm2.getState()).thenReturn(VirtualMachine.State.Running);
-        Mockito.when(vm2.getDetails()).thenReturn(Collections.emptyMap());
 
         List<VirtualMachine> vmList = new ArrayList<>();
         vmList.add(vm1);
@@ -865,15 +875,9 @@ public class ClusterDrsServiceImplTest {
 
         VMInstanceVO vm1 = Mockito.mock(VMInstanceVO.class);
         Mockito.when(vm1.getId()).thenReturn(1L);
-        Mockito.when(vm1.getType()).thenReturn(VirtualMachine.Type.User);
-        Mockito.when(vm1.getState()).thenReturn(VirtualMachine.State.Running);
-        Mockito.when(vm1.getDetails()).thenReturn(Collections.emptyMap());
 
         VMInstanceVO vm2 = Mockito.mock(VMInstanceVO.class);
         Mockito.when(vm2.getId()).thenReturn(2L);
-        Mockito.when(vm2.getType()).thenReturn(VirtualMachine.Type.User);
-        Mockito.when(vm2.getState()).thenReturn(VirtualMachine.State.Running);
-        Mockito.when(vm2.getDetails()).thenReturn(Collections.emptyMap());
 
         List<VirtualMachine> vmList = new ArrayList<>();
         vmList.add(vm1);

test/integration/plugins/linstor/README.md

@@ -48,3 +48,21 @@ nosetests --with-marvin --marvin-config=<marvin-cfg-file> <cloudstack-dir>/test/
 ```
 
 You can also run these tests out of the box with PyDev or PyCharm or whatever.
+
+## Encrypted snapshot tests
+
+`test_linstor_encrypted_snapshots.py` covers the encrypted-volume snapshot round trip
+(create encrypted root disk -> snapshot -> revert / create-volume-from-snapshot) and that the
+backed-up qcow2 on secondary storage is itself LUKS encrypted.
+
+Extra prerequisites:
+
+* At least one KVM host with volume-encryption support (`host.encryptionsupported == true`, i.e.
+  cryptsetup/qemu LUKS available). Tests self-skip if none is found.
+* The Linstor resource group used (`acs-basic`) must be able to add a LUKS layer to its volumes.
+* `lin.backup.snapshots` must be enabled (default) so snapshots are backed up to secondary storage;
+  the test sets it. With it disabled the qcow2 path is not exercised.
+
+```
+nosetests --with-marvin --marvin-config=<marvin-cfg-file> <cloudstack-dir>/test/integration/plugins/linstor/test_linstor_encrypted_snapshots.py --zone=<zone> --hypervisor=kvm
+```

test/integration/plugins/linstor/test_linstor_encrypted_snapshots.py

@@ -0,0 +1,444 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+import logging
+import os
+import random
+import socket
+import time
+
+# All tests inherit from cloudstackTestCase
+from marvin.cloudstackTestCase import cloudstackTestCase
+
+# Import Integration Libraries
+from marvin.cloudstackAPI import createVolume
+from marvin.cloudstackException import CloudstackAPIException
+from marvin.lib.base import Account, Configurations, Host, ServiceOffering, \
+    Snapshot, StoragePool, User, VirtualMachine, Volume
+from marvin.lib.common import get_domain, get_template, get_zone, list_hosts, list_virtual_machines, list_volumes
+from marvin.lib.utils import cleanup_resources
+from marvin.sshClient import SshClient
+from nose.plugins.attrib import attr
+
+# Prerequisites:
+#  Only one zone / pod / cluster
+#  Only KVM hypervisor (Linstor only supports KVM)
+#  At least one KVM host with volume-encryption support (host.encryptionsupported == True),
+#    i.e. cryptsetup/qemu with LUKS available on the host.
+#  One Linstor storage pool whose resource-group can add a LUKS layer (encrypted volumes).
+#  'lin.backup.snapshots' enabled (default true) so snapshots are backed up to secondary storage
+#    as qcow2 -- that is the path these tests are meant to exercise. With it disabled, snapshots
+#    stay on primary as Linstor system snapshots and a different (rollback) code path is used.
+#
+# What this exercises (the encrypted-snapshot round trip):
+#  * backup:  decrypted DRBD device  -> LUKS-encrypted qcow2 on secondary
+#  * revert:  encrypted qcow2        -> decrypted, written to the DRBD device (Linstor re-encrypts)
+#  * create:  encrypted qcow2        -> new volume via createVolumeFromSnapshot (KVMStorageProcessor)
+#
+# Note on verification: Linstor encrypts inside the DRBD stack (LUKS layer), so the libvirt domain
+# XML does NOT carry <encryption format='luks'> like hypervisor-based encryption does. Correctness
+# is therefore verified by a data round trip (write marker -> snapshot -> change -> restore -> read),
+# and encryption-at-rest is verified by inspecting the backed-up qcow2 with 'qemu-img info'.
+
+MARKER_PATH = "/root/cs_enc_marker.txt"
+
+
+class TestData:
+    account = "account"
+    computeOffering = "computeoffering"
+    diskName = "diskname"
+    domainId = "domainId"
+    hypervisor = "hypervisor"
+    provider = "provider"
+    scope = "scope"
+    storageTag = "linstor"
+    tags = "tags"
+    user = "user"
+    virtualMachine = "virtualmachine"
+    zoneId = "zoneId"
+
+    def __init__(self, linstor_controller_url):
+        self.testdata = {
+            TestData.account: {
+                "email": "test-enc@test.com",
+                "firstname": "John",
+                "lastname": "Doe",
+                "username": "test-enc",
+                "password": "test"
+            },
+            TestData.user: {
+                "email": "user-enc@test.com",
+                "firstname": "Jane",
+                "lastname": "Doe",
+                "username": "test-enc-user",
+                "password": "password"
+            },
+            "primarystorage": {
+                "name": "LinstorEncPool-%d" % random.randint(0, 100000),
+                TestData.scope: "ZONE",
+                "url": linstor_controller_url,
+                TestData.provider: "Linstor",
+                TestData.tags: TestData.storageTag,
+                TestData.hypervisor: "KVM",
+                "details": {
+                    "resourceGroup": "acs-basic"
+                }
+            },
+            TestData.virtualMachine: {
+                "name": "TestEncVM",
+                "displayname": "Test Encrypted VM"
+            },
+            # encryptroot=True is passed as a create kwarg, not in this dict
+            TestData.computeOffering: {
+                "name": "Linstor_Compute_Encrypted",
+                "displaytext": "Linstor_Compute_Encrypted",
+                "cpunumber": 1,
+                "cpuspeed": 500,
+                "memory": 512,
+                "storagetype": "shared",
+                TestData.tags: TestData.storageTag
+            },
+            TestData.diskName: "restored-from-enc-snap",
+            TestData.zoneId: 1,
+            TestData.domainId: 1,
+        }
+
+
+class ServiceReady:
+    @classmethod
+    def ready(cls, hostname, port):
+        try:
+            s = socket.create_connection((hostname, port), timeout=1)
+            s.close()
+            return True
+        except (ConnectionRefusedError, socket.timeout, OSError):
+            return False
+
+    @classmethod
+    def wait(cls, hostname, port, wait_interval=5, timeout=120, service_name='ssh'):
+        starttime = int(round(time.time() * 1000))
+        while not cls.ready(hostname, port):
+            if starttime + timeout * 1000 < int(round(time.time() * 1000)):
+                raise RuntimeError("{s} {h} cannot be reached.".format(s=service_name, h=hostname))
+            time.sleep(wait_interval)
+        return True
+
+    @classmethod
+    def wait_ssh_ready(cls, hostname, wait_interval=2, timeout=120):
+        return cls.wait(hostname, 22, wait_interval, timeout, "ssh")
+
+
+class TestLinstorEncryptedSnapshots(cloudstackTestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        testclient = super(TestLinstorEncryptedSnapshots, cls).getClsTestClient()
+
+        cls.apiClient = testclient.getApiClient()
+        cls.dbConnection = testclient.getDbConnection()
+
+        cls._cleanup = []
+        cls.skip_reason = None
+
+        # Linstor is KVM-only, so the hypervisor type is not probed via getHypervisorInfo() (which is
+        # only populated when nosetests is invoked with --hypervisor). Instead we require an actual KVM
+        # host that supports volume encryption below.
+
+        # The first host runs the Linstor controller (per the Linstor test prerequisites).
+        first_host = list_hosts(cls.apiClient)[0]
+        cls.testdata = TestData(first_host.ipaddress).testdata
+
+        cls.zone = get_zone(cls.apiClient, zone_id=cls.testdata[TestData.zoneId])
+        cls.domain = get_domain(cls.apiClient, cls.testdata[TestData.domainId])
+        cls.template = get_template(cls.apiClient, cls.zone.id, hypervisor="KVM")
+
+        # Host SSH credentials, only needed by test_03 to inspect the backed-up qcow2 on secondary
+        # storage. A full marvin config carries these under zones->pods->clusters->hosts, but a
+        # lightweight config may omit them; in that case fall back to HOST_SSH_USER / HOST_SSH_PASSWORD
+        # env vars. Never fail class setup over this - the other tests don't need host SSH.
+        cls.hostConfig = None
+        try:
+            cls.hostConfig = cls.config.__dict__["zones"][0].__dict__["pods"][0].__dict__["clusters"][0] \
+                .__dict__["hosts"][0].__dict__
+        except (KeyError, IndexError, AttributeError, TypeError):
+            host_user = os.environ.get("HOST_SSH_USER")
+            host_pass = os.environ.get("HOST_SSH_PASSWORD")
+            if host_user and host_pass:
+                cls.hostConfig = {"username": host_user, "password": host_pass}
+
+        if not cls._encryption_capable_host_exists():
+            cls.skip_reason = "No KVM host with volume-encryption support found"
+            return
+
+        # Ensure snapshots are backed up to secondary storage (the path under test).
+        Configurations.update(cls.apiClient, name="lin.backup.snapshots", value="true")
+
+        primarystorage = cls.testdata["primarystorage"]
+        # Registering the pool makes the management server call the Linstor controller (to read the
+        # resource-group capacity). If the controller enforces authentication, that call needs an API
+        # token, supplied as the 'lin.auth.apitoken' add-pool detail. Provide it via LINSTOR_API_TOKEN
+        # so it is never hard-coded; leave it unset for an unauthenticated controller.
+        api_token = os.environ.get("LINSTOR_API_TOKEN")
+        if api_token:
+            primarystorage["details"]["lin.auth.apitoken"] = api_token
+
+        try:
+            cls.primary_storage = StoragePool.create(
+                cls.apiClient,
+                primarystorage,
+                scope=primarystorage[TestData.scope],
+                zoneid=cls.zone.id,
+                provider=primarystorage[TestData.provider],
+                tags=primarystorage[TestData.tags],
+                hypervisor=primarystorage[TestData.hypervisor]
+            )
+        except Exception as e:
+            cls.skip_reason = (
+                "Could not register the Linstor primary storage pool (%s). If the Linstor controller "
+                "requires authentication, set the LINSTOR_API_TOKEN env var to a valid controller API "
+                "token before running these tests." % e)
+            return
+
+        # Compute offering with encrypted root, pinned to the Linstor pool via the storage tag.
+        cls.compute_offering_encrypted = ServiceOffering.create(
+            cls.apiClient,
+            cls.testdata[TestData.computeOffering],
+            encryptroot=True
+        )
+
+        cls.account = Account.create(cls.apiClient, cls.testdata[TestData.account], admin=1)
+        cls.user = User.create(
+            cls.apiClient, cls.testdata[TestData.user],
+            account=cls.account.name, domainid=cls.domain.id)
+
+        cls._cleanup = [
+            cls.compute_offering_encrypted,
+            cls.user,
+            cls.account,
+        ]
+
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            cleanup_resources(cls.apiClient, cls._cleanup)
+            if getattr(cls, "primary_storage", None) is not None:
+                cls.primary_storage.delete(cls.apiClient)
+        except Exception as e:
+            logging.debug("Exception in tearDownClass: %s" % e)
+
+    def setUp(self):
+        if self.skip_reason:
+            self.skipTest(self.skip_reason)
+        self.cleanup = []
+
+    def tearDown(self):
+        cleanup_resources(self.apiClient, self.cleanup)
+
+    # --------------------------------------------------------------------- #
+    # Tests
+    # --------------------------------------------------------------------- #
+
+    @attr(tags=['basic'], required_hardware=True)
+    def test_01_revert_encrypted_root_snapshot(self):
+        """Snapshot an encrypted root volume, change it, revert, and verify the data and boot."""
+        vm = self._deploy_encrypted_vm("TestEncVM-revert")
+
+        # 1. write a marker into the encrypted root volume
+        self._write_marker(vm, "linstor-encrypted-v1")
+
+        # 2. snapshot the (stopped) root volume -> encrypted qcow2 on secondary
+        vm.stop(self.apiClient)
+        snapshot = self._snapshot_root_volume(vm)
+
+        # 3. change the data so a successful revert is detectable
+        self._start_vm(vm)
+        self._write_marker(vm, "linstor-encrypted-v2-CHANGED")
+
+        # 4. revert the volume to the snapshot (requires the VM stopped)
+        vm.stop(self.apiClient)
+        Volume.revertToSnapshot(self.apiClient, snapshot.id)
+
+        # 5. the VM must boot again and the original data must be back
+        self._start_vm(vm)
+        restored = self._read_marker(vm)
+        self.assertEqual(
+            "linstor-encrypted-v1", restored,
+            "Reverted encrypted root volume has wrong content (got %r) - decryption/round-trip broken" % restored
+        )
+
+    @attr(tags=['basic'], required_hardware=True)
+    def test_02_create_volume_from_encrypted_snapshot_is_rejected(self):
+        """Creating a new volume from an encrypted volume's snapshot must be rejected by CloudStack.
+
+        CloudStack core (VolumeApiServiceImpl) unconditionally blocks this for any encrypted source
+        volume ("Cannot create new volumes from encrypted volume snapshots"), so the request must never
+        reach the storage layer. This is a guard test: if the limitation is ever lifted, decryption
+        support for the create-from-snapshot path (KVMStorageProcessor / LinstorStorageAdaptor) must be
+        added and this test updated accordingly.
+        """
+        vm = self._deploy_encrypted_vm("TestEncVM-create")
+
+        self._write_marker(vm, "linstor-encrypted-create-src")
+        vm.stop(self.apiClient)
+        snapshot = self._snapshot_root_volume(vm)
+
+        cmd = createVolume.createVolumeCmd()
+        cmd.name = "%s-%d" % (self.testdata[TestData.diskName], random.randint(0, 100000))
+        cmd.zoneid = self.zone.id
+        cmd.account = self.account.name
+        cmd.domainid = self.domain.id
+        cmd.snapshotid = snapshot.id
+
+        try:
+            self.apiClient.createVolume(cmd)
+            self.fail("Creating a volume from an encrypted volume snapshot should have been rejected")
+        except CloudstackAPIException as e:
+            self.assertIn(
+                "encrypted volume snapshots", str(e),
+                "Unexpected error creating volume from encrypted snapshot: %s" % e
+            )
+
+    @attr(tags=['basic'], required_hardware=True)
+    def test_03_backed_up_snapshot_qcow2_is_encrypted(self):
+        """The qcow2 written to secondary storage for an encrypted volume must itself be LUKS encrypted."""
+        if not self.hostConfig:
+            self.skipTest("No host SSH credentials available (set HOST_SSH_USER/HOST_SSH_PASSWORD or "
+                          "provide them in the marvin config) - cannot inspect the secondary-storage qcow2")
+        vm = self._deploy_encrypted_vm("TestEncVM-atrest")
+        self._write_marker(vm, "linstor-encrypted-atrest")
+        vm.stop(self.apiClient)
+        snapshot = self._snapshot_root_volume(vm)
+
+        info = self._qemu_img_info_of_backed_up_snapshot(snapshot)
+        if info is None:
+            self.skipTest("Could not locate the backed-up snapshot on secondary storage to inspect it")
+
+        encrypted = bool(info.get("encrypted")) or "encrypt" in json.dumps(info.get("format-specific", {}))
+        self.assertTrue(
+            encrypted,
+            "Backed-up snapshot qcow2 is NOT encrypted at rest: %s" % json.dumps(info)
+        )
+
+    # --------------------------------------------------------------------- #
+    # Helpers
+    # --------------------------------------------------------------------- #
+
+    def _deploy_encrypted_vm(self, name):
+        vm = VirtualMachine.create(
+            self.apiClient,
+            {"name": name, "displayname": name},
+            accountid=self.account.name,
+            zoneid=self.zone.id,
+            serviceofferingid=self.compute_offering_encrypted.id,
+            templateid=self.template.id,
+            domainid=self.domain.id,
+            startvm=False,
+            mode='basic',
+        )
+        self.cleanup.insert(0, vm)
+        self._start_vm(vm)
+        return vm
+
+    def _snapshot_root_volume(self, vm):
+        root = list_volumes(self.apiClient, virtualmachineid=vm.id, type="ROOT", listall=True)[0]
+        snapshot = Snapshot.create(
+            self.apiClient,
+            volume_id=root.id,
+            account=self.account.name,
+            domainid=self.domain.id,
+        )
+        self.assertIsNotNone(snapshot, "Could not create snapshot of encrypted root volume")
+        self.cleanup.insert(0, snapshot)
+        return snapshot
+
+    def _vm_ssh(self, vm):
+        # The VM is deployed stopped, so its instance has no ssh_ip yet; the IP may also change across
+        # stop/start cycles. Always pass the current address from a fresh lookup.
+        ipaddress = self._get_vm(vm.id).ipaddress
+        return vm.get_ssh_client(ipaddress=ipaddress, reconnect=True, retries=5)
+
+    def _write_marker(self, vm, content):
+        ssh = self._vm_ssh(vm)
+        ssh.execute("echo '%s' > %s" % (content, MARKER_PATH))
+        ssh.execute("sync")
+
+    def _read_marker(self, vm):
+        ssh = self._vm_ssh(vm)
+        result = ssh.execute("cat %s" % MARKER_PATH)
+        return result[0].strip() if result else None
+
+    @classmethod
+    def _encryption_capable_host_exists(cls):
+        hosts = Host.list(cls.apiClient, zoneid=cls.zone.id, type='Routing', hypervisor='KVM', state='Up')
+        return any(getattr(h, "encryptionsupported", False) for h in (hosts or []))
+
+    @classmethod
+    def _get_vm(cls, vm_id):
+        return list_virtual_machines(cls.apiClient, id=vm_id)[0]
+
+    @classmethod
+    def _start_vm(cls, vm):
+        vm_for_check = cls._get_vm(vm.id)
+        if vm_for_check.state == VirtualMachine.STOPPED:
+            vm.start(cls.apiClient)
+            vm_for_check = cls._get_vm(vm.id)
+        ServiceReady.wait_ssh_ready(vm_for_check.ipaddress)
+        return vm_for_check
+
+    def _host_ssh(self):
+        host = list_hosts(self.apiClient, type='Routing', hypervisor='KVM', state='Up')[0]
+        return SshClient(
+            host=host.ipaddress, port=22,
+            user=self.hostConfig['username'], passwd=self.hostConfig['password'])
+
+    def _qemu_img_info_of_backed_up_snapshot(self, snapshot):
+        """Self-mount the secondary NFS export on a host and run 'qemu-img info' on the snapshot file."""
+        # The backed-up snapshot's physical path on secondary storage isn't exposed via the API, so we
+        # read it from the DB. The DB may be unreachable from where the tests run (e.g. MariaDB bound to
+        # localhost on the management server); in that case return None so the test skips.
+        try:
+            rows = self.dbConnection.execute(
+                "SELECT ss.install_path "
+                "FROM snapshot_store_ref ss JOIN snapshots s ON s.id = ss.snapshot_id "
+                "WHERE s.uuid = '%s' AND ss.store_role = 'Image'" % snapshot.id)
+            store = self.dbConnection.execute(
+                "SELECT url FROM image_store WHERE role = 'Image' AND removed IS NULL LIMIT 1")
+        except Exception as e:
+            logging.debug("DB lookup for snapshot install path failed: %s" % e)
+            return None
+
+        if not rows or not rows[0][0] or not store or not store[0][0]:
+            return None
+        install_path = rows[0][0]
+        url = store[0][0]  # e.g. nfs://<server>/<export-path>
+        if not url.startswith("nfs://"):
+            return None
+        server, export = url[len("nfs://"):].split("/", 1)
+
+        ssh = self._host_ssh()
+        mount_point = "/tmp/cs_sectest_%d" % random.randint(0, 100000)
+        try:
+            ssh.execute("mkdir -p %s" % mount_point)
+            ssh.execute("mount -t nfs -o ro %s:/%s %s" % (server, export, mount_point))
+            out = ssh.execute("qemu-img info --output=json %s/%s" % (mount_point, install_path))
+            return json.loads("".join(out)) if out else None
+        except Exception as e:
+            logging.debug("qemu-img info on secondary failed: %s" % e)
+            return None
+        finally:
+            ssh.execute("umount %s 2>/dev/null; rmdir %s 2>/dev/null" % (mount_point, mount_point))

tools/marvin/setup.py

@@ -46,7 +46,7 @@ setup(name="Marvin",
                 "marvin.sandbox.basic"],
       license="LICENSE.txt",
       install_requires=[
-          "mysql-connector-python <= 8.4.0",
+          "mysql-connector-python >= 8.4.0",
           "requests >= 2.2.1",
           "paramiko >= 1.13.0",
           "nose >= 1.3.3",

ui/public/locales/en.json

@@ -1781,6 +1781,8 @@
 "label.offeringid": "Offering ID",
 "label.offeringtype": "Compute Offering type",
 "label.ok": "OK",
+"label.ssvm.open.cert.page": "Open Certificate Page",
+"label.retry.upload": "Retry Upload",
 "label.only.end.date.and.time": "Only end date and time",
 "label.only.start.date.and.time": "Only start date and time",
 "label.open.documentation": "Open documentation",
@@ -4031,6 +4033,9 @@
 "message.upload.iso.failed.description": "Failed to upload ISO.",
 "message.upload.template.failed.description": "Failed to upload Template",
 "message.upload.volume.failed": "Volume upload failed",
+"message.ssvm.cert.untrusted": "Unable to reach the upload server.",
+"message.ssvm.cert.trust.instructions": "The upload server may be using a self-signed or untrusted certificate. Click 'Open Certificate Page' to open the server in a new browser tab, accept the certificate warning, then return here and click 'Retry Upload'. If the server remains unreachable, contact your administrator.",
+"message.ssvm.unreachable.retry": "The upload server is still unreachable. If it uses a self-signed certificate, please accept it in the opened tab and try again.",
 "message.user.not.permitted.api": "User is not permitted to use the API",
 "message.validate.equalto": "Please enter the same value again.",
 "message.validate.max": "Please enter a value less than or equal to {0}.",

ui/src/style/vars.less

@@ -355,7 +355,7 @@ a {
   text-align: right;
   padding-top: 15px;
 
-  button {
+  button, a.ant-btn {
     margin-right: 5px;
   }
 }

ui/src/utils/ssvmProbe.js

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const SSVM_PROBE_TIMEOUT_MS = 5000
+export async function probeSsvmCert (origin) {
+  const controller = new AbortController()
+  const timeoutId = setTimeout(() => controller.abort(), SSVM_PROBE_TIMEOUT_MS)
+  try {
+    await fetch(origin, { method: 'HEAD', mode: 'no-cors', signal: controller.signal })
+    return true
+  } catch (e) {
+    return false
+  } finally {
+    clearTimeout(timeoutId)
+  }
+}

ui/src/views/image/RegisterOrUploadIso.vue

@@ -19,11 +19,27 @@
   <div
     class="form-layout"
     @keyup.ctrl.enter="handleSubmit">
-    <span v-if="uploadPercentage > 0">
+    <span v-if="uploading">
       <loading-outlined />
       {{ $t('message.upload.file.processing') }}
       <a-progress :percent="uploadPercentage" />
     </span>
+    <div v-else-if="ssvmCertUntrusted" class="ssvm-cert-warning">
+      <a-alert
+        type="warning"
+        show-icon
+        :message="$t('message.ssvm.cert.untrusted')"
+        :description="$t('message.ssvm.cert.trust.instructions')" />
+      <div :span="24" class="action-button">
+        <a-button @click="closeAction">{{ $t('label.cancel') }}</a-button>
+        <a-button :href="ssvmOrigin" target="_blank" rel="noopener noreferrer">
+          {{ $t('label.ssvm.open.cert.page') }}
+        </a-button>
+        <a-button type="primary" :loading="loading" @click="retryUpload">
+          {{ $t('label.retry.upload') }}
+        </a-button>
+      </div>
+    </div>
     <a-spin :spinning="loading" v-else>
       <a-form
         :ref="formRef"
@@ -311,6 +327,7 @@ import { getAPI, postAPI } from '@/api'
 import store from '@/store'
 import { axios } from '../../utils/request'
 import { mixinForm } from '@/utils/mixin'
+import { probeSsvmCert } from '@/utils/ssvmProbe'
 import ResourceIcon from '@/components/view/ResourceIcon'
 import TooltipLabel from '@/components/widgets/TooltipLabel'
 
@@ -343,9 +360,12 @@ export default {
       userdatapolicy: null,
       userdatapolicylist: {},
       loading: false,
+      uploading: false,
       allowed: false,
       uploadParams: null,
       uploadPercentage: 0,
+      ssvmCertUntrusted: false,
+      ssvmOrigin: '',
       currentForm: ['plus-outlined', 'PlusOutlined'].includes(this.action.currentAction.icon) ? 'Create' : 'Upload',
       domains: [],
       accounts: [],
@@ -489,6 +509,17 @@ export default {
       this.form.file = file
       return false
     },
+    async retryUpload () {
+      this.loading = true
+      const reachable = await probeSsvmCert(this.ssvmOrigin)
+      this.loading = false
+      if (!reachable) {
+        this.$message.warning(this.$t('message.ssvm.unreachable.retry'))
+        return
+      }
+      this.ssvmCertUntrusted = false
+      this.handleUpload()
+    },
     handleUpload () {
       const { fileList } = this
       if (this.fileList.length > 1) {
@@ -502,6 +533,7 @@ export default {
       fileList.forEach(file => {
         formData.append('files[]', file)
       })
+      this.uploading = true
       this.uploadPercentage = 0
       axios.post(this.uploadParams.postURL,
         formData,
@@ -529,6 +561,8 @@ export default {
           description: `${this.$t('message.upload.iso.failed.description')} -  ${e}`,
           duration: 0
         })
+      }).finally(() => {
+        this.uploading = false
       })
     },
     handleSubmit (e) {
@@ -583,18 +617,18 @@ export default {
           }
           params.format = 'ISO'
           this.loading = true
-          getAPI('getUploadParamsForIso', params).then(json => {
+          getAPI('getUploadParamsForIso', params).then(async json => {
             this.uploadParams = (json.postuploadisoresponse && json.postuploadisoresponse.getuploadparams) ? json.postuploadisoresponse.getuploadparams : ''
-            const response = this.handleUpload()
             if (this.userdataid !== null) {
               this.linkUserdataToTemplate(this.userdataid, json.postuploadisoresponse.iso[0].id)
             }
-            if (response === 'upload successful') {
-              this.$notification.success({
-                message: this.$t('message.success.upload'),
-                description: this.$t('message.success.upload.iso.description')
-              })
+            this.ssvmOrigin = new URL(this.uploadParams.postURL).origin
+            const trusted = await probeSsvmCert(this.ssvmOrigin)
+            if (!trusted) {
+              this.ssvmCertUntrusted = true
+              return
             }
+            this.handleUpload()
           }).catch(error => {
             this.$notifyError(error)
           }).finally(() => {

ui/src/views/image/RegisterOrUploadTemplate.vue

@@ -19,11 +19,27 @@
   <div
     :class="'form-layout'"
     @keyup.ctrl.enter="handleSubmit">
-    <span v-if="uploadPercentage > 0">
+    <span v-if="uploading">
       <loading-outlined />
       {{ $t('message.upload.file.processing') }}
       <a-progress :percent="uploadPercentage" />
     </span>
+    <div v-else-if="ssvmCertUntrusted" class="ssvm-cert-warning">
+      <a-alert
+        type="warning"
+        show-icon
+        :message="$t('message.ssvm.cert.untrusted')"
+        :description="$t('message.ssvm.cert.trust.instructions')" />
+      <div :span="24" class="action-button">
+        <a-button @click="closeAction">{{ $t('label.cancel') }}</a-button>
+        <a-button :href="ssvmOrigin" target="_blank" rel="noopener noreferrer">
+          {{ $t('label.ssvm.open.cert.page') }}
+        </a-button>
+        <a-button type="primary" :loading="loading" @click="retryUpload">
+          {{ $t('label.retry.upload') }}
+        </a-button>
+      </div>
+    </div>
     <a-spin :spinning="loading" v-else>
       <a-form
         :ref="formRef"
@@ -505,6 +521,7 @@ import { getAPI, postAPI } from '@/api'
 import store from '@/store'
 import { axios } from '../../utils/request'
 import { mixinForm } from '@/utils/mixin'
+import { probeSsvmCert } from '@/utils/ssvmProbe'
 import ResourceIcon from '@/components/view/ResourceIcon'
 import TooltipLabel from '@/components/widgets/TooltipLabel'
 import DetailsInput from '@/components/widgets/DetailsInput'
@@ -532,6 +549,8 @@ export default {
       uploadPercentage: 0,
       uploading: false,
       fileList: [],
+      ssvmCertUntrusted: false,
+      ssvmOrigin: '',
       zones: {},
       defaultZone: '',
       hyperVisor: {},
@@ -649,12 +668,24 @@ export default {
       this.form.file = file
       return false
     },
+    async retryUpload () {
+      this.loading = true
+      const reachable = await probeSsvmCert(this.ssvmOrigin)
+      this.loading = false
+      if (!reachable) {
+        this.$message.warning(this.$t('message.ssvm.unreachable.retry'))
+        return
+      }
+      this.ssvmCertUntrusted = false
+      this.handleUpload()
+    },
     handleUpload () {
       const { fileList } = this
       const formData = new FormData()
       fileList.forEach(file => {
         formData.append('files[]', file)
       })
+      this.uploading = true
       this.uploadPercentage = 0
       axios.post(this.uploadParams.postURL,
         formData,
@@ -678,6 +709,8 @@ export default {
         this.closeAction()
       }).catch(e => {
         this.$notifyError(e)
+      }).finally(() => {
+        this.uploading = false
       })
     },
     fetchCustomHypervisorName () {
@@ -1175,12 +1208,18 @@ export default {
               duration: 0
             })
           }
-          getAPI('getUploadParamsForTemplate', params).then(json => {
+          getAPI('getUploadParamsForTemplate', params).then(async json => {
             this.uploadParams = (json.postuploadtemplateresponse && json.postuploadtemplateresponse.getuploadparams) ? json.postuploadtemplateresponse.getuploadparams : ''
-            this.handleUpload()
             if (this.userdataid !== null) {
               this.linkUserdataToTemplate(this.userdataid, json.postuploadtemplateresponse.template[0].id)
             }
+            this.ssvmOrigin = new URL(this.uploadParams.postURL).origin
+            const trusted = await probeSsvmCert(this.ssvmOrigin)
+            if (!trusted) {
+              this.ssvmCertUntrusted = true
+              return
+            }
+            this.handleUpload()
           }).catch(error => {
             this.$notifyError(error)
           }).finally(() => {

ui/src/views/infra/network/IpRangesTabPublic.vue

@@ -610,6 +610,10 @@ export default {
     handleShowAccountFields () {
       if (this.showAccountFields) {
         this.fetchDomains()
+      } else {
+        this.form.account = null
+        this.form.domain = null
+        this.form.forsystemvms = false
       }
     },
     handleOpenAddIpRangeModal () {

ui/src/views/storage/UploadLocalVolume.vue

@@ -16,13 +16,29 @@
 // under the License.
 
 <template>
-  <div class="form-layout" v-ctrl-enter="handleSubmit">
-    <span v-if="uploadPercentage > 0">
+  <div class="form-layout">
+    <span v-if="uploading">
       <loading-outlined />
       {{ $t('message.upload.file.processing') }}
       <a-progress :percent="uploadPercentage" />
     </span>
-    <a-spin :spinning="loading" v-else>
+    <div v-else-if="ssvmCertUntrusted" class="ssvm-cert-warning">
+      <a-alert
+        type="warning"
+        show-icon
+        :message="$t('message.ssvm.cert.untrusted')"
+        :description="$t('message.ssvm.cert.trust.instructions')" />
+      <div :span="24" class="action-button">
+        <a-button @click="closeAction">{{ $t('label.cancel') }}</a-button>
+        <a-button :href="ssvmOrigin" target="_blank" rel="noopener noreferrer">
+          {{ $t('label.ssvm.open.cert.page') }}
+        </a-button>
+        <a-button type="primary" :loading="loading" @click="retryUpload">
+          {{ $t('label.retry.upload') }}
+        </a-button>
+      </div>
+    </div>
+    <a-spin :spinning="loading" v-else v-ctrl-enter="handleSubmit">
       <a-form
         :ref="formRef"
         :model="form"
@@ -156,6 +172,7 @@ import { ref, reactive, toRaw } from 'vue'
 import { getAPI } from '@/api'
 import { axios } from '../../utils/request'
 import { mixinForm } from '@/utils/mixin'
+import { probeSsvmCert } from '@/utils/ssvmProbe'
 import ResourceIcon from '@/components/view/ResourceIcon'
 import TooltipLabel from '@/components/widgets/TooltipLabel'
 import InfiniteScrollSelect from '@/components/widgets/InfiniteScrollSelect.vue'
@@ -178,7 +195,10 @@ export default {
       customDiskOffering: false,
       isCustomizedDiskIOps: false,
       loading: false,
-      uploadPercentage: 0
+      uploading: false,
+      uploadPercentage: 0,
+      ssvmCertUntrusted: false,
+      ssvmOrigin: ''
     }
   },
   beforeCreate () {
@@ -267,6 +287,63 @@ export default {
       this.form.account = accountName
       this.account = accountName
     },
+    async retryUpload () {
+      this.loading = true
+      const reachable = await probeSsvmCert(this.ssvmOrigin)
+      this.loading = false
+      if (!reachable) {
+        this.$message.warning(this.$t('message.ssvm.unreachable.retry'))
+        return
+      }
+      this.ssvmCertUntrusted = false
+      this.handleUpload()
+    },
+    handleUpload () {
+      if (this.fileList.length > 1) {
+        this.$notification.error({
+          message: this.$t('message.upload.volume.failed'),
+          description: this.$t('message.upload.file.limit'),
+          duration: 0
+        })
+        return
+      }
+      const { fileList } = this
+      const formData = new FormData()
+      fileList.forEach(file => {
+        formData.append('files[]', file)
+      })
+      this.uploading = true
+      this.uploadPercentage = 0
+      axios.post(this.uploadParams.postURL,
+        formData,
+        {
+          headers: {
+            'content-type': 'multipart/form-data',
+            'x-signature': this.uploadParams.signature,
+            'x-expires': this.uploadParams.expires,
+            'x-metadata': this.uploadParams.metadata
+          },
+          onUploadProgress: (progressEvent) => {
+            this.uploadPercentage = Number(parseFloat(100 * progressEvent.loaded / progressEvent.total).toFixed(1))
+          },
+          timeout: 86400000
+        }).then((json) => {
+        this.$notification.success({
+          message: this.$t('message.success.upload'),
+          description: this.$t('message.success.upload.volume.description')
+        })
+        this.closeAction()
+      }).catch(e => {
+        this.$notification.error({
+          message: this.$t('message.upload.failed'),
+          description: `${this.$t('message.upload.volume.failed')} -  ${e}`,
+          duration: 0
+        })
+      }).finally(() => {
+        this.uploading = false
+        this.loading = false
+      })
+    },
     handleSubmit (e) {
       e.preventDefault()
       if (this.loading) return
@@ -286,49 +363,15 @@ export default {
         }
         params.domainId = this.domainId
         this.loading = true
-        getAPI('getUploadParamsForVolume', params).then(json => {
+        getAPI('getUploadParamsForVolume', params).then(async json => {
           this.uploadParams = json.postuploadvolumeresponse?.getuploadparams || ''
-          const { fileList } = this
-          if (this.fileList.length > 1) {
-            this.$notification.error({
-              message: this.$t('message.upload.volume.failed'),
-              description: this.$t('message.upload.file.limit'),
-              duration: 0
-            })
+          this.ssvmOrigin = new URL(this.uploadParams.postURL).origin
+          const trusted = await probeSsvmCert(this.ssvmOrigin)
+          if (!trusted) {
+            this.ssvmCertUntrusted = true
+            return
           }
-          const formData = new FormData()
-          fileList.forEach(file => {
-            formData.append('files[]', file)
-          })
-          this.uploadPercentage = 0
-          axios.post(this.uploadParams.postURL,
-            formData,
-            {
-              headers: {
-                'content-type': 'multipart/form-data',
-                'x-signature': this.uploadParams.signature,
-                'x-expires': this.uploadParams.expires,
-                'x-metadata': this.uploadParams.metadata
-              },
-              onUploadProgress: (progressEvent) => {
-                this.uploadPercentage = Number(parseFloat(100 * progressEvent.loaded / progressEvent.total).toFixed(1))
-              },
-              timeout: 86400000
-            }).then((json) => {
-            this.$notification.success({
-              message: this.$t('message.success.upload'),
-              description: this.$t('message.success.upload.volume.description')
-            })
-            this.closeAction()
-          }).catch(e => {
-            this.$notification.error({
-              message: this.$t('message.upload.failed'),
-              description: `${this.$t('message.upload.volume.failed')} -  ${e}`,
-              duration: 0
-            })
-          }).finally(() => {
-            this.loading = false
-          })
+          this.handleUpload()
         }).catch(e => {
           this.$notification.error({
             message: this.$t('message.upload.failed'),