gian/cloudstack-mirror
1
0
Fork 0
mirror of https://github.com/apache/cloudstack synced 2026-08-02 05:26:35 +00:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

Merge branch '4.22' into 'main'

Browse Source
This commit is contained in:
Wei Zhou 2026-07-13 11:03:44 +02:00
commit fe3df6b660
No known key found for this signature in database
GPG Key ID: 1503DFE7C8226103
6 changed files with 1609 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
View File

@ -2319,23 +2319,24 @@ public class KVMStorageProcessor implements StorageProcessor {
logger.debug("Rebasing snapshot [{}] with parent [{}].", snapshotName, parentSnapshotPath);
long snapshotTimeoutInMillis = wait * 1000L;
try {
QemuImg qemuImg = new QemuImg(wait);
QemuImg qemuImg = new QemuImg(snapshotTimeoutInMillis);
qemuImg.rebase(snapshotFile, parentSnapshotFile, PhysicalDiskFormat.QCOW2.toString(), false);
} catch (LibvirtException | QemuImgException e) {
if (!StringUtils.contains(e.getMessage(), "Is another process using the image")) {
logger.error("Exception while rebasing incremental snapshot [{}] due to: [{}].", snapshotName, e.getMessage(), e);
throw new CloudRuntimeException(e);
}
retryRebase(snapshotName, wait, e, snapshotFile, parentSnapshotFile);
retryRebase(snapshotName, snapshotTimeoutInMillis, e, snapshotFile, parentSnapshotFile);
}
}
private void retryRebase(String snapshotName, int wait, Exception e, QemuImgFile snapshotFile, QemuImgFile parentSnapshotFile) {
private void retryRebase(String snapshotName, long waitInMilliseconds, Exception e, QemuImgFile snapshotFile, QemuImgFile parentSnapshotFile) {
logger.warn("Libvirt still has not released the lock, will wait [{}] milliseconds and try again later.", incrementalSnapshotRetryRebaseWait);
try {
Thread.sleep(incrementalSnapshotRetryRebaseWait);
QemuImg qemuImg = new QemuImg(wait);
QemuImg qemuImg = new QemuImg(waitInMilliseconds);
qemuImg.rebase(snapshotFile, parentSnapshotFile, PhysicalDiskFormat.QCOW2.toString(), false);
} catch (LibvirtException | QemuImgException | InterruptedException ex) {
logger.error("Unable to rebase snapshot [{}].", snapshotName, ex);

5
plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java
View File

@ -927,7 +927,10 @@ public class QemuImg {
}
protected static boolean helpSupportsImageFormat(String text, QemuImg.PhysicalDiskFormat format) {
Pattern pattern = Pattern.compile("Supported\\sformats:[a-zA-Z0-9-_\\s]*?\\b" + format + "\\b", CASE_INSENSITIVE);
// QEMU >= 10.1.0 changed the qemu-img --help header from
// "Supported formats:" to "Supported image formats:", so the word
// "image" must be treated as optional here.
Pattern pattern = Pattern.compile("Supported\\s(image\\s)?formats:[a-zA-Z0-9-_\\s]*?\\b" + format + "\\b", CASE_INSENSITIVE);
return pattern.matcher(text).find();
}

15
plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/qemu/QemuImgTest.java
View File

@ -390,6 +390,21 @@ public class QemuImgTest {
Assert.assertFalse("should not support http", QemuImg.helpSupportsImageFormat(partialHelp, PhysicalDiskFormat.SHEEPDOG));
}
@Test
public void testHelpSupportsImageFormatQemu101Header() throws QemuImgException, LibvirtException {
// qemu-img 10.1.0 (e.g. RHEL 9.8: qemu-kvm-10.1.0-17.el9_8.3) changed the
// help header from "Supported formats:" to "Supported image formats:"
String help = "Supported image formats:\n" +
" blkdebug blklogwrites blkverify compress copy-before-write copy-on-read\n" +
" file ftp ftps host_cdrom host_device http https io_uring luks nbd null-aio\n" +
" null-co nvme nvme-io_uring preallocate qcow2 quorum raw rbd\n" +
" snapshot-access throttle vdi vhdx virtio-blk-vfio-pci\n" +
" virtio-blk-vhost-user virtio-blk-vhost-vdpa vmdk vpc\n";
Assert.assertTrue("should support luks", QemuImg.helpSupportsImageFormat(help, PhysicalDiskFormat.LUKS));
Assert.assertTrue("should support qcow2", QemuImg.helpSupportsImageFormat(help, PhysicalDiskFormat.QCOW2));
Assert.assertFalse("should not support sheepdog", QemuImg.helpSupportsImageFormat(help, PhysicalDiskFormat.SHEEPDOG));
}
@Test
public void testCheckAndRepair() throws LibvirtException {
String filename = "/tmp/" + UUID.randomUUID() + ".qcow2";

184
plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayAdapter.java
View File

@ -28,6 +28,8 @@ import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
@ -63,6 +65,7 @@ import org.apache.http.ssl.SSLContextBuilder;
import com.cloud.utils.exception.CloudRuntimeException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -87,6 +90,10 @@ public class FlashArrayAdapter implements ProviderAdapter {
private static final String API_LOGIN_VERSION_DEFAULT = "1.19";
private static final String API_VERSION_DEFAULT = "2.23";
// URLs for which the legacy-auth deprecation WARN has already been emitted,
// so we don't spam the logs once per refresh per pool while it's still configured.
private static final Set<String> WARNED_LEGACY_URLS = ConcurrentHashMap.newKeySet();
static final ObjectMapper mapper = new ObjectMapper();
public String pod = null;
public String hostgroup = null;
@ -589,6 +596,91 @@ public class FlashArrayAdapter implements ProviderAdapter {
return accessToken;
}
/**
* Discover the latest supported Purity REST API version by hitting the unauthenticated
* {@code /api/api_version} endpoint (returns {@code {"version":["1.0",...,"2.36"]}}).
* The discovered version is stored on {@link #apiVersion}; on failure the caller-configured
* default remains in place.
*/
private void fetchApiVersionFromPurity(CloseableHttpClient client) {
HttpGet vReq = new HttpGet(url + "/api_version");
CloseableHttpResponse vResp = null;
try {
vResp = client.execute(vReq);
if (vResp.getStatusLine().getStatusCode() == 200) {
JsonNode root = mapper.readTree(vResp.getEntity().getContent());
JsonNode versions = root.get("version");
if (versions != null && versions.isArray() && versions.size() > 0) {
apiVersion = versions.get(versions.size() - 1).asText();
}
} else {
logger.warn("Unexpected HTTP " + vResp.getStatusLine().getStatusCode()
+ " from FlashArray [" + url + "] /api_version, falling back to default "
+ API_VERSION_DEFAULT);
}
} catch (Exception e) {
logger.warn("Failed to discover Purity REST API version from " + url
+ "/api_version, falling back to default " + API_VERSION_DEFAULT, e);
} finally {
if (vResp != null) {
try {
vResp.close();
} catch (IOException e) {
logger.debug("Error closing /api_version response from FlashArray [" + url + "]", e);
}
}
}
}
/**
* Exchange the operator-configured username/password for a long-lived Purity api-token
* via REST 1.x {@code /auth/apitoken}. Emits the once-per-URL deprecation WARN.
* @return the api-token to feed into the REST 2.x /login exchange.
*/
private String getApiTokenUsingUserPass(CloseableHttpClient client) throws IOException {
if (WARNED_LEGACY_URLS.add(url)) {
logger.warn("FlashArray adapter at [" + url + "] is using deprecated username/password "
+ "login against Purity REST 1.x. Replace with a pre-minted "
+ ProviderAdapter.API_TOKEN_KEY + " detail; the username/password code path will be "
+ "removed in a future release.");
}
HttpPost request = new HttpPost(url + "/" + apiLoginVersion + "/auth/apitoken");
ArrayList<NameValuePair> postParms = new ArrayList<NameValuePair>();
postParms.add(new BasicNameValuePair("username", username));
postParms.add(new BasicNameValuePair("password", password));
request.setEntity(new UrlEncodedFormEntity(postParms, "UTF-8"));
CloseableHttpResponse response = null;
try {
response = client.execute(request);
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == 200 || statusCode == 201) {
FlashArrayApiToken legacyToken = mapper.readValue(response.getEntity().getContent(),
FlashArrayApiToken.class);
if (legacyToken == null || legacyToken.getApiToken() == null) {
throw new CloudRuntimeException(
"Authentication responded successfully but no api token was returned");
}
return legacyToken.getApiToken();
} else if (statusCode == 401 || statusCode == 403) {
throw new CloudRuntimeException(
"Authentication or Authorization to FlashArray [" + url + "] with user [" + username
+ "] failed, unable to retrieve session token");
} else {
throw new CloudRuntimeException(
"Unexpected HTTP response code from FlashArray [" + url + "] - [" + statusCode
+ "] - " + response.getStatusLine().getReasonPhrase());
}
} finally {
if (response != null) {
try {
response.close();
} catch (IOException e) {
logger.debug("Error closing legacy auth/apitoken response from FlashArray [" + url + "]", e);
}
}
}
}
private synchronized void refreshSession(boolean force) {
try {
if (force || keyExpiration < System.currentTimeMillis()) {
@ -663,9 +755,11 @@ public class FlashArrayAdapter implements ProviderAdapter {
}
apiVersion = connectionDetails.get(FlashArrayAdapter.API_VERSION);
if (apiVersion == null) {
boolean apiVersionExplicit = apiVersion != null;
if (!apiVersionExplicit) {
apiVersion = queryParms.get(FlashArrayAdapter.API_VERSION);
if (apiVersion == null) {
apiVersionExplicit = apiVersion != null;
if (!apiVersionExplicit) {
apiVersion = API_VERSION_DEFAULT;
}
}
@ -732,72 +826,66 @@ public class FlashArrayAdapter implements ProviderAdapter {
skipTlsValidation = true;
}
// Resolve the long-lived API token. Prefer a pre-minted api_token (Purity REST 2.x flow);
// fall back to legacy username/password auth via Purity REST 1.x for backward compatibility.
String apiToken = connectionDetails.get(ProviderAdapter.API_TOKEN_KEY);
if (apiToken != null && apiToken.isEmpty()) {
apiToken = null;
}
boolean usingLegacyUserPass = apiToken == null;
if (usingLegacyUserPass && (username == null || password == null)) {
throw new CloudRuntimeException("FlashArray adapter requires either " + ProviderAdapter.API_TOKEN_KEY
+ " (preferred) or both " + ProviderAdapter.API_USERNAME_KEY + " and "
+ ProviderAdapter.API_PASSWORD_KEY + " in the connection details");
}
CloseableHttpClient client = getClient();
CloseableHttpResponse response = null;
try {
HttpPost request = new HttpPost(url + "/" + apiLoginVersion + "/auth/apitoken");
// request.addHeader("Content-Type", "application/json");
// request.addHeader("Accept", "application/json");
ArrayList<NameValuePair> postParms = new ArrayList<NameValuePair>();
postParms.add(new BasicNameValuePair("username", username));
postParms.add(new BasicNameValuePair("password", password));
request.setEntity(new UrlEncodedFormEntity(postParms, "UTF-8"));
CloseableHttpClient client = getClient();
response = (CloseableHttpResponse) client.execute(request);
int statusCode = response.getStatusLine().getStatusCode();
FlashArrayApiToken apitoken = null;
if (statusCode == 200 | statusCode == 201) {
apitoken = mapper.readValue(response.getEntity().getContent(), FlashArrayApiToken.class);
if (apitoken == null) {
throw new CloudRuntimeException(
"Authentication responded successfully but no api token was returned");
}
} else if (statusCode == 401 || statusCode == 403) {
throw new CloudRuntimeException(
"Authentication or Authorization to FlashArray [" + url + "] with user [" + username
+ "] failed, unable to retrieve session token");
} else {
throw new CloudRuntimeException(
"Unexpected HTTP response code from FlashArray [" + url + "] - [" + statusCode
+ "] - " + response.getStatusLine().getReasonPhrase());
// Discover the latest supported API version from the array unless one was explicitly configured.
// GET /api/api_version is unauthenticated and returns {"version":["1.0",...,"2.36"]}.
if (!apiVersionExplicit) {
fetchApiVersionFromPurity(client);
}
// now we need to get the access token
request = new HttpPost(url + "/" + apiVersion + "/login");
request.addHeader("api-token", apitoken.getApiToken());
response = (CloseableHttpResponse) client.execute(request);
if (usingLegacyUserPass) {
apiToken = getApiTokenUsingUserPass(client);
}
statusCode = response.getStatusLine().getStatusCode();
if (statusCode == 200 | statusCode == 201) {
// Exchange the long-lived api-token for a short-lived x-auth-token (REST 2.x).
HttpPost request = new HttpPost(url + "/" + apiVersion + "/login");
request.addHeader("api-token", apiToken);
response = client.execute(request);
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == 200 || statusCode == 201) {
Header[] headers = response.getHeaders("x-auth-token");
if (headers == null || headers.length == 0) {
throw new CloudRuntimeException(
"Getting access token responded successfully but access token was not available");
"FlashArray /login responded successfully but no x-auth-token header was returned");
}
accessToken = headers[0].getValue();
} else if (statusCode == 401 || statusCode == 403) {
throw new CloudRuntimeException(
"Authentication or Authorization to FlashArray [" + url + "] with user [" + username
+ "] failed, unable to retrieve session token");
"FlashArray [" + url + "] rejected the api-token at /" + apiVersion + "/login");
} else {
throw new CloudRuntimeException(
"Unexpected HTTP response code from FlashArray [" + url + "] - [" + statusCode
+ "] - " + response.getStatusLine().getReasonPhrase());
"Unexpected HTTP response code from FlashArray [" + url + "] /" + apiVersion
+ "/login - [" + statusCode + "] - "
+ response.getStatusLine().getReasonPhrase());
}
} catch (UnsupportedEncodingException e) {
throw new CloudRuntimeException("Error creating input for login, check username/password encoding");
throw new CloudRuntimeException("Error encoding login form for FlashArray [" + url + "]", e);
} catch (UnsupportedOperationException e) {
throw new CloudRuntimeException("Error processing login response from FlashArray [" + url + "]", e);
} catch (IOException e) {
throw new CloudRuntimeException("Error sending login request to FlashArray [" + url + "]", e);
} finally {
try {
if (response != null) {
if (response != null) {
try {
response.close();
} catch (IOException e) {
logger.debug("Error closing response from login attempt to FlashArray", e);
}
} catch (IOException e) {
logger.debug("Error closing response from login attempt to FlashArray", e);
}
}
}
@ -965,7 +1053,7 @@ public class FlashArrayAdapter implements ProviderAdapter {
request.setEntity(new StringEntity(data));
CloseableHttpClient client = getClient();
response = (CloseableHttpResponse) client.execute(request);
response = client.execute(request);
final int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == 200 || statusCode == 201) {
@ -1020,7 +1108,7 @@ public class FlashArrayAdapter implements ProviderAdapter {
request.addHeader("X-auth-token", getAccessToken());
CloseableHttpClient client = getClient();
response = (CloseableHttpResponse) client.execute(request);
response = client.execute(request);
final int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == 200) {
try {
@ -1062,7 +1150,7 @@ public class FlashArrayAdapter implements ProviderAdapter {
request.addHeader("X-auth-token", getAccessToken());
CloseableHttpClient client = getClient();
response = (CloseableHttpResponse) client.execute(request);
response = client.execute(request);
final int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == 200 || statusCode == 404 || statusCode == 400) {
// this means the volume was deleted successfully, or doesn't exist (effective

69
server/src/main/java/com/cloud/server/ManagementServerImpl.java
View File

@ -1507,6 +1507,7 @@ public class ManagementServerImpl extends MutualExclusiveIdsManagerBase implemen
*/
Ternary<Pair<List<? extends Host>, Integer>, List<? extends Host>, Map<Host, Boolean>> getTechnicallyCompatibleHosts(
final VirtualMachine vm,
final Host srcHost,
final Long startIndex,
final Long pageSize,
final String keyword) {
@ -1517,31 +1518,6 @@ public class ManagementServerImpl extends MutualExclusiveIdsManagerBase implemen
return new Ternary<>(new Pair<>(new ArrayList<>(), 0), new ArrayList<>(), new HashMap<>());
}
final long srcHostId = vm.getHostId();
final Host srcHost = _hostDao.findById(srcHostId);
if (srcHost == null) {
if (logger.isDebugEnabled()) {
logger.debug("Unable to find the host with ID: " + srcHostId + " of this Instance: " + vm);
}
final InvalidParameterValueException ex = new InvalidParameterValueException("Unable to find the host (with specified ID) of instance with specified ID");
ex.addProxyObject(String.valueOf(srcHostId), "hostId");
ex.addProxyObject(vm.getUuid(), "vmId");
throw ex;
}
String srcHostVersion = srcHost.getHypervisorVersion();
if (HypervisorType.KVM.equals(srcHost.getHypervisorType()) && srcHostVersion == null) {
srcHostVersion = "";
}
// Check if the vm can be migrated with storage.
boolean canMigrateWithStorage = false;
List<HypervisorType> hypervisorTypes = Arrays.asList(new HypervisorType[]{HypervisorType.VMware, HypervisorType.KVM});
if (VirtualMachine.Type.User.equals(vm.getType()) || hypervisorTypes.contains(vm.getHypervisorType())) {
canMigrateWithStorage = _hypervisorCapabilitiesDao.isStorageMotionSupported(srcHost.getHypervisorType(), srcHostVersion);
}
// Check if the vm is using any disks on local storage.
final VirtualMachineProfile vmProfile = new VirtualMachineProfileImpl(vm, null, _offeringDao.findById(vm.getId(), vm.getServiceOfferingId()), null, null);
final List<VolumeVO> volumes = _volumeDao.findCreatedByInstance(vmProfile.getId());
@ -1555,12 +1531,14 @@ public class ManagementServerImpl extends MutualExclusiveIdsManagerBase implemen
}
}
boolean canMigrateWithStorage = isStorageMigrationSupported(vm, srcHost);
if (!canMigrateWithStorage && usesLocal) {
throw new InvalidParameterValueException("Unsupported operation, instance uses Local storage, cannot migrate");
}
validateVgpuProfileForVmMigration(vmProfile);
final String srcHostVersion = getHypervisorVersionOfHost(srcHost);
final Type hostType = srcHost.getType();
Pair<List<HostVO>, Integer> allHostsPair;
List<HostVO> allHosts;
@ -1636,6 +1614,23 @@ public class ManagementServerImpl extends MutualExclusiveIdsManagerBase implemen
return new Ternary<>(allHostsPairResult, filteredHosts, requiresStorageMotion);
}
protected boolean isStorageMigrationSupported(final VirtualMachine vm, final Host srcHost) {
final List<HypervisorType> hypervisorTypes = Arrays.asList(HypervisorType.VMware, HypervisorType.KVM);
if (VirtualMachine.Type.User.equals(vm.getType()) || hypervisorTypes.contains(vm.getHypervisorType())) {
final String srcHostVersion = getHypervisorVersionOfHost(srcHost);
return _hypervisorCapabilitiesDao.isStorageMotionSupported(srcHost.getHypervisorType(), srcHostVersion);
}
return false;
}
protected String getHypervisorVersionOfHost(final Host host) {
final String version = host.getHypervisorVersion();
if (version == null && HypervisorType.KVM.equals(host.getHypervisorType())) {
return "";
}
return version;
}
/**
* Apply affinity group constraints and other exclusion rules for VM migration.
* This builds an ExcludeList based on affinity groups, DPDK requirements, and dedicated resources.
@ -1727,9 +1722,19 @@ public class ManagementServerImpl extends MutualExclusiveIdsManagerBase implemen
validateVmForHostMigration(vm);
final long srcHostId = vm.getHostId();
final Host srcHost = _hostDao.findById(srcHostId);
if (srcHost == null) {
logger.debug("Unable to find the host with ID: {} of this Instance: {}", srcHostId, vm);
final InvalidParameterValueException ex = new InvalidParameterValueException("Unable to find the host (with specified ID) of instance with specified ID");
ex.addProxyObject(String.valueOf(srcHostId), "hostId");
ex.addProxyObject(vm.getUuid(), "vmId");
throw ex;
}
// Get technically compatible hosts (storage, hypervisor, UEFI)
Ternary<Pair<List<? extends Host>, Integer>, List<? extends Host>, Map<Host, Boolean>> compatibilityResult =
getTechnicallyCompatibleHosts(vm, startIndex, pageSize, keyword);
getTechnicallyCompatibleHosts(vm, srcHost, startIndex, pageSize, keyword);
Pair<List<? extends Host>, Integer> allHostsPair = compatibilityResult.first();
List<? extends Host> filteredHosts = compatibilityResult.second();
@ -1742,9 +1747,7 @@ public class ManagementServerImpl extends MutualExclusiveIdsManagerBase implemen
}
// Create deployment plan and VM profile
final Host srcHost = _hostDao.findById(vm.getHostId());
final DataCenterDeployment plan = new DataCenterDeployment(
srcHost.getDataCenterId(), srcHost.getPodId(), srcHost.getClusterId(), null, null, null);
final DataCenterDeployment plan = createDeploymentPlanForMigrationListing(vm, srcHost);
final VirtualMachineProfile vmProfile = new VirtualMachineProfileImpl(
vm, null, _offeringDao.findById(vm.getId(), vm.getServiceOfferingId()), null, null);
@ -1769,6 +1772,14 @@ public class ManagementServerImpl extends MutualExclusiveIdsManagerBase implemen
}
}
protected DataCenterDeployment createDeploymentPlanForMigrationListing(final VirtualMachine vm, final Host srcHost) {
final boolean canMigrateWithStorage = isStorageMigrationSupported(vm, srcHost);
if (canMigrateWithStorage) {
return new DataCenterDeployment(srcHost.getDataCenterId(), srcHost.getPodId(), null, null, null, null);
}
return new DataCenterDeployment(srcHost.getDataCenterId(), srcHost.getPodId(), srcHost.getClusterId(), null, null, null);
}
/**
* Add non DPDK enabled hosts to the avoid list
*/

1414
server/src/test/java/com/cloud/server/ManagementServerImplTest.java
View File

File diff suppressed because it is too large Load Diff