gian/cloudstack-mirror
1
0
Fork 0
mirror of https://github.com/apache/cloudstack synced 2026-08-02 05:26:35 +00:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

optimise prometheus scraping workflow

Browse Source
This commit is contained in:
Daan Hoogland 2026-07-20 12:40:07 +02:00
parent e8df87e89b
commit 4261c3d54d
2 changed files with 33 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java
View File

@ -32,7 +32,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
import org.apache.commons.lang3.StringUtils;
import com.cloud.alert.AlertManager;
import com.cloud.api.ApiDBUtils;
import com.cloud.api.query.dao.DomainJoinDao;
import com.cloud.api.query.dao.StoragePoolJoinDao;
@ -126,8 +125,6 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp
@Inject
private DomainJoinDao domainDao;
@Inject
private AlertManager alertManager;
@Inject
DedicatedResourceDao _dedicatedDao;
@Inject
private AccountDao _accountDao;
@ -494,10 +491,17 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp
public void updateMetrics() {
final List<Item> latestMetricsItems = new ArrayList<Item>();
try {
// NOTE: capacity data is refreshed independently by AlertManagerImpl's own
// periodic CapacityChecker timer (see AlertManagerImpl#start()). Do NOT force a
// synchronous recalculateCapacity() here: it spins up a fresh thread pool per host
// and per storage pool across ALL zones on every single scrape, so with Z zones a
// single Prometheus scrape triggered Z redundant full recalculations. That extra,
// uncoordinated load compounds over time (thread churn + overlapping runs with the
// timer) and was the cause of https://github.com/apache/cloudstack/issues/13586
// (scrape_duration_seconds climbing until a management-server restart).
for (final DataCenterVO dc : dcDao.listAll()) {
final String zoneName = dc.getName();
final String zoneUuid = dc.getUuid();
alertManager.recalculateCapacity();
addHostMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid);
addVMMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid);
addVolumeMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid);

33
server/src/main/java/com/cloud/alert/AlertManagerImpl.java
View File

@ -161,6 +161,8 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi
private final ExecutorService _executor;
private ExecutorService _capacityExecutorService;
protected SMTPMailSender mailSender;
protected String[] recipients = null;
protected String senderAddress = null;
@ -249,6 +251,9 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi
@Override
public boolean stop() {
_timer.cancel();
if (_capacityExecutorService != null) {
_capacityExecutorService.shutdown();
}
return true;
}
@ -281,6 +286,24 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi
}
}
/**
* Shared, long-lived pool for capacity recalculation, reused across every
* recalculateHostCapacities()/recalculateStorageCapacities() call instead of creating and
* tearing down a new thread pool per invocation. Repeatedly creating/shutting down pools was
* unnecessary overhead under frequent callers (e.g. the Prometheus exporter used to trigger a
* full recalculation on every scrape, see https://github.com/apache/cloudstack/issues/13586).
* Lazily created so this remains safe for callers that invoke the recalculate methods directly
* without going through configure()/start() (e.g. unit tests).
*/
private synchronized ExecutorService getCapacityExecutorService() {
if (_capacityExecutorService == null || _capacityExecutorService.isShutdown()) {
_capacityExecutorService = Executors.newFixedThreadPool(
Math.max(1, CapacityManager.CapacityCalculateWorkers.value()),
new NamedThreadFactory("Capacity-Calculator"));
}
return _capacityExecutorService;
}
/**
* Recalculates the capacities of hosts, including CPU and RAM.
*/
@ -290,10 +313,8 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi
return;
}
ConcurrentHashMap<Long, Future<Void>> futures = new ConcurrentHashMap<>();
ExecutorService executorService = Executors.newFixedThreadPool(Math.max(1,
Math.min(CapacityManager.CapacityCalculateWorkers.value(), hostIds.size())));
for (Long hostId : hostIds) {
futures.put(hostId, executorService.submit(() -> {
futures.put(hostId, getCapacityExecutorService().submit(() -> {
final HostVO host = hostDao.findById(hostId);
_capacityMgr.updateCapacityForHost(host);
return null;
@ -307,7 +328,6 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi
entry.getKey(), e.getMessage()), e);
}
}
executorService.shutdown();
}
protected void recalculateStorageCapacities() {
@ -316,10 +336,8 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi
return;
}
ConcurrentHashMap<Long, Future<Void>> futures = new ConcurrentHashMap<>();
ExecutorService executorService = Executors.newFixedThreadPool(Math.max(1,
Math.min(CapacityManager.CapacityCalculateWorkers.value(), storagePoolIds.size())));
for (Long poolId: storagePoolIds) {
futures.put(poolId, executorService.submit(() -> {
futures.put(poolId, getCapacityExecutorService().submit(() -> {
Transaction.execute(new TransactionCallbackNoReturn() {
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {
@ -343,7 +361,6 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi
entry.getKey(), e.getMessage()), e);
}
}
executorService.shutdown();
}
@Override