gian/ceph-mirror
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2026-08-01 22:45:39 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #69650 from baum/nvmeof-host-maintenance

Browse Source 
cephadm: disable/enable NVMe-oF gateways during host maintenance
This commit is contained in:
baum 2026-07-21 14:09:24 +07:00 committed by GitHub
commit c77c5eb8db
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 148 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
src/pybind/mgr/cephadm/module.py
View File

@ -2558,6 +2558,49 @@ Then run the following:
self.log.info(msg)
return msg
def _set_nvmeof_gateways_admin_state(self, hostname: str, enabled: bool) -> Tuple[List[str], Optional[str]]:
"""Enable or disable NVMe-oF gateways on a host via nvme-gw admin state commands.
Returns a tuple of (gateway ids updated, error message if any command failed).
"""
prefix = 'nvme-gw enable' if enabled else 'nvme-gw disable'
nvmeof_daemons = self.cache.get_daemons_by_type('nvmeof', host=hostname)
if not nvmeof_daemons:
return [], None
updated: List[str] = []
errors: List[str] = []
for dd in nvmeof_daemons:
spec = cast(NvmeofServiceSpec, self.spec_store.all_specs.get(dd.service_name(), None))
if not spec:
self.log.warning(
f'No nvmeof spec for {dd.name()}: skipping {prefix} during maintenance on {hostname}')
continue
if not spec.pool or spec.group is None:
self.log.warning(
f'nvmeof spec for {dd.service_name()} missing pool/group: '
f'skipping {prefix} during maintenance on {hostname}')
continue
gw_id = f'{utils.name_to_config_section("nvmeof")}.{dd.daemon_id}'
cmd = {
'prefix': prefix,
'id': gw_id,
'pool': spec.pool,
'group': spec.group,
}
self.log.info(
f'maintenance on {hostname}: {prefix} gateway {gw_id} ({spec.pool}/{spec.group})')
rc, _out, err = self.mon_command(cmd)
if rc:
errors.append(f'{gw_id}: rc={rc} {err}')
else:
updated.append(gw_id)
if errors:
return updated, '; '.join(errors)
return updated, None
def update_maintenance_healthcheck(self) -> None:
"""Raise/update or clear the maintenance health check as needed"""
@ -2575,9 +2618,10 @@ Then run the following:
""" Attempt to place a cluster host in maintenance
Placing a host into maintenance disables the cluster's ceph target in systemd
and stops all ceph daemons. If the host is an osd host we apply the noout flag
for the host subtree in crush to prevent data movement during a host maintenance
window.
and stops all ceph daemons. NVMe-oF gateways on the host are administratively
disabled before daemons are stopped. If the host is an osd host we apply the
noout flag for the host subtree in crush to prevent data movement during a
host maintenance window.
:param hostname: (str) name of the host (must match an inventory hostname)
@ -2608,6 +2652,19 @@ Then run the following:
raise OrchestratorError(
msg + '\nNote: Warnings can be bypassed with the --force flag', errno=rc)
if 'nvmeof' in host_daemons:
_updated, nvmeof_err = self._set_nvmeof_gateways_admin_state(
hostname, enabled=False)
if nvmeof_err:
if yes_i_really_mean_it:
self.log.warning(
f"maintenance mode request for {hostname} failed to disable "
f"some NVMe-oF gateways: {nvmeof_err}")
else:
raise OrchestratorError(
f"Unable to disable NVMe-oF gateways on {hostname}: {nvmeof_err}",
errno=errno.EIO)
# call the host-maintenance function
with self.async_timeout_handler(hostname, 'cephadm host-maintenance enter'):
_out, _err, _code = self.wait_async(
@ -2649,8 +2706,8 @@ Then run the following:
"""Exit maintenance mode and return a host to an operational state
Returning from maintenance will enable the clusters systemd target and
start it, and remove any noout that has been added for the host if the
host has osd daemons
start it, re-enable any NVMe-oF gateways on the host, and remove any
noout that has been added for the host if the host has osd daemons
:param hostname: (str) host name
:param force: (bool) force removal of the host from maintenance mode
@ -2725,6 +2782,18 @@ Then run the following:
self.log.info(
f"exit maintenance request has UNSET for the noout group on host {hostname}")
if 'nvmeof' in self.cache.get_daemon_types(hostname):
_updated, nvmeof_err = self._set_nvmeof_gateways_admin_state(
hostname, enabled=True)
if nvmeof_err:
self.log.warning(
f"exit maintenance request for {hostname} failed to enable "
f"some NVMe-oF gateways: {nvmeof_err}")
if not force:
raise OrchestratorError(
f"Unable to enable NVMe-oF gateways on {hostname}: {nvmeof_err}",
errno=errno.EIO)
# update the host record status
tgt_host['status'] = ""
self.inventory._inventory[hostname] = tgt_host

74
src/pybind/mgr/cephadm/tests/test_cephadm.py
View File

@ -2381,6 +2381,80 @@ class TestCephadm(object):
assert cephadm_module.inventory._inventory[hostname]['status'] == 'maintenance'
@mock.patch("cephadm.module.CephadmOrchestrator.mon_command")
@mock.patch("cephadm.serve.CephadmServe._run_cephadm")
@mock.patch("cephadm.CephadmOrchestrator._host_ok_to_stop")
@mock.patch("cephadm.module.HostCache.get_daemons_by_type")
@mock.patch("cephadm.module.HostCache.get_daemon_types")
@mock.patch("cephadm.module.HostCache.get_hosts")
def test_maintenance_enter_disables_nvmeof_gateways(
self, _hosts, _get_daemon_types, _get_daemons_by_type, _host_ok,
_run_cephadm, _mon_command, cephadm_module: CephadmOrchestrator):
hostname = 'host1'
nvmeof_daemon = DaemonDescription(
daemon_type='nvmeof',
daemon_id='pool1.grp1.host1.abcd',
hostname=hostname,
)
_run_cephadm.side_effect = async_side_effect(
([''], ['something\nsuccess - systemd target xxx disabled'], 0))
_host_ok.return_value = 0, 'it is okay'
_get_daemon_types.return_value = ['nvmeof']
_get_daemons_by_type.return_value = [nvmeof_daemon]
_hosts.return_value = [hostname, 'other_host']
_mon_command.return_value = (0, '', '')
cephadm_module.spec_store.all_specs['nvmeof.pool1.grp1'] = ServiceSpec(
service_type='nvmeof', service_id='pool1.grp1', pool='pool1', group='grp1')
cephadm_module.inventory.add_host(HostSpec(hostname))
retval = cephadm_module.enter_host_maintenance(hostname)
assert retval.result_str().startswith('Daemons for Ceph cluster')
_mon_command.assert_called_once_with({
'prefix': 'nvme-gw disable',
'id': 'client.nvmeof.pool1.grp1.host1.abcd',
'pool': 'pool1',
'group': 'grp1',
})
_run_cephadm.assert_called_once()
@mock.patch("cephadm.module.CephadmOrchestrator.mon_command")
@mock.patch("cephadm.serve.CephadmServe._run_cephadm")
@mock.patch("cephadm.module.HostCache.get_daemon_types")
@mock.patch("cephadm.module.HostCache.get_daemons_by_type")
@mock.patch("cephadm.module.HostCache.get_hosts")
def test_maintenance_exit_enables_nvmeof_gateways(
self, _hosts, _get_daemons_by_type, _get_daemon_types,
_run_cephadm, _mon_command, cephadm_module: CephadmOrchestrator):
hostname = 'host1'
nvmeof_daemon = DaemonDescription(
daemon_type='nvmeof',
daemon_id='pool1.grp1.host1.abcd',
hostname=hostname,
)
_run_cephadm.side_effect = async_side_effect(([''], [
'something\nsuccess - systemd target xxx enabled and started'], 0))
_get_daemon_types.return_value = ['nvmeof']
_get_daemons_by_type.return_value = [nvmeof_daemon]
_hosts.return_value = [hostname, 'other_host']
_mon_command.return_value = (0, '', '')
cephadm_module.spec_store.all_specs['nvmeof.pool1.grp1'] = ServiceSpec(
service_type='nvmeof', service_id='pool1.grp1', pool='pool1', group='grp1')
cephadm_module.inventory.add_host(HostSpec(hostname, status='maintenance'))
retval = cephadm_module.exit_host_maintenance(hostname)
assert retval.result_str().startswith('Ceph cluster')
assert _run_cephadm.call_count == 2
_run_cephadm.assert_any_call(
hostname, cephadmNoImage, 'check-host', [], error_ok=False)
_run_cephadm.assert_any_call(
hostname, cephadmNoImage, 'host-maintenance', ['exit'], error_ok=True)
_mon_command.assert_called_once_with({
'prefix': 'nvme-gw enable',
'id': 'client.nvmeof.pool1.grp1.host1.abcd',
'pool': 'pool1',
'group': 'grp1',
})
@mock.patch("cephadm.ssh.SSHManager._remote_connection")
@mock.patch("cephadm.ssh.SSHManager._execute_command")
@mock.patch("cephadm.ssh.SSHManager._check_execute_command")