gian/ceph-mirror
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2026-08-01 22:45:39 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge branch 'main' into issue-77250-clean

Browse Source 
Signed-off-by: kobi ginon <153318313+kginonredhat@users.noreply.github.com>
This commit is contained in:
kobi ginon 2026-07-11 13:27:02 +03:00 committed by GitHub
commit 93ff14d091
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1321 changed files with 120497 additions and 34155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/check-license.yml vendored
View File

@ -2,6 +2,10 @@
name: "Check for Incompatible Licenses"
on: [pull_request]
permissions:
contents: read
pull-requests: read
jobs:
pull_request:
name: "Check for Incompatible Licenses"

27
.github/workflows/releng-audit.yaml vendored
View File

@ -212,7 +212,11 @@ jobs:
if (!isAuthorized) {
core.info(`[Router] User @${actor} NOT authorized. Removing override label.`);
await github.rest.issues.removeLabel({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, name: labelName });
try {
await github.rest.issues.removeLabel({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, name: labelName });
} catch (e) {
core.info(`[Router] Label removal skipped or already deleted: ${e.message}`);
}
core.setFailed(`User @${actor} is not authorized to override audits.`);
} else {
core.info(`[Router] User @${actor} is authorized. Stripping fail/pass labels to visually unblock PR.`);
@ -224,7 +228,11 @@ jobs:
}
}
core.info(`[Router] Labeled event handled without triggering audit.`);
if (hasFailLabel && labelName !== 'releng-audit-override') {
core.setFailed("PR is currently in a failed audit state. Remove the releng-audit-fail label to re-run.");
} else {
core.info(`[Router] Labeled event handled without triggering audit.`);
}
core.setOutput('run_audit', 'false');
return;
}
@ -250,7 +258,11 @@ jobs:
return;
}
core.info('[Router] Event did not match any trigger criteria. Skipping audit.');
if (hasFailLabel && eventName === 'pull_request_target') {
core.setFailed("PR is currently in a failed audit state. Remove the releng-audit-fail label to re-run.");
} else {
core.info('[Router] Event did not match any trigger criteria. Skipping audit.');
}
core.setOutput('run_audit', 'false');
- name: Checkout Trusted Base Repository
@ -258,7 +270,14 @@ jobs:
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
- name: Fetch main for parity check
if: steps.router.outputs.run_audit == 'true'
# checkout only fetches the PR's base branch (e.g. squid/tentacle/umbrella).
# The parity check needs the main ref to walk the commit history graph and find the upstream
# merge commit for each cherry-pick, so fetch it explicitly as origin/main.
run: git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main
- name: Setup Python
if: steps.router.outputs.run_audit == 'true'
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0

54
CMakeLists.txt
View File

@ -75,6 +75,20 @@ if(MINGW)
link_directories(${MINGW_LINK_DIRECTORIES})
endif()
option(WITH_MOLD "Use the Mold linker" OFF)
if(WITH_MOLD)
find_program(_mold_path mold REQUIRED)
message(STATUS "Mold linker: ${_mold_path}")
set(CMAKE_LINKER ${_mold_path} CACHE FILEPATH "Linker" FORCE)
set(MOLD_FUSE_LD_FLAG "-fuse-ld=mold" CACHE INTERNAL "")
foreach(_flags CMAKE_EXE_LINKER_FLAGS CMAKE_SHARED_LINKER_FLAGS CMAKE_MODULE_LINKER_FLAGS)
if(NOT "${${_flags}}" MATCHES "fuse-ld=")
string(APPEND ${_flags} " ${MOLD_FUSE_LD_FLAG}")
endif()
endforeach()
set(USING_MOLD_LINKER TRUE CACHE INTERNAL "")
endif()
option(WITH_CCACHE "Build with ccache.")
if(WITH_CCACHE)
if(CMAKE_C_COMPILER_LAUNCHER OR CMAKE_CXX_COMPILER_LAUNCHER)
@ -460,6 +474,10 @@ if(ALLOCATOR)
elseif(NOT ALLOCATOR STREQUAL "libc")
message(FATAL_ERROR "Unsupported allocator selected: ${ALLOCATOR}")
endif()
elseif(WITH_ASAN)
# Force libc: tcmalloc/jemalloc still export operator new/delete even when the
# sanitizer shadows malloc, so sanitizer-allocated memory is freed via tcmalloc and SIGSEGVs.
set(ALLOCATOR "libc")
else(ALLOCATOR)
find_package(gperftools 2.6.2)
set(HAVE_LIBTCMALLOC ${gperftools_FOUND})
@ -574,7 +592,7 @@ option(WITH_RADOSGW_DBSTORE "DBStore backend for RADOS Gateway" ON)
option(WITH_RADOSGW_MOTR "CORTX-Motr backend for RADOS Gateway" OFF)
option(WITH_RADOSGW_DAOS "DAOS backend for RADOS Gateway" OFF)
option(WITH_RADOSGW_D4N "D4N wrapper for RADOS Gateway" ON)
cmake_dependent_option(WITH_RADOSGW_POSIX "POSIX backend for RADOS Gateway" ON WITH_RADOSGW_DBSTORE OFF) # posix depends on dbstore
option(WITH_RADOSGW_FDB "FoundationDB support for RADOS Gateway (experimental)" OFF)
option(WITH_RADOSGW_RADOS "RADOS backend for Rados Gateway" ON)
option(WITH_RADOSGW_SELECT_PARQUET "Support for s3 select on parquet objects" ON)
option(WITH_RADOSGW_ARROW_FLIGHT "Build arrow flight when not using system-provided arrow" OFF)
@ -583,6 +601,9 @@ option(WITH_RADOSGW_BACKTRACE_LOGGING "Enable backtraces in rgw logs" OFF)
option(WITH_SYSTEM_ARROW "Use system-provided arrow" OFF)
option(WITH_SYSTEM_UTF8PROC "Use system-provided utf8proc" OFF)
# POSIX depends on DBStore:
cmake_dependent_option(WITH_RADOSGW_POSIX "POSIX backend for RADOS Gateway" ON WITH_RADOSGW_DBSTORE OFF)
if(WITH_RADOSGW)
find_package(EXPAT REQUIRED)
find_package(OATH REQUIRED)
@ -636,6 +657,23 @@ if(WITH_RADOSGW)
message(STATUS "crypto soname: ${LIBCRYPTO_SONAME}")
endif (WITH_RADOSGW)
if(WITH_RADOSGW_FDB)
include(${CMAKE_MODULE_PATH}/CPM.cmake)
CPMAddPackage("gh:eyalz800/zpp_bits@4.5.1")
message("-- enabled zpp_bits")
# JFW: sadly, I'm having trouble getting FoundationDB to build, but this is VERY close
# to "just working". But, how would we install the RPMs, etc.? This will need looking at
# before this can be non-experimental:
# CPMAddPackage("gh:apple/foundationdb@7.3.63")
message("FoundationDB support is EXPERIMENTAL and requires manual setup:")
message(" wget https://github.com/apple/foundationdb/releases/download/7.3.63/foundationdb-clients-7.3.63-1.el7.x86_64.rpm")
message(" wget https://github.com/apple/foundationdb/releases/download/7.3.63/foundationdb-server-7.3.63-1.el7.x86_64.rpm")
message("-- enabled FoundationDB (whether its there or not)")
endif()
#option for CephFS
option(WITH_CEPHFS "CephFS is enabled" ON)
@ -720,6 +758,13 @@ endif(LINUX)
option(WITH_ASAN "build with ASAN" OFF)
if(WITH_ASAN)
list(APPEND sanitizers "address")
# Shared ASan/LSan runtime options: the in-tree suppression files plus the
# flags our tests need. Consumed by add_ceph_test() and baked into bin/ceph
# so both ignore the same still-reachable third-party leaks.
set(CEPH_ASAN_OPTIONS "suppressions=${CMAKE_SOURCE_DIR}/qa/asan.supp,detect_odr_violation=0"
CACHE INTERNAL "ASAN_OPTIONS for ceph tests and the ceph CLI")
set(CEPH_LSAN_OPTIONS "suppressions=${CMAKE_SOURCE_DIR}/qa/lsan.supp,print_suppressions=0"
CACHE INTERNAL "LSAN_OPTIONS for ceph tests and the ceph CLI")
endif()
option(WITH_ASAN_LEAK "explicitly enable ASAN leak detection" OFF)
@ -817,6 +862,12 @@ else()
include(BuildBoost)
build_boost(1.87
COMPONENTS ${BOOST_COMPONENTS} ${BOOST_HEADER_COMPONENTS})
if(WITH_ASAN)
# Boost.Context is built ucontext-only here (context-impl=ucontext); define
# the matching backend tree-wide so every Boost.Context/Coroutine2 consumer
# agrees on it, instead of relying on per-target propagation that is easy to miss.
add_compile_definitions(BOOST_USE_ASAN BOOST_USE_UCONTEXT)
endif()
endif()
include_directories(BEFORE SYSTEM ${Boost_INCLUDE_DIRS})
add_library(Boost::asio INTERFACE IMPORTED)
@ -893,4 +944,3 @@ add_tags(ctags
EXCLUDE_OPTS ${CTAG_EXCLUDES}
EXCLUDES "*.js" "*.css" ".tox" "python-common/build")
add_custom_target(tags DEPENDS ctags)

76
PendingReleaseNotes
View File

@ -1,3 +1,17 @@
* CephFS: The ``client_force_lazyio`` configuration option is now correctly marked
as not supporting runtime updates. Previously, the configuration schema indicated
this option could be changed at runtime, but changes had no effect on opened file
handlers in ceph-fuse or libcephfs clients because there is no logic to propagate
changes to open file handles.
* RADOS: PG autoscaler allows overlapping roots. Each root receives a PG target based
on its OSDs, with OSDs shared across multiple roots contributing proportionally
less to each root's allocation.
* CephFS: CephFS snapshots metadata is now mutable. It is now possible to add,
update and remove existing key-value pairs that are part of a snapshot
metadata via libcephfs API ceph_do_snap_md_op().
* librados/neorados: The C++ APIs for executing Ceph Class (CLS) methods
have undergone a breaking change to enforce compile-time type safety, replacing
legacy string-based parameters with strongly-typed ClsMethod structs. C++
@ -47,6 +61,12 @@
If the default provider is required when also using custom providers,
it must be explicitly loaded in the configuration file or code (see https://github.com/openssl/openssl/blob/master/README-PROVIDERS.md).
* RGW: Fixed bucket notification events so the 'x_amz_request_id' in NotificationEvent now matches the 'x_amz_request_id' returned by the corresponding S3 operation.
* RGW: The rgw_gc_max_deferred and rgw_gc_max_deferred_entries_size options have been removed, as they did not do anything.
- Updated the default for rgw_gc_max_queue_size to account for the extra space from removing rgw_gc_max_deferred_entries_size.
* RGW: The default value of ``rgw_thread_pool_size`` has been reduced from 512 to 128.
Benchmarks showed that the lower thread count improves throughput and reduces latency
on NVMe and HDD based environments. Users with specific workload requirements can
tune this value; see also ``rgw_max_concurrent_requests``.
* DASHBOARD: A new Overview landing page provides an at-a-glance cluster summary
@ -165,6 +185,35 @@
``ceph fs snapshot mirror daemon status`` now shows the remote cluster's
monitor addresses and cluster ID for each configured peer, making it easier
to verify peer connectivity and troubleshoot mirroring issues.
* CephFS Mirroring: The ``fs mirror peer status`` admin socket command reports
additional per-directory sync metrics (sync mode, throughput, crawl and
data-sync queue timing, bytes/files progress, and ETA). Output is grouped
under ``metrics/<mirrored-dir-path>/peer/<peer-uuid>``. For more information,
see https://tracker.ceph.com/issues/73453
* CephFS Mirroring: Per-directory snapshot sync progress is exposed as labeled perf
counters in the ``cephfs_mirror_directory`` group (``counter dump`` on the mirror
daemon admin socket, exportable via ``ceph-exporter``). Counters mirror
``fs mirror peer status`` fields (directory state, current sync, last sync, and
snap summary) and are labeled by peer UUID and mirrored directory path. See
https://tracker.ceph.com/issues/73457
* CephFS Mirroring: The mirroring module provides ``ceph fs snapshot mirror status``,
a Ceph CLI command similar to the ``fs mirror peer status`` admin socket interface
for viewing per-directory snapshot sync metrics. The output layout matches
``fs mirror peer status`` for core sync fields and additionally includes
``metrics_updated_at`` (time of the last omap write). Optional filters by
mirrored directory path and peer UUID are supported. For more information,
see https://tracker.ceph.com/issues/76686
* CephFS Mirroring: Introduced snapshot checkpoints feature that allows users to mark
specific snapshots as important milestones and track their replication status to remote
sites. Checkpoints automatically detect if a snapshot has already been synced and provide
visibility into disaster recovery readiness. The feature includes four new CLI commands:
``ceph fs snapshot mirror checkpoint add`` to mark a snapshot,
``ceph fs snapshot mirror checkpoint now`` to checkpoint the latest snapshot,
``ceph fs snapshot mirror checkpoint ls`` to list all checkpoints with their status
(created/complete/failed), and ``ceph fs snapshot mirror checkpoint remove`` to remove
a checkpoint. Checkpoint metadata is persistent across daemon restarts. This feature is
useful for compliance auditing, application consistency verification, and SLA tracking.
For more information, see https://tracker.ceph.com/issues/73454
* RBD: Mirror snapshot creation and trash purge schedules are now automatically
staggered when no explicit "start-time" is specified. This reduces scheduling
spikes and distributes work more evenly over time.
@ -175,6 +224,17 @@
than instantly aborting I/O and returning an error. It's possible to switch
from ``RBD_LOCK_MODE_EXCLUSIVE`` to ``RBD_LOCK_MODE_EXCLUSIVE_TRANSIENT``
policy and vice versa even if the lock is already held.
* RGW: In multisite deployments, zone endpoints configured in the zonegroup
(e.g. ``https://zone-a.example.com``) can now be resolved to all IP addresses
returned by DNS, with inter-zone traffic distributed across them using
round-robin and per-IP health tracking. This enables DNS-based service discovery
for inter-zone traffic without an external load balancer. This feature is disabled
by default: to opt in, set ``rgw_rest_conn_connect_to_resolved_ips = true``. The
per-IP retry-after-failure timeout is controlled by ``rgw_rest_conn_ip_fail_timeout_secs``
(default: 2 seconds). Deployments that work around libcurl's single-address behavior
by repeating the same endpoint multiple times in zone configuration should review that
setup before enabling, as round-robin distribution makes such duplication unnecessary.
* OSD: A health warning is reported when BlueFS usage exceeds the
configured ratio of the main OSD data device size. This warning is
informational and can be muted with:
@ -196,6 +256,12 @@
state and a subsequent redundancy loss occurs. Used in conjunction with
``last_clean``, the ``last_degraded`` timestamp enables the calculation of
data vulnerability and durability scores.
* RBD: It's possible to specify source cluster's ``mon_host`` and ``key`` for
``native`` format migration via the migration spec now. This eliminates the
dependency on ``<cluster-name>.conf`` file in a known location which is
rather rigid and also challenging to disseminate in some environments. The
key can be embedded in the migration spec or just referenced from there while
stored in the MON config-key store.
>=20.0.0
@ -996,3 +1062,13 @@ can be configured using the `rgw` option `rgw_bucket_persistent_notif_num_shards
https://docs.ceph.com/en/latest/radosgw/notifications/
Relevant tracker: https://tracker.ceph.com/issues/71677
* RGW: Multipart upload part heads are now tracked under a new
``rgw.multipart`` category in bucket stats, separate from completed
objects (``rgw.main``). This makes ``radosgw-admin bucket stats``
and the ``X-RGW-Object-Count`` header on HEAD Bucket more accurate:
in-progress or abandoned multipart parts no longer inflate the
object count. Existing in-progress uploads retain the old category
until they complete or abort.
Relevant tracker: https://tracker.ceph.com/issues/77509

1
admin/doc-read-the-docs.txt
View File

@ -1,2 +1 @@
plantweb
readthedocs-sphinx-search@git+https://github.com/readthedocs/readthedocs-sphinx-search@main

552
ceph.spec.in
View File

File diff suppressed because it is too large Load Diff

9
cmake/modules/AddCephTest.cmake
View File

@ -33,8 +33,8 @@ function(add_ceph_test test_name test_path)
set_property(TEST ${test_name}
APPEND
PROPERTY ENVIRONMENT
ASAN_OPTIONS=suppressions=${CMAKE_SOURCE_DIR}/qa/asan.supp,detect_odr_violation=0
LSAN_OPTIONS=suppressions=${CMAKE_SOURCE_DIR}/qa/lsan.supp,print_suppressions=0)
ASAN_OPTIONS=${CEPH_ASAN_OPTIONS}
LSAN_OPTIONS=${CEPH_LSAN_OPTIONS})
endif()
set_property(TEST ${test_name}
PROPERTY TIMEOUT ${CEPH_TEST_TIMEOUT})
@ -73,8 +73,11 @@ if(WITH_GTEST_PARALLEL)
BUILD_COMMAND ""
INSTALL_COMMAND "")
add_dependencies(tests gtest-parallel_ext)
# CACHE INTERNAL: the set() runs only in the first directory to create the
# target, so a plain variable would be invisible to PARALLEL tests elsewhere.
set(GTEST_PARALLEL_COMMAND
${Python3_EXECUTABLE} ${gtest_parallel_source_dir}/gtest-parallel)
${Python3_EXECUTABLE} ${gtest_parallel_source_dir}/gtest-parallel
CACHE INTERNAL "command to run a gtest binary through gtest-parallel")
endif()
endif()

21
cmake/modules/BuildBoost.cmake
View File

@ -144,15 +144,26 @@ function(do_build_boost root_dir version)
if(WITH_BOOST_VALGRIND)
list(APPEND b2 valgrind=on)
endif()
set(b2_targets headers stage)
set(b2_install_targets install)
if(WITH_ASAN)
list(APPEND b2 context-impl=ucontext)
# build the library with the BOOST_USE_ASAN consumers get from Boost::context,
# so fiber_activation_record has one layout (else heap-buffer-overflow)
list(APPEND b2 define=BOOST_USE_ASAN)
# `context-impl` is declared in libs/context/build/Jamfile.v2; the headers/stage
# and install targets never load it, so b2 aborts with `unknown feature
# "<context-impl>"`. Name the context project as a target so its Jamfile loads
# the feature first.
list(PREPEND b2_targets libs/context/build)
list(PREPEND b2_install_targets libs/context/build)
endif()
set(build_command
${b2} headers stage
${b2} ${b2_targets}
#"--buildid=ceph" # changes lib names--can omit for static
${boost_features})
set(install_command
${b2} install)
${b2} ${b2_install_targets})
if(EXISTS "${PROJECT_SOURCE_DIR}/src/boost/bootstrap.sh")
check_boost_version("${PROJECT_SOURCE_DIR}/src/boost" ${version})
set(source_dir
@ -254,10 +265,8 @@ macro(build_boost version)
set_target_properties(Boost::${c} PROPERTIES
INTERFACE_COMPILE_DEFINITIONS "BOOST_USE_VALGRIND")
endif()
if((c MATCHES "context") AND (WITH_ASAN))
set_target_properties(Boost::${c} PROPERTIES
INTERFACE_COMPILE_DEFINITIONS "BOOST_USE_ASAN;BOOST_USE_UCONTEXT")
endif()
# ASan's BOOST_USE_ASAN/BOOST_USE_UCONTEXT are defined tree-wide in the
# top-level CMakeLists.txt, not per-target.
list(APPEND Boost_LIBRARIES ${Boost_${upper_c}_LIBRARY})
endforeach()
foreach(c ${Boost_BUILD_COMPONENTS})

4
cmake/modules/BuildFIO.cmake
View File

@ -30,10 +30,10 @@ function(build_fio)
file(MAKE_DIRECTORY ${source_dir})
ExternalProject_Add(fio_ext
UPDATE_COMMAND "" # this disables rebuild on each run
GIT_REPOSITORY "https://github.com/ceph/fio.git"
GIT_REPOSITORY "https://github.com/axboe/fio.git"
GIT_CONFIG advice.detachedHead=false
GIT_SHALLOW 1
GIT_TAG "fio-3.27-cxx"
GIT_TAG "fio-3.42"
SOURCE_DIR ${source_dir}
BUILD_IN_SOURCE 1
CONFIGURE_COMMAND <SOURCE_DIR>/configure

9
cmake/modules/CephChecks.cmake
View File

@ -202,3 +202,12 @@ try_compile(HAVE_LINK_EXCLUDE_LIBS
${CMAKE_CURRENT_BINARY_DIR}
SOURCES ${CMAKE_CURRENT_LIST_DIR}/CephCheck_link.c
LINK_LIBRARIES "-Wl,--exclude-libs,ALL")
# Mold linker applies --exclude-libs after version script processing,
# which hides .symver-aliased symbols (e.g. rados_*) even when the
# version script lists them as global. Disable --exclude-libs for Mold;
# version scripts already control symbol visibility.
if(HAVE_LINK_EXCLUDE_LIBS AND USING_MOLD_LINKER)
message(STATUS "Mold linker -- disabling --exclude-libs (incompatible with .symver)")
set(HAVE_LINK_EXCLUDE_LIBS FALSE CACHE INTERNAL "" FORCE)
endif()

12
cmake/modules/Distutils.cmake
View File

@ -72,7 +72,11 @@ function(distutils_add_cython_module target name src)
list(APPEND PY_CPPFLAGS -D'__Pyx_check_single_interpreter\(ARG\)=ARG\#\#0')
set(PY_CC ${compiler_launcher} ${CMAKE_C_COMPILER} ${c_compiler_arg1})
set(PY_CXX ${compiler_launcher} ${CMAKE_CXX_COMPILER} ${cxx_compiler_arg1})
set(PY_LDSHARED ${link_launcher} ${CMAKE_C_COMPILER} ${c_compiler_arg1} "-shared")
if(USING_MOLD_LINKER)
set(PY_LDSHARED ${link_launcher} ${CMAKE_C_COMPILER} ${c_compiler_arg1} "-shared" "${MOLD_FUSE_LD_FLAG}")
else()
set(PY_LDSHARED ${link_launcher} ${CMAKE_C_COMPILER} ${c_compiler_arg1} "-shared")
endif()
string(REPLACE " " ";" PY_LDFLAGS "${CMAKE_SHARED_LINKER_FLAGS}")
list(APPEND PY_LDFLAGS -L${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
@ -117,7 +121,11 @@ function(distutils_install_cython_module name)
get_property(compiler_launcher GLOBAL PROPERTY RULE_LAUNCH_COMPILE)
get_property(link_launcher GLOBAL PROPERTY RULE_LAUNCH_LINK)
set(PY_CC "${compiler_launcher} ${CMAKE_C_COMPILER}")
set(PY_LDSHARED "${link_launcher} ${CMAKE_C_COMPILER} -shared")
if(USING_MOLD_LINKER)
set(PY_LDSHARED "${link_launcher} ${CMAKE_C_COMPILER} -shared ${MOLD_FUSE_LD_FLAG}")
else()
set(PY_LDSHARED "${link_launcher} ${CMAKE_C_COMPILER} -shared")
endif()
set(PY_LDFLAGS "${CMAKE_SHARED_LINKER_FLAGS} -L${CMAKE_LIBRARY_OUTPUT_DIRECTORY}")
cmake_parse_arguments(DU "DISABLE_VTA" "" "" ${ARGN})
if(DU_DISABLE_VTA AND HAS_VTA)

11
container/Containerfile
View File

@ -124,7 +124,16 @@ RUN --mount=type=secret,id=prerelease_creds set -ex && \
source /run/secrets/prerelease_creds; \
REPO_URL="https://${PRERELEASE_USERNAME}:${PRERELEASE_PASSWORD}@download.ceph.com/prerelease/ceph/rpm-${CEPH_REF}/${EL_VER}/" ;\
fi && \
rpm -Uvh "$REPO_URL/noarch/ceph-release-1-${IS_RELEASE}.${EL_VER}.noarch.rpm" ; \
# Trim trailing slashes
REPO_URL="${REPO_URL%/}" && \
if [[ "$REPO_URL" == */pulp/content/* ]] ; then \
# Pulp nests packages under noarch/Packages/<first-letter>/
CEPH_RELEASE_RPM="$REPO_URL/noarch/Packages/c/ceph-release-1-${IS_RELEASE}.${EL_VER}.noarch.rpm" ; \
else \
# chacra serves packages flat under noarch/
CEPH_RELEASE_RPM="$REPO_URL/noarch/ceph-release-1-${IS_RELEASE}.${EL_VER}.noarch.rpm" ; \
fi && \
rpm -Uvh "$CEPH_RELEASE_RPM" ; \
if [[ "$IS_RELEASE" == 1 ]] ; then \
sed -i "s;http://download.ceph.com/;https://${PRERELEASE_USERNAME}:${PRERELEASE_PASSWORD}@download.ceph.com/prerelease/ceph/;" /etc/yum.repos.d/ceph.repo ; \
dnf clean expire-cache ; \

5
debian/copyright vendored
View File

@ -209,9 +209,8 @@ License: GPL-2
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
.
On Debian systems, the complete text of the GNU General Public License
version 2 can be found in `/usr/share/common-licenses/GPL-2' file.

119
doc/cephadm/host-management.rst
View File

@ -551,8 +551,28 @@ cephadm operations. Run a command of the following form:
ceph cephadm set-user <user>
Prior to running this, the cluster SSH key needs to be added to this user's
``authorized_keys`` file and non-root users must have passwordless sudo access.
The ``set-user`` command automatically configures the specified user on all cluster
hosts by calling ``cephadm setup-ssh-user`` on each host. This command includes the following:
- Setting up passwordless sudo access for non-root users
- Authorizing the cluster's SSH public key for the user
If you have already manually configured the user on all hosts, you can skip
the automatic setup by using the ``--skip-pre-steps`` flag:
.. prompt:: bash #
ceph cephadm set-user <user> --skip-pre-steps
For manual setup of SSH users on individual hosts, you can use the
``cephadm setup-ssh-user`` command directly:
.. prompt:: bash #
cephadm setup-ssh-user --ssh-user <user> --ssh-pub-key <public_key>
This command validates that the user exists, configures passwordless sudo access,
and authorizes the SSH public key.
Customizing the SSH Configuration
@ -668,6 +688,95 @@ when executing ``ceph * metadata``. This in turn means cephadm also
requires the bare host name when adding a host to the cluster:
``ceph orch host add <bare-name>``.
..
TODO: This chapter needs to provide way for users to configure
Grafana in the dashboard, as this is right now very hard to do.
Sudo Hardening
==============
Cephadm supports sudo hardening to enhance security by restricting sudo privilege
escalation for non-root SSH users. When sudo hardening is enabled, cephadm uses the
``cephadm_invoker.py`` script to securely execute cephadm commands with controlled
privilege escalation.
Enabling Sudo Hardening
-----------------------
To enable sudo hardening for the entire cluster, use the following command:
.. prompt:: bash #
ceph cephadm prepare-host-and-enable-sudo-hardening <user>
This command performs a comprehensive sudo hardening setup:
1. **Host Preparation**: Prepares all cluster hosts for sudo hardening by:
- Installing/upgrading cephadm RPM with the invoker script
- Configuring restricted sudo access for non-root users
- Setting up SSH key authorization
2. **SSH User Configuration**: Sets the specified user for cluster SSH operations
3. **Global Enablement**: Enables sudo hardening cluster-wide
The ``<user>`` parameter specifies which non-root user should be configured for SSH access.
This user will have restricted sudo access configured through the sudoers file.
You can manually prepare a host for sudo hardening using:
.. prompt:: bash #
cephadm prepare-host-sudo-hardening --ssh-user <user> --ssh-pub-key <pub_key>
.. note:: During initial host addition, the root user is used for setup. After
Sudo hardening is enabled, the specified non-root user with restricted sudo
access will be used for ongoing operations.
Sudo Hardening Workflow
-----------------------
When sudo hardening is enabled, the following workflow is used:
1. **Host Addition**: Before adding a new host with ``ceph orch host add``,
the cluster SSH key needs to be added to this user's ``authorized_keys``
file and non-root users must have passwordless sudo access.
2. **Command Execution**: Instead of executing cephadm directly, commands are
routed through ``cephadm_invoker.py``
3. **Binary Verification**: The invoker validates the cephadm binary's hash before execution
4. **Secure Execution**: Commands are executed with restricted permissions
The ``cephadm_invoker.py`` script provides the following subcommands:
- ``run``: Execute cephadm binary with hash verification
- ``deploy_cephadm_binary``: Deploy cephadm binary to final location
- ``check_existence``: Check if a file exists
Sudo Access Restrictions
-------------------------
Sudo hardening restricts sudo access for non-root users to enhance security. When a
host is prepared for sudo hardening, the sudoers configuration is modified to limit
the commands that can be executed with sudo. This prevents unauthorized command
execution while still allowing necessary cephadm operations.
The sudoers configuration restricts access to only the ``cephadm_invoker.py`` script
and essential system commands, providing a secure execution environment.
Security Benefits
-----------------
Sudo hardening provides the following security benefits:
1. The invoker validates the cephadm binary's hash
2. If validation fails, it signals for binary redeployment
3. Commands are executed with restricted sudo permissions
4. All operations are logged for security auditing
Disabling Sudo Hardening
------------------------
To disable sudo hardening:
.. prompt:: bash #
ceph config set mgr mgr/cephadm/sudo_hardening false
.. note:: Disabling sudo hardening does not automatically revert host configurations.
Hosts that were prepared for sudo hardening will retain the invoker setup.

28
doc/cephadm/operations.rst
View File

@ -393,6 +393,34 @@ For example:
``--log-dest`` options described in the bootstrap section above.
Setting Cephadm Log Verbosity
-----------------------------
The verbosity of cephadm's persistent logging (``/var/log/ceph/cephadm.log``
and syslog, when enabled) can be controlled separately from its terminal
output. Valid levels are ``info``, ``debug``, ``error``, and ``warning``.
The default is ``debug``.
When the cephadm orchestration module runs ``cephadm`` on cluster hosts, it
uses the :confval:`mgr/cephadm/cephadm_binary_logging_level` configuration
value:
.. prompt:: bash #
ceph config set mgr mgr/cephadm/cephadm_binary_logging_level info
For manual invocations, pass ``--logging-level`` on the command line:
.. prompt:: bash #
cephadm --logging-level info check-host
.. note:: The logging level applies only to persistent log destinations
(the log file and syslog). Console output during bootstrap and other
interactive use is unchanged. When you run ``cephadm`` directly, the
mgr configuration does not apply; use ``--logging-level`` as shown above.
Data Location
=============

7
doc/cephadm/services/index.rst
View File

@ -806,6 +806,13 @@ For example:
ceph orch rm rgw.myrgw
The same command accepts ``--force`` and ``--force-delete-data``. Use
``ceph orch rm <service-name> --force --force-delete-data`` when you want
cephadm to remove on-disk data for supported daemon types instead of relocating
it under ``<fsid>/removed/``. The latter flag requires ``--force``. The same
pair of flags applies to ``ceph orch daemon rm <daemon-name>`` when removing
individual daemons.
.. _cephadm-spec-unmanaged:

7
doc/cephadm/services/monitoring.rst
View File

@ -463,11 +463,16 @@ To disable monitoring and remove the software that supports it, run the followin
.. prompt:: bash #
ceph orch rm grafana
ceph orch rm prometheus --force # this will delete metrics data collected so far
ceph orch rm prometheus
ceph orch rm node-exporter
ceph orch rm alertmanager
ceph mgr module disable prometheus
By default, cephadm moves Prometheus data aside under ``<fsid>/removed/`` on the
host. To delete that data instead, use
``ceph orch rm prometheus --force --force-delete-data`` in place of
``ceph orch rm prometheus`` (``--force-delete-data`` requires ``--force``).
See also :ref:`orch-rm`.
Setting up RBD-Image Monitoring

3
doc/cephadm/services/nfs.rst
View File

@ -60,6 +60,7 @@ Alternatively, an NFS service can be applied using a YAML specification.
host2: 10.0.0.124
monitoring_networks:
- 192.168.124.0/24
enable_nfsv3: true
In this example, we run the server on the non-default ``port`` of
@ -78,6 +79,8 @@ IP address is assigned to the host, that IP address will be used. If the IP
address is not present and ``monitoring_networks`` is specified, an IP address
that matches one of the specified networks will be used. If neither condition
is met, the default binding will happen on all available network interfaces.
By default, only the NFSv4 protocol is enabled. NFSv3 can be enabled by setting
``enable_nfsv3`` to ``true`` in the service specification.
NFS over RDMA
-------------

89
doc/cephadm/services/smb.rst
View File

@ -53,13 +53,100 @@ An SMB service can be applied using a specification. An example in YAML follows:
include_ceph_users:
- client.smb.fs.cluster.tango
TLS/SSL Example
---------------
Here's an example SMB service specification with TLS/SSL configuration:
.. code-block:: yaml
service_id: smbcluster
service_type: smb
cluster_id: tango
config_uri: rados://smb/foxtrot/config.json
placement:
hosts:
- host0
spec:
ssl_certificates:
remote_control:
enabled: true
certificate_source: inline
ssl_cert: |
-----BEGIN CERTIFICATE-----
...
-----END CERTIFICATE-----
ssl_key: |
-----BEGIN PRIVATE KEY-----
...
-----END PRIVATE KEY-----
ssl_ca_cert: |
-----BEGIN CERTIFICATE-----
...
-----END CERTIFICATE-----
keybridge:
enabled: true
certificate_source: inline
ssl_cert: |
-----BEGIN CERTIFICATE-----
...
-----END CERTIFICATE-----
ssl_key: |
-----BEGIN PRIVATE KEY-----
...
-----END PRIVATE KEY-----
ssl_ca_cert: |
-----BEGIN CERTIFICATE-----
...
-----END CERTIFICATE-----
This example configures an SMB service with TLS encryption enabled using
inline certificates.
TLS/SSL Parameters
~~~~~~~~~~~~~~~~~~
The following parameters can be used to configure TLS/SSL encryption per sidecar
for the SMB service:
* ``enabled`` (boolean): Enable or disable SSL/TLS encryption. Default is ``false``.
* ``certificate_source`` (string): Specifies the source of the TLS certificates.
Options include:
- ``cephadm-signed``: Use certificates signed by cephadm's internal CA
- ``inline``: Provide certificates directly in the specification using ``ssl_cert``,
``ssl_key`` and ``ssl_ca_cert`` fields
- ``reference``: Users can register their own certificate and key with certmgr and
set the ``certificate_source`` to ``reference`` in the spec.
* ``ssl_cert`` (string): The SSL certificate in PEM format. Required when using
``inline`` certificate source.
* ``ssl_key`` (string): The SSL private key in PEM format. Required when using
``inline`` certificate source.
* ``ssl_ca_cert`` (string): The SSL CA certificate in PEM format. Required when
using ``inline`` certificate source.
.. note::
``ssl_key``, ``ssl_cert`` and ``ssl_ca_cert`` can be set from the smb manager
module. If ``cert`` and ``key`` are specified in the resource_type
``ceph.smb.tls.credential`` and applied from the smb manager will be automatically
configured as ssl_certificate is enabled and update ``ssl_key``, ``ssl_cert`` to
the certificate manager. ``ssl_ca_cert`` will be set if it is specified in the
resource_type ``ceph.smb.tls.credential``
The specification can then be applied by running the following command:
.. prompt:: bash #
ceph orch apply -i smb.yaml
Service Spec Options
--------------------

2
doc/cephadm/upgrade.rst
View File

@ -136,7 +136,7 @@ Requirements:
(same shape as ``ceph_version_short`` in ``ceph osd metadata``).
* If the Monitors indicate to cephadm that no OSDs in the selected CRUSH bucket
are okay to upgrade, cephadm will log details and then retry the operation.
* If the bucket parameters for a ceph ``osd ok-to-upgrade`` upgrade are not provided,
* If the bucket parameters for a ceph ``osd ok-to-upgrade`` upgrade are not provided,
cephadm will fall back to the default ceph osd ok-to-stop gate for OSD upgrades.
* Bucket-scope upgrades apply only to OSDs. CRUSH buckets do not influence upgrades
of other daemon types, for example Monitors, Managers, and MDSes.

360
doc/cephfs/cephfs-mirroring-checkpoints.rst Normal file
View File

@ -0,0 +1,360 @@
========================================
CephFS Snapshot Mirroring Checkpoints
========================================
.. contents::
:depth: 3
:local:
Introduction
============
CephFS Snapshot Mirroring Checkpoints allow you to mark specific snapshots as important
milestones and track whether they have been successfully replicated to your remote site.
This feature provides visibility into your disaster recovery readiness by showing which
critical data states have been safely mirrored.
**Use Cases:**
- **Compliance & Auditing**: Verify that end-of-day or end-of-month snapshots are replicated
- **Application Consistency**: Ensure application-consistent snapshots reach the DR site
- **Migration Tracking**: Monitor progress when migrating data between sites
- **SLA Verification**: Confirm that critical snapshots meet replication SLAs
What is a Checkpoint?
=====================
A checkpoint is a marker you place on a snapshot to track its replication status. When you
checkpoint a snapshot, the system monitors whether that snapshot (and all previous snapshots)
have been successfully synchronized to the remote filesystem.
**Key Concepts:**
- **Automatic Status Tracking**: Checkpoints automatically detect if a snapshot has already been synced
- **Persistent**: Checkpoint information survives daemon restarts
- **Per-Directory**: Each mirrored directory has its own set of checkpoints
Checkpoint Lifecycle
--------------------
Every checkpoint goes through these states:
.. code-block:: text
┌─────────┐
│ CREATED │ ← Checkpoint added, snapshot not yet synced
└────┬────┘
│
├──→ ┌──────────┐
│ │ COMPLETE │ ← Snapshot successfully synced to remote
│ └──────────┘
│
└──→ ┌────────┐
│ FAILED │ ← Sync failed (will be retried automatically)
└────────┘
- **created**: Waiting for synchronization or sync in progress
- **complete**: Successfully replicated to remote site
- **failed**: Sync encountered an error (check error message for details)
Prerequisites
=============
Before using checkpoints, ensure:
1. **CephFS Mirroring is Enabled**:
.. code-block:: bash
ceph fs snapshot mirror enable <fs_name>
2. **Remote Peer is Configured**:
.. code-block:: bash
ceph fs snapshot mirror peer_add <fs_name> <peer_spec>
3. **Directory is Being Mirrored**:
.. code-block:: bash
ceph fs snapshot mirror add <fs_name> <dir_path>
4. **Snapshots Exist**: You need snapshots to checkpoint
For detailed mirroring setup, see :doc:`cephfs-mirroring`.
Getting Started
===============
Basic Workflow
--------------
Here's a typical workflow for using checkpoints:
**Step 1: Create Snapshots**
.. code-block:: bash
# Create snapshots in your mirrored directory
mkdir /mnt/cephfs/data/.snap/daily_backup_2026_06_01
mkdir /mnt/cephfs/data/.snap/daily_backup_2026_06_02
mkdir /mnt/cephfs/data/.snap/daily_backup_2026_06_03
**Step 2: Add Checkpoints**
.. code-block:: bash
# Mark important snapshots as checkpoints
ceph fs snapshot mirror checkpoint add myfs /data daily_backup_2026_06_01
ceph fs snapshot mirror checkpoint add myfs /data daily_backup_2026_06_03
**Step 3: Monitor Status**
.. code-block:: bash
# Check checkpoint status
ceph fs snapshot mirror checkpoint ls myfs /data --format json-pretty
**Example Output:**
.. code-block:: json
{
"dir_root": "/data",
"checkpoints": [
{
"snap_id": 100,
"snap_name": "daily_backup_2026_06_01",
"status": "complete",
"created_at": "2026-06-01T23:00:00.000000+0000",
"updated_at": "2026-06-01T23:15:00.000000+0000"
},
{
"snap_id": 102,
"snap_name": "daily_backup_2026_06_03",
"status": "created",
"created_at": "2026-06-03T23:00:00.000000+0000",
"updated_at": "2026-06-03T23:00:00.000000+0000"
}
]
}
Command Reference
=================
checkpoint add
--------------
Add a checkpoint to a specific snapshot.
**Syntax:**
.. code-block:: bash
ceph fs snapshot mirror checkpoint add <fs_name> <dir_path> <snap_name>
**Parameters:**
- ``fs_name``: Name of the filesystem
- ``dir_path``: Path to the mirrored directory (e.g., ``/data``)
- ``snap_name``: Name of the snapshot to checkpoint
**Example:**
.. code-block:: bash
ceph fs snapshot mirror checkpoint add myfs /data backup_snapshot_1
**Output:**
.. code-block:: json
{
"status": "success",
"message": "checkpoint added for snapshot backup_snapshot_1",
"dir_root": "/data",
"snap_id": 123,
"snap_name": "backup_snapshot_1",
"checkpoint_status": "created"
}
**Notes:**
- If the snapshot has already been synced, status will be ``complete`` immediately
- You cannot checkpoint the same snapshot twice
- The snapshot must exist in the directory
checkpoint now
--------------
Add a checkpoint to the most recent snapshot in a directory.
**Syntax:**
.. code-block:: bash
ceph fs snapshot mirror checkpoint now <fs_name> <dir_path>
**Example:**
.. code-block:: bash
ceph fs snapshot mirror checkpoint now myfs /data
**Output:**
.. code-block:: json
{
"status": "success",
"message": "checkpoint created on latest snapshot",
"dir_root": "/data",
"snap_id": 125,
"snap_name": "daily_backup_2026_06_03",
"checkpoint_status": "created"
}
**Use Case:**
Useful when you want to checkpoint the latest state without knowing the exact snapshot name.
checkpoint ls
-------------
List all checkpoints for a directory.
**Syntax:**
.. code-block:: bash
ceph fs snapshot mirror checkpoint ls <fs_name> <dir_path> [--format <format>]
**Parameters:**
- ``format``: Output format (``json`` or ``json-pretty``), default is ``json``
**Example:**
.. code-block:: bash
ceph fs snapshot mirror checkpoint ls myfs /data --format json-pretty
**Output Fields:**
- ``snap_id``: Internal snapshot ID
- ``snap_name``: Snapshot name
- ``status``: Current status (created/complete/failed)
- ``created_at``: When checkpoint was created
- ``updated_at``: Last status update time
- ``error_msg``: Error description (only present if status is failed)
checkpoint remove
-----------------
Remove a checkpoint from a specific snapshot.
**Syntax:**
.. code-block:: bash
ceph fs snapshot mirror checkpoint remove <fs_name> <dir_path> <snap_name>
**Example:**
.. code-block:: bash
# Remove checkpoint from a specific snapshot
ceph fs snapshot mirror checkpoint remove myfs /data backup_snapshot_1
**Output:**
.. code-block:: json
{
"status": "success",
"message": "checkpoint removed for snapshot backup_snapshot_1",
"dir_root": "/data",
"snap_name": "backup_snapshot_1"
}
**Notes:**
- Removing a checkpoint does NOT delete the snapshot itself
- The snapshot and its data remain intact
- This operation only removes the checkpoint tracking metadata
- This operation cannot be undone
- To remove multiple checkpoints, run the command multiple times
Important Notes
===============
Snapshot Deletion Behavior
---------------------------
.. warning::
**If a snapshot with a checkpoint is deleted, the checkpoint metadata is permanently lost.**
When you delete a snapshot that has a checkpoint:
1. **Before Sync Completes**: The checkpoint is lost and will never reach "complete" status
2. **After Sync Completes**: The checkpoint history is lost, but the data was already replicated
**Impact on Commands:**
- ``checkpoint ls``: Will not show the deleted snapshot's checkpoint
- ``checkpoint remove``: Will fail with "snapshot not found" error
**Best Practice:**
- Do not delete snapshots with active checkpoints until they reach "complete" status
- Use ``checkpoint ls`` to verify checkpoint status before deleting snapshots
- Consider removing checkpoints explicitly before deleting important snapshots
Snapshot Rename Behavior
-------------------------
.. note::
**Checkpoints survive snapshot renames, but you must use the new snapshot name in commands.**
When you rename a snapshot that has a checkpoint:
1. **Checkpoint Persists**: The checkpoint metadata remains attached to the snapshot
2. **Name Updates Automatically**: ``checkpoint ls`` will show the new snapshot name
3. **Old Name No Longer Works**: You must use the new name for ``checkpoint remove``
**Example:**
.. code-block:: bash
# Original snapshot with checkpoint
ceph fs snapshot mirror checkpoint add myfs /data old_name
# Rename the snapshot (using CephFS snapshot rename)
# ... snapshot renamed from 'old_name' to 'new_name' ...
# List checkpoints - shows new name
ceph fs snapshot mirror checkpoint ls myfs /data
# Output shows: "snap_name": "new_name"
# Remove using NEW name (this works)
ceph fs snapshot mirror checkpoint remove myfs /data new_name
# Remove using OLD name (this fails)
ceph fs snapshot mirror checkpoint remove myfs /data old_name
# Error: snapshot 'old_name' not found
**Best Practice:**
- Always use ``checkpoint ls`` to verify current snapshot names before removing checkpoints
- Update any automation scripts if you rename snapshots
Additional Resources
====================
- **CephFS Mirroring Guide**: :doc:`cephfs-mirroring`
- **Snapshot Documentation**: :doc:`/cephfs/snapshots`
- **Tracker Issue**: https://tracker.ceph.com/issues/73454

1018
doc/cephfs/cephfs-mirroring.rst
View File

File diff suppressed because it is too large Load Diff

4
doc/cephfs/experimental-features.rst
View File

@ -24,8 +24,8 @@ failures within it are unlikely to make non-inlined data inaccessible.
Inline data has always been off by default and requires setting
the ``inline_data`` flag.
Inline data has been declared deprecated for the Octopus release, and will
likely be removed altogether in a future release.
Inline data has been deprecated since the Octopus release. Enabling it triggers
a health warning, and the feature will be removed altogether in a future release.
Mantle: Programmable Metadata Load Balancer
-------------------------------------------

1
doc/cephfs/index.rst
View File

@ -93,6 +93,7 @@ Administration
cephfs-tool <cephfs-tool>
Scheduled Snapshots <snap-schedule>
CephFS Snapshot Mirroring <cephfs-mirroring>
CephFS Snapshot Mirroring Checkpoints <cephfs-mirroring-checkpoints>
Purge Queue <purge-queue>
.. raw:: html

6
doc/cephfs/kernel-features.rst
View File

@ -8,9 +8,9 @@ in the kernel driver.
Inline data
-----------
Inline data was introduced by the Firefly release. This feature is being
deprecated in mainline CephFS, and may be removed from a future kernel
release.
Inline data was introduced by the Firefly release. This feature has been
deprecated since the Octopus release: enabling it triggers a health warning,
and it will be removed in a future release.
Linux kernel clients >= 3.19 can read inline data and convert existing
inline data to RADOS objects when file data is modified. At present,

2
doc/cephfs/lazyio.rst
View File

@ -14,7 +14,7 @@ Enable LazyIO
LazyIO can be enabled in the following ways.
- ``client_force_lazyio`` option enables LAZY_IO globally for libcephfs and
ceph-fuse mount.
ceph-fuse mount. This is strictly a startup flag.
- ``ceph_lazyio(...)`` and ``ceph_ll_lazyio(...)`` enable LAZY_IO for file handle
in libcephfs.

34
doc/cephfs/upgrading.rst
View File

@ -54,37 +54,3 @@ command. Older versions of Ceph require you to stop these daemons manually.
ceph fs set <fs_name> max_mds <old_max_mds>
ceph fs set <fs_name> allow_standby_replay <old_allow_standby_replay>
Upgrading pre-Firefly file systems past Jewel
=============================================
.. tip::
This advice only applies to users with file systems
created using versions of Ceph older than *Firefly* (0.80).
Users creating new file systems may disregard this advice.
Pre-firefly versions of Ceph used a now-deprecated format
for storing CephFS directory objects, called TMAPs. Support
for reading these in RADOS will be removed after the Jewel
release of Ceph, so for upgrading CephFS users it is important
to ensure that any old directory objects have been converted.
After installing Jewel on all your MDS and OSD servers, and restarting
the services, run the following command:
::
cephfs-data-scan tmap_upgrade <metadata pool name>
This only needs to be run once, and it is not necessary to
stop any other services while it runs. The command may take some
time to execute, as it iterates over all objects in your metadata
pool. It is safe to continue using your file system as normal while
it executes. If the command aborts for any reason, it is safe
to simply run it again.
If you are upgrading a pre-Firefly CephFS file system to a newer Ceph version
than Jewel, you must first upgrade to Jewel and run the ``tmap_upgrade``
command before completing your upgrade to the latest version.

3
doc/conf.py
View File

@ -165,9 +165,6 @@ else:
'engine': 'ditaa'
}
if build_with_rtd:
extensions += ['sphinx_search.extension']
# sphinx.ext.todo options
todo_include_todos = True

58
doc/dev/cephadm/cephadm-invoker.rst Normal file
View File

@ -0,0 +1,58 @@
Cephadm Invoker
===============
The ``cephadm_invoker.py`` script provides a wrapper intended for executing
cephadm commands with limited sudo priviliges. It is used when sudo hardening
is enabled.
Overview
--------
The cephadm invoker validates the cephadm binary hash before execution and
provides a secure way to run cephadm commands and deploy binaries. It is installed as
part of the cephadm RPM at ``/usr/libexec/cephadm_invoker.py``.
Commands
--------
The invoker supports the following subcommands:
``run``
Execute cephadm binary with arguments after hash verification::
cephadm_invoker.py run <binary> [args...]
The binary path must include a hash in the filename for verification.
``deploy_binary``
Deploy cephadm binary from a temporary file to the final location::
cephadm_invoker.py deploy_binary <temp_file> <final_path>
``check_binary``
Check if a cephadm binary exists::
cephadm_invoker.py check_binary <cephadm_binary_path>
Returns exit code 0 if the file exists, 2 if it does not exist.
Exit Codes
----------
- ``0``: Success
- ``1``: General error (file not found, permission issues, etc.)
- ``2``: Binary hash mismatch or file doesn't exist (triggers redeployment)
- ``126``: Permission denied during execution
Security Features
-----------------
The cephadm invoker provides the following security features:
- **Binary Hash Verification**: Validates the cephadm binary integrity before execution
- **Restricted Execution**: Only allows execution of verified cephadm binaries
- **Secure Deployment**: Safely deploys cephadm binaries with proper permissions
- **Logging**: Comprehensive logging to both console and syslog for audit trails

1
doc/dev/cephadm/index.rst
View File

@ -11,5 +11,6 @@ CEPHADM Developer Documentation
developing-cephadm
host-maintenance
compliance-check
cephadm-invoker
Storage devices and OSDs management <./design/storage_devices_and_osds>
scalability-notes

170
doc/dev/cephfs-mirroring.rst
View File

@ -326,8 +326,48 @@ For example:
]
One entry per mirror-daemon instance is displayed, along with information
including configured peers and basic statistics. For more detailed statistics,
use the admin socket interface as detailed below.
including configured peers and basic statistics.
**Directory snapshot sync metrics (mgr)**
The mirroring module implements ``ceph fs snapshot mirror status``. The
``cephfs-mirror`` daemon persists per-directory sync statistics to the
``cephfs_mirror`` object omap in the metadata pool so the Manager can expose
them through the Ceph CLI without an admin socket on a mirror daemon host. The
``metrics_status`` handler reads that omap, applies stale detection and default
idle metrics for newly mirrored directories, and returns JSON in the same nested
``metrics/<dir>/peer/<uuid>`` format as ``fs mirror peer status``. When
``snapshot_mirror_metrics_cache_enabled`` is true (default TTL 15 seconds via
``snapshot_mirror_metrics_cache_ttl``), responses are served from a complete
cache (full file system snapshot) and, for single-directory queries, a partial
per-directory cache that avoids full omap scans on complete-cache miss. When
caching is disabled, each query reads omap directly. Omap entries are marked
``stale`` when the persisted ``_instance_id`` is not among live mirror instances
or does not match the directory's tracked instance while persisted state is not
``idle``.
Each omap value includes metadata fields written by ``cephfs-mirror``:
- ``_instance_id`` — RADOS client instance of the writer; used for stale
detection and stripped from CLI output.
- ``metrics_updated_at`` — wall-clock time of the last omap write. Exposed in
``ceph fs snapshot mirror status`` output only; omitted from admin socket
``fs mirror peer status``, which reads live in-memory state and does not
need a separate persist timestamp. Operators use this field to judge how
fresh omap-backed metrics are, given ``cephfs_mirror_tick_interval`` omap
persist cadence and Manager caching (``snapshot_mirror_metrics_cache_enabled``
and ``snapshot_mirror_metrics_cache_ttl``).
Omap entries are removed when a directory is removed from mirroring. All metric
fields are written to omap; on daemon restart only ``last_synced_snap`` metadata
is loaded back. Per-session counters (``snaps_synced``, ``snaps_deleted``,
``snaps_renamed``) are persisted but not loaded and therefore start at zero each
session.
See :ref:`Directory snapshot sync metrics<cephfs_mirroring_mgr_snapshot_status>`
and :ref:`Snapshot sync metric fields<cephfs_mirroring_sync_metric_fields>` in
:doc:`/cephfs/cephfs-mirroring` for command syntax, examples, and operator
guidance.
CephFS mirror daemons provide admin socket commands for querying mirror status.
To list the available commands for ``mirror status``, run the following
@ -393,20 +433,35 @@ status. Commands of this kind take the form ``filesystem-name@filesystem-id peer
::
{
"/d0": {
"state": "idle",
"last_synced_snap": {
"id": 120,
"name": "snap1",
"sync_duration": 0.079997898999999997,
"sync_time_stamp": "274900.558797s"
},
"snaps_synced": 2,
"snaps_deleted": 0,
"snaps_renamed": 0
"metrics": {
"/d0": {
"peer": {
"a2dc7784-e7a1-4723-b103-03ee8d8768f8": {
"state": "idle",
"last_synced_snap": {
"id": 120,
"name": "snap1",
"crawl_duration": "2s",
"datasync_queue_wait_duration": "1s",
"sync_duration": "33s",
"sync_time_stamp": "274900.558797s",
"sync_bytes": "149.94 MiB",
"sync_files": 5000
},
"snaps_synced": 2,
"snaps_deleted": 0,
"snaps_renamed": 0
}
}
}
}
}
Several fields in the status output are formatted for readability rather than reported as raw
numbers. See :ref:`Value formatting <cephfs_mirror_peer_status_formatting>` in
:doc:`/cephfs/cephfs-mirroring` for duration, data size, throughput, percentage, count, and
timestamp formats.
Synchronization stats such as ``snaps_synced``, ``snaps_deleted`` and
``snaps_renamed`` are reset when the daemon is restarted or (when multiple
mirror daemons are deployed), when a directory is reassigned to another mirror
@ -419,6 +474,49 @@ A directory can be in one of the following states::
- `syncing`: The directory is currently being synchronized
- `failed`: The directory has hit upper limit of consecutive failures
::
{
"metrics": {
"/d0": {
"peer": {
"a2dc7784-e7a1-4723-b103-03ee8d8768f8": {
"state": "syncing",
"current_syncing_snap": {
"id": 121,
"name": "snap2",
"sync-mode": "full",
"avg_read_throughput_bytes": "13.03 MiB/s",
"avg_write_throughput_bytes": "24.24 MiB/s",
"crawl": {
"state": "completed",
"duration": "2s"
},
"datasync_queue_wait": {
"state": "complete",
"duration": "1s"
},
"bytes": {
"sync_bytes": "60.40 MiB",
"total_bytes": "149.94 MiB",
"sync_percent": "40.29%"
},
"files": {
"sync_files": 2013,
"total_files": 5000,
"sync_percent": "40.26%"
},
"eta": "7s"
},
"snaps_synced": 2,
"snaps_deleted": 0,
"snaps_renamed": 0
}
}
}
}
}
When a directory hits a configured number of consecutive synchronization
failures, the mirror daemon marks it as ``failed``. Synchronization for these
directories is retried. By default, the number of consecutive failures before a
@ -439,23 +537,37 @@ status:
::
{
"/d0": {
"state": "idle",
"last_synced_snap": {
"id": 120,
"name": "snap1",
"sync_duration": 0.079997898999999997,
"sync_time_stamp": "274900.558797s"
"metrics": {
"/d0": {
"peer": {
"a2dc7784-e7a1-4723-b103-03ee8d8768f8": {
"state": "idle",
"last_synced_snap": {
"id": 120,
"name": "snap1",
"crawl_duration": "2s",
"datasync_queue_wait_duration": "1s",
"sync_duration": "33s",
"sync_time_stamp": "274900.558797s",
"sync_bytes": "149.94 MiB",
"sync_files": 5000
},
"snaps_synced": 2,
"snaps_deleted": 0,
"snaps_renamed": 0
}
}
},
"snaps_synced": 2,
"snaps_deleted": 0,
"snaps_renamed": 0
},
"/f0": {
"state": "failed",
"snaps_synced": 0,
"snaps_deleted": 0,
"snaps_renamed": 0
"/f0": {
"peer": {
"a2dc7784-e7a1-4723-b103-03ee8d8768f8": {
"state": "failed",
"snaps_synced": 0,
"snaps_deleted": 0,
"snaps_renamed": 0
}
}
}
}
}

27
doc/dev/continuous-integration.rst
View File

@ -94,6 +94,33 @@ Shaman
for its `Web UI`_. But please note, shaman does not build the
packages, it just offers information on the builds.
ceph-perf-pull-requests
-----------------------
``ceph-perf-pull-requests`` runs CBT performance regression checks on dedicated
``performance`` Jenkins agents. The job definition lives in `ceph-build`_ and
generates two jobs from a single template: ``ceph-perf-classic`` and
``ceph-perf-crimson``.
A pull request can trigger a run with::
jenkins test classic perf
jenkins test crimson perf
The ``performance`` label is also whitelisted for automatic runs. Each job:
#. checks out ``ceph-main`` (``origin/main``) and the PR merge ref
#. builds both trees (classic ``vstart-base`` or Crimson ``crimson-osd``)
#. runs the checked-in ``radosbench_4K_read.yaml`` workload via ``run-cbt.sh``
#. compares PR results against ``main`` with ``cbt/compare.py``
#. publishes a GitHub check named ``perf-test-{classic,crimson}``
The benchmark YAML is always taken from ``ceph-main`` so PR and baseline runs use
the same workload definition. Teuthology-to-CBT translation is handled by
``src/test/crimson/cbt/t2c.py`` in the Ceph tree (not patched at build time).
.. _ceph-build: https://github.com/ceph/ceph-build
As the following shows, `chacra`_ manages multiple projects whose metadata
are stored in a database. These metadata are exposed via Shaman as a web
service. `chacractl`_ is a utility to interact with the `chacra`_ service.

14
doc/dev/crimson/index.rst
View File

@ -253,7 +253,17 @@ In order to use ``fio`` to test ``crimson-store-nbd``, perform the below steps.
CBT
---
We can use `cbt`_ for performance tests::
We can use `cbt`_ for performance tests. Benchmark workloads are checked in under
``src/test/crimson/cbt/`` as teuthology-style YAML files. Before ``run-cbt.sh``
invokes CBT, ``t2c.py`` translates the teuthology ``tasks`` list into a
CBT-ready configuration: it extracts the ``cbt`` task, fills in cluster paths
(``ceph.conf``, ``ceph``/``rados`` binaries, PID directory), and writes the
result to a temporary YAML file consumed by ``cbt.py``.
Unit tests for the translator live in ``src/test/crimson/cbt/test_t2c.py`` and
are registered with ``make check`` via ``add_ceph_test``.
::
$ git checkout main
$ make crimson-osd
@ -410,3 +420,5 @@ Code Walkthroughs
BackfillMachine <backfillmachine>
SeaStore <seastore>
PoseidonStore <poseidonstore>
PG Merge Synchronization <pgmerging>
The Logical Address in SeaStore <seastore_laddr>

106
doc/dev/crimson/pgmerging.rst Normal file
View File

@ -0,0 +1,106 @@
===========================================
PG Merge Synchronization in Crimson
===========================================
This document describes the infrastructure used to coordinate the merging
of Placement Groups (PGs) in Crimson.
Background
----------
Crimson utilizes a shared-nothing memory model where each PG is owned by
a specific CPU core. Merging requires cross-core coordination while
maintaining memory safety and epoch consistency.
Core Concepts
-------------
.. _migration_safety:
Migration Safety (Memory Ownership)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
PGs are managed by their ``birth_shard``. To prevent cross-shard
deallocation crashes:
1. **Extraction**: ``ShardServices::extract_pg()`` removes the source PG
from its shard map so it stops receiving messages.
2. **Transportation**: The PG crosses cores in a ``seastar::foreign_ptr``.
3. **Storage**: The target shard stores each source in
``crimson::local_shared_foreign_ptr<Ref<PG>>`` inside the target PG's
rendezvous map. When the target drops the last reference after
``PG::merge_from()``, destruction is routed back to the source
``birth_shard``.
.. _synchronization:
Target and Source PG Synchronization
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Merging involves a race between source PGs checking in and the target PG
reaching the merge point in ``PGAdvanceMap``.
Synchronization state lives on the **target PG**, not in a per-shard
registry:
* **``merge_rendezvous_t``**: A map of arrived sources plus a
``seastar::semaphore`` counting first-time registrations.
* **``PG::add_merge_source()``**: Called on the target's shard when
``ShardServices::register_merge_source()`` delivers a source PG.
Duplicate source pgids are ignored (idempotent under replay).
* **``PG::collect_merge_sources(n)``**: Waits on the rendezvous semaphore
for ``n`` arrivals (one signal per first insert), asserts the map has
``n`` entries, then returns sources and clears the rendezvous. If
``reset_merge_rendezvous()`` breaks the wait, returns an empty map.
* **``PG::reset_merge_rendezvous()``**: Clears in-flight handoffs and
resets the semaphore. Called on ``PG::stop()`` or after a Seastore
cross-shard abort so a failed try cannot leave stale sources for a
later epoch.
**Cross-shard handoff**: ``register_merge_source()`` resolves the target
shard, extracts the source locally, and uses ``invoke_on`` to call
``add_merge_source()`` on the target PG.
.. _map_consistency:
Map Advancement and Pipeline Stalling
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
To ensure the merge occurs between PGs at the same logical time:
* ``PGAdvanceMap::check_for_merges()`` detects ``pg_num`` shrink between
map epochs and returns a ``merge_result_t`` (source, target, or none).
* ``merge_pg()`` performs merge-specific work only (Seastore eligibility,
rendezvous collection, ``PG::merge_from()`` on the target).
* A single ``PGAdvanceMap`` operation may batch multiple map epochs
(``from`` through ``to``). Only a **merge source** stops the epoch
loop early: ``finish_merge_source()`` commits ``rctx``, stops the PG,
and calls ``register_merge_source()``. A **merge target** runs
``merge_from()`` at the merge epoch, then continues advancing through
the remaining epochs up to ``to`` (matching classic OSD behavior).
Normal completion at the end of ``start()`` activates the final map
and commits ``rctx`` (also covering a completed merge target).
* The target does not call ``merge_from()`` until sources frozen at the
merge epoch have arrived on its rendezvous.
.. _seastore_cross_shard:
Seastore Cross-Shard Merges
~~~~~~~~~~~~~~~~~~~~~~~~~~~
Seastore does not support merging collections across reactor shards.
Before a source hands off or a target collects sources, ``merge_pg()``
calls ``ShardServices::seastore_merge_shards_ok()`` with the full sibling
set. If the target and any source map to different shards, the OSD aborts
the attempt: clears ready-to-merge state, sends
``MOSDPGReadyToMerge{ ready=false }`` for the source so the monitor does
not commit the unsafe ``pg_num`` decrement, resets the target rendezvous,
and sends ``MOSDPGStopMerge`` to the monitor (once per pool per OSD).
The monitor clamps ``pg_num_target`` to ``pg_num`` and clears
``FLAG_CRIMSON_ALLOW_PG_MERGE`` for that pool until operators re-enable
merging (``crimson_allow_pg_merge``).
Cleanup
-------
After ``complete_rctx()``, the target releases its ``local_shared_foreign_ptr``
references; source PG objects are destroyed on their ``birth_shard``.
When the OSD finishes consuming a new OSDMap in ``OSD::committed_osd_maps()``,
``OSDSingletonState::prune_sent_ready_to_merge()`` drops ``sent_ready_to_merge_source``
entries (and stale ``pools_merge_stopped_reported`` pool ids) for PGs that no
longer exist in the committed map, mirroring classic ``OSD::consume_map()``.

3
doc/dev/crimson/seastore_laddr.rst
View File

@ -201,7 +201,8 @@ after the upgrade finishes.
TODO: fsck/tooling must support scanning and migrating all mappings
between upgrades.
//create tracker after PR merged/ready.
//create tracker after PR merged/ready.
Multi-push Recovery
-------------------

25
doc/dev/developer_guide/tests-unit-tests.rst
View File

@ -71,6 +71,31 @@ teuthology using the `cram task`_.
.. _`cram`: https://bitheap.org/cram/
.. _`cram task`: https://github.com/ceph/ceph/blob/master/qa/tasks/cram.py
Running CLI tools tests
-----------------------
A convenience script is provided to run CLI tests easily without having to manually set up the environment.
From the Ceph source tree root directory:
.. prompt:: bash $
# Run all CLI tests
./src/script/run-cli-tests.sh
# Run with custom build directory
./src/script/run-cli-tests.sh -b /path/to/build
Alternatively, you can run the tests directly from the build directory by setting up your PATH:
.. prompt:: bash $
cd build
PATH="$PWD/bin:$PATH" ../src/test/run-cli-tests
The test suite includes tests for: ``ceph-authtool``, ``ceph-conf``,
``ceph-kvstore-tool``, ``crushtool``, ``monmaptool``, ``osdmaptool``,
``radosgw-admin``, and ``rbd``.
Tox-based testing of Python modules
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Some of the Python modules in Ceph use `tox <https://tox.readthedocs.io/en/latest/>`_

0
doc/nvmeof/ha.md → doc/dev/nvmeof-ha-design.md
View File

4
doc/dev/osd_internals/erasure_coding/client_support.rst
View File

@ -466,7 +466,7 @@ These operations provide the same semantics as in replicated pools, ensuring
compatibility with existing applications.
Read Operation Flow
~~~~~~~~~~~~~~
~~~~~~~~~~~~~~~~~~~
Read operations follow a simple flow:
@ -906,7 +906,7 @@ Required Settings
To enable the new features, the following OSDMap pool settings are required:
- ``allows_ec_overwrites = true`
- ``allows_ec_overwrites = true``
- ``allows_ec_optimizations = true``
- ``supports_omap = true``

59
doc/dev/release-process.rst
View File

@ -87,8 +87,8 @@ Once QE has determined a stopping point in the working (e.g., ``squid``) branch,
Notify the "Build Lead" that the release branch is ready.
2. Starting the build
=====================
2a. Starting the build
======================
We'll use a stable/regular 19.2.2 release of Squid as an example throughout this document.
@ -104,25 +104,52 @@ We'll use a stable/regular 19.2.2 release of Squid as an example throughout this
NOTE: if for some reason the build has to be restarted (for example if one distro failed) then the ``TAG`` option has to be unchecked.
4. Use https://docs.ceph.com/en/latest/start/os-recommendations/?highlight=debian#platforms to determine the ``DISTROS`` parameter. For example,
4. Use :ref:`start-platforms` to determine the ``DISTROS`` parameter. For example,
+-------------------+--------------------------------------------------+
| Release | Distro Codemap |
+===================+==================================================+
| pacific (16.X.X) | ``focal bionic buster bullseye`` |
+-------------------+--------------------------------------------------+
| quincy (17.X.X) | ``jammy focal centos9 bullseye`` |
+-------------------+--------------------------------------------------+
| reef (18.X.X) | ``jammy focal centos9 windows bookworm`` |
+-------------------+--------------------------------------------------+
| squid (19.X.X) | ``jammy centos9 windows bookworm`` |
+-------------------+--------------------------------------------------+
| tentacle (20.X.X) | ``jammy centos9 noble windows bookworm rocky10`` |
+-------------------+--------------------------------------------------+
+-------------------+---------------------------------------------------------+
| Release | Distro Codemap |
+===================+=========================================================+
| squid (19.X.X) | ``jammy centos9 windows bookworm`` |
+-------------------+---------------------------------------------------------+
| tentacle (20.X.X) | ``jammy centos9 rocky10 windows bookworm`` |
+-------------------+---------------------------------------------------------+
| umbrella (21.X.X) | ``jammy noble centos9 rocky10 bookworm trixie windows`` |
+-------------------+---------------------------------------------------------+
5. Click ``Build``.
2b. What to do if your build fails
==================================
The ceph-release-pipeline parent job has three stages.
If your build fails during the "create ceph release tag" stage, troubleshoot the child ceph-tag job and re-run the parent ceph-release-pipeline job with the same parameters.
----
If your build fails during the "package build" stage, troubleshoot the child ceph-dev-pipeline job. If only one variant failed to build (e.g., centos9 arm64), start a new ceph-release-pipeline job specifying::
DISTROS=centos9
ARCHS=arm64
TAG=false <-- VERY IMPORTANT
This will leave the version commit and previously-created tag intact in ceph-releases.git. You will want the subsequent ceph-dev-pipeline job to reuse that SHA/tag.
Once all of your variants are successfully built, you will have to manually run the ceph-tag job. For example,::
BRANCH=tentacle
TAG=true
TAG_PHASE=push
VERSION=20.2.0
RELEASE_TYPE=STABLE
Then proceed with the normal release process.
----
If your build fails during the "push ceph release tag" stage, troubleshoot the child ceph-tag job and re-run **just** the ceph-tag job again manually. Do not re-run ceph-release-pipeline.
3. Release Notes
================

65
doc/foundation.rst
View File

@ -20,59 +20,11 @@ cost.
For more information, see `https://ceph.com/foundation
<https://ceph.com/foundation>`_.
Members
=======
Diamond
-------
Visit `https://ceph.io/en/foundation/members/ <https://ceph.io/en/foundation/members/>`_ to see the full list of Ceph Foundation Members
* `45Drives <https://45drives.com/>`_
* `Bloomberg <https://bloomberg.com>`_
* `IBM <https://ibm.com>`_
* `Clyso <https://www.clyso.com/en/>`_
Platinum
--------
* `Western Digital <https://westerndigital.com/>`_
Gold
----
* `42on <https://www.42on.com/>`_
* `OVH <https://www.ovh.com/>`_
* `Samsung Electronics <https://samsung.com/>`_
Silver
------
* `Canonical <https://www.canonical.com/>`_
* `CloudFerro <https://cloudferro.com/>`_
* `croit <http://www.croit.io/>`_
* `Digital Ocean <http://digitalocean.com/>`_
* `ISS <http://iss-integration.com/>`_
* `Intel <http://www.intel.com/>`_
* `OSNexus <https://osnexus.com/>`_
* `Sony Interactive <https://sonyinteractive.com/>`_
Associate
---------
* `Boston University <http://www.bu.com/>`_
* `Center for Research in Open Source Systems (CROSS) <http://cross.ucsc.edu/>`_
* `CERN <https://home.cern/>`_
* `FASRC <https://www.rc.fas.harvard.edu/>`_
* `grnet <https://grnet.gr/>`_
* `Monash University <http://www.monash.edu/>`_
* `NRF SARAO <http://www.ska.ac.za/about/sarao/>`_
* `Open Infrastructure Foundation <http://openinfra.dev>`_
* `Science & Technology Facilities Councel (STFC) <https://stfc.ukri.org/>`_
* `SWITCH <https://switch.ch/>`_
* `University of Michigan <http://www.osris.org/>`_
Governing Board
===============
@ -97,21 +49,6 @@ engineering activities are managed through traditional open source
processes and are overseen by the :ref:`csc`. For more
information see :ref:`governance`.
Members
-------
* Brett Kelly (45Drives)
* Enrico Bocchi (CERN) - Associate member representative
* Dan van der Ster (Clyso) - Ceph Council representative
* Joachim Kraftmayer (Clyso)
* Josh Durgin (IBM) - Ceph Council representative
* Matthew Leonard (Bloomberg)
* Anthony Middleton (Linux Foundation) - Ceph Community Manager
* Neha Ojha (IBM) - Ceph Council Representative
* Steven Umbehocker (OSNexus) - General member representative
* Pawel Sadowski (OVH)
* Sungmin Lee (Samsung Electronics)
* Vincent Hsu (IBM)
Joining
=======

2
doc/governance.rst
View File

@ -99,7 +99,7 @@ Current Members
* Joseph Mundackal <jmundackal@bloomberg.net>
* Josh Durgin <jdurgin@redhat.com>
* João Eduardo Luis <joao@clyso.com>
* Laura Flores <lflores@redhat.com>
* Laura Flores <lflores@ibm.com>
* Mark Nelson <mark.nelson@clyso.com>
* Matan Breizman <mbreizma@redhat.com>
* Matt Benjamin <mbenjami@redhat.com>

8
doc/hardware-monitoring/index.rst
View File

@ -88,7 +88,8 @@ Supported categories are:
* network
* power
* fans
* firmwares
* temperatures
* firmware
* criticals
@ -143,7 +144,7 @@ ________________
.. prompt:: bash # auto
# ceph orch hardware status node-10 --category firmwares
# ceph orch hardware status node-10 --category firmware
+------------+----------------------------------------------------------------------------+--------------------------------------------------------------+----------------------+-------------+--------+
| HOST | COMPONENT | NAME | DATE | VERSION | STATUS |
+------------+----------------------------------------------------------------------------+--------------------------------------------------------------+----------------------+-------------+--------+
@ -187,6 +188,7 @@ For Developers
.. automethod:: NodeProxyEndpoint.power
.. automethod:: NodeProxyEndpoint.processors
.. automethod:: NodeProxyEndpoint.fans
.. automethod:: NodeProxyEndpoint.firmwares
.. automethod:: NodeProxyEndpoint.temperatures
.. automethod:: NodeProxyEndpoint.firmware
.. automethod:: NodeProxyEndpoint.led

10
doc/install/index.rst
View File

@ -24,7 +24,6 @@ in Kubernetes, while also enabling management of storage resources and
provisioning via Kubernetes APIs. We recommend Rook as the way to run Ceph in
Kubernetes or to connect an existing Ceph storage cluster to Kubernetes.
* Rook supports only Nautilus and newer releases of Ceph.
* Rook is the preferred method for running Ceph on Kubernetes, or for
connecting a Kubernetes cluster to an existing (external) Ceph
cluster.
@ -34,15 +33,6 @@ Kubernetes or to connect an existing Ceph storage cluster to Kubernetes.
Other methods
~~~~~~~~~~~~~
`ceph-ansible <https://docs.ceph.com/ceph-ansible/>`_ deploys and manages
Ceph clusters using Ansible.
* ceph-ansible is widely deployed.
* ceph-ansible is not integrated with the orchestrator APIs that were
introduced in Nautilus and Octopus, which means that the management features
and dashboard integration introduced in Nautilus and Octopus are not
available in Ceph clusters deployed by means of ceph-ansible.
`ceph-salt <https://github.com/ceph/ceph-salt>`_ installs Ceph using Salt and cephadm.
`jaas.ai/ceph-mon <https://jaas.ai/ceph-mon>`_ installs Ceph using Juju.

17
doc/install/mirrors.rst
View File

@ -24,17 +24,18 @@ These mirrors are available on the following locations:
- **US-West: US West Coast**: http://us-west.ceph.com/
- **CN: China**: http://mirrors.ustc.edu.cn/ceph/
You can replace all download.ceph.com URLs with any of the mirrors, for example:
You can replace all ``download.ceph.com`` URLs with any of the mirrors,
for example:
- http://download.ceph.com/tarballs/
- http://download.ceph.com/debian-hammer/
- http://download.ceph.com/rpm-hammer/
- https://download.ceph.com/tarballs/
- https://download.ceph.com/debian-tentacle/
- https://download.ceph.com/rpm-tentacle/
Change this to:
- http://eu.ceph.com/tarballs/
- http://eu.ceph.com/debian-hammer/
- http://eu.ceph.com/rpm-hammer/
- https://eu.ceph.com/tarballs/
- https://eu.ceph.com/debian-tentacle/
- https://eu.ceph.com/rpm-tentacle/
Mirroring
@ -62,4 +63,4 @@ set for all mirrors. These can be found on `GitHub`_.
If you want to apply for an official mirror, please contact the ceph-users mailinglist.
.. _GitHub: https://github.com/ceph/ceph/tree/master/mirroring
.. _GitHub: https://github.com/ceph/ceph/tree/main/mirroring

4
doc/man/8/ceph-bluestore-tool.rst
View File

@ -20,6 +20,7 @@ Synopsis
| **ceph-bluestore-tool** qfsck --path *osd path*
| **ceph-bluestore-tool** allocmap --path *osd path*
| **ceph-bluestore-tool** restore_cfb --path *osd path*
| **ceph-bluestore-tool** recovery-compare --path *osd path*
| **ceph-bluestore-tool** show-label --dev *device* ...
| **ceph-bluestore-tool** show-label-at --dev *device* --offset *lba* ...
| **ceph-bluestore-tool** prime-osd-dir --dev *device* --path *osd path*
@ -72,6 +73,9 @@ Commands
Reverses changes done by the new NCB code (either through ceph restart or when running allocmap command) and restores RocksDB B Column-Family (allocator-map).
:command:`recovery-compare`
Runs legacy onode recovery and multithread onode recovery. Prints timings and compares results.
:command:`bluefs-export`

84
doc/man/8/cephadm.rst
View File

@ -13,8 +13,8 @@ Synopsis
| [--log-dir LOG_DIR] [--logrotate-dir LOGROTATE_DIR]
| [--unit-dir UNIT_DIR] [--verbose] [--timeout TIMEOUT]
| [--retry RETRY] [--no-container-init]
| {version,pull,inspect-image,ls,list-networks,list-rdma,adopt,rm-daemon,rm-cluster,run,shell,enter,ceph-volume,unit,logs,bootstrap,deploy,check-host,prepare-host,add-repo,rm-repo,install,list-images,update-osd-service}
| ...
| {version,pull,inspect-image,ls,list-networks,list-rdma,adopt,rm-daemon,rm-cluster,remove-file,deploy-file,run,shell,enter,ceph-volume,unit,logs,bootstrap,deploy,check-host,prepare-host,prepare-host-sudo-hardening,setup-ssh-user,add-repo,rm-repo,install,list-images,update-osd-service}
| ...
| **cephadm** **pull**
@ -90,8 +90,19 @@ Synopsis
| **cephadm** **check-host** [-h] [--expect-hostname EXPECT_HOSTNAME]
| **cephadm** **remove-file** [-h] [--fsid FSID] --path PATH
| **cephadm** **deploy-file** [-h] [--fsid FSID] --path PATH [--mode MODE]
| [--uid UID] [--gid GID]
| **cephadm** **prepare-host**
| **cephadm** **prepare-host-sudo-hardening** [-h] [--ssh-user SSH_USER]
| [--ssh-pub-key SSH_PUB_KEY]
| [--cephadm-version VERSION]
| **cephadm** **setup-ssh-user** [-h] --ssh-user SSH_USER --ssh-pub-key SSH_PUB_KEY
| **cephadm** **add-repo** [-h] [--release RELEASE] [--version VERSION]
| [--dev DEV] [--dev-commit DEV_COMMIT]
| [--gpg-url GPG_URL] [--repo-url REPO_URL]
@ -289,6 +300,32 @@ Arguments:
* [--expect-hostname EXPECT_HOSTNAME] Check that hostname matches an expected value
remove-file
-----------
Remove a regular file on the local host. Missing paths are ignored.
Arguments:
* [--fsid FSID] cluster FSID
* --path PATH absolute path of the file to remove (required)
deploy-file
-----------
Write or replace a file on the local host. The **entire file body** is read from
**standard input** as raw bytes (no encoding or line-ending translation).
Arguments:
* [--fsid FSID] cluster FSID
* --path PATH absolute destination path for the file (required)
* [--mode MODE] octal file mode (for example ``644`` or ``0644``)
* [--uid UID] numeric owner user id (**must** be given together with ``--gid``)
* [--gid GID] numeric owner group id (**must** be given together with ``--uid``)
deploy
------
@ -420,6 +457,49 @@ Arguments:
* [--expect-hostname EXPECT_HOSTNAME] Set hostname
prepare-host-sudo-hardening
---------------------------
Prepare a host for sudo hardening by authorizing SSH keys, installing/upgrading
the cephadm package, and setting up restricted sudoers permissions::
cephadm prepare-host-sudo-hardening --ssh-user cephadm --ssh-pub-key <key>
This command performs the following steps:
1. Authorizes the provided SSH public key for the specified user
2. Installs or upgrades the cephadm package to match the cluster version (includes cephadm_invoker.py)
3. Sets up sudoers with restricted permissions for cephadm_invoker.py
Arguments:
* [--ssh-user SSH_USER] SSH user for key authorization (default: root)
* [--ssh-pub-key SSH_PUB_KEY] SSH public key to authorize
* [--cephadm-version VERSION] Specific cephadm version to install
setup-ssh-user
--------------
Setup SSH user with passwordless sudo and SSH key authorization::
cephadm setup-ssh-user --ssh-user cephadm --ssh-pub-key <public_key>
This command configures an SSH user for cephadm operations by:
1. Validating that the user exists on the system
2. Setting up passwordless sudo for the user (skipped for root)
3. Authorizing the SSH public key for the user
This command is automatically called by ``ceph cephadm set-user`` to configure
SSH users across all cluster hosts.
Arguments:
* [--ssh-user SSH_USER] SSH user to setup (required)
* [--ssh-pub-key SSH_PUB_KEY] SSH public key to add to user's authorized_keys (required)
pull
----

26
doc/man/8/crushtool.rst
View File

@ -151,15 +151,35 @@ pools; it only runs simulations by mapping values in the range
Displays how many attempts were needed to find a device mapping.
For instance::
0: 95224
1: 3745
2: 2225
tries 0: 95224
tries 1: 3745
tries 2: 2225
..
shows that **95224** mappings succeeded without retries, **3745**
mappings succeeded with one attempts, etc. There are as many rows
as the value of the **--set-choose-total-tries** option.
.. option:: --show-retry-exhaustion
Checks whether any PG mappings hit the maximum retry limit
(typically 50 tries) and displays a warning if retry exhaustion
is detected. This is useful for validating CRUSH rules, especially
in stretch cluster configurations with exactly
2 datacenters and 4 replicas can potentially fail to map some PGs due to CRUSH's
pseudorandom selection repeatedly choosing the same datacenter.
Although, the probability is extremely low, we still want to detect this.
Outputs either a warning message if exhaustion is detected::
WARNING: Retry exhaustion detected!
5 PG(s) hit the maximum retry limit of 50
This indicates CRUSH failed to find optimal placement for some PGs.
or a success message showing the maximum tries actually needed::
No retry exhaustion detected (maximum tries needed: 7 / 50)
.. option:: --output-csv
Creates CSV files (in the current directory) containing information

4
doc/mgr/administrator.rst
View File

@ -161,9 +161,9 @@ The Manager automatically adjusts :confval:`mgr_stats_period` based on message q
depth to prevent overload during high cluster activity. This feature is enabled by
default and can be controlled with the following settings:
- :confval:`mgr_stats_period_autotune` (boolean, default: true): Enable or disable
- :confval:`mgr_stats_period_autotune` : Enable or disable
automatic tuning of the stats period.
- :confval:`mgr_stats_period_autotune_queue_threshold` (integer, default: 100):
- :confval:`mgr_stats_period_autotune_queue_threshold` :
The message queue depth threshold that triggers an increase in the stats period.
When the queue depth exceeds this threshold, the stats period is increased to

434
doc/mgr/ceph_secrets.rst Normal file
View File

@ -0,0 +1,434 @@
.. _mgr-ceph-secrets:
===================
Ceph Secrets Module
===================
The ``ceph_secrets`` manager module provides centralised secret storage for
Ceph operators and Ceph manager modules. Instead of embedding plaintext
credentials in service specifications or configuration objects, secrets are
stored once and referenced by URI. Ceph manager modules such as cephadm
and rook can store secret payloads centrally, reference them by URI, and
resolve them to plaintext only when needed at deploy time.
For example, a cephadm-managed service may need an API token. Instead of
embedding that token directly in the service specification, an operator can
store it once:
.. prompt:: bash $
ceph secret set cephadm/service/my-service/api_token -i /tmp/api-token
and then use the URI
``secret:/cephadm/service/my-service/api_token`` in the
service spec instead of the plaintext token. A cephadm integration can then
resolve the URI at deploy time and write the token only into the daemon files
that need it.
Secrets are stored in the Monitor KV store under the ``secret_store/v1/`` prefix
and are organised by namespace, scope, and name. Each secret is versioned and
carries ``created``/``updated`` timestamps. A per-namespace epoch counter
is incremented on every ``set`` and on any ``rm`` that actually removes a
secret, allowing consumers to detect changes without fetching the full secret
list.
.. note::
The ``mon`` backend stores secrets in the Monitor KV store, which is not an
external KMS or vault. Users and MGR modules with sufficient Ceph
permissions can still reveal or resolve stored secret values. Namespaces
provide logical and storage isolation, not an authorisation boundary.
If the module is not already enabled, run::
ceph mgr module enable ceph_secrets
.. contents::
:local:
:depth: 2
Concepts
========
Namespace
---------
A namespace is a logical storage boundary, typically matching the name of
the consuming MGR module (e.g. ``cephadm``, ``rook``). Secrets in different
namespaces are stored independently and have independent epoch counters.
Namespaces are not an authorisation boundary; any caller with access to the
``ceph_secrets`` module can read secrets from any namespace.
Scope
-----
Within a namespace, every secret has a scope that encodes what the secret
belongs to:
.. list-table::
:header-rows: 1
:widths: 15 40 45
* - Scope
- Meaning
- CLI path form
* - ``global``
- Cluster-wide secret, not tied to any specific target
- ``<namespace>/global/<name>``
* - ``service``
- Secret for a specific named service
- ``<namespace>/service/<service-name>/<name>``
* - ``host``
- Secret for a specific host
- ``<namespace>/host/<hostname>/<name>``
* - ``custom``
- Arbitrary slash-delimited path under the namespace
- ``<namespace>/custom/<path>``
Path grammar
------------
All path segments (namespace, scope target, name, and each segment of a
custom path) must match ``[A-Za-z0-9._-]+`` and must not end with ``'.'``.
Empty segments, percent-encoding, and leading/trailing whitespace are
rejected.
Custom scope paths may contain multiple slash-separated segments (e.g.
``cephadm/custom/app/db/password``). Two custom paths that share a common
prefix are independent secrets; ``cephadm/custom/a/b`` and
``cephadm/custom/a/b/c`` do not conflict.
Secret URIs
-----------
Internally, secrets are identified by URIs of the form::
secret:/<namespace>/global/<name>
secret:/<namespace>/service/<target>/<name>
secret:/<namespace>/host/<target>/<name>
secret:/<namespace>/custom/<path>
These URIs appear in the output of :ref:`scan_refs <mgr-ceph-secrets-scan>`
and may be embedded in configuration objects as opaque references to be
resolved at deploy time.
CLI Reference
=============
All CLI commands use the path form described above. Responses are JSON by
default; pass ``--format yaml`` for YAML output.
.. _mgr-ceph-secrets-set:
secret set
----------
Create or update a secret. The input file content is stored as an opaque string::
ceph secret set <path> -i <file>
Secret data must not be empty. Other contents, including leading or trailing
whitespace and final newlines, are preserved exactly. Callers that need to
store structured data should encode it themselves, for example as JSON, and
decode it after retrieval or resolution.
If the secret already exists its data is replaced and its version
incremented, unless the existing secret policy marks it non-editable (set via
the Python API with ``editable=False``), in which case the update is rejected.
The ``created`` timestamp is set on the first write and is never changed
thereafter; ``updated`` is refreshed on every write.
Example:
.. prompt:: bash $
ceph secret set cephadm/service/my-service/api_token -i /tmp/api-token
{"metadata": {"version": 1, "created": "2025-06-01T12:00:00Z", "updated": "2025-06-01T12:00:00Z"}}
.. _mgr-ceph-secrets-get:
secret get
----------
Retrieve the metadata (and, optionally, the data) for a secret::
ceph secret get <path> [--reveal] [--format {json|yaml}]
Without ``--reveal``, the ``data`` field is omitted from the response to
avoid accidental exposure in terminal output or logs. With ``--reveal``,
``data`` contains the stored opaque string.
Example:
.. prompt:: bash $
ceph secret get cephadm/service/my-service/api_token
{
"metadata": {
"version": 1,
"created": "2025-06-01T12:00:00Z",
"updated": "2025-06-01T12:00:00Z"
}
}
ceph secret get cephadm/service/my-service/api_token --reveal
{
"metadata": {
"version": 1,
"created": "2025-06-01T12:00:00Z",
"updated": "2025-06-01T12:00:00Z"
},
"data": "api-token-value"
}
.. _mgr-ceph-secrets-get-value:
secret get-value
----------------
Return the raw secret data string directly, with no JSON envelope::
ceph secret get-value <path>
Unlike ``secret get --reveal``, the output is the stored string itself,
making it suitable for use in shell scripts and pipelines. Returns an
error if the secret does not exist.
Example:
.. prompt:: bash $
ceph secret get-value cephadm/service/my-service/api_token
api-token-value
.. _mgr-ceph-secrets-ls:
secret ls
---------
List secrets, optionally filtered by namespace, scope, and/or target::
ceph secret ls [--namespace <ns>] [--scope <scope>]
[--sec_target <target>] [--reveal] [--show_internals]
[--format {json|yaml}]
Without filters, all secrets across all namespaces are listed.
``--reveal`` includes the stored opaque data string in each output record.
``--show_internals`` additionally includes the ``policy`` object (``user_made``
and ``editable`` flags) in each record.
Example:
.. prompt:: bash $
ceph secret ls --namespace cephadm --scope host --sec_target node1
{
"cephadm/host/node1/ssh_key": {
"metadata": {
"version": 3,
"created": "2025-05-10T08:00:00Z",
"updated": "2025-06-01T09:00:00Z"
},
"ref": {
"namespace": "cephadm",
"scope": "host",
"target": "node1",
"name": "ssh_key"
}
}
}
.. _mgr-ceph-secrets-rm:
secret rm
---------
Remove a secret::
ceph secret rm <path>
The operation is idempotent: removing a secret that does not exist succeeds
and reports ``"status": "not_found"`` rather than an error.
Example:
.. prompt:: bash $
ceph secret rm cephadm/service/my-service/api_token
{"status": "removed"}
ceph secret rm cephadm/service/my-service/api_token
{"status": "not_found"}
.. _mgr-ceph-secrets-get-epoch:
Epoch
-----
The epoch for a namespace is accessible via the Python API only (see
:ref:`Epoch-based change detection <mgr-ceph-secrets-epoch>`). There is
no CLI command for it.
Module API
==========
Other MGR modules should consume ``ceph_secrets`` via ``CephSecretsClient``
(``src/pybind/mgr/ceph_secrets_client.py``) rather than calling
``mgr.remote()`` directly. The client file, along with
``ceph_secrets_types.py``, lives at the top level of ``src/pybind/mgr/`` so
any MGR module can import it without depending on the ``ceph_secrets`` package
internals.
.. code-block:: python
from ceph_secrets_client import CephSecretsClient
from ceph_secrets_types import SecretScope
client = CephSecretsClient(self) # self is a MgrModule instance
# Store a secret as an opaque string
client.secret_set(
namespace="cephadm",
scope=SecretScope.HOST,
target="node1",
name="ssh_key",
data="AQB...==",
)
# Retrieve metadata (no data unless reveal=True)
rec = client.secret_get("cephadm", SecretScope.HOST, "node1", "ssh_key")
if rec:
version = rec["metadata"]["version"]
# Remove
client.secret_rm("cephadm", SecretScope.HOST, "node1", "ssh_key")
CephSecretsClient methods
--------------------------
.. list-table::
:header-rows: 1
:widths: 35 65
* - Method
- Description
* - ``secret_set(namespace, scope, target, name, data, ...)``
- Create or update a secret. ``data`` must be a non-empty opaque
string. Increments the version and refreshes ``updated`` on each
call; ``created`` is set only on the first write. Optional
``user_made`` and ``editable`` flags control the stored policy;
``editable=False`` blocks future updates but does not prevent removal.
* - ``secret_get(namespace, scope, target, name, reveal=False)``
- Return the secret's metadata dict, plus the opaque ``data`` string if
*reveal* is True. Returns ``None`` if the secret does not exist.
* - ``secret_get_value(namespace, scope, target, name)``
- Return the stored opaque string directly, or ``None`` if the secret
does not exist. Use this when only the value is needed and the
metadata envelope is not required.
* - ``secret_get_version(namespace, scope, target, name)``
- Return the current version integer, or ``None`` if not found. A
cheaper alternative to ``secret_get`` when only change-detection is needed.
* - ``secret_get_versions(uris)``
- Batch-fetch version numbers for a list of canonical ``secret:/...``
URIs. Returns a dict keyed by URI; missing secrets map to ``None``.
Malformed URIs are skipped entirely, so a missing key indicates
malformed input rather than an absent secret.
* - ``secret_rm(namespace, scope, target, name)``
- Remove a secret. Idempotent: returns ``False`` if not found.
* - ``secret_get_epoch(namespace)``
- Return the current epoch for the namespace. Use as a cheap
change-detector before a full refresh.
* - ``scan_refs(obj, namespace)``
- Walk a JSON-like object and return secret-like reference strings found
within it. This includes valid whole-value secret URIs and malformed
or embedded secret-like references that should be reported to the
caller. Only canonical ``secret:/...`` URIs should be passed to
``secret_get_versions``.
* - ``scan_unresolved_refs(obj, namespace)``
- Like ``scan_refs`` but returns only references that are missing,
malformed, embedded inside a larger string, or cannot currently be
read successfully. Useful for pre-flight validation.
* - ``resolve_object(obj)``
- Walk a JSON-like object and replace every whole-value ``secret:/...``
URI string with the stored opaque string for the referenced secret.
.. _mgr-ceph-secrets-scan:
Secret URI embedding and resolution
-----------------------------------
Configuration objects that need to reference secrets without embedding
plaintext can store ``secret:/...`` URIs as string values. The module
resolves them on demand:
.. code-block:: python
spec = {
"service_type": "my-service",
"spec": {
"api_token": " secret:/cephadm/service/my-service/api_token ",
},
}
# Check for missing, malformed, embedded, or unresolvable secret references
unresolved = client.scan_unresolved_refs(spec, namespace="cephadm")
if unresolved:
raise RuntimeError(f"Unresolved secret references: {unresolved}")
# Replace URI references with plaintext values
resolved = client.resolve_object(spec)
# resolved["spec"]["api_token"] now holds the stored API token
.. note::
Resolution replaces the entire string value of a field. Surrounding
whitespace around a URI reference is ignored, so values such as
``" secret:/cephadm/global/foo "`` are accepted. Embedding a secret URI
inside a larger string (for example ``"Bearer secret:/..."``) is not
supported; after trimming surrounding whitespace, the field value must be
the URI.
The resolved value is the stored opaque string. The module does not parse
stored data as JSON and does not support field-level URI selection. If a
caller stores structured data, it must encode and decode that structure
itself.
.. _mgr-ceph-secrets-epoch:
Epoch-based change detection
----------------------------
Consumers that maintain a local cache of secrets can use the epoch to avoid
unnecessary full refreshes:
.. code-block:: python
def maybe_refresh(client, namespace, last_epoch):
current_epoch = client.secret_get_epoch(namespace)
if current_epoch == last_epoch:
return last_epoch # nothing changed
# ... do a full refresh ...
return current_epoch
last_epoch = 0
last_epoch = maybe_refresh(client, "cephadm", last_epoch)
The epoch is incremented by every successful ``set`` operation and by
``rm`` only when an existing secret is actually removed (an idempotent
``rm`` returning ``"not_found"`` does not bump the epoch). It starts at 0
for a namespace that has never been written to, and mutations in one namespace
never affect another namespace's epoch.
Configuration
=============
.. mgr_module:: ceph_secrets
.. confval:: secrets_backend
The storage backend used for secrets. Currently only the Monitor KV store
(``mon``) is supported. This option is reserved for future backends
(e.g. HashiCorp Vault).

1
doc/mgr/index.rst
View File

@ -47,5 +47,6 @@ sensible.
MDS Autoscaler module <mds_autoscaler>
NFS module <nfs>
SMB module <smb>
Secrets module <ceph_secrets>
Progress Module <progress>
CLI API Commands module <cli_api>

5
doc/mgr/nfs.rst
View File

@ -31,7 +31,7 @@ Create NFS Ganesha Cluster
.. prompt:: bash #
ceph nfs cluster create <cluster_id> [<placement>] [--ingress] [--virtual_ip <value>] [--ingress-mode {default|keepalive-only|haproxy-standard|haproxy-protocol}] [--port <int>] [--enable-rdma] [--rdma_port <int>] [-i <spec_file>]
ceph nfs cluster create <cluster_id> [<placement>] [--ingress] [--virtual_ip <value>] [--ingress-mode {default|keepalive-only|haproxy-standard|haproxy-protocol}] [--port <int>] [--enable-rdma] [--rdma_port <int>] [-i <spec_file>] [--enable-nfsv3]
This creates a common recovery pool for all NFS Ganesha daemons, new user based on
``cluster_id``, and a common NFS Ganesha config RADOS object.
@ -62,6 +62,9 @@ cluster)::
NFS can be deployed on a port other than 2049 (the default) with ``--port <port>``.
By default, only NFS v4 protocol is enabled. To enable both NFS v3 and v4 protocols,
add the ``--enable-nfsv3`` flag.
To deploy NFS with a high-availability front-end (virtual IP and load balancer), add the
``--ingress`` flag and specify a virtual IP address. This will deploy a combination
of keepalived and haproxy to provide an high-availability NFS frontend for the NFS

11
doc/mgr/orchestrator.rst
View File

@ -113,10 +113,19 @@ Creating/growing/shrinking/removing services:
ceph orch apply mds <fs_name> [--placement=<placement>] [--dry-run]
ceph orch apply rgw <name> [--realm=<realm>] [--zone=<zone>] [--port=<port>] [--ssl] [--placement=<placement>] [--dry-run]
ceph orch apply nfs <name> <pool> [--namespace=<namespace>] [--placement=<placement>] [--dry-run]
ceph orch rm <service_name> [--force]
ceph orch rm <service_name> [--force] [--force-delete-data]
where ``placement`` is a :ref:`orchestrator-cli-placement-spec`.
For ``ceph orch rm``, ``--force`` may be required in some cases (for example when
removing an OSD service that would leave OSDs behind).
The ``--force-delete-data`` flag requires that ``--force`` is also passed.
This directs cephadm to delete on-disk data for certain daemon types instead
of moving it to ``<fsid>/removed/`` on the host. These daemon types include
``mon``, ``osd``, and ``prometheus``. This helps avoid filling up the
underlying filesystem over time.
e.g., ``ceph orch apply mds myfs --placement="3 host1 host2 host3"``
Service Commands:

220
doc/mgr/smb.rst
View File

@ -55,7 +55,7 @@ Create Cluster
.. prompt:: bash #
ceph smb cluster create <cluster_id> {user|active-directory} [--domain-realm=<domain_realm>] [--domain-join-user-pass=<domain_join_user_pass>] [--define-user-pass=<define_user_pass>] [--custom-dns=<custom_dns>] [--placement=<placement>] [--clustering=<clustering>] [--password-filter=<password_filter>] [--password-filter-out=<password_filter_out>]
ceph smb cluster create <cluster_id> {user|active-directory} [--domain-realm=<domain_realm>] [--domain-join-user-pass=<domain_join_user_pass>] [--define-user-pass=<define_user_pass>] [--custom-dns=<custom_dns>] [--placement=<placement>] [--clustering=<clustering>] [--client-compat={default|macos}] [--password-filter=<password_filter>] [--password-filter-out=<password_filter_out>]
Create a new logical cluster, identified by the cluster ID value. The cluster
create command must specify the authentication mode the cluster will use. This
@ -100,6 +100,11 @@ clustering
enables clustering regardless of the placement count. A value of ``never``
disables clustering regardless of the placement count. If unspecified,
``default`` is assumed.
client_compat
Optional. One of ``default`` or ``macos``. Controls client-specific SMB
features and optimizations. The ``default`` mode provides standard SMB
behavior with broad compatibility. The ``macos`` mode enables macOS-specific
optimized settings for macOS clients. If unspecified, ``default`` is assumed.
public_addrs
Optional. A string in the form of <ipaddress/prefixlength>[%<destination address>].
Supported only when using Samba's clustering. Assign "virtual" IP addresses
@ -165,6 +170,15 @@ previous example. Set three CTDB public address values and a custom placement:
--public-address=192.168.76.112/24 \
--placement="3 label:smb"
Create a cluster for macOS clients with user authentication:
.. prompt:: bash #
ceph smb cluster create mac_cluster user \
--define-user-pass=macuser%Passw0rd1 \
--client-compat=macos \
--placement="label:smb"
Update Cluster QoS
++++++++++++++++++
@ -216,6 +230,53 @@ Disable QoS for all shares in a cluster:
--write-bw-limit=0
Update Cluster Client Compatibility
++++++++++++++++++++++++++++++++++++
.. prompt:: bash #
ceph smb cluster update client-compat {default|macos} <cluster_id>
Update the client compatibility mode for an SMB cluster. This setting controls whether client-specific SMB features and optimizations are enabled.
The client compatibility mode determines how the Samba server is configured to optimize for specific client types:
- ``default``: Standard SMB behavior without client-specific optimizations. This is the default mode and provides broad compatibility with all SMB clients.
- ``macos``: Enable macOS-specific features including the Samba's fruit VFS module for proper handling of macOS metadata and optimized settings for macOS clients.
When ``macos`` mode is enabled, the following features are automatically configured:
- Fruit VFS module for proper handling of macOS-specific file attributes
- Streams_xattr VFS module for extended attribute support
Options:
client_compat
One of ``default`` or ``macos``
cluster_id
A short string uniquely identifying the cluster
Examples
~~~~~~~~
Enable macOS compatibility mode for a cluster:
.. prompt:: bash #
ceph smb cluster update client-compat macos mycluster
Revert to default (standard) compatibility mode:
.. prompt:: bash #
ceph smb cluster update client-compat default mycluster
.. note::
The ``macos`` compatibility mode is recommended when the primary clients accessing the SMB shares are macOS systems. Otherwise default mode is recommended.
Remove Cluster
++++++++++++++
@ -321,6 +382,51 @@ Create a read-only share at a custom path in the CephFS volume:
--path=/qbranch/top/secret/plans --readonly
Create RGW Share
++++++++++++++++
.. prompt:: bash #
ceph smb share create rgw <cluster_id> <share_id> <bucket> [--share-name=<share_name>] [--user-id=<user_id>] [--readonly]
Create a new SMB share, hosted by the named cluster, that maps to a RADOS
Gateway (RGW) bucket. This allows S3-compatible object storage to be accessed
via the SMB protocol.
Options:
cluster_id
A short string uniquely identifying the cluster
share_id
A short string uniquely identifying the share
bucket
The name of the RGW bucket to be shared
share_name
Optional. The public name of the share, visible to clients. If not provided
the ``share_id`` will be used automatically
user_id
Optional. The RGW user ID that owns the bucket. If not provided, the system
will attempt to determine the bucket owner automatically
readonly
Creates a read-only share
Examples
~~~~~~~~
Create a share for an RGW bucket:
.. prompt:: bash #
ceph smb share create rgw test1 photos my-photos-bucket
Create a share with a custom name and specific user:
.. prompt:: bash #
ceph smb share create rgw test1 photos my-photos-bucket \
--share-name="Photo Archive" --user-id=s3user
.. _qos-parameters:
QoS Parameters
@ -879,6 +985,12 @@ custom_smb_global_options
indicator that the user is aware that using this option can easily break
things in ways that the Ceph team can not help with. This special key will
automatically be removed from the list of options passed to Samba.
client_compat
Optional. One of ``default`` or ``macos``. Controls client-specific SMB
features and optimizations. The ``default`` mode provides standard SMB
behavior with broad compatibility. The ``macos`` mode enables macOS-specific
features including Samba's fruit VFS module for proper handling of macOS and
optimized settings for macOS clients. If unspecified, ``default`` is assumed.
.. warning::
Setting the ``clustering`` option allows an administrator to choose exactly
@ -977,6 +1089,20 @@ The following is an example of a cluster configured for standalone operation:
hosts:
- node6.mycluster.sink.test
The following is an example of a cluster optimized for macOS clients:
.. code-block:: yaml
resource_type: ceph.smb.cluster
cluster_id: macshare
auth_mode: user
client_compat: macos
user_group_settings:
- source_type: resource
ref: ug1
placement:
count: 1
An example cluster resource with intent to remove:
.. code-block:: yaml
@ -1018,7 +1144,8 @@ max_connections
connections to a specific share. The default value is 0 and it indicates
that there is no limit on the number of connections
cephfs
Required object. Fields:
Object. Configures CephFS-backed storage for the share. Either a ``cephfs``
or ``rgw`` object must be specified, but not both. Fields:
volume
Required string. Name of the cephfs volume to use
@ -1088,6 +1215,22 @@ cephfs
name
String. A value indicating what FSCrypt key to fetch. The specific
value of the name depends on the scope being used.
rgw
Object. Configures RADOS Gateway (RGW) backed storage for the share. Either
a ``cephfs`` or ``rgw`` object must be specified, but not both. This allows
S3-compatible object storage to be accessed via the SMB protocol. Fields:
bucket
Required string. The name of the RGW bucket to be shared.
user_id
Optional string. The RGW user ID that owns the bucket. If not provided,
the system will automatically determine the bucket owner and fetch the
necessary credentials.
credential_ref
Optional string. The ``rgw_credential_id`` value of a
``ceph.smb.rgw.credential`` resource that contains RGW access and
secret key values needed to use the given bucket.
restrict_access
Optional boolean, defaulting to false. If true the share will only permit
access by users explicitly listed in ``login_control``.
@ -1135,7 +1278,32 @@ custom_smb_share_options
things in ways that the Ceph team can not help with. This special key will
automatically be removed from the list of options passed to Samba.
The following is an example of a share with QoS settings including burst
The following is an example of an RGW-backed share with minimal configuration
(credentials auto-fetched):
.. code-block:: yaml
resource_type: ceph.smb.share
cluster_id: tango
share_id: s3share
name: "S3 Storage"
rgw:
bucket: my-bucket
Another example of an RGW-backed share with explicit user_id:
.. code-block:: yaml
resource_type: ceph.smb.share
cluster_id: tango
share_id: s3share
name: "S3 Storage"
rgw:
bucket: my-bucket
user_id: s3user
The following is an example of a CephFS share with QoS settings including burst
multipliers and human-readable bandwidth limits:
.. code-block:: yaml
@ -1285,6 +1453,43 @@ Example:
groups: []
RGW Credential Resource
------------------------
An RGW credential resource stores RADOS Gateway (RGW) access credentials that can be used by RGW-backed shares to authenticate with the object storage system.
A RGW credential resource supports the following fields:
resource_type
A literal string ``ceph.smb.rgw.credential``
rgw_credential_id
A short string identifying the RGW credential resource.
intent
One of ``present`` or ``removed``. If not provided, ``present`` is assumed.
If ``removed`` all following fields are optional
user_id
Required string. The RGW user ID that owns the credentials
access_key_id
Required string. The RGW access key for authentication
secret_access_key
Required string. The RGW secret key for authentication
linked_to_cluster:
Optional. A string containing a cluster ID. If set, the resource may only
be used with the linked cluster and will automatically be removed when the
linked cluster is removed.
Example:
.. code-block:: yaml
resource_type: ceph.smb.rgw.credential
rgw_credential_id: s3user
user_id: s3user
access_key: AKIAIOSFODNN7EXAMPLE
secret_key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
TLS Credential Resource
------------------------
@ -1483,10 +1688,11 @@ cluster has been configured for at least one share. The ``placement`` field of
the cluster resource is passed onto the orchestration layer and is used to
determine on what nodes of the Ceph cluster Samba containers will be run.
At this time Samba services can only listen on port 445. Due to this
restriction only one Samba server, as part of one cluster, may run on a single
Ceph node at a time. Ensure that the placement specs on each cluster do not
overlap.
The Samba containers may run on the same hosts if, and only if, the services
use different IP addresses and/or ports. If the placement spec allows more than
one container to run on the same host, use the ``bind_addrs`` field, the
``custom_ports`` field, or some combination, in the cluster resource to ensure
that the Samba server instances do not conflict.
The ``smb`` clusters are fully isolated from each other. This means that, as
long as you have sufficient resources in your Ceph cluster, you can run multiple

5
doc/rados/api/librados-intro.rst
View File

@ -70,8 +70,7 @@ Getting librados for Python
---------------------------
The ``rados`` module provides ``librados`` support to Python
applications. You may install ``python3-rados`` for Debian, Ubuntu, SLE or
openSUSE or the ``python-rados`` package for CentOS/RHEL.
applications. You may install the ``python3-rados`` package.
To install ``librados`` development support files for Python on Debian/Ubuntu
distributions, execute the following:
@ -85,7 +84,7 @@ distributions, execute the following:
.. prompt:: bash $
sudo yum install python-rados
sudo yum install python3-rados
To install ``librados`` development support files for Python on SLE/openSUSE
distributions, execute the following:

174
doc/rados/bluestore/fast-onode-scan.rst Normal file
View File

@ -0,0 +1,174 @@
============================
Multithread Onode Scanning
============================
.. index:: bluestore; rocksdb; allocator
Beginning with the Pacific release, BlueStore has the option to configure a 5-10% latency reduction
at a cost of significantly longer ``OSD`` recovery after a crash.
.. confval:: bluestore_allocation_from_file
During crash recovery BlueStore must reconstruct allocator state by scanning all stored RADOS objects.
For OSDs storing millions of RADOS objects this process may take several minutes to several hours to complete.
This page describes multithreaded onode recovery that significantly reduces recovery time.
Allocator
=========
In BlueStore, the configured ``Allocator`` is responsible for tracking unused allocation units on the underlying device.
As many as three allocators may be in use concurrently: one for the main block device, one for the DB, and one for the WAL.
Only the main device stores RADOS objects, and thus is the only allocator that is relevant here.
When a BlueStore OSD is running, allocator state is fully loaded in memory. This ensures that the
allocator can quickly respond when there is a need to allocate space for RADOS objects,
or when RADOS objects no longer exist and their underlying space should be reclaimed.
RocksDB-persisted allocator
---------------------------
The original design envisioned that allocator state updates are stored in RocksDB's ``b`` column.
When RADOS objects are written or release allocation units, the relevant state information
is updated with an object-modifying RocksDB atomic transaction. The information transfer is unidirectional:
BlueStore updates allocator state in RocksDB, but never retrieves it.
The exception is at OSD boot when BlueStore retrieves allocator state from the RocksDB ``b`` column.
File-persisted allocator
------------------------
Tests showed that keeping RocksDB busy by updating ``b`` column with allocation data
has observable cost in terms of write latency and CPU burden when compacting.
To avoid this performance impact the new default mode is to update allocator state only in-memory.
Only on shutdown is state saved to a BlueFS file named ``ALLOCATOR_NCB_DIR/ALLOCATOR_NCB_FILE``.
Crash recovery
==============
If BlueStore was not shut down gracefully there will be no updated file from which to read allocator state.
Since RADOS objects are then the only only source of truth regarding underlying device usage,
the recovery procedure must iterate over all RADOS objects in order to determine used vs
available device blocks.
Multi-threaded recovery
=======================
Iterating over all onodes in the RocksDB can take several minutes on
a relatively small SSD; for large and slow devices it may take several *hours*.
To mitigate this significant impact on OSD boot time, a multi-threaded recovery procedure
has been added for the Vampire release and may be backported to Umbrella and Tentacle.
This is a significantly different procedure than the original.
One config option selects and controls this new recovery strategy.
.. confval:: bluestore_allocation_recovery_threads
Performance
-----------
Below are example elapsed recovery timings for an
OSD on an NVMe SSD hosting 7.5M RBD objects, with 4.6TiB used:
+---------------------+------------+-----------------+
| Recovery Mechanism | Threads | Time in seconds |
+=====================+============+=================+
| original | 1 | 76.0 |
+---------------------+------------+-----------------+
| multi-threaded | 1 | 55.8 |
+---------------------+------------+-----------------+
| multi-threaded | 4 | 18.1 |
+---------------------+------------+-----------------+
| multi-threaded | 8 | 10.8 |
+---------------------+------------+-----------------+
| multi-threaded | 12 | 8.0 |
+---------------------+------------+-----------------+
| multi-threaded | 16 | 6.8 |
+---------------------+------------+-----------------+
| multit-hreaded | 32 | 5.3 |
+---------------------+------------+-----------------+
Testing
-------
Concern for long recovery time has led some Ceph operators to configure
legacy synchronous persistance of allocator state in RocksDB.
There is now a procedure for predicting the legacy recovery time for a given OSD,
allowing an informed decision:
.. prompt:: bash #
ceph-bluestore-tool --path <osd-data-path> recovery-compare
.. code-block::
recovery-compare bluestore compare new and legacy onode recovery
Legacy recovery start
Legacy recovery result=0 took 76.033992 seconds
Legacy recovery stats=
==========================================================
onode_count = 7500217
shard_count = 9754950
shared_blob_count = 0
compressed_blob_count = 0
spanning_blob_count = 264519
skipped_illegal_extent = 100815206
extent_count = 188685236
insert_count = 0
store store_statfs(0x0/0x0/0x0, data 0x0/0x0, compress 0x0/0x0/0x0, omap 0x0, meta 0x0)
pool 1 store_statfs(0x0/0x0/0x0, data 0x90220/0x94000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0)
pool 2 store_statfs(0x0/0x0/0x0, data 0x4348a70e000/0x4c7ee684000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0)
pool 18446744073709551615 store_statfs(0x0/0x0/0x0, data 0x26a3f/0x150000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0)
==========================================================
bluestore_allocation_recovery_threads = 0
No multithread recovery to compare.
recovery-compare success
.. prompt:: bash #
ceph-bluestore-tool --path <osd-data-path> --bluestore_allocation_recovery_threads=12 recovery-compare
.. code-block::
recovery-compare bluestore compare new and legacy onode recovery
New recovery start
New recovery result=0 took 8.033309 seconds
New recovery stats=
==========================================================
onode_count = 7500217
shard_count = 9754950
shared_blob_count = 0
compressed_blob_count = 0
spanning_blob_count = 264519
skipped_illegal_extent = 0
extent_count = 188685236
insert_count = 0
store store_statfs(0x0/0x0/0x0, data 0x0/0x0, compress 0x0/0x0/0x0, omap 0x0, meta 0x0)
pool 1 store_statfs(0x0/0x0/0x0, data 0x90220/0x94000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0)
pool 2 store_statfs(0x0/0x0/0x0, data 0x4348a70e000/0x4c7ee684000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0)
pool 18446744073709551615 store_statfs(0x0/0x0/0x0, data 0x26a3f/0x150000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0)
==========================================================
Legacy recovery start
Legacy recovery result=0 took 62.210723 seconds
Legacy recovery stats=
==========================================================
onode_count = 7500217
shard_count = 9754950
shared_blob_count = 0
compressed_blob_count = 0
spanning_blob_count = 264519
skipped_illegal_extent = 100815206
extent_count = 188685236
insert_count = 0
store store_statfs(0x0/0x0/0x0, data 0x0/0x0, compress 0x0/0x0/0x0, omap 0x0, meta 0x0)
pool 1 store_statfs(0x0/0x0/0x0, data 0x90220/0x94000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0)
pool 2 store_statfs(0x0/0x0/0x0, data 0x4348a70e000/0x4c7ee684000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0)
pool 18446744073709551615 store_statfs(0x0/0x0/0x0, data 0x26a3f/0x150000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0)
==========================================================
FSstats the same.
Allocators the same.
recovery-compare success
One can see in the above example the improvement in legacy recovery times: 76.0s vs 62.2s.
The second run shows the effect of RocksDB having the data already cached by the new recovery strategy.

303
doc/rados/bluestore/fcm-plugin.rst Normal file
View File

@ -0,0 +1,303 @@
=======================
EXTBLKDEV, FCM plugin
=======================
.. index:: bluestore; extblkdev
ExtBlkDev
=========
With Pacific release Ceph was extended to handle thinly-provisioned and compressing drives.
The extension is made in form of loadable plugins that are part of Ceph codebase: ``src/extblkdev``.
At startup OSD checks if specific plugins are enabled, as controlled by:
.. confval:: osd_extblkdev_plugins
To let OSD load and check for presence of FCM devices set:
.. code-block::
osd_extblkdev_plugins = fcm
Mode of operation
-----------------
Thin-provisioned and compressing drives have different capacity for data than what is advertised
as addressable logical space. It means that a compressing drive might advertise its size as 10TB,
but in reality only operate on 2TB of physical NAND.
ExtBlkDev plugins provide an additional information channel between devices and BlueStore,
giving Ceph report current disk logical / physical, used / available states.
This information is critical for OSDs: they might otherwise try to write more data to the device
than it can store, which will critically fail when it is really REALLY full.
Without current and accurate drive usage the configurables are useless:
.. confval:: mon_osd_full_ratio
.. confval:: mon_osd_nearfull_ratio
The ExtBlkDev extension is implemented on ``BlockDevice`` level, common for ``Main``, ``DB``, and ``WAL`` devices.
However, BlueStore only uses it in context of ``Main`` device.
For the ``DB`` and ``WAL`` devices, the plugin is detected and loaded, but BlueStore does not interact with it.
Plugin persistence
------------------
.. note::
The examples below are derived from an actual deployment of the FCM plugin.
Having the proper plugin loaded is critical to OSD operation.
.. prompt:: # bash
ceph osd df
.. code-block::
ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS
0 ssd 5.07739 1.00000 31 TiB 4.3 TiB 4.2 TiB 11 KiB 12 GiB 26 TiB 13.92 0.77 129 up
1 ssd 5.07739 1.00000 31 TiB 4.3 TiB 4.2 TiB 12 KiB 12 GiB 26 TiB 13.93 0.77 128 up
2 ssd 5.07739 1.00000 31 TiB 4.3 TiB 4.2 TiB 12 KiB 12 GiB 26 TiB 13.93 0.77 129 up
3 ssd 5.07739 1.00000 31 TiB 4.2 TiB 4.2 TiB 12 KiB 12 GiB 26 TiB 13.90 0.77 128 up
4 ssd 5.07739 1.00000 5.1 TiB 2.2 TiB 4.2 TiB 12 KiB 12 GiB 2.9 TiB 43.08 2.38 128 up
5 ssd 5.07739 1.00000 5.1 TiB 2.2 TiB 4.3 TiB 12 KiB 12 GiB 2.9 TiB 43.25 2.39 128 up
6 ssd 5.07739 1.00000 5.1 TiB 2.2 TiB 4.2 TiB 12 KiB 12 GiB 2.9 TiB 43.06 2.38 128 up
7 ssd 5.07739 1.00000 5.1 TiB 2.2 TiB 4.3 TiB 14 KiB 12 GiB 2.9 TiB 43.44 2.40 129 up
TOTAL 143 TiB 26 TiB 34 TiB 101 KiB 94 GiB 117 TiB 18.09 1027
MIN/MAX VAR: 0.77/2.40 STDDEV: 18.00
In the above case OSDs 0-3 run without the plugin; OSDs 4-7 have the plugin loaded and active.
All eight OSDs participate in a single 6+2 erasure coded pool and therefore carry equivalent utilization.
With EC6+2 this is moot, but for other pool types the balancer will prioritize OSDs 0-3 which seem to be mostly empty
trying to offload seemingly overburdened OSDs 4-7.
Tentacle provides a new configurable that signals BlueStore to expect that an ExtBlkDev plugin will be in use.
.. confval:: bluestore_use_ebd
The ExtBlkDev infrastructure adds new category of health warnings:
.. code-block::
HEALTH_WARN 8 OSD(s) reporting problems with ExtBlkDev plugin;
[WRN] EXTBLKDEV: 8 OSD(s) reporting problems with ExtBlkDev plugin
With the ``bluestore_use_ebd`` set to ``true``, two EXTBLKDEV health warnings may appear:
1) "plugin 'fcm' not loaded"
2) "plugin 'fcm' used on mkfs, but now uses plugin 'disabled'"
Having the configuration option set to ``true`` does not require that a plugin is used.
If a plugin is not found during OSD creation, this is still a proper and acceptable state.
However, once a plugin is detected during the OSD's creation, its usage becomes mandatory.
Should plugin presence be required for specific OSD, one can check for it:
.. prompt:: bash #
ceph osd metadata osd.2
.. code-block::
...
"bluestore_bdev_fcm": "true",
...
or
.. prompt:: bash #
ceph-bluestore-tool --path dev/osd4/ show-label
.. code-block::
...
"extblkdev": "fcm",
...
The first check is somewhat weaker than the second.
In a specific error case that the plugin was detected and memorized at OSD creation,
but is not loaded, one will not see the value in OSD metadata. Instead a health warning appears.
The second check never fails. It simply stores the setting in BlueStore label metadata.
.. _fix-use-ebd:
A fix
-----
If ``bluestore_use_ebd=true`` was not set when OSDs were created on devices that should run with a plugin,
it may be set after the fact with a command of the following form:
.. prompt:: bash #
ceph-bluestore-tool --dev dev/osd4/block -k extblkdev -v fcm set-label-key
.. _fcm-plugin:
FCM plugin
==========
Ceph releases beginning with Tentacle provide a plugin to operate with FlashCoreModule devices (FCMs).
Like all ExtBlkDev plugins, it reports true disk usage to BlueStore, effectively overriding
BlueStore reported metrics of used and available device space.
FCM reported metrics are in physical NAND size, not in logical space size.
In addition the plugin reports fixed stats via OSD metadata:
.. prompt:: bash #
ceph osd metadata osd.2
.. code-block::
...
"bluestore_bdev_fcm_device_logical_size": "33599931809792",
"bluestore_bdev_fcm_device_physical_size": "5582606303232",
"bluestore_bdev_fcm_partition_logical_size": "33599931809792",
"bluestore_bdev_fcm_partition_physical_size": "5582606303232",
...
And dynamic current state via performance counters:
.. prompt:: bash #
ceph tell osd.6 perf dump extblkdev
.. code-block::
{
"extblkdev": {
"fcm": 0, <- value is irrelevant, it is the name that counts
"dev_phy_size": 5582606303232,
"dev_log_size": 33599931809792,
"dev_phy_util": 2403758935846,
"dev_log_util": 4714543366144,
"part_phy_size": 5582606303232,
"part_log_size": 33599931809792,
"part_phy_avail": 3178847367386,
"part_log_avail": 28885388443648
}
}
The FCM plugin provides three new health warnings in the EXTBLKDEV class.
1) "failed accessing FCM utilization log" :ref:`fcm-permissions`
2) "bdev_enable_discard not enabled - free space will leak" :ref:`fcm-discard`
3) "multivolume fcm will not work properly" :ref:`fcm-multivolume`
.. _fcm-permissions:
FCM permissions
---------------
The FCM plugin talks to the NVMe controller within each storage device.
In Linux, this is a privileged operation, tied to the `cap_sys_admin` privilege.
To get the privilege do either:
a) Run Ceph OSD as root.
b) Grant ``setcap "cap_sys_admin=p" ceph-osd``
and run with ``ceph-osd --set-keepcaps=true`` .
.. _fcm-discard:
FCM discard
-----------
FCM devices implement compression; their physical NAND capacity is much smaller than their logical device size.
Conventional SSDs storing data that is no longer used by BlueStore experience may experience suboptimal performance.
With FCMs, orphaned data represents a direct loss of available capacity. To prevent this problem, one should always set:
.. code-block::
bdev_enable_discard = true
.. _fcm-multivolume:
FCM multivolume
---------------
BlueStore works well when there is a one-to-one mapping between FCM devices to BlueStore block devices.
Splitting an FCM device in half by partitioning does not create any boundary for the controller.
Filling one partition will affect available space in the other partition.
Two partitions together will have the total logical capacity exactly the same, but available physical NAND will be counted twice.
This will most likely confuse both balancers and operators.
NOTE: The FCM plugin does not detect this condition.
Combining multiple FCM devices using the device mapper, LVM, or Linux MD software RAID is not recommended. When doing so,
BlueStore has no visibility of the topology of the backing block devices and cannot know to address storage by specific logical device offsets.
For regular drives it does not matter; when the physical:logical ratio is 1:1 BlueStore will eventually run out of logical space on first drive
and without even noticing start consuming the second drive.
With FCMs, and any other compressing drives, it is no longer the case. When BlueStore is not running out of logical space on first drive,
it will keep asking the drive to accomodate more data. The second drive has it, but first does not.
BlueStore will eventually fail to allocate space and fail with a hard ``-ENOSPC`` error.
Trying to run BlueStore with compound block device when one of them is an FCM will raise the warning.
An OSD cannot operate long-term in this way. It must be redeployed.
Redeploy and discard
--------------------
When one redeploys an FCM OSD without first formatting or issuing a whole-device discard operation,
BlueStore will be logically empty, but the underlying device will still hold the burden of previously held data.
.. code-block::
ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS
0 ssd 5.07739 1.00000 5.1 TiB 2.2 TiB 956 KiB 24 KiB 26 MiB 2.9 TiB 43.16 1.00 1 up
1 ssd 5.07739 1.00000 5.1 TiB 2.2 TiB 376 KiB 20 KiB 26 MiB 2.9 TiB 43.14 1.00 0 up
2 ssd 5.07739 1.00000 5.1 TiB 2.2 TiB 956 KiB 20 KiB 26 MiB 2.9 TiB 43.15 1.00 1 up
3 ssd 5.07739 1.00000 5.1 TiB 2.2 TiB 376 KiB 16 KiB 26 MiB 2.9 TiB 43.05 1.00 0 up
4 ssd 5.07739 1.00000 5.1 TiB 2.2 TiB 376 KiB 14 KiB 26 MiB 2.9 TiB 43.08 1.00 0 up
5 ssd 5.07739 1.00000 5.1 TiB 2.2 TiB 376 KiB 11 KiB 26 MiB 2.9 TiB 43.24 1.00 0 up
6 ssd 5.07739 1.00000 5.1 TiB 2.2 TiB 376 KiB 7 KiB 26 MiB 2.9 TiB 43.05 1.00 0 up
7 ssd 5.07739 1.00000 5.1 TiB 2.2 TiB 956 KiB 6 KiB 26 MiB 2.9 TiB 43.44 1.01 1 up
TOTAL 41 TiB 18 TiB 4.6 MiB 121 KiB 210 MiB 23 TiB 43.17 3
MIN/MAX VAR: 1.00/1.01 STDDEV: 0.12
Solution:
a) ``nvme format`` before deploying
b) Set ``bluestore_discard_on_mkfs=true`` (Umbrella+).
CRUSH weight
------------
This section describes :ref:`fix-use-ebd` when using the FCM plugin :ref:`fcm-plugin`.
When an OSD is properly deployed on an FCM, the WEIGHT and SIZE values reflect the physical NAND capacity:
.. code-block::
ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS
0 ssd 5.07739 1.00000 5.1 TiB 12 MiB 944 KiB 21 KiB 26 MiB 5.1 TiB 0 1.00 1 up
When an OSD is deployed without the active FCM plugin, the WEIGHT and SIZE reflect logical advertised values:
.. code-block::
ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS
0 ssd 30.55899 1.00000 31 TiB 27 MiB 908 KiB 16 KiB 26 MiB 31 TiB 0 1.01 1 up
When a plugin is active for the underlying device, the result is that the SIZE
is the proper physical NAND capacity, but the CRUSH weight is still locked to original value at deployment:
.. code-block::
ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS
0 ssd 30.55899 1.00000 5.1 TiB 28 KiB 1.0 MiB 7 KiB 27 MiB 5.1 TiB 0 1.42 1 up
The CRUSH weight of such OSDs must be adjusted.

2
doc/rados/configuration/index.rst
View File

@ -22,6 +22,8 @@ For general object store configuration, refer to the following:
Storage devices <storage-devices>
BlueStore RocksDB cache <../bluestore/rocksdb-config>
BlueFS Spillover Cleaner <../bluestore/bluefs-spillover-cleaner>
Fast Crash Recovery for file-stored allocations <../bluestore/fast-onode-scan>
ExtBlkDev, FCM plugin <../bluestore/fcm-plugin>
ceph-conf

31
doc/rados/configuration/journal-ref.rst
View File

@ -1,34 +1,13 @@
==========================
Journal Config Reference
==========================
.. warning:: Filestore has been deprecated in the Reef release and is no longer supported.
.. warning:: Filestore is not supported. Migrate existing Filestore OSDs to
BlueStore. See :ref:`rados_operations_bluestore_migration`.
.. index:: journal; journal configuration
Filestore OSDs use a journal for two reasons: speed and consistency. Note
that since Luminous, the BlueStore OSD back end has been preferred and default.
This information is provided for pre-existing OSDs and for rare situations where
Filestore is preferred for new deployments.
- **Speed:** The journal enables the Ceph OSD Daemon to commit small writes
quickly. Ceph writes small, random I/O to the journal sequentially, which
tends to speed up bursty workloads by allowing the backing file system more
time to coalesce writes. The Ceph OSD Daemon's journal, however, can lead
to spiky performance with short spurts of high-speed writes followed by
periods without any write progress as the file system catches up to the
journal.
- **Consistency:** Ceph OSD Daemons require a file system interface that
guarantees atomic compound operations. Ceph OSD Daemons write a description
of the operation to the journal and apply the operation to the file system.
This enables atomic updates to an object (for example, placement group
metadata). Every few seconds--between ``filestore max sync interval`` and
``filestore min sync interval``--the Ceph OSD Daemon stops writes and
synchronizes the journal with the file system, allowing Ceph OSD Daemons to
trim operations from the journal and reuse the space. On failure, Ceph
OSD Daemons replay the journal starting after the last synchronization
operation.
Ceph OSD Daemons recognize the following journal settings:
The following journal settings are provided for reference to assist with
migration of existing Filestore OSDs to BlueStore.
.. confval:: journal_dio
.. confval:: journal_aio

22
doc/rados/configuration/mclock-config-ref.rst
View File

@ -4,22 +4,6 @@
.. index:: mclock; configuration
.. warning:: On large clusters with erasure-coded pools, operators may
observe slow ops during recovery or backfill (for example, when an
OSD is drained out). Under mClock, EC sub-operation reads issued
during recovery are currently routed through the ``immediate``
high-priority queue and bypass mClock throttling. When many OSDs
read concurrently from a single source OSD, this can saturate that
OSD's high-priority queue and starve client and background work.
As an interim measure, such deployments are advised to switch to
the ``WeightedPriorityQueue`` (``wpq``) scheduler. The change can
be applied cluster-wide and takes effect after each OSD is
restarted:
.. prompt:: bash #
ceph config set osd osd_op_queue wpq
QoS support in Ceph is implemented using a queuing scheduler based on `the
dmClock algorithm`_. See :ref:`dmclock-qos` section for more details.
@ -128,7 +112,7 @@ built-in profiles may be enabled by following the steps mentioned in next sectio
+------------------------+-------------+--------+-------+
| background recovery | 50% | 1 | MAX |
+------------------------+-------------+--------+-------+
| background best-effort | MIN | 1 | 90% |
| background best-effort | 5% | 2 | 90% |
+------------------------+-------------+--------+-------+
high_client_ops
@ -147,7 +131,7 @@ the resource control parameters set by the profile:
+------------------------+-------------+--------+-------+
| background recovery | 40% | 1 | MAX |
+------------------------+-------------+--------+-------+
| background best-effort | MIN | 1 | 70% |
| background best-effort | 5% | 4 | 70% |
+------------------------+-------------+--------+-------+
high_recovery_ops
@ -165,7 +149,7 @@ parameters set by the profile:
+------------------------+-------------+--------+-------+
| background recovery | 70% | 2 | MAX |
+------------------------+-------------+--------+-------+
| background best-effort | MIN | 1 | MAX |
| background best-effort | 5% | 2 | MAX |
+------------------------+-------------+--------+-------+
.. note:: Across the built-in profiles, internal background best-effort clients

355
doc/rados/configuration/mon-config-ref.rst
View File

@ -600,6 +600,361 @@ is far outweighed by the number of accidental pool (and thus data) deletions it
For more information about the pool flags see :ref:`Pool values <setpoolvalues>`.
Monitor backup
==============
In normal operation, Monitor backups are not required: surviving members
of the Monitor quorum re-sync new or replaced Monitors automatically,
and the Monitor store can be largely rebuilt from the OSDs after total
quorum loss (see :ref:`mon-store-recovery-using-osds`).
However, ``mon-store-recovery-using-osds`` only recovers state that the
OSDs can report: osdmap history, auth keys associated with running
OSDs, and similar. Some Monitor state has no copy outside the Monitor
store and is unrecoverable if all Monitors are lost:
* **Encryption keys for dm-crypt OSDs** are stored only in the
Monitor's ``config-key`` store under
``dm-crypt/osd/<osd-uuid>/luks``. Without these keys the underlying
block devices cannot be unlocked, even when the OSD daemons and data
are physically intact.
* **Cephadm orchestrator state** under ``mgr/cephadm/*`` in the
``config-key`` store, including host inventory, daemon placement
specs, and service definitions.
* **Config-key entries** populated by users or third-party tooling via
``ceph config-key set``.
* **Dashboard and manager module state** persisted to the
``config-key`` store.
A Monitor backup lets an operator restore this state if all running
Monitors are lost. It is most valuable for clusters that use
dm-crypt-encrypted OSDs or that depend heavily on cephadm-managed
deployment state, where loss of the Monitor store would be a
protracted data-availability incident rather than a recoverable inconvenience.
Monitor backups complement, but do not replace, the existing Monitor
recovery procedures. They are not a means of "undoing" cluster-level
operations such as pool deletion or CRUSH changes: once OSDs have
observed and acted on a newer osdmap, restoring an older Monitor store
does not roll back the OSD-side effects.
The Ceph Monitor uses the native RocksDB ``BackupEngine`` to create
consistent snapshots of its store, which can be copied elsewhere
without downtime.
When :confval:`mon_backup_interval` is set, a backup is triggered every
N seconds. Pair it with :confval:`mon_backup_cleanup_interval`; if only
the backup interval is set, backups accumulate indefinitely because
retention is only applied during cleanup.
Backups share table files (``.sst``) within the backup directory: each
new backup only copies SSTables that the running database has produced
since the previous backup. Restoring any individual backup version is
independent of the others, but the on-disk files for every version
live in a single shared tree under ``mon_backup_path``.
Layout of the backup directory
------------------------------
A backup path managed by the RocksDB ``BackupEngine`` contains three
top-level directories plus a per-version copy of the Monitor keyring::
/path/to/backups/
├── meta/
│ ├── 1
│ ├── 2
│ └── 3
├── private/
│ ├── 1/
│ ├── 2/
│ └── 3/
├── shared_checksum/
│ ├── 000007.sst
│ ├── 000010.sst
│ └── ...
├── keyring.1
├── keyring.2
└── keyring.3
* ``meta/<N>`` is a metadata file describing logical backup version
``N``.
* ``private/<N>/`` contains files unique to backup ``N`` (RocksDB
descriptors and other per-version state).
* ``shared_checksum/`` contains SSTables shared across backup
versions. A single SSTable in this directory may belong to several
versions; the BackupEngine deletes a file from here only when no
remaining backup references it.
* ``keyring.<N>`` is a copy of ``$mon_data/keyring`` taken at the time
of backup ``N``. The Monitor needs this key to authenticate at
startup, and it is not stored inside the RocksDB database; restoring
version ``N`` copies the matching ``keyring.<N>`` back into
``$mon_data`` so an older snapshot is paired with the keyring of its
vintage. Cleanup removes ``keyring.<N>`` files whose backup version
has been pruned.
Stashing the keyring is **best effort**: if the copy fails (for
example, permission denied on the backup directory or out of space
after the RocksDB snapshot completed), the backup is still recorded
as successful and the RocksDB data remains usable. On restore, a
missing ``keyring.<N>`` is silently skipped, and the operator must
supply the Monitor keyring out-of-band before starting the daemon.
.. warning::
The backup directory contains the ``[mon.]`` private key. Treat
it with the same access controls as ``$mon_data`` itself; a
complete backup is sufficient material to impersonate a Monitor
in the cluster.
Do not delete a ``private/<N>/`` directory or files under
``shared_checksum/`` by hand: removing a referenced shared file
corrupts every backup that points to it. Use ``backup_cleanup`` or
the configured retention parameters to remove old versions so the
BackupEngine can release shared files safely.
If the Monitor cluster fails and you need to copy a backup elsewhere,
copy the entire ``/path/to/backups/`` directory. Copying only
``private/<N>/`` is not sufficient; the version's SSTables live in
``shared_checksum/``.
The :confval:`mon_backup_cleanup_interval` specifies the interval for
backup cleanup. The cleanup algorithm keeps the last
``mon_backup_keep_last`` backups. It then collects hourly
``mon_backup_keep_hourly`` and daily ``mon_backup_keep_daily``
versions, retaining the newest backup in each time window.
You can trigger ``backup`` and ``backup_cleanup`` through any running
Monitor's admin socket.
.. prompt:: bash #
ceph --admin-daemon .../mon.asok backup
.. prompt:: bash #
ceph --admin-daemon .../mon.asok backup_cleanup
The following metrics related to the monitor backup process are
tracked by ``ceph-mon``.
.. list-table:: Ceph Monitor Backup Metrics
:widths: 30 12 58
:header-rows: 1
* - Name
- Type
- Description
* - ``backup_running``
- Gauge
- ``1`` while a backup is in progress, ``0`` otherwise
* - ``backup_started``
- Counter
- Backup attempts (includes attempts rejected at the :confval:`mon_backup_min_avail` pre-flight check)
* - ``backup_success``
- Counter
- Backups completed by the ``BackupEngine``
* - ``backup_failed``
- Counter
- Failed backup attempts (pre-flight or ``BackupEngine``)
* - ``backup_duration``
- Average
- Backup wall-clock time
* - ``backup_last_success``
- Gauge
- UTC timestamp of the most recent successful backup, or ``0`` if none
* - ``backup_last_success_id``
- Gauge
- ``BackupEngine`` version ID of the most recent successful backup (the value passed to ``--restore-backup --backup-version``)
* - ``backup_last_failed``
- Gauge
- UTC timestamp of the most recent failed attempt, or ``0`` if none
* - ``backup_last_size``
- Gauge
- Payload size in bytes of the most recent attempt (may be partial on failure)
* - ``backup_last_files``
- Gauge
- File count of the most recent attempt (may be partial on failure)
* - ``backup_cleanup_started``
- Counter
- Cleanup invocations
* - ``backup_cleanup_running``
- Gauge
- ``1`` while a cleanup is in progress, ``0`` otherwise
* - ``backup_cleanup_success``
- Counter
- Cleanup passes completed without error
* - ``backup_cleanup_failed``
- Counter
- Failed cleanup passes
* - ``backup_cleanup_duration``
- Average
- Cleanup wall-clock time
* - ``backup_cleanup_kept``
- Gauge
- Backups retained by the most recent cleanup pass
* - ``backup_cleanup_deleted``
- Gauge
- Backups removed by the most recent cleanup pass
* - ``backup_cleanup_freed``
- Gauge
- Bytes released by the most recent cleanup pass (overstated when shared backups are in use, because the ``BackupEngine`` payload sum ignores file sharing)
* - ``backup_cleanup_size``
- Gauge
- Bytes retained by the most recent cleanup pass (same sharing caveat as ``backup_cleanup_freed``)
The ``backup_started``/``backup_success``/``backup_failed`` and
``backup_cleanup_started``/``backup_cleanup_success``/``backup_cleanup_failed``
counters are monotonic and accumulate for the lifetime of the
``ceph-mon`` process. The ``backup_last_*`` fields and the four
cleanup result gauges (``backup_cleanup_kept``, ``backup_cleanup_deleted``,
``backup_cleanup_freed``, ``backup_cleanup_size``) describe only the
most recent invocation and are overwritten on every pass.
To retrieve backup metrics from a running monitor's admin socket:
.. prompt:: bash #
ceph --admin-daemon .../mon.asok perf dump | jq '.["mon"] | with_entries(select(.key | startswith("backup_")))'
.. code-block:: json
{
"backup_running": 0,
"backup_started": 2,
"backup_success": 2,
"backup_failed": 0,
"backup_duration": {
"avgcount": 2,
"sum": 0.149076498,
"avgtime": 0.074538249
},
"backup_last_success": 1722001989.849262,
"backup_last_success_id": 3,
"backup_last_failed": 0,
"backup_last_size": 3924677,
"backup_last_files": 6,
"backup_cleanup_started": 1,
"backup_cleanup_running": 0,
"backup_cleanup_success": 1,
"backup_cleanup_failed": 0,
"backup_cleanup_size": 86144,
"backup_cleanup_kept": 1,
"backup_cleanup_duration": {
"avgcount": 1,
"sum": 0.002031246,
"avgtime": 0.002031246
},
"backup_cleanup_freed": 0,
"backup_cleanup_deleted": 0
}
Monitor Backup Metric Usage Examples
------------------------------------
The following examples show how to use the monitor backup performance
counters. PromQL examples assume that the ``ceph-exporter`` is being
scraped by Prometheus. Admin-socket examples use ``ceph daemon``
directly against a specific ``ceph-mon`` daemon.
``backup_last_success`` (gauge, Unix epoch seconds)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
The timestamp when the most recent successful backup completed. ``0``
means no successful backup has occurred since the mon started.
* Age of the most recent successful backup, per mon:
.. code-block:: promql
time() - mon_backup_last_success
* Detect a stalled backup schedule (no success in over 2 hours; adjust
the threshold to roughly 2× your :confval:`mon_backup_interval`):
.. code-block:: promql
time() - mon_backup_last_success > 7200 and mon_backup_last_success > 0
``backup_failed`` (counter)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
Cumulative count of failed backup attempts (pre-flight or
``BackupEngine``) since the mon started.
* Backup failures per mon in the last hour:
.. code-block:: promql
increase(mon_backup_failed[1h])
* Alert when any mon has logged a backup failure recently:
.. code-block:: promql
increase(mon_backup_failed[15m]) > 0
``backup_last_size`` (gauge, bytes)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Payload size in bytes of the most recent backup attempt.
* Backup size trend across all mons:
.. code-block:: promql
mon_backup_last_size
* Live size check for a single mon via admin socket:
.. code-block:: bash
ceph daemon mon.<id> perf dump | jq '.mon.backup_last_size'
``backup_cleanup_kept`` (gauge)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Number of backups retained by the most recent cleanup pass.
* Mons whose retention has grown beyond an expected ceiling:
.. code-block:: promql
mon_backup_cleanup_kept > 100
``backup_cleanup_freed`` (gauge, bytes)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Bytes released by the most recent cleanup pass. Overstated when shared
backups are in use.
* Bytes reclaimed by the latest cleanup, per mon:
.. code-block:: promql
mon_backup_cleanup_freed
* Total bytes released by the most recent cleanup across all mons:
.. code-block:: promql
sum(mon_backup_cleanup_freed)
Monitor Backup Configuration Options
------------------------------------
The following options control monitor backup behavior. All are
runtime-tunable.
.. confval:: mon_backup_path
.. confval:: mon_backup_min_avail
.. confval:: mon_backup_keep_last
.. confval:: mon_backup_keep_hourly
.. confval:: mon_backup_keep_daily
.. confval:: mon_backup_interval
.. confval:: mon_backup_cleanup_interval
Miscellaneous
=============

33
doc/rados/operations/balancer.rst
View File

@ -39,6 +39,22 @@ combines the upmap balancer with the read balancer so that both writes
and reads are optimized. ``read`` mode can be used when only read optimization
is desired. For more details, see :ref:`read_balancer`.
Limitation: count-based balancing vs. size-based balancing
----------------------------------------------------------
Ceph's built-in balancer optimizes only by **PG shard count**, not by the
actual size of the data stored in each PG.
This can result in clusters whose OSDs are balanced by PG shard count,
but very imbalanced by stored Bytes.
At the pool level, this can cause a pool's ``%USED`` (from ``ceph df``)
to be much higher than the cluster's ``%RAW USED``, because the
pool's fullest OSD (which determines the pool's available space) may be
disproportionately loaded with large PGs.
Size-aware community balancers exist (for example,
`jj-balancer <https://github.com/TheJJ/ceph-balancer>`_).
Throttling
----------
@ -63,9 +79,22 @@ at the potential cost of greater impact on client operations.
There is a separate setting ``upmap_max_deviation`` for how uniform the
distribution of PGs must be for the module to consider the cluster adequately
balanced. At the time of writing (June 2025), this value defaults to ``5``,
which means that if a given OSD's PG replicas vary by five or fewer above or
below the cluster's average, it will be considered sufficiently balanced.
which means that if a given OSD's PG shard count deviates by five or fewer
from its weight-proportional target, it will be considered sufficiently
balanced.
More precisely, the balancer computes a per-OSD target shard count as::
target = osd_weight * (total_shards / total_weight)
where ``osd_weight`` is the OSD's CRUSH weight times its reweight
(the ``REWEIGHT`` value from ``ceph osd df``), ``total_shards`` is
``pool_size * pg_num`` summed over all balanced pools,
and ``total_weight`` is the sum of those per-OSD weights.
The deviation is then ``actual_shard_count - target``.
If no OSD's absolute deviation exceeds
``upmap_max_deviation``, the balancer considers the distribution
sufficiently balanced and makes no changes.
This value of PG replicas/shards (as distinct from logical PGs) is reported
by the ``ceph osd df`` command under the ``PGS`` column and the variance

20
doc/rados/operations/cache-tiering.rst
View File

@ -2,14 +2,16 @@
Cache Tiering
===============
.. warning:: Cache tiering has been deprecated in the Reef release. Cache
tiering has lacked a maintainer for a long time. This does not mean that
it will certainly be removed, but it might be removed without much
notice.
.. warning:: Cache tiering was deprecated in the Reef release and has lacked
a maintainer for a long time. It will be removed in a future release without
notice. **Do not deploy new cache tiers.** Migrate existing cache
tier deployments as soon as possible.
The upstream Ceph community strongly advises against deploying new cache
tiers. The upstream Ceph community also recommends migrating from legacy
deployments.
.. note:: This documentation is retained for existing deployments only.
If you need to remove a cache tier, see :ref:`cache-tiering-removal`.
Some community members have adopted ``dm-cache`` (the Linux kernel's
device-mapper cache target) as a block-level caching alternative, though
this is not an officially supported or endorsed configuration.
A cache tier provides Ceph clients with better I/O performance for a subset of
the data stored in a backing storage tier. Cache tiering involves creating a
@ -450,10 +452,12 @@ For example, to evict objects after 30 minutes, execute the following:
ceph osd pool set hot-storage cache_min_evict_age 1800
.. _cache-tiering-removal:
Removing a Cache Tier
=====================
Removing a cache tier differs depending on whether it is a writeback
Removing a cache tier differs depending on whether it is a writeback
cache or a read-only cache.

35
doc/rados/operations/erasure-code.rst
View File

@ -272,38 +272,41 @@ Erasure-coded pool overhead
---------------------------
The overhead factor (space amplification) of an erasure-coded pool
is `(k+m) / k`. For a 4,2 profile, the overhead is
is ``(k+m) / k``. For a ``4+2`` profile, the overhead is
thus 1.5, which means that 1.5 GiB of underlying storage is used to store
1 GiB of user data. Contrast with default replication with ``size-3``, with
which the overhead factor is 3.0. Do not mistake erasure coding for a free
1 GiB of user data. Contrast with default replication with ``size=3``, with
which the space amplification factor is ``3.0``. Do not mistake erasure coding for a free
lunch: there is a significant performance tradeoff, especially when using HDDs
and when performing cluster recovery or backfill.
Below is a table showing the overhead factors for various values of `k` and `m`.
As `k` increases above 4, the incremental capacity overhead gain quickly
Below is a table showing the overhead factors for various values of ``k`` and ``m``.
As ``k`` increases above ``4``, the incremental space amplification gain
experiences diminishing returns but the performance impact grows proportionally.
We recommend that you do not choose a profile with `k` > 4 or `m` > 2 unless
We recommend that you do not choose a profile with ``k`` > ``4` or ``m`` > 2 unless
and until you fully understand the ramifications, including the number of
failure domains your cluster topology presents. If you choose `m=1`,
failure domains your cluster topology presents. If you choose ``m=1``,
expect data unavailability during maintenance and data loss when component
failures overlap. Profiles with `m=1` are thus strongly discouraged for
failures overlap. Profiles with ``m=1`` are strongly discouraged for
production data.
Deployments that must remain active and avoid data loss when larger numbers
of overlapping component failure must be survived may favor a value of `m` > 2.
Note that such profiles result in lower space efficiency and lessened performance, especially
of overlapping component failure must be survived may favor a value of ``m`` > 2.
Note that such profiles may result in lower space efficiency and lessened performance, especially
during backfill and recovery.
If you are certain that you wish to use erasure coding for one or more pools but
are not certain which profile to use, select `k=4` and `m=2`. You will realize
double the usable space compared to replication with `size=3` with relatively
are not certain which profile to use, select ``4+2`` or ``6+3``. You will realize
double the usable space compared to replication with ``size=3`` with relatively
tolerable write and recovery performance impact.
.. note:: Most erasure-coded pool deployments require at least `k+m` CRUSH failure
domains, which in most cases means `rack`s or `hosts`. There are
.. note:: For a discussion of how durability varies across EC profiles, consult
`this Cephalocon presentation <https://www.youtube.com/watch?v=0y8WMu-hMBQ>`_ .
.. note:: Most erasure-coded pool deployments require at least ``k+m`` CRUSH failure
domains, which in most cases means racks or hosts. There are
operational advantages to planning EC profiles and cluster topology
so that there are at least `k+m+1` failure domains. In most cases
a value of `k` > 8 is discouragd.
so that there are at least ``k+m+1`` failure domains. In most cases
a value of ``k`` > 8 is discouraged.
.. note:: CephFS and RGW deployments with a significant proportion
of very small user files/objects may wish to plan carefully as

54
doc/rados/operations/placement-groups.rst
View File

@ -232,28 +232,48 @@ command:
For all but the very smallest deployments a value of 200 is recommended.
A value above 500 may result in excessive peering traffic and RAM usage.
The autoscaler analyzes pools and adjusts on a per-subtree basis. Because each
pool might map to a different CRUSH rule, and each rule might distribute data
across different and possibly overlapping sets of devices,
Ceph will consider the utilization of each subtree of
the CRUSH hierarchy independently. For example, a pool that maps to OSDs of class
``ssd`` and a pool that maps to OSDs of class ``hdd`` will each have calculated PG
counts that are determined by how many OSDs of these two different device types
there are.
.. _overlapping_crush_roots:
If a pool uses OSDs under two or more CRUSH roots (for example, shadow trees
with both ``ssd`` and ``hdd`` devices), the autoscaler issues a warning to the
user in the manager log. The warning states the name of the pool and the set of
roots that overlap each other. The autoscaler does not scale any pools with
overlapping roots because this condition can cause problems with the scaling
process. We recommend constraining each pool so that it belongs to only one
root (that is, one device OSD class) to silence the warning and ensure successful
scaling.
Overlapping CRUSH Roots PG Budget
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
When OSDs are distributed across multiple CRUSH roots, each root receives a PG target based
on its OSDs, with OSDs shared across multiple roots contributing proportionally less to each
root's allocation. The budget assigned to each root is:
.. math::
\sum_{\text{OSD}_i \in R} \frac{\text{mon_target_pg_per_osd}}{|\text{roots}(\text{OSD}_i)|}
This ensures that the total PG budget is distributed proportionally across all roots.
Consider a cluster with the following topology:
.. prompt:: bash #
mon_target_pg_per_osd = 300
- **rootid -1**: Contains OSDs {0, 1, 2, 3}
- **rootid -2**: Contains OSDs {0, 1}
- **rootid -3**: Contains OSDs {2, 3}
OSD membership:
- **OSD 0**: Belongs to roots {-1, -2}
- **OSD 1**: Belongs to roots {-1, -2}
- **OSD 2**: Belongs to roots {-1, -3}
- **OSD 3**: Belongs to roots {-1, -3}
The PG target allocation for each root is calculated as follows:
- Root -1: pg_target = 600 = (300 / 2 roots for OSD 0) + (300 / 2 roots for OSD 1) + (300 / 2 roots for OSD 2) + (300 / 2 roots for OSD 3)
- Root -2: pg_target = 300 = (300 / 2 roots for OSD 0) + (300 / 2 roots for OSD 1)
- Root -3: pg_target = 300 = (300 / 2 roots for OSD 2) + (300 / 2 roots for OSD 3)
.. _allocation_algorithm:
Allocation Algorithm
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
~~~~~~~~~~~~~~~~~~~~
The autoscaler sets each pool's ``final_pool_pg_target`` to be rounded to the
nearest power of two while ensuring that the number of PGs to be placed on each OSD

20
doc/rados/operations/require-min-compat-client.rst
View File

@ -112,7 +112,10 @@ The Linux kernel Ceph client (``krbd`` for RBD block devices, ``kcephfs``
for CephFS) ships with the kernel, not with Ceph packages, so its version
is independent of the cluster release. New OSD map features reach the
kernel client only after they are implemented and merged upstream, which
can lag the Ceph daemon release by months or years.
can lag the Ceph daemon release by months or years. The userspace clients
(``ceph-fuse`` and ``libcephfs`` for CephFS, ``librbd`` for RBD) link the
Ceph release they were built from and advertise its full feature set, so
they are not subject to this lag.
The *Min kernel* column shows the minimum Linux kernel version required on
kernel client hosts before the corresponding commands are used. If no
@ -164,6 +167,21 @@ kernel clients are present in the cluster, that column can be ignored.
See :ref:`read_balancer` for removal commands.
When a kernel mount blocks the flag
===================================
A single kernel mount can hold the whole cluster below the release you
want, even when every daemon and userspace client is already newer.
``ceph features`` reports such a mount as ``luminous`` because the kernel
client does not advertise the ``pg-upmap-primary`` (reef) feature. To
confirm which entry it adds, run ``ceph features`` with the mount
unmounted, then again with it mounted, and compare the client counts.
Since the userspace clients advertise the full feature set, switching the
holdout to its userspace equivalent (``ceph-fuse`` or ``libcephfs`` for
CephFS, ``librbd`` for RBD) clears the blocker, after which the flag can
be raised and the read balancer enabled.
See :ref:`upmap` and :ref:`read_balancer` for the operational procedures
that depend on raising the flag. CephFS clients must satisfy both this
flag and any per-filesystem :ref:`cephfs_required_client_features` on

91
doc/rados/troubleshooting/troubleshooting-mon.rst
View File

@ -518,6 +518,97 @@ or::
Corruption: 1 missing files; e.g.: /var/lib/ceph/mon/mon.foo/store.db/1234567.ldb
Recovery Using Mon Backup
-------------------------
If Monitor backups are enabled, backups can be found in the configured
``mon_backup_path``. To list the available backup versions, run:
.. code-block:: bash
ceph-mon -i [num] --list-backups /path/to/backups
In containerized deployments, run this from inside the Monitor
container (``cephadm shell --name mon.<id>``), or install
``ceph-common`` on the host.
This invokes the RocksDB ``BackupEngine`` to enumerate the logical
backup versions at the path. Output looks like::
ID: Time: Size:
1 Sun May 18 03:00:01 2026 4 MiB
2 Sun May 18 04:00:02 2026 12 KiB
3 Sun May 18 05:00:01 2026 16 KiB
The ``ID`` column is the value to pass as ``--backup-version`` when
restoring. A plain ``ls`` of the backup path shows the BackupEngine's
internal ``meta/``, ``private/``, and ``shared_checksum/`` directories,
which are not directly usable for restore; always use ``--list-backups``
to obtain the IDs.
To restore a backup, stop the monitor and run:
.. code-block:: bash
ceph-mon -i [num] --restore-backup /path/to/backups --backup-version <version> --yes-i-really-mean-it
The ``--yes-i-really-mean-it`` flag is required because restore overwrites the existing monitor store.
If the ``--backup-version`` argument is omitted, the latest version will be restored.
The restored store contains everything that was in the mon at the time the backup was taken,
including auth records; any changes (auth, pools, CRUSH, etc.) made after that point are lost.
OSDs will reconcile their state with the restored osdmap as the cluster comes back up.
If ``ceph-mon --restore-backup`` is invoked as ``root`` (typical when running
from a service shell), the restored ``kv_backend`` file and the rehydrated
``keyring`` will be owned by ``root``. Before starting the monitor daemon,
``chown -R ceph:ceph <mon_data>`` so the unprivileged ``ceph`` user can read
them.
Restoring a multi-monitor cluster
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Each monitor has its own Paxos state (rank, accepted proposal numbers,
``last_committed``), so backups are per-monitor: a backup taken from
``mon.a`` should be restored into ``mon.a``'s ``mon_data``, not copied
onto ``mon.b`` or ``mon.c``.
To recover a cluster from monitor backups:
#. Stop all monitors.
#. Restore each monitor you have a backup for from its own backup path
using the ``--restore-backup`` command above.
#. Start the restored monitors. Quorum forms once a majority of the
monmap members are running. Restoring and starting a majority is the
simplest path: the recovered monmap is the original one, and the
monitors elect among themselves as normal.
If you cannot bring back a majority of monitors (because backups are
missing or storage is lost on the other hosts), the restored seed will
not form quorum on its own. The monmap recovered from the backup still
lists every monitor in the original cluster, and Paxos requires a
majority to elect. The seed will sit in ``probing`` / ``electing``
indefinitely.
To reduce the monmap on the offline seed so it can elect itself:
#. Stop all monitors.
#. Restore the seed monitor from its backup as above.
#. Edit the monmap directly in the offline mon store::
# extract from the offline mon store
ceph-mon -i <seed-id> --extract-monmap /tmp/monmap
# drop monitors that will not come back
monmaptool /tmp/monmap --rm <other-id-1> --rm <other-id-2>
# write it back into the store
ceph-mon -i <seed-id> --inject-monmap /tmp/monmap
#. Start the seed monitor. With the reduced monmap it can elect itself
and form a single-member quorum.
#. Re-add any remaining monitors as new sync members per
:ref:`adding-and-removing-monitors`; they will synchronize from the
recovered seed rather than reusing their old backups.
Recovery Using Healthy Monitor(s)
---------------------------------

10
doc/radosgw/STS.rst
View File

@ -30,10 +30,10 @@ The following STS REST APIs have been implemented in Ceph Object Gateway:
**DurationSeconds** (Integer/ Optional): The duration in seconds of the session.
Its default value is 3600.
**ExternalId** (String/ Optional): A unique Id that might be used when a role is
**ExternalId** (String/ Optional): A unique ID that might be used when a role is
assumed in another account.
**SerialNumber** (String/ Optional): The Id number of the MFA device associated
**SerialNumber** (String/ Optional): The ID number of the MFA device associated
with the user making the AssumeRole call.
**TokenCode** (String/ Optional): The value provided by the MFA device, if the
@ -91,9 +91,9 @@ Similarly, an example of a policy that uses 'azp' claim in the condition is of t
"{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Federated\":[\"arn:aws:iam:::oidc-provider/<URL of IDP>\"]},\"Action\":[\"sts:AssumeRoleWithWebIdentity\"],\"Condition\":{\"StringEquals\":{\"<URL of IDP> :azp\":\"<azp>\"\}\}\}\]\}"
A shadow user is created corresponding to every federated user. The user id is derived from the 'sub' field of the incoming web token.
The user is created in a separate namespace - 'oidc' such that the user id doesn't clash with any other user ids in RGW. The format of the user id
is - ``<tenant>$<user-namespace>$<sub>`` where user-namespace is 'oidc' for users that authenticate with oidc providers.
A shadow user is created corresponding to every federated user. The user ID is derived from the ``sub`` field of the incoming web token.
The user is created in a separate namespace ``oidc`` such that the user ID doesn't clash with any other user IDs in RGW. The format of the user ID
is ``<tenant>$<user-namespace>$<sub>`` where ``user-namespace`` is ``oidc`` for users that authenticate with OIDC providers.
RGW now supports Session tags that can be passed in the web token to AssumeRoleWithWebIdentity call. More information related to Session Tags can be found here
:doc:`session-tags`.

19
doc/radosgw/adminops.rst
View File

@ -2708,7 +2708,7 @@ Quotas
======
The Admin Operations API enables you to set quotas on users and on buckets owned
by users, and on accounts and on buckets owned by accounts. See `Quota Management`_
by users, and on accounts and on buckets owned by accounts. See :ref:`radosgw-quota-management`
for additional details. Quotas include the maximum number of objects in a bucket
and the maximum storage size in megabytes.
@ -2717,7 +2717,7 @@ modify or disable a quota, the user must have ``users=write`` capability.
To view quotas for accounts, the user must have a ``accounts=read`` capability. To set,
modify or disable a quota, the user must have ``accounts=write`` capability.
See the `Admin Guide`_ for details.
See the :ref:`radosgw-admin-guide` for details.
Valid parameters for quotas include:
@ -2813,13 +2813,13 @@ permission. ::
Rate Limit
==========
The Admin Operations API enables you to set and get ratelimit configurations on users and on buckets and global rate limit configurations. See `Rate Limit Management`_ for additional details.
The Admin Operations API enables you to set and get ratelimit configurations on users and on buckets and global rate limit configurations. See :ref:`radosgw-rate-limit-management` for additional details.
Rate Limit includes the maximum number of operations and/or bytes per accumulation interval, separated by read and/or write (Additionally list and get operations),
to a bucket and/or by a user and the maximum storage size in megabytes.
To view rate limit, the user must have a ``ratelimit=read`` capability. To set,
modify or disable a ratelimit, the user must have ``ratelimit=write`` capability.
See the `Admin Guide`_ for details.
See the :ref:`radosgw-admin-guide` for details.
Valid parameters for quotas include:
@ -2939,14 +2939,12 @@ Dedup
=====
The Admin Operations API can be used to manage RGW object deduplication.
See `Full RGW Object Dedup`_ for additional details on the dedup feature and
See :ref:`radosgw-s3-dedup` for additional details on the dedup feature and
CLI commands.
.. _Full RGW Object Dedup: ../s3_objects_dedup
To view dedup status, the user must have ``dedup=read`` capability. To
control dedup operations, the user must have ``dedup=write`` capability.
See the `Admin Guide`_ for details.
See the :ref:`radosgw-admin-guide` for details.
Get Dedup Stats
~~~~~~~~~~~~~~~
@ -3165,12 +3163,7 @@ Binding libraries
.. _Admin Guide: ../admin
.. _Quota Management: ../admin#quota-management
.. _IrekFasikhov/go-rgwadmin: https://github.com/IrekFasikhov/go-rgwadmin
.. _QuentinPerez/go-radosgw: https://github.com/QuentinPerez/go-radosgw
.. _ceph/go-ceph: https://github.com/ceph/go-ceph/
.. _Rate Limit Management: ../admin#rate-limit-management
.. _IrekFasikhov/go-rgwadmin: https://github.com/IrekFasikhov/go-rgwadmin
.. _QuentinPerez/go-radosgw: https://github.com/QuentinPerez/go-radosgw
.. _twonote/radosgw-admin4j: https://github.com/twonote/radosgw-admin4j

2
doc/radosgw/config-ref.rst
View File

@ -58,6 +58,8 @@ instances or all radosgw-admin options can be put into the ``[global]`` or the
.. confval:: rgw_account_default_quota_max_objects
.. confval:: rgw_account_default_quota_max_size
.. confval:: rgw_verify_ssl
.. confval:: rgw_rest_conn_connect_to_resolved_ips
.. confval:: rgw_rest_conn_ip_fail_timeout_secs
.. confval:: rgw_max_chunk_size
.. confval:: rgw_multi_obj_del_max_aio

4
doc/radosgw/encryption.rst
View File

@ -160,6 +160,7 @@ Secrets are stored using the `Linux Kernel Key Retention Service`_ in
the RGW processes' process keyring. This is subject to a global quota
and must be set in accordance with the configured cache size.
Depending on whether RGW runs as root, these quotas can be managed by adjusting:
- ``/proc/sys/kernel/keys/root_maxkeys`` and ``/proc/sys/kernel/keys/root_maxbytes``
- ``/proc/sys/kernel/keys/maxkeys`` and ``/proc/sys/kernel/keys/maxbytes``
@ -167,6 +168,7 @@ Exceeding a quota will disable the cache, fail the request with an
internal error, and log a failure message.
Three different Cache Time-to-Live (TTL) values can be set:
- **Positive TTL**: How long a successfully retrieved secret remains
in the cache.
- **Negative TTL**: How long to remember that a key does not exist,
@ -178,6 +180,7 @@ Metrics
---------
The cache exports metrics under the ``kms-cache`` collection.
- ``hit``: Hit counter
- ``miss``: Miss counter
- ``expired``: Number of TTL expired entries
@ -187,6 +190,7 @@ The cache exports metrics under the ``kms-cache`` collection.
``miss``, ``expired``
In addition the ``rgw`` collection has:
- ``kms_fetch_lat``: Average KMS fetch latency. Also includes a
successful request counter. Each event results in a positive cache
entry.

18
doc/radosgw/metrics.rst
View File

@ -74,6 +74,12 @@ The following metrics related to S3 or Swift operations are tracked per Ceph Obj
* - list_bucket_lat
- Gauge
- Total latency of list bucket operations
* - head_obj_ops
- Counter
- Number of successful HEAD Object operations
* - head_obj_lat
- Gauge
- Total latency of HEAD Object operations
There are three different sections in the output of the ``counter dump`` and ``counter schema`` commands that show the op metrics and their information.
The sections are ``rgw_op``, ``rgw_op_per_user``, and ``rgw_op_per_bucket``.
@ -109,6 +115,12 @@ To view op metrics in the Ceph Object Gateway go to the ``rgw_op`` sections of t
"avgcount": 1,
"sum": 0.002300000,
"avgtime": 0.002300000
},
"head_obj_ops": 3,
"head_obj_lat": {
"avgcount": 3,
"sum": 0.006123456,
"avgtime": 0.002041152
}
}
},
@ -146,6 +158,12 @@ Op metrics can also be tracked per-user or per-bucket. These metrics are exporte
"avgcount": 1,
"sum": 0.002300000,
"avgtime": 0.002300000
},
"head_obj_ops": 3,
"head_obj_lat": {
"avgcount": 3,
"sum": 0.006123456,
"avgtime": 0.002041152
}
}
},

4
doc/radosgw/s3-notification-compatibility.rst
View File

@ -4,7 +4,7 @@
S3 Bucket Notifications Compatibility
=====================================
Ceph's `Bucket Notifications`_ API follows `AWS S3 Bucket Notifications API`_.
Ceph's :ref:`radosgw-notifications` API follows `AWS S3 Bucket Notifications API`_.
However, some differences exist, as listed below.
@ -193,5 +193,3 @@ We also have the following extensions to topic configuration:
.. _AWS Simple Notification Service API: https://docs.aws.amazon.com/sns/latest/api/API_Operations.html
.. _AWS S3 Bucket Notifications API: https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html
.. _Event Message Structure: https://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
.. _`Bucket Notifications`: ../notifications
.. _`boto3 SDK filter extensions`: https://github.com/ceph/ceph/tree/main/examples/rgw/boto3

1
doc/radosgw/s3.rst
View File

@ -17,6 +17,7 @@ API
Service Ops <s3/serviceops>
Bucket Ops <s3/bucketops>
Object Ops <s3/objectops>
S3 Control <s3/s3control>
C++ <s3/cpp>
C# <s3/csharp>
Java <s3/java>

26
doc/radosgw/s3_objects_dedup.rst
View File

@ -1,3 +1,5 @@
.. _radosgw-s3-dedup:
=====================
Full RGW Object Dedup
=====================
@ -158,19 +160,27 @@ Memory Usage
+------------------+----------+
| RGW Object Count | Memory |
+==================+==========+
| 1M | 8 MB |
| 1M | 8 MB |
+------------------+----------+
| 4M | 16 MB |
| 4M | 16 MB |
+------------------+----------+
| 16M | 32 MB |
| 16M | 32 MB |
+------------------+----------+
| 64M | 64 MB |
| 64M | 64 MB |
+------------------+----------+
| 256M | 128 MB |
| 256M | 128 MB |
+------------------+----------+
| 1024M (1G) | 256 MB |
| 1024M (1G) | 256 MB |
+------------------+----------+
| 4096M (4G) | 512 MB |
| 4096M (4G) | 512 MB |
+------------------+----------+
| 16384M (16G) | 1024 MB |
| 16384M (16G) | 1024 MB |
+------------------+----------+
| 65536M (64G) | 2048 MB |
+------------------+----------+
| 262144M (256G) | 4096 MB |
+------------------+----------+
.. note::
Pools with more than ~213 billion user objects (256B with headroom) exceed the
dedup system's maximum capacity and will be rejected at startup.

28
doc/rbd/rbd-live-migration.rst
View File

@ -157,8 +157,15 @@ as follows::
{
"type": "native",
["mon_host": "<mon-host>",] (specify if image in another cluster,
must be specified together with "key",
mutually exclusive with "cluster_name")
["key": "<key or key-ref>",] (for connecting to another cluster,
must be specified together with "mon_host",
mutually exclusive with "cluster_name")
["cluster_name": "<cluster-name>",] (specify if image in another cluster,
requires ``<cluster-name>.conf`` file)
requires ``<cluster-name>.conf`` file,
mutually exclusive with "mon_host" and "key")
["client_name": "<client-name>",] (for connecting to another cluster,
default is ``client.admin``)
"pool_name": "<pool-name>",
@ -182,6 +189,14 @@ it utilizes native Ceph operations. For example, to import from the image
"snap_name": "snap1"
}
.. note::
The ``key`` parameter supports storing the key in the MON config-key store
by utilizing ``config://`` prefix followed by the path in the MON config-key
store to the key (key reference). Values can be stored in the config-key store
via ``ceph config-key set <key-path> <value>`` (e.g.
``ceph config-key set rbd/native/my_key AQAz7EVWygILFRAAdIcuJ12opU/JKyfFmxhuaw==``
and ``"key": "config://rbd/native/my_key"`` in ``source-spec``).
The ``qcow`` format can be used to describe a QCOW (QEMU copy-on-write) block
device. Both the QCOW (v1) and QCOW2 formats are currently supported with the
exception of advanced features such as compression, encryption, backing
@ -280,8 +295,8 @@ The ``s3`` stream can be used to import from a remote S3 bucket. Its
"stream": {
"type": "s3",
"url": "<url-path>",
"access_key": "<access-key>",
"secret_key": "<secret-key>"
"access_key": "<access-key or access-key-ref>",
"secret_key": "<secret-key or secret-key-ref>"
}
}
@ -301,10 +316,11 @@ as follows::
.. note::
The ``access_key`` and ``secret_key`` parameters support storing the keys in
the MON config-key store by prefixing the key values with ``config://``
followed by the path in the MON config-key store to the value. Values can be
the MON config-key store by utilizing ``config://`` prefix followed by the
path in the MON config-key store to the key (key reference). Values can be
stored in the config-key store via ``ceph config-key set <key-path> <value>``
(e.g. ``ceph config-key set rbd/s3/access_key NX5QOQKC6BH2IDN8HC7A``).
(e.g. ``ceph config-key set rbd/s3/my_access_key NX5QOQKC6BH2IDN8HC7A`` and
``"access_key": "config://rbd/s3/my_access_key"`` in ``source-spec``).
The ``nbd`` stream can be used to import from a remote NBD export. Its
``source-spec`` JSON is encoded as follows::

4
doc/rbd/rbd-mirroring.rst
View File

@ -558,6 +558,10 @@ For example::
local cluster's ``rbd-mirror`` daemon process is responsible for performing
the resync asynchronously.
.. note:: For snapshot-based mirroring, resync replicates the image contents
only up to the most recent mirror-snapshot. Create a new mirror-snapshot on
the primary image to sync the latest updates.
Mirror Status
=============

4
doc/rbd/rbd-openstack.rst
View File

@ -117,8 +117,8 @@ Install Ceph client packages
On the ``glance-api`` node, you will need the Python bindings for ``librbd``::
sudo apt-get install python-rbd
sudo yum install python-rbd
sudo apt-get install python3-rbd
sudo yum install python3-rbd
On the ``nova-compute``, ``cinder-backup`` and on the ``cinder-volume`` node,
use both the Python bindings and the client command line tools::

1
doc/releases/index.rst
View File

@ -67,6 +67,7 @@ Release timeline
.. _Tentacle: tentacle
.. _20.2.0: tentacle#v20-2-0-tentacle
.. _20.2.1: tentacle#v20-2-1-tentacle
.. _20.2.2: tentacle#v20-2-2-tentacle
.. _Squid: squid
.. _19.2.0: squid#v19-2-0-squid

2
doc/releases/releases.yml
View File

@ -19,6 +19,8 @@ releases:
tentacle:
target_eol: 2027-11-18
releases:
- version: 20.2.2
released: 2026-06-16
- version: 20.2.1
released: 2026-04-06
- version: 20.2.0

248
doc/releases/tentacle.rst
View File

@ -4,6 +4,254 @@ Tentacle
Tentacle is the 20th stable release of Ceph.
v20.2.2 Tentacle
================
This is the second minor release in the Tentacle series. We recommend that all users update to this release.
Release Date
------------
June 16, 2026
Notable Changes
---------------
* Rocky 10 package-based installs are now supported starting with v20.2.2. Please see the `supported platforms <https://docs.ceph.com/en/latest/start/os-recommendations/#platforms>`_ for current and planned support in Ceph.
MDS (Metadata Server)
---------------------
* Segmentation fault fixed due to incorrect queueing of request retries.
OSD (Object Storage Daemon)
---------------------------
* PGLog Missed List: Fixed a bug to ensure the correct version is attached to the missing list when ignoring log entries.
* Data Integrity Asserts: Added assertions to explicitly catch potential corruption in the OSD missing list.
RGW (RADOS Gateway)
-------------------
* Lifecycle Management: Fixed lifecycle transition issues affecting encrypted multipart objects.
* REST & Query Handling: RESTArgs::get_string() now properly URL-decodes incoming query parameters.
RADOS / librados / neorados
---------------------------
* Linger Operations: Rewrote safety checks to eliminate use-after-free vulnerabilities and LingerOp memory leaks when an unwatch operation returns ENOTCONN. Replaced the flawed is_valid_watch() check with a safe linger_by_cookie() lookup which safely manages LingerOp references using intrusive_ptr. Ensured librados linger callbacks hold a persistent reference to LingerOp to protect against races with simultaneous linger_cancel() requests. Configured librados::IoCtxImpl::aio_unwatch() to asynchronously deliver ENOTCONN to AioCompletion instead of returning the error directly.
* Watch/Notify: Fixed neorados notification queue bounds enforcement so that an overflow marker is only appended on the first message exceeding capacity, rather than duplicating it on every subsequent message. Prevented double-cleanup triggers in watch/notify operations when incoming errors arrive after maybe_cleanup() runs. Fixed a bug where notify would lose original error values by ensuring it no longer attempts to decode empty responses. Enhanced io_context shutdown procedures to clear handlers and route cleanly through linger_cancel to avoid use-after-free states.
* Async Utilities: Fixed an issue regarding the improper removal of objects from the async service list.
Dashboard
-----------------
* NVMeoF Revamp: Complete UI overhaul with DHCHAP controller key support, namespace encryption, and secure listeners configuration.
* Pools & RGW: Added stretch cluster validation for pools, fixed RGW restart/stop issues, storage class restrictions, sync policy fixes, and MSR EC Profile support.
* NFS: Toggle visibility for CephFS snapshots, fixed export creation and path value consistency issues
ceph-volume
-----------
* Inventory Scanning: Enhanced inventory discovery logic to automatically skip RAM disk devices (/dev/ram*).
External Block Device (extblkdev)
---------------------------------
* Plugin Stability: Fixed an assertion failure in the FCM plugin when encountering multivolume devices.
Changelog
---------
* (tentacle) ceph-volume: backport PRs 67047 and 67240 (`pr#67343 <https://github.com/ceph/ceph/pull/67343>`_, Guillaume Abrioux, Parfait Detchenou)
* [tentacle] bluestore, extblkdev: Now plugins can raise health warnings (`pr#68663 <https://github.com/ceph/ceph/pull/68663>`_, Adam Kupczyk, Igor Fedotov, Martin Ohmacht)
* Backporting PRs 67236 and 67419 (`pr#67533 <https://github.com/ceph/ceph/pull/67533>`_, Adam King)
* Beacon diff + Stretched cluster (`pr#68347 <https://github.com/ceph/ceph/pull/68347>`_, Leonid Chernin, Samuel Just)
* ceph-volume: include LVM mapper devices in get_devices() (`pr#67989 <https://github.com/ceph/ceph/pull/67989>`_, Guillaume Abrioux)
* ceph-volume: skip /dev/ram\* devices in inventory (`pr#68552 <https://github.com/ceph/ceph/pull/68552>`_, Ujjawal Anand)
* ceph-volume: skip mkfs discard for LVM NVMe OSDs (`pr#68286 <https://github.com/ceph/ceph/pull/68286>`_, Ujjawal Anand)
* ceph-volume: skip redundant NVMe mkfs discards (`pr#67341 <https://github.com/ceph/ceph/pull/67341>`_, Ujjawal Anand)
* ceph-volume: skip virtual cdrom devices in inventory (`pr#68108 <https://github.com/ceph/ceph/pull/68108>`_, Ujjawal Anand)
* ceph.spec.in: replace golang github prometheus with promtool binary path (`pr#68420 <https://github.com/ceph/ceph/pull/68420>`_, Nizamudeen A)
* ceph_mon: Fix shutdown order to destroy Monitor before closing mon store (`pr#68399 <https://github.com/ceph/ceph/pull/68399>`_, Prashant D)
* cephadm: wait for latest osd map after ceph-volume before OSD deploy (`pr#68379 <https://github.com/ceph/ceph/pull/68379>`_, Guillaume Abrioux)
* cephfs: MDCache request cleanup (`pr#66469 <https://github.com/ceph/ceph/pull/66469>`_, Abhishek Lekshmanan)
* Check if `HTTP_X_AMZ_COPY_SOURCE` header is empty (`pr#66027 <https://github.com/ceph/ceph/pull/66027>`_, Suyash Dongre)
* client: adjust `Fb` cap ref count check during synchronous fsync() (`issue#73624 <http://tracker.ceph.com/issues/73624>`_, `pr#65913 <https://github.com/ceph/ceph/pull/65913>`_, Venky Shankar)
* client: crash caused by invalid iterator in _readdir_cache_cb (`pr#65957 <https://github.com/ceph/ceph/pull/65957>`_, Zhansong Gao)
* container/build.sh: add 'rocky-10' suffix if necessary (`pr#67895 <https://github.com/ceph/ceph/pull/67895>`_, Dan Mick)
* container/build.sh: FROM_IMAGE=rockylinux-10 default for >=tentacle backports (`pr#67960 <https://github.com/ceph/ceph/pull/67960>`_, Matan Breizman, David Galloway, John Mulligan, Dan Mick)
* debian: package mgr/smb in ceph-mgr-modules-core (`pr#67510 <https://github.com/ceph/ceph/pull/67510>`_, Roland Sommer)
* debian: remove invoke-rc.d calls from postrm scripts (`pr#67354 <https://github.com/ceph/ceph/pull/67354>`_, Kefu Chai)
* debian: remove stale distutils override from py3dist-overrides (`pr#68276 <https://github.com/ceph/ceph/pull/68276>`_, Thomas Lamprecht, Max R. Carrara, Kefu Chai)
* doc: Batch backport for start/hardware-recommendations.rst (`pr#67907 <https://github.com/ceph/ceph/pull/67907>`_, Anthony D'Atri, Marc Methot, Pierre Riteau, Ville Ojamo)
* extblkdev: Fix FCM plugin asserting on multivolume devices (`pr#68877 <https://github.com/ceph/ceph/pull/68877>`_, Adam Kupczyk)
* found duplicate series for the match group {fs_id="-1"} (`pr#68369 <https://github.com/ceph/ceph/pull/68369>`_, bst2002git)
* Implement Admin REST APIs for Setting Account Quota (`pr#66905 <https://github.com/ceph/ceph/pull/66905>`_, Nicholas Liu, Jiffin Tony Thottan)
* kv/RocksDB: Add instrumentation to BinnedLRUCache (`pr#67349 <https://github.com/ceph/ceph/pull/67349>`_, Adam Kupczyk)
* libcephsqlite: ensure atexit handlers are registered after openssl (`pr#68263 <https://github.com/ceph/ceph/pull/68263>`_, Patrick Donnelly)
* librbd/cache/pwl: WriteLogOperationSet::cell can be garbage (`pr#67705 <https://github.com/ceph/ceph/pull/67705>`_, Ilya Dryomov)
* librbd/migration/QCOWFormat: avoid use-after-free in execute_request() (`pr#68188 <https://github.com/ceph/ceph/pull/68188>`_, Ilya Dryomov)
* librbd/mirror: detect trashed snapshots in UnlinkPeerRequest (`pr#67583 <https://github.com/ceph/ceph/pull/67583>`_, Ilya Dryomov)
* librbd: avoid losing sparseness in read_parent() (`pr#68463 <https://github.com/ceph/ceph/pull/68463>`_, Ilya Dryomov)
* librbd: don't complete ImageUpdateWatchers::shut_down() prematurely (`pr#67581 <https://github.com/ceph/ceph/pull/67581>`_, Ilya Dryomov)
* librbd: rbd_aio_write_with_crc32c (`pr#68038 <https://github.com/ceph/ceph/pull/68038>`_, Alexander Indenbaum)
* mds: add ref counting to LogSegment (`pr#68439 <https://github.com/ceph/ceph/pull/68439>`_, Milind Changire)
* mds: add retry request to MDSRank wait queue rather via finisher (`issue#76031 <http://tracker.ceph.com/issues/76031>`_, `pr#68905 <https://github.com/ceph/ceph/pull/68905>`_, Venky Shankar)
* mds: scrub pins more inodes than the mds_cache_memory_limit (`pr#67808 <https://github.com/ceph/ceph/pull/67808>`_, Md Mahamudur Rahaman Sajib)
* mds: use SimpleLock::WAIT_ALL for wait mask (`pr#68313 <https://github.com/ceph/ceph/pull/68313>`_, Patrick Donnelly)
* mgr/cephadm: Add KMIP server support for NVMeoF gateway (`pr#68086 <https://github.com/ceph/ceph/pull/68086>`_, Gil Bregman)
* mgr/cephadm: add rbd_with_crc32c parameter to nvmeof service spec (`pr#66933 <https://github.com/ceph/ceph/pull/66933>`_, Alexander Indenbaum)
* mgr/cephadm: fix mgmt-gateway startup on IPv6 VIP (`pr#68387 <https://github.com/ceph/ceph/pull/68387>`_, Kobi Ginon)
* mgr/cephadm: include cluster FSID in root CA Common Name (CN) (`pr#64692 <https://github.com/ceph/ceph/pull/64692>`_, Redouane Kachach, Kushal Deb)
* mgr/cephadm: serialize OSD class before returning for OSD rm status (`pr#69226 <https://github.com/ceph/ceph/pull/69226>`_, Adam King)
* mgr/DaemonServer: Implement ok-to-upgrade command (`pr#66948 <https://github.com/ceph/ceph/pull/66948>`_, Sridhar Seshasayee)
* mgr/DaemonServer: Limit search for OSDs to upgrade within the crush bucket (`pr#68350 <https://github.com/ceph/ceph/pull/68350>`_, Sridhar Seshasayee)
* mgr/dashboard : Add bottom padding for dashboard screens (`pr#68523 <https://github.com/ceph/ceph/pull/68523>`_, Abhishek Desai)
* mgr/dashboard : add stretch cluster validation for pools form (`pr#68476 <https://github.com/ceph/ceph/pull/68476>`_, Afreen Misbah, Abhishek Desai)
* mgr/dashboard : Fix RGW restart/stop issue (`pr#68554 <https://github.com/ceph/ceph/pull/68554>`_, Abhishek Desai)
* mgr/dashboard : fix-non-versioning-bucket-issue (`pr#68520 <https://github.com/ceph/ceph/pull/68520>`_, Abhishek Desai)
* mgr/dashboard : Fixes EC profile used pool empty (`pr#68730 <https://github.com/ceph/ceph/pull/68730>`_, Abhishek Desai)
* mgr/dashboard : Restrict create storage class with existing name (`pr#68475 <https://github.com/ceph/ceph/pull/68475>`_, Abhishek Desai)
* mgr/dashboard: [RGW-NFS]: User level export creation via UI fails with 500 - Internal Server Error (`pr#67013 <https://github.com/ceph/ceph/pull/67013>`_, Dnyaneshwari Talwekar)
* mgr/dashboard: [snap-visibility]Edit Client config option remains stuck in loading when nfs user is configured (`pr#68542 <https://github.com/ceph/ceph/pull/68542>`_, Dnyaneshwari Talwekar)
* mgr/dashboard: [storage-class]: Deleting local storage class from UI does not remove its entry from zone (`pr#67949 <https://github.com/ceph/ceph/pull/67949>`_, Dnyaneshwari Talwekar)
* mgr/dashboard: Add DHCHAP controller key to NVME host commands (`pr#67600 <https://github.com/ceph/ceph/pull/67600>`_, Gil Bregman)
* mgr/dashboard: add helper text to bucket form > policy and other spacing fixes (`pr#67871 <https://github.com/ceph/ceph/pull/67871>`_, Naman Munet)
* mgr/dashboard: Add location to gateway info command in NVMeoF CLI (`pr#68345 <https://github.com/ceph/ceph/pull/68345>`_, Gil Bregman)
* mgr/dashboard: Add namespace encryption support to NVMeoF CLI (`pr#68339 <https://github.com/ceph/ceph/pull/68339>`_, Gil Bregman)
* mgr/dashboard: Add nvmeof_top_cli service (`pr#67566 <https://github.com/ceph/ceph/pull/67566>`_, Vallari Agrawal)
* mgr/dashboard: Add option to edit zone with keys/ (`pr#68317 <https://github.com/ceph/ceph/pull/68317>`_, Aashish Sharma)
* mgr/dashboard: Add option to set motd via api (`pr#68678 <https://github.com/ceph/ceph/pull/68678>`_, Aashish Sharma)
* mgr/dashboard: Add overview page (`pr#67840 <https://github.com/ceph/ceph/pull/67840>`_, Afreen Misbah, Devika Babrekar, Aashish Sharma, Abhishek Desai, Naman Munet, Nizamudeen A)
* mgr/dashboard: Add port and secure-listeners to subsystem add NVMeoF CLI command (`pr#68370 <https://github.com/ceph/ceph/pull/68370>`_, Vallari Agrawal, Gil Bregman)
* mgr/dashboard: Add restore events in notification screen (`pr#67912 <https://github.com/ceph/ceph/pull/67912>`_, pujashahu, pujaoshahu)
* mgr/dashboard: Add secure and verify-host-name to "listener add" on NVMeoF CLI (`pr#67799 <https://github.com/ceph/ceph/pull/67799>`_, Gil Bregman)
* mgr/dashboard: Adding rados ns option into add_ns_req (`pr#67470 <https://github.com/ceph/ceph/pull/67470>`_, gadi-didi)
* mgr/dashboard: Allow empty port value when adding a listener in NVMEoF CLI (`pr#68766 <https://github.com/ceph/ceph/pull/68766>`_, Gil Bregman)
* mgr/dashboard: Batch backport nvmeof revamp (`pr#67839 <https://github.com/ceph/ceph/pull/67839>`_, Afreen Misbah, Nizamudeen A, Sagar Gopale, pujaoshahu, Puja Shahu, Ville Ojamo)
* mgr/dashboard: Bump lodash (`pr#68695 <https://github.com/ceph/ceph/pull/68695>`_, Afreen Misbah)
* mgr/dashboard: bump nvmeof submodule to 1.6.5 (`pr#67326 <https://github.com/ceph/ceph/pull/67326>`_, Vallari Agrawal, Tomer Haskalovitch)
* mgr/dashboard: carbonize-osd-flags-modal (`pr#68133 <https://github.com/ceph/ceph/pull/68133>`_, Afreen Misbah, Sagar Gopale)
* mgr/dashboard: Difference in "path" value observed when rgw user level export created via dashboard vs cli (`pr#68583 <https://github.com/ceph/ceph/pull/68583>`_, Dnyaneshwari Talwekar)
* mgr/dashboard: fix subvolume group corruption from smb share form (`pr#68103 <https://github.com/ceph/ceph/pull/68103>`_, Nizamudeen A)
* mgr/dashboard: Fix tags in subvolume list and subvolume groups list (`pr#68382 <https://github.com/ceph/ceph/pull/68382>`_, pujaoshahu)
* mgr/dashboard: Making 'ISA' as default plugin for EC profiles created through dashboard (`pr#68373 <https://github.com/ceph/ceph/pull/68373>`_, Devika Babrekar)
* mgr/dashboard: mgr/dashboard: Carbonize Realm Name and Token block in Multi-site Replication Wizard (`pr#68546 <https://github.com/ceph/ceph/pull/68546>`_, Sagar Gopale)
* mgr/dashboard: Misleading warning when no eligible devices are available for OSD creation (`pr#67637 <https://github.com/ceph/ceph/pull/67637>`_, Naman Munet)
* mgr/dashboard: nfs export creation fails with obj deserialization (`pr#67564 <https://github.com/ceph/ceph/pull/67564>`_, Nizamudeen A)
* mgr/dashboard: NFS: Toggle visibility of CephFS snapshots (`pr#67636 <https://github.com/ceph/ceph/pull/67636>`_, Afreen Misbah, Dnyaneshwari Talwekar)
* mgr/dashboard: Option to select archive option while Import Multi-site Token (`pr#68513 <https://github.com/ceph/ceph/pull/68513>`_, Aashish Sharma)
* mgr/dashboard: remove sync_from entry when sync_from_all is true (`pr#68536 <https://github.com/ceph/ceph/pull/68536>`_, Aashish Sharma)
* mgr/dashboard: Rename Alert breadcrumb to Alert Rules (`pr#68238 <https://github.com/ceph/ceph/pull/68238>`_, Sagar Gopale)
* mgr/dashboard: sync policy created for a bucket in Object >> Multi-site >> Sync-policy, is not reflecting under bucket's replication (`pr#68512 <https://github.com/ceph/ceph/pull/68512>`_, Naman Munet)
* mgr/dashboard:Adding MSR EC Profile via dashboard (`pr#68349 <https://github.com/ceph/ceph/pull/68349>`_, Devika Babrekar)
* mgr/smb: fix error handling for fundamental resource parsing (`pr#65861 <https://github.com/ceph/ceph/pull/65861>`_, John Mulligan)
* mgr/test_orchestrator: fixing daemon_action method signature (`pr#69231 <https://github.com/ceph/ceph/pull/69231>`_, Redouane Kachach)
* mgr: add config to load modules in main interpreter instead of subinterpreter (`pr#67515 <https://github.com/ceph/ceph/pull/67515>`_, Nitzan Mordhai, Nitzan Mordechai, Samuel Just)
* mgr: ensure that all modules have started before advertising active mgr (`pr#67850 <https://github.com/ceph/ceph/pull/67850>`_, Laura Flores)
* mgr: fix continous smb MgrDBNotReady (`pr#68598 <https://github.com/ceph/ceph/pull/68598>`_, Pedro Gonzalez Gomez)
* mgr: fix PyObject\* refcounting in TTLCache and cleanup logic (`pr#66482 <https://github.com/ceph/ceph/pull/66482>`_, Nitzan Mordechai)
* mgr: guard close_section calls in get_perf_schema_python (`pr#69436 <https://github.com/ceph/ceph/pull/69436>`_, Lumir Sliva)
* mgr: isolated CherryPy to prevent global state sharing (`pr#67465 <https://github.com/ceph/ceph/pull/67465>`_, Nizamudeen A, Anmol Babu)
* mon [stretch-mode]: Allow a max bucket weight diff threshold (`pr#67790 <https://github.com/ceph/ceph/pull/67790>`_, Kamoltat Sirivadhna, Kamoltat (Junior) Sirivadhna)
* mon/AuthMonitor: add osd w cap for superuser client (`pr#68314 <https://github.com/ceph/ceph/pull/68314>`_, Venky Shankar, Patrick Donnelly)
* monitoring: Fix application overview to show Raw used (`pr#68794 <https://github.com/ceph/ceph/pull/68794>`_, Ankush Behl)
* mr/dashboard: remove rgw_servers filter from radosgw-sync-overview grafana dashboard (`pr#68604 <https://github.com/ceph/ceph/pull/68604>`_, Aashish Sharma)
* neorados: Fix Neorados CephContext leak (`pr#67513 <https://github.com/ceph/ceph/pull/67513>`_, Adam C. Emerson)
* neorados: specify alignments for aligned_storage (`pr#67512 <https://github.com/ceph/ceph/pull/67512>`_, Casey Bodley)
* neorados: Various Fixes to Watch/Notify (`pr#68776 <https://github.com/ceph/ceph/pull/68776>`_, Adam C. Emerson, Casey Bodley)
* node-proxy: major refactor and various fixes (`pr#67418 <https://github.com/ceph/ceph/pull/67418>`_, Guillaume Abrioux)
* nvmeof: Change the NVMEOF image version to 1.6 (`pr#68415 <https://github.com/ceph/ceph/pull/68415>`_, Gil Bregman)
* nvmeofgw: fix issue of delete all gws from the pool/group (`pr#67942 <https://github.com/ceph/ceph/pull/67942>`_, Leonid Chernin)
* orch/cephadm: fix osd.default creation (`pr#68121 <https://github.com/ceph/ceph/pull/68121>`_, Guillaume Abrioux)
* os/bluestore: track compression\_\*blob_size\* parameters for online update (`pr#67888 <https://github.com/ceph/ceph/pull/67888>`_, Igor Fedotov)
* os/bluestore:fix bluestore_volume_selection_reserved_factor usage (`pr#66837 <https://github.com/ceph/ceph/pull/66837>`_, Igor Fedotov)
* osd/scrub: support an operator-abort command (`pr#67031 <https://github.com/ceph/ceph/pull/67031>`_, Ronen Friedman)
* osd: add pg-upmap-primary to clean_pg_upmaps (`pr#67407 <https://github.com/ceph/ceph/pull/67407>`_, Laura Flores)
* osd: Allow multiple objects with same version in missing list (`pr#69450 <https://github.com/ceph/ceph/pull/69450>`_, Alex Ainscow)
* osd: Avoid assertion on empty object read when reading multiple objects (`pr#68714 <https://github.com/ceph/ceph/pull/68714>`_, Alex Ainscow)
* osd: Avoid pwlc spanning intervals (`pr#68708 <https://github.com/ceph/ceph/pull/68708>`_, Bill Scales)
* osd: Change rmissing map key from version_t to eversion_t (`pr#68716 <https://github.com/ceph/ceph/pull/68716>`_, Alex Ainscow)
* osd: Deleting PG should discard pwlc (`pr#68709 <https://github.com/ceph/ceph/pull/68709>`_, Bill Scales)
* osd: FastEC: always update pwlc epoch when activating (`pr#68710 <https://github.com/ceph/ceph/pull/68710>`_, Bill Scales)
* osd: Fix bug when calculating min_peer_features (`pr#69159 <https://github.com/ceph/ceph/pull/69159>`_, Bill Scales)
* osd: Fix incorrect rollback logic for partial write OI (`pr#68715 <https://github.com/ceph/ceph/pull/68715>`_, Alex Ainscow)
* osd: PGLog Attach correct version to missing list when ignoring log entries (`pr#68718 <https://github.com/ceph/ceph/pull/68718>`_, Alex Ainscow)
* osd: Twiddle should create a full sized vector for optimized EC (`pr#68717 <https://github.com/ceph/ceph/pull/68717>`_, Alex Ainscow)
* pybind/mgr: call new _ceph_exit for killpoints (`pr#68518 <https://github.com/ceph/ceph/pull/68518>`_, Patrick Donnelly)
* pybind/mgr: update modules to use independent CLICommand subtypes with distinct COMMAND attributes (`pr#67511 <https://github.com/ceph/ceph/pull/67511>`_, Kefu Chai, Samuel Just)
* qa/cephadm: derive container image from cephadm release (`pr#68328 <https://github.com/ceph/ceph/pull/68328>`_, Lumir Sliva)
* qa/cephfs: lua to respect missing kernel in yaml (`pr#67293 <https://github.com/ceph/ceph/pull/67293>`_, Kyr Shatskyy)
* qa/cephfs: treat "implicit declaration of function" for blogbench workunit for newer gcc version (`issue#75380 <http://tracker.ceph.com/issues/75380>`_, `pr#68820 <https://github.com/ceph/ceph/pull/68820>`_, Venky Shankar)
* qa/distros: add rocky 10.0 as supported distro/container host (`pr#68569 <https://github.com/ceph/ceph/pull/68569>`_, Patrick Donnelly, Casey Bodley, Adam King, David Galloway)
* qa/radosgw_admin: replace boto2 with boto3 (`pr#68739 <https://github.com/ceph/ceph/pull/68739>`_, Adam C. Emerson, Casey Bodley)
* qa/rgw/multisite: remove duplicate test_suspended_delete_marker_incremental_sync (`pr#68846 <https://github.com/ceph/ceph/pull/68846>`_, Oguzhan Ozmen)
* qa/rgw/upgrade: symlinks are explicit about distro versions (`pr#68057 <https://github.com/ceph/ceph/pull/68057>`_, Casey Bodley)
* qa/rgw: Revive crypt kmip (`pr#68371 <https://github.com/ceph/ceph/pull/68371>`_, Kyr Shatskyy)
* qa/suites/fs: fix extraneous distro links (`pr#69251 <https://github.com/ceph/ceph/pull/69251>`_, Patrick Donnelly)
* qa/suites/upgrade: ignore osd in unknown state (`pr#69307 <https://github.com/ceph/ceph/pull/69307>`_, Patrick Donnelly)
* qa/suites/upgrade: ignore undersized PG during stress splits (`pr#69310 <https://github.com/ceph/ceph/pull/69310>`_, Patrick Donnelly)
* qa/suites/upgrade: update upgrade paths and exclude rocky10 from non-supported distros (`pr#68660 <https://github.com/ceph/ceph/pull/68660>`_, Yaarit Hatuka, Laura Flores)
* qa/suites: remove centos restriction from valgrind yaml (`issue#18126 <http://tracker.ceph.com/issues/18126>`_, `issue#20360 <http://tracker.ceph.com/issues/20360>`_, `pr#67519 <https://github.com/ceph/ceph/pull/67519>`_, Samuel Just)
* qa/suites: use tagged version of reef (`pr#68357 <https://github.com/ceph/ceph/pull/68357>`_, Laura Flores)
* qa/tasks/backfill_toofull.py: Fix assert failures with & without compression (`pr#68118 <https://github.com/ceph/ceph/pull/68118>`_, Sridhar Seshasayee)
* qa/tasks/keystone: upgrade keystone to 2025.2 (`pr#67757 <https://github.com/ceph/ceph/pull/67757>`_, Kyr Shatskyy)
* qa/tasks/quiescer: remove racy assertion (`pr#68510 <https://github.com/ceph/ceph/pull/68510>`_, Patrick Donnelly)
* qa/tasks: capture CommandCrashedError when running nvme list cmd (`pr#69232 <https://github.com/ceph/ceph/pull/69232>`_, Redouane Kachach)
* qa/tasks: make rbd_mirror_thrash inherit from ThrasherGreenlet (`pr#67795 <https://github.com/ceph/ceph/pull/67795>`_, Ilya Dryomov)
* qa/tasks: update egrep to 'grep -E' (`pr#67518 <https://github.com/ceph/ceph/pull/67518>`_, Nitzan Mordechai, Samuel Just)
* qa/workunits/ceph-helpers-root: Add Rocky support for install packages (`pr#67507 <https://github.com/ceph/ceph/pull/67507>`_, Nitzan Mordechai)
* qa/workunits/rbd: drop racy assert in test_tasks_recovery() (`pr#68190 <https://github.com/ceph/ceph/pull/68190>`_, Ilya Dryomov)
* qa/workunits: Add updated kernel archive URL (`pr#68169 <https://github.com/ceph/ceph/pull/68169>`_, Brad Hubbard)
* qa: add 'refresh' config to cephadm.wait_for_service (`pr#67038 <https://github.com/ceph/ceph/pull/67038>`_, Vallari Agrawal)
* qa: add MDS_INSUFFICIENT_STANDBY to ignorelist (`pr#69036 <https://github.com/ceph/ceph/pull/69036>`_, Patrick Donnelly)
* qa: Add nvmeof upgrade from v20.2.0 (`pr#68149 <https://github.com/ceph/ceph/pull/68149>`_, Vallari Agrawal)
* qa: allow multiple mgr sessions during eviction test (`pr#68316 <https://github.com/ceph/ceph/pull/68316>`_, Patrick Donnelly)
* qa: cephfs suite changes for rocky (`pr#68374 <https://github.com/ceph/ceph/pull/68374>`_, Patrick Donnelly)
* qa: Fix coredumps caused by udisks (`pr#67526 <https://github.com/ceph/ceph/pull/67526>`_, Vallari Agrawal)
* qa: Fix nvmeof 'errors during thrashing' (`pr#68148 <https://github.com/ceph/ceph/pull/68148>`_, Vallari Agrawal)
* qa: fix setting rbd_sparse_read_threshold_bytes in test_migration_clone() (`pr#68617 <https://github.com/ceph/ceph/pull/68617>`_, Ilya Dryomov)
* qa: fix TypeError in delay (`pr#67617 <https://github.com/ceph/ceph/pull/67617>`_, Jos Collin)
* qa: fixing cephadm mgmt-gateway test to remove openssl dependency (`pr#67820 <https://github.com/ceph/ceph/pull/67820>`_, Redouane Kachach)
* qa: ignore cephadm failed daemon warnings during thrashing (`pr#69309 <https://github.com/ceph/ceph/pull/69309>`_, Patrick Donnelly)
* qa: ignore POOL_FULL for rbd tests exercising full pools (`pr#69304 <https://github.com/ceph/ceph/pull/69304>`_, Patrick Donnelly)
* qa: install nvme-cli only if distro remains rocky10 (`pr#69252 <https://github.com/ceph/ceph/pull/69252>`_, Patrick Donnelly)
* qa: krbd_rxbounce.sh: do more reads to generate more errors (`pr#67455 <https://github.com/ceph/ceph/pull/67455>`_, Ilya Dryomov)
* qa: Leak_StillReachable RocksDB error_handler (`pr#68524 <https://github.com/ceph/ceph/pull/68524>`_, Nitzan Mordechai)
* qa: rbd_mirror_fsx_compare.sh doesn't error out as expected (`pr#67797 <https://github.com/ceph/ceph/pull/67797>`_, Ilya Dryomov)
* qa: Remove cephadm e2e tests from teuthology (`pr#68818 <https://github.com/ceph/ceph/pull/68818>`_, Afreen Misbah)
* qa: resolve py3.12 regression for random.sample (`pr#68315 <https://github.com/ceph/ceph/pull/68315>`_, Patrick Donnelly)
* qa: suppress false positive delete map mismatch errors (`pr#68431 <https://github.com/ceph/ceph/pull/68431>`_, Casey Bodley, Nitzan Mordechai)
* qa: suppress MismatchedFree operator delete RocksDB (`pr#67508 <https://github.com/ceph/ceph/pull/67508>`_, Nitzan Mordechai)
* rgw/beast: use strand executor for timeout timer to prevent concurrent socket access (`pr#68506 <https://github.com/ceph/ceph/pull/68506>`_, Oguzhan Ozmen)
* rgw/lc: Do not delete DM if its at end of pagination list (`pr#67573 <https://github.com/ceph/ceph/pull/67573>`_, kchheda3)
* rgw/multisite: check the local bucket's versioning status when replicating deletion from remote (`pr#66168 <https://github.com/ceph/ceph/pull/66168>`_, Jane Zhu)
* RGW/multisite: fix bucket-full-sync infinite loop caused by stale bucket_list_result reuse (`pr#67923 <https://github.com/ceph/ceph/pull/67923>`_, Oguzhan Ozmen)
* RGW/Multisite: fix uninitialized LatencyMonitor causing spurious "OSD cluster is overloaded" warning (`pr#68803 <https://github.com/ceph/ceph/pull/68803>`_, Oguzhan Ozmen)
* rgw/s3: Always include x-amz-content-sha256 header in AWS v4 signatures (`pr#66358 <https://github.com/ceph/ceph/pull/66358>`_, Shilpa Jagannath, Matthew N. Heler)
* rgw/tests: add os-specific java 1.7 install commands to keycloak task (`pr#67982 <https://github.com/ceph/ceph/pull/67982>`_, J. Eric Ivancich)
* rgw/website: preserve nameservers for future use in dnsmasq (`pr#67061 <https://github.com/ceph/ceph/pull/67061>`_, Kyr Shatskyy)
* rgw/zone: remove duplicated startup logic in RGWSI_Zone (`pr#66300 <https://github.com/ceph/ceph/pull/66300>`_, Casey Bodley)
* rgw: bucket logging fixes (`pr#66769 <https://github.com/ceph/ceph/pull/66769>`_, Nithya Balachandran, N Balachandran, Yuval Lifshitz, Casey Bodley)
* RGW: Change prerequest hook to run after authorization process (`pr#68594 <https://github.com/ceph/ceph/pull/68594>`_, Emin Sunacoglu)
* rgw: discard olh\_ attributes when copying object from a versioning-suspended bucket to a versioning-disabled bucket (`pr#65557 <https://github.com/ceph/ceph/pull/65557>`_, Jane Zhu)
* rgw: fix 'bucket stats' when bucket index doesn't exist (`pr#68505 <https://github.com/ceph/ceph/pull/68505>`_, Casey Bodley)
* rgw: fix lifecycle transition of encrypted multipart objects (`pr#68826 <https://github.com/ceph/ceph/pull/68826>`_, Marcus Watts)
* rgw: handle plain-text object tags in RGWObjTags::decode() (`pr#67927 <https://github.com/ceph/ceph/pull/67927>`_, Oguzhan Ozmen)
* rgw: java s3-tests change setting of JAVA_HOME (`pr#68226 <https://github.com/ceph/ceph/pull/68226>`_, J. Eric Ivancich)
* rgw: read_obj_policy() consults s3:prefix when deciding between 403/404 (`pr#68651 <https://github.com/ceph/ceph/pull/68651>`_, Casey Bodley)
* RGW: remove custom copy constructor for RGWObjectCtx and enforce no copy/move (`pr#67440 <https://github.com/ceph/ceph/pull/67440>`_, Oguzhan Ozmen)
* src/ceph-volume: fast device unavailable as error (`pr#67916 <https://github.com/ceph/ceph/pull/67916>`_, Timothy Q Nguyen)
* test/rgw/kafka: fix kafka relase to more recent one (`pr#67993 <https://github.com/ceph/ceph/pull/67993>`_, Yuval Lifshitz)
* test/rgw/lua: ignore hours for zero mtime (`pr#67468 <https://github.com/ceph/ceph/pull/67468>`_, Kyr Shatskyy)
* test/rgw/notification: do not use netstat in the code (`pr#68142 <https://github.com/ceph/ceph/pull/68142>`_, Yuval Lifshitz)
* test/rgw/notification: fix the cloudevents package version (`pr#68704 <https://github.com/ceph/ceph/pull/68704>`_, Yuval Lifshitz, Adam C. Emerson)
* test/rgw: remove depracated boto dependency (`pr#68344 <https://github.com/ceph/ceph/pull/68344>`_, Yuval Lifshitz)
* test: use json_extract instead of awkward json_tree (`pr#67321 <https://github.com/ceph/ceph/pull/67321>`_, Patrick Donnelly)
* This change introduces the shared memory communication (SMC-D) for the cluster network (`pr#68254 <https://github.com/ceph/ceph/pull/68254>`_, Aliaksei Makarau)
* tools/ceph-kvstore-tool: fix crash on db close (`pr#68406 <https://github.com/ceph/ceph/pull/68406>`_, Igor Fedotov, Max Kellermann)
* upgrade suites update for Rocky10 and missing centos (`pr#68733 <https://github.com/ceph/ceph/pull/68733>`_, Nitzan Mordechai)
v20.2.1 Tentacle
================

2
doc/security/index.rst
View File

@ -7,6 +7,8 @@
Past Vulnerabilities / CVEs <cves>
Vulnerability Management Process <process>
Security Lead <securitylead>
Security Working Group <workinggroup>
Reporting a vulnerability
=========================

4
doc/security/securitylead.rst
View File

@ -1,3 +1,7 @@
================
Security Lead
================
The CSC designates a member as Security Lead, with responsibility for
co-ordinating security posture. The Security Lead also keeps the CSC updated
about vulnerabilities within Ceph and progress toward addressing them. The lead

4
doc/security/workinggroup.rst
View File

@ -1,3 +1,7 @@
==========================
Security Working Group
==========================
In order to fully support Ceph, the security working group
co-ordinates security improvements. This is essential as industry
focuses more on security, and Ceph has become a mature software

2
doc/start/os-recommendations.rst
View File

@ -54,6 +54,8 @@ Linux Kernel
full-time maintainer. As of July 2025 there are no plans to remove this
client but the future is uncertain.
.. _start-platforms:
Platforms
=========

21
examples/libfdb/Makefile Normal file
View File

@ -0,0 +1,21 @@
# Mini-example for libfdb
CXX?=g++
CXXFLAGS+=-std=c++23 -Wno-unused-parameter -Wall -Wextra -Werror -g
CEPH_SRC?=../../src
CEPH_BUILD?=../../build
INCLUDES=-I$(CEPH_SRC) -I$(CEPH_BUILD)/_deps/zpp_bits-src
LIBRARY_PATH=-L$(CEPH_BUILD)/lib/ -Wl,-rpath,$(CEPH_BUILD)/lib
LIBRARIES=-lfdb_c -lfmtd
all: mini_example
.PHONY: clean
mini_example: mini_example.cc
$(CXX) $(CXXFLAGS) $(INCLUDES) $(LIBRARY_PATH) -o mini_example mini_example.cc $(LIBRARIES)
clean:
rm -f mini_example

77
examples/libfdb/mini_example.cc Normal file
View File

@ -0,0 +1,77 @@
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab ft=cpp
/*
* Ceph - scalable distributed file system
*
* Copyright (C) 2026 International Business Machines Corp. (IBM)
*
* This is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License version 2.1, as published by the Free Software
* Foundation. See file COPYING.
*
* Whee! A mini-demo for libfdb!
*
* This shows how to take a user-defined data stucture and both store and
* retrieve it in FoundationDB.
*
*/
#include "rgw/fdb/interface.h"
#include <print>
#include <vector>
#include <string>
#include <cstdio>
namespace lfdb = ceph::libfdb;
using namespace std;
struct person
{
string name;
int year_born = 2000;
vector<string> titles;
};
static const vector<person> nifty_people {
{
.name = "Albrecht Duerer",
.year_born = 1471,
.titles = { "Painter", "Printmaker", "Northern Renaissance artist" },
},
{
.name = "Maria Theresa",
.year_born = 1717,
.titles = { "Habsburg ruler", "Archduchess of Austria", "Queen of Hungary and Bohemia" },
}
};
int main()
try
{
auto dbh = lfdb::create_database();
lfdb::set(dbh, "example/people", nifty_people);
vector<person> people;
if(lfdb::get(dbh, "example/people", people)) {
println("read {} people, some of them are likely nifty", people.size());
}
}
catch(const lfdb::libfdb_exception& e) {
println(stderr, "libfdb exception: {}", e.what());
return 1;
}
catch(const exception& e) {
println(stderr, "exception: {}", e.what());
return 1;
}
catch(...) {
println(stderr, "Well, now we're in a real jam, huh?");
return 1;
}

6
monitoring/ceph-mixin/config.libsonnet
View File

@ -22,5 +22,11 @@
// Read/Write latency is defined in ms
NVMeoFHighClientReadLatency: 10,
NVMeoFHighClientWriteLatency: 20,
//
// Hardware temperature thresholds in Celsius
HWTempMotherboardWarning: 60,
HWTempDIMMWarning: 80,
HWTempProcessorWarning: 80,
HWTempNVMeWarning: 70,
},
}

1
monitoring/ceph-mixin/dashboards.libsonnet
View File

@ -12,5 +12,6 @@
(import 'dashboards/smb-overview.libsonnet') +
(import 'dashboards/ceph-nvmeof.libsonnet') +
(import 'dashboards/ceph-nvmeof-performance.libsonnet') +
(import 'dashboards/hardware.libsonnet') +
{ _config:: $._config },
}

750
monitoring/ceph-mixin/dashboards/hardware.libsonnet Normal file
View File

@ -0,0 +1,750 @@
local g = import 'grafonnet/grafana.libsonnet';
(import 'utils.libsonnet') {
'hardware.json':
$.dashboardSchema(
'Ceph Hardware - Overview',
'Comprehensive hardware monitoring with temperatures, fans, storage, and firmware from node-proxy',
'hardware001',
'now-1h',
'10s',
22,
$._config.dashboardTags,
''
)
.addAnnotation(
$.addAnnotationSchema(
1,
'-- Grafana --',
true,
true,
'rgba(0, 211, 255, 1)',
'Annotations & Alerts',
'dashboard'
)
)
.addLinks([
$.addLinkSchema(
asDropdown=true,
icon='external link',
includeVars=true,
keepTime=true,
tags=[],
targetBlank=false,
title='Ceph Hardware - Overview',
tooltip='',
type='dashboards',
url=''
),
])
.addTemplate(
g.template.datasource('datasource', 'prometheus', 'default', label='Data Source')
)
.addTemplate(
$.addClusterTemplate()
)
.addTemplate(
$.addTemplateSchema(
'hostname',
'$datasource',
'label_values(ceph_hardware_health, hostname)',
1,
false,
1,
'Host',
''
)
)
.addTemplate(
g.template.new(
'fan_speeds',
'$datasource',
'label_values(ceph_hardware_fan_rpm{hostname=~"$hostname"},fan_name)',
label='',
refresh='load',
includeAll=true,
multi=true,
allValues='',
sort=0,
regex='/.*TACH_OUT/',
hide=2
)
)
.addPanels([
// Row 1: Overview (collapsed)
$.addRowSchema(collapse=true, showTitle=true, title='Overview') + { gridPos: { x: 0, y: 0, w: 24, h: 1 }, panels: [
// Health
$.addStatPanel(
title='Health',
unit='short',
datasource='$datasource',
gridPosition={ h: 4, w: 4, x: 0, y: 1 }
)
.addTarget($.addTargetSchema(
'max(ceph_hardware_health)',
legendFormat='Health'
))
+ {
description: 'Overall hardware health across all hosts',
fieldConfig: {
defaults: {
mappings: [
{ type: 'value', options: { '0': { color: 'green', index: 0, text: 'OK' } } },
{ type: 'range', options: { from: 1, to: 999, result: { color: 'semi-dark-red', index: 1, text: 'NOT OK' } } },
],
thresholds: {
mode: 'absolute',
steps: [{ color: 'green' }, { color: 'red', value: 1 }],
},
},
},
options+: { colorMode: 'background_solid' },
},
// CPU Temp
$.addGaugePanel(
title='CPU Temp',
unit='celsius',
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 4, y: 1 },
min=0,
max=100,
)
.addThresholds([
{ color: 'green' },
{ color: '#EAB839', value: 75 },
{ color: 'red', value: 85 },
])
.addTarget($.addTargetSchema(
'max(ceph_hardware_temperature_celsius{sensor_name=~".*CPU_TEMP"})',
legendFormat='CPU'
))
+ { description: 'Highest CPU temperature across all hosts' },
// BMC Versions
$.pieChartPanel(
title='BMC Versions',
datasource='$datasource',
gridPos={ x: 7, y: 1, w: 3, h: 8 }
)
.addTarget($.addTargetSchema(
'count by(version) (ceph_hardware_firmware_info{component=~".*BMC.*"})',
legendFormat='{{version}}'
))
+ {
fieldConfig+: {
defaults+: {
color+: { mode: 'palette-classic' },
},
},
options+: {
pieType: 'donut',
legend+: { showLegend: true, placement: 'bottom' },
displayLabels: ['name', 'value'],
},
},
// BIOS Versions
$.pieChartPanel(
title='BIOS Versions',
datasource='$datasource',
gridPos={ x: 10, y: 1, w: 3, h: 8 }
)
.addTarget($.addTargetSchema(
'count by(version) (ceph_hardware_firmware_info{component=~".*BIOS.*"})',
legendFormat='{{version}}'
))
+ {
fieldConfig+: {
defaults+: {
color+: { mode: 'palette-classic' },
},
},
options+: {
pieType: 'donut',
legend+: { showLegend: true, placement: 'bottom' },
displayLabels: ['name', 'value'],
},
},
// Drive Types
$.pieChartPanel(
title='Drive Types',
datasource='$datasource',
gridPos={ x: 13, y: 1, w: 6, h: 8 }
)
.addTarget($.addTargetSchema(
'count by(model) (ceph_hardware_storage_capacity_bytes)',
legendFormat='{{model}}'
))
+ {
fieldConfig+: {
defaults+: {
color+: { mode: 'palette-classic' },
},
},
options+: {
pieType: 'donut',
legend+: { showLegend: true, placement: 'right' },
displayLabels: ['value'],
},
},
// Drive Firmware
$.pieChartPanel(
title='Drive Firmware',
datasource='$datasource',
gridPos={ x: 19, y: 1, w: 5, h: 8 }
)
.addTarget($.addTargetSchema(
'count by(firmware_version) (ceph_hardware_storage_capacity_bytes{firmware_version!="unknown"})',
legendFormat='{{firmware_version}}',
instant=true
))
+ {
fieldConfig+: {
defaults+: {
color+: { mode: 'palette-classic' },
},
},
options+: {
pieType: 'donut',
legend+: { showLegend: true, placement: 'right' },
displayLabels: ['value'],
},
},
// Hosts
$.addStatPanel(
title='Hosts',
unit='short',
datasource='$datasource',
gridPosition={ h: 4, w: 2, x: 0, y: 5 }
).addTarget($.addTargetSchema(
'count(count by(hostname) (ceph_hardware_health))',
legendFormat='Hosts'
))
+ { description: 'Total hosts with hardware monitoring' },
// Drives
$.addStatPanel(
title='Drives',
unit='short',
datasource='$datasource',
gridPosition={ h: 4, w: 2, x: 2, y: 5 }
).addTarget($.addTargetSchema(
'count(ceph_hardware_storage_capacity_bytes)',
legendFormat='Drives'
))
+ { description: 'Total storage drives detected' },
// NVMe Temp
$.addGaugePanel(
title='NVMe Temp',
unit='celsius',
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 4, y: 5 },
min=0,
max=85,
)
.addThresholds([
{ color: 'green' },
{ color: 'yellow', value: 75 },
{ color: 'red', value: 80 },
])
.addTarget($.addTargetSchema(
'max(ceph_hardware_temperature_celsius{sensor_name=~"NVME.*_TEMP"})',
legendFormat='NVMe'
))
+ { description: 'Highest NVMe drive temperature across all hosts' },
] },
// Row 2: Host Overview
$.addRowSchema(true, true, 'Host Overview: $hostname') + { gridPos: { x: 0, y: 1, w: 24, h: 1 }, panels: [
// Power
$.addStatPanel(
title='Power',
unit='short',
datasource='$datasource',
gridPosition={ h: 3, w: 2, x: 0, y: 2 }
).addTarget($.addTargetSchema(
'max(ceph_hardware_health{hostname=~"$hostname",category="power"})',
legendFormat='Power'
))
+ {
description: 'Power supply health status',
fieldConfig: {
defaults: {
mappings: [
{ type: 'value', options: { '0': { color: 'green', index: 0, text: 'OK' } } },
{ type: 'range', options: { from: 1, to: 9999, result: { color: 'red', index: 1, text: 'NOT OK' } } },
],
thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'red', value: 1 }] },
},
},
options+: { colorMode: 'background_solid' },
},
// MB Temperature
$.addStatPanel(
title='MB Temperature (max)',
unit='celsius',
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 2, y: 2 }
).addTarget($.addTargetSchema(
'max(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*MB_TEMP.*"})',
legendFormat='Motherboard'
))
+ {
description: 'Highest motherboard temperature',
fieldConfig+: { defaults+: { thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: '#EAB839', value: 70 }, { color: 'red', value: 80 }] } } },
options+: { colorMode: 'none', graphMode: 'area' },
},
// CPU Temperature
$.addStatPanel(
title='CPU Temperature',
unit='celsius',
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 5, y: 2 }
).addTarget($.addTargetSchema(
'avg(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*CPU_TEMP"})',
legendFormat='CPU'
))
+ {
description: 'Average CPU temperature',
fieldConfig+: { defaults+: { max: 85, min: 0, thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: '#EAB839', value: 80 }, { color: 'semi-dark-red', value: 90 }] } } },
options+: { colorMode: 'none', graphMode: 'area' },
},
// DIMM Temperature
$.addStatPanel(
title='DIMM Temperature (max)',
unit='celsius',
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 8, y: 2 }
).addTarget($.addTargetSchema(
'max(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*DIMM.*_TEMP"})',
legendFormat='DIMM'
))
+ {
description: 'Highest DIMM temperature',
fieldConfig+: { defaults+: { max: 88, min: 0, thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'yellow', value: 70 }, { color: 'semi-dark-red', value: 80 }] } } },
options+: { colorMode: 'none', graphMode: 'area' },
},
// PSU Fans
$.addStatPanel(
title='PSU Fans',
unit='short',
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 11, y: 2 }
).addTarget($.addTargetSchema(
'count(ceph_hardware_fan_rpm{hostname=~"$hostname", fan_name=~"PSU.*"})',
legendFormat='PSU Fans'
))
+ {
description: 'Number of PSU fans detected',
options+: { colorMode: 'none', graphMode: 'none' },
},
// AVG PSU Temperature
$.addStatPanel(
title='AVG PSU Temperature',
unit='celsius',
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 14, y: 2 }
).addTarget($.addTargetSchema(
'avg(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~"PSU.*_TEMP.*"})',
legendFormat='PSU'
))
+ {
description: 'Average power supply temperature',
fieldConfig+: { defaults+: { color: { fixedColor: 'blue', mode: 'fixed' }, max: 100, min: 0, thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'yellow', value: 80 }, { color: 'semi-dark-red', value: 90 }] } } },
options+: { colorMode: 'none', graphMode: 'area' },
},
// NVMe Drives
$.addStatPanel(
title='NVMe Drives',
unit='short',
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 17, y: 2 }
).addTarget($.addTargetSchema(
'count(ceph_hardware_storage_capacity_bytes{hostname=~"$hostname", protocol="NVMe"})',
legendFormat='NVMe'
))
+ {
description: 'Number of NVMe drives detected',
fieldConfig+: { defaults+: { color: { fixedColor: 'blue', mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: '#EAB839', value: 70 }, { color: 'semi-dark-red', value: 78 }] } } },
options+: { colorMode: 'none', graphMode: 'none' },
},
// NVMe Temperature
$.addStatPanel(
title='NVMe Temperature (max)',
unit='celsius',
datasource='$datasource',
gridPosition={ h: 4, w: 3, x: 20, y: 2 }
).addTarget($.addTargetSchema(
'max(ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~"NVME.*"})',
legendFormat='NVMe'
))
+ {
description: 'Highest NVMe drive temperature',
fieldConfig+: { defaults+: { color: { fixedColor: 'blue', mode: 'thresholds' }, max: 85, min: 0, thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: '#EAB839', value: 70 }, { color: 'semi-dark-red', value: 78 }] } } },
options+: { colorMode: 'none', graphMode: 'area' },
},
// Network
$.addStatPanel(
title='Network',
unit='short',
datasource='$datasource',
gridPosition={ h: 3, w: 2, x: 0, y: 5 }
).addTarget($.addTargetSchema(
'max(ceph_hardware_health{hostname=~"$hostname",category="network"})',
legendFormat='Network'
))
+ {
description: 'Network adapter health status',
fieldConfig: {
defaults: {
mappings: [
{ type: 'value', options: { '0': { color: 'green', index: 0, text: 'OK' } } },
{ type: 'range', options: { from: 1, to: 9999, result: { color: 'red', index: 1, text: 'NOT OK' } } },
],
thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'red', value: 1 }] },
},
},
options+: { colorMode: 'background_solid' },
},
// Cooling
$.addStatPanel(
title='Cooling',
unit='short',
datasource='$datasource',
gridPosition={ h: 3, w: 2, x: 0, y: 8 }
).addTarget($.addTargetSchema(
'max(ceph_hardware_health{hostname=~"$hostname",category="fans"})',
legendFormat='Cooling'
))
+ {
description: 'Cooling fans health status',
fieldConfig: {
defaults: {
mappings: [
{ type: 'value', options: { '0': { color: 'green', index: 0, text: 'OK' } } },
{ type: 'range', options: { from: 1, to: 9999, result: { color: 'red', index: 1, text: 'NOT OK' } } },
],
thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'red', value: 1 }] },
},
},
options+: { colorMode: 'background_solid' },
},
// Drives
$.addStatPanel(
title='Drives',
unit='short',
datasource='$datasource',
gridPosition={ h: 3, w: 2, x: 0, y: 11 }
).addTarget($.addTargetSchema(
'max(ceph_hardware_health{hostname=~"$hostname",category="storage"})',
legendFormat='Drives'
))
+ {
description: 'Storage drives health status',
fieldConfig: {
defaults: {
mappings: [
{ type: 'value', options: { '0': { color: 'green', index: 0, text: 'OK' } } },
{ type: 'range', options: { from: 1, to: 9999, result: { color: 'red', index: 1, text: 'NOT OK' } } },
],
thresholds: { mode: 'absolute', steps: [{ color: 'green' }, { color: 'red', value: 1 }] },
},
},
options+: { colorMode: 'background_solid' },
},
// Device List
$.addTableExtended(
title='Device List',
datasource='$datasource',
gridPosition={ h: 8, w: 11, x: 2, y: 6 },
options={
footer: { show: false },
showHeader: true,
},
custom={ align: 'auto', cellOptions: { type: 'auto' }, filterable: false, inspect: false },
thresholds={
mode: 'absolute',
steps: [{ color: 'green', value: null }],
},
)
.addTarget($.addTargetSchema(
'ceph_hardware_storage_capacity_bytes{hostname=~"$hostname"}',
legendFormat='',
instant=true
) + { format: 'table' })
.addTransformations([
{
id: 'organize',
options: {
excludeByName: {
Time: true,
Value: true,
__name__: true,
cluster: true,
hostname: true,
job: true,
instance: true,
},
renameByName: {
device: 'Device Name',
model: 'Drive Model',
protocol: 'Protocol',
},
},
},
]),
// Platform Firmware
$.addTableExtended(
title='Platform Firmware',
datasource='$datasource',
gridPosition={ h: 8, w: 11, x: 13, y: 6 },
options={
footer: { show: false },
showHeader: true,
},
custom={ align: 'auto', cellOptions: { type: 'auto' }, filterable: false, inspect: false },
thresholds={
mode: 'absolute',
steps: [{ color: 'green', value: null }],
},
)
.addTarget($.addTargetSchema(
'ceph_hardware_firmware_info{hostname=~"$hostname"}',
legendFormat='',
instant=true
) + { format: 'table' })
.addTransformations([
{
id: 'organize',
options: {
excludeByName: {
Time: true,
Value: true,
__name__: true,
cluster: true,
hostname: true,
job: true,
instance: true,
},
renameByName: {
component: 'Component',
version: 'Version',
},
},
},
]),
] },
// Row 3: FAN Speeds
$.addRowSchema(true, true, 'FAN Speeds: $hostname') + { gridPos: { x: 0, y: 2, w: 24, h: 1 }, panels: [
// Repeating panel
$.addStatPanel(
title='FAN: $fan_speeds (RPM)',
unit='locale',
datasource='$datasource',
gridPosition={ h: 5, w: 4, x: 0, y: 3 }
).addTarget($.addTargetSchema(
'ceph_hardware_fan_rpm{hostname=~"$hostname",fan_name=~"$fan_speeds"}',
legendFormat='{{fan_name}}'
))
+ {
fieldConfig+: { defaults+: { color: { fixedColor: 'blue', mode: 'fixed' } } },
options+: { colorMode: 'none', graphMode: 'area' },
repeat: 'fan_speeds',
repeatDirection: 'h',
maxPerRow: 6,
},
// PSU1
$.addStatPanel(
title='PSU1 Fan Speed (RPM)',
unit='locale',
datasource='$datasource',
gridPosition={ h: 5, w: 4, x: 0, y: 8 }
).addTarget($.addTargetSchema(
'ceph_hardware_fan_rpm{hostname=~"$hostname",fan_name=~"PSU1.*"}',
legendFormat='{{fan_name}}'
))
+ {
fieldConfig+: { defaults+: { color: { fixedColor: 'blue', mode: 'fixed' } } },
options+: { colorMode: 'none', graphMode: 'area' },
},
// PSU2
$.addStatPanel(
title='PSU2 Fan Speed (RPM)',
unit='locale',
datasource='$datasource',
gridPosition={ h: 5, w: 4, x: 4, y: 8 }
).addTarget($.addTargetSchema(
'ceph_hardware_fan_rpm{hostname=~"$hostname",fan_name=~"PSU2.*"}',
legendFormat='{{fan_name}}'
))
+ {
fieldConfig+: { defaults+: { color: { fixedColor: 'blue', mode: 'fixed' } } },
options+: { colorMode: 'none', graphMode: 'area' },
},
] },
// Row 4: Temperature History
$.addRowSchema(true, true, 'Temperature History: $hostname') + { gridPos: { x: 0, y: 3, w: 24, h: 1 }, panels: [
// CPU Temperature
g.graphPanel.new(
title='CPU Temperature',
datasource='$datasource',
format='celsius',
).addTarget(
g.prometheus.target(
'ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*CPU_TEMP"}',
legendFormat='{{sensor_name}}'
)
).addTarget(
g.prometheus.target('vector(100)', legendFormat='Critical')
)
+ {
gridPos: { x: 0, y: 4, w: 12, h: 8 },
yaxes: [{ min: 0 }, {}],
seriesOverrides: [
{ alias: 'Critical', color: 'dark-red', dashes: true, fill: 0 },
],
},
// DIMM Temperatures
g.graphPanel.new(
title='DIMM Temperatures',
datasource='$datasource',
format='celsius',
).addTarget(
g.prometheus.target(
'ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*DIMM.*_TEMP"}',
legendFormat='{{sensor_name}}'
)
).addTarget(
g.prometheus.target('vector(88)', legendFormat='Critical')
)
+ {
gridPos: { x: 12, y: 4, w: 12, h: 8 },
yaxes: [{ min: 0 }, {}],
seriesOverrides: [
{ alias: 'Critical', color: 'dark-red', dashes: true, fill: 0 },
],
},
// Motherboard Temperatures
g.graphPanel.new(
title='Motherboard Temperatures',
datasource='$datasource',
format='celsius',
).addTarget(
g.prometheus.target(
'ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~".*MB_TEMP.*"}',
legendFormat='{{sensor_name}}'
)
).addTarget(
g.prometheus.target('vector(85)', legendFormat='Critical')
)
+ {
gridPos: { x: 0, y: 12, w: 12, h: 8 },
yaxes: [{ min: 0 }, {}],
seriesOverrides: [
{ alias: 'Critical', color: 'dark-red', dashes: true, fill: 0 },
],
},
// NVMe Temperatures
g.graphPanel.new(
title='NVMe Temperatures',
datasource='$datasource',
format='celsius',
).addTarget(
g.prometheus.target(
'ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~"NVME.*_TEMP"}',
legendFormat='{{sensor_name}}'
)
).addTarget(
g.prometheus.target('vector(85)', legendFormat='Critical')
)
+ {
gridPos: { x: 12, y: 12, w: 12, h: 8 },
yaxes: [{ min: 0 }, {}],
seriesOverrides: [
{ alias: 'Critical', color: 'dark-red', dashes: true, fill: 0 },
],
},
// PSU Temperatures
g.graphPanel.new(
title='PSU Temperatures',
datasource='$datasource',
format='celsius',
).addTarget(
g.prometheus.target(
'ceph_hardware_temperature_celsius{hostname=~"$hostname", sensor_name=~"PSU.*_TEMP.*"}',
legendFormat='{{sensor_name}}'
)
).addTarget(
g.prometheus.target('vector(65)', legendFormat='Critical')
)
+ {
gridPos: { x: 0, y: 20, w: 12, h: 8 },
yaxes: [{ min: 0 }, {}],
seriesOverrides: [
{ alias: 'Critical', color: 'dark-red', dashes: true, fill: 0 },
],
},
] },
// Row 5: FAN Speed History
$.addRowSchema(true, true, 'FAN Speed History: $hostname') + { gridPos: { x: 0, y: 4, w: 24, h: 1 }, panels: [
// AVG PSU Fan Speed
g.graphPanel.new(
title='AVG PSU Fan Speed',
datasource='$datasource',
format='locale',
fill=1,
).addTarget(
g.prometheus.target(
'avg(ceph_hardware_fan_rpm{hostname=~"$hostname", fan_name=~"PSU.*"})',
legendFormat='PSU Fans'
)
)
+ {
gridPos: { x: 0, y: 5, w: 12, h: 8 },
description: 'AVG across both PSUs',
},
// AVG Cooling Fan RPMs
g.graphPanel.new(
title='AVG Cooling Fan RPMs',
datasource='$datasource',
format='locale',
fill=1,
).addTarget(
g.prometheus.target(
'avg(ceph_hardware_fan_rpm{hostname=~"$hostname", fan_name!~"PSU.*"})',
legendFormat='System Fans'
)
)
+ {
gridPos: { x: 12, y: 5, w: 12, h: 8 },
},
] },
]),
}

1518
monitoring/ceph-mixin/dashboards_out/ceph-hardware-compression.json Normal file
View File

File diff suppressed because it is too large Load Diff

2490
monitoring/ceph-mixin/dashboards_out/hardware.json Normal file
View File

File diff suppressed because it is too large Load Diff

4
monitoring/ceph-mixin/jsonnet-bundler-build.sh
View File

@ -1,8 +1,10 @@
#!/bin/sh -ex
JSONNET_VERSION="v0.4.0"
JSONNET_VERSION="v0.6.0"
OUTPUT_DIR=${1:-$(pwd)}
# clean up leftovers from an aborted run so the clone stays idempotent
rm -rf jsonnet-bundler
git clone -b ${JSONNET_VERSION} --depth 1 https://github.com/jsonnet-bundler/jsonnet-bundler
make -C jsonnet-bundler build
mv jsonnet-bundler/_output/jb ${OUTPUT_DIR}

196
monitoring/ceph-mixin/prometheus_alerts.libsonnet
View File

@ -689,63 +689,163 @@
name: 'hardware',
rules: [
{
alert: 'HardwareStorageError',
'for': '30s',
expr: 'ceph_health_detail{name="HARDWARE_STORAGE"} > 0',
labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.1' },
alert: 'PowerSupplyDegraded',
'for': '1m',
expr: 'ceph_hardware_health{category="power"} == 1',
labels: { severity: 'warning', type: 'ceph_default' },
annotations: {
summary: 'Storage devices error(s) detected%(cluster)s' % $.MultiClusterSummary(),
description: 'Some storage devices are in error. Check `ceph health detail`.',
summary: 'Power supply {{ $labels.component }} degraded on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
description: 'PSU {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}.',
},
},
{
alert: 'HardwareMemoryError',
'for': '30s',
expr: 'ceph_health_detail{name="HARDWARE_MEMORY"} > 0',
labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.2' },
annotations: {
summary: 'DIMM error(s) detected%(cluster)s' % $.MultiClusterSummary(),
description: 'DIMM error(s) detected. Check `ceph health detail`.',
},
},
{
alert: 'HardwareProcessorError',
'for': '30s',
expr: 'ceph_health_detail{name="HARDWARE_PROCESSOR"} > 0',
labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.3' },
annotations: {
summary: 'Processor error(s) detected%(cluster)s' % $.MultiClusterSummary(),
description: 'Processor error(s) detected. Check `ceph health detail`.',
},
},
{
alert: 'HardwareNetworkError',
'for': '30s',
expr: 'ceph_health_detail{name="HARDWARE_NETWORK"} > 0',
labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.4' },
annotations: {
summary: 'Network error(s) detected%(cluster)s' % $.MultiClusterSummary(),
description: 'Network error(s) detected. Check `ceph health detail`.',
},
},
{
alert: 'HardwarePowerError',
'for': '30s',
expr: 'ceph_health_detail{name="HARDWARE_POWER"} > 0',
alert: 'PowerSupplyFailed',
'for': '1m',
expr: 'ceph_hardware_health{category="power"} == 2',
labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.5' },
annotations: {
summary: 'Power supply error(s) detected%(cluster)s' % $.MultiClusterSummary(),
description: 'Power supply error(s) detected. Check `ceph health detail`.',
summary: 'Power supply {{ $labels.component }} failed on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
description: 'PSU fault detected on {{ $labels.hostname }}. PSU {{ $labels.component }} has failed.',
},
},
{
alert: 'HardwareFanError',
'for': '30s',
expr: 'ceph_health_detail{name="HARDWARE_FANS"} > 0',
alert: 'CoolingDegraded',
'for': '1m',
expr: 'ceph_hardware_health{category="fans"} == 1',
labels: { severity: 'warning', type: 'ceph_default' },
annotations: {
summary: 'Fan {{ $labels.component }} degraded on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
description: 'Fan module {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}.',
},
},
{
alert: 'CoolingFailed',
'for': '1m',
expr: 'ceph_hardware_health{category="fans"} == 2',
labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.6' },
annotations: {
summary: 'Fan error(s) detected%(cluster)s' % $.MultiClusterSummary(),
description: 'Fan error(s) detected. Check `ceph health detail`.',
summary: 'Fan {{ $labels.component }} failed on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
description: 'A cooling fault has been detected on {{ $labels.hostname }}. A cooling FAN within the enclosure has failed.',
},
},
{
alert: 'NVMeDriveDegraded',
'for': '1m',
expr: 'ceph_hardware_health{category="storage"} == 1',
labels: { severity: 'warning', type: 'ceph_default' },
annotations: {
summary: 'Storage device {{ $labels.component }} degraded on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
description: 'Storage device {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}.',
},
},
{
alert: 'NVMeDriveFailed',
'for': '1m',
expr: 'ceph_hardware_health{category="storage"} == 2',
labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.1' },
annotations: {
summary: 'Storage device {{ $labels.component }} failed on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
description: 'A drive fault has been detected on {{ $labels.hostname }}. Storage device {{ $labels.component }} has failed.',
},
},
{
alert: 'MemoryDegraded',
'for': '1m',
expr: 'ceph_hardware_health{category="memory"} == 1',
labels: { severity: 'warning', type: 'ceph_default' },
annotations: {
summary: 'Memory module {{ $labels.component }} degraded on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
description: 'Memory module {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}.',
},
},
{
alert: 'MemoryFailed',
'for': '1m',
expr: 'ceph_hardware_health{category="memory"} == 2',
labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.2' },
annotations: {
summary: 'Memory module {{ $labels.component }} failed on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
description: 'A memory fault has been detected on {{ $labels.hostname }}. Memory module {{ $labels.component }} has failed.',
},
},
{
alert: 'ProcessorDegraded',
'for': '1m',
expr: 'ceph_hardware_health{category="processors"} == 1',
labels: { severity: 'warning', type: 'ceph_default' },
annotations: {
summary: 'Processor {{ $labels.component }} degraded on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
description: 'Processor {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}.',
},
},
{
alert: 'ProcessorFailed',
'for': '1m',
expr: 'ceph_hardware_health{category="processors"} == 2',
labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.3' },
annotations: {
summary: 'Processor {{ $labels.component }} failed on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
description: 'A processor fault has been detected on {{ $labels.hostname }}. Processor {{ $labels.component }} has failed.',
},
},
{
alert: 'NetworkDegraded',
'for': '1m',
expr: 'ceph_hardware_health{category="network"} == 1',
labels: { severity: 'warning', type: 'ceph_default' },
annotations: {
summary: 'Network interface {{ $labels.component }} degraded on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
description: 'Network interface {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}.',
},
},
{
alert: 'NetworkFailed',
'for': '1m',
expr: 'ceph_hardware_health{category="network"} == 2',
labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.4' },
annotations: {
summary: 'Network interface {{ $labels.component }} failed on {{ $labels.hostname }}%(cluster)s' % $.MultiClusterSummary(),
description: 'A network fault has been detected on {{ $labels.hostname }}. Network interface {{ $labels.component }} has failed.',
},
},
{
alert: 'MotherboardTemperatureHigh',
'for': '1m',
expr: 'ceph_hardware_temperature_celsius{sensor_name=~".*MB_TEMP.*"} > %d' % $._config.HWTempMotherboardWarning,
labels: { severity: 'warning', type: 'ceph_default' },
annotations: {
summary: 'Motherboard sensor {{ $labels.sensor_name }} above %d°C on {{ $labels.hostname }}%s' % [$._config.HWTempMotherboardWarning, $.MultiClusterSummary()],
description: 'Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall.',
},
},
{
alert: 'DIMMTemperatureHigh',
'for': '1m',
expr: 'ceph_hardware_temperature_celsius{sensor_name=~".*DIMM.*_TEMP"} > %d' % $._config.HWTempDIMMWarning,
labels: { severity: 'warning', type: 'ceph_default' },
annotations: {
summary: 'DIMM sensor {{ $labels.sensor_name }} above %d°C on {{ $labels.hostname }}%s' % [$._config.HWTempDIMMWarning, $.MultiClusterSummary()],
description: 'Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall.',
},
},
{
alert: 'ProcessorTemperatureHigh',
'for': '1m',
expr: 'ceph_hardware_temperature_celsius{sensor_name=~".*CPU_TEMP"} > %d' % $._config.HWTempProcessorWarning,
labels: { severity: 'warning', type: 'ceph_default' },
annotations: {
summary: 'CPU sensor {{ $labels.sensor_name }} above %d°C on {{ $labels.hostname }}%s' % [$._config.HWTempProcessorWarning, $.MultiClusterSummary()],
description: 'Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall.',
},
},
{
alert: 'NVMeTemperatureHigh',
'for': '1m',
expr: 'ceph_hardware_temperature_celsius{sensor_name=~"NVME.*_TEMP"} > %d' % $._config.HWTempNVMeWarning,
labels: { severity: 'warning', type: 'ceph_default' },
annotations: {
summary: 'NVMe sensor {{ $labels.sensor_name }} above %d°C on {{ $labels.hostname }}%s' % [$._config.HWTempNVMeWarning, $.MultiClusterSummary()],
description: 'An NVMe temperature sensor is indicating a potential cooling problem. NVMe may power down if the temperature continues to rise.',
},
},
],
@ -858,10 +958,10 @@
{
alert: 'NVMeoFMultipleNamespacesOfRBDImage',
'for': '1m',
expr: 'count by(pool_name, rbd_name) (count by(bdev_name, pool_name, rbd_name) (ceph_nvmeof_bdev_metadata and on (bdev_name) ceph_nvmeof_subsystem_namespace_metadata)) > 1',
expr: 'count by(pool_name, rbd_name, rados_namespace_name) (count by(bdev_name, pool_name, rbd_name, rados_namespace_name) (ceph_nvmeof_bdev_metadata * on (bdev_name, instance, cluster) group_left(rados_namespace_name) ceph_nvmeof_subsystem_namespace_metadata)) > 1',
labels: { severity: 'warning', type: 'ceph_default' },
annotations: {
summary: 'RBD image {{ $labels.pool_name }}/{{ $labels.rbd_name }} cannot be reused for multiple NVMeoF namespace ',
summary: 'RBD image {{ $labels.pool_name }}/{{ if $labels.rados_namespace_name }}{{ $labels.rados_namespace_name }}/{{ end }}{{ $labels.rbd_name }} cannot be reused for multiple NVMeoF namespaces',
description: 'Each NVMeoF namespace must have a unique RBD pool and image, across all different gateway groups.',
},
},

189
monitoring/ceph-mixin/prometheus_alerts.yml
View File

@ -650,66 +650,156 @@ groups:
type: "ceph_default"
- name: "hardware"
rules:
- alert: "HardwareStorageError"
- alert: "PowerSupplyDegraded"
annotations:
description: "Some storage devices are in error. Check `ceph health detail`."
summary: "Storage devices error(s) detected on cluster {{ $labels.cluster }}"
expr: "ceph_health_detail{name=\"HARDWARE_STORAGE\"} > 0"
for: "30s"
description: "PSU {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}."
summary: "Power supply {{ $labels.component }} degraded on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
expr: "ceph_hardware_health{category=\"power\"} == 1"
for: "1m"
labels:
oid: "1.3.6.1.4.1.50495.1.2.1.13.1"
severity: "critical"
severity: "warning"
type: "ceph_default"
- alert: "HardwareMemoryError"
- alert: "PowerSupplyFailed"
annotations:
description: "DIMM error(s) detected. Check `ceph health detail`."
summary: "DIMM error(s) detected on cluster {{ $labels.cluster }}"
expr: "ceph_health_detail{name=\"HARDWARE_MEMORY\"} > 0"
for: "30s"
labels:
oid: "1.3.6.1.4.1.50495.1.2.1.13.2"
severity: "critical"
type: "ceph_default"
- alert: "HardwareProcessorError"
annotations:
description: "Processor error(s) detected. Check `ceph health detail`."
summary: "Processor error(s) detected on cluster {{ $labels.cluster }}"
expr: "ceph_health_detail{name=\"HARDWARE_PROCESSOR\"} > 0"
for: "30s"
labels:
oid: "1.3.6.1.4.1.50495.1.2.1.13.3"
severity: "critical"
type: "ceph_default"
- alert: "HardwareNetworkError"
annotations:
description: "Network error(s) detected. Check `ceph health detail`."
summary: "Network error(s) detected on cluster {{ $labels.cluster }}"
expr: "ceph_health_detail{name=\"HARDWARE_NETWORK\"} > 0"
for: "30s"
labels:
oid: "1.3.6.1.4.1.50495.1.2.1.13.4"
severity: "critical"
type: "ceph_default"
- alert: "HardwarePowerError"
annotations:
description: "Power supply error(s) detected. Check `ceph health detail`."
summary: "Power supply error(s) detected on cluster {{ $labels.cluster }}"
expr: "ceph_health_detail{name=\"HARDWARE_POWER\"} > 0"
for: "30s"
description: "PSU fault detected on {{ $labels.hostname }}. PSU {{ $labels.component }} has failed."
summary: "Power supply {{ $labels.component }} failed on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
expr: "ceph_hardware_health{category=\"power\"} == 2"
for: "1m"
labels:
oid: "1.3.6.1.4.1.50495.1.2.1.13.5"
severity: "critical"
type: "ceph_default"
- alert: "HardwareFanError"
- alert: "CoolingDegraded"
annotations:
description: "Fan error(s) detected. Check `ceph health detail`."
summary: "Fan error(s) detected on cluster {{ $labels.cluster }}"
expr: "ceph_health_detail{name=\"HARDWARE_FANS\"} > 0"
for: "30s"
description: "Fan module {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}."
summary: "Fan {{ $labels.component }} degraded on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
expr: "ceph_hardware_health{category=\"fans\"} == 1"
for: "1m"
labels:
severity: "warning"
type: "ceph_default"
- alert: "CoolingFailed"
annotations:
description: "A cooling fault has been detected on {{ $labels.hostname }}. A cooling FAN within the enclosure has failed."
summary: "Fan {{ $labels.component }} failed on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
expr: "ceph_hardware_health{category=\"fans\"} == 2"
for: "1m"
labels:
oid: "1.3.6.1.4.1.50495.1.2.1.13.6"
severity: "critical"
type: "ceph_default"
- alert: "NVMeDriveDegraded"
annotations:
description: "Storage device {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}."
summary: "Storage device {{ $labels.component }} degraded on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
expr: "ceph_hardware_health{category=\"storage\"} == 1"
for: "1m"
labels:
severity: "warning"
type: "ceph_default"
- alert: "NVMeDriveFailed"
annotations:
description: "A drive fault has been detected on {{ $labels.hostname }}. Storage device {{ $labels.component }} has failed."
summary: "Storage device {{ $labels.component }} failed on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
expr: "ceph_hardware_health{category=\"storage\"} == 2"
for: "1m"
labels:
oid: "1.3.6.1.4.1.50495.1.2.1.13.1"
severity: "critical"
type: "ceph_default"
- alert: "MemoryDegraded"
annotations:
description: "Memory module {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}."
summary: "Memory module {{ $labels.component }} degraded on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
expr: "ceph_hardware_health{category=\"memory\"} == 1"
for: "1m"
labels:
severity: "warning"
type: "ceph_default"
- alert: "MemoryFailed"
annotations:
description: "A memory fault has been detected on {{ $labels.hostname }}. Memory module {{ $labels.component }} has failed."
summary: "Memory module {{ $labels.component }} failed on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
expr: "ceph_hardware_health{category=\"memory\"} == 2"
for: "1m"
labels:
oid: "1.3.6.1.4.1.50495.1.2.1.13.2"
severity: "critical"
type: "ceph_default"
- alert: "ProcessorDegraded"
annotations:
description: "Processor {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}."
summary: "Processor {{ $labels.component }} degraded on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
expr: "ceph_hardware_health{category=\"processors\"} == 1"
for: "1m"
labels:
severity: "warning"
type: "ceph_default"
- alert: "ProcessorFailed"
annotations:
description: "A processor fault has been detected on {{ $labels.hostname }}. Processor {{ $labels.component }} has failed."
summary: "Processor {{ $labels.component }} failed on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
expr: "ceph_hardware_health{category=\"processors\"} == 2"
for: "1m"
labels:
oid: "1.3.6.1.4.1.50495.1.2.1.13.3"
severity: "critical"
type: "ceph_default"
- alert: "NetworkDegraded"
annotations:
description: "Network interface {{ $labels.component }} is in a degraded state on {{ $labels.hostname }}."
summary: "Network interface {{ $labels.component }} degraded on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
expr: "ceph_hardware_health{category=\"network\"} == 1"
for: "1m"
labels:
severity: "warning"
type: "ceph_default"
- alert: "NetworkFailed"
annotations:
description: "A network fault has been detected on {{ $labels.hostname }}. Network interface {{ $labels.component }} has failed."
summary: "Network interface {{ $labels.component }} failed on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
expr: "ceph_hardware_health{category=\"network\"} == 2"
for: "1m"
labels:
oid: "1.3.6.1.4.1.50495.1.2.1.13.4"
severity: "critical"
type: "ceph_default"
- alert: "MotherboardTemperatureHigh"
annotations:
description: "Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall."
summary: "Motherboard sensor {{ $labels.sensor_name }} above 60°C on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
expr: "ceph_hardware_temperature_celsius{sensor_name=~\".*MB_TEMP.*\"} > 60"
for: "1m"
labels:
severity: "warning"
type: "ceph_default"
- alert: "DIMMTemperatureHigh"
annotations:
description: "Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall."
summary: "DIMM sensor {{ $labels.sensor_name }} above 80°C on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
expr: "ceph_hardware_temperature_celsius{sensor_name=~\".*DIMM.*_TEMP\"} > 80"
for: "1m"
labels:
severity: "warning"
type: "ceph_default"
- alert: "ProcessorTemperatureHigh"
annotations:
description: "Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall."
summary: "CPU sensor {{ $labels.sensor_name }} above 80°C on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
expr: "ceph_hardware_temperature_celsius{sensor_name=~\".*CPU_TEMP\"} > 80"
for: "1m"
labels:
severity: "warning"
type: "ceph_default"
- alert: "NVMeTemperatureHigh"
annotations:
description: "An NVMe temperature sensor is indicating a potential cooling problem. NVMe may power down if the temperature continues to rise."
summary: "NVMe sensor {{ $labels.sensor_name }} above 70°C on {{ $labels.hostname }} on cluster {{ $labels.cluster }}"
expr: "ceph_hardware_temperature_celsius{sensor_name=~\"NVME.*_TEMP\"} > 70"
for: "1m"
labels:
severity: "warning"
type: "ceph_default"
- name: "PrometheusServer"
rules:
- alert: "PrometheusJobMissing"
@ -804,8 +894,8 @@ groups:
- alert: "NVMeoFMultipleNamespacesOfRBDImage"
annotations:
description: "Each NVMeoF namespace must have a unique RBD pool and image, across all different gateway groups."
summary: "RBD image {{ $labels.pool_name }}/{{ $labels.rbd_name }} cannot be reused for multiple NVMeoF namespace "
expr: "count by(pool_name, rbd_name) (count by(bdev_name, pool_name, rbd_name) (ceph_nvmeof_bdev_metadata and on (bdev_name) ceph_nvmeof_subsystem_namespace_metadata)) > 1"
summary: "RBD image {{ $labels.pool_name }}/{{ if $labels.rados_namespace_name }}{{ $labels.rados_namespace_name }}/{{ end }}{{ $labels.rbd_name }} cannot be reused for multiple NVMeoF namespaces"
expr: "count by(pool_name, rbd_name, rados_namespace_name) (count by(bdev_name, pool_name, rbd_name, rados_namespace_name) (ceph_nvmeof_bdev_metadata * on (bdev_name, instance, cluster) group_left(rados_namespace_name) ceph_nvmeof_subsystem_namespace_metadata)) > 1"
for: "1m"
labels:
severity: "warning"
@ -995,4 +1085,3 @@ groups:
oid: "1.3.6.1.4.1.50495.1.2.1.15.2"
severity: "warning"
type: "ceph_default"

534
monitoring/ceph-mixin/tests_alerts/test_alerts.yml
View File

@ -2087,156 +2087,440 @@ tests:
summary: "The replication network usage on cluster mycluster has been increased over 80% in the last 30 minutes. Review the number of images being replicated. This alert will be cleaned automatically after 30 minutes"
description: "Detected a heavy increase in bandwidth for rbd replications (over 80%) in the last 30 min. This might not be a problem, but it is good to review the number of images being replicated simultaneously"
- interval: 30s
# Power supply degraded
- interval: 1m
input_series:
- series: 'ceph_health_detail{name="HARDWARE_STORAGE", cluster="mycluster"}'
values: '1+0x40'
- series: 'ceph_hardware_health{category="power",component="PSU0",hostname="node1",cluster="mycluster"}'
values: '1 1 1 1 1 1 1 1 1'
promql_expr_test:
- expr: ceph_health_detail{name="HARDWARE_STORAGE"} > 0
- expr: ceph_hardware_health{category="power"} == 1
eval_time: 2m
exp_samples:
- labels: '{__name__="ceph_health_detail", name="HARDWARE_STORAGE", cluster="mycluster"}'
- labels: 'ceph_hardware_health{category="power",component="PSU0",hostname="node1",cluster="mycluster"}'
value: 1
alert_rule_test:
- eval_time: 1m
alertname: HardwareStorageError
alertname: PowerSupplyDegraded
- eval_time: 5m
alertname: HardwareStorageError
alertname: PowerSupplyDegraded
exp_alerts:
- exp_labels:
name: HARDWARE_STORAGE
severity: critical
category: power
component: PSU0
hostname: node1
severity: warning
cluster: mycluster
type: ceph_default
oid: 1.3.6.1.4.1.50495.1.2.1.13.1
exp_annotations:
summary: Storage devices error(s) detected on cluster mycluster
description: "Some storage devices are in error. Check `ceph health detail`."
- interval: 30s
summary: "Power supply PSU0 degraded on node1 on cluster mycluster"
description: "PSU PSU0 is in a degraded state on node1."
# Power supply failed
- interval: 1m
input_series:
- series: 'ceph_health_detail{name="HARDWARE_MEMORY", cluster="mycluster"}'
values: '1+0x40'
- series: 'ceph_hardware_health{category="power",component="PSU1",hostname="node2",cluster="mycluster"}'
values: '2 2 2 2 2 2 2 2 2'
promql_expr_test:
- expr: ceph_health_detail{name="HARDWARE_MEMORY"} > 0
- expr: ceph_hardware_health{category="power"} == 2
eval_time: 2m
exp_samples:
- labels: '{__name__="ceph_health_detail", name="HARDWARE_MEMORY", cluster="mycluster"}'
value: 1
- labels: 'ceph_hardware_health{category="power",component="PSU1",hostname="node2",cluster="mycluster"}'
value: 2
alert_rule_test:
- eval_time: 1m
alertname: HardwareMemoryError
alertname: PowerSupplyFailed
- eval_time: 5m
alertname: HardwareMemoryError
alertname: PowerSupplyFailed
exp_alerts:
- exp_labels:
name: HARDWARE_MEMORY
severity: critical
cluster: mycluster
type: ceph_default
oid: 1.3.6.1.4.1.50495.1.2.1.13.2
exp_annotations:
summary: DIMM error(s) detected on cluster mycluster
description: "DIMM error(s) detected. Check `ceph health detail`."
- interval: 30s
input_series:
- series: 'ceph_health_detail{name="HARDWARE_PROCESSOR", cluster="mycluster"}'
values: '1+0x40'
promql_expr_test:
- expr: ceph_health_detail{name="HARDWARE_PROCESSOR"} > 0
eval_time: 2m
exp_samples:
- labels: '{__name__="ceph_health_detail", name="HARDWARE_PROCESSOR", cluster="mycluster"}'
value: 1
alert_rule_test:
- eval_time: 1m
alertname: HardwareProcessorError
- eval_time: 5m
alertname: HardwareProcessorError
exp_alerts:
- exp_labels:
name: HARDWARE_PROCESSOR
severity: critical
cluster: mycluster
type: ceph_default
oid: 1.3.6.1.4.1.50495.1.2.1.13.3
exp_annotations:
summary: Processor error(s) detected on cluster mycluster
description: "Processor error(s) detected. Check `ceph health detail`."
- interval: 30s
input_series:
- series: 'ceph_health_detail{name="HARDWARE_NETWORK", cluster="mycluster"}'
values: '1+0x40'
promql_expr_test:
- expr: ceph_health_detail{name="HARDWARE_NETWORK"} > 0
eval_time: 2m
exp_samples:
- labels: '{__name__="ceph_health_detail", name="HARDWARE_NETWORK", cluster="mycluster"}'
value: 1
alert_rule_test:
- eval_time: 1m
alertname: HardwareNetworkError
- eval_time: 5m
alertname: HardwareNetworkError
exp_alerts:
- exp_labels:
name: HARDWARE_NETWORK
severity: critical
cluster: mycluster
type: ceph_default
oid: 1.3.6.1.4.1.50495.1.2.1.13.4
exp_annotations:
summary: Network error(s) detected on cluster mycluster
description: "Network error(s) detected. Check `ceph health detail`."
- interval: 30s
input_series:
- series: 'ceph_health_detail{name="HARDWARE_POWER", cluster="mycluster"}'
values: '1+0x40'
promql_expr_test:
- expr: ceph_health_detail{name="HARDWARE_POWER"} > 0
eval_time: 2m
exp_samples:
- labels: '{__name__="ceph_health_detail", name="HARDWARE_POWER", cluster="mycluster"}'
value: 1
alert_rule_test:
- eval_time: 1m
alertname: HardwarePowerError
- eval_time: 5m
alertname: HardwarePowerError
exp_alerts:
- exp_labels:
name: HARDWARE_POWER
category: power
component: PSU1
hostname: node2
severity: critical
cluster: mycluster
type: ceph_default
oid: 1.3.6.1.4.1.50495.1.2.1.13.5
exp_annotations:
summary: Power supply error(s) detected on cluster mycluster
description: "Power supply error(s) detected. Check `ceph health detail`."
- interval: 30s
summary: "Power supply PSU1 failed on node2 on cluster mycluster"
description: "PSU fault detected on node2. PSU PSU1 has failed."
# Cooling degraded
- interval: 1m
input_series:
- series: 'ceph_health_detail{name="HARDWARE_FANS", cluster="mycluster"}'
values: '1+0x40'
- series: 'ceph_hardware_health{category="fans",component="FAN0",hostname="node1",cluster="mycluster"}'
values: '1 1 1 1 1 1 1 1 1'
promql_expr_test:
- expr: ceph_health_detail{name="HARDWARE_FANS"} > 0
- expr: ceph_hardware_health{category="fans"} == 1
eval_time: 2m
exp_samples:
- labels: '{__name__="ceph_health_detail", name="HARDWARE_FANS", cluster="mycluster"}'
- labels: 'ceph_hardware_health{category="fans",component="FAN0",hostname="node1",cluster="mycluster"}'
value: 1
alert_rule_test:
- eval_time: 1m
alertname: HardwareFanError
alertname: CoolingDegraded
- eval_time: 5m
alertname: HardwareFanError
alertname: CoolingDegraded
exp_alerts:
- exp_labels:
name: HARDWARE_FANS
category: fans
component: FAN0
hostname: node1
severity: warning
cluster: mycluster
type: ceph_default
exp_annotations:
summary: "Fan FAN0 degraded on node1 on cluster mycluster"
description: "Fan module FAN0 is in a degraded state on node1."
# Cooling failed
- interval: 1m
input_series:
- series: 'ceph_hardware_health{category="fans",component="FAN1",hostname="node2",cluster="mycluster"}'
values: '2 2 2 2 2 2 2 2 2'
promql_expr_test:
- expr: ceph_hardware_health{category="fans"} == 2
eval_time: 2m
exp_samples:
- labels: 'ceph_hardware_health{category="fans",component="FAN1",hostname="node2",cluster="mycluster"}'
value: 2
alert_rule_test:
- eval_time: 1m
alertname: CoolingFailed
- eval_time: 5m
alertname: CoolingFailed
exp_alerts:
- exp_labels:
category: fans
component: FAN1
hostname: node2
severity: critical
cluster: mycluster
type: ceph_default
oid: 1.3.6.1.4.1.50495.1.2.1.13.6
exp_annotations:
summary: Fan error(s) detected on cluster mycluster
description: "Fan error(s) detected. Check `ceph health detail`."
summary: "Fan FAN1 failed on node2 on cluster mycluster"
description: "A cooling fault has been detected on node2. A cooling FAN within the enclosure has failed."
# NVMe drive degraded
- interval: 1m
input_series:
- series: 'ceph_hardware_health{category="storage",component="nvme0n1",hostname="node1",cluster="mycluster"}'
values: '1 1 1 1 1 1 1 1 1'
promql_expr_test:
- expr: ceph_hardware_health{category="storage"} == 1
eval_time: 2m
exp_samples:
- labels: 'ceph_hardware_health{category="storage",component="nvme0n1",hostname="node1",cluster="mycluster"}'
value: 1
alert_rule_test:
- eval_time: 1m
alertname: NVMeDriveDegraded
- eval_time: 5m
alertname: NVMeDriveDegraded
exp_alerts:
- exp_labels:
category: storage
component: nvme0n1
hostname: node1
severity: warning
cluster: mycluster
type: ceph_default
exp_annotations:
summary: "Storage device nvme0n1 degraded on node1 on cluster mycluster"
description: "Storage device nvme0n1 is in a degraded state on node1."
# NVMe drive failed
- interval: 1m
input_series:
- series: 'ceph_hardware_health{category="storage",component="nvme1n1",hostname="node2",cluster="mycluster"}'
values: '2 2 2 2 2 2 2 2 2'
promql_expr_test:
- expr: ceph_hardware_health{category="storage"} == 2
eval_time: 2m
exp_samples:
- labels: 'ceph_hardware_health{category="storage",component="nvme1n1",hostname="node2",cluster="mycluster"}'
value: 2
alert_rule_test:
- eval_time: 1m
alertname: NVMeDriveFailed
- eval_time: 5m
alertname: NVMeDriveFailed
exp_alerts:
- exp_labels:
category: storage
component: nvme1n1
hostname: node2
severity: critical
cluster: mycluster
type: ceph_default
oid: 1.3.6.1.4.1.50495.1.2.1.13.1
exp_annotations:
summary: "Storage device nvme1n1 failed on node2 on cluster mycluster"
description: "A drive fault has been detected on node2. Storage device nvme1n1 has failed."
# Memory degraded
- interval: 1m
input_series:
- series: 'ceph_hardware_health{category="memory",component="DIMM_A0",hostname="node1",cluster="mycluster"}'
values: '1 1 1 1 1 1 1 1 1'
promql_expr_test:
- expr: ceph_hardware_health{category="memory"} == 1
eval_time: 2m
exp_samples:
- labels: 'ceph_hardware_health{category="memory",component="DIMM_A0",hostname="node1",cluster="mycluster"}'
value: 1
alert_rule_test:
- eval_time: 1m
alertname: MemoryDegraded
- eval_time: 5m
alertname: MemoryDegraded
exp_alerts:
- exp_labels:
category: memory
component: DIMM_A0
hostname: node1
severity: warning
cluster: mycluster
type: ceph_default
exp_annotations:
summary: "Memory module DIMM_A0 degraded on node1 on cluster mycluster"
description: "Memory module DIMM_A0 is in a degraded state on node1."
# Memory failed
- interval: 1m
input_series:
- series: 'ceph_hardware_health{category="memory",component="DIMM_B0",hostname="node2",cluster="mycluster"}'
values: '2 2 2 2 2 2 2 2 2'
promql_expr_test:
- expr: ceph_hardware_health{category="memory"} == 2
eval_time: 2m
exp_samples:
- labels: 'ceph_hardware_health{category="memory",component="DIMM_B0",hostname="node2",cluster="mycluster"}'
value: 2
alert_rule_test:
- eval_time: 1m
alertname: MemoryFailed
- eval_time: 5m
alertname: MemoryFailed
exp_alerts:
- exp_labels:
category: memory
component: DIMM_B0
hostname: node2
severity: critical
cluster: mycluster
type: ceph_default
oid: 1.3.6.1.4.1.50495.1.2.1.13.2
exp_annotations:
summary: "Memory module DIMM_B0 failed on node2 on cluster mycluster"
description: "A memory fault has been detected on node2. Memory module DIMM_B0 has failed."
# Processor degraded
- interval: 1m
input_series:
- series: 'ceph_hardware_health{category="processors",component="CPU0",hostname="node1",cluster="mycluster"}'
values: '1 1 1 1 1 1 1 1 1'
promql_expr_test:
- expr: ceph_hardware_health{category="processors"} == 1
eval_time: 2m
exp_samples:
- labels: 'ceph_hardware_health{category="processors",component="CPU0",hostname="node1",cluster="mycluster"}'
value: 1
alert_rule_test:
- eval_time: 1m
alertname: ProcessorDegraded
- eval_time: 5m
alertname: ProcessorDegraded
exp_alerts:
- exp_labels:
category: processors
component: CPU0
hostname: node1
severity: warning
cluster: mycluster
type: ceph_default
exp_annotations:
summary: "Processor CPU0 degraded on node1 on cluster mycluster"
description: "Processor CPU0 is in a degraded state on node1."
# Processor failed
- interval: 1m
input_series:
- series: 'ceph_hardware_health{category="processors",component="CPU1",hostname="node2",cluster="mycluster"}'
values: '2 2 2 2 2 2 2 2 2'
promql_expr_test:
- expr: ceph_hardware_health{category="processors"} == 2
eval_time: 2m
exp_samples:
- labels: 'ceph_hardware_health{category="processors",component="CPU1",hostname="node2",cluster="mycluster"}'
value: 2
alert_rule_test:
- eval_time: 1m
alertname: ProcessorFailed
- eval_time: 5m
alertname: ProcessorFailed
exp_alerts:
- exp_labels:
category: processors
component: CPU1
hostname: node2
severity: critical
cluster: mycluster
type: ceph_default
oid: 1.3.6.1.4.1.50495.1.2.1.13.3
exp_annotations:
summary: "Processor CPU1 failed on node2 on cluster mycluster"
description: "A processor fault has been detected on node2. Processor CPU1 has failed."
# Network degraded
- interval: 1m
input_series:
- series: 'ceph_hardware_health{category="network",component="eth0",hostname="node1",cluster="mycluster"}'
values: '1 1 1 1 1 1 1 1 1'
promql_expr_test:
- expr: ceph_hardware_health{category="network"} == 1
eval_time: 2m
exp_samples:
- labels: 'ceph_hardware_health{category="network",component="eth0",hostname="node1",cluster="mycluster"}'
value: 1
alert_rule_test:
- eval_time: 1m
alertname: NetworkDegraded
- eval_time: 5m
alertname: NetworkDegraded
exp_alerts:
- exp_labels:
category: network
component: eth0
hostname: node1
severity: warning
cluster: mycluster
type: ceph_default
exp_annotations:
summary: "Network interface eth0 degraded on node1 on cluster mycluster"
description: "Network interface eth0 is in a degraded state on node1."
# Network failed
- interval: 1m
input_series:
- series: 'ceph_hardware_health{category="network",component="eth1",hostname="node2",cluster="mycluster"}'
values: '2 2 2 2 2 2 2 2 2'
promql_expr_test:
- expr: ceph_hardware_health{category="network"} == 2
eval_time: 2m
exp_samples:
- labels: 'ceph_hardware_health{category="network",component="eth1",hostname="node2",cluster="mycluster"}'
value: 2
alert_rule_test:
- eval_time: 1m
alertname: NetworkFailed
- eval_time: 5m
alertname: NetworkFailed
exp_alerts:
- exp_labels:
category: network
component: eth1
hostname: node2
severity: critical
cluster: mycluster
type: ceph_default
oid: 1.3.6.1.4.1.50495.1.2.1.13.4
exp_annotations:
summary: "Network interface eth1 failed on node2 on cluster mycluster"
description: "A network fault has been detected on node2. Network interface eth1 has failed."
# Motherboard temperature high
- interval: 1m
input_series:
- series: 'ceph_hardware_temperature_celsius{sensor_name="MB_TEMP1",hostname="node1",cluster="mycluster"}'
values: '65 65 65 65 65 65 65 65 65'
promql_expr_test:
- expr: ceph_hardware_temperature_celsius{sensor_name=~".*MB_TEMP.*"} > 60
eval_time: 2m
exp_samples:
- labels: 'ceph_hardware_temperature_celsius{sensor_name="MB_TEMP1",hostname="node1",cluster="mycluster"}'
value: 65
alert_rule_test:
- eval_time: 1m
alertname: MotherboardTemperatureHigh
- eval_time: 5m
alertname: MotherboardTemperatureHigh
exp_alerts:
- exp_labels:
sensor_name: MB_TEMP1
hostname: node1
severity: warning
cluster: mycluster
type: ceph_default
exp_annotations:
summary: "Motherboard sensor MB_TEMP1 above 60°C on node1 on cluster mycluster"
description: "Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall."
# DIMM temperature high
- interval: 1m
input_series:
- series: 'ceph_hardware_temperature_celsius{sensor_name="DIMM_A0_TEMP",hostname="node1",cluster="mycluster"}'
values: '85 85 85 85 85 85 85 85 85'
promql_expr_test:
- expr: ceph_hardware_temperature_celsius{sensor_name=~".*DIMM.*_TEMP"} > 80
eval_time: 2m
exp_samples:
- labels: 'ceph_hardware_temperature_celsius{sensor_name="DIMM_A0_TEMP",hostname="node1",cluster="mycluster"}'
value: 85
alert_rule_test:
- eval_time: 1m
alertname: DIMMTemperatureHigh
- eval_time: 5m
alertname: DIMMTemperatureHigh
exp_alerts:
- exp_labels:
sensor_name: DIMM_A0_TEMP
hostname: node1
severity: warning
cluster: mycluster
type: ceph_default
exp_annotations:
summary: "DIMM sensor DIMM_A0_TEMP above 80°C on node1 on cluster mycluster"
description: "Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall."
# Processor temperature high
- interval: 1m
input_series:
- series: 'ceph_hardware_temperature_celsius{sensor_name="CPU_TEMP",hostname="node1",cluster="mycluster"}'
values: '85 85 85 85 85 85 85 85 85'
promql_expr_test:
- expr: ceph_hardware_temperature_celsius{sensor_name=~".*CPU_TEMP"} > 80
eval_time: 2m
exp_samples:
- labels: 'ceph_hardware_temperature_celsius{sensor_name="CPU_TEMP",hostname="node1",cluster="mycluster"}'
value: 85
alert_rule_test:
- eval_time: 1m
alertname: ProcessorTemperatureHigh
- eval_time: 5m
alertname: ProcessorTemperatureHigh
exp_alerts:
- exp_labels:
sensor_name: CPU_TEMP
hostname: node1
severity: warning
cluster: mycluster
type: ceph_default
exp_annotations:
summary: "CPU sensor CPU_TEMP above 80°C on node1 on cluster mycluster"
description: "Prolonged high temperatures could lead to platform instability. Investigate cooling issues and hot-spots in the server room/hall."
# NVMe temperature high
- interval: 1m
input_series:
- series: 'ceph_hardware_temperature_celsius{sensor_name="NVME0_TEMP",hostname="node1",cluster="mycluster"}'
values: '75 75 75 75 75 75 75 75 75'
promql_expr_test:
- expr: ceph_hardware_temperature_celsius{sensor_name=~"NVME.*_TEMP"} > 70
eval_time: 2m
exp_samples:
- labels: 'ceph_hardware_temperature_celsius{sensor_name="NVME0_TEMP",hostname="node1",cluster="mycluster"}'
value: 75
alert_rule_test:
- eval_time: 1m
alertname: NVMeTemperatureHigh
- eval_time: 5m
alertname: NVMeTemperatureHigh
exp_alerts:
- exp_labels:
sensor_name: NVME0_TEMP
hostname: node1
severity: warning
cluster: mycluster
type: ceph_default
exp_annotations:
summary: "NVMe sensor NVME0_TEMP above 70°C on node1 on cluster mycluster"
description: "An NVMe temperature sensor is indicating a potential cooling problem. NVMe may power down if the temperature continues to rise."
# nvmeof Tests
# NVMeoFSubsystemNamespaceLimit
@ -2278,19 +2562,27 @@ tests:
# NVMeoFMultipleNamespacesOfRBDImage
- interval: 1m
input_series:
- series: 'ceph_nvmeof_bdev_metadata{bdev_name="bdev1", instance="ceph-nvme-vm1", pool_name="mypool", rbd_name="myimage1"}'
- series: 'ceph_nvmeof_bdev_metadata{bdev_name="bdev1", instance="ceph-nvme-vm1", cluster="mycluster", pool_name="mypool", rbd_name="myimage1"}'
values: '1x10'
- series: 'ceph_nvmeof_bdev_metadata{bdev_name="bdev1", instance="ceph-nvme-vm2", pool_name="mypool", rbd_name="myimage1"}'
- series: 'ceph_nvmeof_bdev_metadata{bdev_name="bdev1", instance="ceph-nvme-vm2", cluster="mycluster", pool_name="mypool", rbd_name="myimage1"}'
values: '1x10'
- series: 'ceph_nvmeof_bdev_metadata{bdev_name="bdev2", instance="ceph-nvme-vm1", pool_name="mypool", rbd_name="myimage2"}'
- series: 'ceph_nvmeof_bdev_metadata{bdev_name="bdev2", instance="ceph-nvme-vm1", cluster="mycluster", pool_name="mypool", rbd_name="myimage2"}'
values: '1x10'
- series: 'ceph_nvmeof_bdev_metadata{bdev_name="bdev2", instance="ceph-nvme-vm2", pool_name="mypool", rbd_name="myimage2"}'
- series: 'ceph_nvmeof_bdev_metadata{bdev_name="bdev2", instance="ceph-nvme-vm2", cluster="mycluster", pool_name="mypool", rbd_name="myimage2"}'
values: '1x10'
- series: 'ceph_nvmeof_bdev_metadata{bdev_name="bdev3", instance="ceph-nvme-vm1", pool_name="mypool", rbd_name="myimage1"}'
- series: 'ceph_nvmeof_bdev_metadata{bdev_name="bdev3", instance="ceph-nvme-vm1", cluster="mycluster", pool_name="mypool", rbd_name="myimage1"}'
values: '1x10'
- series: 'ceph_nvmeof_bdev_metadata{bdev_name="bdev3", instance="ceph-nvme-vm2", pool_name="mypool", rbd_name="myimage1"}'
- series: 'ceph_nvmeof_bdev_metadata{bdev_name="bdev3", instance="ceph-nvme-vm2", cluster="mycluster", pool_name="mypool", rbd_name="myimage1"}'
values: '1x10'
- series: 'ceph_nvmeof_bdev_metadata{bdev_name="bdev4", instance="ceph-nvme-vm1", pool_name="mypool", rbd_name="myimage1"}' # bdev with no ns
- series: 'ceph_nvmeof_bdev_metadata{bdev_name="bdev4", instance="ceph-nvme-vm1", cluster="mycluster", pool_name="mypool", rbd_name="myimage1"}' # bdev with no ns
values: '1x10'
- series: 'ceph_nvmeof_bdev_metadata{bdev_name="bdev5", instance="ceph-nvme-vm1", cluster="mycluster", pool_name="mypool", rbd_name="myimage5"}'
values: '1x10'
- series: 'ceph_nvmeof_bdev_metadata{bdev_name="bdev5", instance="ceph-nvme-vm2", cluster="mycluster", pool_name="mypool", rbd_name="myimage5"}'
values: '1x10'
- series: 'ceph_nvmeof_bdev_metadata{bdev_name="bdev6", instance="ceph-nvme-vm1", cluster="mycluster", pool_name="mypool", rbd_name="myimage5"}'
values: '1x10'
- series: 'ceph_nvmeof_bdev_metadata{bdev_name="bdev6", instance="ceph-nvme-vm2", cluster="mycluster", pool_name="mypool", rbd_name="myimage5"}'
values: '1x10'
- series: 'ceph_nvmeof_subsystem_namespace_metadata{nqn="nqn1", nsid="1", bdev_name="bdev1", instance="ceph-nvme-vm1", cluster="mycluster"}'
values: '1x10'
@ -2304,8 +2596,16 @@ tests:
values: '1x10'
- series: 'ceph_nvmeof_subsystem_namespace_metadata{nqn="nqn2", nsid="1", bdev_name="bdev3", instance="ceph-nvme-vm2", cluster="mycluster"}'
values: '1x10'
- series: 'ceph_nvmeof_subsystem_namespace_metadata{nqn="nqn3", nsid="1", bdev_name="bdev5", instance="ceph-nvme-vm1", cluster="mycluster", rados_namespace_name="rados1"}'
values: '1x10'
- series: 'ceph_nvmeof_subsystem_namespace_metadata{nqn="nqn3", nsid="1", bdev_name="bdev5", instance="ceph-nvme-vm2", cluster="mycluster", rados_namespace_name="rados1"}'
values: '1x10'
- series: 'ceph_nvmeof_subsystem_namespace_metadata{nqn="nqn3", nsid="2", bdev_name="bdev6", instance="ceph-nvme-vm1", cluster="mycluster", rados_namespace_name="rados2"}'
values: '1x10'
- series: 'ceph_nvmeof_subsystem_namespace_metadata{nqn="nqn3", nsid="2", bdev_name="bdev6", instance="ceph-nvme-vm2", cluster="mycluster", rados_namespace_name="rados2"}'
values: '1x10'
promql_expr_test:
- expr: count by(pool_name, rbd_name) (count by(bdev_name, pool_name, rbd_name) (ceph_nvmeof_bdev_metadata and on (bdev_name) ceph_nvmeof_subsystem_namespace_metadata)) > 1
- expr: count by(pool_name, rbd_name, rados_namespace_name) (count by(bdev_name, pool_name, rbd_name, rados_namespace_name) (ceph_nvmeof_bdev_metadata * on (bdev_name, instance, cluster) group_left(rados_namespace_name) ceph_nvmeof_subsystem_namespace_metadata)) > 1
eval_time: 1m
exp_samples:
- labels: '{pool_name="mypool", rbd_name="myimage1"}'
@ -2320,7 +2620,7 @@ tests:
severity: warning
type: ceph_default
exp_annotations:
summary: "RBD image mypool/myimage1 cannot be reused for multiple NVMeoF namespace "
summary: "RBD image mypool/myimage1 cannot be reused for multiple NVMeoF namespaces"
description: "Each NVMeoF namespace must have a unique RBD pool and image, across all different gateway groups."
# NVMeoFTooManyGateways

3
monitoring/ceph-mixin/tests_dashboards/features/ceph-cluster.feature
View File

@ -1,5 +1,8 @@
Feature: Ceph Cluster Dashboard
Background:
Given the dashboard is `ceph-cluster-advanced`
Scenario: "Test cluster health"
Given the following series:
| metrics | values |

26
monitoring/ceph-mixin/tests_dashboards/features/environment.py
View File

@ -15,6 +15,7 @@ class GlobalContext:
self.promql_expr_test = None
self.data = get_dashboards_data()
self.query_map = self.data['queries']
self.dashboard = None
def reset_promql_test(self):
self.promql_expr_test = PromqlTest()
@ -53,6 +54,7 @@ global_context = GlobalContext()
def before_scenario(context, scenario):
global_context.reset_promql_test()
global_context.dashboard = None
def after_scenario(context, scenario):
@ -63,6 +65,14 @@ def after_all(context):
global_context.print_query_stats()
@given('the dashboard is `{dashboard}`')
def step_impl(context, dashboard):
if dashboard not in global_context.query_map:
raise KeyError(f'Unknown dashboard "{dashboard}". Known: '
f'{sorted(global_context.query_map)}')
global_context.dashboard = dashboard
@given("the following series")
def step_impl(context):
for row in context.table:
@ -111,19 +121,23 @@ def step_impl(context, panel_name, legend):
"""
if legend == "EMPTY":
legend = ''
if global_context.dashboard is None:
raise KeyError('No dashboard selected; add "Given the dashboard is '
'`<name>`" to the feature Background')
query_id = panel_name + '-' + legend
if query_id not in global_context.query_map:
print(f"QueryMap: {global_context.query_map}")
raise KeyError((f'Query with legend {legend} in panel "{panel_name}"'
'couldn\'t be found'))
dashboard_queries = global_context.query_map[global_context.dashboard]
if query_id not in dashboard_queries:
raise KeyError(f'Query with legend {legend} in panel "{panel_name}" '
f'couldn\'t be found in dashboard '
f'"{global_context.dashboard}"')
expr = global_context.query_map[query_id]['query']
expr = dashboard_queries[query_id]['query']
global_context.promql_expr_test.set_expression(expr)
for row in context.table:
metric = row['metrics']
value = row['values']
global_context.promql_expr_test.add_exp_samples(metric, float(value))
path = global_context.query_map[query_id]['path']
path = dashboard_queries[query_id]['path']
global_context.data['stats'][path]['tested'] += 1

3
monitoring/ceph-mixin/tests_dashboards/features/host-details.feature
View File

@ -1,5 +1,8 @@
Feature: Host Details Dashboard
Background:
Given the dashboard is `host-details`
Scenario: "Test OSD"
Given the following series:
| metrics | values |

3
monitoring/ceph-mixin/tests_dashboards/features/hosts_overview.feature
View File

@ -1,5 +1,8 @@
Feature: Hosts Overview Dashboard
Background:
Given the dashboard is `hosts-overview`
Scenario: "Test network load succeeds"
Given the following series:
| metrics | values |

3
monitoring/ceph-mixin/tests_dashboards/features/osd-device-details.feature
View File

@ -1,5 +1,8 @@
Feature: OSD device details
Background:
Given the dashboard is `osd-device-details`
Scenario: "Test Physical Device Latency for $osd - Reads"
Given the following series:
| metrics | values |

Some files were not shown because too many files have changed in this diff Show More