From 80b90672860cafe32118c9a0f2f80afdd9b57f68 Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Thu, 25 Jun 2026 14:59:35 +0800 Subject: [PATCH] blk,build: drop BlueStore PMEM support BlueStore's PMEMDevice backend targets byte-addressable persistent memory exposed as a DAX device, in practice Intel Optane DC persistent memory. Intel Optane has been discontinued since 2021 and there is no mainstream replacement, so the backend has effectively no hardware left to run on. It was always experimental and has seen no functional change since the DML/DSA offload landed in 2022; every commit to it since has been a formatting sweep. Keeping it builds in a chain of dependencies that nothing else needs: libpmem, libndctl and libdaxctl, plus dml for the DSA offload that only works on a Sapphire Rapids CPU. Remove the backend and everything tied to it. RBD persistent write-back cache (WITH_RBD_RWL) still uses PMDK's libpmemobj, so the pmdk submodule, WITH_SYSTEM_PMDK, Buildpmdk and Findpmdk stay. WITH_SYSTEM_PMDK now depends on WITH_RBD_RWL alone, and Buildpmdk no longer builds ndctl support. - remove the WITH_BLUESTORE_PMEM option, HAVE_BLUESTORE_PMEM and HAVE_LIBDML - delete src/blk/pmem/PMEMDevice.{cc,h} and the pmem block_device_t - drop the pmem value from the bdev_type option - remove the --bluestore-pmem vstart flag - drop the libpmem, libndctl and libdaxctl build deps from debian and rpm, keeping libpmemobj for RWL - delete the Finddml, Findndctl and Finddaxctl cmake modules - remove the DSA/DML and ndctl documentation Signed-off-by: Kefu Chai --- CMakeLists.txt | 18 +- PendingReleaseNotes | 12 + ceph.spec.in | 7 - cmake/modules/Buildpmdk.cmake | 10 +- cmake/modules/Finddaxctl.cmake | 42 -- cmake/modules/Finddml.cmake | 58 --- cmake/modules/Findndctl.cmake | 42 -- debian/control | 3 - doc/dev/continuous-integration.rst | 4 +- .../configuration/bluestore-config-ref.rst | 21 - src/CMakeLists.txt | 20 +- src/blk/BlockDevice.cc | 18 - src/blk/BlockDevice.h | 3 - src/blk/CMakeLists.txt | 14 - src/blk/pmem/PMEMDevice.cc | 379 ------------------ src/blk/pmem/PMEMDevice.h | 79 ---- src/common/options/global.yaml.in | 1 - src/include/config-h.in.cmake | 6 - src/vstart.sh | 14 +- 19 files changed, 22 insertions(+), 729 deletions(-) delete mode 100644 cmake/modules/Finddaxctl.cmake delete mode 100644 cmake/modules/Finddml.cmake delete mode 100644 cmake/modules/Findndctl.cmake delete mode 100644 src/blk/pmem/PMEMDevice.cc delete mode 100644 src/blk/pmem/PMEMDevice.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 43ce6ef4147..993ff40bece 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -223,7 +223,6 @@ if(LINUX) elseif(FREEBSD) set(HAVE_UDEV OFF) set(HAVE_LIBAIO OFF) - set(HAVE_LIBDML OFF) set(HAVE_BLKID OFF) set(HAVE_KEYUTILS OFF) else() @@ -304,13 +303,6 @@ if(WITH_LIBURING) endif() endif() -CMAKE_DEPENDENT_OPTION(WITH_BLUESTORE_PMEM "Enable PMDK libraries" OFF - "WITH_BLUESTORE" OFF) -if(WITH_BLUESTORE_PMEM) - find_package(dml) - set(HAVE_LIBDML ${DML_FOUND}) -endif() - CMAKE_DEPENDENT_OPTION(WITH_RBD_MIRROR "Enable build for rbd-mirror daemon executable" OFF "WITH_RBD" OFF) @@ -325,7 +317,7 @@ CMAKE_DEPENDENT_OPTION(WITH_RBD_SSD_CACHE "Enable librbd persistent write back c "WITH_RBD" OFF) CMAKE_DEPENDENT_OPTION(WITH_SYSTEM_PMDK "Require and build with system PMDK" OFF - "WITH_RBD_RWL OR WITH_BLUESTORE_PMEM" OFF) + "WITH_RBD_RWL" OFF) CMAKE_DEPENDENT_OPTION(WITH_RBD_UBBD "Enable ubbd support for rbd device utility" OFF "WITH_RBD" OFF) @@ -335,10 +327,6 @@ if(WITH_RBD_UBBD) build_ubbd() endif() -if(WITH_BLUESTORE_PMEM) - set(HAVE_BLUESTORE_PMEM ON) -endif() - CMAKE_DEPENDENT_OPTION(WITH_SPDK "Enable SPDK" OFF "CMAKE_SYSTEM_PROCESSOR MATCHES i386|i686|amd64|x86_64|AMD64|aarch64|riscv64" OFF) option(WITH_SYSTEM_SPDK "Require a system SPDK instead of building bundled src/spdk" OFF) @@ -356,10 +344,10 @@ if(WITH_SPDK) endif(WITH_SPDK) if(WITH_BLUESTORE) - if(NOT AIO_FOUND AND NOT HAVE_POSIXAIO AND NOT WITH_SPDK AND NOT WITH_BLUESTORE_PMEM) + if(NOT AIO_FOUND AND NOT HAVE_POSIXAIO AND NOT WITH_SPDK) message(SEND_ERROR "WITH_BLUESTORE is ON, " "but none of the bluestore backends is enabled. " - "Please install libaio, or enable WITH_SPDK or WITH_BLUESTORE_PMEM (experimental)") + "Please install libaio, or enable WITH_SPDK") endif() endif() diff --git a/PendingReleaseNotes b/PendingReleaseNotes index 8afbcfe66de..d0758af5bf1 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -262,6 +262,18 @@ rather rigid and also challenging to disseminate in some environments. The key can be embedded in the migration spec or just referenced from there while stored in the MON config-key store. +* BlueStore: The experimental PMEM device backend has been removed, together + with the ``pmem`` value of the ``bdev_type`` option and the DML/DSA offload + path. The hardware it targeted, Intel Optane DC persistent memory, was + discontinued in 2022. The RBD persistent write-back cache, which also uses + PMDK, is unaffected. + + Existing OSDs backed by persistent memory in fsdax mode (a block device such + as ``/dev/pmem0``) keep working on the default ``aio`` backend over the same + data. Remove any explicit ``bdev_type = pmem`` from their configuration. OSDs + on a devdax namespace (a character device such as ``/dev/dax0.0``) will no + longer start, because no remaining backend can open one; reconfigure the + namespace to fsdax with ``ndctl`` or re-provision them before upgrading. >=20.0.0 diff --git a/ceph.spec.in b/ceph.spec.in index 10ccf569956..9033a3c9e3a 100644 --- a/ceph.spec.in +++ b/ceph.spec.in @@ -358,13 +358,6 @@ BuildRequires: pkgconfig(nlohmann_json) BuildRequires: pkgconfig(libevent) %endif %if 0%{with system_pmdk} -%if 0%{?suse_version} -BuildRequires: pkgconfig(libndctl) >= 63 -%else -BuildRequires: pkgconfig(libndctl) >= 63 -BuildRequires: pkgconfig(libdaxctl) >= 63 -%endif -BuildRequires: pkgconfig(libpmem) BuildRequires: pkgconfig(libpmemobj) >= 1.8 %endif %if 0%{with system_arrow} diff --git a/cmake/modules/Buildpmdk.cmake b/cmake/modules/Buildpmdk.cmake index 03a17b99436..7377fc9d5c8 100644 --- a/cmake/modules/Buildpmdk.cmake +++ b/cmake/modules/Buildpmdk.cmake @@ -1,4 +1,4 @@ -function(build_pmdk enable_ndctl) +function(build_pmdk) include(FindMake) find_make("MAKE_EXECUTABLE" "make_cmd") @@ -15,12 +15,6 @@ function(build_pmdk enable_ndctl) endif() set(LIBPMEM_INTERFACE_LINK_LIBRARIES Threads::Threads) - if(${enable_ndctl}) - set(ndctl "y") - list(APPEND LIBPMEM_INTERFACE_LINK_LIBRARIES ndctl::ndctl daxctl::daxctl) - else() - set(ndctl "n") - endif() # Use debug PMDK libs in debug lib/rbd builds if(CMAKE_BUILD_TYPE STREQUAL Debug) @@ -34,7 +28,7 @@ function(build_pmdk enable_ndctl) ExternalProject_Add(pmdk_ext ${source_dir_args} CONFIGURE_COMMAND "" - BUILD_COMMAND ${make_cmd} CC=${CMAKE_C_COMPILER} "EXTRA_CFLAGS=${pmdk_cflags}" NDCTL_ENABLE=${ndctl} BUILD_EXAMPLES=n BUILD_BENCHMARKS=n DOC=n + BUILD_COMMAND ${make_cmd} CC=${CMAKE_C_COMPILER} "EXTRA_CFLAGS=${pmdk_cflags}" NDCTL_ENABLE=n BUILD_EXAMPLES=n BUILD_BENCHMARKS=n DOC=n BUILD_IN_SOURCE 1 BUILD_BYPRODUCTS "/src/${PMDK_LIB_DIR}/libpmem.a" "/src/${PMDK_LIB_DIR}/libpmemobj.a" INSTALL_COMMAND "") diff --git a/cmake/modules/Finddaxctl.cmake b/cmake/modules/Finddaxctl.cmake deleted file mode 100644 index fbe58042466..00000000000 --- a/cmake/modules/Finddaxctl.cmake +++ /dev/null @@ -1,42 +0,0 @@ -# - Find libdaxctl -# Find the daxctl libraries and includes -# -# daxctl_INCLUDE_DIR - where to find libdaxctl.h etc. -# daxctl_LIBRARIES - List of libraries when using daxctl. -# daxctl_FOUND - True if daxctl found. - -find_path(daxctl_INCLUDE_DIR daxctl/libdaxctl.h) - -if(daxctl_INCLUDE_DIR AND EXISTS "${daxctl_INCLUDE_DIR}/libdaxctl.h") - foreach(ver "MAJOR" "MINOR" "RELEASE") - file(STRINGS "${daxctl_INCLUDE_DIR}/libdaxctl.h" daxctl_VER_${ver}_LINE - REGEX "^#define[ \t]+daxctl_VERSION_${ver}[ \t]+[0-9]+[ \t]+.*$") - string(REGEX REPLACE "^#define[ \t]+daxctl_VERSION_${ver}[ \t]+([0-9]+)[ \t]+.*$" - "\\1" daxctl_VERSION_${ver} "${daxctl_VER_${ver}_LINE}") - unset(${daxctl_VER_${ver}_LINE}) - endforeach() - set(daxctl_VERSION_STRING - "${daxctl_VERSION_MAJOR}.${daxctl_VERSION_MINOR}.${daxctl_VERSION_RELEASE}") -endif() - -find_library(daxctl_LIBRARY daxctl) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(daxctl - REQUIRED_VARS daxctl_LIBRARY daxctl_INCLUDE_DIR - VERSION_VAR daxctl_VERSION_STRING) - -mark_as_advanced(daxctl_INCLUDE_DIR daxctl_LIBRARY) - -if(daxctl_FOUND) - set(daxctl_INCLUDE_DIRS ${daxctl_INCLUDE_DIR}) - set(daxctl_LIBRARIES ${daxctl_LIBRARY}) - if(NOT (TARGET daxctl::daxctl)) - add_library(daxctl::daxctl UNKNOWN IMPORTED) - set_target_properties(daxctl::daxctl PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${daxctl_INCLUDE_DIRS}" - IMPORTED_LINK_INTERFACE_LANGUAGES "C" - IMPORTED_LOCATION "${daxctl_LIBRARIES}" - VERSION "${daxctl_VERSION_STRING}") - endif() -endif() diff --git a/cmake/modules/Finddml.cmake b/cmake/modules/Finddml.cmake deleted file mode 100644 index 8e94ad26d6c..00000000000 --- a/cmake/modules/Finddml.cmake +++ /dev/null @@ -1,58 +0,0 @@ -# - Find libdml -# Find the dml and dmlhl libraries and includes -# -# DML_INCLUDE_DIR - where to find dml.hpp etc. -# DML_LIBRARIES - List of libraries when using dml. -# DML_HL_LIBRARIES - List of libraries when using dmlhl. -# DML_FOUND - True if DML found. - - -find_path(DML_INCLUDE_DIR - dml/dml.hpp - PATHS - /usr/include - /usr/local/include) - -find_library(DML_LIBRARIES NAMES dml libdml PATHS - /usr/local/ - /usr/local/lib64 - /usr/lib64 - /usr/lib) - -find_library(DML_HL_LIBRARIES NAMES dmlhl libdmlhl PATHS - /usr/local/ - /usr/local/lib64 - /usr/lib64 - /usr/lib) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(dml DEFAULT_MSG - DML_LIBRARIES - DML_INCLUDE_DIR - DML_HL_LIBRARIES) - -mark_as_advanced( - DML_LIBRARIES - DML_INCLUDE_DIR - DML_HL_LIBRARIES) - -if(DML_FOUND) - if(NOT (TARGET dml::dml)) - add_library(dml::dml UNKNOWN IMPORTED) - set_target_properties(dml::dml PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${DML_INCLUDE_DIR}" - IMPORTED_LINK_INTERFACE_LANGUAGES "C" - IMPORTED_LOCATION "${DML_LIBRARIES}") - endif() - - if(NOT (TARGET dml::dmlhl)) - add_library(dml::dmlhl UNKNOWN IMPORTED) - set_target_properties(dml::dmlhl PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${DML_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES ${CMAKE_DL_LIBS} - INTERFACE_COMPILE_FEATURES cxx_std_17 - INTERFACE_COMPILE_DEFINITIONS "DML_HW" - IMPORTED_LINK_INTERFACE_LANGUAGES "CXX" - IMPORTED_LOCATION "${DML_HL_LIBRARIES}") - endif() -endif() diff --git a/cmake/modules/Findndctl.cmake b/cmake/modules/Findndctl.cmake deleted file mode 100644 index 12afa1781f7..00000000000 --- a/cmake/modules/Findndctl.cmake +++ /dev/null @@ -1,42 +0,0 @@ -# - Find libndctl -# Find the ndctl libraries and includes -# -# ndctl_INCLUDE_DIR - where to find libndctl.h etc. -# ndctl_LIBRARIES - List of libraries when using ndctl. -# ndctl_FOUND - True if ndctl found. - -find_path(ndctl_INCLUDE_DIR ndctl/libndctl.h) - -if(ndctl_INCLUDE_DIR AND EXISTS "${ndctl_INCLUDE_DIR}/libndctl.h") - foreach(ver "MAJOR" "MINOR" "RELEASE") - file(STRINGS "${ndctl_INCLUDE_DIR}/libndctl.h" ndctl_VER_${ver}_LINE - REGEX "^#define[ \t]+ndctl_VERSION_${ver}[ \t]+[0-9]+[ \t]+.*$") - string(REGEX REPLACE "^#define[ \t]+ndctl_VERSION_${ver}[ \t]+([0-9]+)[ \t]+.*$" - "\\1" ndctl_VERSION_${ver} "${ndctl_VER_${ver}_LINE}") - unset(${ndctl_VER_${ver}_LINE}) - endforeach() - set(ndctl_VERSION_STRING - "${ndctl_VERSION_MAJOR}.${ndctl_VERSION_MINOR}.${ndctl_VERSION_RELEASE}") -endif() - -find_library(ndctl_LIBRARY ndctl) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(ndctl - REQUIRED_VARS ndctl_LIBRARY ndctl_INCLUDE_DIR - VERSION_VAR ndctl_VERSION_STRING) - -mark_as_advanced(ndctl_INCLUDE_DIR ndctl_LIBRARY) - -if(ndctl_FOUND) - set(ndctl_INCLUDE_DIRS ${ndctl_INCLUDE_DIR}) - set(ndctl_LIBRARIES ${ndctl_LIBRARY}) - if(NOT (TARGET ndctl::ndctl)) - add_library(ndctl::ndctl UNKNOWN IMPORTED) - set_target_properties(ndctl::ndctl PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${ndctl_INCLUDE_DIRS}" - IMPORTED_LINK_INTERFACE_LANGUAGES "C" - IMPORTED_LOCATION "${ndctl_LIBRARIES}" - VERSION "${ndctl_VERSION_STRING}") - endif() -endif() diff --git a/debian/control b/debian/control index b91fd483239..9ae0fd90eed 100644 --- a/debian/control +++ b/debian/control @@ -80,9 +80,6 @@ Build-Depends: automake, libthrift-dev (>= 0.13.0), libyaml-cpp-dev (>= 0.6), libzstd-dev , - libdaxctl-dev (>= 63) , - libndctl-dev (>= 63) , - libpmem-dev , libpmemobj-dev (>= 1.8) , libprotobuf-dev, libxsimd-dev , diff --git a/doc/dev/continuous-integration.rst b/doc/dev/continuous-integration.rst index 091f4f154a4..a33ba85e711 100644 --- a/doc/dev/continuous-integration.rst +++ b/doc/dev/continuous-integration.rst @@ -206,9 +206,7 @@ libzbd packages `libzbd`_ . The upstream libzbd includes debian packaging already. libpmem - packages `pmdk`_ . Please note, ``ndctl`` is one of the build dependencies of - pmdk, for an updated debian packaging, please see - https://github.com/ceph/ceph-ndctl . + packages `pmdk`_ . .. note:: diff --git a/doc/rados/configuration/bluestore-config-ref.rst b/doc/rados/configuration/bluestore-config-ref.rst index 9bb8b20d64e..6dcc11036cb 100644 --- a/doc/rados/configuration/bluestore-config-ref.rst +++ b/doc/rados/configuration/bluestore-config-ref.rst @@ -535,24 +535,3 @@ were deployed under older releases or with other settings. .. confval:: bluestore_min_alloc_size_hdd .. confval:: bluestore_min_alloc_size_ssd .. confval:: bluestore_use_optimal_io_size_for_min_alloc_size - -DSA (Data Streaming Accelerator) Usage -====================================== - -If you want to use the DML library to drive the DSA device for offloading -read/write operations on persistent memory (PMEM) in BlueStore, you need to -install `DML`_ and the `idxd-config`_ library. This will work only on machines -that have a SPR (Sapphire Rapids) CPU. - -.. _DML: https://github.com/intel/DML -.. _idxd-config: https://github.com/intel/idxd-config - -After installing the DML software, configure the shared work queues (WQs) with -reference to the following WQ configuration example: - -.. prompt:: bash $ - - accel-config config-wq --group-id=1 --mode=shared --wq-size=16 --threshold=15 --type=user --name="MyApp1" --priority=10 --block-on-fault=1 dsa0/wq0.1 - accel-config config-engine dsa0/engine0.1 --group-id=1 - accel-config enable-device dsa0 - accel-config enable-wq dsa0/wq0.1 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4eb283d4959..03c2ee878e3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -654,26 +654,12 @@ if(WIN32) list(APPEND ceph_common_deps dlfcn_win32) endif() -if(WITH_BLUESTORE_PMEM OR WITH_RBD_RWL) +if(WITH_RBD_RWL) if(WITH_SYSTEM_PMDK) - set(pmdk_COMPONENTS) - if(WITH_BLUESTORE_PMEM) - list(APPEND pmdk_COMPONENTS pmem) - endif() - if(WITH_RBD_RWL) - list(APPEND pmdk_COMPONENTS pmemobj) - endif() - find_package(pmdk 1.8 REQUIRED COMPONENTS ${pmdk_COMPONENTS}) + find_package(pmdk 1.8 REQUIRED COMPONENTS pmemobj) else() include(Buildpmdk) - if(WITH_BLUESTORE_PMEM) - set(enable_ndctl ON) - find_package(ndctl 63 REQUIRED) - find_package(daxctl 63 REQUIRED) - else() - set(enable_ndctl OFF) - endif() - build_pmdk(${enable_ndctl}) + build_pmdk() endif() endif() diff --git a/src/blk/BlockDevice.cc b/src/blk/BlockDevice.cc index 871287cd73a..ebc073700b2 100644 --- a/src/blk/BlockDevice.cc +++ b/src/blk/BlockDevice.cc @@ -28,10 +28,6 @@ #include "spdk/NVMEDevice.h" #endif -#if defined(HAVE_BLUESTORE_PMEM) -#include "pmem/PMEMDevice.h" -#endif - #include "common/debug.h" #include "common/EventTrace.h" #include "common/errno.h" @@ -106,11 +102,6 @@ BlockDevice::detect_device_type(const std::string& path) return block_device_t::spdk; } #endif -#if defined(HAVE_BLUESTORE_PMEM) - if (PMEMDevice::support(path)) { - return block_device_t::pmem; - } -#endif #if defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO) return block_device_t::aio; #else @@ -130,11 +121,6 @@ BlockDevice::device_type_from_name(const std::string& blk_dev_name) if (blk_dev_name == "spdk") { return block_device_t::spdk; } -#endif -#if defined(HAVE_BLUESTORE_PMEM) - if (blk_dev_name == "pmem") { - return block_device_t::pmem; - } #endif return block_device_t::unknown; } @@ -152,10 +138,6 @@ BlockDevice* BlockDevice::create_with_type(block_device_t device_type, #if defined(HAVE_SPDK) case block_device_t::spdk: return new NVMEDevice(cct, cb, cbpriv); -#endif -#if defined(HAVE_BLUESTORE_PMEM) - case block_device_t::pmem: - return new PMEMDevice(cct, cb, cbpriv); #endif default: ceph_abort_msg("unsupported device"); diff --git a/src/blk/BlockDevice.h b/src/blk/BlockDevice.h index c5ff531e7e7..6c9725a0a32 100644 --- a/src/blk/BlockDevice.h +++ b/src/blk/BlockDevice.h @@ -164,9 +164,6 @@ private: #endif #if defined(HAVE_SPDK) spdk, -#endif -#if defined(HAVE_BLUESTORE_PMEM) - pmem, #endif }; std::queue stalled_read_event_queue; diff --git a/src/blk/CMakeLists.txt b/src/blk/CMakeLists.txt index 6a4ebd502b9..01680c1d039 100644 --- a/src/blk/CMakeLists.txt +++ b/src/blk/CMakeLists.txt @@ -10,11 +10,6 @@ if(HAVE_LIBAIO OR HAVE_POSIXAIO) aio/aio.cc) endif() -if(WITH_BLUESTORE_PMEM) - list(APPEND libblk_srcs - pmem/PMEMDevice.cc) -endif() - if(WITH_SPDK) list(APPEND libblk_srcs spdk/NVMEDevice.cc) @@ -36,15 +31,6 @@ if(WITH_SPDK) PRIVATE spdk::spdk) endif() -if(WITH_BLUESTORE_PMEM) - if(HAVE_LIBDML) - target_link_libraries(blk PRIVATE dml::dml dml::dmlhl) - endif() - - target_link_libraries(blk - PRIVATE pmdk::pmem) -endif() - if(WITH_EVENTTRACE) add_dependencies(blk eventtrace_tp) endif() diff --git a/src/blk/pmem/PMEMDevice.cc b/src/blk/pmem/PMEMDevice.cc deleted file mode 100644 index 0f309d2c444..00000000000 --- a/src/blk/pmem/PMEMDevice.cc +++ /dev/null @@ -1,379 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*- -// vim: ts=8 sw=2 sts=2 expandtab - -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2015 Intel - * - * Author: Jianpeng Ma - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "PMEMDevice.h" -#include "libpmem.h" -#include "include/types.h" -#include "include/compat.h" -#include "include/stringify.h" -#include "common/errno.h" -#include "common/debug.h" -#include "common/blkdev.h" - -#if defined(HAVE_LIBDML) -#include -using execution_path = dml::automatic; -#endif - -#define dout_context cct -#define dout_subsys ceph_subsys_bdev -#undef dout_prefix -#define dout_prefix *_dout << "bdev-PMEM(" << path << ") " - -PMEMDevice::PMEMDevice(CephContext *cct, aio_callback_t cb, void *cbpriv) - : BlockDevice(cct, cb, cbpriv), - fd(-1), addr(0), - injecting_crash(0) -{ -} - -int PMEMDevice::_lock() -{ - struct flock l; - memset(&l, 0, sizeof(l)); - l.l_type = F_WRLCK; - l.l_whence = SEEK_SET; - l.l_start = 0; - l.l_len = 0; - int r = ::fcntl(fd, F_SETLK, &l); - if (r < 0) - return -errno; - return 0; -} - -static int pmem_check_file_type(int fd, const char *pmem_file, uint64_t *total_size) -{ - namespace fs = std::filesystem; - if (!fs::is_character_file(pmem_file)) { - return -EINVAL; - } - struct stat file_stat; - if (::fstat(fd, &file_stat)) { - return -EINVAL; - } - fs::path char_dir = fmt::format("/sys/dev/char/{}:{}", - major(file_stat.st_rdev), - minor(file_stat.st_rdev)); - // Need to check if it is a DAX device - if (auto subsys_path = char_dir / "subsystem"; - fs::read_symlink(subsys_path).filename().string() != "dax") { - return -EINVAL; - } - if (total_size == nullptr) { - return 0; - } - if (std::ifstream size_file(char_dir / "size"); size_file) { - size_file >> *total_size; - return size_file ? 0 : -EINVAL; - } else { - return -EINVAL; - } -} - -int PMEMDevice::open(const std::string& p) -{ - path = p; - int r = 0; - dout(1) << __func__ << " path " << path << dendl; - - fd = ::open(path.c_str(), O_RDWR | O_CLOEXEC); - if (fd < 0) { - r = -errno; - derr << __func__ << " open got: " << cpp_strerror(r) << dendl; - return r; - } - - r = pmem_check_file_type(fd, path.c_str(), &size); - if (!r) { - dout(1) << __func__ << " This path " << path << " is a devdax dev " << dendl; - devdax_device = true; - // If using devdax char device, set it to not rotational device. - rotational = false; - } - - r = _lock(); - if (r < 0) { - derr << __func__ << " failed to lock " << path << ": " << cpp_strerror(r) - << dendl; - goto out_fail; - } - - struct stat st; - r = ::fstat(fd, &st); - if (r < 0) { - r = -errno; - derr << __func__ << " fstat got " << cpp_strerror(r) << dendl; - goto out_fail; - } - - size_t map_len; - addr = (char *)pmem_map_file(path.c_str(), 0, - devdax_device ? 0: PMEM_FILE_EXCL, O_RDWR, - &map_len, NULL); - if (addr == NULL) { - derr << __func__ << " pmem_map_file failed: " << pmem_errormsg() << dendl; - goto out_fail; - } - size = map_len; - - // Operate as though the block size is 4 KB. The backing file - // blksize doesn't strictly matter except that some file systems may - // require a read/modify/write if we write something smaller than - // it. - block_size = g_conf()->bdev_block_size; - if (block_size != (unsigned)st.st_blksize) { - dout(1) << __func__ << " backing device/file reports st_blksize " - << st.st_blksize << ", using bdev_block_size " - << block_size << " anyway" << dendl; - } - - dout(1) << __func__ - << " size " << size - << " (" << byte_u_t(size) << ")" - << " block_size " << block_size - << " (" << byte_u_t(block_size) << ")" - << dendl; - return 0; - - out_fail: - VOID_TEMP_FAILURE_RETRY(::close(fd)); - fd = -1; - return r; -} - -void PMEMDevice::close() -{ - dout(1) << __func__ << dendl; - - ceph_assert(addr != NULL); - if (devdax_device) { - devdax_device = false; - } - pmem_unmap(addr, size); - - ceph_assert(fd >= 0); - VOID_TEMP_FAILURE_RETRY(::close(fd)); - fd = -1; - - path.clear(); -} - -int PMEMDevice::collect_metadata(const std::string& prefix, std::map *pm) const -{ - (*pm)[prefix + "rotational"] = stringify((int)(bool)rotational); - (*pm)[prefix + "size"] = stringify(get_size()); - (*pm)[prefix + "block_size"] = stringify(get_block_size()); - (*pm)[prefix + "driver"] = "PMEMDevice"; - (*pm)[prefix + "type"] = "ssd"; - - struct stat st; - int r = ::fstat(fd, &st); - if (r < 0) - return -errno; - if (S_ISBLK(st.st_mode)) { - (*pm)[prefix + "access_mode"] = "blk"; - char buffer[1024] = {0}; - BlkDev blkdev(fd); - - blkdev.model(buffer, sizeof(buffer)); - (*pm)[prefix + "model"] = buffer; - - buffer[0] = '\0'; - blkdev.dev(buffer, sizeof(buffer)); - (*pm)[prefix + "dev"] = buffer; - - // nvme exposes a serial number - buffer[0] = '\0'; - blkdev.serial(buffer, sizeof(buffer)); - (*pm)[prefix + "serial"] = buffer; - - } else if (S_ISCHR(st.st_mode)) { - (*pm)[prefix + "access_mode"] = "chardevice"; - (*pm)[prefix + "path"] = path; - - } else { - (*pm)[prefix + "access_mode"] = "file"; - (*pm)[prefix + "path"] = path; - } - return 0; -} - -bool PMEMDevice::support(const std::string &path) -{ - int is_pmem = 0; - size_t map_len = 0; - int r = 0; - int local_fd; - - local_fd = ::open(path.c_str(), O_RDWR); - if (local_fd < 0) { - return false; - } - - r = pmem_check_file_type(local_fd, path.c_str(), NULL); - VOID_TEMP_FAILURE_RETRY(::close(local_fd)); - int flags = PMEM_FILE_EXCL; - if (r == 0) { - flags = 0; - } - - void *addr = pmem_map_file(path.c_str(), 0, flags, O_RDONLY, &map_len, &is_pmem); - if (addr != NULL) { - pmem_unmap(addr, map_len); - if (is_pmem) { - return true; - } - } - - return false; -} - -int PMEMDevice::flush() -{ - //Because all write is persist. So no need - return 0; -} - - -void PMEMDevice::aio_submit(IOContext *ioc) -{ - if (ioc->priv) { - ceph_assert(ioc->num_running == 0); - aio_callback(aio_callback_priv, ioc->priv); - } else { - ioc->try_aio_wake(); - } - return; -} - -int PMEMDevice::write(uint64_t off, bufferlist& bl, bool buffered, int write_hint) -{ - uint64_t len = bl.length(); - dout(20) << __func__ << " " << off << "~" << len << dendl; - ceph_assert(is_valid_io(off, len)); - - dout(40) << "data:\n"; - bl.hexdump(*_dout); - *_dout << dendl; - - if (g_conf()->bdev_inject_crash && - rand() % g_conf()->bdev_inject_crash == 0) { - derr << __func__ << " bdev_inject_crash: dropping io " << off << "~" << len - << dendl; - ++injecting_crash; - return 0; - } - - bufferlist::iterator p = bl.begin(); - uint64_t off1 = off; - while (len) { - const char *data; - uint32_t l = p.get_ptr_and_advance(len, &data); - -#if defined(HAVE_LIBDML) - // Take care of the persistency issue - auto result = dml::execute(dml::mem_move, dml::make_view(data, l), dml::make_view(addr + off1, l)); - ceph_assert(result.status == dml::status_code::ok); -#else - pmem_memcpy_persist(addr + off1, data, l); -#endif - len -= l; - off1 += l; - } - return 0; -} - -int PMEMDevice::aio_write( - uint64_t off, - bufferlist &bl, - IOContext *ioc, - bool buffered, - int write_hint) -{ - return write(off, bl, buffered); -} - - -int PMEMDevice::read(uint64_t off, uint64_t len, bufferlist *pbl, - IOContext *ioc, - bool buffered) -{ - dout(5) << __func__ << " " << off << "~" << len << dendl; - ceph_assert(is_valid_io(off, len)); - - bufferptr p = buffer::create_small_page_aligned(len); - -#if defined(HAVE_LIBDML) - auto result = dml::execute(dml::mem_move, dml::make_view(addr + off, len), dml::make_view(p.c_str(), len)); - ceph_assert(result.status == dml::status_code::ok); -#else - memcpy(p.c_str(), addr + off, len); -#endif - - pbl->clear(); - pbl->push_back(std::move(p)); - - dout(40) << "data:\n"; - pbl->hexdump(*_dout); - *_dout << dendl; - - return 0; -} - -int PMEMDevice::aio_read(uint64_t off, uint64_t len, bufferlist *pbl, - IOContext *ioc) -{ - return read(off, len, pbl, ioc, false); -} - -int PMEMDevice::read_random(uint64_t off, uint64_t len, char *buf, bool buffered) -{ - dout(5) << __func__ << " " << off << "~" << len << dendl; - ceph_assert(is_valid_io(off, len)); - - -#if defined(HAVE_LIBDML) - auto result = dml::execute(dml::mem_move, dml::make_view(addr + off, len), dml::make_view(buf, len)); - ceph_assert(result.status == dml::status_code::ok); -#else - memcpy(buf, addr + off, len); -#endif - return 0; -} - - -int PMEMDevice::invalidate_cache(uint64_t off, uint64_t len) -{ - dout(5) << __func__ << " " << off << "~" << len << dendl; - return 0; -} - - diff --git a/src/blk/pmem/PMEMDevice.h b/src/blk/pmem/PMEMDevice.h deleted file mode 100644 index 75e96cc4f45..00000000000 --- a/src/blk/pmem/PMEMDevice.h +++ /dev/null @@ -1,79 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*- -// vim: ts=8 sw=2 sts=2 expandtab - -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2015 Intel - * - * Author: Jianpeng Ma - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - * - */ - -#ifndef CEPH_BLK_PMEMDEVICE_H -#define CEPH_BLK_PMEMDEVICE_H - -#include -#include -#include - -#include "os/fs/FS.h" -#include "include/interval_set.h" -#include "aio/aio.h" -#include "BlockDevice.h" - -class PMEMDevice : public BlockDevice { - int fd; - char *addr; //the address of mmap - std::string path; - bool devdax_device = false; - - ceph::mutex debug_lock = ceph::make_mutex("PMEMDevice::debug_lock"); - interval_set debug_inflight; - - std::atomic_int injecting_crash; - int _lock(); - -public: - PMEMDevice(CephContext *cct, aio_callback_t cb, void *cbpriv); - - bool supported_bdev_label() override { return !devdax_device; } - void aio_submit(IOContext *ioc) override; - - int collect_metadata(const std::string& prefix, std::map *pm) const override; - - static bool support(const std::string& path); - - int read(uint64_t off, uint64_t len, bufferlist *pbl, - IOContext *ioc, - bool buffered) override; - int aio_read(uint64_t off, uint64_t len, bufferlist *pbl, - IOContext *ioc) override; - - int read_random(uint64_t off, uint64_t len, char *buf, bool buffered) override; - int write(uint64_t off, bufferlist& bl, bool buffered, int write_hint = WRITE_LIFE_NOT_SET) override; - int aio_write(uint64_t off, bufferlist& bl, - IOContext *ioc, - bool buffered, - int write_hint = WRITE_LIFE_NOT_SET) override; - int flush() override; - - // for managing buffered readers/writers - int invalidate_cache(uint64_t off, uint64_t len) override; - int open(const std::string &path) override; - void close() override; - -private: - bool is_valid_io(uint64_t off, uint64_t len) const { - return (len > 0 && - off < size && - off + len <= size); - } -}; - -#endif diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in index 59317a8d6b2..8fa84eb501c 100644 --- a/src/common/options/global.yaml.in +++ b/src/common/options/global.yaml.in @@ -6981,7 +6981,6 @@ options: enum_values: - aio - spdk - - pmem - name: bdev_stalled_read_warn_lifetime type: uint level: advanced diff --git a/src/include/config-h.in.cmake b/src/include/config-h.in.cmake index 1ae58be6201..bf8706312f0 100644 --- a/src/include/config-h.in.cmake +++ b/src/include/config-h.in.cmake @@ -84,9 +84,6 @@ /* Defined if you have libaio */ #cmakedefine HAVE_LIBAIO -/* Defined if you have libdml */ -#cmakedefine HAVE_LIBDML - /* Defined if you have liburing */ #cmakedefine HAVE_LIBURING @@ -114,9 +111,6 @@ /* DPDK conditional compilation */ #cmakedefine HAVE_DPDK -/* PMEM_DEVICE (OSD) conditional compilation */ -#cmakedefine HAVE_BLUESTORE_PMEM - /* Define if you have tcmalloc */ #cmakedefine HAVE_LIBTCMALLOC #cmakedefine LIBTCMALLOC_MISSING_ALIGNED_ALLOC diff --git a/src/vstart.sh b/src/vstart.sh index c64c9cf9983..859c804d2e4 100755 --- a/src/vstart.sh +++ b/src/vstart.sh @@ -187,7 +187,6 @@ rgw_compression="" rgw_store="rados" lockdep=${LOCKDEP:-1} spdk_enabled=0 # disable SPDK by default -pmem_enabled=0 io_uring_enabled=0 with_jaeger=0 force_addr=0 @@ -259,7 +258,6 @@ options: --multimds allow multimds with maximum active count --without-dashboard: do not run using mgr dashboard --bluestore-spdk: enable SPDK and with a comma-delimited list of PCI-IDs of NVME device (e.g, 0000:81:00.0) - --bluestore-pmem: enable PMEM and with path to a file mapped to PMEM --msgr1: use msgr1 only --msgr2: use msgr2 only --msgr21: use msgr2 and msgr1 @@ -657,12 +655,6 @@ case $1 in spdk_enabled=1 shift ;; - --bluestore-pmem) - [ -z "$2" ] && usage_exit - bluestore_pmem_file="$2" - pmem_enabled=1 - shift - ;; --bluestore-devs) parse_block_devs --bluestore-devs "$2" shift @@ -911,7 +903,7 @@ EOF osd max object namespace len = 64" fi if [ "$objectstore" == "bluestore" ]; then - if [ "$spdk_enabled" -eq 1 ] || [ "$pmem_enabled" -eq 1 ]; then + if [ "$spdk_enabled" -eq 1 ]; then BLUESTORE_OPTS=" bluestore_block_db_path = \"\" bluestore_block_db_size = 0 bluestore_block_db_create = false @@ -1290,10 +1282,6 @@ EOF if [ "$spdk_enabled" -eq 1 ]; then wconf <