mirror of
https://github.com/ceph/ceph
synced 2026-08-01 22:45:39 +00:00
blk,build: drop BlueStore PMEM support
BlueStore's PMEMDevice backend targets byte-addressable persistent
memory exposed as a DAX device, in practice Intel Optane DC persistent
memory. Intel Optane has been discontinued since 2021 and there is no
mainstream replacement, so the backend has effectively no hardware left
to run on. It was always experimental and has seen no functional change
since the DML/DSA offload landed in 2022; every commit to it since has
been a formatting sweep.
Keeping it builds in a chain of dependencies that nothing else needs:
libpmem, libndctl and libdaxctl, plus dml for the DSA offload that only
works on a Sapphire Rapids CPU. Remove the backend and everything tied
to it.
RBD persistent write-back cache (WITH_RBD_RWL) still uses PMDK's
libpmemobj, so the pmdk submodule, WITH_SYSTEM_PMDK, Buildpmdk and
Findpmdk stay. WITH_SYSTEM_PMDK now depends on WITH_RBD_RWL alone, and
Buildpmdk no longer builds ndctl support.
- remove the WITH_BLUESTORE_PMEM option, HAVE_BLUESTORE_PMEM and
HAVE_LIBDML
- delete src/blk/pmem/PMEMDevice.{cc,h} and the pmem block_device_t
- drop the pmem value from the bdev_type option
- remove the --bluestore-pmem vstart flag
- drop the libpmem, libndctl and libdaxctl build deps from debian and
rpm, keeping libpmemobj for RWL
- delete the Finddml, Findndctl and Finddaxctl cmake modules
- remove the DSA/DML and ndctl documentation
Signed-off-by: Kefu Chai <k.chai@proxmox.com>
This commit is contained in:
Kefu Chai
parent
c11c7144fc
commit
80b9067286
@ -223,7 +223,6 @@ if(LINUX)
|
||||
elseif(FREEBSD)
|
||||
set(HAVE_UDEV OFF)
|
||||
set(HAVE_LIBAIO OFF)
|
||||
set(HAVE_LIBDML OFF)
|
||||
set(HAVE_BLKID OFF)
|
||||
set(HAVE_KEYUTILS OFF)
|
||||
else()
|
||||
@ -304,13 +303,6 @@ if(WITH_LIBURING)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
CMAKE_DEPENDENT_OPTION(WITH_BLUESTORE_PMEM "Enable PMDK libraries" OFF
|
||||
"WITH_BLUESTORE" OFF)
|
||||
if(WITH_BLUESTORE_PMEM)
|
||||
find_package(dml)
|
||||
set(HAVE_LIBDML ${DML_FOUND})
|
||||
endif()
|
||||
|
||||
CMAKE_DEPENDENT_OPTION(WITH_RBD_MIRROR "Enable build for rbd-mirror daemon executable" OFF
|
||||
"WITH_RBD" OFF)
|
||||
|
||||
@ -325,7 +317,7 @@ CMAKE_DEPENDENT_OPTION(WITH_RBD_SSD_CACHE "Enable librbd persistent write back c
|
||||
"WITH_RBD" OFF)
|
||||
|
||||
CMAKE_DEPENDENT_OPTION(WITH_SYSTEM_PMDK "Require and build with system PMDK" OFF
|
||||
"WITH_RBD_RWL OR WITH_BLUESTORE_PMEM" OFF)
|
||||
"WITH_RBD_RWL" OFF)
|
||||
|
||||
CMAKE_DEPENDENT_OPTION(WITH_RBD_UBBD "Enable ubbd support for rbd device utility" OFF
|
||||
"WITH_RBD" OFF)
|
||||
@ -335,10 +327,6 @@ if(WITH_RBD_UBBD)
|
||||
build_ubbd()
|
||||
endif()
|
||||
|
||||
if(WITH_BLUESTORE_PMEM)
|
||||
set(HAVE_BLUESTORE_PMEM ON)
|
||||
endif()
|
||||
|
||||
CMAKE_DEPENDENT_OPTION(WITH_SPDK "Enable SPDK" OFF
|
||||
"CMAKE_SYSTEM_PROCESSOR MATCHES i386|i686|amd64|x86_64|AMD64|aarch64|riscv64" OFF)
|
||||
option(WITH_SYSTEM_SPDK "Require a system SPDK instead of building bundled src/spdk" OFF)
|
||||
@ -356,10 +344,10 @@ if(WITH_SPDK)
|
||||
endif(WITH_SPDK)
|
||||
|
||||
if(WITH_BLUESTORE)
|
||||
if(NOT AIO_FOUND AND NOT HAVE_POSIXAIO AND NOT WITH_SPDK AND NOT WITH_BLUESTORE_PMEM)
|
||||
if(NOT AIO_FOUND AND NOT HAVE_POSIXAIO AND NOT WITH_SPDK)
|
||||
message(SEND_ERROR "WITH_BLUESTORE is ON, "
|
||||
"but none of the bluestore backends is enabled. "
|
||||
"Please install libaio, or enable WITH_SPDK or WITH_BLUESTORE_PMEM (experimental)")
|
||||
"Please install libaio, or enable WITH_SPDK")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
|
||||
@ -262,6 +262,18 @@
|
||||
rather rigid and also challenging to disseminate in some environments. The
|
||||
key can be embedded in the migration spec or just referenced from there while
|
||||
stored in the MON config-key store.
|
||||
* BlueStore: The experimental PMEM device backend has been removed, together
|
||||
with the ``pmem`` value of the ``bdev_type`` option and the DML/DSA offload
|
||||
path. The hardware it targeted, Intel Optane DC persistent memory, was
|
||||
discontinued in 2022. The RBD persistent write-back cache, which also uses
|
||||
PMDK, is unaffected.
|
||||
|
||||
Existing OSDs backed by persistent memory in fsdax mode (a block device such
|
||||
as ``/dev/pmem0``) keep working on the default ``aio`` backend over the same
|
||||
data. Remove any explicit ``bdev_type = pmem`` from their configuration. OSDs
|
||||
on a devdax namespace (a character device such as ``/dev/dax0.0``) will no
|
||||
longer start, because no remaining backend can open one; reconfigure the
|
||||
namespace to fsdax with ``ndctl`` or re-provision them before upgrading.
|
||||
|
||||
>=20.0.0
|
||||
|
||||
|
||||
@ -358,13 +358,6 @@ BuildRequires: pkgconfig(nlohmann_json)
|
||||
BuildRequires: pkgconfig(libevent)
|
||||
%endif
|
||||
%if 0%{with system_pmdk}
|
||||
%if 0%{?suse_version}
|
||||
BuildRequires: pkgconfig(libndctl) >= 63
|
||||
%else
|
||||
BuildRequires: pkgconfig(libndctl) >= 63
|
||||
BuildRequires: pkgconfig(libdaxctl) >= 63
|
||||
%endif
|
||||
BuildRequires: pkgconfig(libpmem)
|
||||
BuildRequires: pkgconfig(libpmemobj) >= 1.8
|
||||
%endif
|
||||
%if 0%{with system_arrow}
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
function(build_pmdk enable_ndctl)
|
||||
function(build_pmdk)
|
||||
include(FindMake)
|
||||
find_make("MAKE_EXECUTABLE" "make_cmd")
|
||||
|
||||
@ -15,12 +15,6 @@ function(build_pmdk enable_ndctl)
|
||||
endif()
|
||||
|
||||
set(LIBPMEM_INTERFACE_LINK_LIBRARIES Threads::Threads)
|
||||
if(${enable_ndctl})
|
||||
set(ndctl "y")
|
||||
list(APPEND LIBPMEM_INTERFACE_LINK_LIBRARIES ndctl::ndctl daxctl::daxctl)
|
||||
else()
|
||||
set(ndctl "n")
|
||||
endif()
|
||||
|
||||
# Use debug PMDK libs in debug lib/rbd builds
|
||||
if(CMAKE_BUILD_TYPE STREQUAL Debug)
|
||||
@ -34,7 +28,7 @@ function(build_pmdk enable_ndctl)
|
||||
ExternalProject_Add(pmdk_ext
|
||||
${source_dir_args}
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND ${make_cmd} CC=${CMAKE_C_COMPILER} "EXTRA_CFLAGS=${pmdk_cflags}" NDCTL_ENABLE=${ndctl} BUILD_EXAMPLES=n BUILD_BENCHMARKS=n DOC=n
|
||||
BUILD_COMMAND ${make_cmd} CC=${CMAKE_C_COMPILER} "EXTRA_CFLAGS=${pmdk_cflags}" NDCTL_ENABLE=n BUILD_EXAMPLES=n BUILD_BENCHMARKS=n DOC=n
|
||||
BUILD_IN_SOURCE 1
|
||||
BUILD_BYPRODUCTS "<SOURCE_DIR>/src/${PMDK_LIB_DIR}/libpmem.a" "<SOURCE_DIR>/src/${PMDK_LIB_DIR}/libpmemobj.a"
|
||||
INSTALL_COMMAND "")
|
||||
|
||||
@ -1,42 +0,0 @@
|
||||
# - Find libdaxctl
|
||||
# Find the daxctl libraries and includes
|
||||
#
|
||||
# daxctl_INCLUDE_DIR - where to find libdaxctl.h etc.
|
||||
# daxctl_LIBRARIES - List of libraries when using daxctl.
|
||||
# daxctl_FOUND - True if daxctl found.
|
||||
|
||||
find_path(daxctl_INCLUDE_DIR daxctl/libdaxctl.h)
|
||||
|
||||
if(daxctl_INCLUDE_DIR AND EXISTS "${daxctl_INCLUDE_DIR}/libdaxctl.h")
|
||||
foreach(ver "MAJOR" "MINOR" "RELEASE")
|
||||
file(STRINGS "${daxctl_INCLUDE_DIR}/libdaxctl.h" daxctl_VER_${ver}_LINE
|
||||
REGEX "^#define[ \t]+daxctl_VERSION_${ver}[ \t]+[0-9]+[ \t]+.*$")
|
||||
string(REGEX REPLACE "^#define[ \t]+daxctl_VERSION_${ver}[ \t]+([0-9]+)[ \t]+.*$"
|
||||
"\\1" daxctl_VERSION_${ver} "${daxctl_VER_${ver}_LINE}")
|
||||
unset(${daxctl_VER_${ver}_LINE})
|
||||
endforeach()
|
||||
set(daxctl_VERSION_STRING
|
||||
"${daxctl_VERSION_MAJOR}.${daxctl_VERSION_MINOR}.${daxctl_VERSION_RELEASE}")
|
||||
endif()
|
||||
|
||||
find_library(daxctl_LIBRARY daxctl)
|
||||
|
||||
include(FindPackageHandleStandardArgs)
|
||||
find_package_handle_standard_args(daxctl
|
||||
REQUIRED_VARS daxctl_LIBRARY daxctl_INCLUDE_DIR
|
||||
VERSION_VAR daxctl_VERSION_STRING)
|
||||
|
||||
mark_as_advanced(daxctl_INCLUDE_DIR daxctl_LIBRARY)
|
||||
|
||||
if(daxctl_FOUND)
|
||||
set(daxctl_INCLUDE_DIRS ${daxctl_INCLUDE_DIR})
|
||||
set(daxctl_LIBRARIES ${daxctl_LIBRARY})
|
||||
if(NOT (TARGET daxctl::daxctl))
|
||||
add_library(daxctl::daxctl UNKNOWN IMPORTED)
|
||||
set_target_properties(daxctl::daxctl PROPERTIES
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${daxctl_INCLUDE_DIRS}"
|
||||
IMPORTED_LINK_INTERFACE_LANGUAGES "C"
|
||||
IMPORTED_LOCATION "${daxctl_LIBRARIES}"
|
||||
VERSION "${daxctl_VERSION_STRING}")
|
||||
endif()
|
||||
endif()
|
||||
@ -1,58 +0,0 @@
|
||||
# - Find libdml
|
||||
# Find the dml and dmlhl libraries and includes
|
||||
#
|
||||
# DML_INCLUDE_DIR - where to find dml.hpp etc.
|
||||
# DML_LIBRARIES - List of libraries when using dml.
|
||||
# DML_HL_LIBRARIES - List of libraries when using dmlhl.
|
||||
# DML_FOUND - True if DML found.
|
||||
|
||||
|
||||
find_path(DML_INCLUDE_DIR
|
||||
dml/dml.hpp
|
||||
PATHS
|
||||
/usr/include
|
||||
/usr/local/include)
|
||||
|
||||
find_library(DML_LIBRARIES NAMES dml libdml PATHS
|
||||
/usr/local/
|
||||
/usr/local/lib64
|
||||
/usr/lib64
|
||||
/usr/lib)
|
||||
|
||||
find_library(DML_HL_LIBRARIES NAMES dmlhl libdmlhl PATHS
|
||||
/usr/local/
|
||||
/usr/local/lib64
|
||||
/usr/lib64
|
||||
/usr/lib)
|
||||
|
||||
include(FindPackageHandleStandardArgs)
|
||||
find_package_handle_standard_args(dml DEFAULT_MSG
|
||||
DML_LIBRARIES
|
||||
DML_INCLUDE_DIR
|
||||
DML_HL_LIBRARIES)
|
||||
|
||||
mark_as_advanced(
|
||||
DML_LIBRARIES
|
||||
DML_INCLUDE_DIR
|
||||
DML_HL_LIBRARIES)
|
||||
|
||||
if(DML_FOUND)
|
||||
if(NOT (TARGET dml::dml))
|
||||
add_library(dml::dml UNKNOWN IMPORTED)
|
||||
set_target_properties(dml::dml PROPERTIES
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${DML_INCLUDE_DIR}"
|
||||
IMPORTED_LINK_INTERFACE_LANGUAGES "C"
|
||||
IMPORTED_LOCATION "${DML_LIBRARIES}")
|
||||
endif()
|
||||
|
||||
if(NOT (TARGET dml::dmlhl))
|
||||
add_library(dml::dmlhl UNKNOWN IMPORTED)
|
||||
set_target_properties(dml::dmlhl PROPERTIES
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${DML_INCLUDE_DIR}"
|
||||
INTERFACE_LINK_LIBRARIES ${CMAKE_DL_LIBS}
|
||||
INTERFACE_COMPILE_FEATURES cxx_std_17
|
||||
INTERFACE_COMPILE_DEFINITIONS "DML_HW"
|
||||
IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
|
||||
IMPORTED_LOCATION "${DML_HL_LIBRARIES}")
|
||||
endif()
|
||||
endif()
|
||||
@ -1,42 +0,0 @@
|
||||
# - Find libndctl
|
||||
# Find the ndctl libraries and includes
|
||||
#
|
||||
# ndctl_INCLUDE_DIR - where to find libndctl.h etc.
|
||||
# ndctl_LIBRARIES - List of libraries when using ndctl.
|
||||
# ndctl_FOUND - True if ndctl found.
|
||||
|
||||
find_path(ndctl_INCLUDE_DIR ndctl/libndctl.h)
|
||||
|
||||
if(ndctl_INCLUDE_DIR AND EXISTS "${ndctl_INCLUDE_DIR}/libndctl.h")
|
||||
foreach(ver "MAJOR" "MINOR" "RELEASE")
|
||||
file(STRINGS "${ndctl_INCLUDE_DIR}/libndctl.h" ndctl_VER_${ver}_LINE
|
||||
REGEX "^#define[ \t]+ndctl_VERSION_${ver}[ \t]+[0-9]+[ \t]+.*$")
|
||||
string(REGEX REPLACE "^#define[ \t]+ndctl_VERSION_${ver}[ \t]+([0-9]+)[ \t]+.*$"
|
||||
"\\1" ndctl_VERSION_${ver} "${ndctl_VER_${ver}_LINE}")
|
||||
unset(${ndctl_VER_${ver}_LINE})
|
||||
endforeach()
|
||||
set(ndctl_VERSION_STRING
|
||||
"${ndctl_VERSION_MAJOR}.${ndctl_VERSION_MINOR}.${ndctl_VERSION_RELEASE}")
|
||||
endif()
|
||||
|
||||
find_library(ndctl_LIBRARY ndctl)
|
||||
|
||||
include(FindPackageHandleStandardArgs)
|
||||
find_package_handle_standard_args(ndctl
|
||||
REQUIRED_VARS ndctl_LIBRARY ndctl_INCLUDE_DIR
|
||||
VERSION_VAR ndctl_VERSION_STRING)
|
||||
|
||||
mark_as_advanced(ndctl_INCLUDE_DIR ndctl_LIBRARY)
|
||||
|
||||
if(ndctl_FOUND)
|
||||
set(ndctl_INCLUDE_DIRS ${ndctl_INCLUDE_DIR})
|
||||
set(ndctl_LIBRARIES ${ndctl_LIBRARY})
|
||||
if(NOT (TARGET ndctl::ndctl))
|
||||
add_library(ndctl::ndctl UNKNOWN IMPORTED)
|
||||
set_target_properties(ndctl::ndctl PROPERTIES
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${ndctl_INCLUDE_DIRS}"
|
||||
IMPORTED_LINK_INTERFACE_LANGUAGES "C"
|
||||
IMPORTED_LOCATION "${ndctl_LIBRARIES}"
|
||||
VERSION "${ndctl_VERSION_STRING}")
|
||||
endif()
|
||||
endif()
|
||||
3
debian/control
vendored
3
debian/control
vendored
@ -80,9 +80,6 @@ Build-Depends: automake,
|
||||
libthrift-dev (>= 0.13.0),
|
||||
libyaml-cpp-dev (>= 0.6),
|
||||
libzstd-dev <pkg.ceph.check>,
|
||||
libdaxctl-dev (>= 63) <pkg.ceph.pmdk>,
|
||||
libndctl-dev (>= 63) <pkg.ceph.pmdk>,
|
||||
libpmem-dev <pkg.ceph.pmdk>,
|
||||
libpmemobj-dev (>= 1.8) <pkg.ceph.pmdk>,
|
||||
libprotobuf-dev,
|
||||
libxsimd-dev <!pkg.ceph.arrow>,
|
||||
|
||||
@ -206,9 +206,7 @@ libzbd
|
||||
packages `libzbd`_ . The upstream libzbd includes debian packaging already.
|
||||
|
||||
libpmem
|
||||
packages `pmdk`_ . Please note, ``ndctl`` is one of the build dependencies of
|
||||
pmdk, for an updated debian packaging, please see
|
||||
https://github.com/ceph/ceph-ndctl .
|
||||
packages `pmdk`_ .
|
||||
|
||||
.. note::
|
||||
|
||||
|
||||
@ -535,24 +535,3 @@ were deployed under older releases or with other settings.
|
||||
.. confval:: bluestore_min_alloc_size_hdd
|
||||
.. confval:: bluestore_min_alloc_size_ssd
|
||||
.. confval:: bluestore_use_optimal_io_size_for_min_alloc_size
|
||||
|
||||
DSA (Data Streaming Accelerator) Usage
|
||||
======================================
|
||||
|
||||
If you want to use the DML library to drive the DSA device for offloading
|
||||
read/write operations on persistent memory (PMEM) in BlueStore, you need to
|
||||
install `DML`_ and the `idxd-config`_ library. This will work only on machines
|
||||
that have a SPR (Sapphire Rapids) CPU.
|
||||
|
||||
.. _DML: https://github.com/intel/DML
|
||||
.. _idxd-config: https://github.com/intel/idxd-config
|
||||
|
||||
After installing the DML software, configure the shared work queues (WQs) with
|
||||
reference to the following WQ configuration example:
|
||||
|
||||
.. prompt:: bash $
|
||||
|
||||
accel-config config-wq --group-id=1 --mode=shared --wq-size=16 --threshold=15 --type=user --name="MyApp1" --priority=10 --block-on-fault=1 dsa0/wq0.1
|
||||
accel-config config-engine dsa0/engine0.1 --group-id=1
|
||||
accel-config enable-device dsa0
|
||||
accel-config enable-wq dsa0/wq0.1
|
||||
|
||||
@ -654,26 +654,12 @@ if(WIN32)
|
||||
list(APPEND ceph_common_deps dlfcn_win32)
|
||||
endif()
|
||||
|
||||
if(WITH_BLUESTORE_PMEM OR WITH_RBD_RWL)
|
||||
if(WITH_RBD_RWL)
|
||||
if(WITH_SYSTEM_PMDK)
|
||||
set(pmdk_COMPONENTS)
|
||||
if(WITH_BLUESTORE_PMEM)
|
||||
list(APPEND pmdk_COMPONENTS pmem)
|
||||
endif()
|
||||
if(WITH_RBD_RWL)
|
||||
list(APPEND pmdk_COMPONENTS pmemobj)
|
||||
endif()
|
||||
find_package(pmdk 1.8 REQUIRED COMPONENTS ${pmdk_COMPONENTS})
|
||||
find_package(pmdk 1.8 REQUIRED COMPONENTS pmemobj)
|
||||
else()
|
||||
include(Buildpmdk)
|
||||
if(WITH_BLUESTORE_PMEM)
|
||||
set(enable_ndctl ON)
|
||||
find_package(ndctl 63 REQUIRED)
|
||||
find_package(daxctl 63 REQUIRED)
|
||||
else()
|
||||
set(enable_ndctl OFF)
|
||||
endif()
|
||||
build_pmdk(${enable_ndctl})
|
||||
build_pmdk()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
|
||||
@ -28,10 +28,6 @@
|
||||
#include "spdk/NVMEDevice.h"
|
||||
#endif
|
||||
|
||||
#if defined(HAVE_BLUESTORE_PMEM)
|
||||
#include "pmem/PMEMDevice.h"
|
||||
#endif
|
||||
|
||||
#include "common/debug.h"
|
||||
#include "common/EventTrace.h"
|
||||
#include "common/errno.h"
|
||||
@ -106,11 +102,6 @@ BlockDevice::detect_device_type(const std::string& path)
|
||||
return block_device_t::spdk;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAVE_BLUESTORE_PMEM)
|
||||
if (PMEMDevice::support(path)) {
|
||||
return block_device_t::pmem;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO)
|
||||
return block_device_t::aio;
|
||||
#else
|
||||
@ -130,11 +121,6 @@ BlockDevice::device_type_from_name(const std::string& blk_dev_name)
|
||||
if (blk_dev_name == "spdk") {
|
||||
return block_device_t::spdk;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAVE_BLUESTORE_PMEM)
|
||||
if (blk_dev_name == "pmem") {
|
||||
return block_device_t::pmem;
|
||||
}
|
||||
#endif
|
||||
return block_device_t::unknown;
|
||||
}
|
||||
@ -152,10 +138,6 @@ BlockDevice* BlockDevice::create_with_type(block_device_t device_type,
|
||||
#if defined(HAVE_SPDK)
|
||||
case block_device_t::spdk:
|
||||
return new NVMEDevice(cct, cb, cbpriv);
|
||||
#endif
|
||||
#if defined(HAVE_BLUESTORE_PMEM)
|
||||
case block_device_t::pmem:
|
||||
return new PMEMDevice(cct, cb, cbpriv);
|
||||
#endif
|
||||
default:
|
||||
ceph_abort_msg("unsupported device");
|
||||
|
||||
@ -164,9 +164,6 @@ private:
|
||||
#endif
|
||||
#if defined(HAVE_SPDK)
|
||||
spdk,
|
||||
#endif
|
||||
#if defined(HAVE_BLUESTORE_PMEM)
|
||||
pmem,
|
||||
#endif
|
||||
};
|
||||
std::queue <ceph::mono_clock::time_point> stalled_read_event_queue;
|
||||
|
||||
@ -10,11 +10,6 @@ if(HAVE_LIBAIO OR HAVE_POSIXAIO)
|
||||
aio/aio.cc)
|
||||
endif()
|
||||
|
||||
if(WITH_BLUESTORE_PMEM)
|
||||
list(APPEND libblk_srcs
|
||||
pmem/PMEMDevice.cc)
|
||||
endif()
|
||||
|
||||
if(WITH_SPDK)
|
||||
list(APPEND libblk_srcs
|
||||
spdk/NVMEDevice.cc)
|
||||
@ -36,15 +31,6 @@ if(WITH_SPDK)
|
||||
PRIVATE spdk::spdk)
|
||||
endif()
|
||||
|
||||
if(WITH_BLUESTORE_PMEM)
|
||||
if(HAVE_LIBDML)
|
||||
target_link_libraries(blk PRIVATE dml::dml dml::dmlhl)
|
||||
endif()
|
||||
|
||||
target_link_libraries(blk
|
||||
PRIVATE pmdk::pmem)
|
||||
endif()
|
||||
|
||||
if(WITH_EVENTTRACE)
|
||||
add_dependencies(blk eventtrace_tp)
|
||||
endif()
|
||||
|
||||
@ -1,379 +0,0 @@
|
||||
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
|
||||
// vim: ts=8 sw=2 sts=2 expandtab
|
||||
|
||||
/*
|
||||
* Ceph - scalable distributed file system
|
||||
*
|
||||
* Copyright (C) 2015 Intel <jianpeng.ma@intel.com>
|
||||
*
|
||||
* Author: Jianpeng Ma <jianpeng.ma@intel.com>
|
||||
*
|
||||
* This is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License version 2.1, as published by the Free Software
|
||||
* Foundation. See file COPYING.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <unistd.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/sysmacros.h>
|
||||
#include <stdio.h>
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <string.h>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
|
||||
#include <fmt/format.h>
|
||||
|
||||
#include "PMEMDevice.h"
|
||||
#include "libpmem.h"
|
||||
#include "include/types.h"
|
||||
#include "include/compat.h"
|
||||
#include "include/stringify.h"
|
||||
#include "common/errno.h"
|
||||
#include "common/debug.h"
|
||||
#include "common/blkdev.h"
|
||||
|
||||
#if defined(HAVE_LIBDML)
|
||||
#include <dml/dml.hpp>
|
||||
using execution_path = dml::automatic;
|
||||
#endif
|
||||
|
||||
#define dout_context cct
|
||||
#define dout_subsys ceph_subsys_bdev
|
||||
#undef dout_prefix
|
||||
#define dout_prefix *_dout << "bdev-PMEM(" << path << ") "
|
||||
|
||||
PMEMDevice::PMEMDevice(CephContext *cct, aio_callback_t cb, void *cbpriv)
|
||||
: BlockDevice(cct, cb, cbpriv),
|
||||
fd(-1), addr(0),
|
||||
injecting_crash(0)
|
||||
{
|
||||
}
|
||||
|
||||
int PMEMDevice::_lock()
|
||||
{
|
||||
struct flock l;
|
||||
memset(&l, 0, sizeof(l));
|
||||
l.l_type = F_WRLCK;
|
||||
l.l_whence = SEEK_SET;
|
||||
l.l_start = 0;
|
||||
l.l_len = 0;
|
||||
int r = ::fcntl(fd, F_SETLK, &l);
|
||||
if (r < 0)
|
||||
return -errno;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int pmem_check_file_type(int fd, const char *pmem_file, uint64_t *total_size)
|
||||
{
|
||||
namespace fs = std::filesystem;
|
||||
if (!fs::is_character_file(pmem_file)) {
|
||||
return -EINVAL;
|
||||
}
|
||||
struct stat file_stat;
|
||||
if (::fstat(fd, &file_stat)) {
|
||||
return -EINVAL;
|
||||
}
|
||||
fs::path char_dir = fmt::format("/sys/dev/char/{}:{}",
|
||||
major(file_stat.st_rdev),
|
||||
minor(file_stat.st_rdev));
|
||||
// Need to check if it is a DAX device
|
||||
if (auto subsys_path = char_dir / "subsystem";
|
||||
fs::read_symlink(subsys_path).filename().string() != "dax") {
|
||||
return -EINVAL;
|
||||
}
|
||||
if (total_size == nullptr) {
|
||||
return 0;
|
||||
}
|
||||
if (std::ifstream size_file(char_dir / "size"); size_file) {
|
||||
size_file >> *total_size;
|
||||
return size_file ? 0 : -EINVAL;
|
||||
} else {
|
||||
return -EINVAL;
|
||||
}
|
||||
}
|
||||
|
||||
int PMEMDevice::open(const std::string& p)
|
||||
{
|
||||
path = p;
|
||||
int r = 0;
|
||||
dout(1) << __func__ << " path " << path << dendl;
|
||||
|
||||
fd = ::open(path.c_str(), O_RDWR | O_CLOEXEC);
|
||||
if (fd < 0) {
|
||||
r = -errno;
|
||||
derr << __func__ << " open got: " << cpp_strerror(r) << dendl;
|
||||
return r;
|
||||
}
|
||||
|
||||
r = pmem_check_file_type(fd, path.c_str(), &size);
|
||||
if (!r) {
|
||||
dout(1) << __func__ << " This path " << path << " is a devdax dev " << dendl;
|
||||
devdax_device = true;
|
||||
// If using devdax char device, set it to not rotational device.
|
||||
rotational = false;
|
||||
}
|
||||
|
||||
r = _lock();
|
||||
if (r < 0) {
|
||||
derr << __func__ << " failed to lock " << path << ": " << cpp_strerror(r)
|
||||
<< dendl;
|
||||
goto out_fail;
|
||||
}
|
||||
|
||||
struct stat st;
|
||||
r = ::fstat(fd, &st);
|
||||
if (r < 0) {
|
||||
r = -errno;
|
||||
derr << __func__ << " fstat got " << cpp_strerror(r) << dendl;
|
||||
goto out_fail;
|
||||
}
|
||||
|
||||
size_t map_len;
|
||||
addr = (char *)pmem_map_file(path.c_str(), 0,
|
||||
devdax_device ? 0: PMEM_FILE_EXCL, O_RDWR,
|
||||
&map_len, NULL);
|
||||
if (addr == NULL) {
|
||||
derr << __func__ << " pmem_map_file failed: " << pmem_errormsg() << dendl;
|
||||
goto out_fail;
|
||||
}
|
||||
size = map_len;
|
||||
|
||||
// Operate as though the block size is 4 KB. The backing file
|
||||
// blksize doesn't strictly matter except that some file systems may
|
||||
// require a read/modify/write if we write something smaller than
|
||||
// it.
|
||||
block_size = g_conf()->bdev_block_size;
|
||||
if (block_size != (unsigned)st.st_blksize) {
|
||||
dout(1) << __func__ << " backing device/file reports st_blksize "
|
||||
<< st.st_blksize << ", using bdev_block_size "
|
||||
<< block_size << " anyway" << dendl;
|
||||
}
|
||||
|
||||
dout(1) << __func__
|
||||
<< " size " << size
|
||||
<< " (" << byte_u_t(size) << ")"
|
||||
<< " block_size " << block_size
|
||||
<< " (" << byte_u_t(block_size) << ")"
|
||||
<< dendl;
|
||||
return 0;
|
||||
|
||||
out_fail:
|
||||
VOID_TEMP_FAILURE_RETRY(::close(fd));
|
||||
fd = -1;
|
||||
return r;
|
||||
}
|
||||
|
||||
void PMEMDevice::close()
|
||||
{
|
||||
dout(1) << __func__ << dendl;
|
||||
|
||||
ceph_assert(addr != NULL);
|
||||
if (devdax_device) {
|
||||
devdax_device = false;
|
||||
}
|
||||
pmem_unmap(addr, size);
|
||||
|
||||
ceph_assert(fd >= 0);
|
||||
VOID_TEMP_FAILURE_RETRY(::close(fd));
|
||||
fd = -1;
|
||||
|
||||
path.clear();
|
||||
}
|
||||
|
||||
int PMEMDevice::collect_metadata(const std::string& prefix, std::map<std::string,std::string> *pm) const
|
||||
{
|
||||
(*pm)[prefix + "rotational"] = stringify((int)(bool)rotational);
|
||||
(*pm)[prefix + "size"] = stringify(get_size());
|
||||
(*pm)[prefix + "block_size"] = stringify(get_block_size());
|
||||
(*pm)[prefix + "driver"] = "PMEMDevice";
|
||||
(*pm)[prefix + "type"] = "ssd";
|
||||
|
||||
struct stat st;
|
||||
int r = ::fstat(fd, &st);
|
||||
if (r < 0)
|
||||
return -errno;
|
||||
if (S_ISBLK(st.st_mode)) {
|
||||
(*pm)[prefix + "access_mode"] = "blk";
|
||||
char buffer[1024] = {0};
|
||||
BlkDev blkdev(fd);
|
||||
|
||||
blkdev.model(buffer, sizeof(buffer));
|
||||
(*pm)[prefix + "model"] = buffer;
|
||||
|
||||
buffer[0] = '\0';
|
||||
blkdev.dev(buffer, sizeof(buffer));
|
||||
(*pm)[prefix + "dev"] = buffer;
|
||||
|
||||
// nvme exposes a serial number
|
||||
buffer[0] = '\0';
|
||||
blkdev.serial(buffer, sizeof(buffer));
|
||||
(*pm)[prefix + "serial"] = buffer;
|
||||
|
||||
} else if (S_ISCHR(st.st_mode)) {
|
||||
(*pm)[prefix + "access_mode"] = "chardevice";
|
||||
(*pm)[prefix + "path"] = path;
|
||||
|
||||
} else {
|
||||
(*pm)[prefix + "access_mode"] = "file";
|
||||
(*pm)[prefix + "path"] = path;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool PMEMDevice::support(const std::string &path)
|
||||
{
|
||||
int is_pmem = 0;
|
||||
size_t map_len = 0;
|
||||
int r = 0;
|
||||
int local_fd;
|
||||
|
||||
local_fd = ::open(path.c_str(), O_RDWR);
|
||||
if (local_fd < 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
r = pmem_check_file_type(local_fd, path.c_str(), NULL);
|
||||
VOID_TEMP_FAILURE_RETRY(::close(local_fd));
|
||||
int flags = PMEM_FILE_EXCL;
|
||||
if (r == 0) {
|
||||
flags = 0;
|
||||
}
|
||||
|
||||
void *addr = pmem_map_file(path.c_str(), 0, flags, O_RDONLY, &map_len, &is_pmem);
|
||||
if (addr != NULL) {
|
||||
pmem_unmap(addr, map_len);
|
||||
if (is_pmem) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
int PMEMDevice::flush()
|
||||
{
|
||||
//Because all write is persist. So no need
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
void PMEMDevice::aio_submit(IOContext *ioc)
|
||||
{
|
||||
if (ioc->priv) {
|
||||
ceph_assert(ioc->num_running == 0);
|
||||
aio_callback(aio_callback_priv, ioc->priv);
|
||||
} else {
|
||||
ioc->try_aio_wake();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
int PMEMDevice::write(uint64_t off, bufferlist& bl, bool buffered, int write_hint)
|
||||
{
|
||||
uint64_t len = bl.length();
|
||||
dout(20) << __func__ << " " << off << "~" << len << dendl;
|
||||
ceph_assert(is_valid_io(off, len));
|
||||
|
||||
dout(40) << "data:\n";
|
||||
bl.hexdump(*_dout);
|
||||
*_dout << dendl;
|
||||
|
||||
if (g_conf()->bdev_inject_crash &&
|
||||
rand() % g_conf()->bdev_inject_crash == 0) {
|
||||
derr << __func__ << " bdev_inject_crash: dropping io " << off << "~" << len
|
||||
<< dendl;
|
||||
++injecting_crash;
|
||||
return 0;
|
||||
}
|
||||
|
||||
bufferlist::iterator p = bl.begin();
|
||||
uint64_t off1 = off;
|
||||
while (len) {
|
||||
const char *data;
|
||||
uint32_t l = p.get_ptr_and_advance(len, &data);
|
||||
|
||||
#if defined(HAVE_LIBDML)
|
||||
// Take care of the persistency issue
|
||||
auto result = dml::execute<execution_path>(dml::mem_move, dml::make_view(data, l), dml::make_view(addr + off1, l));
|
||||
ceph_assert(result.status == dml::status_code::ok);
|
||||
#else
|
||||
pmem_memcpy_persist(addr + off1, data, l);
|
||||
#endif
|
||||
len -= l;
|
||||
off1 += l;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int PMEMDevice::aio_write(
|
||||
uint64_t off,
|
||||
bufferlist &bl,
|
||||
IOContext *ioc,
|
||||
bool buffered,
|
||||
int write_hint)
|
||||
{
|
||||
return write(off, bl, buffered);
|
||||
}
|
||||
|
||||
|
||||
int PMEMDevice::read(uint64_t off, uint64_t len, bufferlist *pbl,
|
||||
IOContext *ioc,
|
||||
bool buffered)
|
||||
{
|
||||
dout(5) << __func__ << " " << off << "~" << len << dendl;
|
||||
ceph_assert(is_valid_io(off, len));
|
||||
|
||||
bufferptr p = buffer::create_small_page_aligned(len);
|
||||
|
||||
#if defined(HAVE_LIBDML)
|
||||
auto result = dml::execute<execution_path>(dml::mem_move, dml::make_view(addr + off, len), dml::make_view(p.c_str(), len));
|
||||
ceph_assert(result.status == dml::status_code::ok);
|
||||
#else
|
||||
memcpy(p.c_str(), addr + off, len);
|
||||
#endif
|
||||
|
||||
pbl->clear();
|
||||
pbl->push_back(std::move(p));
|
||||
|
||||
dout(40) << "data:\n";
|
||||
pbl->hexdump(*_dout);
|
||||
*_dout << dendl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int PMEMDevice::aio_read(uint64_t off, uint64_t len, bufferlist *pbl,
|
||||
IOContext *ioc)
|
||||
{
|
||||
return read(off, len, pbl, ioc, false);
|
||||
}
|
||||
|
||||
int PMEMDevice::read_random(uint64_t off, uint64_t len, char *buf, bool buffered)
|
||||
{
|
||||
dout(5) << __func__ << " " << off << "~" << len << dendl;
|
||||
ceph_assert(is_valid_io(off, len));
|
||||
|
||||
|
||||
#if defined(HAVE_LIBDML)
|
||||
auto result = dml::execute<execution_path>(dml::mem_move, dml::make_view(addr + off, len), dml::make_view(buf, len));
|
||||
ceph_assert(result.status == dml::status_code::ok);
|
||||
#else
|
||||
memcpy(buf, addr + off, len);
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int PMEMDevice::invalidate_cache(uint64_t off, uint64_t len)
|
||||
{
|
||||
dout(5) << __func__ << " " << off << "~" << len << dendl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@ -1,79 +0,0 @@
|
||||
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
|
||||
// vim: ts=8 sw=2 sts=2 expandtab
|
||||
|
||||
/*
|
||||
* Ceph - scalable distributed file system
|
||||
*
|
||||
* Copyright (C) 2015 Intel <jianpeng.ma@intel.com>
|
||||
*
|
||||
* Author: Jianpeng Ma <jianpeng.ma@intel.com>
|
||||
*
|
||||
* This is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License version 2.1, as published by the Free Software
|
||||
* Foundation. See file COPYING.
|
||||
*
|
||||
*/
|
||||
|
||||
#ifndef CEPH_BLK_PMEMDEVICE_H
|
||||
#define CEPH_BLK_PMEMDEVICE_H
|
||||
|
||||
#include <atomic>
|
||||
#include <map>
|
||||
#include <string>
|
||||
|
||||
#include "os/fs/FS.h"
|
||||
#include "include/interval_set.h"
|
||||
#include "aio/aio.h"
|
||||
#include "BlockDevice.h"
|
||||
|
||||
class PMEMDevice : public BlockDevice {
|
||||
int fd;
|
||||
char *addr; //the address of mmap
|
||||
std::string path;
|
||||
bool devdax_device = false;
|
||||
|
||||
ceph::mutex debug_lock = ceph::make_mutex("PMEMDevice::debug_lock");
|
||||
interval_set<uint64_t> debug_inflight;
|
||||
|
||||
std::atomic_int injecting_crash;
|
||||
int _lock();
|
||||
|
||||
public:
|
||||
PMEMDevice(CephContext *cct, aio_callback_t cb, void *cbpriv);
|
||||
|
||||
bool supported_bdev_label() override { return !devdax_device; }
|
||||
void aio_submit(IOContext *ioc) override;
|
||||
|
||||
int collect_metadata(const std::string& prefix, std::map<std::string,std::string> *pm) const override;
|
||||
|
||||
static bool support(const std::string& path);
|
||||
|
||||
int read(uint64_t off, uint64_t len, bufferlist *pbl,
|
||||
IOContext *ioc,
|
||||
bool buffered) override;
|
||||
int aio_read(uint64_t off, uint64_t len, bufferlist *pbl,
|
||||
IOContext *ioc) override;
|
||||
|
||||
int read_random(uint64_t off, uint64_t len, char *buf, bool buffered) override;
|
||||
int write(uint64_t off, bufferlist& bl, bool buffered, int write_hint = WRITE_LIFE_NOT_SET) override;
|
||||
int aio_write(uint64_t off, bufferlist& bl,
|
||||
IOContext *ioc,
|
||||
bool buffered,
|
||||
int write_hint = WRITE_LIFE_NOT_SET) override;
|
||||
int flush() override;
|
||||
|
||||
// for managing buffered readers/writers
|
||||
int invalidate_cache(uint64_t off, uint64_t len) override;
|
||||
int open(const std::string &path) override;
|
||||
void close() override;
|
||||
|
||||
private:
|
||||
bool is_valid_io(uint64_t off, uint64_t len) const {
|
||||
return (len > 0 &&
|
||||
off < size &&
|
||||
off + len <= size);
|
||||
}
|
||||
};
|
||||
|
||||
#endif
|
||||
@ -6981,7 +6981,6 @@ options:
|
||||
enum_values:
|
||||
- aio
|
||||
- spdk
|
||||
- pmem
|
||||
- name: bdev_stalled_read_warn_lifetime
|
||||
type: uint
|
||||
level: advanced
|
||||
|
||||
@ -84,9 +84,6 @@
|
||||
/* Defined if you have libaio */
|
||||
#cmakedefine HAVE_LIBAIO
|
||||
|
||||
/* Defined if you have libdml */
|
||||
#cmakedefine HAVE_LIBDML
|
||||
|
||||
/* Defined if you have liburing */
|
||||
#cmakedefine HAVE_LIBURING
|
||||
|
||||
@ -114,9 +111,6 @@
|
||||
/* DPDK conditional compilation */
|
||||
#cmakedefine HAVE_DPDK
|
||||
|
||||
/* PMEM_DEVICE (OSD) conditional compilation */
|
||||
#cmakedefine HAVE_BLUESTORE_PMEM
|
||||
|
||||
/* Define if you have tcmalloc */
|
||||
#cmakedefine HAVE_LIBTCMALLOC
|
||||
#cmakedefine LIBTCMALLOC_MISSING_ALIGNED_ALLOC
|
||||
|
||||
@ -187,7 +187,6 @@ rgw_compression=""
|
||||
rgw_store="rados"
|
||||
lockdep=${LOCKDEP:-1}
|
||||
spdk_enabled=0 # disable SPDK by default
|
||||
pmem_enabled=0
|
||||
io_uring_enabled=0
|
||||
with_jaeger=0
|
||||
force_addr=0
|
||||
@ -259,7 +258,6 @@ options:
|
||||
--multimds <count> allow multimds with maximum active count
|
||||
--without-dashboard: do not run using mgr dashboard
|
||||
--bluestore-spdk: enable SPDK and with a comma-delimited list of PCI-IDs of NVME device (e.g, 0000:81:00.0)
|
||||
--bluestore-pmem: enable PMEM and with path to a file mapped to PMEM
|
||||
--msgr1: use msgr1 only
|
||||
--msgr2: use msgr2 only
|
||||
--msgr21: use msgr2 and msgr1
|
||||
@ -657,12 +655,6 @@ case $1 in
|
||||
spdk_enabled=1
|
||||
shift
|
||||
;;
|
||||
--bluestore-pmem)
|
||||
[ -z "$2" ] && usage_exit
|
||||
bluestore_pmem_file="$2"
|
||||
pmem_enabled=1
|
||||
shift
|
||||
;;
|
||||
--bluestore-devs)
|
||||
parse_block_devs --bluestore-devs "$2"
|
||||
shift
|
||||
@ -911,7 +903,7 @@ EOF
|
||||
osd max object namespace len = 64"
|
||||
fi
|
||||
if [ "$objectstore" == "bluestore" ]; then
|
||||
if [ "$spdk_enabled" -eq 1 ] || [ "$pmem_enabled" -eq 1 ]; then
|
||||
if [ "$spdk_enabled" -eq 1 ]; then
|
||||
BLUESTORE_OPTS=" bluestore_block_db_path = \"\"
|
||||
bluestore_block_db_size = 0
|
||||
bluestore_block_db_create = false
|
||||
@ -1290,10 +1282,6 @@ EOF
|
||||
if [ "$spdk_enabled" -eq 1 ]; then
|
||||
wconf <<EOF
|
||||
bluestore_block_path = spdk:${bluestore_spdk_dev[$osd]}
|
||||
EOF
|
||||
elif [ "$pmem_enabled" -eq 1 ]; then
|
||||
wconf <<EOF
|
||||
bluestore_block_path = ${bluestore_pmem_file}
|
||||
EOF
|
||||
fi
|
||||
rm -rf $CEPH_DEV_DIR/osd$osd || true
|
||||
|
||||
Loading…
Reference in New Issue
Block a user
