mirror of
https://github.com/ceph/ceph
synced 2026-08-01 22:45:39 +00:00
Merge pull request #69726 from tchaikov/wip-drop-bluestore-pmem
blk,build: drop BlueStore PMEM support Reviewed-by: Igor Fedotov <igor.fedotov@croit.io>
This commit is contained in:
Kefu Chai
GitHub
commit
757d192b43
@ -223,7 +223,6 @@ if(LINUX)
|
||||
elseif(FREEBSD)
|
||||
set(HAVE_UDEV OFF)
|
||||
set(HAVE_LIBAIO OFF)
|
||||
set(HAVE_LIBDML OFF)
|
||||
set(HAVE_BLKID OFF)
|
||||
set(HAVE_KEYUTILS OFF)
|
||||
else()
|
||||
@ -304,13 +303,6 @@ if(WITH_LIBURING)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
CMAKE_DEPENDENT_OPTION(WITH_BLUESTORE_PMEM "Enable PMDK libraries" OFF
|
||||
"WITH_BLUESTORE" OFF)
|
||||
if(WITH_BLUESTORE_PMEM)
|
||||
find_package(dml)
|
||||
set(HAVE_LIBDML ${DML_FOUND})
|
||||
endif()
|
||||
|
||||
CMAKE_DEPENDENT_OPTION(WITH_RBD_MIRROR "Enable build for rbd-mirror daemon executable" OFF
|
||||
"WITH_RBD" OFF)
|
||||
|
||||
@ -325,7 +317,7 @@ CMAKE_DEPENDENT_OPTION(WITH_RBD_SSD_CACHE "Enable librbd persistent write back c
|
||||
"WITH_RBD" OFF)
|
||||
|
||||
CMAKE_DEPENDENT_OPTION(WITH_SYSTEM_PMDK "Require and build with system PMDK" OFF
|
||||
"WITH_RBD_RWL OR WITH_BLUESTORE_PMEM" OFF)
|
||||
"WITH_RBD_RWL" OFF)
|
||||
|
||||
CMAKE_DEPENDENT_OPTION(WITH_RBD_UBBD "Enable ubbd support for rbd device utility" OFF
|
||||
"WITH_RBD" OFF)
|
||||
@ -335,10 +327,6 @@ if(WITH_RBD_UBBD)
|
||||
build_ubbd()
|
||||
endif()
|
||||
|
||||
if(WITH_BLUESTORE_PMEM)
|
||||
set(HAVE_BLUESTORE_PMEM ON)
|
||||
endif()
|
||||
|
||||
CMAKE_DEPENDENT_OPTION(WITH_SPDK "Enable SPDK" OFF
|
||||
"CMAKE_SYSTEM_PROCESSOR MATCHES i386|i686|amd64|x86_64|AMD64|aarch64|riscv64" OFF)
|
||||
option(WITH_SYSTEM_SPDK "Require a system SPDK instead of building bundled src/spdk" OFF)
|
||||
@ -356,10 +344,10 @@ if(WITH_SPDK)
|
||||
endif(WITH_SPDK)
|
||||
|
||||
if(WITH_BLUESTORE)
|
||||
if(NOT AIO_FOUND AND NOT HAVE_POSIXAIO AND NOT WITH_SPDK AND NOT WITH_BLUESTORE_PMEM)
|
||||
if(NOT AIO_FOUND AND NOT HAVE_POSIXAIO AND NOT WITH_SPDK)
|
||||
message(SEND_ERROR "WITH_BLUESTORE is ON, "
|
||||
"but none of the bluestore backends is enabled. "
|
||||
"Please install libaio, or enable WITH_SPDK or WITH_BLUESTORE_PMEM (experimental)")
|
||||
"Please install libaio, or enable WITH_SPDK")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
|
||||
@ -269,6 +269,18 @@
|
||||
rather rigid and also challenging to disseminate in some environments. The
|
||||
key can be embedded in the migration spec or just referenced from there while
|
||||
stored in the MON config-key store.
|
||||
* BlueStore: The experimental PMEM device backend has been removed, together
|
||||
with the ``pmem`` value of the ``bdev_type`` option and the DML/DSA offload
|
||||
path. The hardware it targeted, Intel Optane DC persistent memory, was
|
||||
discontinued in 2022. The RBD persistent write-back cache, which also uses
|
||||
PMDK, is unaffected.
|
||||
|
||||
Existing OSDs backed by persistent memory in fsdax mode (a block device such
|
||||
as ``/dev/pmem0``) keep working on the default ``aio`` backend over the same
|
||||
data. Remove any explicit ``bdev_type = pmem`` from their configuration. OSDs
|
||||
on a devdax namespace (a character device such as ``/dev/dax0.0``) will no
|
||||
longer start, because no remaining backend can open one; reconfigure the
|
||||
namespace to fsdax with ``ndctl`` or re-provision them before upgrading.
|
||||
|
||||
>=20.0.0
|
||||
|
||||
|
||||
@ -384,13 +384,6 @@ BuildRequires: pkgconfig(nlohmann_json)
|
||||
BuildRequires: pkgconfig(libevent)
|
||||
%endif
|
||||
%if 0%{with system_pmdk}
|
||||
%if 0%{?suse_version}
|
||||
BuildRequires: pkgconfig(libndctl) >= 63
|
||||
%else
|
||||
BuildRequires: pkgconfig(libndctl) >= 63
|
||||
BuildRequires: pkgconfig(libdaxctl) >= 63
|
||||
%endif
|
||||
BuildRequires: pkgconfig(libpmem)
|
||||
BuildRequires: pkgconfig(libpmemobj) >= 1.8
|
||||
%endif
|
||||
%if 0%{with system_arrow}
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
function(build_pmdk enable_ndctl)
|
||||
function(build_pmdk)
|
||||
include(FindMake)
|
||||
find_make("MAKE_EXECUTABLE" "make_cmd")
|
||||
|
||||
@ -15,12 +15,6 @@ function(build_pmdk enable_ndctl)
|
||||
endif()
|
||||
|
||||
set(LIBPMEM_INTERFACE_LINK_LIBRARIES Threads::Threads)
|
||||
if(${enable_ndctl})
|
||||
set(ndctl "y")
|
||||
list(APPEND LIBPMEM_INTERFACE_LINK_LIBRARIES ndctl::ndctl daxctl::daxctl)
|
||||
else()
|
||||
set(ndctl "n")
|
||||
endif()
|
||||
|
||||
# Use debug PMDK libs in debug lib/rbd builds
|
||||
if(CMAKE_BUILD_TYPE STREQUAL Debug)
|
||||
@ -34,7 +28,7 @@ function(build_pmdk enable_ndctl)
|
||||
ExternalProject_Add(pmdk_ext
|
||||
${source_dir_args}
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND ${make_cmd} CC=${CMAKE_C_COMPILER} "EXTRA_CFLAGS=${pmdk_cflags}" NDCTL_ENABLE=${ndctl} BUILD_EXAMPLES=n BUILD_BENCHMARKS=n DOC=n
|
||||
BUILD_COMMAND ${make_cmd} CC=${CMAKE_C_COMPILER} "EXTRA_CFLAGS=${pmdk_cflags}" NDCTL_ENABLE=n BUILD_EXAMPLES=n BUILD_BENCHMARKS=n DOC=n
|
||||
BUILD_IN_SOURCE 1
|
||||
BUILD_BYPRODUCTS "<SOURCE_DIR>/src/${PMDK_LIB_DIR}/libpmem.a" "<SOURCE_DIR>/src/${PMDK_LIB_DIR}/libpmemobj.a"
|
||||
INSTALL_COMMAND "")
|
||||
|
||||
@ -1,42 +0,0 @@
|
||||
# - Find libdaxctl
|
||||
# Find the daxctl libraries and includes
|
||||
#
|
||||
# daxctl_INCLUDE_DIR - where to find libdaxctl.h etc.
|
||||
# daxctl_LIBRARIES - List of libraries when using daxctl.
|
||||
# daxctl_FOUND - True if daxctl found.
|
||||
|
||||
find_path(daxctl_INCLUDE_DIR daxctl/libdaxctl.h)
|
||||
|
||||
if(daxctl_INCLUDE_DIR AND EXISTS "${daxctl_INCLUDE_DIR}/libdaxctl.h")
|
||||
foreach(ver "MAJOR" "MINOR" "RELEASE")
|
||||
file(STRINGS "${daxctl_INCLUDE_DIR}/libdaxctl.h" daxctl_VER_${ver}_LINE
|
||||
REGEX "^#define[ \t]+daxctl_VERSION_${ver}[ \t]+[0-9]+[ \t]+.*$")
|
||||
string(REGEX REPLACE "^#define[ \t]+daxctl_VERSION_${ver}[ \t]+([0-9]+)[ \t]+.*$"
|
||||
"\\1" daxctl_VERSION_${ver} "${daxctl_VER_${ver}_LINE}")
|
||||
unset(${daxctl_VER_${ver}_LINE})
|
||||
endforeach()
|
||||
set(daxctl_VERSION_STRING
|
||||
"${daxctl_VERSION_MAJOR}.${daxctl_VERSION_MINOR}.${daxctl_VERSION_RELEASE}")
|
||||
endif()
|
||||
|
||||
find_library(daxctl_LIBRARY daxctl)
|
||||
|
||||
include(FindPackageHandleStandardArgs)
|
||||
find_package_handle_standard_args(daxctl
|
||||
REQUIRED_VARS daxctl_LIBRARY daxctl_INCLUDE_DIR
|
||||
VERSION_VAR daxctl_VERSION_STRING)
|
||||
|
||||
mark_as_advanced(daxctl_INCLUDE_DIR daxctl_LIBRARY)
|
||||
|
||||
if(daxctl_FOUND)
|
||||
set(daxctl_INCLUDE_DIRS ${daxctl_INCLUDE_DIR})
|
||||
set(daxctl_LIBRARIES ${daxctl_LIBRARY})
|
||||
if(NOT (TARGET daxctl::daxctl))
|
||||
add_library(daxctl::daxctl UNKNOWN IMPORTED)
|
||||
set_target_properties(daxctl::daxctl PROPERTIES
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${daxctl_INCLUDE_DIRS}"
|
||||
IMPORTED_LINK_INTERFACE_LANGUAGES "C"
|
||||
IMPORTED_LOCATION "${daxctl_LIBRARIES}"
|
||||
VERSION "${daxctl_VERSION_STRING}")
|
||||
endif()
|
||||
endif()
|
||||
@ -1,58 +0,0 @@
|
||||
# - Find libdml
|
||||
# Find the dml and dmlhl libraries and includes
|
||||
#
|
||||
# DML_INCLUDE_DIR - where to find dml.hpp etc.
|
||||
# DML_LIBRARIES - List of libraries when using dml.
|
||||
# DML_HL_LIBRARIES - List of libraries when using dmlhl.
|
||||
# DML_FOUND - True if DML found.
|
||||
|
||||
|
||||
find_path(DML_INCLUDE_DIR
|
||||
dml/dml.hpp
|
||||
PATHS
|
||||
/usr/include
|
||||
/usr/local/include)
|
||||
|
||||
find_library(DML_LIBRARIES NAMES dml libdml PATHS
|
||||
/usr/local/
|
||||
/usr/local/lib64
|
||||
/usr/lib64
|
||||
/usr/lib)
|
||||
|
||||
find_library(DML_HL_LIBRARIES NAMES dmlhl libdmlhl PATHS
|
||||
/usr/local/
|
||||
/usr/local/lib64
|
||||
/usr/lib64
|
||||
/usr/lib)
|
||||
|
||||
include(FindPackageHandleStandardArgs)
|
||||
find_package_handle_standard_args(dml DEFAULT_MSG
|
||||
DML_LIBRARIES
|
||||
DML_INCLUDE_DIR
|
||||
DML_HL_LIBRARIES)
|
||||
|
||||
mark_as_advanced(
|
||||
DML_LIBRARIES
|
||||
DML_INCLUDE_DIR
|
||||
DML_HL_LIBRARIES)
|
||||
|
||||
if(DML_FOUND)
|
||||
if(NOT (TARGET dml::dml))
|
||||
add_library(dml::dml UNKNOWN IMPORTED)
|
||||
set_target_properties(dml::dml PROPERTIES
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${DML_INCLUDE_DIR}"
|
||||
IMPORTED_LINK_INTERFACE_LANGUAGES "C"
|
||||
IMPORTED_LOCATION "${DML_LIBRARIES}")
|
||||
endif()
|
||||
|
||||
if(NOT (TARGET dml::dmlhl))
|
||||
add_library(dml::dmlhl UNKNOWN IMPORTED)
|
||||
set_target_properties(dml::dmlhl PROPERTIES
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${DML_INCLUDE_DIR}"
|
||||
INTERFACE_LINK_LIBRARIES ${CMAKE_DL_LIBS}
|
||||
INTERFACE_COMPILE_FEATURES cxx_std_17
|
||||
INTERFACE_COMPILE_DEFINITIONS "DML_HW"
|
||||
IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
|
||||
IMPORTED_LOCATION "${DML_HL_LIBRARIES}")
|
||||
endif()
|
||||
endif()
|
||||
@ -1,42 +0,0 @@
|
||||
# - Find libndctl
|
||||
# Find the ndctl libraries and includes
|
||||
#
|
||||
# ndctl_INCLUDE_DIR - where to find libndctl.h etc.
|
||||
# ndctl_LIBRARIES - List of libraries when using ndctl.
|
||||
# ndctl_FOUND - True if ndctl found.
|
||||
|
||||
find_path(ndctl_INCLUDE_DIR ndctl/libndctl.h)
|
||||
|
||||
if(ndctl_INCLUDE_DIR AND EXISTS "${ndctl_INCLUDE_DIR}/libndctl.h")
|
||||
foreach(ver "MAJOR" "MINOR" "RELEASE")
|
||||
file(STRINGS "${ndctl_INCLUDE_DIR}/libndctl.h" ndctl_VER_${ver}_LINE
|
||||
REGEX "^#define[ \t]+ndctl_VERSION_${ver}[ \t]+[0-9]+[ \t]+.*$")
|
||||
string(REGEX REPLACE "^#define[ \t]+ndctl_VERSION_${ver}[ \t]+([0-9]+)[ \t]+.*$"
|
||||
"\\1" ndctl_VERSION_${ver} "${ndctl_VER_${ver}_LINE}")
|
||||
unset(${ndctl_VER_${ver}_LINE})
|
||||
endforeach()
|
||||
set(ndctl_VERSION_STRING
|
||||
"${ndctl_VERSION_MAJOR}.${ndctl_VERSION_MINOR}.${ndctl_VERSION_RELEASE}")
|
||||
endif()
|
||||
|
||||
find_library(ndctl_LIBRARY ndctl)
|
||||
|
||||
include(FindPackageHandleStandardArgs)
|
||||
find_package_handle_standard_args(ndctl
|
||||
REQUIRED_VARS ndctl_LIBRARY ndctl_INCLUDE_DIR
|
||||
VERSION_VAR ndctl_VERSION_STRING)
|
||||
|
||||
mark_as_advanced(ndctl_INCLUDE_DIR ndctl_LIBRARY)
|
||||
|
||||
if(ndctl_FOUND)
|
||||
set(ndctl_INCLUDE_DIRS ${ndctl_INCLUDE_DIR})
|
||||
set(ndctl_LIBRARIES ${ndctl_LIBRARY})
|
||||
if(NOT (TARGET ndctl::ndctl))
|
||||
add_library(ndctl::ndctl UNKNOWN IMPORTED)
|
||||
set_target_properties(ndctl::ndctl PROPERTIES
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${ndctl_INCLUDE_DIRS}"
|
||||
IMPORTED_LINK_INTERFACE_LANGUAGES "C"
|
||||
IMPORTED_LOCATION "${ndctl_LIBRARIES}"
|
||||
VERSION "${ndctl_VERSION_STRING}")
|
||||
endif()
|
||||
endif()
|
||||
3
debian/control
vendored
3
debian/control
vendored
@ -81,9 +81,6 @@ Build-Depends: automake,
|
||||
libthrift-dev (>= 0.13.0),
|
||||
libyaml-cpp-dev (>= 0.6),
|
||||
libzstd-dev <pkg.ceph.check>,
|
||||
libdaxctl-dev (>= 63) <pkg.ceph.pmdk>,
|
||||
libndctl-dev (>= 63) <pkg.ceph.pmdk>,
|
||||
libpmem-dev <pkg.ceph.pmdk>,
|
||||
libpmemobj-dev (>= 1.8) <pkg.ceph.pmdk>,
|
||||
libprotobuf-dev,
|
||||
libxsimd-dev <!pkg.ceph.arrow>,
|
||||
|
||||
@ -206,9 +206,7 @@ libzbd
|
||||
packages `libzbd`_ . The upstream libzbd includes debian packaging already.
|
||||
|
||||
libpmem
|
||||
packages `pmdk`_ . Please note, ``ndctl`` is one of the build dependencies of
|
||||
pmdk, for an updated debian packaging, please see
|
||||
https://github.com/ceph/ceph-ndctl .
|
||||
packages `pmdk`_ .
|
||||
|
||||
.. note::
|
||||
|
||||
|
||||
@ -535,24 +535,3 @@ were deployed under older releases or with other settings.
|
||||
.. confval:: bluestore_min_alloc_size_hdd
|
||||
.. confval:: bluestore_min_alloc_size_ssd
|
||||
.. confval:: bluestore_use_optimal_io_size_for_min_alloc_size
|
||||
|
||||
DSA (Data Streaming Accelerator) Usage
|
||||
======================================
|
||||
|
||||
If you want to use the DML library to drive the DSA device for offloading
|
||||
read/write operations on persistent memory (PMEM) in BlueStore, you need to
|
||||
install `DML`_ and the `idxd-config`_ library. This will work only on machines
|
||||
that have a SPR (Sapphire Rapids) CPU.
|
||||
|
||||
.. _DML: https://github.com/intel/DML
|
||||
.. _idxd-config: https://github.com/intel/idxd-config
|
||||
|
||||
After installing the DML software, configure the shared work queues (WQs) with
|
||||
reference to the following WQ configuration example:
|
||||
|
||||
.. prompt:: bash $
|
||||
|
||||
accel-config config-wq --group-id=1 --mode=shared --wq-size=16 --threshold=15 --type=user --name="MyApp1" --priority=10 --block-on-fault=1 dsa0/wq0.1
|
||||
accel-config config-engine dsa0/engine0.1 --group-id=1
|
||||
accel-config enable-device dsa0
|
||||
accel-config enable-wq dsa0/wq0.1
|
||||
|
||||
@ -654,26 +654,12 @@ if(WIN32)
|
||||
list(APPEND ceph_common_deps dlfcn_win32)
|
||||
endif()
|
||||
|
||||
if(WITH_BLUESTORE_PMEM OR WITH_RBD_RWL)
|
||||
if(WITH_SYSTEM_PMDK)
|
||||
set(pmdk_COMPONENTS)
|
||||
if(WITH_BLUESTORE_PMEM)
|
||||
list(APPEND pmdk_COMPONENTS pmem)
|
||||
endif()
|
||||
if(WITH_RBD_RWL)
|
||||
list(APPEND pmdk_COMPONENTS pmemobj)
|
||||
endif()
|
||||
find_package(pmdk 1.8 REQUIRED COMPONENTS ${pmdk_COMPONENTS})
|
||||
if(WITH_SYSTEM_PMDK)
|
||||
find_package(pmdk 1.8 REQUIRED COMPONENTS pmemobj)
|
||||
else()
|
||||
include(Buildpmdk)
|
||||
if(WITH_BLUESTORE_PMEM)
|
||||
set(enable_ndctl ON)
|
||||
find_package(ndctl 63 REQUIRED)
|
||||
find_package(daxctl 63 REQUIRED)
|
||||
else()
|
||||
set(enable_ndctl OFF)
|
||||
endif()
|
||||
build_pmdk(${enable_ndctl})
|
||||
build_pmdk()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
|
||||
@ -28,10 +28,6 @@
|
||||
#include "spdk/NVMEDevice.h"
|
||||
#endif
|
||||
|
||||
#if defined(HAVE_BLUESTORE_PMEM)
|
||||
#include "pmem/PMEMDevice.h"
|
||||
#endif
|
||||
|
||||
#include "common/debug.h"
|
||||
#include "common/EventTrace.h"
|
||||
#include "common/errno.h"
|
||||
@ -106,11 +102,6 @@ BlockDevice::detect_device_type(const std::string& path)
|
||||
return block_device_t::spdk;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAVE_BLUESTORE_PMEM)
|
||||
if (PMEMDevice::support(path)) {
|
||||
return block_device_t::pmem;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO)
|
||||
return block_device_t::aio;
|
||||
#else
|
||||
@ -130,11 +121,6 @@ BlockDevice::device_type_from_name(const std::string& blk_dev_name)
|
||||
if (blk_dev_name == "spdk") {
|
||||
return block_device_t::spdk;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAVE_BLUESTORE_PMEM)
|
||||
if (blk_dev_name == "pmem") {
|
||||
return block_device_t::pmem;
|
||||
}
|
||||
#endif
|
||||
return block_device_t::unknown;
|
||||
}
|
||||
@ -152,10 +138,6 @@ BlockDevice* BlockDevice::create_with_type(block_device_t device_type,
|
||||
#if defined(HAVE_SPDK)
|
||||
case block_device_t::spdk:
|
||||
return new NVMEDevice(cct, cb, cbpriv);
|
||||
#endif
|
||||
#if defined(HAVE_BLUESTORE_PMEM)
|
||||
case block_device_t::pmem:
|
||||
return new PMEMDevice(cct, cb, cbpriv);
|
||||
#endif
|
||||
default:
|
||||
ceph_abort_msg("unsupported device");
|
||||
|
||||
@ -164,9 +164,6 @@ private:
|
||||
#endif
|
||||
#if defined(HAVE_SPDK)
|
||||
spdk,
|
||||
#endif
|
||||
#if defined(HAVE_BLUESTORE_PMEM)
|
||||
pmem,
|
||||
#endif
|
||||
};
|
||||
std::queue <ceph::mono_clock::time_point> stalled_read_event_queue;
|
||||
|
||||
@ -10,11 +10,6 @@ if(HAVE_LIBAIO OR HAVE_POSIXAIO)
|
||||
aio/aio.cc)
|
||||
endif()
|
||||
|
||||
if(WITH_BLUESTORE_PMEM)
|
||||
list(APPEND libblk_srcs
|
||||
pmem/PMEMDevice.cc)
|
||||
endif()
|
||||
|
||||
if(WITH_SPDK)
|
||||
list(APPEND libblk_srcs
|
||||
spdk/NVMEDevice.cc)
|
||||
@ -36,15 +31,6 @@ if(WITH_SPDK)
|
||||
PRIVATE spdk::spdk)
|
||||
endif()
|
||||
|
||||
if(WITH_BLUESTORE_PMEM)
|
||||
if(HAVE_LIBDML)
|
||||
target_link_libraries(blk PRIVATE dml::dml dml::dmlhl)
|
||||
endif()
|
||||
|
||||
target_link_libraries(blk
|
||||
PRIVATE pmdk::pmem)
|
||||
endif()
|
||||
|
||||
if(WITH_EVENTTRACE)
|
||||
add_dependencies(blk eventtrace_tp)
|
||||
endif()
|
||||
|
||||
@ -1,379 +0,0 @@
|
||||
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
|
||||
// vim: ts=8 sw=2 sts=2 expandtab
|
||||
|
||||
/*
|
||||
* Ceph - scalable distributed file system
|
||||
*
|
||||
* Copyright (C) 2015 Intel <jianpeng.ma@intel.com>
|
||||
*
|
||||
* Author: Jianpeng Ma <jianpeng.ma@intel.com>
|
||||
*
|
||||
* This is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License version 2.1, as published by the Free Software
|
||||
* Foundation. See file COPYING.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <unistd.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/sysmacros.h>
|
||||
#include <stdio.h>
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <string.h>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
|
||||
#include <fmt/format.h>
|
||||
|
||||
#include "PMEMDevice.h"
|
||||
#include "libpmem.h"
|
||||
#include "include/types.h"
|
||||
#include "include/compat.h"
|
||||
#include "include/stringify.h"
|
||||
#include "common/errno.h"
|
||||
#include "common/debug.h"
|
||||
#include "common/blkdev.h"
|
||||
|
||||
#if defined(HAVE_LIBDML)
|
||||
#include <dml/dml.hpp>
|
||||
using execution_path = dml::automatic;
|
||||
#endif
|
||||
|
||||
#define dout_context cct
|
||||
#define dout_subsys ceph_subsys_bdev
|
||||
#undef dout_prefix
|
||||
#define dout_prefix *_dout << "bdev-PMEM(" << path << ") "
|
||||
|
||||
PMEMDevice::PMEMDevice(CephContext *cct, aio_callback_t cb, void *cbpriv)
|
||||
: BlockDevice(cct, cb, cbpriv),
|
||||
fd(-1), addr(0),
|
||||
injecting_crash(0)
|
||||
{
|
||||
}
|
||||
|
||||
int PMEMDevice::_lock()
|
||||
{
|
||||
struct flock l;
|
||||
memset(&l, 0, sizeof(l));
|
||||
l.l_type = F_WRLCK;
|
||||
l.l_whence = SEEK_SET;
|
||||
l.l_start = 0;
|
||||
l.l_len = 0;
|
||||
int r = ::fcntl(fd, F_SETLK, &l);
|
||||
if (r < 0)
|
||||
return -errno;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int pmem_check_file_type(int fd, const char *pmem_file, uint64_t *total_size)
|
||||
{
|
||||
namespace fs = std::filesystem;
|
||||
if (!fs::is_character_file(pmem_file)) {
|
||||
return -EINVAL;
|
||||
}
|
||||
struct stat file_stat;
|
||||
if (::fstat(fd, &file_stat)) {
|
||||
return -EINVAL;
|
||||
}
|
||||
fs::path char_dir = fmt::format("/sys/dev/char/{}:{}",
|
||||
major(file_stat.st_rdev),
|
||||
minor(file_stat.st_rdev));
|
||||
// Need to check if it is a DAX device
|
||||
if (auto subsys_path = char_dir / "subsystem";
|
||||
fs::read_symlink(subsys_path).filename().string() != "dax") {
|
||||
return -EINVAL;
|
||||
}
|
||||
if (total_size == nullptr) {
|
||||
return 0;
|
||||
}
|
||||
if (std::ifstream size_file(char_dir / "size"); size_file) {
|
||||
size_file >> *total_size;
|
||||
return size_file ? 0 : -EINVAL;
|
||||
} else {
|
||||
return -EINVAL;
|
||||
}
|
||||
}
|
||||
|
||||
int PMEMDevice::open(const std::string& p)
|
||||
{
|
||||
path = p;
|
||||
int r = 0;
|
||||
dout(1) << __func__ << " path " << path << dendl;
|
||||
|
||||
fd = ::open(path.c_str(), O_RDWR | O_CLOEXEC);
|
||||
if (fd < 0) {
|
||||
r = -errno;
|
||||
derr << __func__ << " open got: " << cpp_strerror(r) << dendl;
|
||||
return r;
|
||||
}
|
||||
|
||||
r = pmem_check_file_type(fd, path.c_str(), &size);
|
||||
if (!r) {
|
||||
dout(1) << __func__ << " This path " << path << " is a devdax dev " << dendl;
|
||||
devdax_device = true;
|
||||
// If using devdax char device, set it to not rotational device.
|
||||
rotational = false;
|
||||
}
|
||||
|
||||
r = _lock();
|
||||
if (r < 0) {
|
||||
derr << __func__ << " failed to lock " << path << ": " << cpp_strerror(r)
|
||||
<< dendl;
|
||||
goto out_fail;
|
||||
}
|
||||
|
||||
struct stat st;
|
||||
r = ::fstat(fd, &st);
|
||||
if (r < 0) {
|
||||
r = -errno;
|
||||
derr << __func__ << " fstat got " << cpp_strerror(r) << dendl;
|
||||
goto out_fail;
|
||||
}
|
||||
|
||||
size_t map_len;
|
||||
addr = (char *)pmem_map_file(path.c_str(), 0,
|
||||
devdax_device ? 0: PMEM_FILE_EXCL, O_RDWR,
|
||||
&map_len, NULL);
|
||||
if (addr == NULL) {
|
||||
derr << __func__ << " pmem_map_file failed: " << pmem_errormsg() << dendl;
|
||||
goto out_fail;
|
||||
}
|
||||
size = map_len;
|
||||
|
||||
// Operate as though the block size is 4 KB. The backing file
|
||||
// blksize doesn't strictly matter except that some file systems may
|
||||
// require a read/modify/write if we write something smaller than
|
||||
// it.
|
||||
block_size = g_conf()->bdev_block_size;
|
||||
if (block_size != (unsigned)st.st_blksize) {
|
||||
dout(1) << __func__ << " backing device/file reports st_blksize "
|
||||
<< st.st_blksize << ", using bdev_block_size "
|
||||
<< block_size << " anyway" << dendl;
|
||||
}
|
||||
|
||||
dout(1) << __func__
|
||||
<< " size " << size
|
||||
<< " (" << byte_u_t(size) << ")"
|
||||
<< " block_size " << block_size
|
||||
<< " (" << byte_u_t(block_size) << ")"
|
||||
<< dendl;
|
||||
return 0;
|
||||
|
||||
out_fail:
|
||||
VOID_TEMP_FAILURE_RETRY(::close(fd));
|
||||
fd = -1;
|
||||
return r;
|
||||
}
|
||||
|
||||
void PMEMDevice::close()
|
||||
{
|
||||
dout(1) << __func__ << dendl;
|
||||
|
||||
ceph_assert(addr != NULL);
|
||||
if (devdax_device) {
|
||||
devdax_device = false;
|
||||
}
|
||||
pmem_unmap(addr, size);
|
||||
|
||||
ceph_assert(fd >= 0);
|
||||
VOID_TEMP_FAILURE_RETRY(::close(fd));
|
||||
fd = -1;
|
||||
|
||||
path.clear();
|
||||
}
|
||||
|
||||
int PMEMDevice::collect_metadata(const std::string& prefix, std::map<std::string,std::string> *pm) const
|
||||
{
|
||||
(*pm)[prefix + "rotational"] = stringify((int)(bool)rotational);
|
||||
(*pm)[prefix + "size"] = stringify(get_size());
|
||||
(*pm)[prefix + "block_size"] = stringify(get_block_size());
|
||||
(*pm)[prefix + "driver"] = "PMEMDevice";
|
||||
(*pm)[prefix + "type"] = "ssd";
|
||||
|
||||
struct stat st;
|
||||
int r = ::fstat(fd, &st);
|
||||
if (r < 0)
|
||||
return -errno;
|
||||
if (S_ISBLK(st.st_mode)) {
|
||||
(*pm)[prefix + "access_mode"] = "blk";
|
||||
char buffer[1024] = {0};
|
||||
BlkDev blkdev(fd);
|
||||
|
||||
blkdev.model(buffer, sizeof(buffer));
|
||||
(*pm)[prefix + "model"] = buffer;
|
||||
|
||||
buffer[0] = '\0';
|
||||
blkdev.dev(buffer, sizeof(buffer));
|
||||
(*pm)[prefix + "dev"] = buffer;
|
||||
|
||||
// nvme exposes a serial number
|
||||
buffer[0] = '\0';
|
||||
blkdev.serial(buffer, sizeof(buffer));
|
||||
(*pm)[prefix + "serial"] = buffer;
|
||||
|
||||
} else if (S_ISCHR(st.st_mode)) {
|
||||
(*pm)[prefix + "access_mode"] = "chardevice";
|
||||
(*pm)[prefix + "path"] = path;
|
||||
|
||||
} else {
|
||||
(*pm)[prefix + "access_mode"] = "file";
|
||||
(*pm)[prefix + "path"] = path;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool PMEMDevice::support(const std::string &path)
|
||||
{
|
||||
int is_pmem = 0;
|
||||
size_t map_len = 0;
|
||||
int r = 0;
|
||||
int local_fd;
|
||||
|
||||
local_fd = ::open(path.c_str(), O_RDWR);
|
||||
if (local_fd < 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
r = pmem_check_file_type(local_fd, path.c_str(), NULL);
|
||||
VOID_TEMP_FAILURE_RETRY(::close(local_fd));
|
||||
int flags = PMEM_FILE_EXCL;
|
||||
if (r == 0) {
|
||||
flags = 0;
|
||||
}
|
||||
|
||||
void *addr = pmem_map_file(path.c_str(), 0, flags, O_RDONLY, &map_len, &is_pmem);
|
||||
if (addr != NULL) {
|
||||
pmem_unmap(addr, map_len);
|
||||
if (is_pmem) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
int PMEMDevice::flush()
|
||||
{
|
||||
//Because all write is persist. So no need
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
void PMEMDevice::aio_submit(IOContext *ioc)
|
||||
{
|
||||
if (ioc->priv) {
|
||||
ceph_assert(ioc->num_running == 0);
|
||||
aio_callback(aio_callback_priv, ioc->priv);
|
||||
} else {
|
||||
ioc->try_aio_wake();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
int PMEMDevice::write(uint64_t off, bufferlist& bl, bool buffered, int write_hint)
|
||||
{
|
||||
uint64_t len = bl.length();
|
||||
dout(20) << __func__ << " " << off << "~" << len << dendl;
|
||||
ceph_assert(is_valid_io(off, len));
|
||||
|
||||
dout(40) << "data:\n";
|
||||
bl.hexdump(*_dout);
|
||||
*_dout << dendl;
|
||||
|
||||
if (g_conf()->bdev_inject_crash &&
|
||||
rand() % g_conf()->bdev_inject_crash == 0) {
|
||||
derr << __func__ << " bdev_inject_crash: dropping io " << off << "~" << len
|
||||
<< dendl;
|
||||
++injecting_crash;
|
||||
return 0;
|
||||
}
|
||||
|
||||
bufferlist::iterator p = bl.begin();
|
||||
uint64_t off1 = off;
|
||||
while (len) {
|
||||
const char *data;
|
||||
uint32_t l = p.get_ptr_and_advance(len, &data);
|
||||
|
||||
#if defined(HAVE_LIBDML)
|
||||
// Take care of the persistency issue
|
||||
auto result = dml::execute<execution_path>(dml::mem_move, dml::make_view(data, l), dml::make_view(addr + off1, l));
|
||||
ceph_assert(result.status == dml::status_code::ok);
|
||||
#else
|
||||
pmem_memcpy_persist(addr + off1, data, l);
|
||||
#endif
|
||||
len -= l;
|
||||
off1 += l;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int PMEMDevice::aio_write(
|
||||
uint64_t off,
|
||||
bufferlist &bl,
|
||||
IOContext *ioc,
|
||||
bool buffered,
|
||||
int write_hint)
|
||||
{
|
||||
return write(off, bl, buffered);
|
||||
}
|
||||
|
||||
|
||||
int PMEMDevice::read(uint64_t off, uint64_t len, bufferlist *pbl,
|
||||
IOContext *ioc,
|
||||
bool buffered)
|
||||
{
|
||||
dout(5) << __func__ << " " << off << "~" << len << dendl;
|
||||
ceph_assert(is_valid_io(off, len));
|
||||
|
||||
bufferptr p = buffer::create_small_page_aligned(len);
|
||||
|
||||
#if defined(HAVE_LIBDML)
|
||||
auto result = dml::execute<execution_path>(dml::mem_move, dml::make_view(addr + off, len), dml::make_view(p.c_str(), len));
|
||||
ceph_assert(result.status == dml::status_code::ok);
|
||||
#else
|
||||
memcpy(p.c_str(), addr + off, len);
|
||||
#endif
|
||||
|
||||
pbl->clear();
|
||||
pbl->push_back(std::move(p));
|
||||
|
||||
dout(40) << "data:\n";
|
||||
pbl->hexdump(*_dout);
|
||||
*_dout << dendl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int PMEMDevice::aio_read(uint64_t off, uint64_t len, bufferlist *pbl,
|
||||
IOContext *ioc)
|
||||
{
|
||||
return read(off, len, pbl, ioc, false);
|
||||
}
|
||||
|
||||
int PMEMDevice::read_random(uint64_t off, uint64_t len, char *buf, bool buffered)
|
||||
{
|
||||
dout(5) << __func__ << " " << off << "~" << len << dendl;
|
||||
ceph_assert(is_valid_io(off, len));
|
||||
|
||||
|
||||
#if defined(HAVE_LIBDML)
|
||||
auto result = dml::execute<execution_path>(dml::mem_move, dml::make_view(addr + off, len), dml::make_view(buf, len));
|
||||
ceph_assert(result.status == dml::status_code::ok);
|
||||
#else
|
||||
memcpy(buf, addr + off, len);
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int PMEMDevice::invalidate_cache(uint64_t off, uint64_t len)
|
||||
{
|
||||
dout(5) << __func__ << " " << off << "~" << len << dendl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@ -1,79 +0,0 @@
|
||||
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
|
||||
// vim: ts=8 sw=2 sts=2 expandtab
|
||||
|
||||
/*
|
||||
* Ceph - scalable distributed file system
|
||||
*
|
||||
* Copyright (C) 2015 Intel <jianpeng.ma@intel.com>
|
||||
*
|
||||
* Author: Jianpeng Ma <jianpeng.ma@intel.com>
|
||||
*
|
||||
* This is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License version 2.1, as published by the Free Software
|
||||
* Foundation. See file COPYING.
|
||||
*
|
||||
*/
|
||||
|
||||
#ifndef CEPH_BLK_PMEMDEVICE_H
|
||||
#define CEPH_BLK_PMEMDEVICE_H
|
||||
|
||||
#include <atomic>
|
||||
#include <map>
|
||||
#include <string>
|
||||
|
||||
#include "os/fs/FS.h"
|
||||
#include "include/interval_set.h"
|
||||
#include "aio/aio.h"
|
||||
#include "BlockDevice.h"
|
||||
|
||||
class PMEMDevice : public BlockDevice {
|
||||
int fd;
|
||||
char *addr; //the address of mmap
|
||||
std::string path;
|
||||
bool devdax_device = false;
|
||||
|
||||
ceph::mutex debug_lock = ceph::make_mutex("PMEMDevice::debug_lock");
|
||||
interval_set<uint64_t> debug_inflight;
|
||||
|
||||
std::atomic_int injecting_crash;
|
||||
int _lock();
|
||||
|
||||
public:
|
||||
PMEMDevice(CephContext *cct, aio_callback_t cb, void *cbpriv);
|
||||
|
||||
bool supported_bdev_label() override { return !devdax_device; }
|
||||
void aio_submit(IOContext *ioc) override;
|
||||
|
||||
int collect_metadata(const std::string& prefix, std::map<std::string,std::string> *pm) const override;
|
||||
|
||||
static bool support(const std::string& path);
|
||||
|
||||
int read(uint64_t off, uint64_t len, bufferlist *pbl,
|
||||
IOContext *ioc,
|
||||
bool buffered) override;
|
||||
int aio_read(uint64_t off, uint64_t len, bufferlist *pbl,
|
||||
IOContext *ioc) override;
|
||||
|
||||
int read_random(uint64_t off, uint64_t len, char *buf, bool buffered) override;
|
||||
int write(uint64_t off, bufferlist& bl, bool buffered, int write_hint = WRITE_LIFE_NOT_SET) override;
|
||||
int aio_write(uint64_t off, bufferlist& bl,
|
||||
IOContext *ioc,
|
||||
bool buffered,
|
||||
int write_hint = WRITE_LIFE_NOT_SET) override;
|
||||
int flush() override;
|
||||
|
||||
// for managing buffered readers/writers
|
||||
int invalidate_cache(uint64_t off, uint64_t len) override;
|
||||
int open(const std::string &path) override;
|
||||
void close() override;
|
||||
|
||||
private:
|
||||
bool is_valid_io(uint64_t off, uint64_t len) const {
|
||||
return (len > 0 &&
|
||||
off < size &&
|
||||
off + len <= size);
|
||||
}
|
||||
};
|
||||
|
||||
#endif
|
||||
@ -6992,7 +6992,6 @@ options:
|
||||
enum_values:
|
||||
- aio
|
||||
- spdk
|
||||
- pmem
|
||||
- name: bdev_stalled_read_warn_lifetime
|
||||
type: uint
|
||||
level: advanced
|
||||
|
||||
@ -84,9 +84,6 @@
|
||||
/* Defined if you have libaio */
|
||||
#cmakedefine HAVE_LIBAIO
|
||||
|
||||
/* Defined if you have libdml */
|
||||
#cmakedefine HAVE_LIBDML
|
||||
|
||||
/* Defined if you have liburing */
|
||||
#cmakedefine HAVE_LIBURING
|
||||
|
||||
@ -114,9 +111,6 @@
|
||||
/* DPDK conditional compilation */
|
||||
#cmakedefine HAVE_DPDK
|
||||
|
||||
/* PMEM_DEVICE (OSD) conditional compilation */
|
||||
#cmakedefine HAVE_BLUESTORE_PMEM
|
||||
|
||||
/* Define if you have tcmalloc */
|
||||
#cmakedefine HAVE_LIBTCMALLOC
|
||||
#cmakedefine LIBTCMALLOC_MISSING_ALIGNED_ALLOC
|
||||
|
||||
@ -187,7 +187,6 @@ rgw_compression=""
|
||||
rgw_store="rados"
|
||||
lockdep=${LOCKDEP:-1}
|
||||
spdk_enabled=0 # disable SPDK by default
|
||||
pmem_enabled=0
|
||||
io_uring_enabled=0
|
||||
with_jaeger=0
|
||||
force_addr=0
|
||||
@ -259,7 +258,6 @@ options:
|
||||
--multimds <count> allow multimds with maximum active count
|
||||
--without-dashboard: do not run using mgr dashboard
|
||||
--bluestore-spdk: enable SPDK and with a comma-delimited list of PCI-IDs of NVME device (e.g, 0000:81:00.0)
|
||||
--bluestore-pmem: enable PMEM and with path to a file mapped to PMEM
|
||||
--msgr1: use msgr1 only
|
||||
--msgr2: use msgr2 only
|
||||
--msgr21: use msgr2 and msgr1
|
||||
@ -657,12 +655,6 @@ case $1 in
|
||||
spdk_enabled=1
|
||||
shift
|
||||
;;
|
||||
--bluestore-pmem)
|
||||
[ -z "$2" ] && usage_exit
|
||||
bluestore_pmem_file="$2"
|
||||
pmem_enabled=1
|
||||
shift
|
||||
;;
|
||||
--bluestore-devs)
|
||||
parse_block_devs --bluestore-devs "$2"
|
||||
shift
|
||||
@ -911,7 +903,7 @@ EOF
|
||||
osd max object namespace len = 64"
|
||||
fi
|
||||
if [ "$objectstore" == "bluestore" ]; then
|
||||
if [ "$spdk_enabled" -eq 1 ] || [ "$pmem_enabled" -eq 1 ]; then
|
||||
if [ "$spdk_enabled" -eq 1 ]; then
|
||||
BLUESTORE_OPTS=" bluestore_block_db_path = \"\"
|
||||
bluestore_block_db_size = 0
|
||||
bluestore_block_db_create = false
|
||||
@ -1290,10 +1282,6 @@ EOF
|
||||
if [ "$spdk_enabled" -eq 1 ]; then
|
||||
wconf <<EOF
|
||||
bluestore_block_path = spdk:${bluestore_spdk_dev[$osd]}
|
||||
EOF
|
||||
elif [ "$pmem_enabled" -eq 1 ]; then
|
||||
wconf <<EOF
|
||||
bluestore_block_path = ${bluestore_pmem_file}
|
||||
EOF
|
||||
fi
|
||||
rm -rf $CEPH_DEV_DIR/osd$osd || true
|
||||
|
||||
Loading…
Reference in New Issue
Block a user
