Merge pull request #69726 from tchaikov/wip-drop-bluestore-pmem

blk,build: drop BlueStore PMEM support

Reviewed-by: Igor Fedotov <igor.fedotov@croit.io>

CMakeLists.txt

@@ -223,7 +223,6 @@ if(LINUX)
 elseif(FREEBSD)
   set(HAVE_UDEV OFF)
   set(HAVE_LIBAIO OFF)
-  set(HAVE_LIBDML OFF)
   set(HAVE_BLKID OFF)
   set(HAVE_KEYUTILS OFF)
 else()
@@ -304,13 +303,6 @@ if(WITH_LIBURING)
   endif()
 endif()
 
-CMAKE_DEPENDENT_OPTION(WITH_BLUESTORE_PMEM "Enable PMDK libraries" OFF
-  "WITH_BLUESTORE" OFF)
-if(WITH_BLUESTORE_PMEM)
-  find_package(dml)
-  set(HAVE_LIBDML ${DML_FOUND})
-endif()
-
 CMAKE_DEPENDENT_OPTION(WITH_RBD_MIRROR "Enable build for rbd-mirror daemon executable" OFF
     "WITH_RBD" OFF)
 
@@ -325,7 +317,7 @@ CMAKE_DEPENDENT_OPTION(WITH_RBD_SSD_CACHE "Enable librbd persistent write back c
     "WITH_RBD" OFF)
 
 CMAKE_DEPENDENT_OPTION(WITH_SYSTEM_PMDK "Require and build with system PMDK" OFF
-  "WITH_RBD_RWL OR WITH_BLUESTORE_PMEM" OFF)
+  "WITH_RBD_RWL" OFF)
 
 CMAKE_DEPENDENT_OPTION(WITH_RBD_UBBD "Enable ubbd support for rbd device utility" OFF
     "WITH_RBD" OFF)
@@ -335,10 +327,6 @@ if(WITH_RBD_UBBD)
   build_ubbd()
 endif()
 
-if(WITH_BLUESTORE_PMEM)
-  set(HAVE_BLUESTORE_PMEM ON)
-endif()
-
 CMAKE_DEPENDENT_OPTION(WITH_SPDK "Enable SPDK" OFF
   "CMAKE_SYSTEM_PROCESSOR MATCHES i386|i686|amd64|x86_64|AMD64|aarch64|riscv64" OFF)
 option(WITH_SYSTEM_SPDK "Require a system SPDK instead of building bundled src/spdk" OFF)
@@ -356,10 +344,10 @@ if(WITH_SPDK)
 endif(WITH_SPDK)
 
 if(WITH_BLUESTORE)
-  if(NOT AIO_FOUND AND NOT HAVE_POSIXAIO AND NOT WITH_SPDK AND NOT WITH_BLUESTORE_PMEM)
+  if(NOT AIO_FOUND AND NOT HAVE_POSIXAIO AND NOT WITH_SPDK)
     message(SEND_ERROR "WITH_BLUESTORE is ON, "
       "but none of the bluestore backends is enabled. "
-      "Please install libaio, or enable WITH_SPDK or WITH_BLUESTORE_PMEM (experimental)")
+      "Please install libaio, or enable WITH_SPDK")
   endif()
 endif()
 

PendingReleaseNotes

@@ -269,6 +269,18 @@
   rather rigid and also challenging to disseminate in some environments.  The
   key can be embedded in the migration spec or just referenced from there while
   stored in the MON config-key store.
+* BlueStore: The experimental PMEM device backend has been removed, together
+  with the ``pmem`` value of the ``bdev_type`` option and the DML/DSA offload
+  path. The hardware it targeted, Intel Optane DC persistent memory, was
+  discontinued in 2022. The RBD persistent write-back cache, which also uses
+  PMDK, is unaffected.
+
+  Existing OSDs backed by persistent memory in fsdax mode (a block device such
+  as ``/dev/pmem0``) keep working on the default ``aio`` backend over the same
+  data. Remove any explicit ``bdev_type = pmem`` from their configuration. OSDs
+  on a devdax namespace (a character device such as ``/dev/dax0.0``) will no
+  longer start, because no remaining backend can open one; reconfigure the
+  namespace to fsdax with ``ndctl`` or re-provision them before upgrading.
 
 >=20.0.0
 

ceph.spec.in

@@ -384,13 +384,6 @@ BuildRequires:	pkgconfig(nlohmann_json)
 BuildRequires:  pkgconfig(libevent)
 %endif
 %if 0%{with system_pmdk}
-%if 0%{?suse_version}
-BuildRequires:  pkgconfig(libndctl) >= 63
-%else
-BuildRequires:  pkgconfig(libndctl) >= 63
-BuildRequires:  pkgconfig(libdaxctl) >= 63
-%endif
-BuildRequires:  pkgconfig(libpmem)
 BuildRequires:  pkgconfig(libpmemobj) >= 1.8
 %endif
 %if 0%{with system_arrow}

cmake/modules/Buildpmdk.cmake

@@ -1,4 +1,4 @@
-function(build_pmdk enable_ndctl)
+function(build_pmdk)
   include(FindMake)
   find_make("MAKE_EXECUTABLE" "make_cmd")
 
@@ -15,12 +15,6 @@ function(build_pmdk enable_ndctl)
   endif()
 
   set(LIBPMEM_INTERFACE_LINK_LIBRARIES Threads::Threads)
-  if(${enable_ndctl})
-    set(ndctl "y")
-    list(APPEND LIBPMEM_INTERFACE_LINK_LIBRARIES ndctl::ndctl daxctl::daxctl)
-  else()
-    set(ndctl "n")
-  endif()
 
   # Use debug PMDK libs in debug lib/rbd builds
   if(CMAKE_BUILD_TYPE STREQUAL Debug)
@@ -34,7 +28,7 @@ function(build_pmdk enable_ndctl)
   ExternalProject_Add(pmdk_ext
       ${source_dir_args}
       CONFIGURE_COMMAND ""
-      BUILD_COMMAND ${make_cmd} CC=${CMAKE_C_COMPILER} "EXTRA_CFLAGS=${pmdk_cflags}" NDCTL_ENABLE=${ndctl} BUILD_EXAMPLES=n BUILD_BENCHMARKS=n DOC=n
+      BUILD_COMMAND ${make_cmd} CC=${CMAKE_C_COMPILER} "EXTRA_CFLAGS=${pmdk_cflags}" NDCTL_ENABLE=n BUILD_EXAMPLES=n BUILD_BENCHMARKS=n DOC=n
       BUILD_IN_SOURCE 1
       BUILD_BYPRODUCTS "<SOURCE_DIR>/src/${PMDK_LIB_DIR}/libpmem.a" "<SOURCE_DIR>/src/${PMDK_LIB_DIR}/libpmemobj.a"
       INSTALL_COMMAND "")

cmake/modules/Finddaxctl.cmake

@@ -1,42 +0,0 @@
-# - Find libdaxctl
-# Find the daxctl libraries and includes
-#
-# daxctl_INCLUDE_DIR - where to find libdaxctl.h etc.
-# daxctl_LIBRARIES - List of libraries when using daxctl.
-# daxctl_FOUND - True if daxctl found.
-
-find_path(daxctl_INCLUDE_DIR daxctl/libdaxctl.h)
-
-if(daxctl_INCLUDE_DIR AND EXISTS "${daxctl_INCLUDE_DIR}/libdaxctl.h")
-  foreach(ver "MAJOR" "MINOR" "RELEASE")
-    file(STRINGS "${daxctl_INCLUDE_DIR}/libdaxctl.h" daxctl_VER_${ver}_LINE
-      REGEX "^#define[ \t]+daxctl_VERSION_${ver}[ \t]+[0-9]+[ \t]+.*$")
-    string(REGEX REPLACE "^#define[ \t]+daxctl_VERSION_${ver}[ \t]+([0-9]+)[ \t]+.*$"
-      "\\1" daxctl_VERSION_${ver} "${daxctl_VER_${ver}_LINE}")
-    unset(${daxctl_VER_${ver}_LINE})
-  endforeach()
-  set(daxctl_VERSION_STRING
-    "${daxctl_VERSION_MAJOR}.${daxctl_VERSION_MINOR}.${daxctl_VERSION_RELEASE}")
-endif()
-
-find_library(daxctl_LIBRARY daxctl)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(daxctl
-  REQUIRED_VARS daxctl_LIBRARY daxctl_INCLUDE_DIR
-  VERSION_VAR daxctl_VERSION_STRING)
-
-mark_as_advanced(daxctl_INCLUDE_DIR daxctl_LIBRARY)
-
-if(daxctl_FOUND)
-  set(daxctl_INCLUDE_DIRS ${daxctl_INCLUDE_DIR})
-  set(daxctl_LIBRARIES ${daxctl_LIBRARY})
-  if(NOT (TARGET daxctl::daxctl))
-    add_library(daxctl::daxctl UNKNOWN IMPORTED)
-    set_target_properties(daxctl::daxctl PROPERTIES
-      INTERFACE_INCLUDE_DIRECTORIES "${daxctl_INCLUDE_DIRS}"
-      IMPORTED_LINK_INTERFACE_LANGUAGES "C"
-      IMPORTED_LOCATION "${daxctl_LIBRARIES}"
-      VERSION "${daxctl_VERSION_STRING}")
-  endif()
-endif()

cmake/modules/Finddml.cmake

@@ -1,58 +0,0 @@
-# - Find libdml
-# Find the dml and dmlhl libraries and includes
-#
-# DML_INCLUDE_DIR - where to find dml.hpp etc.
-# DML_LIBRARIES - List of libraries when using dml.
-# DML_HL_LIBRARIES - List of libraries when using dmlhl.
-# DML_FOUND - True if DML found.
-
-
-find_path(DML_INCLUDE_DIR
-  dml/dml.hpp
-  PATHS
-  /usr/include
-  /usr/local/include)
-
-find_library(DML_LIBRARIES NAMES dml libdml PATHS
-  /usr/local/
-  /usr/local/lib64
-  /usr/lib64
-  /usr/lib)
-
-find_library(DML_HL_LIBRARIES NAMES dmlhl libdmlhl PATHS
-  /usr/local/
-  /usr/local/lib64
-  /usr/lib64
-  /usr/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(dml DEFAULT_MSG
-  DML_LIBRARIES
-  DML_INCLUDE_DIR
-  DML_HL_LIBRARIES)
-
-mark_as_advanced(
-  DML_LIBRARIES
-  DML_INCLUDE_DIR
-  DML_HL_LIBRARIES)
-
-if(DML_FOUND)
-  if(NOT (TARGET dml::dml))
-    add_library(dml::dml UNKNOWN IMPORTED)
-    set_target_properties(dml::dml PROPERTIES
-      INTERFACE_INCLUDE_DIRECTORIES "${DML_INCLUDE_DIR}"
-      IMPORTED_LINK_INTERFACE_LANGUAGES "C"
-      IMPORTED_LOCATION "${DML_LIBRARIES}")
-  endif()
-
-  if(NOT (TARGET dml::dmlhl))
-    add_library(dml::dmlhl UNKNOWN IMPORTED)
-    set_target_properties(dml::dmlhl PROPERTIES
-      INTERFACE_INCLUDE_DIRECTORIES "${DML_INCLUDE_DIR}"
-      INTERFACE_LINK_LIBRARIES ${CMAKE_DL_LIBS}
-      INTERFACE_COMPILE_FEATURES cxx_std_17
-      INTERFACE_COMPILE_DEFINITIONS "DML_HW"
-      IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
-      IMPORTED_LOCATION "${DML_HL_LIBRARIES}")
-  endif()
-endif()

cmake/modules/Findndctl.cmake

@@ -1,42 +0,0 @@
-# - Find libndctl
-# Find the ndctl libraries and includes
-#
-# ndctl_INCLUDE_DIR - where to find libndctl.h etc.
-# ndctl_LIBRARIES - List of libraries when using ndctl.
-# ndctl_FOUND - True if ndctl found.
-
-find_path(ndctl_INCLUDE_DIR ndctl/libndctl.h)
-
-if(ndctl_INCLUDE_DIR AND EXISTS "${ndctl_INCLUDE_DIR}/libndctl.h")
-  foreach(ver "MAJOR" "MINOR" "RELEASE")
-    file(STRINGS "${ndctl_INCLUDE_DIR}/libndctl.h" ndctl_VER_${ver}_LINE
-      REGEX "^#define[ \t]+ndctl_VERSION_${ver}[ \t]+[0-9]+[ \t]+.*$")
-    string(REGEX REPLACE "^#define[ \t]+ndctl_VERSION_${ver}[ \t]+([0-9]+)[ \t]+.*$"
-      "\\1" ndctl_VERSION_${ver} "${ndctl_VER_${ver}_LINE}")
-    unset(${ndctl_VER_${ver}_LINE})
-  endforeach()
-  set(ndctl_VERSION_STRING
-    "${ndctl_VERSION_MAJOR}.${ndctl_VERSION_MINOR}.${ndctl_VERSION_RELEASE}")
-endif()
-
-find_library(ndctl_LIBRARY ndctl)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(ndctl
-  REQUIRED_VARS ndctl_LIBRARY ndctl_INCLUDE_DIR
-  VERSION_VAR ndctl_VERSION_STRING)
-
-mark_as_advanced(ndctl_INCLUDE_DIR ndctl_LIBRARY)
-
-if(ndctl_FOUND)
-  set(ndctl_INCLUDE_DIRS ${ndctl_INCLUDE_DIR})
-  set(ndctl_LIBRARIES ${ndctl_LIBRARY})
-  if(NOT (TARGET ndctl::ndctl))
-    add_library(ndctl::ndctl UNKNOWN IMPORTED)
-    set_target_properties(ndctl::ndctl PROPERTIES
-      INTERFACE_INCLUDE_DIRECTORIES "${ndctl_INCLUDE_DIRS}"
-      IMPORTED_LINK_INTERFACE_LANGUAGES "C"
-      IMPORTED_LOCATION "${ndctl_LIBRARIES}"
-      VERSION "${ndctl_VERSION_STRING}")
-  endif()
-endif()

debian/control

@@ -81,9 +81,6 @@ Build-Depends: automake,
                libthrift-dev (>= 0.13.0),
                libyaml-cpp-dev (>= 0.6),
                libzstd-dev <pkg.ceph.check>,
-               libdaxctl-dev (>= 63) <pkg.ceph.pmdk>,
-               libndctl-dev (>= 63) <pkg.ceph.pmdk>,
-               libpmem-dev <pkg.ceph.pmdk>,
                libpmemobj-dev (>= 1.8) <pkg.ceph.pmdk>,
                libprotobuf-dev,
                libxsimd-dev <!pkg.ceph.arrow>,

doc/dev/continuous-integration.rst

@@ -206,9 +206,7 @@ libzbd
     packages `libzbd`_ . The upstream libzbd includes debian packaging already.
 
 libpmem
-    packages `pmdk`_ . Please note, ``ndctl`` is one of the build dependencies of
+    packages `pmdk`_ .
-    pmdk, for an updated debian packaging, please see
-    https://github.com/ceph/ceph-ndctl .
 
 .. note::
 

doc/rados/configuration/bluestore-config-ref.rst

@@ -535,24 +535,3 @@ were deployed under older releases or with other settings.
 .. confval:: bluestore_min_alloc_size_hdd
 .. confval:: bluestore_min_alloc_size_ssd
 .. confval:: bluestore_use_optimal_io_size_for_min_alloc_size
-
-DSA (Data Streaming Accelerator) Usage
-======================================
-
-If you want to use the DML library to drive the DSA device for offloading
-read/write operations on persistent memory (PMEM) in BlueStore, you need to
-install `DML`_ and the `idxd-config`_ library. This will work only on machines
-that have a SPR (Sapphire Rapids) CPU.
-
-.. _DML: https://github.com/intel/DML
-.. _idxd-config: https://github.com/intel/idxd-config
-
-After installing the DML software, configure the shared work queues (WQs) with
-reference to the following WQ configuration example:
-
-.. prompt:: bash $
-
-   accel-config config-wq --group-id=1 --mode=shared --wq-size=16 --threshold=15 --type=user --name="MyApp1" --priority=10 --block-on-fault=1 dsa0/wq0.1
-   accel-config config-engine dsa0/engine0.1 --group-id=1
-   accel-config enable-device dsa0
-   accel-config enable-wq dsa0/wq0.1

src/CMakeLists.txt

@@ -654,26 +654,12 @@ if(WIN32)
   list(APPEND ceph_common_deps dlfcn_win32)
 endif()
 
-if(WITH_BLUESTORE_PMEM OR WITH_RBD_RWL)
+if(WITH_RBD_RWL)
   if(WITH_SYSTEM_PMDK)
-    set(pmdk_COMPONENTS)
+    find_package(pmdk 1.8 REQUIRED COMPONENTS pmemobj)
-    if(WITH_BLUESTORE_PMEM)
-      list(APPEND pmdk_COMPONENTS pmem)
-    endif()
-    if(WITH_RBD_RWL)
-      list(APPEND pmdk_COMPONENTS pmemobj)
-    endif()
-    find_package(pmdk 1.8 REQUIRED COMPONENTS ${pmdk_COMPONENTS})
   else()
     include(Buildpmdk)
-    if(WITH_BLUESTORE_PMEM)
+    build_pmdk()
-      set(enable_ndctl ON)
-      find_package(ndctl 63 REQUIRED)
-      find_package(daxctl 63 REQUIRED)
-    else()
-      set(enable_ndctl OFF)
-    endif()
-    build_pmdk(${enable_ndctl})
   endif()
 endif()
 

src/blk/BlockDevice.cc

@@ -28,10 +28,6 @@
 #include "spdk/NVMEDevice.h"
 #endif
 
-#if defined(HAVE_BLUESTORE_PMEM)
-#include "pmem/PMEMDevice.h"
-#endif
-
 #include "common/debug.h"
 #include "common/EventTrace.h"
 #include "common/errno.h"
@@ -106,11 +102,6 @@ BlockDevice::detect_device_type(const std::string& path)
     return block_device_t::spdk;
   }
 #endif
-#if defined(HAVE_BLUESTORE_PMEM)
-  if (PMEMDevice::support(path)) {
-    return block_device_t::pmem;
-  }
-#endif
 #if defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO)
   return block_device_t::aio;
 #else
@@ -130,11 +121,6 @@ BlockDevice::device_type_from_name(const std::string& blk_dev_name)
   if (blk_dev_name == "spdk") {
     return block_device_t::spdk;
   }
-#endif
-#if defined(HAVE_BLUESTORE_PMEM)
-  if (blk_dev_name == "pmem") {
-    return block_device_t::pmem;
-  }
 #endif
   return block_device_t::unknown;
 }
@@ -152,10 +138,6 @@ BlockDevice* BlockDevice::create_with_type(block_device_t device_type,
 #if defined(HAVE_SPDK)
   case block_device_t::spdk:
     return new NVMEDevice(cct, cb, cbpriv);
-#endif
-#if defined(HAVE_BLUESTORE_PMEM)
-  case block_device_t::pmem:
-    return new PMEMDevice(cct, cb, cbpriv);
 #endif
   default:
     ceph_abort_msg("unsupported device");

src/blk/BlockDevice.h

@@ -164,9 +164,6 @@ private:
 #endif
 #if defined(HAVE_SPDK)
     spdk,
-#endif
-#if defined(HAVE_BLUESTORE_PMEM)
-    pmem,
 #endif
   };
   std::queue <ceph::mono_clock::time_point> stalled_read_event_queue;

src/blk/CMakeLists.txt

@@ -10,11 +10,6 @@ if(HAVE_LIBAIO OR HAVE_POSIXAIO)
     aio/aio.cc)
 endif()
 
-if(WITH_BLUESTORE_PMEM)
-  list(APPEND libblk_srcs
-    pmem/PMEMDevice.cc)
-endif()
-
 if(WITH_SPDK)
   list(APPEND libblk_srcs
     spdk/NVMEDevice.cc)
@@ -36,15 +31,6 @@ if(WITH_SPDK)
     PRIVATE spdk::spdk)
 endif()
 
-if(WITH_BLUESTORE_PMEM)
-  if(HAVE_LIBDML)
-    target_link_libraries(blk PRIVATE dml::dml dml::dmlhl)
-  endif()
-
-  target_link_libraries(blk
-    PRIVATE pmdk::pmem)
-endif()
-
 if(WITH_EVENTTRACE)
   add_dependencies(blk eventtrace_tp)
 endif()

src/blk/pmem/PMEMDevice.cc

@@ -1,379 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
-// vim: ts=8 sw=2 sts=2 expandtab
-
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2015 Intel <jianpeng.ma@intel.com>
- *
- * Author: Jianpeng Ma <jianpeng.ma@intel.com>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-#include <unistd.h>
-#include <stdlib.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/sysmacros.h>
-#include <stdio.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <string.h>
-#include <filesystem>
-#include <fstream>
-
-#include <fmt/format.h>
-
-#include "PMEMDevice.h"
-#include "libpmem.h"
-#include "include/types.h"
-#include "include/compat.h"
-#include "include/stringify.h"
-#include "common/errno.h"
-#include "common/debug.h"
-#include "common/blkdev.h"
-
-#if defined(HAVE_LIBDML)
-#include <dml/dml.hpp>
-using execution_path = dml::automatic;
-#endif
-
-#define dout_context cct
-#define dout_subsys ceph_subsys_bdev
-#undef dout_prefix
-#define dout_prefix *_dout << "bdev-PMEM("  << path << ") "
-
-PMEMDevice::PMEMDevice(CephContext *cct, aio_callback_t cb, void *cbpriv)
-  : BlockDevice(cct, cb, cbpriv),
-    fd(-1), addr(0),
-    injecting_crash(0)
-{
-}
-
-int PMEMDevice::_lock()
-{
-  struct flock l;
-  memset(&l, 0, sizeof(l));
-  l.l_type = F_WRLCK;
-  l.l_whence = SEEK_SET;
-  l.l_start = 0;
-  l.l_len = 0;
-  int r = ::fcntl(fd, F_SETLK, &l);
-  if (r < 0)
-    return -errno;
-  return 0;
-}
-
-static int pmem_check_file_type(int fd, const char *pmem_file, uint64_t *total_size)
-{
-  namespace fs = std::filesystem;
-  if (!fs::is_character_file(pmem_file)) {
-    return -EINVAL;
-  }
-  struct stat file_stat;
-  if (::fstat(fd, &file_stat)) {
-    return -EINVAL;
-  }
-  fs::path char_dir = fmt::format("/sys/dev/char/{}:{}",
-				  major(file_stat.st_rdev),
-				  minor(file_stat.st_rdev));
-  // Need to check if it is a DAX device
-  if (auto subsys_path = char_dir / "subsystem";
-      fs::read_symlink(subsys_path).filename().string() != "dax") {
-    return -EINVAL;
-  }
-  if (total_size == nullptr) {
-    return 0;
-  }
-  if (std::ifstream size_file(char_dir / "size"); size_file) {
-    size_file >> *total_size;
-    return size_file ? 0 : -EINVAL;
-  } else {
-    return -EINVAL;
-  }
-}
-
-int PMEMDevice::open(const std::string& p)
-{
-  path = p;
-  int r = 0;
-  dout(1) << __func__ << " path " << path << dendl;
-
-  fd = ::open(path.c_str(), O_RDWR | O_CLOEXEC);
-  if (fd < 0) {
-    r = -errno;
-    derr << __func__ << " open got: " << cpp_strerror(r) << dendl;
-    return r;
-  }
-
-  r = pmem_check_file_type(fd, path.c_str(), &size);
-  if (!r) {
-    dout(1) << __func__ << " This path " << path << " is a devdax dev " << dendl;
-    devdax_device = true;
-    // If using devdax char device, set it to not rotational device.
-    rotational = false;
-  }
-
-  r = _lock();
-  if (r < 0) {
-    derr << __func__ << " failed to lock " << path << ": " << cpp_strerror(r)
-	 << dendl;
-    goto out_fail;
-  }
-
-  struct stat st;
-  r = ::fstat(fd, &st);
-  if (r < 0) {
-    r = -errno;
-    derr << __func__ << " fstat got " << cpp_strerror(r) << dendl;
-    goto out_fail;
-  }
-
-  size_t map_len;
-  addr = (char *)pmem_map_file(path.c_str(), 0,
-                               devdax_device ? 0: PMEM_FILE_EXCL, O_RDWR,
-			       &map_len, NULL);
-  if (addr == NULL) {
-    derr << __func__ << " pmem_map_file failed: " << pmem_errormsg() << dendl;
-    goto out_fail;
-  }
-  size = map_len;
-
-  // Operate as though the block size is 4 KB.  The backing file
-  // blksize doesn't strictly matter except that some file systems may
-  // require a read/modify/write if we write something smaller than
-  // it.
-  block_size = g_conf()->bdev_block_size;
-  if (block_size != (unsigned)st.st_blksize) {
-    dout(1) << __func__ << " backing device/file reports st_blksize "
-      << st.st_blksize << ", using bdev_block_size "
-      << block_size << " anyway" << dendl;
-  }
-
-  dout(1) << __func__
-    << " size " << size
-    << " (" << byte_u_t(size) << ")"
-    << " block_size " << block_size
-    << " (" << byte_u_t(block_size) << ")"
-    << dendl;
-  return 0;
-
- out_fail:
-  VOID_TEMP_FAILURE_RETRY(::close(fd));
-  fd = -1;
-  return r;
-}
-
-void PMEMDevice::close()
-{
-  dout(1) << __func__ << dendl;
-
-  ceph_assert(addr != NULL);
-  if (devdax_device) {
-    devdax_device = false;
-  }
-  pmem_unmap(addr, size);
-
-  ceph_assert(fd >= 0);
-  VOID_TEMP_FAILURE_RETRY(::close(fd));
-  fd = -1;
-
-  path.clear();
-}
-
-int PMEMDevice::collect_metadata(const std::string& prefix, std::map<std::string,std::string> *pm) const
-{
-  (*pm)[prefix + "rotational"] = stringify((int)(bool)rotational);
-  (*pm)[prefix + "size"] = stringify(get_size());
-  (*pm)[prefix + "block_size"] = stringify(get_block_size());
-  (*pm)[prefix + "driver"] = "PMEMDevice";
-  (*pm)[prefix + "type"] = "ssd";
-
-  struct stat st;
-  int r = ::fstat(fd, &st);
-  if (r < 0)
-    return -errno;
-  if (S_ISBLK(st.st_mode)) {
-    (*pm)[prefix + "access_mode"] = "blk";
-    char buffer[1024] = {0};
-    BlkDev blkdev(fd);
-
-    blkdev.model(buffer, sizeof(buffer));
-    (*pm)[prefix + "model"] = buffer;
-
-    buffer[0] = '\0';
-    blkdev.dev(buffer, sizeof(buffer));
-    (*pm)[prefix + "dev"] = buffer;
-
-    // nvme exposes a serial number
-    buffer[0] = '\0';
-    blkdev.serial(buffer, sizeof(buffer));
-    (*pm)[prefix + "serial"] = buffer;
-
-  } else if (S_ISCHR(st.st_mode)) {
-    (*pm)[prefix + "access_mode"] = "chardevice";
-    (*pm)[prefix + "path"] = path;
-
-  } else {
-    (*pm)[prefix + "access_mode"] = "file";
-    (*pm)[prefix + "path"] = path;
-  }
-  return 0;
-}
-
-bool PMEMDevice::support(const std::string &path)
-{
-  int is_pmem = 0;
-  size_t map_len = 0;
-  int r = 0;
-  int local_fd;
-
-  local_fd = ::open(path.c_str(), O_RDWR);
-  if (local_fd < 0) {
-    return false;
-  }
-
-  r = pmem_check_file_type(local_fd, path.c_str(), NULL);
-  VOID_TEMP_FAILURE_RETRY(::close(local_fd));
-  int flags = PMEM_FILE_EXCL;
-  if (r == 0) {
-    flags = 0;
-  }
-
-  void *addr = pmem_map_file(path.c_str(), 0, flags, O_RDONLY, &map_len, &is_pmem);
-  if (addr != NULL) {
-    pmem_unmap(addr, map_len);
-    if (is_pmem) {
-      return true;
-    }
-  }
-
-  return false;
-}
-
-int PMEMDevice::flush()
-{
-  //Because all write is persist. So no need
-  return 0;
-}
-
-
-void PMEMDevice::aio_submit(IOContext *ioc)
-{
-  if (ioc->priv) {
-    ceph_assert(ioc->num_running == 0);
-    aio_callback(aio_callback_priv, ioc->priv);
-  } else {
-    ioc->try_aio_wake();
-  }
-  return;
-}
-
-int PMEMDevice::write(uint64_t off, bufferlist& bl, bool buffered, int write_hint)
-{
-  uint64_t len = bl.length();
-  dout(20) << __func__ << " " << off << "~" << len  << dendl;
-  ceph_assert(is_valid_io(off, len));
-
-  dout(40) << "data:\n";
-  bl.hexdump(*_dout);
-  *_dout << dendl;
-
-  if (g_conf()->bdev_inject_crash &&
-      rand() % g_conf()->bdev_inject_crash == 0) {
-    derr << __func__ << " bdev_inject_crash: dropping io " << off << "~" << len
-      << dendl;
-    ++injecting_crash;
-    return 0;
-  }
-
-  bufferlist::iterator p = bl.begin();
-  uint64_t off1 = off;
-  while (len) {
-    const char *data;
-    uint32_t l = p.get_ptr_and_advance(len, &data);
-
-#if defined(HAVE_LIBDML)
-    // Take care of the persistency issue
-    auto result = dml::execute<execution_path>(dml::mem_move, dml::make_view(data, l), dml::make_view(addr + off1, l));
-    ceph_assert(result.status == dml::status_code::ok);
-#else
-    pmem_memcpy_persist(addr + off1, data, l);
-#endif
-    len -= l;
-    off1 += l;
-  }
-  return 0;
-}
-
-int PMEMDevice::aio_write(
-  uint64_t off,
-  bufferlist &bl,
-  IOContext *ioc,
-  bool buffered,
-  int write_hint)
-{
-  return write(off, bl, buffered);
-}
-
-
-int PMEMDevice::read(uint64_t off, uint64_t len, bufferlist *pbl,
-		      IOContext *ioc,
-		      bool buffered)
-{
-  dout(5) << __func__ << " " << off << "~" << len  << dendl;
-  ceph_assert(is_valid_io(off, len));
-
-  bufferptr p = buffer::create_small_page_aligned(len);
-
-#if defined(HAVE_LIBDML)
-  auto result = dml::execute<execution_path>(dml::mem_move, dml::make_view(addr + off, len), dml::make_view(p.c_str(), len));
-  ceph_assert(result.status == dml::status_code::ok);
-#else
-  memcpy(p.c_str(), addr + off, len);
-#endif
-
-  pbl->clear();
-  pbl->push_back(std::move(p));
-
-  dout(40) << "data:\n";
-  pbl->hexdump(*_dout);
-  *_dout << dendl;
-
-  return 0;
-}
-
-int PMEMDevice::aio_read(uint64_t off, uint64_t len, bufferlist *pbl,
-		      IOContext *ioc)
-{
-  return read(off, len, pbl, ioc, false);
-}
-
-int PMEMDevice::read_random(uint64_t off, uint64_t len, char *buf, bool buffered)
-{
-  dout(5) << __func__ << " " << off << "~" << len << dendl;
-  ceph_assert(is_valid_io(off, len));
-
-
-#if defined(HAVE_LIBDML)
-  auto result = dml::execute<execution_path>(dml::mem_move, dml::make_view(addr + off, len), dml::make_view(buf, len));
-  ceph_assert(result.status == dml::status_code::ok);
-#else
-  memcpy(buf, addr + off, len);
-#endif
-  return 0;
-}
-
-
-int PMEMDevice::invalidate_cache(uint64_t off, uint64_t len)
-{
-  dout(5) << __func__ << " " << off << "~" << len << dendl;
-  return 0;
-}
-
-

src/blk/pmem/PMEMDevice.h

@@ -1,79 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
-// vim: ts=8 sw=2 sts=2 expandtab
-
-/*
- * Ceph - scalable distributed file system
- *
- *  Copyright (C) 2015 Intel <jianpeng.ma@intel.com>
- *
- * Author: Jianpeng Ma <jianpeng.ma@intel.com>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-#ifndef CEPH_BLK_PMEMDEVICE_H
-#define CEPH_BLK_PMEMDEVICE_H
-
-#include <atomic>
-#include <map>
-#include <string>
-
-#include "os/fs/FS.h"
-#include "include/interval_set.h"
-#include "aio/aio.h"
-#include "BlockDevice.h"
-
-class PMEMDevice : public BlockDevice {
-  int fd;
-  char *addr; //the address of mmap
-  std::string path;
-  bool devdax_device = false;
-
-  ceph::mutex debug_lock = ceph::make_mutex("PMEMDevice::debug_lock");
-  interval_set<uint64_t> debug_inflight;
-
-  std::atomic_int injecting_crash;
-  int _lock();
-
-public:
-  PMEMDevice(CephContext *cct, aio_callback_t cb, void *cbpriv);
-
-  bool supported_bdev_label() override { return !devdax_device; }
-  void aio_submit(IOContext *ioc) override;
-
-  int collect_metadata(const std::string& prefix, std::map<std::string,std::string> *pm) const override;
-
-  static bool support(const std::string& path);
-
-  int read(uint64_t off, uint64_t len, bufferlist *pbl,
-	   IOContext *ioc,
-	   bool buffered) override;
-  int aio_read(uint64_t off, uint64_t len, bufferlist *pbl,
-	       IOContext *ioc) override;
-
-  int read_random(uint64_t off, uint64_t len, char *buf, bool buffered) override;
-  int write(uint64_t off, bufferlist& bl, bool buffered, int write_hint = WRITE_LIFE_NOT_SET) override;
-  int aio_write(uint64_t off, bufferlist& bl,
-		IOContext *ioc,
-		bool buffered,
-		int write_hint = WRITE_LIFE_NOT_SET) override;
-  int flush() override;
-
-  // for managing buffered readers/writers
-  int invalidate_cache(uint64_t off, uint64_t len) override;
-  int open(const std::string &path) override;
-  void close() override;
-
-private:
-  bool is_valid_io(uint64_t off, uint64_t len) const {
-    return (len > 0 &&
-            off < size &&
-            off + len <= size);
-  }
-};
-
-#endif

src/common/options/global.yaml.in

@@ -6992,7 +6992,6 @@ options:
   enum_values:
   - aio
   - spdk
-  - pmem
 - name: bdev_stalled_read_warn_lifetime
   type: uint
   level: advanced

src/include/config-h.in.cmake

@@ -84,9 +84,6 @@
 /* Defined if you have libaio */
 #cmakedefine HAVE_LIBAIO
 
-/* Defined if you have libdml */
-#cmakedefine HAVE_LIBDML
-
 /* Defined if you have liburing */
 #cmakedefine HAVE_LIBURING
 
@@ -114,9 +111,6 @@
 /* DPDK conditional compilation */
 #cmakedefine HAVE_DPDK
 
-/* PMEM_DEVICE (OSD) conditional compilation */
-#cmakedefine HAVE_BLUESTORE_PMEM
-
 /* Define if you have tcmalloc */
 #cmakedefine HAVE_LIBTCMALLOC
 #cmakedefine LIBTCMALLOC_MISSING_ALIGNED_ALLOC

src/vstart.sh

@@ -187,7 +187,6 @@ rgw_compression=""
 rgw_store="rados"
 lockdep=${LOCKDEP:-1}
 spdk_enabled=0 # disable SPDK by default
-pmem_enabled=0
 io_uring_enabled=0
 with_jaeger=0
 force_addr=0
@@ -259,7 +258,6 @@ options:
 	--multimds <count> allow multimds with maximum active count
 	--without-dashboard: do not run using mgr dashboard
 	--bluestore-spdk: enable SPDK and with a comma-delimited list of PCI-IDs of NVME device (e.g, 0000:81:00.0)
-	--bluestore-pmem: enable PMEM and with path to a file mapped to PMEM
 	--msgr1: use msgr1 only
 	--msgr2: use msgr2 only
 	--msgr21: use msgr2 and msgr1
@@ -657,12 +655,6 @@ case $1 in
         spdk_enabled=1
         shift
         ;;
-    --bluestore-pmem)
-        [ -z "$2" ] && usage_exit
-        bluestore_pmem_file="$2"
-        pmem_enabled=1
-        shift
-        ;;
     --bluestore-devs)
         parse_block_devs --bluestore-devs "$2"
         shift
@@ -911,7 +903,7 @@ EOF
         osd max object namespace len = 64"
     fi
     if [ "$objectstore" == "bluestore" ]; then
-        if [ "$spdk_enabled" -eq 1 ] || [ "$pmem_enabled" -eq 1 ]; then
+        if [ "$spdk_enabled" -eq 1 ]; then
             BLUESTORE_OPTS="        bluestore_block_db_path = \"\"
         bluestore_block_db_size = 0
         bluestore_block_db_create = false
@@ -1290,10 +1282,6 @@ EOF
             if [ "$spdk_enabled" -eq 1 ]; then
                 wconf <<EOF
         bluestore_block_path = spdk:${bluestore_spdk_dev[$osd]}
-EOF
-            elif [ "$pmem_enabled" -eq 1 ]; then
-                wconf <<EOF
-        bluestore_block_path = ${bluestore_pmem_file}
 EOF
             fi
             rm -rf $CEPH_DEV_DIR/osd$osd || true