rbd: allow mounting images on Windows

This change will allow mapping rbd images on Windows, leveraging the
WNBD[1] Virtual Storport Miniport driver [2].

The behavior and CLI is similar to the Linux rbd-nbd, with a few
notable differences:

* device paths cannot be requested. The disk number and path will
  be picked by Windows. If a device path is provided by the user
  when mapping an image, it will be used as an identifier, which
  can also be used when unmapping the image.
* the "show" command was added, which describes a specific mapping.
  This can be used for retrieving the disk path.
* the "service" command was added, allowing rbd-wnbd to run as a
  Windows service. All mappings are currently perisistent, being
  recreated when the service stops, unless explicitly unmapped.
  The service disconnects the mappings when being stopped.
* the "list" command also includes a "status" column.

The purpose of the "service" mode is to ensure that mappings survive
reboots and that the Windows service start order can be adjusted so
that rbd images can be mapped before starting services that may depend
on it, such as VMMS.

The mapped images can either be consumed by the host directly or exposed
to Hyper-V VMs.

While at it, we'll skip building rbd-mirror as it's quite unlikely that
this daemon is going to be used on Windows for now.

[1] https://github.com/cloudbase/wnbd
[2] https://docs.microsoft.com/en-us/windows-hardware/drivers/storage/overview-of-storage-virtual-miniport-drivers

Signed-off-by: Lucian Petrut <lpetrut@cloudbasesolutions.com>
Signed-off-by: Alin Gabriel Serdean <aserdean@cloudbasesolutions.com>

src/common/win32/service.cc

@@ -61,18 +61,22 @@ void WINAPI ServiceBase::run()
   s_service->hstatus = RegisterServiceCtrlHandler(
     "", (LPHANDLER_FUNCTION)control_handler);
   if (!s_service->hstatus) {
+    lderr(s_service->cct) << "Could not initialize service control handler. "
+                          << "Error: " << GetLastError() << dendl;
     return;
   }
 
   s_service->set_status(SERVICE_START_PENDING);
 
   // TODO: should we expect exceptions?
+  ldout(s_service->cct, 5) << "Starting service." << dendl;
   int err = s_service->run_hook();
   if (err) {
     lderr(s_service->cct) << "Failed to start service. Error code: "
                           << err << dendl;
     s_service->set_status(SERVICE_STOPPED);
   } else {
+    ldout(s_service->cct, 5) << "Successfully started service." << dendl;
     s_service->set_status(SERVICE_RUNNING);
   }
 }
@@ -87,6 +91,7 @@ void ServiceBase::shutdown()
     derr << "Shutdown service hook failed. Error code: " << err << dendl;
     set_status(original_state);
   } else {
+    dout(5) << "Shutdown hook completed." << dendl;
     set_status(SERVICE_STOPPED);
   }
 }
@@ -101,6 +106,7 @@ void ServiceBase::stop()
     derr << "Service stop hook failed. Error code: " << err << dendl;
     set_status(original_state);
   } else {
+    dout(5) << "Successfully stopped service." << dendl;
     set_status(SERVICE_STOPPED);
   }
 }
@@ -132,6 +138,12 @@ void ServiceBase::set_status(DWORD current_state, DWORD exit_code) {
   status.dwWin32ExitCode = exit_code;
 
   if (hstatus) {
+    dout(5) << "Updating service service status (" << current_state
+             << ") and exit code(" << exit_code << ")." << dendl;
     ::SetServiceStatus(hstatus, &status);
+  } else {
+    derr << "Service control handler not initialized. Cannot "
+         << "update service status (" << current_state
+         << ") and exit code(" << exit_code << ")." << dendl;
   }
 }

src/tools/CMakeLists.txt

@@ -137,6 +137,9 @@ if(WITH_RBD)
   if(LINUX)
     add_subdirectory(rbd_nbd)
   endif()
+  if(WIN32)
+    add_subdirectory(rbd_wnbd)
+  endif()
   if(FREEBSD)
     add_subdirectory(rbd_ggate)
   endif()

src/tools/rbd_wnbd/CMakeLists.txt

@@ -0,0 +1,10 @@
+add_executable(rbd-wnbd wnbd_handler.cc rbd_wnbd.cc)
+set_target_properties(
+    rbd-wnbd PROPERTIES COMPILE_FLAGS
+    "-fpermissive -I${WNBD_INCLUDE_DIRS}")
+target_link_libraries(
+    rbd-wnbd setupapi rpcrt4
+    ${WNBD_LIBRARIES}
+    ${Boost_FILESYSTEM_LIBRARY}
+    librbd librados global)
+install(TARGETS rbd-wnbd DESTINATION bin)

src/tools/rbd_wnbd/rbd_wnbd.h

@@ -0,0 +1,169 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2020 SUSE LINUX GmbH
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef RBD_WNBD_H
+#define RBD_WNBD_H
+
+#include <string.h>
+#include <iostream>
+#include <vector>
+
+#include "include/compat.h"
+#include "common/win32/registry.h"
+
+#include "wnbd_handler.h"
+
+#define SERVICE_REG_KEY "SYSTEM\\CurrentControlSet\\Services\\rbd-wnbd"
+
+#define RBD_WNBD_BLKSIZE 512UL
+
+#define HELP_INFO 1
+#define VERSION_INFO 2
+
+#define WNBD_STATUS_ACTIVE "active"
+#define WNBD_STATUS_INACTIVE "inactive"
+
+#define DEFAULT_SERVICE_THREAD_COUNT 8
+
+static WnbdHandler* handler = nullptr;
+ceph::mutex shutdown_lock = ceph::make_mutex("RbdWnbd::ShutdownLock");
+
+struct Config {
+  bool exclusive = false;
+  bool readonly = false;
+
+  intptr_t parent_pipe = 0;
+
+  std::string poolname;
+  std::string nsname;
+  std::string imgname;
+  std::string snapname;
+  std::string devpath;
+
+  std::string format;
+  bool pretty_format = false;
+
+  bool hard_disconnect = false;
+  int soft_disconnect_timeout = DEFAULT_SOFT_REMOVE_TIMEOUT;
+  bool hard_disconnect_fallback = true;
+
+  // TODO: consider moving those fields to a separate structure. Those
+  // provide connection information without actually being configurable.
+  // The disk number is provided by Windows.
+  int disk_number = -1;
+  int pid = 0;
+  std::string serial_number;
+  bool active = false;
+  bool wnbd_mapped = false;
+  std::string command_line;
+  std::string admin_sock_path;
+
+  WnbdLogLevel wnbd_log_level = WnbdLogLevelInfo;
+  int io_req_workers = DEFAULT_IO_WORKER_COUNT;
+  int io_reply_workers = DEFAULT_IO_WORKER_COUNT;
+  int service_thread_count = DEFAULT_SERVICE_THREAD_COUNT;
+};
+
+enum Command {
+  None,
+  Connect,
+  Disconnect,
+  List,
+  Show,
+  Service,
+  Stats
+};
+
+bool is_process_running(DWORD pid);
+
+void daemonize_complete(HANDLE parent_pipe);
+void unmap_at_exit();
+
+int disconnect_all_mappings(
+  bool unregister,
+  bool hard_disconnect,
+  int soft_disconnect_timeout,
+  int worker_count);
+int restart_registered_mappings(int worker_count);
+int map_device_using_suprocess(std::string command_line);
+
+int construct_devpath_if_missing(Config* cfg);
+int save_config_to_registry(Config* cfg);
+int remove_config_from_registry(Config* cfg);
+int load_mapping_config_from_registry(std::string devpath, Config* cfg);
+
+BOOL WINAPI console_handler_routine(DWORD dwCtrlType);
+
+static int parse_args(std::vector<const char*>& args,
+                      std::ostream *err_msg,
+                      Command *command, Config *cfg);
+static int do_unmap(Config *cfg, bool unregister);
+
+
+class BaseIterator {
+  public:
+    virtual ~BaseIterator() {};
+    virtual bool get(Config *cfg) = 0;
+
+    int get_error() {
+      return error;
+    }
+  protected:
+    int error = 0;
+    int index = -1;
+};
+
+// Iterate over mapped devices, retrieving info from the driver.
+class WNBDActiveDiskIterator : public BaseIterator {
+  public:
+    WNBDActiveDiskIterator();
+    ~WNBDActiveDiskIterator();
+
+    bool get(Config *cfg);
+
+  private:
+    PWNBD_CONNECTION_LIST conn_list = NULL;
+
+    static DWORD fetch_list(PWNBD_CONNECTION_LIST* conn_list);
+};
+
+
+// Iterate over the Windows registry key, retrieving registered mappings.
+class RegistryDiskIterator : public BaseIterator {
+  public:
+    RegistryDiskIterator();
+    ~RegistryDiskIterator() {
+      delete reg_key;
+    }
+
+    bool get(Config *cfg);
+  private:
+    DWORD subkey_count = 0;
+    char subkey_name[MAX_PATH];
+
+  RegistryKey* reg_key = NULL;
+};
+
+// Iterate over all RBD mappings, getting info from the registry and driver.
+class WNBDDiskIterator : public BaseIterator {
+  public:
+    bool get(Config *cfg);
+
+  private:
+    // We'll keep track of the active devices.
+    std::set<std::string> active_devices;
+
+    WNBDActiveDiskIterator active_iterator;
+    RegistryDiskIterator registry_iterator;
+};
+
+#endif // RBD_WNBD_H

src/tools/rbd_wnbd/wnbd_handler.cc

@@ -0,0 +1,430 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2020 SUSE LINUX GmbH
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd
+
+#include "wnbd_handler.h"
+
+#define _NTSCSI_USER_MODE_
+#include <rpc.h>
+#include <ddk/scsi.h>
+
+#include <boost/thread/tss.hpp>
+
+#include "common/debug.h"
+#include "common/errno.h"
+#include "common/safe_io.h"
+#include "common/SubProcess.h"
+#include "common/Formatter.h"
+
+#include "global/global_context.h"
+
+WnbdHandler::~WnbdHandler()
+{
+  if (started && wnbd_disk) {
+    dout(10) << __func__ << ": terminating" << dendl;
+
+    shutdown();
+    reply_tpool->join();
+
+    WnbdClose(wnbd_disk);
+
+    started = false;
+
+    delete reply_tpool;
+    delete admin_hook;
+  }
+}
+
+int WnbdHandler::wait()
+{
+  int err = 0;
+  if (started && wnbd_disk) {
+    dout(10) << __func__ << ": waiting" << dendl;
+
+    err = WnbdWaitDispatcher(wnbd_disk);
+    if (err) {
+      derr << __func__ << " failed waiting for dispatcher to stop: "
+           << err << dendl;
+    }
+  }
+
+  return err;
+}
+
+int WnbdAdminHook::call (std::string_view command, const cmdmap_t& cmdmap,
+     Formatter *f,
+     std::ostream& errss,
+     bufferlist& out) {
+    if (command == "wnbd stats") {
+      return m_handler->dump_stats(f);
+    }
+    return -ENOSYS;
+  }
+
+int WnbdHandler::dump_stats(Formatter *f)
+{
+  if (!f) {
+    return -EINVAL;
+  }
+
+  WNBD_USR_STATS stats = { 0 };
+  DWORD err = WnbdGetUserspaceStats(wnbd_disk, &stats);
+  if (err) {
+    derr << "Failed to retrieve WNBD userspace stats. Error: " << err << dendl;
+    return -EINVAL;
+  }
+
+  f->open_object_section("stats");
+  f->dump_int("TotalReceivedRequests", stats.TotalReceivedRequests);
+  f->dump_int("TotalSubmittedRequests", stats.TotalSubmittedRequests);
+  f->dump_int("TotalReceivedReplies", stats.TotalReceivedReplies);
+  f->dump_int("UnsubmittedRequests", stats.UnsubmittedRequests);
+  f->dump_int("PendingSubmittedRequests", stats.PendingSubmittedRequests);
+  f->dump_int("PendingReplies", stats.PendingReplies);
+  f->dump_int("ReadErrors", stats.ReadErrors);
+  f->dump_int("WriteErrors", stats.WriteErrors);
+  f->dump_int("FlushErrors", stats.FlushErrors);
+  f->dump_int("UnmapErrors", stats.UnmapErrors);
+  f->dump_int("InvalidRequests", stats.InvalidRequests);
+  f->dump_int("TotalRWRequests", stats.TotalRWRequests);
+  f->dump_int("TotalReadBlocks", stats.TotalReadBlocks);
+  f->dump_int("TotalWrittenBlocks", stats.TotalWrittenBlocks);
+
+  f->close_section();
+  return 0;
+}
+
+void WnbdHandler::shutdown()
+{
+  std::unique_lock l{shutdown_lock};
+  if (!terminated && wnbd_disk) {
+    // We're requesting the disk to be removed but continue serving IO
+    // requests until the driver sends us the "Disconnect" event.
+    // TODO: expose PWNBD_REMOVE_OPTIONS, we're using the defaults ATM.
+    WnbdRemove(wnbd_disk, NULL);
+    wait();
+    terminated = true;
+  }
+}
+
+void WnbdHandler::aio_callback(librbd::completion_t cb, void *arg)
+{
+  librbd::RBD::AioCompletion *aio_completion =
+    reinterpret_cast<librbd::RBD::AioCompletion*>(cb);
+
+  WnbdHandler::IOContext* ctx = static_cast<WnbdHandler::IOContext*>(arg);
+  int ret = aio_completion->get_return_value();
+
+  dout(20) << __func__ << ": " << *ctx << dendl;
+
+  if (ret == -EINVAL) {
+    // if shrinking an image, a pagecache writeback might reference
+    // extents outside of the range of the new image extents
+    dout(0) << __func__ << ": masking IO out-of-bounds error" << *ctx << dendl;
+    ctx->data.clear();
+    ret = 0;
+  }
+
+  if (ret < 0) {
+    ctx->err_code = -ret;
+    // TODO: check the actual error.
+    ctx->set_sense(SCSI_SENSE_MEDIUM_ERROR,
+                   SCSI_ADSENSE_UNRECOVERED_ERROR);
+  } else if ((ctx->req_type == WnbdReqTypeRead) &&
+              ret < static_cast<int>(ctx->req_size)) {
+    int pad_byte_count = static_cast<int> (ctx->req_size) - ret;
+    ctx->data.append_zero(pad_byte_count);
+    dout(20) << __func__ << ": " << *ctx << ": Pad byte count: "
+             << pad_byte_count << dendl;
+    ctx->err_code = 0;
+  } else {
+    ctx->err_code = 0;
+  }
+
+  boost::asio::post(
+    *ctx->handler->reply_tpool,
+    [&, ctx]()
+    {
+      ctx->handler->send_io_response(ctx);
+    });
+
+  aio_completion->release();
+}
+
+void WnbdHandler::send_io_response(WnbdHandler::IOContext *ctx) {
+  std::unique_ptr<WnbdHandler::IOContext> pctx{ctx};
+  ceph_assert(WNBD_DEFAULT_MAX_TRANSFER_LENGTH >= pctx->data.length());
+
+  WNBD_IO_RESPONSE wnbd_rsp = {0};
+  wnbd_rsp.RequestHandle = pctx->req_handle;
+  wnbd_rsp.RequestType = pctx->req_type;
+  wnbd_rsp.Status = pctx->wnbd_status;
+  int err = 0;
+
+  // Use TLS to store an overlapped structure so that we avoid
+  // recreating one each time we send a reply.
+  static boost::thread_specific_ptr<OVERLAPPED> overlapped_tls(
+    // Cleanup routine
+    [](LPOVERLAPPED p_overlapped)
+    {
+      if (p_overlapped->hEvent) {
+        CloseHandle(p_overlapped->hEvent);
+      }
+      delete p_overlapped;
+    });
+
+  LPOVERLAPPED overlapped = overlapped_tls.get();
+  if (!overlapped)
+  {
+    overlapped = new OVERLAPPED{0};
+    HANDLE overlapped_evt = CreateEventA(0, TRUE, TRUE, NULL);
+    if (!overlapped_evt) {
+      err = GetLastError();
+      derr << "Could not create event. Error: " << err << dendl;
+      return;
+    }
+
+    overlapped->hEvent = overlapped_evt;
+    overlapped_tls.reset(overlapped);
+  }
+
+  if (!ResetEvent(overlapped->hEvent)) {
+    err = GetLastError();
+    derr << "Could not reset event. Error: " << err << dendl;
+    return;
+  }
+
+  err = WnbdSendResponseEx(
+    pctx->handler->wnbd_disk,
+    &wnbd_rsp,
+    pctx->data.c_str(),
+    pctx->data.length(),
+    overlapped);
+  if (err == ERROR_IO_PENDING) {
+    DWORD returned_bytes = 0;
+    err = 0;
+    // We've got ERROR_IO_PENDING, which means that the operation is in
+    // progress. We'll use GetOverlappedResult to wait for it to complete
+    // and then retrieve the result.
+    if (!GetOverlappedResult(pctx->handler->wnbd_disk, overlapped,
+                             &returned_bytes, TRUE)) {
+      err = GetLastError();
+      derr << "Could not send response. Request id: " << wnbd_rsp.RequestHandle
+           << ". Error: " << err << dendl;
+    }
+  }
+}
+
+void WnbdHandler::IOContext::set_sense(uint8_t sense_key, uint8_t asc, uint64_t info)
+{
+  WnbdSetSenseEx(&wnbd_status, sense_key, asc, info);
+}
+
+void WnbdHandler::IOContext::set_sense(uint8_t sense_key, uint8_t asc)
+{
+  WnbdSetSense(&wnbd_status, sense_key, asc);
+}
+
+void WnbdHandler::Read(
+  PWNBD_DISK Disk,
+  UINT64 RequestHandle,
+  PVOID Buffer,
+  UINT64 BlockAddress,
+  UINT32 BlockCount,
+  BOOLEAN ForceUnitAccess)
+{
+  WnbdHandler* handler = nullptr;
+  ceph_assert(!WnbdGetUserContext(Disk, (PVOID*)&handler));
+
+  WnbdHandler::IOContext* ctx = new WnbdHandler::IOContext();
+  ctx->handler = handler;
+  ctx->req_handle = RequestHandle;
+  ctx->req_type = WnbdReqTypeRead;
+  ctx->req_size = BlockCount * handler->block_size;
+  ctx->req_from = BlockAddress * handler->block_size;
+  ceph_assert(ctx->req_size <= WNBD_DEFAULT_MAX_TRANSFER_LENGTH);
+
+  int op_flags = 0;
+  if (ForceUnitAccess) {
+    op_flags |= LIBRADOS_OP_FLAG_FADVISE_FUA;
+  }
+
+  dout(20) << *ctx << ": start" << dendl;
+
+  librbd::RBD::AioCompletion *c = new librbd::RBD::AioCompletion(ctx, aio_callback);
+  handler->image.aio_read2(ctx->req_from, ctx->req_size, ctx->data, c, op_flags);
+
+  dout(20) << *ctx << ": submitted" << dendl;
+}
+
+void WnbdHandler::Write(
+  PWNBD_DISK Disk,
+  UINT64 RequestHandle,
+  PVOID Buffer,
+  UINT64 BlockAddress,
+  UINT32 BlockCount,
+  BOOLEAN ForceUnitAccess)
+{
+  WnbdHandler* handler = nullptr;
+  ceph_assert(!WnbdGetUserContext(Disk, (PVOID*)&handler));
+
+  WnbdHandler::IOContext* ctx = new WnbdHandler::IOContext();
+  ctx->handler = handler;
+  ctx->req_handle = RequestHandle;
+  ctx->req_type = WnbdReqTypeWrite;
+  ctx->req_size = BlockCount * handler->block_size;
+  ctx->req_from = BlockAddress * handler->block_size;
+
+  bufferptr ptr((char*)Buffer, ctx->req_size);
+  ctx->data.push_back(ptr);
+
+  int op_flags = 0;
+  if (ForceUnitAccess) {
+    op_flags |= LIBRADOS_OP_FLAG_FADVISE_FUA;
+  }
+
+  dout(20) << *ctx << ": start" << dendl;
+
+  librbd::RBD::AioCompletion *c = new librbd::RBD::AioCompletion(ctx, aio_callback);
+  handler->image.aio_write2(ctx->req_from, ctx->req_size, ctx->data, c, op_flags);
+
+  dout(20) << *ctx << ": submitted" << dendl;
+}
+
+void WnbdHandler::Flush(
+  PWNBD_DISK Disk,
+  UINT64 RequestHandle,
+  UINT64 BlockAddress,
+  UINT32 BlockCount)
+{
+  WnbdHandler* handler = nullptr;
+  ceph_assert(!WnbdGetUserContext(Disk, (PVOID*)&handler));
+
+  WnbdHandler::IOContext* ctx = new WnbdHandler::IOContext();
+  ctx->handler = handler;
+  ctx->req_handle = RequestHandle;
+  ctx->req_type = WnbdReqTypeFlush;
+  ctx->req_size = BlockCount * handler->block_size;
+  ctx->req_from = BlockAddress * handler->block_size;
+
+  dout(20) << *ctx << ": start" << dendl;
+
+  librbd::RBD::AioCompletion *c = new librbd::RBD::AioCompletion(ctx, aio_callback);
+  handler->image.aio_flush(c);
+
+  dout(20) << *ctx << ": submitted" << dendl;
+}
+
+void WnbdHandler::Unmap(
+  PWNBD_DISK Disk,
+  UINT64 RequestHandle,
+  PWNBD_UNMAP_DESCRIPTOR Descriptors,
+  UINT32 Count)
+{
+  WnbdHandler* handler = nullptr;
+  ceph_assert(!WnbdGetUserContext(Disk, (PVOID*)&handler));
+  ceph_assert(1 == Count);
+
+  WnbdHandler::IOContext* ctx = new WnbdHandler::IOContext();
+  ctx->handler = handler;
+  ctx->req_handle = RequestHandle;
+  ctx->req_type = WnbdReqTypeUnmap;
+  ctx->req_size = Descriptors[0].BlockCount * handler->block_size;
+  ctx->req_from = Descriptors[0].BlockAddress * handler->block_size;
+
+  dout(20) << *ctx << ": start" << dendl;
+
+  librbd::RBD::AioCompletion *c = new librbd::RBD::AioCompletion(ctx, aio_callback);
+  handler->image.aio_discard(ctx->req_from, ctx->req_size, c);
+
+  dout(20) << *ctx << ": submitted" << dendl;
+}
+
+void WnbdHandler::LogMessage(
+  WnbdLogLevel LogLevel,
+  const char* Message,
+  const char* FileName,
+  UINT32 Line,
+  const char* FunctionName)
+{
+  // We're already passing the log level to WNBD, so we'll use the highest
+  // log level here.
+  dout(0) << "libwnbd.dll!" << FunctionName << " "
+          << WnbdLogLevelToStr(LogLevel) << " " << Message << dendl;
+}
+
+
+int WnbdHandler::start()
+{
+  int err = 0;
+  WNBD_PROPERTIES wnbd_props = {0};
+
+  instance_name.copy(wnbd_props.InstanceName, sizeof(wnbd_props.InstanceName));
+  ceph_assert(strlen(RBD_WNBD_OWNER_NAME) < WNBD_MAX_OWNER_LENGTH);
+  strncpy(wnbd_props.Owner, RBD_WNBD_OWNER_NAME, WNBD_MAX_OWNER_LENGTH);
+
+  wnbd_props.BlockCount = block_count;
+  wnbd_props.BlockSize = block_size;
+  wnbd_props.MaxUnmapDescCount = 1;
+
+  wnbd_props.Flags.ReadOnly = readonly;
+  wnbd_props.Flags.UnmapSupported = 1;
+  if (rbd_cache_enabled) {
+    wnbd_props.Flags.FUASupported = 1;
+    wnbd_props.Flags.FlushSupported = 1;
+  }
+
+  err = WnbdCreate(&wnbd_props, &RbdWnbdInterface, this, &wnbd_disk);
+  if (err)
+    goto exit;
+
+  started = true;
+
+  err = WnbdStartDispatcher(wnbd_disk, io_req_workers);
+  if (err) {
+      derr << "Could not start WNBD dispatcher. Error: " << err << dendl;
+  }
+
+exit:
+  return err;
+}
+
+std::ostream &operator<<(std::ostream &os, const WnbdHandler::IOContext &ctx) {
+
+  os << "[" << std::hex << ctx.req_handle;
+
+  switch (ctx.req_type)
+  {
+  case WnbdReqTypeRead:
+    os << " READ ";
+    break;
+  case WnbdReqTypeWrite:
+    os << " WRITE ";
+    break;
+  case WnbdReqTypeFlush:
+    os << " FLUSH ";
+    break;
+  case WnbdReqTypeUnmap:
+    os << " TRIM ";
+    break;
+  default:
+    os << " UNKNOWN(" << ctx.req_type << ") ";
+    break;
+  }
+
+  os << ctx.req_from << "~" << ctx.req_size << " "
+     << std::dec << ntohl(ctx.err_code) << "]";
+
+  return os;
+}

src/tools/rbd_wnbd/wnbd_handler.h

@@ -0,0 +1,186 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2020 SUSE LINUX GmbH
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef WNBD_HANDLER_H
+#define WNBD_HANDLER_H
+
+#include <wnbd.h>
+
+#include "common/admin_socket.h"
+#include "common/ceph_context.h"
+#include "common/Thread.h"
+
+#include "include/rbd/librbd.hpp"
+#include "include/xlist.h"
+
+#include "global/global_context.h"
+
+// TODO: make this configurable.
+#define RBD_WNBD_MAX_TRANSFER 2 * 1024 * 1024
+#define SOFT_REMOVE_RETRY_INTERVAL 2
+#define DEFAULT_SOFT_REMOVE_TIMEOUT 15
+#define DEFAULT_IO_WORKER_COUNT 4
+
+// Not defined by mingw.
+#ifndef SCSI_ADSENSE_UNRECOVERED_ERROR
+#define SCSI_ADSENSE_UNRECOVERED_ERROR 0x11
+#endif
+
+// The following will be assigned to the "Owner" field of the WNBD
+// parameters, which can be used to determine the application managing
+// a disk. We'll ignore other disks.
+#define RBD_WNBD_OWNER_NAME "ceph-rbd-wnbd"
+
+class WnbdHandler;
+
+class WnbdAdminHook : public AdminSocketHook {
+  WnbdHandler *m_handler;
+
+public:
+  explicit WnbdAdminHook(WnbdHandler *handler) :
+        m_handler(handler) {
+    g_ceph_context->get_admin_socket()->register_command(
+      "wnbd stats", this, "get WNBD stats");
+  }
+  ~WnbdAdminHook() override {
+    g_ceph_context->get_admin_socket()->unregister_commands(this);
+  }
+
+  int call(std::string_view command, const cmdmap_t& cmdmap,
+     Formatter *f, std::ostream& errss, bufferlist& out) override;
+};
+
+
+class WnbdHandler
+{
+private:
+  librbd::Image &image;
+  std::string instance_name;
+  uint32_t block_count;
+  uint32_t block_size;
+  bool readonly;
+  bool rbd_cache_enabled;
+  uint32_t io_req_workers;
+  uint32_t io_reply_workers;
+  WnbdAdminHook* admin_hook;
+  boost::asio::thread_pool* reply_tpool;
+
+public:
+  WnbdHandler(librbd::Image& _image, std::string _instance_name,
+              uint32_t _block_count, uint32_t _block_size,
+              bool _readonly, bool _rbd_cache_enabled,
+              uint32_t _io_req_workers,
+              uint32_t _io_reply_workers)
+    : image(_image)
+    , instance_name(_instance_name)
+    , block_count(_block_count)
+    , block_size(_block_size)
+    , readonly(_readonly)
+    , rbd_cache_enabled(_rbd_cache_enabled)
+    , io_req_workers(_io_req_workers)
+    , io_reply_workers(_io_reply_workers)
+  {
+    admin_hook = new WnbdAdminHook(this);
+    // Instead of relying on librbd's own thread pool, we're going to use a
+    // separate one. This allows us to make assumptions on the threads that
+    // are going to send the IO replies and thus be able to cache Windows
+    // OVERLAPPED structures.
+    reply_tpool = new boost::asio::thread_pool(_io_reply_workers);
+  }
+
+  int start();
+  // Wait for the handler to stop, which normally happens when the driver
+  // passes the "Disconnect" request.
+  int wait();
+  void shutdown();
+
+  int dump_stats(Formatter *f);
+
+  ~WnbdHandler();
+
+  static VOID LogMessage(
+    WnbdLogLevel LogLevel,
+    const char* Message,
+    const char* FileName,
+    UINT32 Line,
+    const char* FunctionName);
+
+private:
+  ceph::mutex shutdown_lock = ceph::make_mutex("WnbdHandler::DisconnectLocker");
+  bool started = false;
+  bool terminated = false;
+  WNBD_DISK* wnbd_disk = nullptr;
+
+  struct IOContext
+  {
+    xlist<IOContext*>::item item;
+    WnbdHandler *handler = nullptr;
+    WNBD_STATUS wnbd_status = {0};
+    WnbdRequestType req_type = WnbdReqTypeUnknown;
+    uint64_t req_handle = 0;
+    uint32_t err_code = 0;
+    uint32_t req_size;
+    uint32_t req_from;
+    bufferlist data;
+
+    IOContext()
+      : item(this)
+    {}
+
+    void set_sense(uint8_t sense_key, uint8_t asc, uint64_t info);
+    void set_sense(uint8_t sense_key, uint8_t asc);
+  };
+
+  friend std::ostream &operator<<(std::ostream &os, const IOContext &ctx);
+
+  void send_io_response(IOContext *ctx);
+
+  static void aio_callback(librbd::completion_t cb, void *arg);
+
+  // WNBD IO entry points
+  static void Read(
+    PWNBD_DISK Disk,
+    UINT64 RequestHandle,
+    PVOID Buffer,
+    UINT64 BlockAddress,
+    UINT32 BlockCount,
+    BOOLEAN ForceUnitAccess);
+  static void Write(
+    PWNBD_DISK Disk,
+    UINT64 RequestHandle,
+    PVOID Buffer,
+    UINT64 BlockAddress,
+    UINT32 BlockCount,
+    BOOLEAN ForceUnitAccess);
+  static void Flush(
+    PWNBD_DISK Disk,
+    UINT64 RequestHandle,
+    UINT64 BlockAddress,
+    UINT32 BlockCount);
+  static void Unmap(
+    PWNBD_DISK Disk,
+    UINT64 RequestHandle,
+    PWNBD_UNMAP_DESCRIPTOR Descriptors,
+    UINT32 Count);
+
+  static constexpr WNBD_INTERFACE RbdWnbdInterface =
+  {
+    Read,
+    Write,
+    Flush,
+    Unmap,
+  };
+};
+
+std::ostream &operator<<(std::ostream &os, const WnbdHandler::IOContext &ctx);
+
+#endif // WNBD_HANDLER_H

win32_build.sh

@@ -36,8 +36,8 @@ backtraceDir="${depsToolsetDir}/backtrace"
 snappyDir="${depsToolsetDir}/snappy"
 winLibDir="${depsToolsetDir}/windows/lib"
 generatorUsed="Unix Makefiles"
-
-pyVersion=`python -c "import sys; print('%d.%d' % (sys.version_info.major, sys.version_info.minor))"`
+wnbdSrcDir="${depsSrcDir}/wnbd"
+wnbdLibDir="${depsToolsetDir}/wnbd/lib"
 
 depsDirs="$lz4Dir;$curlDir;$sslDir;$boostDir;$zlibDir;$backtraceDir;$snappyDir"
 depsDirs+=";$winLibDir"
@@ -83,7 +83,6 @@ fi
 # or circular), we'll have to stick to static linking.
 cmake -D CMAKE_PREFIX_PATH=$depsDirs \
       -D CMAKE_TOOLCHAIN_FILE="$CEPH_DIR/cmake/toolchains/mingw32.cmake" \
-      -D MGR_PYTHON_VERSION=$pyVersion \
       -D WITH_RDMA=OFF -D WITH_OPENLDAP=OFF \
       -D WITH_GSSAPI=OFF -D WITH_FUSE=OFF -D WITH_XFS=OFF \
       -D WITH_BLUESTORE=OFF -D WITH_LEVELDB=OFF \
@@ -100,6 +99,8 @@ cmake -D CMAKE_PREFIX_PATH=$depsDirs \
       -D Boost_THREADAPI="pthread" \
       -D ENABLE_GIT_VERSION=$ENABLE_GIT_VERSION \
       -D ALLOCATOR="$ALLOCATOR" -D CMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \
+      -D WNBD_INCLUDE_DIRS="$wnbdSrcDir/include" \
+      -D WNBD_LIBRARIES="$wnbdLibDir/libwnbd.a" \
       -G "$generatorUsed" \
       $CEPH_DIR  2>&1 | tee "${BUILD_DIR}/cmake.log"
 
@@ -113,7 +114,7 @@ if [[ -z $SKIP_BUILD ]]; then
     make_targets["src/tools"]="ceph-conf ceph_radosacl ceph_scratchtool rados"
     make_targets["src/tools/immutable_object_cache"]="all"
     make_targets["src/tools/rbd"]="all"
-    make_targets["src/tools/rbd_mirror"]="all"
+    make_targets["src/tools/rbd_wnbd"]="all"
     make_targets["src/compressor"]="all"
 
     for target_subdir in "${!make_targets[@]}"; do

win32_deps_build.sh

@@ -37,8 +37,10 @@ snappyTag="1.1.7"
 # Additional Windows libraries, which aren't provided by Mingw
 winLibDir="${depsToolsetDir}/windows/lib"
 
-
-MINGW_PREFIX="x86_64-w64-mingw32-"
+wnbdUrl="https://github.com/cloudbase/wnbd"
+wnbdTag="master"
+wnbdSrcDir="${depsSrcDir}/wnbd"
+wnbdLibDir="${depsToolsetDir}/wnbd/lib"
 
 function _make() {
   make -j $NUM_WORKERS $@