Merge pull request #69253 from cbodley/wip-76725

osdc: deliver neorados completions to associated executor

Reviewed-by: Adam Emerson <aemerson@redhat.com>
Reviewed-by: Shilpa Jagannath <smanjara@redhat.com>

qa/workunits/rados/test.sh

@@ -24,7 +24,7 @@ RADOS_TESTS=(
 )
 
 NEORADOS_TESTS=(
-    cls cmd handler_error io ec_io list ec_list misc pool
+    cls cmd completions handler_error io ec_io list ec_list misc pool
     read_operations snapshots watch_notify write_operations
 )
 

src/osdc/Objecter.h

@@ -2080,8 +2080,9 @@ public:
 			      fu2::unique_function<OpSig>>) {
 		     std::move(arg)(ec);
                    } else {
-		     boost::asio::defer(e,
-					boost::asio::append(std::move(arg), ec));
+		     auto ex = boost::asio::get_associated_executor(arg, e);
+		     boost::asio::dispatch(ex,
+					   boost::asio::append(std::move(arg), ec));
 		   }
 		 }, std::move(f));
     }

src/test/neorados/CMakeLists.txt

@@ -10,10 +10,6 @@ add_executable(ceph_test_neorados_start_stop start_stop.cc)
 target_link_libraries(ceph_test_neorados_start_stop global libneorados
   ${unittest_libs})
 
-add_executable(ceph_test_neorados_completions completions.cc)
-target_link_libraries(ceph_test_neorados_completions Boost::boost pthread
-  ${unittest_libs})
-
 add_executable(ceph_test_neorados_op_speed op_speed.cc)
 target_link_libraries(ceph_test_neorados_op_speed
   libneorados ${FMT_LIB} ${unittest_libs})
@@ -22,6 +18,10 @@ add_library(neoradostest-support STATIC common_tests.cc)
 target_link_libraries(neoradostest-support
   libneorados ${FMT_LIB} GTest::GTest)
 
+add_executable(ceph_test_neorados_completions completions.cc)
+target_link_libraries(ceph_test_neorados_completions
+  libneorados neoradostest-support GTest::Main)
+
 add_executable(ceph_test_neorados_list_pool list_pool.cc)
 target_link_libraries(ceph_test_neorados_list_pool
   libneorados neoradostest-support global ${FMT_LIB} ${unittest_libs})

src/test/neorados/completions.cc

@@ -1,19 +1,62 @@
-#include <boost/asio/io_context.hpp>
-#include <boost/asio/post.hpp>
+#include <optional>
+#include <boost/asio/bind_executor.hpp>
+#include "include/neorados/RADOS.hpp"
+#include "test/neorados/common_tests.h"
 
-constexpr int max_completions = 10'000'000;
-int completed = 0;
+TEST(NeoRadosCompletion, CrossExecutor)
+{
+  boost::asio::io_context rados_context;
+  std::optional<neorados::RADOS> rados;
+  std::optional<neorados::IOContext> pool;
 
-boost::asio::io_context c;
+  neorados::RADOS::Builder{}.build(rados_context,
+      [&rados, &pool] (boost::system::error_code ec, neorados::RADOS r) {
+        if (ec) {
+          throw boost::system::system_error(ec);
+        }
+        rados = std::move(r);
 
-void nested_cb() {
-  if (++completed < max_completions)
-    boost::asio::post(c, &nested_cb);
-}
-
-int main(void) {
-  boost::asio::post(c, &nested_cb);
-  c.run();
-  assert(completed == max_completions);
-  return 0;
+        const char* pool_name = "ceph_test_neorados_completions";
+        rados->lookup_pool(pool_name,
+            [&rados, &pool, pool_name] (boost::system::error_code ec, int64_t id) {
+              if (ec == boost::system::errc::no_such_file_or_directory) {
+                create_pool(*rados, pool_name,
+                    [&pool] (boost::system::error_code ec, int64_t id) {
+                      if (ec) {
+                        throw boost::system::system_error(ec);
+                      }
+                      pool.emplace(id);
+                    });
+              } else if (ec) {
+                throw boost::system::system_error(ec);
+              } else {
+                pool.emplace(id);
+              }
+            });
+      });
+  rados_context.run();
+
+  // expect ReadOp completion on a separate execution context
+  boost::asio::io_context completion_context;
+  std::optional<boost::system::error_code> ec;
+  uint64_t size;
+  ceph::real_time mtime;
+
+  rados->execute("nonexistent", *pool, neorados::ReadOp{}.stat(&size, &mtime), nullptr,
+      boost::asio::bind_executor(completion_context,
+          [&ec] (boost::system::error_code e) { ec = e; }));
+
+  rados_context.restart();
+  rados_context.run();
+
+  EXPECT_FALSE(ec); // completion does not run on rados_context
+
+  completion_context.run();
+
+  EXPECT_TRUE(ec);
+
+  rados->delete_pool(pool->get_pool(), boost::asio::detached);
+
+  rados_context.restart();
+  rados_context.run();
 }