Merge pull request #69827 from mohit84/mclock_perf_counter

crimson/osd: add per-class mClock scheduler perf counters

src/crimson/osd/osd_operation.cc

@@ -167,9 +167,61 @@ void OperationThrottler::start()
   return;
 }
 
+void OperationThrottler::register_metrics(const std::string &sched_type) {
+  namespace sm = seastar::metrics;
+
+  LOG_PREFIX(OperationThrottler::register_metrics);
+  INFO("registering metrics for scheduler {}", sched_type);
+  const std::string group_name =
+    (sched_type == "mclock_scheduler") ? "osd_mclock" : "osd_wpq";
+
+  for (auto& [op_class, name] : {
+    std::pair{SchedulerClass::background_recovery,    "background_recovery"},
+    std::pair{SchedulerClass::background_best_effort, "background_best_effort"},
+    std::pair{SchedulerClass::client,                 "client"},
+    std::pair{SchedulerClass::repop,                  "repop"},
+    std::pair{SchedulerClass::immediate,              "immediate"},
+  }) {
+    auto label = sm::label("op_class")(name);
+    metrics.add_group(group_name, {
+      sm::make_counter("throttled_ops",
+        [this, op_class] { return throttled_ops[op_class]; },
+        sm::description("ops delayed by mClock scheduler"), {label}),
+      sm::make_counter("total_wait_ms",
+        [this, op_class] { return total_wait_ms[op_class]; },
+        sm::description("total ms spent waiting in mClock"), {label}),
+      sm::make_gauge("max_wait_ms",
+        [this, op_class] { return max_wait_ms[op_class]; },
+        sm::description("max wait ms in mClock by op class"), {label}),
+      sm::make_histogram("throttle_wait_latency",
+        [this, op_class]() -> seastar::metrics::histogram& {
+          return wait_hist[op_class]; },
+        sm::description("mClock throttle wait distribution"), {label}),
+    });
+  }
+}
+
+
 OperationThrottler::OperationThrottler(ConfigProxy &conf)
 {
   conf.add_observer(this);
+  for (auto op_class : {SchedulerClass::background_recovery,
+                        SchedulerClass::background_best_effort,
+                        SchedulerClass::client,
+                        SchedulerClass::repop,
+                        SchedulerClass::immediate}) {
+    wait_hist[op_class].buckets = {
+      {0, 1},
+      {0, 5},
+      {0, 10},
+      {0, 50},
+      {0, 100},
+      {0, 500},
+      {0, 1000},
+    };
+  }
+  register_metrics(conf.get_val<std::string>("osd_op_queue"));
+
 }
 
 void OperationThrottler::initialize_scheduler(CephContext *cct, ConfigProxy &conf, bool is_rotational, int whoami)
@@ -262,6 +314,26 @@ seastar::future<> OperationThrottler::stop()
   co_return;
 }
 
+void OperationThrottler::record_throttle_wait(
+  SchedulerClass op_class, uint64_t wait_ms)
+{
+  if (wait_ms == 0) {
+    return;
+  }
+  throttled_ops[op_class]++;
+  total_wait_ms[op_class] += wait_ms;
+  max_wait_ms[op_class] = std::max(max_wait_ms[op_class], wait_ms);
+
+  auto& h = wait_hist[op_class];
+  h.sample_count++;
+  h.sample_sum += wait_ms;
+  for (auto& bucket : h.buckets) {
+    if (wait_ms <= bucket.upper_bound) {
+      bucket.count++;
+    }
+  }
+}
+
 void OperationThrottler::dump_detail(Formatter *f) const
 {
   f->dump_unsigned("max_in_progress", max_in_progress);

src/crimson/osd/osd_operation.h

@@ -362,6 +362,13 @@ class OperationThrottler : public BlockerT<OperationThrottler>,
   friend BlockerT<OperationThrottler>;
   static constexpr const char* type_name = "OperationThrottler";
 
+  // Perf counters used to capture mClock throttling behavior per op class
+  std::unordered_map<SchedulerClass, uint64_t> throttled_ops;
+  std::unordered_map<SchedulerClass, uint64_t> total_wait_ms;
+  std::unordered_map<SchedulerClass, uint64_t> max_wait_ms;
+  std::unordered_map<SchedulerClass, seastar::metrics::histogram> wait_hist;
+  seastar::metrics::metric_groups metrics;
+
 public:
   OperationThrottler(ConfigProxy &conf);
   void start();
@@ -396,14 +403,19 @@ public:
     }
   };
 
+  void record_throttle_wait(SchedulerClass op_class, uint64_t wait_ms);
   auto get_throttle(crimson::osd::scheduler::params_t params) {
+    auto start = seastar::steady_clock_type::now();
     return acquire_throttle(
       params
-    ).then([this] {
+    ).then([this, start, klass = params.klass] {
+      auto wait_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
+      seastar::steady_clock_type::now() - start).count();
+      record_throttle_wait(klass, wait_ms);
       return ThrottleReleaser{this};
     });
   }
-
+  void register_metrics(const std::string &sched_type);
   void initialize_scheduler(CephContext* cct, ConfigProxy &conf, bool is_rotational, int whoami);
 private:
   void dump_detail(Formatter *f) const final;