From f17f12b4a189bef1ff1eea7f8e39767112d66268 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Wed, 1 Jul 2026 19:52:31 +0800 Subject: [PATCH 01/51] doc/dev/crimson: add design the document for LogicalBucketCache Signed-off-by: Xuehan Xu --- .../crimson/seastore_logical_bucket_cache.rst | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 doc/dev/crimson/seastore_logical_bucket_cache.rst diff --git a/doc/dev/crimson/seastore_logical_bucket_cache.rst b/doc/dev/crimson/seastore_logical_bucket_cache.rst new file mode 100644 index 00000000000..6324c78b8bf --- /dev/null +++ b/doc/dev/crimson/seastore_logical_bucket_cache.rst @@ -0,0 +1,85 @@ +===================================== + The Logical Bucket Cache in SeaStore +===================================== + +This document introduces the LogicalBucketCache machinery which is +mainly for using SSDs as non-volatile cache for the relatively slow +HDD devices. Specifically, it serves to: + +#. Load frequently accessed data from the cold tier to the Logical + Bucket Cache. +#. Evict relatively cold data from the Logical Bucket Cache to the + cold tier based on the heat of both data read and write. + +The Logical Bucket Cache is enabled when there is at least one cold +tier. + +General Approach +================ + +#. Cache line: a range of continuous laddr address space, which we + call logical bucket. For now, that continuous laddr range is the + scope of a RADOS object. +#. Extents are loaded to the cache device when: 1) they are newly + written by clients and don't trigger the write_through machinery. + For now, the write_through machinery is triggered when the newly + written data is larger than a specific threshold; 2) they are on + the slow cold device and read to or evicted from the in-memory Cache. +#. Extents are evicted from the cache device in units of logical buckets, + that is, extents of the same logical bucket are evicted from the cache + device altogether. +#. The process that write extents evicted from Cache down to the cache + device is called promotion. +#. The process that evict extents from the cache device to the cold device + is called demotion. +#. Extents that are loaded from the cold tier to the Logical Bucket Cache + are NOT removed from the cold tier, which means they'd have two paddrs, + one in the Logical Bucket Cache, the other in the cold tier, and the + promoted data is stored twice. This is designed to make the demote process + cheap. We call paddrs corresponding to the cold tier shadow paddrs. + So lba mappings have two paddrs now, the primary paddr and the shadow + paddr. + +Promotion +========= + +Promotion is the process that load the extents in the cold tier to the +Logical Bucket Cache. + +Extents that are read from the cold tier are first added to the in-memory +cache; when evicted from the in-memory cache, they are added to the Promoter. + +The Promoter write its extents down to the Logical Bucket Cache when the size +of its extents grows larger than ``seastore_cache_promotion_size``. + +Demotion +======== + +Demotion happens when: + +#. The usage of the Logical Bucket Cache reaches ``seastore_multiple_tiers_fast_evict_ratio``. +#. The in-memory data index of the Logical Bucket Cache reaches ``seastore_logical_bucket_capacity``. + +Demoting extents that are live in both the Logical Bucket Cache and the +cold tier only involves setting the shadow paddr to the primary paddr and +removing the shadow paddr itself. + +Demoting extents that are only live in the Logical Bucket Cache would involve +rewriting the extents to the cold tier. + +WriteThrough +============ + +When the newly written data is larger than ``seastore_write_through_size``, +it would be written directly to the cold tier. + +Internal Test Workload +======================= + +To test the correctness of the whole Logical Bucket Cache machinery, we also +implemented an internal stress test workload. When turned on, SeaStore would +aggressively load the extents from the cold tier to the Logical Bucket Cache, +and demote extents back to the cold tier. The newly written data may also be +written directly to the cold tier based on ``seastore_test_workload_write_through_probability``. + +The internal test workload is turned on by ``crimson_test_workload``. From ece1a684970671ccf888c6381f7c1268e8802e78 Mon Sep 17 00:00:00 2001 From: Zhang Song Date: Wed, 10 Apr 2024 11:17:30 +0800 Subject: [PATCH 02/51] crimson/os/seastore: introduce PROMOTE and DEMOTE transaction type Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/cache.cc | 6 ++++++ src/crimson/os/seastore/cache.h | 4 ++++ src/crimson/os/seastore/seastore_types.cc | 4 ++++ src/crimson/os/seastore/seastore_types.h | 2 ++ 4 files changed, 16 insertions(+) diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc index bcc2adef7d1..4bcf21f098f 100644 --- a/src/crimson/os/seastore/cache.cc +++ b/src/crimson/os/seastore/cache.cc @@ -133,6 +133,8 @@ void Cache::register_metrics(store_index_t store_index) {src_t::TRIM_ALLOC, {sm::label_instance("src", "TRIM_ALLOC")}}, {src_t::CLEANER_MAIN, {sm::label_instance("src", "CLEANER_MAIN")}}, {src_t::CLEANER_COLD, {sm::label_instance("src", "CLEANER_COLD")}}, + {src_t::PROMOTE, {sm::label_instance("src", "PROMOTE")}}, + {src_t::DEMOTE, {sm::label_instance("src", "DEMOTE")}}, }; assert(labels_by_src.size() == (std::size_t)src_t::MAX); @@ -672,6 +674,10 @@ void Cache::register_metrics(store_index_t store_index) src2 == Transaction::src_t::CLEANER_MAIN) || (src1 == Transaction::src_t::CLEANER_COLD && src2 == Transaction::src_t::CLEANER_COLD) || + (src1 == Transaction::src_t::PROMOTE && + src2 == Transaction::src_t::PROMOTE) || + (src1 == Transaction::src_t::DEMOTE && + src2 == Transaction::src_t::DEMOTE) || (src1 == Transaction::src_t::TRIM_ALLOC && src2 == Transaction::src_t::TRIM_ALLOC)) { continue; diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h index c878ad6d51f..72c5d174db6 100644 --- a/src/crimson/os/seastore/cache.h +++ b/src/crimson/os/seastore/cache.h @@ -1905,6 +1905,10 @@ private: src2 == Transaction::src_t::CLEANER_MAIN)); assert(!(src1 == Transaction::src_t::CLEANER_COLD && src2 == Transaction::src_t::CLEANER_COLD)); + assert(!(src1 == Transaction::src_t::PROMOTE && + src2 == Transaction::src_t::PROMOTE)); + assert(!(src1 == Transaction::src_t::DEMOTE && + src2 == Transaction::src_t::DEMOTE)); assert(!(src1 == Transaction::src_t::TRIM_ALLOC && src2 == Transaction::src_t::TRIM_ALLOC)); diff --git a/src/crimson/os/seastore/seastore_types.cc b/src/crimson/os/seastore/seastore_types.cc index d205d009f11..adddf6fd3fd 100644 --- a/src/crimson/os/seastore/seastore_types.cc +++ b/src/crimson/os/seastore/seastore_types.cc @@ -816,6 +816,10 @@ std::ostream &operator<<(std::ostream &os, transaction_type_t type) return os << "CLEANER_MAIN"; case transaction_type_t::CLEANER_COLD: return os << "CLEANER_COLD"; + case transaction_type_t::PROMOTE: + return os << "PROMOTE"; + case transaction_type_t::DEMOTE: + return os << "DEMOTE"; case transaction_type_t::MAX: return os << "TRANS_TYPE_NULL"; default: diff --git a/src/crimson/os/seastore/seastore_types.h b/src/crimson/os/seastore/seastore_types.h index 58b23da3814..76f1dbe7ebc 100644 --- a/src/crimson/os/seastore/seastore_types.h +++ b/src/crimson/os/seastore/seastore_types.h @@ -2718,6 +2718,8 @@ enum class transaction_type_t : uint8_t { TRIM_ALLOC, CLEANER_MAIN, CLEANER_COLD, + PROMOTE, + DEMOTE, MAX }; From 6c526c811b570afc9b786f90f0afc5c93ffaac31 Mon Sep 17 00:00:00 2001 From: Zhang Song Date: Wed, 3 Sep 2025 15:58:59 +0800 Subject: [PATCH 03/51] crimson/os/seastore: rename extent_2q_state_t to extent_pin_state_t This is because, even for LRU caches, extents still have pin states, although it's only Fresh. What's more, in the logical bucket cache world, extents in both LRU and 2Q caches may be in the Fresh/PendingPromote/Promoting state. Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/cached_extent.cc | 16 ++++++ src/crimson/os/seastore/cached_extent.h | 67 +++++++++++----------- src/crimson/os/seastore/extent_pinboard.cc | 36 ++++++------ 3 files changed, 68 insertions(+), 51 deletions(-) diff --git a/src/crimson/os/seastore/cached_extent.cc b/src/crimson/os/seastore/cached_extent.cc index 09caf5f6635..84a9ecace43 100644 --- a/src/crimson/os/seastore/cached_extent.cc +++ b/src/crimson/os/seastore/cached_extent.cc @@ -66,6 +66,22 @@ std::ostream &operator<<(std::ostream &out, CachedExtent::extent_state_t state) } } +std::ostream &operator<<(std::ostream &out, const extent_pin_state_t &s) { + switch (s) { + case extent_pin_state_t::Fresh: + return out << "Fresh"; + case extent_pin_state_t::WarmIn: + return out << "WarmIn"; + case extent_pin_state_t::Hot: + return out << "Hot"; + case extent_pin_state_t::Max: + return out << "Max"; + default: + __builtin_unreachable(); + return out; + } +} + std::ostream &operator<<(std::ostream &out, const CachedExtent &ext) { return ext.print(out); diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h index 91416f87060..bda0b676a2d 100644 --- a/src/crimson/os/seastore/cached_extent.h +++ b/src/crimson/os/seastore/cached_extent.h @@ -266,12 +266,15 @@ private: map_t buffer_map; }; -enum class extent_2q_state_t : uint8_t { +enum class extent_pin_state_t : uint8_t { + // shared state between LRU and 2Q impl Fresh = 0, + // 2Q impl only WarmIn, Hot, Max }; +std::ostream &operator<<(std::ostream &, const extent_pin_state_t &); class ExtentCommitter : public boost::intrusive_ref_counter< ExtentCommitter, boost::thread_unsafe_counter> { @@ -529,6 +532,7 @@ public: << std::hex << ", length=0x" << get_length() << ", loaded=0x" << get_loaded_length() << std::dec << ", state=" << state + << ", pin_state=" << pin_state << ", last_committed_crc=" << last_committed_crc << ", refcount=" << use_count() << ", user_hint=" << user_hint @@ -864,17 +868,34 @@ public: std::pair is_viewable_by_trans(Transaction &t); - extent_2q_state_t get_2q_state() const { - assert("2Q" == crimson::common::get_conf - ("seastore_cachepin_type")); - return cache_state; + extent_pin_state_t get_pin_state() const { +#ifndef NDEBUG + using crimson::common::get_conf; + auto type = get_conf("seastore_cachepin_type"); + if (type == "LRU") { + assert(pin_state == extent_pin_state_t::Fresh); + } else if (type == "2Q") { + assert(pin_state < extent_pin_state_t::Max); + } else { + ceph_abort("invalid seastore_cachepin_type(LRU or 2Q)"); + } +#endif + return pin_state; } - void set_2q_state(extent_2q_state_t state) { - assert("2Q" == crimson::common::get_conf - ("seastore_cachepin_type")); - assert(state < extent_2q_state_t::Max); - cache_state = state; + void set_pin_state(extent_pin_state_t state) { + pin_state = state; +#ifndef NDEBUG + using crimson::common::get_conf; + auto type = get_conf("seastore_cachepin_type"); + if (type == "LRU") { + assert(pin_state == extent_pin_state_t::Fresh); + } else if (type == "2Q") { + assert(pin_state < extent_pin_state_t::Max); + } else { + ceph_abort("invalid seastore_cachepin_type(LRU or 2Q)"); + } +#endif } extent_len_t get_last_touch_end() const { @@ -1005,8 +1026,8 @@ private: // see Cache::touch_extent_by_range() and ExtentPinboardTwoQ. extent_len_t last_touch_end = 0; - // This field is unused when the ExtentPinboard use LRU algorithm - extent_2q_state_t cache_state = extent_2q_state_t::Fresh; + // This field is used by ExtentPinboard + extent_pin_state_t pin_state = extent_pin_state_t::Fresh; ExtentCommitterRef committer; @@ -1621,25 +1642,5 @@ using lextent_list_t = addr_extent_list_base_t< template <> struct fmt::formatter : fmt::ostream_formatter {}; template <> struct fmt::formatter : fmt::ostream_formatter {}; template <> struct fmt::formatter : fmt::ostream_formatter {}; +template <> struct fmt::formatter : fmt::ostream_formatter {}; #endif - -template <> -struct fmt::formatter - : public fmt::formatter { - using State = crimson::os::seastore::extent_2q_state_t; - auto format(const State &s, auto &ctx) const { - switch (s) { - case State::Fresh: - return fmt::format_to(ctx.out(), "Fresh"); - case State::WarmIn: - return fmt::format_to(ctx.out(), "WarmIn"); - case State::Hot: - return fmt::format_to(ctx.out(), "Hot"); - case State::Max: - return fmt::format_to(ctx.out(), "Max"); - default: - __builtin_unreachable(); - return ctx.out(); - } - } -}; diff --git a/src/crimson/os/seastore/extent_pinboard.cc b/src/crimson/os/seastore/extent_pinboard.cc index 418afd928ba..c75786685be 100644 --- a/src/crimson/os/seastore/extent_pinboard.cc +++ b/src/crimson/os/seastore/extent_pinboard.cc @@ -513,17 +513,17 @@ public: double seconds) const final; void remove(CachedExtent &extent) final { - auto s = extent.get_2q_state(); + auto s = extent.get_pin_state(); if (extent.is_linked_to_list()) { - if (s == extent_2q_state_t::WarmIn) { + if (s == extent_pin_state_t::WarmIn) { warm_in.remove(extent); } else { - ceph_assert(s == extent_2q_state_t::Hot); + ceph_assert(s == extent_pin_state_t::Hot); hot.remove(extent); } - extent.set_2q_state(extent_2q_state_t::Fresh); + extent.set_pin_state(extent_pin_state_t::Fresh); } else { - ceph_assert(s == extent_2q_state_t::Fresh); + ceph_assert(s == extent_pin_state_t::Fresh); } } @@ -532,14 +532,14 @@ public: const Transaction::src_t* p_src, extent_len_t load_start, extent_len_t load_length) final { - auto state = extent.get_2q_state(); + auto state = extent.get_pin_state(); auto type = extent.get_type(); if (extent.is_linked_to_list()) { - if (state == extent_2q_state_t::Hot) { + if (state == extent_pin_state_t::Hot) { hot.move_to_top(extent, p_src); hit_queue(overall_hits.hot_hits, p_src, type); } else { - ceph_assert(state == extent_2q_state_t::WarmIn); + ceph_assert(state == extent_pin_state_t::WarmIn); hit_queue(overall_hits.warm_in_hits, p_src, type); // warm_in is a FIFO queue, do nothing here // In the standard 2Q algorithm, the extent won't be considerred @@ -549,28 +549,28 @@ public: hit++; } else if (!is_logical_type(extent.get_type())) { // put physical extents to hot queue directly - ceph_assert(state == extent_2q_state_t::Fresh); - extent.set_2q_state(extent_2q_state_t::Hot); + ceph_assert(state == extent_pin_state_t::Fresh); + extent.set_pin_state(extent_pin_state_t::Hot); auto trimmed_extents = hot.add_to_top(extent, p_src); on_update_hot(trimmed_extents); hit_queue(overall_hits.absent, p_src, type); miss++; } else { // the logical extent which is not in warm_in and not in hot - ceph_assert(state == extent_2q_state_t::Fresh); + ceph_assert(state == extent_pin_state_t::Fresh); auto lext = extent.cast(); auto m = warm_out.accessed_recently(lext->get_laddr(), load_start); using AccessMode = IndexedFifoQueue::AccessMode; if (m == AccessMode::Again) { // This extent was accessed recently, consider it's hot enough to // promote to hot queue. - extent.set_2q_state(extent_2q_state_t::Hot); + extent.set_pin_state(extent_pin_state_t::Hot); auto trimmed_extents = hot.add_to_top(extent, p_src); on_update_hot(trimmed_extents); hit_queue(overall_hits.hot_absent, p_src, type); } else { // This extent didn't be accessed recently, put it warm_in queue // by default. - extent.set_2q_state(extent_2q_state_t::WarmIn); + extent.set_pin_state(extent_pin_state_t::WarmIn); auto trimmed_extents = warm_in.add_to_top(extent, p_src); on_update_warm_in(trimmed_extents); if (m == AccessMode::Missing) { @@ -593,13 +593,13 @@ public: extent_len_t increased_length, const Transaction::src_t* p_src) final { if (extent.is_linked_to_list()) { - auto state = extent.get_2q_state(); - if (state == extent_2q_state_t::WarmIn) { + auto state = extent.get_pin_state(); + if (state == extent_pin_state_t::WarmIn) { auto trimmed_extents = warm_in.increase_cached_size( extent, increased_length, p_src); on_update_warm_in(trimmed_extents); } else { - ceph_assert(state == extent_2q_state_t::Hot); + ceph_assert(state == extent_pin_state_t::Hot); auto trimmed_extents = hot.increase_cached_size( extent, increased_length, p_src); on_update_hot(trimmed_extents); @@ -624,13 +624,13 @@ public: private: void on_update_hot(std::list &extents) { for (auto extent : extents) { - extent->set_2q_state(extent_2q_state_t::Fresh); + extent->set_pin_state(extent_pin_state_t::Fresh); } } void on_update_warm_in(std::list &extents) { for (auto extent : extents) { ceph_assert(is_logical_type(extent->get_type())); - extent->set_2q_state(extent_2q_state_t::Fresh); + extent->set_pin_state(extent_pin_state_t::Fresh); auto len = extent->get_loaded_length(); if (len == 0) { // The extent is possibly empty after being initially split/remapped From 2d629af107b2691fbc4d67f8ef4114998e8cef74 Mon Sep 17 00:00:00 2001 From: Zhang Song Date: Thu, 3 Jul 2025 15:41:43 +0800 Subject: [PATCH 04/51] crimson/os/seastore: support promote extents purged from pinboard Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- src/common/options/crimson.yaml.in | 5 + src/crimson/os/seastore/async_cleaner.h | 16 + src/crimson/os/seastore/cache.cc | 3 +- src/crimson/os/seastore/cached_extent.cc | 2 + src/crimson/os/seastore/cached_extent.h | 6 +- src/crimson/os/seastore/extent_pinboard.cc | 307 +++++++++++++++++- src/crimson/os/seastore/extent_pinboard.h | 11 +- .../os/seastore/extent_placement_manager.h | 17 +- .../os/seastore/transaction_manager.cc | 9 + src/crimson/os/seastore/transaction_manager.h | 5 + 10 files changed, 359 insertions(+), 22 deletions(-) diff --git a/src/common/options/crimson.yaml.in b/src/common/options/crimson.yaml.in index aa94ec0c438..9651bfc69d3 100644 --- a/src/common/options/crimson.yaml.in +++ b/src/common/options/crimson.yaml.in @@ -342,3 +342,8 @@ options: desc: disable osd shards changes upon restart. flags: - startup +- name: seastore_cache_promotion_size + type: size + level: advanced + desc: Size in bytes of extents to be promoted from cold devices, set 0 to disable promotion + default: 2_M diff --git a/src/crimson/os/seastore/async_cleaner.h b/src/crimson/os/seastore/async_cleaner.h index 8006215c320..b45ad20e373 100644 --- a/src/crimson/os/seastore/async_cleaner.h +++ b/src/crimson/os/seastore/async_cleaner.h @@ -351,6 +351,22 @@ public: rewrite_gen_t target_generation, sea_time_point modify_time) = 0; + /** + * promote_extent + * + * Promote an extent located from slow devices to the faster devices. + * When the type of extent is ObjectDataBlock, the original extent won't + * be retired, so that this extent is located in two differenct devices + * at the same time, which is helpful to reduce the cost of cleaner process. + * The others follow the normal rewrite process but its rewrite generation + * will be INIT_GENERATION(XXX: or the maximum generation of the hot tier?). + */ + using promote_extent_iertr = base_iertr; + using promote_extent_ret = promote_extent_iertr::future<>; + virtual promote_extent_ret promote_extent( + Transaction &t, + CachedExtentRef extent) = 0; + /** * get_extents_if_live * diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc index 4bcf21f098f..2226744081a 100644 --- a/src/crimson/os/seastore/cache.cc +++ b/src/crimson/os/seastore/cache.cc @@ -38,7 +38,8 @@ Cache::Cache( "seastore_data_delta_based_overwrite") > 0), pinboard(create_extent_pinboard( crimson::common::get_conf( - "seastore_cachepin_size_pershard"))) + "seastore_cachepin_size_pershard"), + &epm)) { register_metrics(store_index); segment_providers_by_device_id.resize(DEVICE_ID_MAX, nullptr); diff --git a/src/crimson/os/seastore/cached_extent.cc b/src/crimson/os/seastore/cached_extent.cc index 84a9ecace43..4854bc8055f 100644 --- a/src/crimson/os/seastore/cached_extent.cc +++ b/src/crimson/os/seastore/cached_extent.cc @@ -70,6 +70,8 @@ std::ostream &operator<<(std::ostream &out, const extent_pin_state_t &s) { switch (s) { case extent_pin_state_t::Fresh: return out << "Fresh"; + case extent_pin_state_t::PendingPromote: + return out << "PendingPromote"; case extent_pin_state_t::WarmIn: return out << "WarmIn"; case extent_pin_state_t::Hot: diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h index bda0b676a2d..cfb07b509d5 100644 --- a/src/crimson/os/seastore/cached_extent.h +++ b/src/crimson/os/seastore/cached_extent.h @@ -269,6 +269,7 @@ private: enum class extent_pin_state_t : uint8_t { // shared state between LRU and 2Q impl Fresh = 0, + PendingPromote, // 2Q impl only WarmIn, Hot, @@ -873,7 +874,7 @@ public: using crimson::common::get_conf; auto type = get_conf("seastore_cachepin_type"); if (type == "LRU") { - assert(pin_state == extent_pin_state_t::Fresh); + assert(pin_state <= extent_pin_state_t::PendingPromote); } else if (type == "2Q") { assert(pin_state < extent_pin_state_t::Max); } else { @@ -889,7 +890,7 @@ public: using crimson::common::get_conf; auto type = get_conf("seastore_cachepin_type"); if (type == "LRU") { - assert(pin_state == extent_pin_state_t::Fresh); + assert(pin_state <= extent_pin_state_t::PendingPromote); } else if (type == "2Q") { assert(pin_state < extent_pin_state_t::Max); } else { @@ -1117,6 +1118,7 @@ protected: friend class ExtentQueue; friend class ExtentPinboardLRU; friend class ExtentPinboardTwoQ; + friend class ExtentPromoter; template static TCachedExtentRef make_cached_extent_ref( Args&&... args) { diff --git a/src/crimson/os/seastore/extent_pinboard.cc b/src/crimson/os/seastore/extent_pinboard.cc index c75786685be..d65ad2a4956 100644 --- a/src/crimson/os/seastore/extent_pinboard.cc +++ b/src/crimson/os/seastore/extent_pinboard.cc @@ -3,6 +3,7 @@ #include "crimson/os/seastore/extent_pinboard.h" #include "crimson/os/seastore/transaction.h" +#include "crimson/os/seastore/transaction_manager.h" #include @@ -266,8 +267,150 @@ void ExtentQueue::get_stats( last_overall_io = overall_io; } +class ExtentPromoter { +public: + ExtentPromoter(size_t promotion_size, ExtentPlacementManager &epm) + : promotion_size(promotion_size), epm(epm) {} + + ~ExtentPromoter() { + clear(); + } + + bool enabled() const { + return ecb != nullptr; + } + + bool should_promote_extent(const CachedExtent &extent) { + return enabled() && epm.is_cold_device(extent.get_paddr().get_device_id()); + } + + size_t get_promotion_size() const { + return current_contents; + } + + void set_background_callback(BackgroundListener *l) { + listener = l; + } + + void set_extent_callback(ExtentCallbackInterface *cb) { + ecb = cb; + } + + bool should_run_promote() const { + return enabled() && current_contents >= promotion_size; + } + + std::size_t get_promoted_size() const { + return promoted_size; + } + + std::size_t get_promoted_count() const { + return promoted_count; + } + + void add_extent(CachedExtent &extent) { + assert(!extent.is_linked_to_list()); + assert(extent.is_stable_clean()); + extent.set_pin_state(extent_pin_state_t::PendingPromote); + list.push_back(extent); + current_contents += extent.get_length(); + intrusive_ptr_add_ref(&extent); + while (current_contents > promotion_size) { + remove_extent(list.front(), extent_pin_state_t::Fresh); + } + if (should_run_promote()) { + // TODO: wake promote background process + } + } + + void remove_extent(CachedExtent &extent, extent_pin_state_t new_state) { + assert(extent.is_linked_to_list()); + assert(extent.get_pin_state() == extent_pin_state_t::PendingPromote); + assert(current_contents >= extent.get_length()); + extent.set_pin_state(new_state); + list.erase(list.s_iterator_to(extent)); + current_contents -= extent.get_length(); + intrusive_ptr_release(&extent); + } + + void clear() { + for (auto iter = list.begin(); iter != list.end();) { + remove_extent(*(iter++), extent_pin_state_t::Fresh); + } + } + + using run_promote_ret = base_iertr::future<>; + run_promote_ret run_promote( + Transaction &t, + std::list &promoting_extents) + { + LOG_PREFIX(ExtentPromoter::run_promote); + std::size_t promote_size = 0; + DEBUGT("start promote", t); + std::list extents; + for (auto &extent : list) { + DEBUGT("promote {} to the hot tier", t, extent); + ceph_assert(extent.is_stable_clean()); + ceph_assert(extent.get_pin_state() == extent_pin_state_t::PendingPromote); + extents.emplace_back(&extent); + } + for (auto &extent : extents) { + remove_extent(*extent, extent_pin_state_t::Fresh); + } + promoting_extents.insert( + promoting_extents.end(), + extents.begin(), + extents.end()); + for (auto it = promoting_extents.begin(); + it != promoting_extents.end();) { + auto &extent = *it; + if (!extent->is_valid()) { + it = promoting_extents.erase(it); + continue; + } + promote_size += extent->get_length(); + t.add_to_read_set(extent); + co_await trans_intr::make_interruptible(extent->wait_io()); + co_await ecb->promote_extent(t, extent); + it++; + } + // existing extents in lru will be retired after transaction submitted + co_await ecb->submit_transaction_direct(t); + promoted_count += promoting_extents.size(); + promoted_size += promote_size; + DEBUGT("finish promoting {} {}B extents", + t, promoting_extents.size(), promote_size); + co_return; + } + + seastar::future<> promote() { + assert(enabled()); + std::list promoting_extents; + co_await repeat_eagain([this, &promoting_extents] { + return ecb->with_transaction_intr( + Transaction::src_t::PROMOTE, + "promote", cache_hint_t::get_nocache(), + [this, &promoting_extents](auto &t) { + return run_promote(t, promoting_extents); + }); + }).handle_error(crimson::ct_error::assert_all("error occupied during promotion")); + } + +private: + const size_t promotion_size = 0; + ExtentPlacementManager &epm; + ExtentCallbackInterface *ecb = nullptr; + BackgroundListener *listener = nullptr; + CachedExtent::primary_ref_list list; + size_t current_contents = 0; + + size_t promoted_count = 0; + size_t promoted_size = 0; +}; + class ExtentPinboardLRU : public ExtentPinboard { ExtentQueue lru; + ExtentPromoter promoter; seastar::metrics::metric_group metrics; // hit and miss indicates if an extent is linked when touching it @@ -275,9 +418,11 @@ class ExtentPinboardLRU : public ExtentPinboard { uint64_t miss = 0; public: - ExtentPinboardLRU(std::size_t capacity) : lru(capacity) { + ExtentPinboardLRU(std::size_t capacity, size_t promotion_size, ExtentPlacementManager &epm) + : lru(capacity), promoter(promotion_size, epm) { LOG_PREFIX(ExtentPinboardLRU::ExtentPinboardLRU); - INFO("created, lru_capacity=0x{:x}B", capacity); + INFO("created, lru_capacity=0x{:x}B, promotion_size=0x{:x}B", + capacity, promotion_size); } std::size_t get_capacity_bytes() const { @@ -325,6 +470,26 @@ public: ), } ); + if (promoter.enabled()) { + metrics.add_group( + "cache", + { + sm::make_counter( + "promoted_size", + [this] { + return promoter.get_promoted_size(); + }, + sm::description("total bytes promoted by the lru"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), + sm::make_counter( + "promoted_count", + [this] { + return promoter.get_promoted_count(); + }, + sm::description("total extents promoted by the lru"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), + }); + } } void get_stats( @@ -335,8 +500,28 @@ public: } void remove(CachedExtent &extent) final { + auto s = extent.get_pin_state(); if (extent.is_linked_to_list()) { - lru.remove(extent); + if (s == extent_pin_state_t::Fresh) { + lru.remove(extent); + } else { + promoter.remove_extent(extent, extent_pin_state_t::Fresh); + } + } else { + ceph_assert(s == extent_pin_state_t::Fresh); + } + } + + void add_to_top( + CachedExtent &extent, + const Transaction::src_t* p_src) { + auto trimmed = lru.add_to_top(extent, p_src); + if (promoter.enabled()) { + for (auto &extent : trimmed) { + if (promoter.should_promote_extent(*extent)) { + promoter.add_extent(*extent); + } + } } } @@ -346,10 +531,17 @@ public: extent_len_t /*load_start*/, extent_len_t /*load_length*/) final { if (extent.is_linked_to_list()) { - lru.move_to_top(extent, p_src); + auto s = extent.get_pin_state(); + assert(s <= extent_pin_state_t::PendingPromote); + if (s == extent_pin_state_t::Fresh) { + lru.move_to_top(extent, p_src); + } else { + promoter.remove_extent(extent, extent_pin_state_t::Fresh); + add_to_top(extent, p_src); + } hit++; } else { - lru.add_to_top(extent, p_src); + add_to_top(extent, p_src); miss++; } } @@ -358,13 +550,38 @@ public: CachedExtent &extent, extent_len_t increased_length, const Transaction::src_t* p_src) final { - if (extent.is_linked_to_list()) { + if (extent.is_linked_to_list() && + extent.get_pin_state() == extent_pin_state_t::Fresh) { lru.increase_cached_size(extent, increased_length, p_src); + } else { + // promoter take the complete extent size for content size calculation + assert(extent.get_pin_state() <= extent_pin_state_t::PendingPromote); } } void clear() final { lru.clear(); + promoter.clear(); + } + + void set_background_callback(BackgroundListener *listener) final { + promoter.set_background_callback(listener); + } + + void set_extent_callback(ExtentCallbackInterface *cb) final { + promoter.set_extent_callback(cb); + } + + std::size_t get_promotion_size() const final { + return promoter.get_promotion_size(); + } + + bool should_promote() const final { + return promoter.should_run_promote(); + } + + seastar::future<> promote() final { + return promoter.promote(); } ~ExtentPinboardLRU() { @@ -482,15 +699,18 @@ public: ExtentPinboardTwoQ( std::size_t warm_in_capacity, std::size_t warm_out_capacity, - std::size_t hot_capacity) + std::size_t hot_capacity, + std::size_t promotion_size, + ExtentPlacementManager &epm) : warm_in(warm_in_capacity), warm_out(warm_out_capacity), - hot(hot_capacity) + hot(hot_capacity), + promoter(promotion_size, epm) { LOG_PREFIX(ExtentPinboardTwoQ::ExtentPinboardTwoQ); - INFO("created, warm_in_capacity=0x{:x}B, " - "warm_out_capacity=0x{:x}B, hot_capacity=0x{:x}B", - warm_in_capacity, warm_out_capacity, hot_capacity); + INFO("created, warm_in_capacity=0x{:x}B, warm_out_capacity=0x{:x}B, " + "hot_capacity=0x{:x}B, promotion_size=0x{:x}B", + warm_in_capacity, warm_out_capacity, hot_capacity, promotion_size); } std::size_t get_capacity_bytes() const { @@ -517,6 +737,8 @@ public: if (extent.is_linked_to_list()) { if (s == extent_pin_state_t::WarmIn) { warm_in.remove(extent); + } else if (s == extent_pin_state_t::PendingPromote) { + promoter.remove_extent(extent, extent_pin_state_t::Fresh); } else { ceph_assert(s == extent_pin_state_t::Hot); hot.remove(extent); @@ -538,6 +760,11 @@ public: if (state == extent_pin_state_t::Hot) { hot.move_to_top(extent, p_src); hit_queue(overall_hits.hot_hits, p_src, type); + } else if (state == extent_pin_state_t::PendingPromote) { + promoter.remove_extent(extent, extent_pin_state_t::Hot); + auto trimmed_extents = hot.add_to_top(extent, p_src); + on_update_hot(trimmed_extents); + hit_queue(overall_hits.hot_hits, p_src, type); } else { ceph_assert(state == extent_pin_state_t::WarmIn); hit_queue(overall_hits.warm_in_hits, p_src, type); @@ -598,11 +825,13 @@ public: auto trimmed_extents = warm_in.increase_cached_size( extent, increased_length, p_src); on_update_warm_in(trimmed_extents); - } else { - ceph_assert(state == extent_pin_state_t::Hot); + } else if (state == extent_pin_state_t::Hot) { auto trimmed_extents = hot.increase_cached_size( extent, increased_length, p_src); on_update_hot(trimmed_extents); + } else { + ceph_assert(extent.get_pin_state() == + extent_pin_state_t::PendingPromote); } } } @@ -616,6 +845,23 @@ public: warm_in.clear(); warm_out.clear(); hot.clear(); + promoter.clear(); + } + + void set_background_callback(BackgroundListener *listener) final { + promoter.set_background_callback(listener); + } + void set_extent_callback(ExtentCallbackInterface *cb) final { + promoter.set_extent_callback(cb); + } + std::size_t get_promotion_size() const final { + return promoter.get_promotion_size(); + } + bool should_promote() const final { + return promoter.should_run_promote(); + } + seastar::future<> promote() final { + return promoter.promote(); } ~ExtentPinboardTwoQ() { @@ -624,7 +870,11 @@ public: private: void on_update_hot(std::list &extents) { for (auto extent : extents) { - extent->set_pin_state(extent_pin_state_t::Fresh); + if (promoter.should_promote_extent(*extent)) { + promoter.add_extent(*extent); + } else { + extent->set_pin_state(extent_pin_state_t::Fresh); + } } } void on_update_warm_in(std::list &extents) { @@ -663,6 +913,7 @@ private: ExtentQueue warm_in; IndexedFifoQueue warm_out; ExtentQueue hot; + ExtentPromoter promoter; seastar::metrics::metric_group metrics; struct QueueCounter { @@ -852,13 +1103,34 @@ void ExtentPinboardTwoQ::register_metrics(store_index_t store_index) { ), } ); + if (promoter.enabled()) { + metrics.add_group( + "cache", + { + sm::make_counter( + "promoted_size", + [this] { + return promoter.get_promoted_size(); + }, + sm::description("total bytes promoted by the lru"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), + sm::make_counter( + "promoted_count", + [this] { + return promoter.get_promoted_count(); + }, + sm::description("total extents promoted by the lru"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), + }); + } } -ExtentPinboardRef create_extent_pinboard(std::size_t capacity) { +ExtentPinboardRef create_extent_pinboard(std::size_t capacity, ExtentPlacementManager *epm) { using crimson::common::get_conf; + size_t promotion_size = get_conf("seastore_cache_promotion_size"); auto algorithm = get_conf("seastore_cachepin_type"); if (algorithm == "LRU") { - return std::make_unique(capacity); + return std::make_unique(capacity, promotion_size, *epm); } else if (algorithm == "2Q") { auto warm_in_ratio = get_conf("seastore_cachepin_2q_in_ratio"); auto warm_out_ratio = get_conf("seastore_cachepin_2q_out_ratio"); @@ -867,7 +1139,8 @@ ExtentPinboardRef create_extent_pinboard(std::size_t capacity) { return std::make_unique( capacity * warm_in_ratio, capacity * warm_out_ratio, - capacity * (1 - warm_in_ratio)); + capacity * (1 - warm_in_ratio), + promotion_size, *epm); } else { ceph_abort("invalid seastore_cachepin_type(LRU or 2Q)"); return nullptr; diff --git a/src/crimson/os/seastore/extent_pinboard.h b/src/crimson/os/seastore/extent_pinboard.h index 3d2dd542b26..a766e486d4f 100644 --- a/src/crimson/os/seastore/extent_pinboard.h +++ b/src/crimson/os/seastore/extent_pinboard.h @@ -7,6 +7,9 @@ #include "crimson/os/seastore/transaction.h" namespace crimson::os::seastore { +class BackgroundListener; +class ExtentCallbackInterface; +class ExtentPlacementManager; struct ExtentPinboard { virtual ~ExtentPinboard() = default; @@ -28,8 +31,14 @@ struct ExtentPinboard { extent_len_t increased_length, const Transaction::src_t *p_src) = 0; virtual void clear() = 0; + virtual void set_background_callback(BackgroundListener *listener) = 0; + virtual void set_extent_callback(ExtentCallbackInterface *cb) = 0; + virtual std::size_t get_promotion_size() const = 0; + virtual bool should_promote() const = 0; + virtual seastar::future<> promote() = 0; }; using ExtentPinboardRef = std::unique_ptr; -ExtentPinboardRef create_extent_pinboard(std::size_t capacity); +ExtentPinboardRef create_extent_pinboard( + std::size_t capacity, ExtentPlacementManager *epm); } // namespace crimson::os::seastore diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index df32bcd7ba1..aba38a43e1a 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -571,13 +571,20 @@ public: return primary_device->get_backend_type(); } - bool is_pure_rbm() const { return get_main_backend_type() == backend_type_t::RANDOM_BLOCK && // as of now, cold tier can only be segmented. !background_process.has_cold_tier(); } + bool has_cold_tier() const { + return background_process.has_cold_tier(); + } + + bool is_cold_device(device_id_t id) const { + return background_process.is_cold_device(id); + } + // Testing interfaces void test_init_no_background(Device *test_device) { @@ -770,6 +777,14 @@ private: return cold_cleaner.get() != nullptr; } + bool is_cold_device(device_id_t id) const { + if (!has_cold_tier()) { + return false; + } + assert(cleaners_by_device_id[id]); + return cleaners_by_device_id[id] != main_cleaner.get(); + } + void set_extent_callback(ExtentCallbackInterface *cb) { trimmer->set_extent_callback(cb); main_cleaner->set_extent_callback(cb); diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index b4bfd50a2a5..ab0c043b4de 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -1094,6 +1094,15 @@ TransactionManager::move_region( co_return; } +TransactionManager::promote_extent_ret +TransactionManager::promote_extent( + Transaction &t, + CachedExtentRef extent) +{ + // TODO + return rewrite_extent_iertr::make_ready_future(); +} + TransactionManager::get_extents_if_live_ret TransactionManager::get_extents_if_live( Transaction &t, diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index a08734f0d21..82a51cb1e57 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -900,6 +900,11 @@ public: rewrite_gen_t target_generation, sea_time_point modify_time) final; + using ExtentCallbackInterface::promote_extent_ret; + promote_extent_ret promote_extent( + Transaction &t, + CachedExtentRef extent); + using ExtentCallbackInterface::get_extents_if_live_ret; get_extents_if_live_ret get_extents_if_live( Transaction &t, From 671b1b4ffcf37d83bd7622b36c81b2876fe6a1a2 Mon Sep 17 00:00:00 2001 From: Zhang Song Date: Thu, 3 Jul 2025 18:09:17 +0800 Subject: [PATCH 05/51] crimson/os/seastore: introduce logical bucket and demote process Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/CMakeLists.txt | 1 + src/crimson/os/seastore/async_cleaner.h | 19 ++ .../os/seastore/extent_placement_manager.cc | 3 + src/crimson/os/seastore/logical_bucket.cc | 233 ++++++++++++++++++ src/crimson/os/seastore/logical_bucket.h | 29 +++ .../os/seastore/transaction_manager.cc | 11 + src/crimson/os/seastore/transaction_manager.h | 7 + 7 files changed, 303 insertions(+) create mode 100644 src/crimson/os/seastore/logical_bucket.cc create mode 100644 src/crimson/os/seastore/logical_bucket.h diff --git a/src/crimson/os/seastore/CMakeLists.txt b/src/crimson/os/seastore/CMakeLists.txt index 89a2ae0b5a6..dbac365a3b2 100644 --- a/src/crimson/os/seastore/CMakeLists.txt +++ b/src/crimson/os/seastore/CMakeLists.txt @@ -9,6 +9,7 @@ set(crimson_seastore_srcs transaction_manager.cc cache.cc extent_pinboard.cc + logical_bucket.cc root_block.cc lba_manager.cc async_cleaner.cc diff --git a/src/crimson/os/seastore/async_cleaner.h b/src/crimson/os/seastore/async_cleaner.h index b45ad20e373..7c54d7d33ae 100644 --- a/src/crimson/os/seastore/async_cleaner.h +++ b/src/crimson/os/seastore/async_cleaner.h @@ -367,6 +367,25 @@ public: Transaction &t, CachedExtentRef extent) = 0; + /** + * demote_region + * + * Demote the logical extents promoted from the slower device and evict + * the extents to the cold tier under the given laddr prefix. + */ + struct demote_region_res_t { + std::size_t demoted_size = 0; + std::size_t evicted_size = 0; + bool complete = false; + }; + using demote_region_iertr = base_iertr; + using demote_region_ret = demote_region_iertr::future< + demote_region_res_t>; + virtual demote_region_ret demote_region( + Transaction &t, + laddr_t prefix, + std::size_t max_proceed_size) = 0; + /** * get_extents_if_live * diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc index c6110a28de3..822a0a3a375 100644 --- a/src/crimson/os/seastore/extent_placement_manager.cc +++ b/src/crimson/os/seastore/extent_placement_manager.cc @@ -607,6 +607,9 @@ ExtentPlacementManager::mount_ret ExtentPlacementManager::BackgroundProcess::mou trimmer->reset(); stats = {}; register_metrics(store_index); + if (logical_bucket) { + logical_bucket->mount(store_index); + } DEBUG("mounting main cleaner"); co_await main_cleaner->mount(); if (has_cold_tier()) { diff --git a/src/crimson/os/seastore/logical_bucket.cc b/src/crimson/os/seastore/logical_bucket.cc new file mode 100644 index 00000000000..36a37abffb9 --- /dev/null +++ b/src/crimson/os/seastore/logical_bucket.cc @@ -0,0 +1,233 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "seastar/core/metrics.hh" + +#include "crimson/common/coroutine.h" +#include "crimson/os/seastore/logging.h" +#include "crimson/os/seastore/logical_bucket.h" +#include "crimson/os/seastore/transaction_manager.h" + +#include + +namespace crimson::os::seastore { + +SET_SUBSYS(seastore_cache); + +struct bucket_t { + laddr_t laddr = L_ADDR_NULL; + bool demoting = false; +}; + +class LogicalBucketCache : public LogicalBucket { +public: + LogicalBucketCache(std::size_t memory_capacity, + std::size_t demote_size_per_cycle) + : memory_capacity(memory_capacity), + demote_size_per_cycle(demote_size_per_cycle) { + LOG_PREFIX(LogicalBucketCache); + INFO("init memory_capacity={}, demote_size_per_cycle={}", + memory_capacity, demote_size_per_cycle); + } + + void mount(store_index_t store_index) final { + register_metrics(store_index); + } + + ~LogicalBucketCache() { + clear(); + } + + void move_to_top(laddr_t laddr) final { + LOG_PREFIX(LogicalBucketCache::move_to_top); + assert(laddr != L_ADDR_NULL); + assert(laddr == laddr.get_object_prefix()); + auto iter = index.find(laddr); + if (iter != index.end()) { + TRACE("find bucket: {}", iter->first); + iter->second->demoting = false; + lru.splice(lru.end(), lru, iter->second); + } else { + TRACE("create bucket: {}", laddr); + index[laddr] = lru.emplace(lru.end(), laddr); + } + } + + bool is_cached(laddr_t laddr) final { + assert(laddr != L_ADDR_NULL); + assert(laddr == laddr.get_object_prefix()); + return index.contains(laddr); + } + + void clear() final { + index.clear(); + lru.clear(); + } + + void set_background_callback(BackgroundListener *l) final { + listener = l; + } + + void set_extent_callback(ExtentCallbackInterface *cb) final { + ecb = cb; + } + + bool could_demote() const final { + return !lru.empty(); + } + + bool should_demote() const final { + // lru element: laddr_t + pointer * 2 + // index element: laddr_t + lru iterator(void*) + auto element_size = (sizeof(laddr_t) * 2 + sizeof(void*) * 3); + return element_size * lru.size() > memory_capacity; + } + + using run_demote_iertr = base_iertr; + using run_demote_ret = run_demote_iertr::future<>; + run_demote_ret run_demote(Transaction &t) { + LOG_PREFIX(LogicalBucketCache::run_demote); + std::vector pending_buckets; + std::vector completed_buckets; + ceph_assert(pending_buckets_target > 0); + ceph_assert(!lru.empty()); + for (auto &b : lru) { + if (pending_buckets.size() == (uint32_t)pending_buckets_target) { + break; + } + b.demoting = true; + pending_buckets.push_back(b.laddr); + } + + DEBUGT("start demote {} buckets", t, pending_buckets.size()); + std::size_t demoted_size = 0; + std::size_t evicted_size = 0; + for (auto &bucket : pending_buckets) { + TRACET("start demote {}", t, bucket); + auto res = co_await ecb->demote_region( + t, + bucket, + demote_size_per_cycle - demoted_size - evicted_size); + + TRACET("demote_size: {}, evicted_size: {}, complete: {}", + t, res.demoted_size, res.evicted_size, res.complete); + demoted_size += res.demoted_size; + evicted_size += res.evicted_size; + if (res.complete) { + completed_buckets.push_back(bucket); + } + if (demoted_size + evicted_size >= demote_size_per_cycle) { + break; + } + } + + co_await ecb->submit_transaction_direct(t); + + DEBUGT("finish demoting {} buckets with {} bytes evicted and {} bytes demoted", + t, completed_buckets.size(), evicted_size, demoted_size); + stat.demoted_bucket_count += completed_buckets.size(); + stat.demoted_size += demoted_size; + stat.evicted_size += evicted_size; + for (auto &p : completed_buckets) { + maybe_remove(p); + } + auto old_count = pending_buckets_target; + if (demoted_size != 0 && !completed_buckets.empty()) { + auto demote_ratio = (double)(demoted_size + evicted_size) / + (double)completed_buckets.size(); + assert(!std::isnan(demote_ratio)); + pending_buckets_target = (demote_size_per_cycle / demote_ratio) + 1; + } + DEBUGT("update init buckets count {} -> {}", + t, old_count, pending_buckets_target); + co_return; + } + + seastar::future<> demote() final { + return repeat_eagain([this] { + return ecb->with_transaction_intr( + Transaction::src_t::DEMOTE, + "demote", cache_hint_t::get_nocache(), + [this](auto &t) { + return run_demote(t); + }); + }).handle_error(crimson::ct_error::assert_all("impossible")); + } + +private: + using laddr_lru_t = std::list; + laddr_lru_t lru; + boost::unordered_flat_map index; + int32_t pending_buckets_target = 20; + + void maybe_remove(laddr_t laddr) { + LOG_PREFIX(LogicalBucketCache::maybe_remove); + TRACE("laddr: {}", laddr); + assert(laddr != L_ADDR_NULL); + assert(laddr == laddr.get_object_prefix()); + auto iter = index.find(laddr); + if (iter != index.end()) { + if (iter->second->demoting) { + DEBUG("remove bucket: {}", laddr); + lru.erase(iter->second); + index.erase(iter); + } else { + DEBUG("bucket {} not removed, user accesses to the " + "bucket must have happened during the demotion", laddr); + } + } + } + + + void register_metrics(store_index_t store_index) { + namespace sm = seastar::metrics; + metrics.add_group( + "cache", + { + sm::make_gauge( + "non_volatile_cache_buckets_count", + [this] { return lru.size(); }, + sm::description("the count of laddr bucket used by non volatile cache"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), + sm::make_counter( + "non_volatile_cache_evicted_size", + [this] { return stat.evicted_size; }, + sm::description("total bytes of extents evicted by non volatile cache"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), + sm::make_counter( + "non_volatile_cache_demoted_size", + [this] { return stat.demoted_size; }, + sm::description("total bytes of extents demoted by non volatile cache"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), + sm::make_counter( + "non_volatile_cache_demoted_bucket_count", + [this] { return stat.demoted_bucket_count; }, + sm::description("the count of laddr bucket demoted by non volatile cache"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), + }); + } + + struct { + uint64_t evicted_size = 0; + uint64_t demoted_size = 0; + uint64_t demoted_bucket_count = 0; + } stat; + + seastar::metrics::metric_group metrics; + + const std::size_t memory_capacity = 0; + const std::size_t demote_size_per_cycle = 0; + + ExtentCallbackInterface *ecb = nullptr; + BackgroundListener *listener = nullptr; +}; + +LogicalBucketRef create_logical_bucket( + std::size_t memory_capacity, + std::size_t demote_size_per_cycle) +{ + return std::make_unique( + memory_capacity, demote_size_per_cycle); +} + +} diff --git a/src/crimson/os/seastore/logical_bucket.h b/src/crimson/os/seastore/logical_bucket.h new file mode 100644 index 00000000000..9a0ef1e11ee --- /dev/null +++ b/src/crimson/os/seastore/logical_bucket.h @@ -0,0 +1,29 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab expandtab + +#pragma once + +#include "crimson/common/smp_helpers.h" +#include "crimson/os/seastore/seastore_types.h" + +namespace crimson::os::seastore { +class BackgroundListener; +class ExtentCallbackInterface; + +struct LogicalBucket { + virtual ~LogicalBucket() = default; + virtual void move_to_top(laddr_t laddr) = 0; + virtual bool is_cached(laddr_t laddr) = 0; + virtual void clear() = 0; + virtual void set_background_callback(BackgroundListener *listener) = 0; + virtual void set_extent_callback(ExtentCallbackInterface *cb) = 0; + virtual bool could_demote() const = 0; + virtual bool should_demote() const = 0; + virtual void mount(store_index_t store_index) = 0; + virtual seastar::future<> demote() = 0; +}; +using LogicalBucketRef = std::unique_ptr; +LogicalBucketRef create_logical_bucket( + std::size_t memory_capacity, + std::size_t demote_size_per_cycle); +} diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index ab0c043b4de..16d0447f5da 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -1103,6 +1103,17 @@ TransactionManager::promote_extent( return rewrite_extent_iertr::make_ready_future(); } +TransactionManager::demote_region_ret +TransactionManager::demote_region( + Transaction &t, + laddr_t start, + loffset_t max_proceed_size) +{ + // TODO + return demote_region_iertr::make_ready_future( + demote_region_res_t{0, false}); +} + TransactionManager::get_extents_if_live_ret TransactionManager::get_extents_if_live( Transaction &t, diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index 82a51cb1e57..e87a4f19531 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -905,6 +905,13 @@ public: Transaction &t, CachedExtentRef extent); + using ExtentCallbackInterface::demote_region_res_t; + using ExtentCallbackInterface::demote_region_ret; + demote_region_ret demote_region( + Transaction &t, + laddr_t start, + std::size_t max_proceed_size) final; + using ExtentCallbackInterface::get_extents_if_live_ret; get_extents_if_live_ret get_extents_if_live( Transaction &t, From ad7469ca5974e7f81afb4da8c9099ee7c4662c1d Mon Sep 17 00:00:00 2001 From: Zhang Song Date: Wed, 3 Sep 2025 16:01:35 +0800 Subject: [PATCH 06/51] crimson/os/seastore/lba: introduce shadow paddr Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/btree/btree_types.cc | 10 ++++-- src/crimson/os/seastore/btree/btree_types.h | 9 ++++-- .../os/seastore/lba/btree_lba_manager.cc | 27 +++++++++++++--- .../os/seastore/lba/btree_lba_manager.h | 5 ++- src/crimson/os/seastore/lba/lba_btree_node.h | 18 ++++++++--- src/crimson/os/seastore/lba_manager.h | 2 +- src/crimson/os/seastore/lba_mapping.h | 12 +++++++ .../os/seastore/transaction_manager.cc | 27 ++++++++++++++++ src/crimson/os/seastore/transaction_manager.h | 31 +++++++++++++++++++ .../seastore/test_btree_lba_manager.cc | 4 +-- .../seastore/test_transaction_manager.cc | 6 ++-- 11 files changed, 130 insertions(+), 21 deletions(-) diff --git a/src/crimson/os/seastore/btree/btree_types.cc b/src/crimson/os/seastore/btree/btree_types.cc index bc5d0cafab7..56f4382ed9d 100644 --- a/src/crimson/os/seastore/btree/btree_types.cc +++ b/src/crimson/os/seastore/btree/btree_types.cc @@ -12,9 +12,13 @@ namespace lba { std::ostream& operator<<(std::ostream& out, const lba_map_val_t& v) { - return out << "lba_map_val_t(" - << v.pladdr - << "~0x" << std::hex << v.len + out << "lba_map_val_t("; + if (v.shadow_paddr != P_ADDR_NULL) { + out << '[' << v.pladdr << ',' << v.shadow_paddr << ']'; + } else { + out << v.pladdr; + } + return out << "~0x" << std::hex << v.len << ", type=" << (extent_types_t)v.type << ", checksum=0x" << v.checksum << ", refcount=" << std::dec << v.refcount diff --git a/src/crimson/os/seastore/btree/btree_types.h b/src/crimson/os/seastore/btree/btree_types.h index ade1e759899..0958e1933e4 100644 --- a/src/crimson/os/seastore/btree/btree_types.h +++ b/src/crimson/os/seastore/btree/btree_types.h @@ -110,6 +110,7 @@ struct lba_map_val_t { extent_len_t len = 0; ///< length of mapping pladdr_t pladdr; ///< direct addr of mapping or // laddr of a direct lba mapping(see btree_lba_manager.h) + paddr_t shadow_paddr; extent_ref_count_t refcount = 0; ///< refcount checksum_t checksum = 0; ///< checksum of original block written at paddr (TODO) extent_types_t type = extent_types_t::NONE; @@ -118,11 +119,12 @@ struct lba_map_val_t { lba_map_val_t( extent_len_t len, pladdr_t pladdr, + paddr_t shadow_paddr, extent_ref_count_t refcount, checksum_t checksum, extent_types_t type) - : len(len), pladdr(pladdr), refcount(refcount), - checksum(checksum), type(type) {} + : len(len), pladdr(pladdr), shadow_paddr(shadow_paddr), + refcount(refcount), checksum(checksum), type(type) {} bool operator==(const lba_map_val_t&) const = default; }; @@ -136,6 +138,7 @@ std::ostream& operator<<(std::ostream& out, const lba_map_val_t&); struct __attribute__((packed)) lba_map_val_le_t { extent_len_le_t len = init_extent_len_le(0); pladdr_le_t pladdr; + paddr_le_t shadow_paddr; extent_ref_count_le_t refcount{0}; checksum_le_t checksum{0}; extent_types_le_t type{EXTENT_TYPES_MAX}; @@ -145,6 +148,7 @@ struct __attribute__((packed)) lba_map_val_le_t { explicit lba_map_val_le_t(const lba_map_val_t &val) : len(init_extent_len_le(val.len)), pladdr(pladdr_le_t(val.pladdr)), + shadow_paddr(paddr_le_t(val.shadow_paddr)), refcount(val.refcount), checksum(val.checksum), type(static_cast(val.type)) {} @@ -153,6 +157,7 @@ struct __attribute__((packed)) lba_map_val_le_t { return lba_map_val_t{ len, pladdr, + shadow_paddr, refcount, checksum, static_cast(type)}; diff --git a/src/crimson/os/seastore/lba/btree_lba_manager.cc b/src/crimson/os/seastore/lba/btree_lba_manager.cc index 480d40f36db..8ce285943f8 100644 --- a/src/crimson/os/seastore/lba/btree_lba_manager.cc +++ b/src/crimson/os/seastore/lba/btree_lba_manager.cc @@ -300,6 +300,7 @@ BtreeLBAManager::resolve_indirect_cursor( const LBACursor &indirect_cursor) { ceph_assert(indirect_cursor.is_indirect()); + ceph_assert(!indirect_cursor.has_shadow_paddr()); return get_cursors( c, btree, @@ -351,6 +352,7 @@ BtreeLBAManager::reserve_region( lba_map_val_t val{ len, pladdr_t{P_ADDR_ZERO}, + P_ADDR_NULL, EXTENT_DEFAULT_REF_COUNT, 0, type}; @@ -392,6 +394,7 @@ BtreeLBAManager::alloc_extents( lba_map_val_t{ ext->get_length(), pladdr_t{ext->get_paddr()}, + P_ADDR_NULL, EXTENT_DEFAULT_REF_COUNT, ext->get_last_committed_crc(), ext->get_type()}, @@ -451,6 +454,7 @@ BtreeLBAManager::clone_mapping( lba_map_val_t{ len, pladdr_t{inter_key.get_local_clone_id()}, + P_ADDR_NULL, EXTENT_DEFAULT_REF_COUNT, 0, mapping->get_extent_type()}, @@ -880,7 +884,8 @@ BtreeLBAManager::scan_mappings( } ceph_assert((pos.get_key() + pos.get_val().len) > begin); if (pos.get_val().pladdr.is_paddr()) { - f(pos.get_key(), pos.get_val().pladdr.get_paddr(), pos.get_val().len); + f(pos.get_key(), pos.get_val().pladdr.get_paddr(), + pos.get_val().shadow_paddr, pos.get_val().len); } return LBABtree::iterate_repeat_ret_inner( interruptible::ready_future_marker{}, @@ -956,11 +961,19 @@ BtreeLBAManager::update_mapping( assert(!addr.is_null()); lba_map_val_t ret = in; ceph_assert(in.pladdr.is_paddr()); - ceph_assert(in.pladdr.get_paddr() == prev_addr); ceph_assert(in.len == prev_len); - ret.pladdr = addr; - ret.len = len; - ret.checksum = checksum; + if (prev_addr == in.pladdr.get_paddr()) { + ret.pladdr = addr; + ret.len = len; + ret.checksum = checksum; + if (ret.shadow_paddr != P_ADDR_NULL) { + ceph_assert( + addr.get_device_id() != ret.shadow_paddr.get_device_id()); + } + } else { + ceph_assert(in.shadow_paddr == prev_addr); + ret.shadow_paddr = addr; + } return ret; }, &nextent @@ -1314,6 +1327,9 @@ BtreeLBAManager::remap_mappings( } else { auto paddr = val.pladdr.get_paddr(); val.pladdr = paddr + cur_off; + if (val.shadow_paddr != P_ADDR_NULL) { + val.shadow_paddr = val.shadow_paddr.add_offset(cur_off); + } } val.len = remap.len; val.refcount = EXTENT_DEFAULT_REF_COUNT; @@ -1539,6 +1555,7 @@ BtreeLBAManager::move_and_clone_direct_mapping( lba_map_val_t val = in; val.pladdr = ret.dest->get_key().get_local_clone_id(); val.checksum = 0; + val.shadow_paddr = P_ADDR_NULL; return val; }, nullptr diff --git a/src/crimson/os/seastore/lba/btree_lba_manager.h b/src/crimson/os/seastore/lba/btree_lba_manager.h index 8b150d95e25..f402612c820 100644 --- a/src/crimson/os/seastore/lba/btree_lba_manager.h +++ b/src/crimson/os/seastore/lba/btree_lba_manager.h @@ -209,6 +209,7 @@ public: laddr_t dest_laddr, LBACursorRef dest) final { assert(src->is_indirect()); + assert(!src->has_shadow_paddr()); return _move_mapping( t, std::move(src), dest_laddr, std::move(dest), nullptr); } @@ -503,6 +504,7 @@ private: { len, pladdr_t(P_ADDR_ZERO), + P_ADDR_NULL, EXTENT_DEFAULT_REF_COUNT, 0, type @@ -517,6 +519,7 @@ private: { len, pladdr_t(intermediate_key.get_local_clone_id()), + P_ADDR_NULL, EXTENT_DEFAULT_REF_COUNT, 0, // crc will only be used and checked with LBA direct mappings // also see pin_to_extent(_by_type) @@ -533,7 +536,7 @@ private: LogicalChildNode& extent) { return { laddr, - {len, pladdr_t(paddr), refcount, checksum, extent.get_type()}, + {len, pladdr_t(paddr), P_ADDR_NULL, refcount, checksum, extent.get_type()}, &extent }; } diff --git a/src/crimson/os/seastore/lba/lba_btree_node.h b/src/crimson/os/seastore/lba/lba_btree_node.h index 4513afdc4ef..53eb51c9d05 100644 --- a/src/crimson/os/seastore/lba/lba_btree_node.h +++ b/src/crimson/os/seastore/lba/lba_btree_node.h @@ -101,14 +101,14 @@ using LBAInternalNodeRef = LBAInternalNode::Ref; * checksum : ceph_le32[1] 4B * size : ceph_le32[1] 4B * meta : lba_node_meta_le_t[1] 36B - * keys : laddr_le_t[CAPACITY] (106*16)B - * values : lba_map_val_le_t[CAPACITY] (106*21)B - * = 4077B + * keys : laddr_le_t[CAPACITY] (88*16)B + * values : lba_map_val_le_t[CAPACITY] (88*30)B + * = 4092B * * TODO: update FixedKVNodeLayout to handle the above calculation * TODO: the above alignment probably isn't portable without further work */ -constexpr size_t LEAF_NODE_CAPACITY = 106; +constexpr size_t LEAF_NODE_CAPACITY = 88; struct LBALeafNode : FixedKVLeafNode< @@ -480,12 +480,22 @@ struct LBACursor : BtreeCursor { } extent_types_t get_extent_type() const { + assert(iter.get_val().type != extent_types_t::NONE); assert(is_viewable()); assert(!is_end()); assert(iter.get_val().type != extent_types_t::NONE); return iter.get_val().type; } + bool has_shadow_paddr() const { + return iter.get_val().shadow_paddr != P_ADDR_NULL; + } + + paddr_t get_shadow_paddr() const { + assert(has_shadow_paddr()); + return iter.get_val().shadow_paddr; + } + base_iertr::future<> refresh(); private: diff --git a/src/crimson/os/seastore/lba_manager.h b/src/crimson/os/seastore/lba_manager.h index 8826d96a2b4..a537a351418 100644 --- a/src/crimson/os/seastore/lba_manager.h +++ b/src/crimson/os/seastore/lba_manager.h @@ -253,7 +253,7 @@ public: using scan_mappings_iertr = base_iertr; using scan_mappings_ret = scan_mappings_iertr::future<>; using scan_mappings_func_t = std::function< - void(laddr_t, paddr_t, extent_len_t)>; + void(laddr_t, paddr_t, paddr_t, extent_len_t)>; virtual scan_mappings_ret scan_mappings( Transaction &t, laddr_t begin, diff --git a/src/crimson/os/seastore/lba_mapping.h b/src/crimson/os/seastore/lba_mapping.h index 395e51e9534..d72198ef101 100644 --- a/src/crimson/os/seastore/lba_mapping.h +++ b/src/crimson/os/seastore/lba_mapping.h @@ -132,6 +132,18 @@ public: return direct_cursor->get_length(); } + bool has_shadow_val() const { + assert(is_linked_direct()); + assert(!direct_cursor->is_end()); + return direct_cursor->has_shadow_paddr(); + } + + paddr_t get_shadow_val() const { + assert(is_linked_direct()); + assert(!direct_cursor->is_end()); + return direct_cursor->get_shadow_paddr(); + } + checksum_t get_checksum() const { assert(is_linked_direct()); assert(!direct_cursor->is_end()); diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index 16d0447f5da..097d467640e 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -229,12 +229,33 @@ TransactionManager::ref_ret TransactionManager::remove( extent_ref_count_t refcount = cursor->get_refcount(); auto laddr = cursor->get_laddr(); auto length = cursor->get_length(); + paddr_t shadow_addr = P_ADDR_NULL; + if (cursor->has_shadow_paddr()) { + shadow_addr = cursor->get_shadow_paddr(); + } assert(refcount > 0); --refcount; co_await lba_manager->update_mapping_refcount( t, std::move(cursor), -1); if (refcount == 0) { cache->retire_extent(t, ref); + if (shadow_addr != P_ADDR_NULL) { + if (auto shadow = ref->get_shadow(); shadow) { + cache->retire_extent(t, shadow); + } else { + auto laddr = ref->get_laddr(); + cache->retire_absent_extent_addr_by_type( + t, laddr, shadow_addr, length, ref->get_type(), + [laddr](auto &extent) { + auto lextent = extent.template cast(); + assert(extent.is_logical()); + assert(!lextent->has_laddr()); + assert(!extent.has_been_invalidated()); + lextent->set_laddr(laddr); + extent.set_shadow_extent(true); + }); + } + } } DEBUGT("removed {}~0x{:x} refcount={} -- {}", t, laddr, length, @@ -317,6 +338,12 @@ TransactionManager::_remove( } ); } + if (mapping.has_shadow_val()) { + cache->retire_absent_extent_addr( + t, mapping.get_intermediate_base(), + mapping.get_shadow_val(), + mapping.get_intermediate_length()); + } } LBACursorRef indirect_cursor; diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index e87a4f19531..3fb426ffd08 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -1103,6 +1103,7 @@ public: if (!mapping.is_indirect() && mapping.is_zero_reserved()) { SUBDEBUGT(seastore_tm, "zero reserved, mapping {}, {} remaps", t, mapping, remaps); + ceph_assert(!mapping.has_shadow_val()); //TODO: drop this assert assert(mapping.get_extent_type() == extent_types_t::OBJECT_DATA_BLOCK); auto type = mapping.get_extent_type(); @@ -1514,6 +1515,20 @@ private: lextent->set_laddr(laddr); } ); + if (pin.has_shadow_val()) { + cache->retire_absent_extent_addr_by_type( + t, pin.get_key(), pin.get_shadow_val(), + original_len, pin.get_extent_type(), + [laddr](auto &extent) { + auto lextent = extent.template cast(); + assert(extent.is_logical()); + assert(!lextent->has_laddr()); + assert(!extent.has_been_invalidated()); + lextent->set_laddr(laddr); + } + ); + } + } } @@ -1543,6 +1558,11 @@ private: auto remap_len = remap.len; auto remap_laddr = (original_laddr + remap_offset).checked_to_laddr(); auto remap_paddr = pin.get_val().add_offset(remap_offset); + auto shadow_paddr = P_ADDR_NULL; + if (pin.has_shadow_val()) { + assert(pin.get_shadow_val() != P_ADDR_NULL); + shadow_paddr = pin.get_shadow_val().add_offset(remap_offset); + } SUBDEBUGT(seastore_tm, "remap direct pin into {}~0x{:x} {} ...", t, remap_laddr, remap_len, remap_paddr); ceph_assert(remap_len < original_len); @@ -1557,6 +1577,17 @@ private: remap_offset, remap_len, original_bptr); + if (shadow_paddr != P_ADDR_NULL) { + SUBTRACET(seastore_tm, "remap shadow {}", t, shadow_paddr); + auto cold_ext = cache->alloc_remapped_extent( + t, + remap_laddr, + shadow_paddr, + remap_offset, + remap_len, + std::nullopt); + boost::ignore_unused(cold_ext); + } // user must initialize the logical extent themselves. remapped_extent->set_seen_by_users(); remap.extent = remapped_extent.get(); diff --git a/src/test/crimson/seastore/test_btree_lba_manager.cc b/src/test/crimson/seastore/test_btree_lba_manager.cc index 6d02ec53c2e..4fb774930e5 100644 --- a/src/test/crimson/seastore/test_btree_lba_manager.cc +++ b/src/test/crimson/seastore/test_btree_lba_manager.cc @@ -313,7 +313,7 @@ struct lba_btree_test : btree_test_base { } static auto get_map_val(extent_len_t len, extent_types_t type) { - return lba_map_val_t{0, (pladdr_t)P_ADDR_NULL, len, 0, type}; + return lba_map_val_t{0, (pladdr_t)P_ADDR_NULL, P_ADDR_NULL, len, 0, type}; } device_off_t next_off = 0; @@ -716,7 +716,7 @@ struct btree_lba_manager_test : btree_test_base { *t.t, L_ADDR_MIN, L_ADDR_MAX, - [iter=t.mappings.begin(), &t](auto l, auto p, auto len) mutable { + [iter=t.mappings.begin(), &t](auto l, auto p, auto s, auto len) mutable { EXPECT_NE(iter, t.mappings.end()); EXPECT_EQ(l, iter->first); EXPECT_EQ(p, iter->second.addr); diff --git a/src/test/crimson/seastore/test_transaction_manager.cc b/src/test/crimson/seastore/test_transaction_manager.cc index 52a734ceec1..dd7ef23c37d 100644 --- a/src/test/crimson/seastore/test_transaction_manager.cc +++ b/src/test/crimson/seastore/test_transaction_manager.cc @@ -806,7 +806,7 @@ struct transaction_manager_test_t : t, get_laddr_hint(0), L_ADDR_MAX, - [iter=overlay.begin(), &overlay](auto l, auto p, auto len) mutable { + [iter=overlay.begin(), &overlay](auto l, auto p, auto s, auto len) mutable { EXPECT_NE(iter, overlay.end()); logger().debug( "check_mappings: scan {}", @@ -1901,10 +1901,10 @@ TEST_P(tm_random_block_device_test_t, scatter_allocation) laddr_t ADDR = get_laddr_hint(0xFF * 4096); epm->prefill_fragmented_devices(); auto t = create_transaction(); - for (int i = 0; i < 1974; i++) { + for (int i = 0; i < 1958; i++) { auto extents = alloc_extents(t, (ADDR + i * 16384).checked_to_laddr(), 16384, 'a'); } - alloc_extents_deemed_fail(t, (ADDR + 1974 * 16384).checked_to_laddr(), 16384, 'a'); + alloc_extents_deemed_fail(t, (ADDR + 1958 * 16384).checked_to_laddr(), 16384, 'a'); check_mappings(t); check(); submit_transaction(std::move(t)); From c3e971a84c435fcdcf9f3ddefd197c43fc114faa Mon Sep 17 00:00:00 2001 From: Zhang Song Date: Wed, 3 Sep 2025 16:02:01 +0800 Subject: [PATCH 07/51] crimson/os/seastore: implement promote extent Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- .../os/seastore/lba/btree_lba_manager.cc | 53 +++++++++ .../os/seastore/lba/btree_lba_manager.h | 5 + src/crimson/os/seastore/lba_manager.h | 7 ++ .../os/seastore/transaction_manager.cc | 102 +++++++++++++++++- 4 files changed, 165 insertions(+), 2 deletions(-) diff --git a/src/crimson/os/seastore/lba/btree_lba_manager.cc b/src/crimson/os/seastore/lba/btree_lba_manager.cc index 8ce285943f8..6f49128ae14 100644 --- a/src/crimson/os/seastore/lba/btree_lba_manager.cc +++ b/src/crimson/os/seastore/lba/btree_lba_manager.cc @@ -7,6 +7,7 @@ #include #include "include/buffer.h" +#include "crimson/common/coroutine.h" #include "crimson/os/seastore/lba/btree_lba_manager.h" #include "crimson/os/seastore/lba/lba_btree_node.h" #include "crimson/os/seastore/logging.h" @@ -330,6 +331,58 @@ BtreeLBAManager::lower_bound( co_return iter.get_cursor(c); } +BtreeLBAManager::promote_extent_ret +BtreeLBAManager::promote_extent( + Transaction &t, + LBACursor &cursor, + std::vector extents) +{ + LOG_PREFIX(BtreeLBAManager::promote_extent); + auto laddr = cursor.get_laddr(); + ceph_assert(!extents.empty()); + ceph_assert(!cursor.is_indirect()); + ceph_assert(laddr == extents.front()->get_laddr()); + DEBUGT("promote cursor {} with {} extents", + t, cursor, extents.size()); + auto c = get_context(t); + auto btree = co_await get_btree(c); + auto iter = btree.make_partial_iter(c, cursor); + auto orig_val = iter.get_val(); + if (extents.size() == 1) { + auto new_val = orig_val; + ceph_assert(new_val.pladdr.is_paddr()); + new_val.shadow_paddr = new_val.pladdr.get_paddr(); + auto extent = extents.front().get(); + auto paddr = extent->get_paddr(); + new_val.pladdr = pladdr_t(paddr); + TRACET("promote {} from {} to {}", + t, iter.get_key(), new_val.shadow_paddr, paddr); + assert(extent->is_pending()); + assert(!extent->has_parent_tracker()); + iter = btree.update(c, iter, new_val, extent); + assert(extent->has_parent_tracker()); + } else { + auto insert_iter = co_await btree.remove(c, std::move(iter)); + for (auto &extent : extents) { + auto offset = extent->get_laddr().get_byte_distance(laddr); + auto new_val = orig_val; + new_val.shadow_paddr = orig_val.pladdr.get_paddr().add_offset(offset); + new_val.pladdr = pladdr_t(extent->get_paddr()); + new_val.len = extent->get_length(); + new_val.checksum = extent->get_last_committed_crc(); + TRACET("insert promoted cursor {} {}", + c.trans, extent->get_laddr(), new_val); + assert(!extent->has_parent_tracker()); + auto [iter, inserted] = co_await btree.insert( + c, std::move(insert_iter), extent->get_laddr(), new_val, extent.get()); + ceph_assert(inserted); + assert(extent->has_parent_tracker()); + insert_iter = co_await iter.next(c); + } + } + co_return; +} + /** * reserve_region: insert a zero-mapping (P_ADDR_ZERO) at the specified laddr. * Uses the cursor as a btree insertion hint. The reserved_ptr child pointer diff --git a/src/crimson/os/seastore/lba/btree_lba_manager.h b/src/crimson/os/seastore/lba/btree_lba_manager.h index f402612c820..e109c3adfa2 100644 --- a/src/crimson/os/seastore/lba/btree_lba_manager.h +++ b/src/crimson/os/seastore/lba/btree_lba_manager.h @@ -148,6 +148,11 @@ public: Transaction &t, laddr_t laddr) final; + promote_extent_ret promote_extent( + Transaction &t, + LBACursor &cursor, + std::vector extents) final; + // --------------------------------------------------------------------------- // Allocation operations - insert new mappings into the LBA tree. // --------------------------------------------------------------------------- diff --git a/src/crimson/os/seastore/lba_manager.h b/src/crimson/os/seastore/lba_manager.h index a537a351418..59f6f251176 100644 --- a/src/crimson/os/seastore/lba_manager.h +++ b/src/crimson/os/seastore/lba_manager.h @@ -167,6 +167,13 @@ public: LBACursorRef dest, LogicalChildNode &extent) = 0; + using promote_extent_iertr = base_iertr; + using promote_extent_ret = promote_extent_iertr::future<>; + virtual promote_extent_ret promote_extent( + Transaction &t, + LBACursor &cursor, + std::vector extents) = 0; + virtual alloc_extent_ret reserve_region( Transaction &t, laddr_hint_t hint, diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index 097d467640e..b247b8cf060 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -1126,8 +1126,106 @@ TransactionManager::promote_extent( Transaction &t, CachedExtentRef extent) { - // TODO - return rewrite_extent_iertr::make_ready_future(); + LOG_PREFIX(TransactionManager::promote_extent); + assert(epm->is_cold_device(extent->get_paddr().get_device_id())); + DEBUGT("promote extent: {}", t, *extent); + ceph_assert(extent->is_logical()); + + std::vector promoted_extents; + auto orig_ext = extent->cast(); + // fill extent if it's not fully loaded + if (!extent->is_fully_loaded()) { + ceph_assert(extent->get_type() == extent_types_t::OBJECT_DATA_BLOCK); + extent = co_await cache->read_extent_maybe_partial( + t, extent->cast(), 0, extent->get_length()); + } + + cache->retire_extent(t, extent); + + if (get_extent_category(extent->get_type()) == data_category_t::DATA) { + auto promoted_raw_extents = cache->alloc_new_data_extents_by_type( + t, + orig_ext->get_type(), + orig_ext->get_length(), + placement_hint_t::HOT, + INIT_GENERATION); + + promoted_extents.reserve(promoted_raw_extents.size()); + + extent_len_t offset = 0; + auto orig_laddr = orig_ext->get_laddr(); + auto orig_paddr = orig_ext->get_paddr(); + auto orig_length = orig_ext->get_length(); + t.force_rewrite_conflict |= (promoted_raw_extents.size() > 1); + for (auto &extent : promoted_raw_extents) { + auto slice_laddr = (orig_laddr + offset).checked_to_laddr(); + auto slice_length = extent->get_length(); + extent->rewrite(t, *orig_ext, offset); + assert(!extent->get_paddr().is_absolute() || + !cache->is_on_cold_tier(extent->get_paddr())); + + auto lext = extent->cast(); + lext->set_laddr(slice_laddr); + //TODO: this memory copy should be saved + orig_ext->get_bptr().copy_out( + offset, slice_length, lext->get_bptr().c_str()); + lext->set_last_committed_crc(lext->calc_crc32c()); + + promoted_extents.push_back(lext); + + auto remapped_cold_extent = cache->alloc_remapped_extent_by_type( + t, + orig_ext->get_type(), + slice_laddr, + orig_paddr.add_offset(offset), + offset, + slice_length, + std::nullopt); + remapped_cold_extent->set_shadow_extent(true); + + offset += slice_length; + } + ceph_assert(offset == orig_length); + } else { + auto promoted_extent = cache->alloc_new_non_data_extent_by_type( + t, + orig_ext->get_type(), + orig_ext->get_length(), + placement_hint_t::HOT, + INIT_GENERATION); + auto lext = promoted_extent->cast(); + lext->set_laddr(orig_ext->get_laddr()); + lext->rewrite(t, *orig_ext, 0); + assert(!extent->get_paddr().is_absolute() || + !cache->is_on_cold_tier(lext->get_paddr())); + //TODO: this memory copy should be saved + orig_ext->get_bptr().copy_out( + 0, + orig_ext->get_length(), + lext->get_bptr().c_str()); + promoted_extents.push_back(lext); + auto remapped_cold_extent = cache->alloc_remapped_extent_by_type( + t, + orig_ext->get_type(), + orig_ext->get_laddr(), + orig_ext->get_paddr(), + 0, + orig_ext->get_length(), + std::nullopt); + boost::ignore_unused(remapped_cold_extent); + + remapped_cold_extent->set_shadow_extent(true); + } + + auto cursor = co_await lba_manager->get_cursor( + t, *orig_ext + ).handle_error_interruptible( + promote_extent_iertr::pass_further(), + crimson::ct_error::assert_all("invalid error")); + auto mapping = co_await resolve_cursor_to_mapping(t, std::move(cursor)); + assert(!mapping.is_indirect()); + co_return co_await lba_manager->promote_extent( + t, *mapping.direct_cursor, std::move(promoted_extents)); } TransactionManager::demote_region_ret From 1af4a9779275f489bd787acf4c43f3873f1c7881 Mon Sep 17 00:00:00 2001 From: Zhang Song Date: Wed, 3 Sep 2025 16:02:29 +0800 Subject: [PATCH 08/51] crimson/os/seastore: implement demote region Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- .../os/seastore/extent_placement_manager.h | 8 +- .../os/seastore/lba/btree_lba_manager.cc | 44 +++++++++ .../os/seastore/lba/btree_lba_manager.h | 9 ++ src/crimson/os/seastore/lba_manager.h | 14 +++ .../os/seastore/transaction_manager.cc | 97 ++++++++++++++++++- src/crimson/os/seastore/transaction_manager.h | 4 + 6 files changed, 172 insertions(+), 4 deletions(-) diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index aba38a43e1a..2a48510a17f 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -179,7 +179,9 @@ public: if (!extent->is_stable_dirty()) { return false; } - assert(t.get_src() == transaction_type_t::TRIM_DIRTY); + assert((t.get_src() == transaction_type_t::TRIM_DIRTY) || + (t.get_src() == transaction_type_t::DEMOTE) || + (t.get_src() == transaction_type_t::PROMOTE)); ceph_assert_always(is_root_type(extent->get_type()) || extent->get_paddr().is_absolute()); return crimson::os::seastore::can_inplace_rewrite(extent->get_type()); @@ -616,6 +618,10 @@ public: return !devices_by_id[addr.get_device_id()]->is_end_to_end_data_protection(); } + rewrite_gen_t get_max_hot_gen() const { + return hot_tier_generations - 1; + } + private: rewrite_gen_t adjust_generation( data_category_t category, diff --git a/src/crimson/os/seastore/lba/btree_lba_manager.cc b/src/crimson/os/seastore/lba/btree_lba_manager.cc index 6f49128ae14..6120eeb3749 100644 --- a/src/crimson/os/seastore/lba/btree_lba_manager.cc +++ b/src/crimson/os/seastore/lba/btree_lba_manager.cc @@ -331,6 +331,22 @@ BtreeLBAManager::lower_bound( co_return iter.get_cursor(c); } +BtreeLBAManager::upper_bound_right_ret +BtreeLBAManager::upper_bound_right( + Transaction &t, + laddr_t laddr) +{ + auto c = get_context(t); + auto btree = co_await get_btree(c); + auto iter = co_await btree.upper_bound_right(c, laddr); + if (iter.is_end()) { + co_await upper_bound_right_iertr::future( + crimson::ct_error::enoent::make()); + } + assert(iter.get_key() >= laddr); + co_return iter.get_cursor(c); +} + BtreeLBAManager::promote_extent_ret BtreeLBAManager::promote_extent( Transaction &t, @@ -382,6 +398,34 @@ BtreeLBAManager::promote_extent( } co_return; } +BtreeLBAManager::demote_extent_ret +BtreeLBAManager::demote_extent( + Transaction &t, + LBACursor &cursor, + LogicalChildNode &extent) +{ + assert(cursor.is_viewable()); + assert(!cursor.is_end()); + assert(!cursor.is_indirect()); + assert(cursor.has_shadow_paddr()); + auto c = get_context(t); + auto btree = co_await get_btree(c); + auto ret = co_await _update_mapping( + t, + cursor, + [&extent](lba_map_val_t val) { + assert(val.pladdr.is_paddr()); + assert(val.shadow_paddr == extent.get_paddr()); + val.pladdr = pladdr_t(val.shadow_paddr); + val.shadow_paddr = P_ADDR_NULL; + return val; + }, + &extent + ).handle_error_interruptible( + demote_extent_iertr::pass_further{}, + crimson::ct_error::assert_all("unexpected enoent")); + co_return ret; +} /** * reserve_region: insert a zero-mapping (P_ADDR_ZERO) at the specified laddr. diff --git a/src/crimson/os/seastore/lba/btree_lba_manager.h b/src/crimson/os/seastore/lba/btree_lba_manager.h index e109c3adfa2..c78289bbf41 100644 --- a/src/crimson/os/seastore/lba/btree_lba_manager.h +++ b/src/crimson/os/seastore/lba/btree_lba_manager.h @@ -148,11 +148,20 @@ public: Transaction &t, laddr_t laddr) final; + upper_bound_right_ret upper_bound_right( + Transaction &t, + laddr_t laddr) final; + promote_extent_ret promote_extent( Transaction &t, LBACursor &cursor, std::vector extents) final; + demote_extent_ret demote_extent( + Transaction &t, + LBACursor &cursor, + LogicalChildNode &extent) final; + // --------------------------------------------------------------------------- // Allocation operations - insert new mappings into the LBA tree. // --------------------------------------------------------------------------- diff --git a/src/crimson/os/seastore/lba_manager.h b/src/crimson/os/seastore/lba_manager.h index 59f6f251176..39ddbed6c0d 100644 --- a/src/crimson/os/seastore/lba_manager.h +++ b/src/crimson/os/seastore/lba_manager.h @@ -60,6 +60,13 @@ public: Transaction &t, laddr_t laddr) = 0; + using upper_bound_right_iertr = base_iertr::extend< + crimson::ct_error::enoent>; + using upper_bound_right_ret = upper_bound_right_iertr::future; + virtual upper_bound_right_ret upper_bound_right( + Transaction &t, + laddr_t laddr) = 0; + #ifdef UNIT_TESTS_BUILT using get_end_mapping_iertr = base_iertr; using get_end_mapping_ret = get_end_mapping_iertr::future; @@ -174,6 +181,13 @@ public: LBACursor &cursor, std::vector extents) = 0; + using demote_extent_iertr = base_iertr; + using demote_extent_ret = demote_extent_iertr::future; + virtual demote_extent_ret demote_extent( + Transaction &t, + LBACursor &cursor, + LogicalChildNode &extent) = 0; + virtual alloc_extent_ret reserve_region( Transaction &t, laddr_hint_t hint, diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index b247b8cf060..2454c7d1cc9 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -502,6 +502,57 @@ TransactionManager::relocate_logical_extent( } } +base_iertr::future +TransactionManager::relocate_shadow_extent( + Transaction &t, LBAMapping mapping) +{ + LOG_PREFIX(TransactionManager::relocate_shadow_extent); + SUBDEBUGT(seastore_tm, "relocate {}", t, mapping); + assert(mapping.has_shadow_val()); + assert(!mapping.is_zero_reserved()); + assert(mapping.is_viewable()); + assert(!mapping.is_indirect()); + auto v = get_extent_if_linked(t, *mapping.direct_cursor); + CachedExtentRef extent; + auto laddr = mapping.get_intermediate_base(); + if (!v.has_child()) { + auto &child_pos = v.get_child_pos(); + extent = cache->retire_absent_extent_addr_by_type( + t, + laddr, + mapping.get_val(), + mapping.get_length(), + mapping.get_extent_type(), + [laddr, &child_pos](auto &extent) { + auto lextent = extent.template cast(); + assert(extent.is_logical()); + assert(!lextent->has_laddr()); + assert(!extent.has_been_invalidated()); + child_pos.link_child(lextent.get()); + lextent->set_laddr(laddr); + } + ); + } else { + auto extent = co_await std::move(v.get_child_fut()); + cache->retire_extent(t, extent); + } + auto shadow_paddr = mapping.get_shadow_val(); + std::ignore = cache->retire_absent_extent_addr_by_type( + t, laddr, shadow_paddr, mapping.get_length(), mapping.get_extent_type(), + [laddr](auto &ext) { + auto lextent = ext.template cast(); + assert(ext.is_logical()); + assert(!lextent->has_laddr()); + assert(!ext.has_been_invalidated()); + lextent->set_laddr(laddr); + } + ); + co_return cache->alloc_remapped_extent_by_type( + t, mapping.get_extent_type(), laddr, + mapping.get_shadow_val(), 0, mapping.get_length(), std::nullopt + )->cast(); +} + TransactionManager::submit_transaction_iertr::future<> TransactionManager::submit_transaction( Transaction &t) @@ -1234,9 +1285,49 @@ TransactionManager::demote_region( laddr_t start, loffset_t max_proceed_size) { - // TODO - return demote_region_iertr::make_ready_future( - demote_region_res_t{0, false}); + LOG_PREFIX(TransactionManager::demote_region); + auto prefix = start.get_object_prefix(); + DEBUGT("start demote {}", t, prefix); + auto cursor = co_await lba_manager->upper_bound_right( + t, start + ).handle_error_interruptible( + demote_region_iertr::pass_further{}, + crimson::ct_error::assert_all("unexpected enoent")); + auto it = co_await resolve_cursor_to_mapping(t, std::move(cursor)); + demote_region_res_t ret{0, 0, false}; + while ((ret.demoted_size + ret.evicted_size) < max_proceed_size) { + if (it.is_end() || it.get_key().get_object_prefix() != prefix) { + ret.complete = true; + break; + } + if (it.is_indirect()) { + it = co_await it.next(); + continue; + } + if (it.has_shadow_val()) { + DEBUGT("demote shadow {}", t, it); + auto extent = co_await relocate_shadow_extent(t, it); + ret.demoted_size += extent->get_length(); + auto cursor = co_await lba_manager->demote_extent( + t, *it.direct_cursor, *extent); + auto nit = co_await resolve_cursor_to_mapping(t, std::move(cursor)); + it = co_await nit.next(); + } else if (!it.is_indirect() && !it.is_zero_reserved() && + !epm->is_cold_device(it.get_val().get_device_id())) { + DEBUGT("demote hot {}", t, it); + auto extent = co_await read_cursor_by_type( + t, it.direct_cursor, it.get_extent_type()); + ret.evicted_size += extent->get_length(); + extent->set_target_rewrite_generation(epm->get_max_hot_gen() + 1); + co_await rewrite_logical_extent(t, extent); + it = co_await it.next(); + } else { + DEBUGT("skip {}", t, it); + it = co_await it.next(); + } + } + + co_return ret; } TransactionManager::get_extents_if_live_ret diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index 3fb426ffd08..bbdfa20498d 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -114,6 +114,10 @@ public: Transaction &t, LBAMapping mapping); + base_iertr::future relocate_shadow_extent( + Transaction &t, + LBAMapping mapping); + /** * get_pin * From d088cfc46db678114a80fa5011841a1e3e63296c Mon Sep 17 00:00:00 2001 From: Zhang Song Date: Wed, 30 Jul 2025 14:38:51 +0800 Subject: [PATCH 09/51] crimson/os/seastore: consolidate parameters on extent allocation path Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- .../os/seastore/btree/fixed_kv_btree.h | 16 ++-- src/crimson/os/seastore/btree/fixed_kv_node.h | 20 ++--- src/crimson/os/seastore/cache.cc | 23 +++--- src/crimson/os/seastore/cache.h | 43 +++-------- .../os/seastore/extent_placement_manager.h | 75 +++++++++---------- src/crimson/os/seastore/transaction_manager.h | 10 +-- .../seastore/test_btree_lba_manager.cc | 15 +--- .../crimson/seastore/test_seastore_cache.cc | 6 +- 8 files changed, 83 insertions(+), 125 deletions(-) diff --git a/src/crimson/os/seastore/btree/fixed_kv_btree.h b/src/crimson/os/seastore/btree/fixed_kv_btree.h index 3c6149ba4b6..d8256c7552a 100644 --- a/src/crimson/os/seastore/btree/fixed_kv_btree.h +++ b/src/crimson/os/seastore/btree/fixed_kv_btree.h @@ -653,10 +653,7 @@ public: static mkfs_ret mkfs(RootBlockRef &root_block, op_context_t c) { assert(root_block->is_mutation_pending()); auto root_leaf = c.cache.template alloc_new_non_data_extent( - c.trans, - node_size, - placement_hint_t::HOT, - INIT_GENERATION); + c.trans, node_size, {placement_hint_t::HOT, INIT_GENERATION}); root_leaf->set_size(0); fixed_kv_node_meta_t meta{min_max_t::min, min_max_t::max, 1}; root_leaf->set_meta(meta); @@ -1731,14 +1728,17 @@ public: assert(is_lba_backref_node(e->get_type())); auto do_rewrite = [&](auto &fixed_kv_extent) { + auto opt = Cache::alloc_option_t { + fixed_kv_extent.get_user_hint(), + // get target rewrite generation + fixed_kv_extent.get_rewrite_generation() + }; auto n_fixed_kv_extent = c.cache.template alloc_new_non_data_extent< std::remove_reference_t >( c.trans, fixed_kv_extent.get_length(), - fixed_kv_extent.get_user_hint(), - // get target rewrite generation - fixed_kv_extent.get_rewrite_generation()); + opt); n_fixed_kv_extent->rewrite(c.trans, fixed_kv_extent, 0); SUBTRACET( @@ -2609,7 +2609,7 @@ private: if (split_from == iter.get_depth()) { assert(iter.is_full()); auto nroot = c.cache.template alloc_new_non_data_extent( - c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION); + c.trans, node_size, {placement_hint_t::HOT, INIT_GENERATION}); fixed_kv_node_meta_t meta{ min_max_t::min, min_max_t::max, iter.get_depth() + 1}; nroot->set_meta(meta); diff --git a/src/crimson/os/seastore/btree/fixed_kv_node.h b/src/crimson/os/seastore/btree/fixed_kv_node.h index 5a729c70520..d2e4c272b59 100644 --- a/src/crimson/os/seastore/btree/fixed_kv_node.h +++ b/src/crimson/os/seastore/btree/fixed_kv_node.h @@ -329,9 +329,9 @@ struct FixedKVInternalNode std::tuple make_split_children(op_context_t c) { auto left = c.cache.template alloc_new_non_data_extent( - c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION); + c.trans, node_size, {placement_hint_t::HOT, INIT_GENERATION}); auto right = c.cache.template alloc_new_non_data_extent( - c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION); + c.trans, node_size, {placement_hint_t::HOT, INIT_GENERATION}); this->split_child_ptrs(c.trans, *left, *right); auto pivot = this->split_into(*left, *right); left->range = left->get_meta(); @@ -347,7 +347,7 @@ struct FixedKVInternalNode op_context_t c, Ref &right) { auto replacement = c.cache.template alloc_new_non_data_extent( - c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION); + c.trans, node_size, {placement_hint_t::HOT, INIT_GENERATION}); replacement->merge_child_ptrs( c.trans, static_cast(*this), *right); replacement->merge_from(*this, *right->template cast()); @@ -365,9 +365,9 @@ struct FixedKVInternalNode ceph_assert(_right->get_type() == this->get_type()); auto &right = *_right->template cast(); auto replacement_left = c.cache.template alloc_new_non_data_extent( - c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION); + c.trans, node_size, {placement_hint_t::HOT, INIT_GENERATION}); auto replacement_right = c.cache.template alloc_new_non_data_extent( - c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION); + c.trans, node_size, {placement_hint_t::HOT, INIT_GENERATION}); // We should do full merge if pivot_idx == right.get_size(). ceph_assert(pivot_idx != right.get_size()); @@ -750,9 +750,9 @@ struct FixedKVLeafNode std::tuple make_split_children(op_context_t c) { auto left = c.cache.template alloc_new_non_data_extent( - c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION); + c.trans, node_size, {placement_hint_t::HOT, INIT_GENERATION}); auto right = c.cache.template alloc_new_non_data_extent( - c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION); + c.trans, node_size, {placement_hint_t::HOT, INIT_GENERATION}); this->on_split(c.trans, *left, *right); auto pivot = this->split_into(*left, *right); left->range = left->get_meta(); @@ -777,7 +777,7 @@ struct FixedKVLeafNode op_context_t c, Ref &right) { auto replacement = c.cache.template alloc_new_non_data_extent( - c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION); + c.trans, node_size, {placement_hint_t::HOT, INIT_GENERATION}); replacement->on_merge(c.trans, static_cast(*this), *right); replacement->merge_from(*this, *right->template cast()); replacement->range = replacement->get_meta(); @@ -809,9 +809,9 @@ struct FixedKVLeafNode ceph_assert(_right->get_type() == this->get_type()); auto &right = *_right->template cast(); auto replacement_left = c.cache.template alloc_new_non_data_extent( - c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION); + c.trans, node_size, {placement_hint_t::HOT, INIT_GENERATION}); auto replacement_right = c.cache.template alloc_new_non_data_extent( - c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION); + c.trans, node_size, {placement_hint_t::HOT, INIT_GENERATION}); this->on_balance( c.trans, diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc index 2226744081a..1ec98a9b5a2 100644 --- a/src/crimson/os/seastore/cache.cc +++ b/src/crimson/os/seastore/cache.cc @@ -1219,35 +1219,36 @@ CachedExtentRef Cache::alloc_new_non_data_extent_by_type( SUBDEBUGT(seastore_cache, "allocate {} 0x{:x}B, hint={}, gen={}", t, type, length, hint, rewrite_gen_printer_t{gen}); ceph_assert(get_extent_category(type) == data_category_t::METADATA); + auto opt = alloc_option_t{hint, gen}; switch (type) { case extent_types_t::ROOT: ceph_assert(0 == "ROOT is never directly alloc'd"); return CachedExtentRef(); case extent_types_t::LADDR_INTERNAL: - return alloc_new_non_data_extent(t, length, hint, gen); + return alloc_new_non_data_extent(t, length, opt); case extent_types_t::LADDR_LEAF: return alloc_new_non_data_extent( - t, length, hint, gen); + t, length, opt); case extent_types_t::ROOT_META: return alloc_new_non_data_extent( - t, length, hint, gen); + t, length, opt); case extent_types_t::ONODE_BLOCK_STAGED: return alloc_new_non_data_extent( - t, length, hint, gen); + t, length, opt); case extent_types_t::OMAP_INNER: return alloc_new_non_data_extent( - t, length, hint, gen); + t, length, opt); case extent_types_t::OMAP_LEAF: return alloc_new_non_data_extent( - t, length, hint, gen); + t, length, opt); case extent_types_t::COLL_BLOCK: return alloc_new_non_data_extent( - t, length, hint, gen); + t, length, opt); case extent_types_t::TEST_BLOCK_PHYSICAL: - return alloc_new_non_data_extent(t, length, hint, gen); + return alloc_new_non_data_extent(t, length, opt); case extent_types_t::LOG_NODE: return alloc_new_non_data_extent( - t, length, hint, gen); + t, length, opt); case extent_types_t::NONE: { ceph_assert(0 == "NONE is an invalid extent type"); return CachedExtentRef(); @@ -1275,14 +1276,14 @@ std::vector Cache::alloc_new_data_extents_by_type( case extent_types_t::OBJECT_DATA_BLOCK: { auto extents = alloc_new_data_extents< - ObjectDataBlock>(t, length, hint, gen); + ObjectDataBlock>(t, length, {hint, gen}); res.insert(res.begin(), extents.begin(), extents.end()); } return res; case extent_types_t::TEST_BLOCK: { auto extents = alloc_new_data_extents< - TestBlock>(t, length, hint, gen); + TestBlock>(t, length, {hint, gen}); res.insert(res.begin(), extents.begin(), extents.end()); } return res; diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h index 72c5d174db6..8e70ba9434e 100644 --- a/src/crimson/os/seastore/cache.h +++ b/src/crimson/os/seastore/cache.h @@ -1142,6 +1142,7 @@ public: } } + using alloc_option_t = ExtentPlacementManager::alloc_option_t; /** * alloc_new_non_data_extent * @@ -1152,22 +1153,12 @@ public: TCachedExtentRef alloc_new_non_data_extent( Transaction &t, ///< [in, out] current transaction extent_len_t length, ///< [in] length - placement_hint_t hint, ///< [in] user hint -#ifdef UNIT_TESTS_BUILT - rewrite_gen_t gen, ///< [in] rewrite generation - std::optional epaddr = std::nullopt ///< [in] paddr fed by callers -#else - rewrite_gen_t gen -#endif + alloc_option_t opt ///< [in] allocation options ) { LOG_PREFIX(Cache::alloc_new_non_data_extent); - SUBTRACET(seastore_cache, "allocate {} 0x{:x}B, hint={}, gen={}", - t, T::TYPE, length, hint, rewrite_gen_printer_t{gen}); -#ifdef UNIT_TESTS_BUILT - auto result = epm.alloc_new_non_data_extent(t, T::TYPE, length, hint, gen, epaddr); -#else - auto result = epm.alloc_new_non_data_extent(t, T::TYPE, length, hint, gen); -#endif + SUBTRACET(seastore_cache, "allocate {} 0x{:x}B, opt.hint={}, gen={}", + t, T::TYPE, length, opt.hint, rewrite_gen_printer_t{opt.gen}); + auto result = epm.alloc_new_non_data_extent(t, T::TYPE, length, opt); if (!result) { SUBERRORT(seastore_cache, "insufficient space", t); std::rethrow_exception(crimson::ct_error::enospc::exception_ptr()); @@ -1178,14 +1169,14 @@ public: epm.dynamic_max_rewrite_generation)); ret->init(CachedExtent::extent_state_t::INITIAL_WRITE_PENDING, result->paddr, - hint, + opt.hint, result->gen, t.get_trans_id()); t.add_fresh_extent(ret); SUBDEBUGT(seastore_cache, "allocated {} 0x{:x}B extent at {}, hint={}, gen={} -- {}", t, T::TYPE, length, result->paddr, - hint, rewrite_gen_printer_t{result->gen}, *ret); + opt.hint, rewrite_gen_printer_t{result->gen}, *ret); return ret; } /** @@ -1198,22 +1189,12 @@ public: std::vector> alloc_new_data_extents( Transaction &t, ///< [in, out] current transaction extent_len_t length, ///< [in] length - placement_hint_t hint, ///< [in] user hint -#ifdef UNIT_TESTS_BUILT - rewrite_gen_t gen, ///< [in] rewrite generation - std::optional epaddr = std::nullopt ///< [in] paddr fed by callers -#else - rewrite_gen_t gen -#endif + alloc_option_t opt ///< [in] allocation options ) { LOG_PREFIX(Cache::alloc_new_data_extents); SUBTRACET(seastore_cache, "allocate {} 0x{:x}B, hint={}, gen={}", - t, T::TYPE, length, hint, rewrite_gen_printer_t{gen}); -#ifdef UNIT_TESTS_BUILT - auto results = epm.alloc_new_data_extents(t, T::TYPE, length, hint, gen, epaddr); -#else - auto results = epm.alloc_new_data_extents(t, T::TYPE, length, hint, gen); -#endif + t, T::TYPE, length, opt.hint, rewrite_gen_printer_t{opt.gen}); + auto results = epm.alloc_new_data_extents(t, T::TYPE, length, opt); if (results.empty()) { SUBERRORT(seastore_cache, "insufficient space", t); std::rethrow_exception(crimson::ct_error::enospc::exception_ptr()); @@ -1226,14 +1207,14 @@ public: epm.dynamic_max_rewrite_generation)); ret->init(CachedExtent::extent_state_t::INITIAL_WRITE_PENDING, result.paddr, - hint, + opt.hint, result.gen, t.get_trans_id()); t.add_fresh_extent(ret); SUBDEBUGT(seastore_cache, "allocated {} 0x{:x}B extent at {}, hint={}, gen={} -- {}", t, T::TYPE, length, result.paddr, - hint, rewrite_gen_printer_t{result.gen}, *ret); + opt.hint, rewrite_gen_printer_t{result.gen}, *ret); extents.emplace_back(std::move(ret)); } return extents; diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index 2a48510a17f..8685a3d04a2 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -351,6 +351,13 @@ public: return background_process.start_background(); } + struct alloc_option_t { + placement_hint_t hint; + rewrite_gen_t gen; +#ifdef UNIT_TESTS_BUILT + std::optional external_paddr = std::nullopt; +#endif + }; struct alloc_result_t { paddr_t paddr; bufferptr bp; @@ -360,34 +367,27 @@ public: Transaction& t, extent_types_t type, extent_len_t length, - placement_hint_t hint, -#ifdef UNIT_TESTS_BUILT - rewrite_gen_t gen, - std::optional external_paddr = std::nullopt -#else - rewrite_gen_t gen -#endif + alloc_option_t opt ) { - assert(hint < placement_hint_t::NUM_HINTS); - assert(is_target_rewrite_generation(gen, dynamic_max_rewrite_generation)); - assert(gen == INIT_GENERATION || hint == placement_hint_t::REWRITE); + assert(opt.hint < placement_hint_t::NUM_HINTS); + assert(is_target_rewrite_generation(opt.gen, dynamic_max_rewrite_generation)); + assert(opt.gen == INIT_GENERATION || opt.hint == placement_hint_t::REWRITE); data_category_t category = get_extent_category(type); - gen = adjust_generation(category, type, hint, gen); + opt.gen = adjust_generation(category, type, opt.hint, opt.gen); paddr_t addr; #ifdef UNIT_TESTS_BUILT - if (unlikely(external_paddr.has_value())) { - assert(external_paddr->is_fake()); - addr = *external_paddr; - } else if (gen == INLINE_GENERATION) { -#else - if (gen == INLINE_GENERATION) { + if (unlikely(opt.external_paddr.has_value())) { + assert(opt.external_paddr->is_fake()); + addr = *opt.external_paddr; + } else #endif + if (opt.gen == INLINE_GENERATION) { addr = make_record_relative_paddr(0); } else { assert(category == data_category_t::METADATA); - addr = get_writer(hint, category, gen)->alloc_paddr(length); + addr = get_writer(opt.hint, category, opt.gen)->alloc_paddr(length); } assert(!(category == data_category_t::DATA)); @@ -399,44 +399,37 @@ public: // according to the allocator. auto bp = create_extent_ptr_zero(length); - return alloc_result_t{addr, std::move(bp), gen}; + return alloc_result_t{addr, std::move(bp), opt.gen}; } std::list alloc_new_data_extents( Transaction& t, extent_types_t type, extent_len_t length, - placement_hint_t hint, -#ifdef UNIT_TESTS_BUILT - rewrite_gen_t gen, - std::optional external_paddr = std::nullopt -#else - rewrite_gen_t gen -#endif + alloc_option_t opt ) { LOG_PREFIX(ExtentPlacementManager::alloc_new_data_extents); - assert(hint < placement_hint_t::NUM_HINTS); - assert(is_target_rewrite_generation(gen, dynamic_max_rewrite_generation)); - assert(gen == INIT_GENERATION || hint == placement_hint_t::REWRITE); + assert(opt.hint < placement_hint_t::NUM_HINTS); + assert(is_target_rewrite_generation(opt.gen, dynamic_max_rewrite_generation)); + assert(opt.gen == INIT_GENERATION || opt.hint == placement_hint_t::REWRITE); data_category_t category = get_extent_category(type); - gen = adjust_generation(category, type, hint, gen); - assert(gen != INLINE_GENERATION); + opt.gen = adjust_generation(category, type, opt.hint, opt.gen); + assert(opt.gen != INLINE_GENERATION); // XXX: bp might be extended to point to different memory (e.g. PMem) // according to the allocator. std::list allocs; #ifdef UNIT_TESTS_BUILT - if (unlikely(external_paddr.has_value())) { - assert(external_paddr->is_fake()); + if (unlikely(opt.external_paddr.has_value())) { + assert(opt.external_paddr->is_fake()); auto bp = create_extent_ptr_zero(length); - allocs.emplace_back(alloc_result_t{*external_paddr, std::move(bp), gen}); - } else { -#else - { + allocs.emplace_back(alloc_result_t{*opt.external_paddr, std::move(bp), opt.gen}); + } else #endif + { assert(category == data_category_t::DATA); - auto addrs = get_writer(hint, category, gen)->alloc_paddrs(length); + auto addrs = get_writer(opt.hint, category, opt.gen)->alloc_paddrs(length); for (auto &ext : addrs) { auto left = ext.len; while (left > 0) { @@ -448,10 +441,10 @@ public: auto start = ext.start.is_delayed() ? ext.start : ext.start + (ext.len - left); - allocs.emplace_back(alloc_result_t{start, std::move(bp), gen}); + allocs.emplace_back(alloc_result_t{start, std::move(bp), opt.gen}); SUBDEBUGT(seastore_epm, - "allocated {} 0x{:x}B extent at {}, hint={}, gen={}", - t, type, len, start, hint, gen); + "allocated {} 0x{:x}B extent at {}, opt.hint={}, opt.gen={}", + t, type, len, start, opt.hint, opt.gen); left -= len; } } diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index bbdfa20498d..fe6f816efc3 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -552,10 +552,7 @@ public: SUBDEBUGT(seastore_tm, "{} hint {}~0x{:x} phint={} ...", t, T::TYPE, laddr_hint, len, placement_hint); auto ext = cache->alloc_new_non_data_extent( - t, - len, - placement_hint, - INIT_GENERATION); + t, len, {placement_hint, INIT_GENERATION}); // user must initialize the logical extent themselves. assert(is_user_transaction(t.get_src())); ext->set_seen_by_users(); @@ -593,10 +590,7 @@ public: SUBDEBUGT(seastore_tm, "{} hint {}~0x{:x} phint={} ...", t, T::TYPE, laddr_hint, len, placement_hint); auto exts = cache->alloc_new_data_extents( - t, - len, - placement_hint, - INIT_GENERATION); + t, len, {placement_hint, INIT_GENERATION}); // user must initialize the logical extent themselves assert(is_user_transaction(t.get_src())); for (auto& ext : exts) { diff --git a/src/test/crimson/seastore/test_btree_lba_manager.cc b/src/test/crimson/seastore/test_btree_lba_manager.cc index 4fb774930e5..c79ce6e5e25 100644 --- a/src/test/crimson/seastore/test_btree_lba_manager.cc +++ b/src/test/crimson/seastore/test_btree_lba_manager.cc @@ -327,11 +327,7 @@ struct lba_btree_test : btree_test_base { check.emplace(addr, get_map_val(len, TestBlock::TYPE)); lba_btree_update([=, this](auto &btree, auto &t) { auto extents = cache->alloc_new_data_extents( - t, - TestBlock::SIZE, - placement_hint_t::HOT, - 0, - get_paddr()); + t, TestBlock::SIZE, {placement_hint_t::HOT, 0, false, get_paddr()}); return seastar::do_with( std::move(extents), [this, addr, &t, len, &btree](auto &extents) { @@ -454,8 +450,7 @@ struct btree_lba_manager_test : btree_test_base { cache->alloc_new_non_data_extent( *t.t, TestBlockPhysical::SIZE, - placement_hint_t::HOT, - 0); + {placement_hint_t::HOT, 0}); }; return t; } @@ -550,11 +545,7 @@ struct btree_lba_manager_test : btree_test_base { *t.t, [=, this](auto &t) { auto extents = cache->alloc_new_data_extents( - t, - TestBlock::SIZE, - placement_hint_t::HOT, - 0, - get_paddr()); + t, TestBlock::SIZE, {placement_hint_t::HOT, 0, false, get_paddr()}); return seastar::do_with( std::vector( extents.begin(), extents.end()), diff --git a/src/test/crimson/seastore/test_seastore_cache.cc b/src/test/crimson/seastore/test_seastore_cache.cc index 20b99e50ee5..4de65875e05 100644 --- a/src/test/crimson/seastore/test_seastore_cache.cc +++ b/src/test/crimson/seastore/test_seastore_cache.cc @@ -157,8 +157,7 @@ TEST_F(cache_test_t, test_addr_fixup) auto extent = cache->alloc_new_non_data_extent( *t, TestBlockPhysical::SIZE, - placement_hint_t::HOT, - 0); + {placement_hint_t::HOT, 0}); extent->set_contents('c'); csum = extent->calc_crc32c(); submit_transaction(std::move(t)).get(); @@ -188,8 +187,7 @@ TEST_F(cache_test_t, test_dirty_extent) auto extent = cache->alloc_new_non_data_extent( *t, TestBlockPhysical::SIZE, - placement_hint_t::HOT, - 0); + {placement_hint_t::HOT, 0}); extent->set_contents('c'); csum = extent->calc_crc32c(); auto reladdr = extent->get_paddr(); From 3f2b9584783bf475c5e4e7e023c585ba753c29a4 Mon Sep 17 00:00:00 2001 From: Zhang Song Date: Wed, 3 Sep 2025 16:04:12 +0800 Subject: [PATCH 10/51] crimson/os/seastore: avoid evicting promoted extent to the cold tier Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/cache.cc | 12 ++++++----- src/crimson/os/seastore/cache.h | 6 ++++-- .../os/seastore/extent_placement_manager.h | 13 +++++++++--- .../os/seastore/transaction_manager.cc | 21 +++++++++++++++---- src/crimson/os/seastore/transaction_manager.h | 6 ++++++ .../seastore/test_transaction_manager.cc | 3 ++- 6 files changed, 46 insertions(+), 15 deletions(-) diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc index 1ec98a9b5a2..5bcd0adf5b6 100644 --- a/src/crimson/os/seastore/cache.cc +++ b/src/crimson/os/seastore/cache.cc @@ -1212,14 +1212,15 @@ CachedExtentRef Cache::alloc_new_non_data_extent_by_type( extent_types_t type, ///< [in] type tag extent_len_t length, ///< [in] length placement_hint_t hint, ///< [in] user hint - rewrite_gen_t gen ///< [in] rewrite generation + rewrite_gen_t gen, ///< [in] rewrite generation + bool is_tracked ) { LOG_PREFIX(Cache::alloc_new_non_data_extent_by_type); SUBDEBUGT(seastore_cache, "allocate {} 0x{:x}B, hint={}, gen={}", t, type, length, hint, rewrite_gen_printer_t{gen}); ceph_assert(get_extent_category(type) == data_category_t::METADATA); - auto opt = alloc_option_t{hint, gen}; + auto opt = alloc_option_t{hint, gen, is_tracked}; switch (type) { case extent_types_t::ROOT: ceph_assert(0 == "ROOT is never directly alloc'd"); @@ -1264,7 +1265,8 @@ std::vector Cache::alloc_new_data_extents_by_type( extent_types_t type, ///< [in] type tag extent_len_t length, ///< [in] length placement_hint_t hint, ///< [in] user hint - rewrite_gen_t gen ///< [in] rewrite generation + rewrite_gen_t gen, ///< [in] rewrite generation + bool is_tracked ) { LOG_PREFIX(Cache::alloc_new_data_extents_by_type); @@ -1276,14 +1278,14 @@ std::vector Cache::alloc_new_data_extents_by_type( case extent_types_t::OBJECT_DATA_BLOCK: { auto extents = alloc_new_data_extents< - ObjectDataBlock>(t, length, {hint, gen}); + ObjectDataBlock>(t, length, {hint, gen, is_tracked}); res.insert(res.begin(), extents.begin(), extents.end()); } return res; case extent_types_t::TEST_BLOCK: { auto extents = alloc_new_data_extents< - TestBlock>(t, length, {hint, gen}); + TestBlock>(t, length, {hint, gen, is_tracked}); res.insert(res.begin(), extents.begin(), extents.end()); } return res; diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h index 8e70ba9434e..873bc7294f9 100644 --- a/src/crimson/os/seastore/cache.h +++ b/src/crimson/os/seastore/cache.h @@ -1280,7 +1280,8 @@ public: extent_types_t type, ///< [in] type tag extent_len_t length, ///< [in] length placement_hint_t hint, ///< [in] user hint - rewrite_gen_t gen ///< [in] rewrite generation + rewrite_gen_t gen, ///< [in] rewrite generation + bool is_tracked ); /** @@ -1293,7 +1294,8 @@ public: extent_types_t type, ///< [in] type tag extent_len_t length, ///< [in] length placement_hint_t hint, ///< [in] user hint - rewrite_gen_t gen ///< [in] rewrite generation + rewrite_gen_t gen, ///< [in] rewrite generation + bool is_tracked ); /** diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index 8685a3d04a2..0f80e1c7125 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -354,6 +354,7 @@ public: struct alloc_option_t { placement_hint_t hint; rewrite_gen_t gen; + bool is_tracked; #ifdef UNIT_TESTS_BUILT std::optional external_paddr = std::nullopt; #endif @@ -374,7 +375,7 @@ public: assert(opt.gen == INIT_GENERATION || opt.hint == placement_hint_t::REWRITE); data_category_t category = get_extent_category(type); - opt.gen = adjust_generation(category, type, opt.hint, opt.gen); + opt.gen = adjust_generation(category, type, opt.hint, opt.gen, opt.is_tracked); paddr_t addr; #ifdef UNIT_TESTS_BUILT @@ -414,7 +415,7 @@ public: assert(opt.gen == INIT_GENERATION || opt.hint == placement_hint_t::REWRITE); data_category_t category = get_extent_category(type); - opt.gen = adjust_generation(category, type, opt.hint, opt.gen); + opt.gen = adjust_generation(category, type, opt.hint, opt.gen, opt.is_tracked); assert(opt.gen != INLINE_GENERATION); // XXX: bp might be extended to point to different memory (e.g. PMem) @@ -620,7 +621,8 @@ private: data_category_t category, extent_types_t type, placement_hint_t hint, - rewrite_gen_t gen) { + rewrite_gen_t gen, + bool is_tracked) { assert(is_real_type(type)); if (is_root_type(type)) { gen = INLINE_GENERATION; @@ -655,6 +657,11 @@ private: gen = background_process.adjust_generation(gen); } + if (is_tracked && gen >= hot_tier_generations && + hint != placement_hint_t::REWRITE) { + gen = hot_tier_generations - 1; + } + if (gen > dynamic_max_rewrite_generation) { gen = dynamic_max_rewrite_generation; } diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index 2454c7d1cc9..cf99f120272 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -39,6 +39,7 @@ TransactionManager::TransactionManager( journal(std::move(_journal)), epm(std::move(_epm)), backref_manager(std::move(_backref_manager)), + logical_bucket(nullptr), full_extent_integrity_check( crimson::common::get_conf( "seastore_full_integrity_check")), @@ -854,6 +855,14 @@ TransactionManager::rewrite_logical_extent( ceph_abort(); } + bool is_tracked = + support_logical_bucket() && + // lextent is from hot tier + !epm->is_cold_device(extent->get_paddr().get_device_id()) && + // lextent is cached by non volatile cache + logical_bucket->is_cached( + extent->get_laddr().get_object_prefix()); + if (get_extent_category(extent->get_type()) == data_category_t::METADATA) { assert(extent->is_fully_loaded()); cache->retire_extent(t, extent); @@ -863,7 +872,8 @@ TransactionManager::rewrite_logical_extent( extent->get_length(), extent->get_user_hint(), // get target rewrite generation - extent->get_rewrite_generation())->cast(); + extent->get_rewrite_generation(), + is_tracked)->cast(); nextent->rewrite(t, *extent, 0); DEBUGT("rewriting meta -- {} to {}", t, *extent, *nextent); @@ -907,7 +917,8 @@ TransactionManager::rewrite_logical_extent( extent->get_length(), extent->get_user_hint(), // get target rewrite generation - extent->get_rewrite_generation()); + extent->get_rewrite_generation(), + is_tracked); extent_len_t off = 0; auto left = extent->get_length(); extent_ref_count_t refcount = 0; @@ -1199,7 +1210,8 @@ TransactionManager::promote_extent( orig_ext->get_type(), orig_ext->get_length(), placement_hint_t::HOT, - INIT_GENERATION); + INIT_GENERATION, + true); promoted_extents.reserve(promoted_raw_extents.size()); @@ -1243,7 +1255,8 @@ TransactionManager::promote_extent( orig_ext->get_type(), orig_ext->get_length(), placement_hint_t::HOT, - INIT_GENERATION); + INIT_GENERATION, + true); auto lext = promoted_extent->cast(); lext->set_laddr(orig_ext->get_laddr()); lext->rewrite(t, *orig_ext, 0); diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index fe6f816efc3..985cf5321ed 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -26,6 +26,7 @@ #include "crimson/os/seastore/logging.h" #include "crimson/os/seastore/seastore_types.h" #include "crimson/os/seastore/cache.h" +#include "crimson/os/seastore/logical_bucket.h" #include "crimson/os/seastore/root_meta.h" #include "crimson/os/seastore/lba_manager.h" #include "crimson/os/seastore/backref_manager.h" @@ -1293,6 +1294,10 @@ public: co_return mapping; } + bool support_logical_bucket() const { + return logical_bucket != nullptr; + } + ~TransactionManager(); private: @@ -1305,6 +1310,7 @@ private: BackrefManagerRef backref_manager; WritePipeline write_pipeline; + LogicalBucket *logical_bucket; bool full_extent_integrity_check = true; diff --git a/src/test/crimson/seastore/test_transaction_manager.cc b/src/test/crimson/seastore/test_transaction_manager.cc index dd7ef23c37d..834304bcdb4 100644 --- a/src/test/crimson/seastore/test_transaction_manager.cc +++ b/src/test/crimson/seastore/test_transaction_manager.cc @@ -1149,7 +1149,8 @@ struct transaction_manager_test_t : get_extent_category(t), t, placement_hint_t::HOT, - gen); + gen, + false); if (expected_generations[t][gen] != epm_gen) { logger().error("caller: {}, extent type: {}, input generation: {}, " "expected generation : {}, adjust result from EPM: {}", From c7b3fd1350099290fdb93abe705817f0574026c1 Mon Sep 17 00:00:00 2001 From: Zhang Song Date: Wed, 30 Jul 2025 15:39:55 +0800 Subject: [PATCH 11/51] crimson/os/seastore: don't add shadow extent to cache Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/cache.h | 3 +++ src/crimson/os/seastore/cached_extent.h | 10 ++++++++++ src/crimson/os/seastore/logical_child_node.h | 2 +- src/crimson/os/seastore/transaction_manager.cc | 2 +- src/crimson/os/seastore/transaction_manager.h | 2 +- 5 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h index 873bc7294f9..1b195dffb0f 100644 --- a/src/crimson/os/seastore/cache.h +++ b/src/crimson/os/seastore/cache.h @@ -1705,6 +1705,9 @@ private: extent_len_t load_length) { assert(ext.get_paddr().is_absolute()); + if (ext.is_shadow_extent()) { + return; + } if (hint == CACHE_HINT_NOCACHE && is_logical_type(ext.get_type())) { return; } diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h index cfb07b509d5..3d69ea73f32 100644 --- a/src/crimson/os/seastore/cached_extent.h +++ b/src/crimson/os/seastore/cached_extent.h @@ -908,6 +908,14 @@ public: last_touch_end = touch_end; } + bool is_shadow_extent() const { + return is_shadow; + } + + void set_shadow_extent(bool b) { + is_shadow = b; + } + private: template friend class read_set_item_t; @@ -1036,6 +1044,8 @@ private: seastar::shared_mutex commit_lock; + bool is_shadow = false; + protected: trans_view_set_t mutation_pending_extents; trans_view_set_t retired_transactions; diff --git a/src/crimson/os/seastore/logical_child_node.h b/src/crimson/os/seastore/logical_child_node.h index 43721747faf..ea4bb9c6a76 100644 --- a/src/crimson/os/seastore/logical_child_node.h +++ b/src/crimson/os/seastore/logical_child_node.h @@ -28,7 +28,7 @@ public: } virtual ~LogicalChildNode() { - if (this->is_stable()) { + if (this->is_stable() && !is_shadow_extent()) { lba_child_node_t::destroy(); } } diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index cf99f120272..9657a79c84b 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -1276,7 +1276,7 @@ TransactionManager::promote_extent( 0, orig_ext->get_length(), std::nullopt); - boost::ignore_unused(remapped_cold_extent); + remapped_cold_extent->set_shadow_extent(true); remapped_cold_extent->set_shadow_extent(true); } diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index 985cf5321ed..07c1a11f455 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -1590,7 +1590,7 @@ private: remap_offset, remap_len, std::nullopt); - boost::ignore_unused(cold_ext); + cold_ext->set_shadow_extent(true); } // user must initialize the logical extent themselves. remapped_extent->set_seen_by_users(); From 91eb5d2a9c44ace4d26905a50f0bd0b691839833 Mon Sep 17 00:00:00 2001 From: Zhang Song Date: Thu, 31 Jul 2025 11:07:44 +0800 Subject: [PATCH 12/51] crimson/os/seastore/EPM: initialize logical bucket and pinboard Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- src/common/options/crimson.yaml.in | 10 ++++++ src/crimson/os/seastore/cache.h | 4 +++ .../os/seastore/extent_placement_manager.cc | 6 ++-- .../os/seastore/extent_placement_manager.h | 33 +++++++++++++++++-- .../os/seastore/transaction_manager.cc | 7 +++- 5 files changed, 54 insertions(+), 6 deletions(-) diff --git a/src/common/options/crimson.yaml.in b/src/common/options/crimson.yaml.in index 9651bfc69d3..bfb658f3f1e 100644 --- a/src/common/options/crimson.yaml.in +++ b/src/common/options/crimson.yaml.in @@ -347,3 +347,13 @@ options: level: advanced desc: Size in bytes of extents to be promoted from cold devices, set 0 to disable promotion default: 2_M +- name: seastore_logical_bucket_capacity + type: size + level: advanced + desc: Size in bytes of memory used by logical bucket + default: 200_M +- name: seastore_logical_bucket_proceed_size_per_cycle + type: size + level: advanced + desc: Size in bytes of extents to be demoted from logical bucket + default: 2_M diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h index 1b195dffb0f..6f893bc6c3c 100644 --- a/src/crimson/os/seastore/cache.h +++ b/src/crimson/os/seastore/cache.h @@ -746,6 +746,10 @@ public: return epm.get_block_size(); } + ExtentPinboard *get_extent_pinboard() { + return pinboard.get(); + } + // Interfaces only for tests. public: CachedExtentRef test_query_cache(paddr_t offset) { diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc index 822a0a3a375..92c69b5287d 100644 --- a/src/crimson/os/seastore/extent_placement_manager.cc +++ b/src/crimson/os/seastore/extent_placement_manager.cc @@ -192,7 +192,8 @@ SegmentedOolWriter::alloc_write_ool_extents( void ExtentPlacementManager::init( JournalTrimmerImplRef &&trimmer, AsyncCleanerRef &&cleaner, - AsyncCleanerRef &&cold_cleaner) + AsyncCleanerRef &&cold_cleaner, + ExtentPinboard *pinboard) { LOG_PREFIX(ExtentPlacementManager::init); writer_refs.clear(); @@ -279,7 +280,8 @@ void ExtentPlacementManager::init( background_process.init(std::move(trimmer), std::move(cleaner), std::move(cold_cleaner), - hot_tier_generations); + hot_tier_generations, + pinboard); if (cold_segment_cleaner) { ceph_assert(get_main_backend_type() == backend_type_t::SEGMENTED); ceph_assert(background_process.has_cold_tier()); diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index 0f80e1c7125..7c512356507 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -11,6 +11,8 @@ #include "crimson/os/seastore/journal/segment_allocator.h" #include "crimson/os/seastore/journal/record_submitter.h" #include "crimson/os/seastore/transaction.h" +#include "crimson/os/seastore/extent_pinboard.h" +#include "crimson/os/seastore/logical_bucket.h" #include "crimson/os/seastore/random_block_manager.h" #include "crimson/os/seastore/random_block_manager/block_rb_manager.h" #include "crimson/os/seastore/randomblock_manager_group.h" @@ -290,7 +292,8 @@ public: cold_tier_generations, hot_tier_generations); } - void init(JournalTrimmerImplRef &&, AsyncCleanerRef &&, AsyncCleanerRef &&); + void init(JournalTrimmerImplRef &&, AsyncCleanerRef &&, AsyncCleanerRef &&, + ExtentPinboard *pinboard); SegmentSeqAllocator &get_ool_segment_seq_allocator() const { return *ool_segment_seq_allocator; @@ -333,6 +336,10 @@ public: bool report_detail, double seconds) const; + LogicalBucket *get_logical_bucket() { + return background_process.get_logical_bucket(); + } + using mount_ertr = crimson::errorator< crimson::ct_error::input_output_error>; using mount_ret = mount_ertr::future<>; @@ -742,7 +749,8 @@ private: void init(JournalTrimmerImplRef &&_trimmer, AsyncCleanerRef &&_cleaner, AsyncCleanerRef &&_cold_cleaner, - rewrite_gen_t hot_tier_generations) { + rewrite_gen_t hot_tier_generations, + ExtentPinboard *_pinboard) { trimmer = std::move(_trimmer); trimmer->set_background_callback(this); main_cleaner = std::move(_cleaner); @@ -759,6 +767,7 @@ private: cleaners_by_device_id[id] = cold_cleaner.get(); } + using crimson::common::get_conf; eviction_state.init( crimson::common::get_conf( "seastore_multiple_tiers_stop_evict_ratio"), @@ -767,9 +776,22 @@ private: crimson::common::get_conf( "seastore_multiple_tiers_fast_evict_ratio"), hot_tier_generations); + + pinboard = _pinboard; + ceph_assert(pinboard != nullptr); + pinboard->set_background_callback(this); + + logical_bucket = create_logical_bucket( + get_conf("seastore_logical_bucket_capacity"), + get_conf("seastore_logical_bucket_proceed_size_per_cycle")); + logical_bucket->set_background_callback(this); } } + LogicalBucket *get_logical_bucket() { + return logical_bucket.get(); + } + backend_type_t get_backend_type() const { return trimmer->get_backend_type(); } @@ -796,6 +818,8 @@ private: main_cleaner->set_extent_callback(cb); if (has_cold_tier()) { cold_cleaner->set_extent_callback(cb); + pinboard->set_extent_callback(cb); + logical_bucket->set_extent_callback(cb); } } @@ -972,7 +996,8 @@ private: bool main_cleaner_should_fast_evict() const { return has_cold_tier() && - main_cleaner->can_clean_space() && + (main_cleaner->can_clean_space() || + (logical_bucket && logical_bucket->could_demote())) && eviction_state.is_fast_mode(); } @@ -1135,6 +1160,8 @@ private: JournalTrimmerImplRef trimmer; AsyncCleanerRef main_cleaner; + ExtentPinboard *pinboard = nullptr; + LogicalBucketRef logical_bucket; /* * cold tier (optional, see has_cold_tier()) diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index 9657a79c84b..35960eff498 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -47,6 +47,10 @@ TransactionManager::TransactionManager( { epm->set_extent_callback(this); journal->set_write_pipeline(&write_pipeline); + if (epm->has_cold_tier()) { + logical_bucket = epm->get_logical_bucket(); + assert(support_logical_bucket()); + } } TransactionManager::mkfs_ertr::future<> TransactionManager::mkfs() @@ -1582,7 +1586,8 @@ TransactionManagerRef make_transaction_manager( epm->init(std::move(journal_trimmer), std::move(cleaner), - std::move(cold_segment_cleaner)); + std::move(cold_segment_cleaner), + cache->get_extent_pinboard()); epm->set_primary_device(primary_device); INFO("main backend type: {}, cold tier: {}", From 051db088b311e17becf8e940acf9f86b99c2ed9b Mon Sep 17 00:00:00 2001 From: Zhang Song Date: Thu, 7 Aug 2025 19:22:58 +0800 Subject: [PATCH 13/51] crimson/os/seastore/EPM/BackgroundProcess: add promote and demote process Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/async_cleaner.h | 1 + src/crimson/os/seastore/extent_pinboard.cc | 3 +- .../os/seastore/extent_placement_manager.cc | 105 +++++++++++++----- .../os/seastore/extent_placement_manager.h | 29 ++++- src/crimson/os/seastore/logical_bucket.cc | 4 + 5 files changed, 114 insertions(+), 28 deletions(-) diff --git a/src/crimson/os/seastore/async_cleaner.h b/src/crimson/os/seastore/async_cleaner.h index 7c54d7d33ae..df978020be9 100644 --- a/src/crimson/os/seastore/async_cleaner.h +++ b/src/crimson/os/seastore/async_cleaner.h @@ -453,6 +453,7 @@ struct BackgroundListener { virtual ~BackgroundListener() = default; virtual void maybe_wake_background() = 0; virtual void maybe_wake_blocked_io() = 0; + virtual void maybe_wake_promote() = 0; virtual state_t get_state() const = 0; bool is_ready() const { diff --git a/src/crimson/os/seastore/extent_pinboard.cc b/src/crimson/os/seastore/extent_pinboard.cc index d65ad2a4956..e03ffbf618f 100644 --- a/src/crimson/os/seastore/extent_pinboard.cc +++ b/src/crimson/os/seastore/extent_pinboard.cc @@ -319,7 +319,8 @@ public: remove_extent(list.front(), extent_pin_state_t::Fresh); } if (should_run_promote()) { - // TODO: wake promote background process + assert(listener); + listener->maybe_wake_promote(); } } diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc index 92c69b5287d..0995ada0413 100644 --- a/src/crimson/os/seastore/extent_placement_manager.cc +++ b/src/crimson/os/seastore/extent_placement_manager.cc @@ -634,41 +634,49 @@ void ExtentPlacementManager::BackgroundProcess::start_background() ceph_assert(state == state_t::SCAN_SPACE); assert(!is_running()); process_join = seastar::now(); + promote_process_join = seastar::now(); state = state_t::RUNNING; assert(is_running()); process_join = run(); + if (has_cold_tier()) { + promote_process_join = run_promote(); + } } seastar::future<> ExtentPlacementManager::BackgroundProcess::stop_background() { LOG_PREFIX(BackgroundProcess::stop_background); - return seastar::futurize_invoke([this, FNAME] { - if (!is_running()) { - if (state != state_t::HALT) { - INFO("isn't RUNNING or HALT, STOP"); - state = state_t::STOP; - } else { - INFO("isn't RUNNING, already HALT"); - } - return seastar::now(); + if (!is_running()) { + if (state != state_t::HALT) { + INFO("isn't RUNNING or HALT, STOP"); + state = state_t::STOP; + } else { + INFO("isn't RUNNING, already HALT"); } - INFO("is RUNNING, going to HALT..."); - auto ret = std::move(*process_join); - process_join.reset(); - state = state_t::HALT; - assert(!is_running()); - do_wake_background(); - return ret; - }).then([this, FNAME] { - INFO("done, {}, {}", - JournalTrimmerImpl::stat_printer_t{*trimmer, true}, - AsyncCleaner::stat_printer_t{*main_cleaner, true}); - if (has_cold_tier()) { - INFO("done, cold_cleaner: {}", - AsyncCleaner::stat_printer_t{*cold_cleaner, true}); - } - }); + co_return; + } + INFO("is RUNNING, going to HALT..."); + std::vector> futs; + futs.emplace_back(std::move(*process_join)); + process_join.reset(); + if (promote_process_join) { + futs.emplace_back(std::move(*promote_process_join)); + promote_process_join.reset(); + } + state = state_t::HALT; + assert(!is_running()); + do_wake_background(); + do_wake_promote(); + co_await seastar::when_all(futs.begin(), futs.end()); + INFO("done, {}, {}", + JournalTrimmerImpl::stat_printer_t{*trimmer, true}, + AsyncCleaner::stat_printer_t{*main_cleaner, true}); + if (has_cold_tier()) { + INFO("done, cold_cleaner: {}", + AsyncCleaner::stat_printer_t{*cold_cleaner, true}); + } + co_return; } seastar::future<> @@ -1020,7 +1028,12 @@ ExtentPlacementManager::BackgroundProcess::do_background_cycle() proceed_clean_cold = true; } - if (!proceed_clean_main && !proceed_clean_cold) { + bool proceed_demote = false; + if (demote_should_run()) { + proceed_demote = true; + } + + if (!proceed_clean_main && !proceed_clean_cold && !proceed_demote) { ceph_abort_msg("no background process will start"); } return seastar::when_all( @@ -1062,11 +1075,51 @@ ExtentPlacementManager::BackgroundProcess::do_background_cycle() ).finally([FNAME] { DEBUG("finished clean cold"); }); + }, + [this, proceed_demote] { + if (!proceed_demote) { + return seastar::now(); + } + return logical_bucket->demote(); } ).discard_result(); } } +seastar::future<> ExtentPlacementManager::BackgroundProcess::run_promote() +{ + assert(pinboard); + assert(is_running()); + return seastar::repeat([this] { + if (!is_running()) { + return seastar::make_ready_future( + seastar::stop_iteration::yes); + } + + return seastar::futurize_invoke([this] { + if (pinboard->should_promote()) { + auto usage = cleaner_usage_t{pinboard->get_promotion_size(), 0}; + auto res = try_reserve_cleaner(usage); + if (res.is_successful()) { + return pinboard->promote( + ).finally([this, usage, res] { + abort_cleaner_usage(usage, res); + }); + } else { + // reserve usage failed, block + abort_cleaner_usage(usage, res); + } + } // shouldn't promote, block + + ceph_assert(!blocking_promote); + blocking_promote = seastar::promise<>(); + return blocking_promote->get_future(); + }).then([] { + return seastar::stop_iteration::no; + }); + }); +} + void ExtentPlacementManager::BackgroundProcess::register_metrics(store_index_t store_index) { namespace sm = seastar::metrics; diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index 7c512356507..001f2c15b46 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -950,6 +950,15 @@ private: void maybe_wake_blocked_io() final; + void maybe_wake_promote() final { + if (!is_ready()) { + return; + } + if (pinboard && pinboard->should_promote()) { + do_wake_promote(); + } + } + private: // reserve helpers bool try_reserve_cold(std::size_t usage); @@ -984,6 +993,13 @@ private: } } + void do_wake_promote() { + if (blocking_promote) { + blocking_promote->set_value(); + blocking_promote = std::nullopt; + } + } + // background_should_run() should be atomic with do_background_cycle() // to make sure the condition is consistent. bool background_should_run() { @@ -991,7 +1007,8 @@ private: maybe_update_eviction_mode(); return main_cleaner_should_run() || cold_cleaner_should_run() - || trimmer->should_trim(); + || trimmer->should_trim() + || demote_should_run(); } bool main_cleaner_should_fast_evict() const { @@ -1013,6 +1030,13 @@ private: cold_cleaner->should_clean_space(); } + bool demote_should_run() const { + return has_cold_tier() && + logical_bucket->could_demote() && + (eviction_state.is_fast_mode() || + logical_bucket->should_demote()); + } + bool should_block_io() const { assert(is_ready()); return trimmer->should_block_io_on_trim() || @@ -1142,6 +1166,7 @@ private: }; seastar::future<> do_background_cycle(); + seastar::future<> run_promote(); void register_metrics(store_index_t store_index); @@ -1177,6 +1202,8 @@ private: // giving the woken continuation a chance to retry the reservation // before the next background cycle. bool pending_user_io_wake = false; + std::optional> promote_process_join; + std::optional> blocking_promote; bool is_running_until_halt = false; state_t state = state_t::STOP; eviction_state_t eviction_state; diff --git a/src/crimson/os/seastore/logical_bucket.cc b/src/crimson/os/seastore/logical_bucket.cc index 36a37abffb9..76cfb3a65b9 100644 --- a/src/crimson/os/seastore/logical_bucket.cc +++ b/src/crimson/os/seastore/logical_bucket.cc @@ -50,6 +50,10 @@ public: } else { TRACE("create bucket: {}", laddr); index[laddr] = lru.emplace(lru.end(), laddr); + if (should_demote()) { + assert(listener); + listener->maybe_wake_background(); + } } } From eca32e1fa9894481d6a7d7545fbb9a913b61c63f Mon Sep 17 00:00:00 2001 From: Zhang Song Date: Thu, 31 Jul 2025 16:06:13 +0800 Subject: [PATCH 14/51] crimson/os/seastore: set backend type explicitly Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- doc/dev/crimson/index.rst | 2 +- doc/dev/crimson/seastore.rst | 2 +- src/common/options/crimson.yaml.in | 22 ++++- src/crimson/os/seastore/device.cc | 22 +++-- src/crimson/os/seastore/device.h | 9 ++- .../os/seastore/extent_placement_manager.cc | 1 + .../random_block_manager/block_rb_manager.cc | 4 +- src/crimson/os/seastore/seastore.cc | 80 ++++++++++++------- src/crimson/os/seastore/seastore_types.cc | 20 ++++- src/crimson/os/seastore/seastore_types.h | 7 +- .../os/seastore/segment_manager/ephemeral.cc | 2 + src/crimson/tools/store_nbd/tm_driver.cc | 19 +++-- .../seastore/nvmedevice/test_nvmedevice.cc | 1 + src/vstart.sh | 35 ++++++-- 14 files changed, 165 insertions(+), 61 deletions(-) diff --git a/doc/dev/crimson/index.rst b/doc/dev/crimson/index.rst index 0facd5e7388..045fc1f80aa 100644 --- a/doc/dev/crimson/index.rst +++ b/doc/dev/crimson/index.rst @@ -101,7 +101,7 @@ run:: $ MDS=0 MON=1 OSD=1 MGR=1 taskset -ac '0-95' /ceph/src/vstart.sh --new -x \ --localhost --without-dashboard --redirect-output --seastore --osd-args \ "--seastore_max_concurrent_transactions=128 --seastore_cachepin_type=LRU \ - --seastore_main_device_type=RANDOM_BLOCK_SSD" --seastore-devs /dev/nvme0n1 \ + --seastore_hot_device_type=RANDOM_BLOCK_SSD" --seastore-devs /dev/nvme0n1 \ --crimson --crimson-smp 1 --no-restart Another SeaStore example:: diff --git a/doc/dev/crimson/seastore.rst b/doc/dev/crimson/seastore.rst index cc54904cff7..91b29624cca 100644 --- a/doc/dev/crimson/seastore.rst +++ b/doc/dev/crimson/seastore.rst @@ -204,7 +204,7 @@ Metadata Structures Device Types ------------ -Configured via the ``seastore_main_device_type`` option, the device types are +Configured via the ``seastore_hot_device_type`` option, the device types are separated into **Segmented** and **RBM** backend types as follows: diff --git a/src/common/options/crimson.yaml.in b/src/common/options/crimson.yaml.in index bfb658f3f1e..7f4f1af71cc 100644 --- a/src/common/options/crimson.yaml.in +++ b/src/common/options/crimson.yaml.in @@ -245,15 +245,31 @@ options: level: advanced desc: maximum concurrent transactions that seastore allows (per reactor) default: 128 -- name: seastore_main_device_type +- name: seastore_hot_device_type type: str level: dev - desc: The main device type seastore uses (SSD or RANDOM_BLOCK_SSD) + desc: The type of the devices in the hot (or only) tier, valid values are HDD, SSD, RANDOM_BLOCK_SSD and RANDOM_BLOCK_HDD default: SSD +- name: seastore_hot_backend_type + type: str + level: dev + desc: The back end used by the devices in the hot (or only) tier, valid values are SEGMENTED and RANDOM_BLOCK + default: SEGMENTED +- name: seastore_cold_device_type + type: str + level: dev + desc: The cold device type SeaStore uses, valid values are RANDOM_BLOCK_SSD, RANDOM_BLOCK_HDD, SSD and HDD. + This is only effective when there are two tiers, a hot one and a cold one. + default: RANDOM_BLOCK_HDD +- name: seastore_cold_backend_type + type: str + level: dev + desc: The backend used by cold devices (SEGMENTED or RANDOM_BLOCK) + default: RANDOM_BLOCK - name: seastore_cbjournal_size type: size level: dev - desc: Total size to use for CircularBoundedJournal if created, it is valid only if seastore_main_device_type is RANDOM_BLOCK + desc: Total size to use for CircularBoundedJournal if created, it is valid only if seastore_hot_device_type is RANDOM_BLOCK default: 5_G - name: seastore_multiple_tiers_stop_evict_ratio type: float diff --git a/src/crimson/os/seastore/device.cc b/src/crimson/os/seastore/device.cc index 2da0593a008..57303c37ce1 100644 --- a/src/crimson/os/seastore/device.cc +++ b/src/crimson/os/seastore/device.cc @@ -58,6 +58,8 @@ void device_superblock_t::validate() const } ceph_assert(block_size > 0); ceph_assert(config.spec.magic != 0); + // allow HDD devices use segmented backend + ceph_assert(config.spec.btype != backend_type_t::NONE); ceph_assert(config.spec.id <= DEVICE_ID_MAX_VALID); if (!config.major_dev) { ceph_assert(config.secondary_devices.empty()); @@ -114,19 +116,23 @@ void device_superblock_t::validate() const } seastar::future -Device::make_device(const std::string& device, device_type_t dtype) +Device::make_device( + const std::string& device, + device_type_t dtype, + backend_type_t btype) { - if (get_default_backend_of_device(dtype) == backend_type_t::SEGMENTED) { + if (btype == backend_type_t::SEGMENTED) { return SegmentManager::get_segment_manager(device, dtype ).then([](DeviceRef ret) { return ret; }); - } - assert(get_default_backend_of_device(dtype) == backend_type_t::RANDOM_BLOCK); - return get_rb_device(device - ).then([](DeviceRef ret) { - return ret; - }); + } else { + ceph_assert(btype != backend_type_t::NONE); + return get_rb_device(device + ).then([](DeviceRef ret) { + return ret; + }); + } } check_create_device_ret check_create_device( diff --git a/src/crimson/os/seastore/device.h b/src/crimson/os/seastore/device.h index 19dd1c9b2c5..c6df8109bc5 100644 --- a/src/crimson/os/seastore/device.h +++ b/src/crimson/os/seastore/device.h @@ -23,11 +23,13 @@ using magic_t = uint64_t; struct device_spec_t { magic_t magic = 0; device_type_t dtype = device_type_t::NONE; + backend_type_t btype = backend_type_t::NONE; device_id_t id = DEVICE_ID_NULL; DENC(device_spec_t, v, p) { DENC_START(1, 1, p); denc(v.magic, p); denc(v.dtype, p); + denc(v.btype, p); denc(v.id, p); DENC_FINISH(p); } @@ -55,12 +57,14 @@ struct device_config_t { uuid_d new_osd_fsid, device_id_t id, device_type_t d_type, + backend_type_t b_type, secondary_device_set_t sds) { return device_config_t{ true, device_spec_t{ (magic_t)std::rand(), d_type, + b_type, id}, seastore_meta_t{new_osd_fsid}, sds}; @@ -69,12 +73,14 @@ struct device_config_t { uuid_d new_osd_fsid, device_id_t id, device_type_t d_type, + backend_type_t b_type, magic_t magic) { return device_config_t{ false, device_spec_t{ magic, d_type, + b_type, id}, seastore_meta_t{new_osd_fsid}, secondary_device_set_t()}; @@ -329,7 +335,8 @@ public: static seastar::future make_device( const std::string &device, - device_type_t dtype); + device_type_t dtype, + backend_type_t btype); // interfaces used by each device shard public: diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc index 0995ada0413..2e62c2452dd 100644 --- a/src/crimson/os/seastore/extent_placement_manager.cc +++ b/src/crimson/os/seastore/extent_placement_manager.cc @@ -282,6 +282,7 @@ void ExtentPlacementManager::init( std::move(cold_cleaner), hot_tier_generations, pinboard); + ceph_assert(get_main_backend_type() != backend_type_t::NONE); if (cold_segment_cleaner) { ceph_assert(get_main_backend_type() == backend_type_t::SEGMENTED); ceph_assert(background_process.has_cold_tier()); diff --git a/src/crimson/os/seastore/random_block_manager/block_rb_manager.cc b/src/crimson/os/seastore/random_block_manager/block_rb_manager.cc index db8775a1b18..74bcbddb917 100644 --- a/src/crimson/os/seastore/random_block_manager/block_rb_manager.cc +++ b/src/crimson/os/seastore/random_block_manager/block_rb_manager.cc @@ -31,7 +31,7 @@ device_config_t get_rbm_ephemeral_device_config( ++secondary_index) { device_id_t secondary_id = static_cast(secondary_index); secondary_devices.insert({ - secondary_index, device_spec_t{magic, type, secondary_id} + secondary_index, device_spec_t{magic, type, backend_type_t::RANDOM_BLOCK, secondary_id} }); } } else { // index > 0 @@ -41,7 +41,7 @@ device_config_t get_rbm_ephemeral_device_config( device_id_t id = static_cast(DEVICE_ID_RANDOM_BLOCK_MIN + index); seastore_meta_t meta = {}; return {is_major_device, - device_spec_t{magic, type, id}, + device_spec_t{magic, type, backend_type_t::RANDOM_BLOCK, id}, meta, secondary_devices}; } diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc index ed483cb10c4..d8992f148e8 100644 --- a/src/crimson/os/seastore/seastore.cc +++ b/src/crimson/os/seastore/seastore.cc @@ -388,13 +388,16 @@ seastar::future SeaStore::start() bool is_test = false; #endif using crimson::common::get_conf; - std::string type = get_conf("seastore_main_device_type"); + std::string type = get_conf("seastore_hot_device_type"); device_type_t d_type = string_to_device_type(type); assert(d_type == device_type_t::SSD || d_type == device_type_t::RANDOM_BLOCK_SSD); + type = get_conf("seastore_hot_backend_type"); + auto b_type = string_to_backend_type(type); + INFO("main device type: {}, main backend type: {}", d_type, b_type); ceph_assert(root != ""); - DeviceRef device_obj = co_await Device::make_device(root, d_type); + DeviceRef device_obj = co_await Device::make_device(root, d_type, b_type); device = std::move(device_obj); co_await get_shard_nums(); co_await device->start(store_shard_nums); @@ -467,8 +470,11 @@ Device::access_ertr::future<> SeaStore::_mount() device_id_t id = device_entry.first; [[maybe_unused]] magic_t magic = device_entry.second.magic; device_type_t dtype = device_entry.second.dtype; - std::string path = fmt::format("{}/block.{}.{}", root, dtype, std::to_string(id)); - DeviceRef sec_dev = co_await Device::make_device(path, dtype); + backend_type_t btype = device_entry.second.btype; + auto btype_conf_str = get_conf("seastore_cold_backend_type"); + ceph_assert(string_to_backend_type(btype_conf_str) == btype); + std::string path = fmt::format("{}/block.{}", root, std::to_string(id)); + DeviceRef sec_dev = co_await Device::make_device(path, dtype, btype); co_await sec_dev->start(store_shard_nums); co_await sec_dev->mount(); auto sec_block_size = sec_dev->get_sharded_device(0).get_block_size(); @@ -662,6 +668,19 @@ seastar::future<> SeaStore::prepare_meta(uuid_d new_osd_fsid) co_await write_meta("mkfs_done", "yes"); } +std::optional parse_device_id(const seastar::sstring &name) { + auto prefix_len = sizeof("block.") - 1; + if (name.starts_with("block.") && name.length() > prefix_len) { + int id = 0; + std::string id_str = name.substr(prefix_len); + std::istringstream iss(id_str); + iss >> id; + assert(id < std::numeric_limits::max()); + return std::make_optional(id); + } + return std::nullopt; +} + Device::access_ertr::future<> SeaStore::_mkfs(uuid_d new_osd_fsid) { LOG_PREFIX(SeaStore::_mkfs); @@ -674,6 +693,12 @@ Device::access_ertr::future<> SeaStore::_mkfs(uuid_d new_osd_fsid) co_return; } DEBUG("mkfs_done does not exist, starting mkfs"); + auto dtype_str = get_conf("seastore_cold_device_type"); + auto dtype = string_to_device_type(dtype_str); + auto btype_str = get_conf("seastore_cold_backend_type"); + auto btype = string_to_backend_type(btype_str); + ceph_assert(!root.empty()); + INFO("secondary device type: {}, secondary backend type: {}", dtype, btype); secondary_device_set_t sds; if (!root.empty()) { seastar::file rdir = co_await seastar::open_directory(root); @@ -682,43 +707,40 @@ Device::access_ertr::future<> SeaStore::_mkfs(uuid_d new_osd_fsid) while (auto de = co_await lister()) { auto& entry = *de; DEBUG("found file: {}", entry.name); - if (entry.name.find("block.") == 0 && entry.name.length() > 6 ) { - // 6 for "block." - std::string entry_name = entry.name; - auto dtype_end = entry_name.find_first_of('.', 6); - device_type_t dtype = - string_to_device_type( - entry_name.substr(6, dtype_end - 6)); - if (dtype == device_type_t::NONE) { - // invalid device type - co_return; - } - auto id = std::stoi(entry_name.substr(dtype_end + 1)); - std::string path = fmt::format("{}/{}", root, entry_name); - DeviceRef sec_dev = co_await Device::make_device(path, dtype); - auto p_sec_dev = sec_dev.get(); - secondaries.emplace_back(std::move(sec_dev)); - co_await p_sec_dev->start(store_shard_nums); - magic_t magic = (magic_t)std::rand(); - sds.emplace((device_id_t)id, device_spec_t{magic, dtype, (device_id_t)id}); - co_await p_sec_dev->mkfs( - device_config_t::create_secondary(new_osd_fsid, id, dtype, magic) - ).handle_error(crimson::ct_error::assert_all("not possible")); - co_await set_secondaries(); + auto p = parse_device_id(entry.name); + if (!p) { + continue; } + std::string path = fmt::format("{}/{}", root, entry.name); + DeviceRef sec_dev = co_await Device::make_device(path, dtype, btype); + auto p_sec_dev = sec_dev.get(); + secondaries.emplace_back(std::move(sec_dev)); + co_await p_sec_dev->start(store_shard_nums); + magic_t magic = (magic_t)std::rand(); + auto id = *p; + sds.emplace((device_id_t)id, + device_spec_t{magic, dtype, btype, (device_id_t)id}); + co_await p_sec_dev->mkfs( + device_config_t::create_secondary(new_osd_fsid, id, dtype, btype, magic) + ).handle_error(crimson::ct_error::assert_all("not possible")); + co_await set_secondaries(); } co_await rdir.close(); } device_id_t id = 0; device_type_t d_type = device->get_device_type(); + backend_type_t b_type = device->get_backend_type(); assert(d_type == device_type_t::SSD || d_type == device_type_t::RANDOM_BLOCK_SSD); + assert(b_type != backend_type_t::NONE); if (d_type == device_type_t::RANDOM_BLOCK_SSD) { id = static_cast(DEVICE_ID_RANDOM_BLOCK_MIN); } DEBUG("creating primary device"); - co_await device->mkfs(device_config_t::create_primary(new_osd_fsid, id, d_type, sds)); + co_await device->mkfs( + device_config_t::create_primary( + new_osd_fsid, id, d_type, b_type, sds)); DEBUG("mounting {} secondaries", secondaries.size()); for (auto& sec_dev : secondaries) { co_await sec_dev->mount(); @@ -3007,7 +3029,7 @@ SeaStore::read_meta(const std::string& key) seastar::future SeaStore::get_default_device_class() { using crimson::common::get_conf; - std::string type = get_conf("seastore_main_device_type"); + std::string type = get_conf("seastore_hot_device_type"); return seastar::make_ready_future(type); } diff --git a/src/crimson/os/seastore/seastore_types.cc b/src/crimson/os/seastore/seastore_types.cc index adddf6fd3fd..6ac435cd069 100644 --- a/src/crimson/os/seastore/seastore_types.cc +++ b/src/crimson/os/seastore/seastore_types.cc @@ -1270,10 +1270,24 @@ std::ostream& operator<<(std::ostream& out, device_type_t t) } } -std::ostream& operator<<(std::ostream& out, backend_type_t btype) { - if (btype == backend_type_t::SEGMENTED) { - return out << "SEGMENTED"; +backend_type_t string_to_backend_type(const std::string &str) { + if (str == "SEGMENTED") { + return backend_type_t::SEGMENTED; + } else if (str == "RANDOM_BLOCK") { + return backend_type_t::RANDOM_BLOCK; } else { + ceph_abort("backend str not valid"); + return backend_type_t::SEGMENTED; + } +} + +std::ostream& operator<<(std::ostream& out, backend_type_t btype) { + switch (btype) { + case backend_type_t::NONE: + return out << "NONE"; + case backend_type_t::SEGMENTED: + return out << "SEGMENTED"; + case backend_type_t::RANDOM_BLOCK: return out << "RANDOM_BLOCK"; } } diff --git a/src/crimson/os/seastore/seastore_types.h b/src/crimson/os/seastore/seastore_types.h index 76f1dbe7ebc..72bee6d640b 100644 --- a/src/crimson/os/seastore/seastore_types.h +++ b/src/crimson/os/seastore/seastore_types.h @@ -974,7 +974,8 @@ std::ostream& operator<<(std::ostream& out, device_type_t t); bool can_delay_allocation(device_type_t type); device_type_t string_to_device_type(std::string type); -enum class backend_type_t { +enum class backend_type_t : uint8_t { + NONE, SEGMENTED, // SegmentManager: SSD, ZBD, HDD RANDOM_BLOCK // RBMDevice: RANDOM_BLOCK_SSD }; @@ -983,7 +984,7 @@ std::ostream& operator<<(std::ostream& out, backend_type_t); constexpr backend_type_t get_default_backend_of_device(device_type_t dtype) { assert(dtype != device_type_t::NONE && - dtype != device_type_t::NUM_TYPES); + dtype != device_type_t::NUM_TYPES); if (dtype >= device_type_t::HDD && dtype <= device_type_t::EPHEMERAL_MAIN) { return backend_type_t::SEGMENTED; @@ -992,6 +993,8 @@ constexpr backend_type_t get_default_backend_of_device(device_type_t dtype) { } } +backend_type_t string_to_backend_type(const std::string &str); + /** * Monotonically increasing identifier for the location of a * journal_record. diff --git a/src/crimson/os/seastore/segment_manager/ephemeral.cc b/src/crimson/os/seastore/segment_manager/ephemeral.cc index 393c20ed042..3144b9b9de4 100644 --- a/src/crimson/os/seastore/segment_manager/ephemeral.cc +++ b/src/crimson/os/seastore/segment_manager/ephemeral.cc @@ -59,6 +59,7 @@ device_config_t get_ephemeral_device_config( device_spec_t{ magic, get_sec_dtype(secondary_index), + backend_type_t::SEGMENTED, secondary_id } }); @@ -73,6 +74,7 @@ device_config_t get_ephemeral_device_config( device_spec_t{ magic, get_sec_dtype(index), + backend_type_t::SEGMENTED, id }, meta, diff --git a/src/crimson/tools/store_nbd/tm_driver.cc b/src/crimson/tools/store_nbd/tm_driver.cc index 47b3e8a7e4c..45fd8007684 100644 --- a/src/crimson/tools/store_nbd/tm_driver.cc +++ b/src/crimson/tools/store_nbd/tm_driver.cc @@ -168,7 +168,10 @@ seastar::future<> TMDriver::mkfs() { assert(config.path); logger().debug("mkfs"); - return Device::make_device(*config.path, device_type_t::SSD + return Device::make_device( + *config.path, + device_type_t::SSD, + backend_type_t::SEGMENTED ).then([this](DeviceRef dev) { device = std::move(dev); seastore_meta_t meta; @@ -176,9 +179,12 @@ seastar::future<> TMDriver::mkfs() return device->mkfs( device_config_t{ true, - (magic_t)std::rand(), - device_type_t::SSD, - 0, + device_spec_t{ + (magic_t)std::rand(), + device_type_t::SSD, + backend_type_t::SEGMENTED, + 0 + }, meta, secondary_device_set_t()}); }).safe_then([this] { @@ -210,7 +216,10 @@ seastar::future<> TMDriver::mount() { return (config.mkfs ? mkfs() : seastar::now() ).then([this] { - return Device::make_device(*config.path, device_type_t::SSD); + return Device::make_device( + *config.path, + device_type_t::SSD, + backend_type_t::SEGMENTED); }).then([this](DeviceRef dev) { device = std::move(dev); return device->mount(); diff --git a/src/test/crimson/seastore/nvmedevice/test_nvmedevice.cc b/src/test/crimson/seastore/nvmedevice/test_nvmedevice.cc index 4eb771c0b6e..ea13b065e00 100644 --- a/src/test/crimson/seastore/nvmedevice/test_nvmedevice.cc +++ b/src/test/crimson/seastore/nvmedevice/test_nvmedevice.cc @@ -65,6 +65,7 @@ TEST_F(nvdev_test_t, write_and_verify_test) device_spec_t{ (magic_t)std::rand(), device_type_t::RANDOM_BLOCK_SSD, + backend_type_t::RANDOM_BLOCK, static_cast(DEVICE_ID_RANDOM_BLOCK_MIN)}, seastore_meta_t{uuid_d()}, secondary_device_set_t()} diff --git a/src/vstart.sh b/src/vstart.sh index 859c804d2e4..76cf76bcd97 100755 --- a/src/vstart.sh +++ b/src/vstart.sh @@ -206,7 +206,10 @@ declare -a bluestore_db_devs declare -a bluestore_wal_devs declare -a secondary_block_devs declare -a cpu_table -secondary_block_devs_type="SSD" +seastore_hot_device_type="SSD" +seastore_hot_backend_type="SEGMENTED" +seastore_cold_device_type="SSD" +seastore_cold_backend_type="SEGMENTED" VSTART_SEC="client.vstart.sh" @@ -276,7 +279,10 @@ options: --seastore-device-size: set total size of seastore --seastore-devs: comma-separated list of blockdevs to use for seastore --seastore-secondary-devs: comma-separated list of secondary blockdevs to use for seastore - --seastore-secondary-devs-type: device type of all secondary blockdevs. HDD, SSD(default), ZNS or RANDOM_BLOCK_SSD + --seastore-main-device-type: device type of main blockdevs. (SSD or RANDOM_BLOCK_SSD) + --seastore-main-backend-type: the driver used by main blockdevs (SEGMENTED or RANDOM_BLOCK) + --seastore-secondary-device-type: device type of all secondary blockdevs. HDD, SSD(default), ZNS or RANDOM_BLOCK_SSD + --seastore-secondary-backend-type: the driver used by secondary blockdevs (SEGMENTED or RANDOM_BLOCK) --crimson-smp: number of cores to use for crimson --crimson-alien-num-threads: number of alien-tp threads --crimson-reactor-physical-only: use only one cpu per physical core for seastar reactors @@ -611,12 +617,24 @@ case $1 in parse_block_devs --seastore-devs "$2" shift ;; + --seastore-main-device-type) + seastore_hot_device_type="$2" + shift + ;; + --seastore-main-backend-type) + seastore_hot_backend_type="$2" + shift + ;; --seastore-secondary-devs) parse_secondary_devs --seastore-devs "$2" shift ;; - --seastore-secondary-devs-type) - secondary_block_devs_type="$2" + --seastore-secondary-device-type) + seastore_cold_device_type="$2" + shift + ;; + --seastore-secondary-backend-type) + seastore_cold_backend_type="$2" shift ;; --crimson-smp) @@ -936,6 +954,11 @@ EOF SEASTORE_OPTS=" seastore device size = $seastore_size" fi + SEASTORE_OPTS+=" + seastore_hot_device_type=$seastore_hot_device_type + seastore_hot_backend_type=$seastore_hot_backend_type + seastore_cold_device_type=$seastore_cold_device_type + seastore_cold_backend_type=$seastore_cold_backend_type" fi wconf < Date: Thu, 31 Jul 2025 16:16:56 +0800 Subject: [PATCH 15/51] crimson/os/seastore: introduce RotationalDevice Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/CMakeLists.txt | 1 + src/crimson/os/seastore/async_cleaner.h | 10 ++ src/crimson/os/seastore/device.cc | 5 +- .../os/seastore/extent_placement_manager.cc | 61 ++++--- .../os/seastore/random_block_manager.cc | 18 +- .../os/seastore/random_block_manager.h | 2 +- .../random_block_manager/hdd_device.cc | 156 ++++++++++++++++++ .../random_block_manager/hdd_device.h | 100 +++++++++++ .../random_block_manager/nvme_block_device.h | 20 +-- .../random_block_manager/rbm_device.cc | 5 +- .../random_block_manager/rbm_device.h | 22 ++- src/crimson/os/seastore/seastore_types.cc | 5 + src/crimson/os/seastore/seastore_types.h | 1 + .../os/seastore/transaction_manager.cc | 36 +++- 14 files changed, 386 insertions(+), 56 deletions(-) create mode 100644 src/crimson/os/seastore/random_block_manager/hdd_device.cc create mode 100644 src/crimson/os/seastore/random_block_manager/hdd_device.h diff --git a/src/crimson/os/seastore/CMakeLists.txt b/src/crimson/os/seastore/CMakeLists.txt index dbac365a3b2..1a707cfea5b 100644 --- a/src/crimson/os/seastore/CMakeLists.txt +++ b/src/crimson/os/seastore/CMakeLists.txt @@ -46,6 +46,7 @@ set(crimson_seastore_srcs random_block_manager/block_rb_manager.cc random_block_manager/rbm_device.cc random_block_manager/nvme_block_device.cc + random_block_manager/hdd_device.cc random_block_manager/avlallocator.cc journal/segmented_journal.cc journal/segment_allocator.cc diff --git a/src/crimson/os/seastore/async_cleaner.h b/src/crimson/os/seastore/async_cleaner.h index df978020be9..095164f32d8 100644 --- a/src/crimson/os/seastore/async_cleaner.h +++ b/src/crimson/os/seastore/async_cleaner.h @@ -1279,6 +1279,8 @@ public: virtual const std::set& get_device_ids() const = 0; + virtual backend_type_t get_backend_type() const = 0; + virtual std::size_t get_reclaim_size_per_cycle() const = 0; // Periodic hook for adaptive threshold control. Default: no-op. @@ -1503,6 +1505,10 @@ public: return sm_group->get_device_ids(); } + backend_type_t get_backend_type() const final { + return backend_type_t::SEGMENTED; + } + std::size_t get_reclaim_size_per_cycle() const final { return config.reclaim_bytes_per_cycle; } @@ -1851,6 +1857,10 @@ public: return rb_group->get_device_ids(); } + backend_type_t get_backend_type() const final { + return backend_type_t::RANDOM_BLOCK; + } + std::size_t get_reclaim_size_per_cycle() const final { return 0; } diff --git a/src/crimson/os/seastore/device.cc b/src/crimson/os/seastore/device.cc index 57303c37ce1..d9c16bc4a69 100644 --- a/src/crimson/os/seastore/device.cc +++ b/src/crimson/os/seastore/device.cc @@ -108,7 +108,8 @@ void device_superblock_t::validate() const ceph_assert(shard_infos[i].size > block_size && shard_infos[i].size % block_size == 0); ceph_assert_always(shard_infos[i].size <= DEVICE_OFF_MAX); - ceph_assert(journal_size > 0 && journal_size % block_size == 0); + ceph_assert((journal_size > 0 && journal_size % block_size == 0) || + config.spec.dtype == device_type_t::RANDOM_BLOCK_HDD); ceph_assert(shard_infos[i].start_offset < total_size && shard_infos[i].start_offset % block_size == 0); } @@ -128,7 +129,7 @@ Device::make_device( }); } else { ceph_assert(btype != backend_type_t::NONE); - return get_rb_device(device + return get_rb_device(device, dtype ).then([](DeviceRef ret) { return ret; }); diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc index 2e62c2452dd..2d29dbdb048 100644 --- a/src/crimson/os/seastore/extent_placement_manager.cc +++ b/src/crimson/os/seastore/extent_placement_manager.cc @@ -197,9 +197,8 @@ void ExtentPlacementManager::init( { LOG_PREFIX(ExtentPlacementManager::init); writer_refs.clear(); - auto cold_segment_cleaner = dynamic_cast(cold_cleaner.get()); dynamic_max_rewrite_generation = hot_tier_generations - 1; - if (cold_segment_cleaner) { + if (cold_cleaner) { dynamic_max_rewrite_generation = hot_tier_generations + cold_tier_generations - 1; } DEBUG("dynamic_max_rewrite_generation: {}, " @@ -254,36 +253,58 @@ void ExtentPlacementManager::init( for (auto *rb : rb_cleaner->get_rb_group()->get_rb_managers()) { add_device(rb->get_device()); } - } - - if (cold_segment_cleaner) { - // Cold DATA Segments - for (rewrite_gen_t gen = hot_tier_generations; gen <= dynamic_max_rewrite_generation; ++gen) { - writer_refs.emplace_back(std::make_unique(store_index, - data_category_t::DATA, gen, *cold_segment_cleaner, - *ool_segment_seq_allocator)); + for (rewrite_gen_t gen = OOL_GENERATION; gen < hot_tier_generations; ++gen) { data_writers_by_gen[generation_to_writer(gen)] = writer_refs.back().get(); } - for (rewrite_gen_t gen = hot_tier_generations; gen <= dynamic_max_rewrite_generation; ++gen) { - // Cold METADATA Segments - writer_refs.emplace_back(std::make_unique(store_index, - data_category_t::METADATA, gen, *cold_segment_cleaner, - *ool_segment_seq_allocator)); + for (rewrite_gen_t gen = OOL_GENERATION; gen < hot_tier_generations; ++gen) { md_writers_by_gen[generation_to_writer(gen)] = writer_refs.back().get(); } - for (auto *device : cold_segment_cleaner->get_segment_manager_group() - ->get_segment_managers()) { - add_device(device); - } } + if (cold_cleaner) { + if (cold_cleaner->get_backend_type() == backend_type_t::SEGMENTED) { + auto cold_segment_cleaner = static_cast(cold_cleaner.get()); + for (rewrite_gen_t gen = hot_tier_generations; gen <= dynamic_max_rewrite_generation; ++gen) { + writer_refs.emplace_back(std::make_unique(store_index, + data_category_t::DATA, gen, *cold_segment_cleaner, + *ool_segment_seq_allocator)); + data_writers_by_gen[generation_to_writer(gen)] = writer_refs.back().get(); + } + for (rewrite_gen_t gen = hot_tier_generations; gen <= dynamic_max_rewrite_generation; ++gen) { + writer_refs.emplace_back(std::make_unique(store_index, + data_category_t::METADATA, gen, *cold_segment_cleaner, + *ool_segment_seq_allocator)); + md_writers_by_gen[generation_to_writer(gen)] = writer_refs.back().get(); + } + for (auto *device : cold_segment_cleaner->get_segment_manager_group() + ->get_segment_managers()) { + add_device(device); + } + } else { + ceph_assert(cold_cleaner->get_backend_type() == backend_type_t::RANDOM_BLOCK); + auto rb_cleaner = static_cast(cold_cleaner.get()); + ceph_assert(rb_cleaner); + writer_refs.emplace_back(std::make_unique(rb_cleaner)); + for (rewrite_gen_t gen = hot_tier_generations; gen <= dynamic_max_rewrite_generation; ++gen) { + data_writers_by_gen[generation_to_writer(gen)] = writer_refs.back().get(); + } + for (rewrite_gen_t gen = hot_tier_generations; gen <= dynamic_max_rewrite_generation; ++gen) { + md_writers_by_gen[generation_to_writer(gen)] = writer_refs.back().get(); + } + for (auto *rb : rb_cleaner->get_rb_group()->get_rb_managers()) { + add_device(rb->get_device()); + } + } + } + + auto cold_cleaner_ = cold_cleaner.get(); background_process.init(std::move(trimmer), std::move(cleaner), std::move(cold_cleaner), hot_tier_generations, pinboard); ceph_assert(get_main_backend_type() != backend_type_t::NONE); - if (cold_segment_cleaner) { + if (cold_cleaner_) { ceph_assert(get_main_backend_type() == backend_type_t::SEGMENTED); ceph_assert(background_process.has_cold_tier()); } else { diff --git a/src/crimson/os/seastore/random_block_manager.cc b/src/crimson/os/seastore/random_block_manager.cc index b849c187843..64196db5ae6 100644 --- a/src/crimson/os/seastore/random_block_manager.cc +++ b/src/crimson/os/seastore/random_block_manager.cc @@ -5,17 +5,25 @@ #include "crimson/os/seastore/random_block_manager.h" #include "crimson/os/seastore/random_block_manager/nvme_block_device.h" #include "crimson/os/seastore/random_block_manager/rbm_device.h" +#include "crimson/os/seastore/random_block_manager/hdd_device.h" namespace crimson::os::seastore { seastar::future get_rb_device( - const std::string &device) + const std::string &device, device_type_t dtype) { - return seastar::make_ready_future( - std::make_unique< - random_block_device::nvme::NVMeBlockDevice - >(device + "/block")); + if (dtype == device_type_t::RANDOM_BLOCK_HDD) { + return seastar::make_ready_future( + std::make_unique< + random_block_device::RotationalDevice + >(device + "/block")); + } else { + return seastar::make_ready_future( + std::make_unique< + random_block_device::nvme::NVMeBlockDevice + >(device + "/block")); + } } } diff --git a/src/crimson/os/seastore/random_block_manager.h b/src/crimson/os/seastore/random_block_manager.h index dadd2578f45..f776483edf2 100644 --- a/src/crimson/os/seastore/random_block_manager.h +++ b/src/crimson/os/seastore/random_block_manager.h @@ -114,7 +114,7 @@ namespace random_block_device { } seastar::future> - get_rb_device(const std::string &device); + get_rb_device(const std::string &device, device_type_t dtype); std::ostream &operator<<(std::ostream &out, const rbm_extent_state_t &state); } diff --git a/src/crimson/os/seastore/random_block_manager/hdd_device.cc b/src/crimson/os/seastore/random_block_manager/hdd_device.cc new file mode 100644 index 00000000000..8a7d2133575 --- /dev/null +++ b/src/crimson/os/seastore/random_block_manager/hdd_device.cc @@ -0,0 +1,156 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "crimson/common/errorator-utils.h" +#include "crimson/os/seastore/logging.h" +#include "crimson/os/seastore/random_block_manager/hdd_device.h" + +SET_SUBSYS(seastore_device); + +namespace crimson::os::seastore::random_block_device { + +seastar::future<> RotationalDevice::start(uint32_t shard_nums) { + device_shard_nums = shard_nums; + auto num_shard_services = + (device_shard_nums + seastar::this_smp_shard_count() - 1 ) / + seastar::this_smp_shard_count(); + LOG_PREFIX(NVMeBlockDevice::start); + DEBUG("device_shard_nums={} seastar::smp={}, num_shard_services={}", + device_shard_nums, seastar::this_smp_shard_count(), num_shard_services); + return shard_devices.start(num_shard_services, device_path); +} + +RotationalDevice::mkfs_ret RotationalDevice::mkfs(device_config_t config) { + LOG_PREFIX(RotationalDevice::mkfs); + INFO("{}", config); + return shard_devices.local().mshard_devices[0]->do_primary_mkfs( + config, seastar::this_smp_shard_count(), 0); +} + +RotationalDevice::mount_ret RotationalDevice::mount() { + LOG_PREFIX(RotationalDevice::mount); + DEBUG("mount"); + return shard_devices.invoke_on_all([](auto &local_device) { + return seastar::do_for_each( + local_device.mshard_devices, + [](auto &mshard_device) { + return mshard_device->do_shard_mount( + ).handle_error( + crimson::ct_error::assert_all( + "Invalid error in RBMDevice::do_mount" + ) + ); + }); + }); +} + +read_ertr::future<> RotationalDevice::read( + uint64_t offset, + bufferptr &bptr) +{ + auto length = bptr.length(); + return device.dma_read(offset, bptr.c_str(), length + ).handle_exception([](auto e) -> read_ertr::future { + return crimson::ct_error::input_output_error::make(); + }).then([length](auto result) -> read_ertr::future<> { + if (result != length) { + return crimson::ct_error::input_output_error::make(); + } + return read_ertr::now(); + }); +} + +read_ertr::future<> RotationalDevice::_readv( + uint64_t offset, + std::vector ptrs) { + LOG_PREFIX(NVMeBlockDevice::_readv); + DEBUG("block: read offset {}, {} buffers", offset, ptrs.size()); + if (ptrs.size() == 0) { + return read_ertr::now(); + } + + std::vector iov; + size_t length = 0; + for (auto &ptr : ptrs) { + length += ptr.length(); + assert((ptr.length() % super.block_size) == 0); + iov.emplace_back(ptr.c_str(), ptr.length()); + } + return device.dma_read(offset, std::move(iov) + ).handle_exception( + [FNAME](auto e) -> read_ertr::future { + ERROR("read: dma_read got error{}", e); + return crimson::ct_error::input_output_error::make(); + }).then([length, FNAME](auto result) -> read_ertr::future<> { + if (result != length) { + ERROR("read: dma_read got error with not proper length"); + return crimson::ct_error::input_output_error::make(); + } + return read_ertr::now(); + }); +} + +write_ertr::future<> RotationalDevice::write( + uint64_t offset, + bufferptr bptr, + uint16_t stream) +{ + auto length = bptr.length(); + return seastar::do_with(std::move(bptr), [this, offset, length](auto &bptr) { + return device.dma_write(offset, bptr.c_str(), length + ).handle_exception([](auto e) -> write_ertr::future { + return crimson::ct_error::input_output_error::make(); + }).then([length](auto result) -> write_ertr::future<> { + if (result != length) { + return crimson::ct_error::input_output_error::make(); + } + return write_ertr::now(); + }); + }); +} + +open_ertr::future<> RotationalDevice::open( + const std::string &path, + seastar::open_flags mode) +{ + return seastar::open_file_dma(path, mode).then([this](auto file) { + device = std::move(file); + }).handle_exception([](auto e) -> open_ertr::future<> { + return crimson::ct_error::input_output_error::make(); + }); +} + +write_ertr::future<> RotationalDevice::writev( + uint64_t offset, + ceph::bufferlist bl, + uint16_t stream) { + bl.rebuild_aligned(super.block_size); + + return seastar::do_with( + bl.prepare_iovs(), + std::move(bl), + [this, offset](auto& iovs, auto& bl) + { + return write_ertr::parallel_for_each( + iovs, + [this, offset](auto& p) mutable + { + auto off = offset + p.offset; + auto len = p.length; + auto& iov = p.iov; + return device.dma_write(off, std::move(iov) + ).handle_exception( + [](auto e) -> write_ertr::future + { + return crimson::ct_error::input_output_error::make(); + }).then([len](size_t written) -> write_ertr::future<> { + if (written != len) { + return crimson::ct_error::input_output_error::make(); + } + return write_ertr::now(); + }); + }); + }); +} + +} diff --git a/src/crimson/os/seastore/random_block_manager/hdd_device.h b/src/crimson/os/seastore/random_block_manager/hdd_device.h new file mode 100644 index 00000000000..bd3755b0421 --- /dev/null +++ b/src/crimson/os/seastore/random_block_manager/hdd_device.h @@ -0,0 +1,100 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#pragma once + +#include "crimson/os/seastore/random_block_manager/rbm_device.h" + +namespace crimson::os::seastore::random_block_device { +class RotationalDevice : public RBMDevice { +public: + RotationalDevice( + std::string device_path, + store_index_t store_index = 0) + : RBMDevice(store_index), + device_path(device_path) + {} + ~RotationalDevice() = default; + + /// Device interface + + seastar::future<> start(uint32_t shard_nums) final; + + seastar::future<> stop() final { + return shard_devices.stop(); + } + + Device& get_sharded_device(store_index_t store_index) final { + assert(store_index < shard_devices.local().mshard_devices.size()); + return *shard_devices.local().mshard_devices[store_index]; + } + + mkfs_ret mkfs(device_config_t config) final; + + mount_ret mount() final; + + device_type_t get_device_type() const final { + return device_type_t::RANDOM_BLOCK_HDD; + } + + close_ertr::future<> close() final { + return device.close(); + } + + /// RBMDevice interface + + read_ertr::future<> read( + uint64_t offset, + bufferptr &bptr) final; + read_ertr::future<> _readv( + uint64_t offset, + std::vector ptrs) final; + + write_ertr::future<> write( + uint64_t offset, + bufferptr bptr, + uint16_t stream = 0) final; + + open_ertr::future<> open( + const std::string &path, + seastar::open_flags mode) final; + + write_ertr::future<> writev( + uint64_t offset, + ceph::bufferlist bl, + uint16_t stream = 0) final; + + stat_device_ret stat_device() final { + return seastar::file_stat(device_path, seastar::follow_symlink::yes + ).then([this](auto stat) { + return seastar::open_file_dma( + device_path, + seastar::open_flags::rw | seastar::open_flags::dsync + ).then([stat](auto file) mutable { + return seastar::do_with(std::move(file), [stat](auto &file) mutable { + return file.size().then([stat](auto size) mutable { + stat.size = size; + return stat; + }).then([file](auto stat) mutable { + return file.close().then([stat] { + return stat; + }); + }); + }); + }); + }).handle_exception([](auto e) -> stat_device_ret { + return crimson::ct_error::input_output_error::make(); + }); + } + + std::string get_device_path() const final { + return device_path; + } + +private: + std::string device_path; + seastar::file device; + seastar::sharded> shard_devices; +}; + +} diff --git a/src/crimson/os/seastore/random_block_manager/nvme_block_device.h b/src/crimson/os/seastore/random_block_manager/nvme_block_device.h index 3031b8d0191..21eae2fec3a 100644 --- a/src/crimson/os/seastore/random_block_manager/nvme_block_device.h +++ b/src/crimson/os/seastore/random_block_manager/nvme_block_device.h @@ -374,25 +374,7 @@ private: int namespace_id; // TODO: multi namespaces std::string device_path; - class MultiShardDevices { - public: - std::vector> mshard_devices; - - public: - MultiShardDevices(size_t count, - const std::string path) - : mshard_devices() { - mshard_devices.reserve(count); - for (size_t store_index = 0; store_index < count; ++store_index) { - mshard_devices.emplace_back(std::make_unique( - path, store_index)); - } - } - ~MultiShardDevices() { - mshard_devices.clear(); - } - }; - seastar::sharded shard_devices; + seastar::sharded> shard_devices; }; } diff --git a/src/crimson/os/seastore/random_block_manager/rbm_device.cc b/src/crimson/os/seastore/random_block_manager/rbm_device.cc index 083386b212d..2175565f8ec 100644 --- a/src/crimson/os/seastore/random_block_manager/rbm_device.cc +++ b/src/crimson/os/seastore/random_block_manager/rbm_device.cc @@ -29,7 +29,6 @@ RBMDevice::mkfs_ret RBMDevice::do_primary_mkfs(device_config_t config, maybe_create = check_create_device(get_device_path(), size); } - co_await std::move(maybe_create); auto st = co_await stat_device( ).safe_then([] (auto st) mutable { @@ -44,9 +43,11 @@ RBMDevice::mkfs_ret RBMDevice::do_primary_mkfs(device_config_t config, ); } + config.spec.id |= 0x80; const size_t cur_block_size = (*st).block_size; const size_t cur_total_size = (*st).size; - ceph_assert_always(journal_size > 0); + ceph_assert_always(journal_size > 0 || + config.spec.dtype == device_type_t::RANDOM_BLOCK_HDD); ceph_assert_always(cur_total_size >= journal_size); ceph_assert_always(shard_num > 0); diff --git a/src/crimson/os/seastore/random_block_manager/rbm_device.h b/src/crimson/os/seastore/random_block_manager/rbm_device.h index 3e87960849d..d380b2a78ac 100644 --- a/src/crimson/os/seastore/random_block_manager/rbm_device.h +++ b/src/crimson/os/seastore/random_block_manager/rbm_device.h @@ -111,7 +111,7 @@ public: return super.config.spec.magic; } - device_type_t get_device_type() const final { + virtual device_type_t get_device_type() const { return device_type_t::RANDOM_BLOCK_SSD; } @@ -278,4 +278,24 @@ EphemeralRBMDeviceRef create_test_ephemeral( uint64_t journal_size = DEFAULT_TEST_CBJOURNAL_SIZE, uint64_t data_size = DEFAULT_TEST_CBJOURNAL_SIZE); +template +class MultiShardDevices { + public: + std::vector> mshard_devices; + + public: + MultiShardDevices(size_t count, + const std::string path) + : mshard_devices() { + mshard_devices.reserve(count); + for (size_t store_index = 0; store_index < count; ++store_index) { + mshard_devices.emplace_back(std::make_unique( + path, store_index)); + } + } + ~MultiShardDevices() { + mshard_devices.clear(); + } +}; + } diff --git a/src/crimson/os/seastore/seastore_types.cc b/src/crimson/os/seastore/seastore_types.cc index 6ac435cd069..df83d1e5825 100644 --- a/src/crimson/os/seastore/seastore_types.cc +++ b/src/crimson/os/seastore/seastore_types.cc @@ -1243,6 +1243,9 @@ device_type_t string_to_device_type(std::string type) { if (type == "RANDOM_BLOCK_SSD") { return device_type_t::RANDOM_BLOCK_SSD; } + if (type == "RANDOM_BLOCK_HDD") { + return device_type_t::RANDOM_BLOCK_HDD; + } return device_type_t::NONE; } @@ -1265,6 +1268,8 @@ std::ostream& operator<<(std::ostream& out, device_type_t t) return out << "RANDOM_BLOCK_SSD"; case device_type_t::RANDOM_BLOCK_EPHEMERAL: return out << "RANDOM_BLOCK_EPHEMERAL"; + case device_type_t::RANDOM_BLOCK_HDD: + return out << "RANDOM_BLOCK_HDD"; default: return out << "INVALID_DEVICE_TYPE!"; } diff --git a/src/crimson/os/seastore/seastore_types.h b/src/crimson/os/seastore/seastore_types.h index 72bee6d640b..92661894dd8 100644 --- a/src/crimson/os/seastore/seastore_types.h +++ b/src/crimson/os/seastore/seastore_types.h @@ -966,6 +966,7 @@ enum class device_type_t : uint8_t { EPHEMERAL_MAIN, RANDOM_BLOCK_SSD, RANDOM_BLOCK_EPHEMERAL, + RANDOM_BLOCK_HDD, NUM_TYPES }; diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index 35960eff498..5dbfee0d8fa 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -1452,9 +1452,11 @@ TransactionManagerRef make_transaction_manager( auto rbs = std::make_unique(); auto backref_manager = create_backref_manager(*cache); SegmentManagerGroupRef cold_sms = nullptr; + RBMDeviceGroupRef cold_rbs = nullptr; std::vector segment_providers_by_id{DEVICE_ID_MAX, nullptr}; auto p_backend_type = primary_device->get_backend_type(); + INFO("primary backend: {}", p_backend_type); if (p_backend_type == backend_type_t::SEGMENTED) { auto dtype = primary_device->get_device_type(); @@ -1462,6 +1464,7 @@ TransactionManagerRef make_transaction_manager( dtype != device_type_t::EPHEMERAL_COLD); sms->add_segment_manager(static_cast(primary_device)); } else { + assert(p_backend_type != backend_type_t::NONE); auto rbm = std::make_unique( static_cast(primary_device), "", is_test); rbs->add_rb_manager(std::move(rbm)); @@ -1470,17 +1473,29 @@ TransactionManagerRef make_transaction_manager( for (auto &p_dev : secondary_devices) { if (p_dev->get_backend_type() == backend_type_t::SEGMENTED) { if (p_dev->get_device_type() == primary_device->get_device_type()) { + INFO("add {} to main segment backend", device_id_printer_t{p_dev->get_device_id()}); sms->add_segment_manager(static_cast(p_dev)); } else { if (!cold_sms) { cold_sms = std::make_unique(); } + INFO("add {} to cold segment backend", device_id_printer_t{p_dev->get_device_id()}); cold_sms->add_segment_manager(static_cast(p_dev)); } } else { + assert(p_backend_type != backend_type_t::NONE); auto rbm = std::make_unique( static_cast(p_dev), "", is_test); - rbs->add_rb_manager(std::move(rbm)); + if (p_dev->get_device_type() == primary_device->get_device_type()) { + INFO("add {} to rbm backend", device_id_printer_t{p_dev->get_device_id()}); + rbs->add_rb_manager(std::move(rbm)); + } else { + if (!cold_rbs) { + cold_rbs = std::make_unique(); + } + INFO("add {} to cold rbm backend", device_id_printer_t{p_dev->get_device_id()}); + cold_rbs->add_rb_manager(std::move(rbm)); + } } } @@ -1529,10 +1544,11 @@ TransactionManagerRef make_transaction_manager( AsyncCleanerRef cleaner; JournalRef journal; - SegmentCleanerRef cold_segment_cleaner = nullptr; + AsyncCleanerRef cold_cleaner = nullptr; if (cold_sms) { - cold_segment_cleaner = SegmentCleaner::create( + assert(!cold_rbs); + auto segment_cleaner = SegmentCleaner::create( store_index, cleaner_config, std::move(cold_sms), @@ -1542,11 +1558,19 @@ TransactionManagerRef make_transaction_manager( cleaner_is_detailed, /* is_cold = */ true); if (backend_type == backend_type_t::SEGMENTED) { - for (auto id : cold_segment_cleaner->get_device_ids()) { + for (auto id : segment_cleaner->get_device_ids()) { segment_providers_by_id[id] = - static_cast(cold_segment_cleaner.get()); + static_cast(segment_cleaner.get()); } } + cold_cleaner = std::move(segment_cleaner); + } else if (cold_rbs) { + cold_cleaner = RBMCleaner::create( + store_index, + std::move(cold_rbs), + *backref_manager, + *lba_manager, + cleaner_is_detailed); } if (backend_type == backend_type_t::SEGMENTED) { @@ -1586,7 +1610,7 @@ TransactionManagerRef make_transaction_manager( epm->init(std::move(journal_trimmer), std::move(cleaner), - std::move(cold_segment_cleaner), + std::move(cold_cleaner), cache->get_extent_pinboard()); epm->set_primary_device(primary_device); From 98f1892fb6b71cf1f9914e1b07a16fbbe610b6c8 Mon Sep 17 00:00:00 2001 From: Zhang Song Date: Thu, 31 Jul 2025 16:22:28 +0800 Subject: [PATCH 16/51] crimson/os/seastore: throttle bandwidth to secondary devices Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- src/common/options/crimson.yaml.in | 10 ++ .../os/seastore/extent_placement_manager.cc | 93 +++++++++++++------ .../os/seastore/extent_placement_manager.h | 93 ++++++++++++++++++- 3 files changed, 167 insertions(+), 29 deletions(-) diff --git a/src/common/options/crimson.yaml.in b/src/common/options/crimson.yaml.in index 7f4f1af71cc..c4bad3917c9 100644 --- a/src/common/options/crimson.yaml.in +++ b/src/common/options/crimson.yaml.in @@ -255,6 +255,11 @@ options: level: dev desc: The back end used by the devices in the hot (or only) tier, valid values are SEGMENTED and RANDOM_BLOCK default: SEGMENTED +- name: seastore_hot_backend_bw_throttle + type: size + level: advanced + desc: Size in bytes per second written to the hot devices, 0 indicates no limit + default: 0 - name: seastore_cold_device_type type: str level: dev @@ -266,6 +271,11 @@ options: level: dev desc: The backend used by cold devices (SEGMENTED or RANDOM_BLOCK) default: RANDOM_BLOCK +- name: seastore_cold_backend_bw_throttle + type: size + level: advanced + desc: Size in bytes per second written to cold devices, 0 indicates no limit + default: 100_M - name: seastore_cbjournal_size type: size level: dev diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc index 2d29dbdb048..554a479be8b 100644 --- a/src/crimson/os/seastore/extent_placement_manager.cc +++ b/src/crimson/os/seastore/extent_placement_manager.cc @@ -16,7 +16,8 @@ SegmentedOolWriter::SegmentedOolWriter( data_category_t category, rewrite_gen_t gen, SegmentProvider& sp, - SegmentSeqAllocator &ssa) + SegmentSeqAllocator &ssa, + TokenBucket &bucket) : store_index(store_index), segment_allocator(nullptr, category, gen, sp, ssa), record_submitter(crimson::common::get_conf( @@ -27,7 +28,8 @@ SegmentedOolWriter::SegmentedOolWriter( "seastore_journal_batch_flush_size"), crimson::common::get_conf( "seastore_journal_batch_preferred_fullness"), - segment_allocator) + segment_allocator), + token_bucket(bucket) { } @@ -182,10 +184,23 @@ SegmentedOolWriter::alloc_write_ool_extents( std::list& extents) { if (extents.empty()) { - return alloc_write_iertr::now(); + co_return; } - return seastar::with_gate(write_guard, [this, &t, &extents] { - return do_write(t, extents); + co_await seastar::with_gate( + write_guard, + [this, &t, &extents] -> alloc_write_iertr::future<> { + uint64_t size = 0; + for (auto &e : extents) { + size += e->get_length(); + } + try { + co_await trans_intr::make_interruptible( + token_bucket.get(size)); + co_await do_write(t, extents); + } catch (...) { + token_bucket.release(size); + throw; + } }); } @@ -207,6 +222,14 @@ void ExtentPlacementManager::init( cold_tier_generations); ceph_assert(dynamic_max_rewrite_generation > MIN_REWRITE_GENERATION); + auto main_bw_limit = crimson::common::get_conf< + Option::size_t>("seastore_hot_backend_bw_throttle"); + auto secondary_bw_limit = crimson::common::get_conf< + Option::size_t>("seastore_cold_backend_bw_throttle"); + + token_buckets.emplace_back(std::make_unique(main_bw_limit)); + token_buckets.back()->start(); + if (trimmer->get_backend_type() == backend_type_t::SEGMENTED) { DEBUG("initiating SegmentCleaner"); auto segment_cleaner = dynamic_cast(cleaner.get()); @@ -219,7 +242,7 @@ void ExtentPlacementManager::init( for (rewrite_gen_t gen = OOL_GENERATION; gen < hot_tier_generations; ++gen) { writer_refs.emplace_back(std::make_unique(store_index, data_category_t::DATA, gen, *segment_cleaner, - *ool_segment_seq_allocator)); + *ool_segment_seq_allocator, *token_buckets.back())); data_writers_by_gen[generation_to_writer(gen)] = writer_refs.back().get(); } @@ -228,7 +251,7 @@ void ExtentPlacementManager::init( for (rewrite_gen_t gen = OOL_GENERATION; gen < hot_tier_generations; ++gen) { writer_refs.emplace_back(std::make_unique(store_index, data_category_t::METADATA, gen, *segment_cleaner, - *ool_segment_seq_allocator)); + *ool_segment_seq_allocator, *token_buckets.back())); md_writers_by_gen[generation_to_writer(gen)] = writer_refs.back().get(); } @@ -246,7 +269,7 @@ void ExtentPlacementManager::init( data_writers_by_gen.resize(num_writers, nullptr); md_writers_by_gen.resize(num_writers, {}); writer_refs.emplace_back(std::make_unique( - rb_cleaner)); + rb_cleaner, *token_buckets.back())); // TODO: implement eviction in RBCleaner and introduce further writers data_writers_by_gen[generation_to_writer(OOL_GENERATION)] = writer_refs.back().get(); md_writers_by_gen[generation_to_writer(OOL_GENERATION)] = writer_refs.back().get(); @@ -262,18 +285,20 @@ void ExtentPlacementManager::init( } if (cold_cleaner) { + token_buckets.emplace_back(std::make_unique(secondary_bw_limit)); + token_buckets.back()->start(); if (cold_cleaner->get_backend_type() == backend_type_t::SEGMENTED) { auto cold_segment_cleaner = static_cast(cold_cleaner.get()); for (rewrite_gen_t gen = hot_tier_generations; gen <= dynamic_max_rewrite_generation; ++gen) { writer_refs.emplace_back(std::make_unique(store_index, data_category_t::DATA, gen, *cold_segment_cleaner, - *ool_segment_seq_allocator)); + *ool_segment_seq_allocator, *token_buckets.back())); data_writers_by_gen[generation_to_writer(gen)] = writer_refs.back().get(); } for (rewrite_gen_t gen = hot_tier_generations; gen <= dynamic_max_rewrite_generation; ++gen) { writer_refs.emplace_back(std::make_unique(store_index, data_category_t::METADATA, gen, *cold_segment_cleaner, - *ool_segment_seq_allocator)); + *ool_segment_seq_allocator, *token_buckets.back())); md_writers_by_gen[generation_to_writer(gen)] = writer_refs.back().get(); } for (auto *device : cold_segment_cleaner->get_segment_manager_group() @@ -284,7 +309,7 @@ void ExtentPlacementManager::init( ceph_assert(cold_cleaner->get_backend_type() == backend_type_t::RANDOM_BLOCK); auto rb_cleaner = static_cast(cold_cleaner.get()); ceph_assert(rb_cleaner); - writer_refs.emplace_back(std::make_unique(rb_cleaner)); + writer_refs.emplace_back(std::make_unique(rb_cleaner, *token_buckets.back())); for (rewrite_gen_t gen = hot_tier_generations; gen <= dynamic_max_rewrite_generation; ++gen) { data_writers_by_gen[generation_to_writer(gen)] = writer_refs.back().get(); } @@ -594,6 +619,9 @@ ExtentPlacementManager::close() { LOG_PREFIX(ExtentPlacementManager::close); INFO("started"); + for (auto &token_bucket : token_buckets) { + token_bucket->stop(); + } return crimson::do_for_each(data_writers_by_gen, [](auto &writer) { if (writer) { return writer->close(); @@ -1179,22 +1207,35 @@ RandomBlockOolWriter::alloc_write_ool_extents( std::list& extents) { if (extents.empty()) { - return alloc_write_iertr::now(); + co_return; } - return seastar::with_gate(write_guard, [this, &t, &extents] { - seastar::lw_shared_ptr ptr = - seastar::make_lw_shared(); - ptr->pending_extents = t.get_pre_alloc_list(); - assert(!t.is_conflicted()); - t.set_pending_ool(ptr); - return do_write(t, extents - ).finally([this, ptr=ptr] { - if (ptr->is_conflicted) { - for (auto &e : ptr->pending_extents) { - rb_cleaner->mark_space_free(e->get_paddr(), e->get_length()); - } - } - }); + co_await seastar::with_gate( + write_guard, + [this, &t, &extents] -> alloc_write_iertr::future<> { + uint64_t size = 0; + for (auto &extent : extents) { + size += extent->get_length(); + } + try { + co_await trans_intr::make_interruptible( + token_bucket.get(size)); + seastar::lw_shared_ptr ptr = + seastar::make_lw_shared(); + ptr->pending_extents = t.get_pre_alloc_list(); + assert(!t.is_conflicted()); + t.set_pending_ool(ptr); + co_await do_write(t, extents + ).finally([this, ptr=ptr] { + if (ptr->is_conflicted) { + for (auto &e : ptr->pending_extents) { + rb_cleaner->mark_space_free(e->get_paddr(), e->get_length()); + } + } + }); + } catch (...) { + token_bucket.release(size); + throw; + } }); } diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index 001f2c15b46..3b94ae99384 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -23,6 +23,89 @@ namespace crimson::os::seastore { class Cache; +class TokenBucket { + struct Blocker { + uint64_t size = 0; + seastar::promise<> pr; + }; +public: + TokenBucket(uint64_t mt) : + tokens(mt), max_tokens(mt), timer() {} + + void start() { + if (max_tokens != 0) { + tokens = max_tokens; + timer.set_callback([this] { + if (tokens == max_tokens) { + return; + } + assert(tokens < max_tokens); + tokens += std::min( + max_tokens / 10 + 1, + max_tokens - tokens); + do_wake(); + }); + if (!timer.armed()) { + timer.arm_periodic(std::chrono::milliseconds(100)); + } + } + } + + void stop() { + if (max_tokens != 0) { + timer.cancel(); + tokens = std::numeric_limits::max(); + do_wake(); + } + } + + seastar::future<> get(uint64_t size) { + if (max_tokens == 0) { + return seastar::now(); + } + if (tokens < size) { + size -= tokens; + tokens = 0; + blockers.emplace_back(size); + return blockers.back().pr.get_future(); + } else { + tokens -= size; + return seastar::now(); + } + } + + void release(uint64_t size) { + if (tokens == max_tokens) { + return; + } + assert(tokens < max_tokens); + tokens += std::min(size, max_tokens - tokens); + do_wake(); + } + +private: + void do_wake() { + while (!blockers.empty()) { + auto &next = blockers.front(); + if (tokens < next.size) { + next.size -= tokens; + tokens = 0; + break; + } else { + tokens -= next.size; + next.pr.set_value(); + blockers.pop_front(); + } + } + } + uint64_t tokens; + const uint64_t max_tokens; + seastar::timer timer; + std::list blockers; +}; + +using TokenBucketRef = std::unique_ptr; + /** * ExtentOolWriter * @@ -75,7 +158,8 @@ public: data_category_t category, rewrite_gen_t gen, SegmentProvider &sp, - SegmentSeqAllocator &ssa); + SegmentSeqAllocator &ssa, + TokenBucket &buckets); backend_type_t get_type() const final { return backend_type_t::SEGMENTED; @@ -129,13 +213,14 @@ private: journal::SegmentAllocator segment_allocator; journal::RecordSubmitter record_submitter; seastar::gate write_guard; + TokenBucket &token_bucket; }; class RandomBlockOolWriter : public ExtentOolWriter { public: - RandomBlockOolWriter(RBMCleaner* rb_cleaner) : - rb_cleaner(rb_cleaner) {} + RandomBlockOolWriter(RBMCleaner* rb_cleaner, TokenBucket &bucket) : + rb_cleaner(rb_cleaner), token_bucket(bucket) {} backend_type_t get_type() const final { return backend_type_t::RANDOM_BLOCK; @@ -222,6 +307,7 @@ private: seastar::gate write_guard; writer_stats_t w_stats; mutable writer_stats_t last_w_stats; + TokenBucket &token_bucket; }; struct cleaner_usage_t { @@ -1215,6 +1301,7 @@ private: std::vector data_writers_by_gen; // gen 0 METADATA writer is the journal writer std::vector md_writers_by_gen; + std::vector token_buckets; std::vector devices_by_id; Device* primary_device = nullptr; From 43d0b3316af876682d088c3279cd1763db9bc767 Mon Sep 17 00:00:00 2001 From: Zhang Song Date: Tue, 26 Mar 2024 11:52:15 +0800 Subject: [PATCH 17/51] crimson/osd: create cold devices when mkfs Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- src/common/options/crimson.yaml.in | 5 +++++ src/crimson/osd/main.cc | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/src/common/options/crimson.yaml.in b/src/common/options/crimson.yaml.in index c4bad3917c9..755fb0d5f94 100644 --- a/src/common/options/crimson.yaml.in +++ b/src/common/options/crimson.yaml.in @@ -260,6 +260,11 @@ options: level: advanced desc: Size in bytes per second written to the hot devices, 0 indicates no limit default: 0 +- name: seastore_cold_devices_count + type: uint + level: dev + desc: create cold devices during mkfs + default: 0 - name: seastore_cold_device_type type: str level: dev diff --git a/src/crimson/osd/main.cc b/src/crimson/osd/main.cc index 86bbbe1ab98..744cddb9a42 100644 --- a/src/crimson/osd/main.cc +++ b/src/crimson/osd/main.cc @@ -264,6 +264,15 @@ int main(int argc, const char* argv[]) // use a random osd uuid if not specified osd_uuid.generate_random(); } + if (auto c = local_conf().get_val("seastore_cold_devices_count"); + c != 0) { + auto root = local_conf().get_val("osd_data"); + for (size_t i = 1; i <= c; i++) { + auto path = fmt::format("{}/block.{}", root, i); + seastar::touch_directory(path).get(); + } + seastar::sync_directory(root).get(); + } osd.mkfs( *store, whoami, From afde4c6dcb94dbbed154764454c5ea57cc6e2cdb Mon Sep 17 00:00:00 2001 From: Zhang Song Date: Wed, 3 Sep 2025 16:05:38 +0800 Subject: [PATCH 18/51] crimson/os/seastore: add write through policy Write extents larger than some threshold to the cold tier directly Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- src/common/options/crimson.yaml.in | 5 +++ src/crimson/os/seastore/cache.cc | 16 +++++----- src/crimson/os/seastore/cache.h | 19 ++++++----- src/crimson/os/seastore/cached_extent.h | 19 ++++++++++- .../os/seastore/extent_placement_manager.h | 32 ++++++++++++++++--- src/crimson/os/seastore/seastore_types.cc | 10 ++++++ src/crimson/os/seastore/seastore_types.h | 8 +++++ .../os/seastore/transaction_manager.cc | 24 ++++++++++---- src/crimson/os/seastore/transaction_manager.h | 6 +++- .../seastore/test_btree_lba_manager.cc | 8 +++-- .../seastore/test_transaction_manager.cc | 1 + 11 files changed, 117 insertions(+), 31 deletions(-) diff --git a/src/common/options/crimson.yaml.in b/src/common/options/crimson.yaml.in index 755fb0d5f94..1c7c3a18240 100644 --- a/src/common/options/crimson.yaml.in +++ b/src/common/options/crimson.yaml.in @@ -388,3 +388,8 @@ options: level: advanced desc: Size in bytes of extents to be demoted from logical bucket default: 2_M +- name: seastore_write_through_size + type: size + level: dev + desc: Select write through policy when data length is greater than this value. + default: 512_K diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc index 5bcd0adf5b6..10db3a0732b 100644 --- a/src/crimson/os/seastore/cache.cc +++ b/src/crimson/os/seastore/cache.cc @@ -1264,28 +1264,26 @@ std::vector Cache::alloc_new_data_extents_by_type( Transaction &t, ///< [in, out] current transaction extent_types_t type, ///< [in] type tag extent_len_t length, ///< [in] length - placement_hint_t hint, ///< [in] user hint - rewrite_gen_t gen, ///< [in] rewrite generation - bool is_tracked + alloc_option_t opt ///< [in] allocation options ) { LOG_PREFIX(Cache::alloc_new_data_extents_by_type); SUBDEBUGT(seastore_cache, "allocate {} 0x{:x}B, hint={}, gen={}", - t, type, length, hint, rewrite_gen_printer_t{gen}); + t, type, length, opt.hint, rewrite_gen_printer_t{opt.gen}); ceph_assert(get_extent_category(type) == data_category_t::DATA); std::vector res; switch (type) { case extent_types_t::OBJECT_DATA_BLOCK: { auto extents = alloc_new_data_extents< - ObjectDataBlock>(t, length, {hint, gen, is_tracked}); + ObjectDataBlock>(t, length, std::move(opt)); res.insert(res.begin(), extents.begin(), extents.end()); } return res; case extent_types_t::TEST_BLOCK: { auto extents = alloc_new_data_extents< - TestBlock>(t, length, {hint, gen, is_tracked}); + TestBlock>(t, length, std::move(opt)); res.insert(res.begin(), extents.begin(), extents.end()); } return res; @@ -2200,7 +2198,8 @@ void Cache::init() P_ADDR_ROOT, PLACEMENT_HINT_NULL, NULL_GENERATION, - TRANS_ID_NULL); + TRANS_ID_NULL, + write_policy_t::WRITE_BACK); root->set_modify_time(seastar::lowres_system_clock::now()); INFO("init root -- {}", *root); add_extent(root); @@ -2624,7 +2623,8 @@ Cache::_get_absent_extent_by_type( offset, PLACEMENT_HINT_NULL, NULL_GENERATION, - TRANS_ID_NULL); + TRANS_ID_NULL, + write_policy_t::WRITE_BACK); DEBUGT("{} length=0x{:x} is absent, add extent ... -- {}", t, type, length, *ret); add_extent(ret); diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h index 6f893bc6c3c..df3fdf90e98 100644 --- a/src/crimson/os/seastore/cache.h +++ b/src/crimson/os/seastore/cache.h @@ -857,7 +857,8 @@ private: offset, PLACEMENT_HINT_NULL, NULL_GENERATION, - TRANS_ID_NULL); + TRANS_ID_NULL, + write_policy_t::WRITE_BACK); SUBDEBUGT(seastore_cache, "{} {}~0x{:x} is absent, add extent and reading range 0x{:x}~0x{:x} ... -- {}", t, T::TYPE, offset, length, partial_off, partial_len, *ret); @@ -896,7 +897,8 @@ private: offset, PLACEMENT_HINT_NULL, NULL_GENERATION, - TRANS_ID_NULL); + TRANS_ID_NULL, + write_policy_t::WRITE_BACK); SUBDEBUG(seastore_cache, "{} {}~0x{:x} is absent, add extent and reading range 0x{:x}~0x{:x} ... -- {}", T::TYPE, offset, length, partial_off, partial_len, *ret); @@ -1175,7 +1177,8 @@ public: result->paddr, opt.hint, result->gen, - t.get_trans_id()); + t.get_trans_id(), + write_policy_t::WRITE_BACK); t.add_fresh_extent(ret); SUBDEBUGT(seastore_cache, "allocated {} 0x{:x}B extent at {}, hint={}, gen={} -- {}", @@ -1213,7 +1216,8 @@ public: result.paddr, opt.hint, result.gen, - t.get_trans_id()); + t.get_trans_id(), + opt.write_policy); t.add_fresh_extent(ret); SUBDEBUGT(seastore_cache, "allocated {} 0x{:x}B extent at {}, hint={}, gen={} -- {}", @@ -1255,7 +1259,8 @@ public: remap_paddr, PLACEMENT_HINT_NULL, NULL_GENERATION, - t.get_trans_id()); + t.get_trans_id(), + write_policy_t::WRITE_BACK); auto extent = ext->template cast(); extent->set_laddr(remap_laddr); @@ -1297,9 +1302,7 @@ public: Transaction &t, ///< [in, out] current transaction extent_types_t type, ///< [in] type tag extent_len_t length, ///< [in] length - placement_hint_t hint, ///< [in] user hint - rewrite_gen_t gen, ///< [in] rewrite generation - bool is_tracked + alloc_option_t opt ///< [in] allocation options ); /** diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h index 3d69ea73f32..d40e9e632bf 100644 --- a/src/crimson/os/seastore/cached_extent.h +++ b/src/crimson/os/seastore/cached_extent.h @@ -360,12 +360,14 @@ public: paddr_t paddr, placement_hint_t hint, rewrite_gen_t gen, - transaction_id_t trans_id) { + transaction_id_t trans_id, + write_policy_t policy) { state = _state; set_paddr(paddr); user_hint = hint; rewrite_generation = gen; pending_for_transaction = trans_id; + write_policy = policy; } void set_modify_time(sea_time_point t) { @@ -537,6 +539,7 @@ public: << ", last_committed_crc=" << last_committed_crc << ", refcount=" << use_count() << ", user_hint=" << user_hint + << ", write_policy=" << write_policy << ", rewrite_gen=" << rewrite_gen_printer_t{rewrite_generation} << ", pending_io="; if (is_pending_io()) { @@ -916,6 +919,18 @@ public: is_shadow = b; } + write_policy_t get_write_policy() const { + return write_policy; + } + + void set_write_policy(write_policy_t w) { + write_policy = w; + } + + void reset_write_policy() { + write_policy = write_policy_t::WRITE_BACK; + } + private: template friend class read_set_item_t; @@ -1027,6 +1042,8 @@ private: placement_hint_t user_hint = PLACEMENT_HINT_NULL; + write_policy_t write_policy = write_policy_t::WRITE_BACK; + // the target rewrite generation for the followup rewrite // or the rewrite generation for the fresh write rewrite_gen_t rewrite_generation = NULL_GENERATION; diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index 3b94ae99384..210e8d38442 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -370,7 +370,9 @@ public: ool_segment_seq_allocator( std::make_unique(segment_type_t::OOL)), max_data_allocation_size(crimson::common::get_conf( - "seastore_max_data_allocation_size")) + "seastore_max_data_allocation_size")), + write_through_size(crimson::common::get_conf( + "seastore_write_through_size")) { LOG_PREFIX(ExtentPlacementManager::ExtentPlacementManager); devices_by_id.resize(DEVICE_ID_MAX, nullptr); @@ -448,6 +450,7 @@ public: placement_hint_t hint; rewrite_gen_t gen; bool is_tracked; + write_policy_t write_policy = write_policy_t::WRITE_BACK; #ifdef UNIT_TESTS_BUILT std::optional external_paddr = std::nullopt; #endif @@ -468,7 +471,8 @@ public: assert(opt.gen == INIT_GENERATION || opt.hint == placement_hint_t::REWRITE); data_category_t category = get_extent_category(type); - opt.gen = adjust_generation(category, type, opt.hint, opt.gen, opt.is_tracked); + opt.gen = adjust_generation( + category, type, opt.hint, opt.gen, opt.write_policy, opt.is_tracked); paddr_t addr; #ifdef UNIT_TESTS_BUILT @@ -508,7 +512,8 @@ public: assert(opt.gen == INIT_GENERATION || opt.hint == placement_hint_t::REWRITE); data_category_t category = get_extent_category(type); - opt.gen = adjust_generation(category, type, opt.hint, opt.gen, opt.is_tracked); + opt.gen = adjust_generation( + category, type, opt.hint, opt.gen, opt.write_policy, opt.is_tracked); assert(opt.gen != INLINE_GENERATION); // XXX: bp might be extended to point to different memory (e.g. PMem) @@ -546,6 +551,13 @@ public: return allocs; } + write_policy_t get_write_policy(extent_types_t type, extent_len_t length) const { + if (has_cold_tier() && length >= write_through_size && is_data_type(type)) { + return write_policy_t::WRITE_THROUGH; + } + return write_policy_t::WRITE_BACK; + } + #ifdef UNIT_TESTS_BUILT void prefill_fragmented_devices() { LOG_PREFIX(ExtentPlacementManager::prefill_fragmented_devices); @@ -715,6 +727,7 @@ private: extent_types_t type, placement_hint_t hint, rewrite_gen_t gen, + write_policy_t policy, bool is_tracked) { assert(is_real_type(type)); if (is_root_type(type)) { @@ -744,10 +757,20 @@ private: } } else { assert(category == data_category_t::DATA); - gen = OOL_GENERATION; + if (background_process.has_cold_tier() && + policy == write_policy_t::WRITE_THROUGH) { + gen = hot_tier_generations; + } else { + assert(policy != write_policy_t::WRITE_THROUGH); + gen = OOL_GENERATION; + } } } else if (background_process.has_cold_tier()) { gen = background_process.adjust_generation(gen); + if (gen <= hot_tier_generations && + policy == write_policy_t::WRITE_THROUGH) { + gen = hot_tier_generations; + } } if (is_tracked && gen >= hot_tier_generations && @@ -1316,6 +1339,7 @@ private: // TODO: drop once paddr->journal_seq_t is introduced SegmentSeqAllocatorRef ool_segment_seq_allocator; extent_len_t max_data_allocation_size = 0; + std::size_t write_through_size = 0; friend class ::transaction_manager_test_t; friend class Cache; diff --git a/src/crimson/os/seastore/seastore_types.cc b/src/crimson/os/seastore/seastore_types.cc index df83d1e5825..65f9a39b12a 100644 --- a/src/crimson/os/seastore/seastore_types.cc +++ b/src/crimson/os/seastore/seastore_types.cc @@ -1224,6 +1224,16 @@ std::ostream& operator<<(std::ostream& out, placement_hint_t h) } } +std::ostream& operator<<(std::ostream& out, write_policy_t w) +{ + switch(w) { + case write_policy_t::WRITE_BACK: + return out << "WRITE_BACK"; + case write_policy_t::WRITE_THROUGH: + return out << "WRITE_THROUGH"; + } +} + bool can_delay_allocation(device_type_t type) { // Some types of device may not support delayed allocation, for example PMEM. // All types of device currently support delayed allocation. diff --git a/src/crimson/os/seastore/seastore_types.h b/src/crimson/os/seastore/seastore_types.h index 92661894dd8..4f02157111d 100644 --- a/src/crimson/os/seastore/seastore_types.h +++ b/src/crimson/os/seastore/seastore_types.h @@ -2070,6 +2070,13 @@ constexpr bool is_real_type(extent_types_t type) { std::ostream &operator<<(std::ostream &out, extent_types_t t); +enum class write_policy_t { + WRITE_BACK, + WRITE_THROUGH +}; + +std::ostream& operator<<(std::ostream& out, write_policy_t w); + /** * rewrite_gen_t * @@ -3654,6 +3661,7 @@ template <> struct fmt::formatter : f template <> struct fmt::formatter : fmt::ostream_formatter {}; template <> struct fmt::formatter : fmt::ostream_formatter {}; template <> struct fmt::formatter : fmt::ostream_formatter {}; +template <> struct fmt::formatter : fmt::ostream_formatter {}; #endif namespace fmt { diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index 5dbfee0d8fa..a32573a1cc9 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -878,6 +878,7 @@ TransactionManager::rewrite_logical_extent( // get target rewrite generation extent->get_rewrite_generation(), is_tracked)->cast(); + assert(nextent->get_write_policy() != write_policy_t::WRITE_THROUGH); nextent->rewrite(t, *extent, 0); DEBUGT("rewriting meta -- {} to {}", t, *extent, *nextent); @@ -919,10 +920,15 @@ TransactionManager::rewrite_logical_extent( t, extent->get_type(), extent->get_length(), - extent->get_user_hint(), - // get target rewrite generation - extent->get_rewrite_generation(), - is_tracked); + { + extent->get_user_hint(), + // get target rewrite generation + extent->get_rewrite_generation(), + is_tracked, + // WRITH_THROUGH is only effective for client io, so + // always set the write policy to WRITE_BACK here + write_policy_t::WRITE_BACK + }); extent_len_t off = 0; auto left = extent->get_length(); extent_ref_count_t refcount = 0; @@ -934,6 +940,7 @@ TransactionManager::rewrite_logical_extent( t.force_rewrite_conflict = (extents.size() > 1); for (auto &_nextent : extents) { auto nextent = _nextent->template cast(); + assert(nextent->get_write_policy() != write_policy_t::WRITE_THROUGH); bool first_extent = (off == 0); ceph_assert(left >= nextent->get_length()); nextent->rewrite(t, *extent, off); @@ -1213,9 +1220,12 @@ TransactionManager::promote_extent( t, orig_ext->get_type(), orig_ext->get_length(), - placement_hint_t::HOT, - INIT_GENERATION, - true); + { + placement_hint_t::HOT, + INIT_GENERATION, + true, + write_policy_t::WRITE_BACK + }); promoted_extents.reserve(promoted_raw_extents.size()); diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index 07c1a11f455..5c88925a7dd 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -591,7 +591,11 @@ public: SUBDEBUGT(seastore_tm, "{} hint {}~0x{:x} phint={} ...", t, T::TYPE, laddr_hint, len, placement_hint); auto exts = cache->alloc_new_data_extents( - t, len, {placement_hint, INIT_GENERATION}); + t, len, + { + placement_hint, INIT_GENERATION, false, + epm->get_write_policy(T::TYPE, len) + }); // user must initialize the logical extent themselves assert(is_user_transaction(t.get_src())); for (auto& ext : exts) { diff --git a/src/test/crimson/seastore/test_btree_lba_manager.cc b/src/test/crimson/seastore/test_btree_lba_manager.cc index c79ce6e5e25..ee27a850e76 100644 --- a/src/test/crimson/seastore/test_btree_lba_manager.cc +++ b/src/test/crimson/seastore/test_btree_lba_manager.cc @@ -327,7 +327,9 @@ struct lba_btree_test : btree_test_base { check.emplace(addr, get_map_val(len, TestBlock::TYPE)); lba_btree_update([=, this](auto &btree, auto &t) { auto extents = cache->alloc_new_data_extents( - t, TestBlock::SIZE, {placement_hint_t::HOT, 0, false, get_paddr()}); + t, TestBlock::SIZE, + {placement_hint_t::HOT, 0, false, + write_policy_t::WRITE_BACK, get_paddr()}); return seastar::do_with( std::move(extents), [this, addr, &t, len, &btree](auto &extents) { @@ -545,7 +547,9 @@ struct btree_lba_manager_test : btree_test_base { *t.t, [=, this](auto &t) { auto extents = cache->alloc_new_data_extents( - t, TestBlock::SIZE, {placement_hint_t::HOT, 0, false, get_paddr()}); + t, TestBlock::SIZE, + {placement_hint_t::HOT, 0, false, + write_policy_t::WRITE_BACK, get_paddr()}); return seastar::do_with( std::vector( extents.begin(), extents.end()), diff --git a/src/test/crimson/seastore/test_transaction_manager.cc b/src/test/crimson/seastore/test_transaction_manager.cc index 834304bcdb4..f5b5c9f77a3 100644 --- a/src/test/crimson/seastore/test_transaction_manager.cc +++ b/src/test/crimson/seastore/test_transaction_manager.cc @@ -1150,6 +1150,7 @@ struct transaction_manager_test_t : t, placement_hint_t::HOT, gen, + write_policy_t::WRITE_BACK, false); if (expected_generations[t][gen] != epm_gen) { logger().error("caller: {}, extent type: {}, input generation: {}, " From c969a8b9b5784c4df75ccf80e89e3ae4f9c11b92 Mon Sep 17 00:00:00 2001 From: Zhang Song Date: Fri, 8 Aug 2025 15:50:52 +0800 Subject: [PATCH 19/51] crimson/os/seastore: update logical bucket cache when processing read/write transactions All client accessed extents are to be tracked by logical bucket cache before eviction Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- src/common/hobject.h | 11 ++++--- .../os/seastore/object_data_handler.cc | 9 ++++++ src/crimson/os/seastore/transaction.h | 11 +++++++ .../os/seastore/transaction_manager.cc | 15 +++++++++ src/crimson/os/seastore/transaction_manager.h | 31 ++++++++++++++++++- 5 files changed, 72 insertions(+), 5 deletions(-) diff --git a/src/common/hobject.h b/src/common/hobject.h index 2dbd2e298e0..cb37b00cd8c 100644 --- a/src/common/hobject.h +++ b/src/common/hobject.h @@ -60,6 +60,12 @@ public: static bool is_meta_pool(int64_t pool) { return pool == POOL_META; } + static int64_t get_logical_pool(int64_t pool) { + if (is_temp_pool(pool)) + return get_temp_pool(pool); // it's reversible + else + return pool; + } public: object_t oid; @@ -114,10 +120,7 @@ public: return is_meta_pool(pool); } int64_t get_logical_pool() const { - if (is_temp_pool(pool)) - return get_temp_pool(pool); // it's reversible - else - return pool; + return get_logical_pool(pool); } hobject_t() : snap(0), hash(0), max(false), pool(INT64_MIN) { diff --git a/src/crimson/os/seastore/object_data_handler.cc b/src/crimson/os/seastore/object_data_handler.cc index dd615395060..8a0d683c065 100644 --- a/src/crimson/os/seastore/object_data_handler.cc +++ b/src/crimson/os/seastore/object_data_handler.cc @@ -1590,6 +1590,8 @@ ObjectDataHandler::read_ret ObjectDataHandler::read( "ObjectDataHandler::read hit invalid error" ) ); + auto prefix = l_start.get_laddr().get_object_prefix(); + bool all_cold = true; for (auto &pin : rpins) { if (pin.mapping.is_zero_reserved()) { ret.append_zero(pin.unaligned_len); @@ -1608,6 +1610,13 @@ ObjectDataHandler::read_ret ObjectDataHandler::read( assert(pin.unaligned_start_offset == 0); ret.append(std::move(aligned_bl)); } + DEBUGT("got extent: {}", ctx.t, *maybe_indirect_extent.extent); + auto paddr = maybe_indirect_extent.extent->get_paddr(); + all_cold &= ctx.tm.is_cold_device(paddr.get_device_id()); + } + if (!all_cold) { + assert(ctx.tm.is_prefix_cached(prefix)); + ctx.tm.update_logical_bucket_for_read(prefix); } co_return std::move(ret); } diff --git a/src/crimson/os/seastore/transaction.h b/src/crimson/os/seastore/transaction.h index c7ff9071af9..73beb38d0f2 100644 --- a/src/crimson/os/seastore/transaction.h +++ b/src/crimson/os/seastore/transaction.h @@ -576,6 +576,7 @@ public: views.clear(); copied_lba_keys.clear(); update_copied_lba_key = nullptr; + touched_prefix.clear(); } bool did_reset() const { @@ -692,6 +693,14 @@ public: return cache_hint; } + void touch_laddr_prefix(laddr_t laddr) { + touched_prefix.insert(laddr.get_object_prefix()); + } + + std::unordered_set &get_touched_laddr_prefix() { + return touched_prefix; + } + btree_cursor_stats_t cursor_stats; bool force_rewrite_conflict = false; @@ -918,6 +927,8 @@ private: */ retired_extent_set_t retired_set; + std::unordered_set touched_prefix; + /// stats to collect when commit or invalidate tree_stats_t onode_tree_stats; tree_stats_t omap_tree_stats; // exclude omap tree depth diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index a32573a1cc9..8b9facf3f48 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -162,6 +162,10 @@ TransactionManager::mount() assert(paddr.is_absolute()); cache->update_tree_extents_num(type, 1); epm->mark_space_used(paddr, len); + if (support_logical_bucket() && + !epm->is_cold_device(paddr.get_device_id())) { + logical_bucket->move_to_top(laddr.get_object_prefix()); + } }); } else { return backref_manager->scan_mapped_space( @@ -188,6 +192,10 @@ TransactionManager::mount() assert(backref_key == P_ADDR_NULL); cache->update_tree_extents_num(type, 1); epm->mark_space_used(paddr, len); + if (support_logical_bucket() && + !epm->is_cold_device(paddr.get_device_id())) { + logical_bucket->move_to_top(laddr.get_object_prefix()); + } } }); } @@ -809,6 +817,11 @@ TransactionManager::do_submit_transaction( journal->get_trimmer().update_journal_tails( cache->get_oldest_dirty_from().value_or(start_seq), cache->get_oldest_backref_dirty_from().value_or(start_seq)); + if (support_logical_bucket()) { + for (auto &prefix : tref.get_touched_laddr_prefix()) { + logical_bucket->move_to_top(prefix.get_object_prefix()); + } + } }).handle_error( submit_transaction_iertr::pass_further{}, crimson::ct_error::assert_all("Hit error submitting to journal") @@ -1226,6 +1239,7 @@ TransactionManager::promote_extent( true, write_policy_t::WRITE_BACK }); + t.touch_laddr_prefix(orig_ext->get_laddr().get_object_prefix()); promoted_extents.reserve(promoted_raw_extents.size()); @@ -1276,6 +1290,7 @@ TransactionManager::promote_extent( lext->rewrite(t, *orig_ext, 0); assert(!extent->get_paddr().is_absolute() || !cache->is_on_cold_tier(lext->get_paddr())); + t.touch_laddr_prefix(orig_ext->get_laddr().get_object_prefix()); //TODO: this memory copy should be saved orig_ext->get_bptr().copy_out( 0, diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index 5c88925a7dd..53272f5a6f4 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -625,6 +625,17 @@ public: exts.begin(), exts.end()), EXTENT_DEFAULT_REF_COUNT); } + auto &front = exts.front(); + if (front->get_write_policy() != write_policy_t::WRITE_THROUGH && + front->get_rewrite_generation() <= epm->get_max_hot_gen()) { + auto prefix = front->get_laddr().get_object_prefix(); + t.touch_laddr_prefix(prefix); + if (auto pool = prefix.get_pool(); + unlikely(hobject_t::is_temp_pool(pool))) { + prefix.set_pool(hobject_t::get_logical_pool(pool)); + t.touch_laddr_prefix(prefix); + } + } for (auto &ext : exts) { SUBDEBUGT(seastore_tm, "allocated {}", t, *ext); } @@ -851,6 +862,24 @@ public: }); } + // non-trivial method, should only be used in DEBUG build + bool is_prefix_cached(laddr_t prefix) { + if (!logical_bucket) { + return true; + } + return logical_bucket->is_cached(prefix); + } + + void update_logical_bucket_for_read(laddr_t prefix) { + if (logical_bucket) { + logical_bucket->move_to_top(prefix.get_object_prefix()); + } + } + + bool is_cold_device(device_id_t id) const { + return epm->is_cold_device(id); + } + /** * submit_transaction * @@ -1299,7 +1328,7 @@ public: } bool support_logical_bucket() const { - return logical_bucket != nullptr; + return epm->has_cold_tier() && logical_bucket != nullptr; } ~TransactionManager(); From a8b0baca915780b908257ab4bf87aeb7d87f17ba Mon Sep 17 00:00:00 2001 From: Zhang Song Date: Fri, 8 Aug 2025 16:36:27 +0800 Subject: [PATCH 20/51] crimson/os/seastore: allow RBM backend as hot tier Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/extent_placement_manager.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc index 554a479be8b..24c1e9d848c 100644 --- a/src/crimson/os/seastore/extent_placement_manager.cc +++ b/src/crimson/os/seastore/extent_placement_manager.cc @@ -330,7 +330,6 @@ void ExtentPlacementManager::init( pinboard); ceph_assert(get_main_backend_type() != backend_type_t::NONE); if (cold_cleaner_) { - ceph_assert(get_main_backend_type() == backend_type_t::SEGMENTED); ceph_assert(background_process.has_cold_tier()); } else { ceph_assert(!background_process.has_cold_tier()); From 2a5257a04c4187f374089837794daf25c75a9458 Mon Sep 17 00:00:00 2001 From: Zhang Song Date: Fri, 8 Aug 2025 16:58:22 +0800 Subject: [PATCH 21/51] crimson/os/seastore: make cold rbm cleaner use different metric prefix Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/async_cleaner.cc | 12 ++++++++++-- src/crimson/os/seastore/async_cleaner.h | 9 ++++++--- src/crimson/os/seastore/transaction_manager.cc | 6 ++++-- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/crimson/os/seastore/async_cleaner.cc b/src/crimson/os/seastore/async_cleaner.cc index dff8bbb9678..febc6e96112 100644 --- a/src/crimson/os/seastore/async_cleaner.cc +++ b/src/crimson/os/seastore/async_cleaner.cc @@ -1962,9 +1962,11 @@ RBMCleaner::RBMCleaner( RBMDeviceGroupRef&& rb_group, BackrefManager &backref_manager, LBAManager &lba_manager, - bool detailed) + bool detailed, + bool is_cold) : store_index(store_index), detailed(detailed), + is_cold(is_cold), rb_group(std::move(rb_group)), backref_manager(backref_manager), lba_manager(lba_manager) @@ -2194,7 +2196,13 @@ void RBMCleaner::register_metrics() { namespace sm = seastar::metrics; - metrics.add_group("rbm_cleaner", { + std::string prefix; + if (is_cold) { + prefix.append("cold_"); + } + prefix.append("rbm_cleaner"); + + metrics.add_group(prefix, { sm::make_counter("total_bytes", [this] { return get_total_bytes(); }, sm::description("the size of the space"), diff --git a/src/crimson/os/seastore/async_cleaner.h b/src/crimson/os/seastore/async_cleaner.h index 095164f32d8..51c43d6a1aa 100644 --- a/src/crimson/os/seastore/async_cleaner.h +++ b/src/crimson/os/seastore/async_cleaner.h @@ -1783,17 +1783,19 @@ public: RBMDeviceGroupRef&& rb_group, BackrefManager &backref_manager, LBAManager &lba_manager, - bool detailed); + bool detailed, + bool is_cold); static RBMCleanerRef create( store_index_t store_index, RBMDeviceGroupRef&& rb_group, BackrefManager &backref_manager, LBAManager &lba_manager, - bool detailed) { + bool detailed, + bool is_cold) { return std::make_unique( store_index, - std::move(rb_group), backref_manager, lba_manager, detailed); + std::move(rb_group), backref_manager, lba_manager, detailed, is_cold); } RBMDeviceGroup* get_rb_group() { @@ -1939,6 +1941,7 @@ private: store_index_t store_index; const bool detailed; + const bool is_cold; RBMDeviceGroupRef rb_group; BackrefManager &backref_manager; LBAManager &lba_manager; diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index 8b9facf3f48..cf2b774e411 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -1595,7 +1595,8 @@ TransactionManagerRef make_transaction_manager( std::move(cold_rbs), *backref_manager, *lba_manager, - cleaner_is_detailed); + cleaner_is_detailed, + true); } if (backend_type == backend_type_t::SEGMENTED) { @@ -1623,7 +1624,8 @@ TransactionManagerRef make_transaction_manager( std::move(rbs), *backref_manager, *lba_manager, - cleaner_is_detailed); + cleaner_is_detailed, + false); journal = journal::make_circularbounded( store_index, *journal_trimmer, From eb9e9b9d063df808d7cba804acb0d26cfff95d76 Mon Sep 17 00:00:00 2001 From: Zhang Song Date: Wed, 3 Sep 2025 16:06:54 +0800 Subject: [PATCH 22/51] crimson/os/seastore/cache: add hit ratio metric Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/cache.cc | 26 +++++++++++++++++++ src/crimson/os/seastore/cache.h | 10 +++++++ .../os/seastore/object_data_handler.cc | 7 +++++ src/crimson/os/seastore/transaction.h | 9 +++++++ src/crimson/os/seastore/transaction_manager.h | 25 ++++++++++++++++++ 5 files changed, 77 insertions(+) diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc index 10db3a0732b..02fbe9a59b4 100644 --- a/src/crimson/os/seastore/cache.cc +++ b/src/crimson/os/seastore/cache.cc @@ -231,6 +231,29 @@ void Cache::register_metrics(store_index_t store_index) } ); + metrics.add_group("cache", { + sm::make_counter( + "write_hit_hot", + stats.write_hit_hot, + sm::description("the number of the lbc hits of data writes"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), + sm::make_counter( + "write_hit_cold", + stats.write_hit_cold, + sm::description("the number of the lbc misses of data writes"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), + sm::make_counter( + "read_hit_hot", + stats.read_hit_hot, + sm::description("the number of the lbc hits of data reads"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), + sm::make_counter( + "read_hit_cold", + stats.read_hit_cold, + sm::description("the number of the lbc misses of data reads"), + {sm::label_instance("shard_store_index", std::to_string(store_index))}), + }); + { /* * efforts discarded/committed @@ -2181,6 +2204,9 @@ void Cache::complete_commit( i->set_invalid(t); } } + + stats.write_hit_hot += t.write_hit_hot; + stats.write_hit_cold += t.write_hit_cold; } void Cache::init() diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h index df3fdf90e98..0b67f16ccc2 100644 --- a/src/crimson/os/seastore/cache.h +++ b/src/crimson/os/seastore/cache.h @@ -761,6 +761,11 @@ public: return epm.is_pure_rbm(); } + void update_read_ratio(Transaction &t) { + stats.read_hit_hot += t.read_hit_hot; + stats.read_hit_cold += t.read_hit_cold; + } + private: using get_extent_ertr = base_ertr; template @@ -1868,6 +1873,11 @@ private: std::array trans_conflicts_by_srcs; counter_by_src_t trans_conflicts_by_unknown; + uint64_t write_hit_hot = 0; + uint64_t write_hit_cold = 0; + uint64_t read_hit_hot = 0; + uint64_t read_hit_cold = 0; + rewrite_stats_t trim_rewrites; rewrite_stats_t reclaim_rewrites; } stats; diff --git a/src/crimson/os/seastore/object_data_handler.cc b/src/crimson/os/seastore/object_data_handler.cc index 8a0d683c065..28d1a7b0ef4 100644 --- a/src/crimson/os/seastore/object_data_handler.cc +++ b/src/crimson/os/seastore/object_data_handler.cc @@ -1578,6 +1578,7 @@ ObjectDataHandler::read_ret ObjectDataHandler::read( read_len, pin_start, pin_len); + rpins.emplace_back( pin, read_start_aligned, read_len_aligned, unalign_start_offset, read_len); @@ -1613,11 +1614,17 @@ ObjectDataHandler::read_ret ObjectDataHandler::read( DEBUGT("got extent: {}", ctx.t, *maybe_indirect_extent.extent); auto paddr = maybe_indirect_extent.extent->get_paddr(); all_cold &= ctx.tm.is_cold_device(paddr.get_device_id()); + + if (paddr.is_absolute()) { + ctx.tm.update_read_ratio(ctx.t, paddr.get_device_id()); + } + } if (!all_cold) { assert(ctx.tm.is_prefix_cached(prefix)); ctx.tm.update_logical_bucket_for_read(prefix); } + ctx.tm.submit_read_ratio(ctx.t); co_return std::move(ret); } diff --git a/src/crimson/os/seastore/transaction.h b/src/crimson/os/seastore/transaction.h index 73beb38d0f2..aee35094449 100644 --- a/src/crimson/os/seastore/transaction.h +++ b/src/crimson/os/seastore/transaction.h @@ -540,7 +540,16 @@ public: friend class crimson::os::seastore::SeaStore; friend class TransactionConflictCondition; + uint64_t write_hit_hot = 0; + uint64_t write_hit_cold = 0; + uint64_t read_hit_hot = 0; + uint64_t read_hit_cold = 0; + void reset_preserve_handle() { + write_hit_hot = 0; + write_hit_cold = 0; + read_hit_hot = 0; + read_hit_cold = 0; root.reset(); offset = 0; delayed_temp_offset = 0; diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index 53272f5a6f4..3c576d78b23 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -245,6 +245,28 @@ public: } }; + void update_hit_ratio(Transaction& t, device_id_t id) { + if (epm->is_cold_device(id)) { + t.write_hit_cold++; + } else { + t.write_hit_hot++; + } + } + + void update_read_ratio(Transaction& t, device_id_t id) { + if (epm->is_cold_device(id)) { + t.read_hit_cold++; + } else { + t.read_hit_hot++; + } + } + + void submit_read_ratio(Transaction& t) { + if (cache) { + cache->update_read_ratio(t); + } + } + template using lextent_init_func_t = std::function; /** @@ -1223,6 +1245,9 @@ public: return cut_mapping( t, (laddr + aligned_len).checked_to_laddr(), std::move(mapping), false); } else { + if (mapping.is_linked_direct() && mapping.get_val().is_absolute()) { + update_hit_ratio(t, mapping.get_val().get_device_id()); + } return remove(t, std::move(mapping) ).handle_error_interruptible( punch_mappings_iertr::pass_further{}, From d86fb5fd801e3a49477098a87ec683603e74ae57 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Wed, 3 Sep 2025 16:10:28 +0800 Subject: [PATCH 23/51] crimson/os/seastore/random_block_manager: try to allocate consecutive rbm space when rewriting extents Add the address of the last allocation as the hint to the current round of allocation in rbm Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/async_cleaner.cc | 14 +-- src/crimson/os/seastore/async_cleaner.h | 13 ++- src/crimson/os/seastore/cache.cc | 3 +- src/crimson/os/seastore/cache.h | 1 + .../os/seastore/extent_placement_manager.h | 15 ++- .../os/seastore/random_block_manager.h | 2 +- .../random_block_manager/avlallocator.cc | 15 ++- .../random_block_manager/avlallocator.h | 2 +- .../random_block_manager/block_rb_manager.cc | 6 +- .../random_block_manager/block_rb_manager.h | 3 +- .../random_block_manager/extent_allocator.h | 2 +- .../os/seastore/transaction_manager.cc | 101 +++++++++++++++++- src/crimson/os/seastore/transaction_manager.h | 15 ++- .../seastore/test_btree_lba_manager.cc | 4 +- .../crimson/seastore/test_extent_allocator.cc | 2 +- 15 files changed, 161 insertions(+), 37 deletions(-) diff --git a/src/crimson/os/seastore/async_cleaner.cc b/src/crimson/os/seastore/async_cleaner.cc index febc6e96112..d630a84b494 100644 --- a/src/crimson/os/seastore/async_cleaner.cc +++ b/src/crimson/os/seastore/async_cleaner.cc @@ -1388,15 +1388,11 @@ do_reclaim_space_ret do_reclaim_space( &reclaimed, &t, modify_time, target_generation] { DEBUGT("reclaim {} extents", t, extents.size()); // rewrite live extents - return trans_intr::do_for_each( - extents, - [&extent_callback, modify_time, &t, - &reclaimed, target_generation](auto ext) - { - reclaimed += ext->get_length(); - return extent_callback.rewrite_extent( - t, ext, target_generation, modify_time); - }); + for (auto &ext : extents) { + reclaimed += ext->get_length(); + } + return extent_callback.rewrite_extents( + t, extents, target_generation, modify_time); }); }).si_then([&extent_callback, &t] { return extent_callback.submit_transaction_direct(t); diff --git a/src/crimson/os/seastore/async_cleaner.h b/src/crimson/os/seastore/async_cleaner.h index 51c43d6a1aa..3e1ee3e993c 100644 --- a/src/crimson/os/seastore/async_cleaner.h +++ b/src/crimson/os/seastore/async_cleaner.h @@ -351,6 +351,14 @@ public: rewrite_gen_t target_generation, sea_time_point modify_time) = 0; + using rewrite_extents_iertr = base_iertr; + using rewrite_extents_ret = rewrite_extents_iertr::future<>; + virtual rewrite_extents_ret rewrite_extents( + Transaction &t, + std::vector &extents, + rewrite_gen_t target_generation, + sea_time_point modify_time) = 0; + /** * promote_extent * @@ -1898,10 +1906,11 @@ public: return paddr; } - std::list alloc_paddrs(extent_len_t length) { + std::list alloc_paddrs( + extent_len_t length, paddr_t hint) { // TODO: implement allocation strategy (dirty metadata and multiple devices) auto rbs = rb_group->get_rb_managers(); - auto ret = rbs[0]->alloc_extents(length); + auto ret = rbs[0]->alloc_extents(length, hint); if (!ret.empty()) { stats.used_bytes += length; } diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc index 02fbe9a59b4..4208d899efb 100644 --- a/src/crimson/os/seastore/cache.cc +++ b/src/crimson/os/seastore/cache.cc @@ -1236,6 +1236,7 @@ CachedExtentRef Cache::alloc_new_non_data_extent_by_type( extent_len_t length, ///< [in] length placement_hint_t hint, ///< [in] user hint rewrite_gen_t gen, ///< [in] rewrite generation + paddr_t paddr_hint, bool is_tracked ) { @@ -1243,7 +1244,7 @@ CachedExtentRef Cache::alloc_new_non_data_extent_by_type( SUBDEBUGT(seastore_cache, "allocate {} 0x{:x}B, hint={}, gen={}", t, type, length, hint, rewrite_gen_printer_t{gen}); ceph_assert(get_extent_category(type) == data_category_t::METADATA); - auto opt = alloc_option_t{hint, gen, is_tracked}; + auto opt = alloc_option_t{hint, gen, is_tracked, paddr_hint}; switch (type) { case extent_types_t::ROOT: ceph_assert(0 == "ROOT is never directly alloc'd"); diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h index 0b67f16ccc2..b33d8d59d1b 100644 --- a/src/crimson/os/seastore/cache.h +++ b/src/crimson/os/seastore/cache.h @@ -1295,6 +1295,7 @@ public: extent_len_t length, ///< [in] length placement_hint_t hint, ///< [in] user hint rewrite_gen_t gen, ///< [in] rewrite generation + paddr_t paddr_hint, bool is_tracked ); diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index 210e8d38442..5cea7b0ccbf 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -126,7 +126,9 @@ public: virtual paddr_t alloc_paddr(extent_len_t length) = 0; - virtual std::list alloc_paddrs(extent_len_t length) = 0; + virtual std::list alloc_paddrs( + extent_len_t length, + paddr_t hint) = 0; using alloc_write_ertr = base_ertr; using alloc_write_iertr = trans_iertr; @@ -189,7 +191,7 @@ public: return make_delayed_temp_paddr(0); } - std::list alloc_paddrs(extent_len_t length) final { + std::list alloc_paddrs(extent_len_t length, paddr_t) final { return {alloc_paddr_result{make_delayed_temp_paddr(0), length}}; } @@ -256,9 +258,10 @@ public: return rb_cleaner->alloc_paddr(length); } - std::list alloc_paddrs(extent_len_t length) final { + std::list alloc_paddrs( + extent_len_t length, paddr_t hint) final { assert(rb_cleaner); - return rb_cleaner->alloc_paddrs(length); + return rb_cleaner->alloc_paddrs(length, hint); } bool can_inplace_rewrite(Transaction& t, @@ -450,6 +453,7 @@ public: placement_hint_t hint; rewrite_gen_t gen; bool is_tracked; + paddr_t paddr_hint = P_ADDR_NULL; write_policy_t write_policy = write_policy_t::WRITE_BACK; #ifdef UNIT_TESTS_BUILT std::optional external_paddr = std::nullopt; @@ -528,7 +532,8 @@ public: #endif { assert(category == data_category_t::DATA); - auto addrs = get_writer(opt.hint, category, opt.gen)->alloc_paddrs(length); + auto addrs = get_writer(opt.hint, category, opt.gen)->alloc_paddrs( + length, opt.paddr_hint); for (auto &ext : addrs) { auto left = ext.len; while (left > 0) { diff --git a/src/crimson/os/seastore/random_block_manager.h b/src/crimson/os/seastore/random_block_manager.h index f776483edf2..8ccdbf1639f 100644 --- a/src/crimson/os/seastore/random_block_manager.h +++ b/src/crimson/os/seastore/random_block_manager.h @@ -77,7 +77,7 @@ public: using allocate_ret_bare = std::list; using allo_extents_ret = allocate_ertr::future; - virtual allocate_ret_bare alloc_extents(size_t size) = 0; + virtual allocate_ret_bare alloc_extents(size_t size, paddr_t hint) = 0; virtual void mark_space_used(paddr_t paddr, size_t len) = 0; virtual void mark_space_free(paddr_t paddr, size_t len) = 0; diff --git a/src/crimson/os/seastore/random_block_manager/avlallocator.cc b/src/crimson/os/seastore/random_block_manager/avlallocator.cc index e23b2a76426..046c65d73c4 100644 --- a/src/crimson/os/seastore/random_block_manager/avlallocator.cc +++ b/src/crimson/os/seastore/random_block_manager/avlallocator.cc @@ -99,6 +99,15 @@ extent_len_t AvlAllocator::find_block( if (p != extent_size_tree.rend()) { max_size = p->end - p->start; } + const auto compare = extent_tree.key_comp(); + auto rs = extent_tree.lower_bound(extent_range_t{start, size}, compare); + if (rs != extent_tree.end()) { + uint64_t offset = rs->start; + if (offset + size <= rs->end) { + start = offset; + return size; + } + } assert(max_size); if (max_size <= size) { @@ -210,7 +219,7 @@ std::optional> AvlAllocator::alloc_extent( } std::optional> AvlAllocator::alloc_extents( - size_t size) + size_t size, rbm_abs_addr hint) { LOG_PREFIX(AvlAllocator::alloc_extents); if (available_size < size) { @@ -224,10 +233,10 @@ std::optional> AvlAllocator::alloc_extents( interval_set result; - auto try_to_alloc_block = [this, &result, FNAME] (uint64_t alloc_size) + auto try_to_alloc_block = [this, hint, &result, FNAME] (uint64_t alloc_size) { + rbm_abs_addr start = hint; while (alloc_size) { - rbm_abs_addr start = 0; extent_len_t len = find_block(std::min(max_alloc_size, alloc_size), start); ceph_assert(len); _remove_from_tree(start, len); diff --git a/src/crimson/os/seastore/random_block_manager/avlallocator.h b/src/crimson/os/seastore/random_block_manager/avlallocator.h index 8f4eedc4f7a..dab0e36be5b 100644 --- a/src/crimson/os/seastore/random_block_manager/avlallocator.h +++ b/src/crimson/os/seastore/random_block_manager/avlallocator.h @@ -65,7 +65,7 @@ public: std::optional> alloc_extent( size_t size) final; std::optional> alloc_extents( - size_t size) final; + size_t size, rbm_abs_addr hint) final; void free_extent(rbm_abs_addr addr, size_t size) final; void mark_extent_used(rbm_abs_addr addr, size_t size) final; diff --git a/src/crimson/os/seastore/random_block_manager/block_rb_manager.cc b/src/crimson/os/seastore/random_block_manager/block_rb_manager.cc index 74bcbddb917..c1604a40f73 100644 --- a/src/crimson/os/seastore/random_block_manager/block_rb_manager.cc +++ b/src/crimson/os/seastore/random_block_manager/block_rb_manager.cc @@ -66,11 +66,13 @@ paddr_t BlockRBManager::alloc_extent(size_t size) } BlockRBManager::allocate_ret_bare -BlockRBManager::alloc_extents(size_t size) +BlockRBManager::alloc_extents(size_t size, paddr_t hint) { LOG_PREFIX(BlockRBManager::alloc_extents); assert(allocator); - auto alloc = allocator->alloc_extents(size); + rbm_abs_addr rbm_hint = + (hint == P_ADDR_NULL ? 0 : convert_paddr_to_abs_addr(hint)); + auto alloc = allocator->alloc_extents(size, rbm_hint); if (!alloc) { return {}; } diff --git a/src/crimson/os/seastore/random_block_manager/block_rb_manager.h b/src/crimson/os/seastore/random_block_manager/block_rb_manager.h index 03b2285d8ce..09ad50f75bd 100644 --- a/src/crimson/os/seastore/random_block_manager/block_rb_manager.h +++ b/src/crimson/os/seastore/random_block_manager/block_rb_manager.h @@ -58,7 +58,8 @@ public: */ paddr_t alloc_extent(size_t size) override; // allocator, return blocks - allocate_ret_bare alloc_extents(size_t size) override; // allocator, return blocks + allocate_ret_bare alloc_extents( + size_t size, paddr_t hint) override; // allocator, return blocks void complete_allocation(paddr_t addr, size_t size) override; diff --git a/src/crimson/os/seastore/random_block_manager/extent_allocator.h b/src/crimson/os/seastore/random_block_manager/extent_allocator.h index 2797b8822fd..d0a06fd4cf0 100644 --- a/src/crimson/os/seastore/random_block_manager/extent_allocator.h +++ b/src/crimson/os/seastore/random_block_manager/extent_allocator.h @@ -35,7 +35,7 @@ public: * */ virtual std::optional> alloc_extents( - size_t size) = 0; + size_t size, rbm_abs_addr hint) = 0; /** * free_extent diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index cf2b774e411..dedd6060b9e 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -861,10 +861,12 @@ TransactionManager::get_next_dirty_extents( return cache->get_next_dirty_extents(t, seq, max_bytes); } -TransactionManager::rewrite_extent_ret +TransactionManager::rewrite_extent_iertr::future< + std::vector> TransactionManager::rewrite_logical_extent( Transaction& t, - LogicalChildNodeRef extent) + LogicalChildNodeRef extent, + paddr_t paddr_hint) { LOG_PREFIX(TransactionManager::rewrite_logical_extent); if (extent->has_been_invalidated()) { @@ -890,6 +892,7 @@ TransactionManager::rewrite_logical_extent( extent->get_user_hint(), // get target rewrite generation extent->get_rewrite_generation(), + paddr_hint, is_tracked)->cast(); assert(nextent->get_write_policy() != write_policy_t::WRITE_THROUGH); nextent->rewrite(t, *extent, 0); @@ -921,6 +924,7 @@ TransactionManager::rewrite_logical_extent( extent->get_paddr(), *nextent ); + co_return std::vector{nextent}; } else { assert(get_extent_category(extent->get_type()) == data_category_t::DATA); @@ -938,6 +942,7 @@ TransactionManager::rewrite_logical_extent( // get target rewrite generation extent->get_rewrite_generation(), is_tracked, + paddr_hint, // WRITH_THROUGH is only effective for client io, so // always set the write policy to WRITE_BACK here write_policy_t::WRITE_BACK @@ -996,6 +1001,7 @@ TransactionManager::rewrite_logical_extent( off += nextent->get_length(); left -= nextent->get_length(); } + co_return std::move(extents); } } @@ -1062,7 +1068,9 @@ TransactionManager::rewrite_extent_ret TransactionManager::rewrite_extent( auto fut = rewrite_extent_iertr::now(); if (extent->is_logical()) { assert(is_logical_type(extent->get_type())); - fut = rewrite_logical_extent(t, extent->cast()); + fut = rewrite_logical_extent( + t, extent->cast(), P_ADDR_NULL + ).discard_result(); } else if (is_backref_node(extent->get_type())) { fut = backref_manager->rewrite_extent(t, extent); } else { @@ -1237,6 +1245,7 @@ TransactionManager::promote_extent( placement_hint_t::HOT, INIT_GENERATION, true, + P_ADDR_NULL, write_policy_t::WRITE_BACK }); t.touch_laddr_prefix(orig_ext->get_laddr().get_object_prefix()); @@ -1284,6 +1293,7 @@ TransactionManager::promote_extent( orig_ext->get_length(), placement_hint_t::HOT, INIT_GENERATION, + P_ADDR_NULL, true); auto lext = promoted_extent->cast(); lext->set_laddr(orig_ext->get_laddr()); @@ -1321,6 +1331,83 @@ TransactionManager::promote_extent( t, *mapping.direct_cursor, std::move(promoted_extents)); } +TransactionManager::rewrite_extents_ret TransactionManager::rewrite_extents( + Transaction &t, + std::vector &extents, + rewrite_gen_t target_generation, + sea_time_point modify_time) +{ + LOG_PREFIX(TransactionManager::rewrite_extents); + return seastar::do_with( + P_ADDR_NULL, + L_ADDR_NULL, + [this, &t, target_generation, modify_time, &extents, FNAME] + (auto &paddr_hint, auto &next_laddr) { + return trans_intr::do_for_each( + extents, + [this, &t, target_generation, modify_time, FNAME, + &paddr_hint, &next_laddr](auto &extent) { + { + auto updated = cache->update_extent_from_transaction(t, extent); + if (!updated) { + DEBUGT("extent is already retired, skipping -- {}", t, *extent); + return rewrite_extent_iertr::now(); + } + extent = updated; + ceph_assert(!extent->is_pending_io()); + } + + assert(extent->is_valid() && !extent->is_initial_pending()); + if (extent->is_stable_dirty()) { + if (epm->can_inplace_rewrite(t, extent)) { + DEBUGT("delta overwriting extent -- {}", t, *extent); + t.add_inplace_rewrite_extent(extent); + extent->set_inplace_rewrite_generation(); + return rewrite_extent_iertr::now(); + } + if (extent->get_version() == 1 && extent->has_mutation()) { + t.get_rewrite_stats().account_n_dirty(); + } else { + // extent->get_version() > 1 or DIRTY + t.get_rewrite_stats().account_dirty(extent->get_version()); + } + extent->set_target_rewrite_generation(INIT_GENERATION); + } else { + extent->set_target_rewrite_generation(target_generation); + ceph_assert(modify_time != NULL_TIME); + extent->set_modify_time(modify_time); + } + + if (is_backref_node(extent->get_type())) { + DEBUGT("rewriting backref extent -- {}", t, *extent); + return backref_manager->rewrite_extent(t, extent); + } + + if (extent->get_type() == extent_types_t::ROOT) { + DEBUGT("rewriting root extent -- {}", t, *extent); + cache->duplicate_for_write(t, extent); + return rewrite_extent_iertr::now(); + } + + if (extent->is_logical()) { + auto ext = extent->template cast(); + if (next_laddr != ext->get_laddr()) { + paddr_hint = P_ADDR_NULL; + } + next_laddr = (ext->get_laddr() + ext->get_length()).checked_to_laddr(); + return rewrite_logical_extent(t, ext, paddr_hint + ).si_then([&paddr_hint](auto nlextents) { + for (auto &nlextent : nlextents) { + paddr_hint = nlextent->get_paddr() + nlextent->get_length(); + } + }); + } else { + DEBUGT("rewriting physical extent -- {}", t, *extent); + return lba_manager->rewrite_extent(t, extent); + } + }); + }); +} TransactionManager::demote_region_ret TransactionManager::demote_region( Transaction &t, @@ -1337,6 +1424,7 @@ TransactionManager::demote_region( crimson::ct_error::assert_all("unexpected enoent")); auto it = co_await resolve_cursor_to_mapping(t, std::move(cursor)); demote_region_res_t ret{0, 0, false}; + std::vector extents; while ((ret.demoted_size + ret.evicted_size) < max_proceed_size) { if (it.is_end() || it.get_key().get_object_prefix() != prefix) { ret.complete = true; @@ -1360,8 +1448,7 @@ TransactionManager::demote_region( auto extent = co_await read_cursor_by_type( t, it.direct_cursor, it.get_extent_type()); ret.evicted_size += extent->get_length(); - extent->set_target_rewrite_generation(epm->get_max_hot_gen() + 1); - co_await rewrite_logical_extent(t, extent); + extents.push_back(extent); it = co_await it.next(); } else { DEBUGT("skip {}", t, it); @@ -1369,6 +1456,10 @@ TransactionManager::demote_region( } } + co_await rewrite_extents( + t, extents, epm->get_max_hot_gen() + 1, + seastar::lowres_system_clock::now()); + co_return ret; } diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index 3c576d78b23..9ffb08719a6 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -615,7 +615,7 @@ public: auto exts = cache->alloc_new_data_extents( t, len, { - placement_hint, INIT_GENERATION, false, + placement_hint, INIT_GENERATION, false, P_ADDR_NULL, epm->get_write_policy(T::TYPE, len) }); // user must initialize the logical extent themselves @@ -954,6 +954,13 @@ public: rewrite_gen_t target_generation, sea_time_point modify_time) final; + using ExtentCallbackInterface::rewrite_extents_ret; + rewrite_extents_ret rewrite_extents( + Transaction &t, + std::vector &extents, + rewrite_gen_t target_generation, + sea_time_point modify_time) final; + using ExtentCallbackInterface::promote_extent_ret; promote_extent_ret promote_extent( Transaction &t, @@ -1709,9 +1716,11 @@ private: Transaction &t, LBAMapping mapping); - rewrite_extent_ret rewrite_logical_extent( + rewrite_extent_iertr::future> + rewrite_logical_extent( Transaction& t, - LogicalChildNodeRef extent); + LogicalChildNodeRef extent, + paddr_t hint); submit_transaction_direct_ret do_submit_transaction( Transaction &t, diff --git a/src/test/crimson/seastore/test_btree_lba_manager.cc b/src/test/crimson/seastore/test_btree_lba_manager.cc index ee27a850e76..c37758fc5d6 100644 --- a/src/test/crimson/seastore/test_btree_lba_manager.cc +++ b/src/test/crimson/seastore/test_btree_lba_manager.cc @@ -328,7 +328,7 @@ struct lba_btree_test : btree_test_base { lba_btree_update([=, this](auto &btree, auto &t) { auto extents = cache->alloc_new_data_extents( t, TestBlock::SIZE, - {placement_hint_t::HOT, 0, false, + {placement_hint_t::HOT, 0, false, P_ADDR_NULL, write_policy_t::WRITE_BACK, get_paddr()}); return seastar::do_with( std::move(extents), @@ -548,7 +548,7 @@ struct btree_lba_manager_test : btree_test_base { [=, this](auto &t) { auto extents = cache->alloc_new_data_extents( t, TestBlock::SIZE, - {placement_hint_t::HOT, 0, false, + {placement_hint_t::HOT, 0, false, P_ADDR_NULL, write_policy_t::WRITE_BACK, get_paddr()}); return seastar::do_with( std::vector( diff --git a/src/test/crimson/seastore/test_extent_allocator.cc b/src/test/crimson/seastore/test_extent_allocator.cc index 17eb105e7b2..63c86e660c1 100644 --- a/src/test/crimson/seastore/test_extent_allocator.cc +++ b/src/test/crimson/seastore/test_extent_allocator.cc @@ -58,7 +58,7 @@ struct allocator_test_t : return allocator->alloc_extent(size); } auto allocates(size_t size) { - return allocator->alloc_extents(size); + return allocator->alloc_extents(size, 0); } void free(uint64_t start, uint64_t length) { allocator->free_extent(start, length); From 5314e38d39db000ab1a0bf84968a2e5e6d57e2d6 Mon Sep 17 00:00:00 2001 From: Zhang Song Date: Wed, 3 Sep 2025 16:10:54 +0800 Subject: [PATCH 24/51] crimson/os/seastore: add test workload to promote/evict aggressively Signed-off-by: Zhang Song Signed-off-by: Xuehan Xu --- src/common/options/crimson.yaml.in | 22 ++++ src/crimson/os/seastore/async_cleaner.cc | 10 +- src/crimson/os/seastore/async_cleaner.h | 6 +- .../seastore/backref/btree_backref_manager.cc | 45 ++++++++ .../seastore/backref/btree_backref_manager.h | 5 + src/crimson/os/seastore/backref_manager.h | 9 ++ src/crimson/os/seastore/cache.cc | 2 + src/crimson/os/seastore/cache.h | 22 +++- src/crimson/os/seastore/cached_extent.cc | 8 +- src/crimson/os/seastore/extent_pinboard.cc | 104 +++++++++++++----- .../os/seastore/extent_placement_manager.cc | 56 ++++++++-- .../os/seastore/extent_placement_manager.h | 91 ++++++++++++++- src/crimson/os/seastore/seastore.cc | 17 +++ .../os/seastore/transaction_manager.cc | 58 +++++++++- src/crimson/os/seastore/transaction_manager.h | 6 +- .../seastore/test_transaction_manager.cc | 2 +- 16 files changed, 411 insertions(+), 52 deletions(-) diff --git a/src/common/options/crimson.yaml.in b/src/common/options/crimson.yaml.in index 1c7c3a18240..1660d8fe03a 100644 --- a/src/common/options/crimson.yaml.in +++ b/src/common/options/crimson.yaml.in @@ -154,6 +154,13 @@ options: desc: CPU cores on which POSIX threads alienized to seastar will run in cpuset(7) format flags: - startup +- name: seastore_logical_bucket_cache_test_stress + type: bool + level: dev + desc: Enable SeaStore Logical Bucket Cache stress mode. When enabled, SeaStore exercises cold-tier + cache paths by forcing promotion/demotion activity and probabilistic write-through behavior. + Intended for testing only. + default: false # Seastore options @@ -393,3 +400,18 @@ options: level: dev desc: Select write through policy when data length is greater than this value. default: 512_K +- name: seastore_test_workload_write_through_probability + type: float + level: dev + desc: The percentage of writes that are applied directly to slow devices + default: 0.5 +- name: seastore_test_workload_2Q_promote_probability + type: float + level: dev + desc: The probability of promoting the extents evicted from the warm_in queue to faster devices + default: 0.5 +- name: seastore_test_workload_force_process_background_tasks_period + type: uint + level: dev + desc: Seconds of the period to force process background tasks + default: 5 diff --git a/src/crimson/os/seastore/async_cleaner.cc b/src/crimson/os/seastore/async_cleaner.cc index d630a84b494..9d515e13cd9 100644 --- a/src/crimson/os/seastore/async_cleaner.cc +++ b/src/crimson/os/seastore/async_cleaner.cc @@ -603,10 +603,10 @@ std::size_t JournalTrimmerImpl::get_alloc_journal_size() const return static_cast(ret); } -seastar::future<> JournalTrimmerImpl::trim() { +seastar::future<> JournalTrimmerImpl::trim(bool force) { return seastar::when_all( - [this] { - if (should_trim_alloc()) { + [this, force] { + if (force || should_trim_alloc()) { return trim_alloc( ).handle_error( crimson::ct_error::assert_all( @@ -617,8 +617,8 @@ seastar::future<> JournalTrimmerImpl::trim() { return seastar::now(); } }, - [this] { - if (should_start_trim_dirty()) { + [this, force] { + if (force || should_start_trim_dirty()) { return trim_dirty( ).handle_error( crimson::ct_error::assert_all( diff --git a/src/crimson/os/seastore/async_cleaner.h b/src/crimson/os/seastore/async_cleaner.h index 3e1ee3e993c..59f0311a72b 100644 --- a/src/crimson/os/seastore/async_cleaner.h +++ b/src/crimson/os/seastore/async_cleaner.h @@ -375,6 +375,10 @@ public: Transaction &t, CachedExtentRef extent) = 0; + virtual promote_extent_ret promote_extents_from_disk( + Transaction &t, + paddr_t paddr) = 0; + /** * demote_region * @@ -666,7 +670,7 @@ public: reserved_usage -= usage; } - seastar::future<> trim(); + seastar::future<> trim(bool force); static JournalTrimmerImplRef create( store_index_t store_index, diff --git a/src/crimson/os/seastore/backref/btree_backref_manager.cc b/src/crimson/os/seastore/backref/btree_backref_manager.cc index e049d43362a..2190d0327b8 100644 --- a/src/crimson/os/seastore/backref/btree_backref_manager.cc +++ b/src/crimson/os/seastore/backref/btree_backref_manager.cc @@ -1,6 +1,7 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*- // vim: ts=8 sw=2 sts=2 expandtab +#include "crimson/common/coroutine.h" #include "crimson/os/seastore/backref/btree_backref_manager.h" SET_SUBSYS(seastore_backref); @@ -456,6 +457,50 @@ BtreeBackrefManager::scan_mapped_space( }); } +BtreeBackrefManager::scan_device_ret +BtreeBackrefManager::scan_device( + Transaction &t, + paddr_t paddr, + scan_device_func_t &f) +{ + LOG_PREFIX(BtreeBackrefManager::scan_device); + auto c = get_context(t); + auto croot = co_await cache.get_root(t); + auto btree = BackrefBtree(croot); + auto iter = co_await btree.lower_bound(c, paddr); + while (!iter.is_end()) { + auto key = iter.get_key(); + auto bentry = cache.get_cached_backref_entry(key); + if (bentry) { + assert(bentry->paddr == key); + DEBUGT("found in cache: {} {}", t, bentry->paddr, bentry->laddr); + } + if (bentry && bentry->laddr == L_ADDR_NULL) { + DEBUGT("{} is removed", t, bentry->paddr); + iter = co_await iter.next(c); + continue; + } + if (key.get_device_id() == paddr.get_device_id()) { + auto val = iter.get_val(); + if (bentry && bentry->laddr != val.laddr) { + DEBUGT("{} changed from {} to {}", + t, bentry->paddr, val.laddr, bentry->laddr); + iter = co_await iter.next(c); + continue; + } + DEBUGT("scanned {}, {}", t, key, val.laddr); + auto ret = co_await f(key, val.len, val.type, val.laddr); + if (ret == seastar::stop_iteration::yes) { + break; + } + } else if (key.get_device_id() > paddr.get_device_id()) { + break; + } + iter = co_await iter.next(c); + } + co_return; +} + base_iertr::future<> _init_cached_extent( op_context_t c, const CachedExtentRef &e, diff --git a/src/crimson/os/seastore/backref/btree_backref_manager.h b/src/crimson/os/seastore/backref/btree_backref_manager.h index b8b69af77d9..28d89afdc0c 100644 --- a/src/crimson/os/seastore/backref/btree_backref_manager.h +++ b/src/crimson/os/seastore/backref/btree_backref_manager.h @@ -55,6 +55,11 @@ public: Transaction &t, scan_mapped_space_func_t &&f) final; + scan_device_ret scan_device( + Transaction &t, + paddr_t paddr, + scan_device_func_t &f) final; + init_cached_extent_ret init_cached_extent( Transaction &t, CachedExtentRef e) final; diff --git a/src/crimson/os/seastore/backref_manager.h b/src/crimson/os/seastore/backref_manager.h index 6b6f1676189..b2314e5e789 100644 --- a/src/crimson/os/seastore/backref_manager.h +++ b/src/crimson/os/seastore/backref_manager.h @@ -139,6 +139,15 @@ public: Transaction &t, scan_mapped_space_func_t &&f) = 0; + using scan_device_ret = base_iertr::future<>; + using scan_device_func_t = std::function< + base_iertr::future( + paddr_t, extent_len_t, extent_types_t, laddr_t)>; + virtual scan_device_ret scan_device( + Transaction &t, + paddr_t start, + scan_device_func_t &f) = 0; + virtual ~BackrefManager() {} }; diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc index 4208d899efb..7c7d44d32ed 100644 --- a/src/crimson/os/seastore/cache.cc +++ b/src/crimson/os/seastore/cache.cc @@ -36,6 +36,8 @@ Cache::Cache( delta_based_overwrite_enabled( crimson::common::get_conf( "seastore_data_delta_based_overwrite") > 0), + force_backref(crimson::common::get_conf( + "seastore_logical_bucket_cache_test_stress")), pinboard(create_extent_pinboard( crimson::common::get_conf( "seastore_cachepin_size_pershard"), diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h index b33d8d59d1b..d92e42802b3 100644 --- a/src/crimson/os/seastore/cache.h +++ b/src/crimson/os/seastore/cache.h @@ -758,7 +758,7 @@ public: } bool can_drop_backref() const { - return epm.is_pure_rbm(); + return epm.is_pure_rbm() && !force_backref; } void update_read_ratio(Transaction &t) { @@ -1076,6 +1076,24 @@ private: return res; } + std::optional get_cached_backref_entry(paddr_t addr) { + auto it = backref_entry_mset.lower_bound( + addr, + backref_entry_t::cmp_t()); + if (it == backref_entry_mset.end()) { + return std::nullopt; + } + while (it->paddr == addr) { + auto &backref_entry = *it; + ++it; + if (it == backref_entry_mset.end() || + it->paddr != addr) { + return backref_entry; + } + } + return std::nullopt; + } + const backref_entry_mset_t& get_backref_entry_mset() { return backref_entry_mset; } @@ -1741,6 +1759,8 @@ private: transaction_id_t next_id = 0; + const bool force_backref = false; + /** * dirty * diff --git a/src/crimson/os/seastore/cached_extent.cc b/src/crimson/os/seastore/cached_extent.cc index 4854bc8055f..70e9e101035 100644 --- a/src/crimson/os/seastore/cached_extent.cc +++ b/src/crimson/os/seastore/cached_extent.cc @@ -415,8 +415,12 @@ void ExtentCommitter::commit_state() { prior.last_committed_crc = extent.last_committed_crc; prior.dirty_from = extent.dirty_from; prior.length = extent.length; - prior.loaded_length = extent.loaded_length; - prior.buffer_space = std::move(extent.buffer_space); + // XXX: at present, zero loaded_length extents here + // must have been created by promoting/demoting them. + if (likely(extent.loaded_length != 0)) { + assert(prior.loaded_length == extent.loaded_length); + prior.buffer_space = std::move(extent.buffer_space); + } // XXX: We can go ahead and change the prior's version because // transactions don't hold a local view of the version field, // unlike FixedKVLeafNode::modifications diff --git a/src/crimson/os/seastore/extent_pinboard.cc b/src/crimson/os/seastore/extent_pinboard.cc index e03ffbf618f..e149d3b0bad 100644 --- a/src/crimson/os/seastore/extent_pinboard.cc +++ b/src/crimson/os/seastore/extent_pinboard.cc @@ -269,8 +269,14 @@ void ExtentQueue::get_stats( class ExtentPromoter { public: - ExtentPromoter(size_t promotion_size, ExtentPlacementManager &epm) - : promotion_size(promotion_size), epm(epm) {} + ExtentPromoter( + size_t promotion_size, + ExtentPlacementManager &epm) + : promotion_size(promotion_size), + epm(epm), + test_workload(crimson::common::get_conf( + "seastore_logical_bucket_cache_test_stress")) + {} ~ExtentPromoter() { clear(); @@ -285,7 +291,13 @@ public: } size_t get_promotion_size() const { - return current_contents; + if (unlikely(test_workload)) { + return current_contents >= promotion_size + ? current_contents + : promotion_size; + } else { + return current_contents; + } } void set_background_callback(BackgroundListener *l) { @@ -297,7 +309,9 @@ public: } bool should_run_promote() const { - return enabled() && current_contents >= promotion_size; + return enabled() && + (current_contents >= promotion_size + || test_workload); } std::size_t get_promoted_size() const { @@ -348,32 +362,49 @@ public: LOG_PREFIX(ExtentPromoter::run_promote); std::size_t promote_size = 0; DEBUGT("start promote", t); - std::list extents; - for (auto &extent : list) { - DEBUGT("promote {} to the hot tier", t, extent); - ceph_assert(extent.is_stable_clean()); - ceph_assert(extent.get_pin_state() == extent_pin_state_t::PendingPromote); - extents.emplace_back(&extent); - } - for (auto &extent : extents) { - remove_extent(*extent, extent_pin_state_t::Fresh); - } - promoting_extents.insert( - promoting_extents.end(), - extents.begin(), - extents.end()); - for (auto it = promoting_extents.begin(); - it != promoting_extents.end();) { - auto &extent = *it; - if (!extent->is_valid()) { - it = promoting_extents.erase(it); - continue; + if (unlikely(test_workload && + current_contents < promotion_size && + // if promoting_extents is not empty, it means the last + // round of promotion was not a test workload and was + // interrupted, so this round shouldn't be a test workload + // too. + promoting_extents.empty())) { + auto id = epm.get_cold_device_id(); + paddr_t start = P_ADDR_NULL; + if (device_id_to_paddr_type(id) == paddr_types_t::SEGMENT) { + start = paddr_t::make_seg_paddr(id, 0, 0); + } else { + start = paddr_t::make_blk_paddr(id, 0); + } + co_await ecb->promote_extents_from_disk(t, start); + } else { + std::list extents; + for (auto &extent : list) { + DEBUGT("promote {} to the hot tier", t, extent); + ceph_assert(extent.is_stable_clean()); + ceph_assert(extent.get_pin_state() == extent_pin_state_t::PendingPromote); + extents.emplace_back(&extent); + } + for (auto &extent : extents) { + remove_extent(*extent, extent_pin_state_t::Fresh); + } + promoting_extents.insert( + promoting_extents.end(), + extents.begin(), + extents.end()); + for (auto it = promoting_extents.begin(); + it != promoting_extents.end();) { + auto &extent = *it; + if (!extent->is_valid()) { + it = promoting_extents.erase(it); + continue; + } + promote_size += extent->get_length(); + t.add_to_read_set(extent); + co_await trans_intr::make_interruptible(extent->wait_io()); + co_await ecb->promote_extent(t, extent); + it++; } - promote_size += extent->get_length(); - t.add_to_read_set(extent); - co_await trans_intr::make_interruptible(extent->wait_io()); - co_await ecb->promote_extent(t, extent); - it++; } // existing extents in lru will be retired after transaction submitted co_await ecb->submit_transaction_direct(t); @@ -407,6 +438,7 @@ private: size_t promoted_count = 0; size_t promoted_size = 0; + bool test_workload = false; }; class ExtentPinboardLRU : public ExtentPinboard { @@ -706,7 +738,11 @@ public: : warm_in(warm_in_capacity), warm_out(warm_out_capacity), hot(hot_capacity), - promoter(promotion_size, epm) + promoter(promotion_size, epm), + test_workload(crimson::common::get_conf( + "seastore_logical_bucket_cache_test_stress")), + TwoQ_promote_probability(crimson::common::get_conf( + "seastore_test_workload_2Q_promote_probability")) { LOG_PREFIX(ExtentPinboardTwoQ::ExtentPinboardTwoQ); INFO("created, warm_in_capacity=0x{:x}B, warm_out_capacity=0x{:x}B, " @@ -889,6 +925,12 @@ private: // to the warm out queue. continue; } + if (promoter.should_promote_extent(*extent) + && test_workload + && (double(std::rand() % 100) / 100.0) <= TwoQ_promote_probability) { + promoter.add_extent(*extent); + continue; + } auto lext = extent->cast(); auto laddr = lext->get_laddr(); auto end = extent->get_last_touch_end(); @@ -975,6 +1017,8 @@ private: // hit and miss indicates if an extent is linked when touching it uint64_t hit = 0; uint64_t miss = 0; + bool test_workload = false; + double TwoQ_promote_probability = 0; }; void ExtentPinboardTwoQ::get_stats( diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc index 24c1e9d848c..7ece18312ab 100644 --- a/src/crimson/os/seastore/extent_placement_manager.cc +++ b/src/crimson/os/seastore/extent_placement_manager.cc @@ -1,6 +1,9 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*- // vim: ts=8 sw=2 sts=2 expandtab expandtab +#include +#include + #include "crimson/os/seastore/extent_placement_manager.h" #include "crimson/common/errorator-utils.h" @@ -882,7 +885,8 @@ ExtentPlacementManager::BackgroundProcess::run() { assert(is_running()); while (is_running()) { - if (background_should_run()) { + if (background_should_run() + || force_run_background()) { log_state("run(background)"); co_await do_background_cycle(); // Edge-triggered: yield only when a blocked IO was actually woken, so the @@ -898,9 +902,13 @@ ExtentPlacementManager::BackgroundProcess::run() if (cold_cleaner) { cold_cleaner->maybe_adjust_thresholds(); } + maybe_reschedule_force_process(); } else { log_state("run(block)"); assert(!blocking_background); + if (unlikely(test_workload)) { + set_next_force_process(); + } blocking_background = seastar::promise<>(); co_await blocking_background->get_future(); // After waking (typically because arm_blocking_io_and_wake() kicked us), @@ -1042,12 +1050,24 @@ ExtentPlacementManager::BackgroundProcess::do_background_cycle() } } + bool force_trim = false; + bool should_abort_cleaner_usage = true; + if (unlikely(should_force_trim())) { + if (!proceed_trim) { + should_abort_cleaner_usage = false; + } + proceed_trim = true; + force_trim = true; + } + if (proceed_trim) { DEBUG("started trimming..."); - return trimmer->trim( - ).finally([this, trim_usage, FNAME] { + return trimmer->trim(force_trim + ).finally([this, trim_usage, should_abort_cleaner_usage, FNAME] { DEBUG("finished trimming"); - abort_cleaner_usage(trim_usage, {true, true}); + if (should_abort_cleaner_usage) { + abort_cleaner_usage(trim_usage, {true, true}); + } }); } else { assert(!proceed_trim); @@ -1082,11 +1102,25 @@ ExtentPlacementManager::BackgroundProcess::do_background_cycle() proceed_demote = true; } + bool abort_cold_cleaner_usage = true; + if (unlikely(should_force_clean())) { + if (!proceed_clean_main) { + abort_cold_cleaner_usage = false; + } + proceed_clean_main = main_cleaner->can_clean_space(); + if (has_cold_tier()) { + proceed_clean_cold = cold_cleaner->can_clean_space(); + } + if (logical_bucket) { + proceed_demote = logical_bucket->could_demote(); + } + } + if (!proceed_clean_main && !proceed_clean_cold && !proceed_demote) { ceph_abort_msg("no background process will start"); } return seastar::when_all( - [this, FNAME, proceed_clean_main, + [this, FNAME, proceed_clean_main, abort_cold_cleaner_usage, should_clean_main_for_trim, main_cold_usage] { if (!proceed_clean_main) { return seastar::now(); @@ -1101,9 +1135,11 @@ ExtentPlacementManager::BackgroundProcess::do_background_cycle() crimson::ct_error::assert_all( "do_background_cycle encountered invalid error in main clean_space" ) - ).finally([this, main_cold_usage, FNAME] { + ).finally([this, main_cold_usage, abort_cold_cleaner_usage, FNAME] { DEBUG("finished clean main"); - abort_cold_usage(main_cold_usage, true); + if (abort_cold_cleaner_usage) { + abort_cold_usage(main_cold_usage, true); + } }); }, [this, FNAME, proceed_clean_cold, @@ -1163,6 +1199,12 @@ seastar::future<> ExtentPlacementManager::BackgroundProcess::run_promote() ceph_assert(!blocking_promote); blocking_promote = seastar::promise<>(); return blocking_promote->get_future(); + }).then([this] { + if (unlikely(test_workload)) { + return seastar::sleep(std::chrono::seconds( + force_process_half_life)); + } + return seastar::now(); }).then([] { return seastar::stop_iteration::no; }); diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index 5cea7b0ccbf..11463df8fb3 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -375,7 +375,11 @@ public: max_data_allocation_size(crimson::common::get_conf( "seastore_max_data_allocation_size")), write_through_size(crimson::common::get_conf( - "seastore_write_through_size")) + "seastore_write_through_size")), + test_workload(crimson::common::get_conf( + "seastore_logical_bucket_cache_test_stress")), + write_through_probability(crimson::common::get_conf( + "seastore_test_workload_write_through_probability")) { LOG_PREFIX(ExtentPlacementManager::ExtentPlacementManager); devices_by_id.resize(DEVICE_ID_MAX, nullptr); @@ -557,8 +561,14 @@ public: } write_policy_t get_write_policy(extent_types_t type, extent_len_t length) const { - if (has_cold_tier() && length >= write_through_size && is_data_type(type)) { - return write_policy_t::WRITE_THROUGH; + if (has_cold_tier() && is_data_type(type)) { + if (length >= write_through_size + || (test_workload + && (double(std::rand() % 100) / 100.0) <= + write_through_probability)) + { + return write_policy_t::WRITE_THROUGH; + } } return write_policy_t::WRITE_BACK; } @@ -726,6 +736,10 @@ public: return hot_tier_generations - 1; } + device_id_t get_cold_device_id() const { + return background_process.get_cold_device_id(); + } + private: rewrite_gen_t adjust_generation( data_category_t category, @@ -900,6 +914,18 @@ private: get_conf("seastore_logical_bucket_proceed_size_per_cycle")); logical_bucket->set_background_callback(this); } + LOG_PREFIX(BackgroundProcess::init); + test_workload = crimson::common::get_conf( + "seastore_logical_bucket_cache_test_stress"); + force_process_half_life = crimson::common::get_conf( + "seastore_test_workload_force_process_background_tasks_period"); + force_background_timer.set_callback([this] { wake_half_life(); }); + write_through_probability = crimson::common::get_conf( + "seastore_test_workload_write_through_probability"); + SUBINFO(seastore_epm, "crimson test workload supported, enabled: {}", test_workload); + if (test_workload) { + set_next_force_process(); + } } LogicalBucket *get_logical_bucket() { @@ -1016,6 +1042,12 @@ private: } } + + device_id_t get_cold_device_id() const { + assert(has_cold_tier()); + return *cold_cleaner->get_device_ids().begin(); + } + seastar::future<> reserve_projected_usage(io_usage_t usage); void release_projected_usage(const io_usage_t &usage) { @@ -1322,6 +1354,57 @@ private: state_t state = state_t::STOP; eviction_state_t eviction_state; + enum class ForceProcessState : uint8_t{ + STOP, + TRIM, + CLEAN, + }; + bool test_workload = false; + double write_through_probability = 0; + ForceProcessState force_process_state = ForceProcessState::STOP; + ForceProcessState last_process_state = ForceProcessState::STOP; + seastar::timer force_background_timer; + int force_process_half_life; + + void set_next_force_process() { + assert(test_workload); + force_background_timer.rearm( + seastar::steady_clock_type::now() + + std::chrono::seconds(force_process_half_life)); + } + + void wake_half_life() { + assert(test_workload); + if (last_process_state == ForceProcessState::TRIM) { + force_process_state = ForceProcessState::CLEAN; + } else { + force_process_state = ForceProcessState::TRIM; + } + + do_wake_background(); + } + + void maybe_reschedule_force_process() { + if (unlikely(test_workload && + force_process_state != ForceProcessState::STOP)) { + last_process_state = force_process_state; + force_process_state = ForceProcessState::STOP; + set_next_force_process(); + } + } + + bool should_force_trim() const { + return test_workload && force_process_state == ForceProcessState::TRIM; + } + + bool should_force_clean() const { + return test_workload && force_process_state == ForceProcessState::CLEAN; + } + + bool force_run_background() const { + return test_workload && force_process_state != ForceProcessState::STOP + && (logical_bucket && logical_bucket->could_demote()); + } friend class ::transaction_manager_test_t; }; @@ -1345,6 +1428,8 @@ private: SegmentSeqAllocatorRef ool_segment_seq_allocator; extent_len_t max_data_allocation_size = 0; std::size_t write_through_size = 0; + bool test_workload = false; + double write_through_probability = 0; friend class ::transaction_manager_test_t; friend class Cache; diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc index d8992f148e8..e367dea437c 100644 --- a/src/crimson/os/seastore/seastore.cc +++ b/src/crimson/os/seastore/seastore.cc @@ -728,6 +728,23 @@ Device::access_ertr::future<> SeaStore::_mkfs(uuid_d new_osd_fsid) co_await rdir.close(); } + if (sds.empty() && crimson::common::get_conf( + "seastore_logical_bucket_cache_test_stress")) { + // lbc test workload enabled while no secondary devices indicated, create one + std::string path = fmt::format("{}/block.1", root); + co_await seastar::make_directory(path); + DeviceRef sec_dev = co_await Device::make_device(path, dtype, btype); + auto p_sec_dev = sec_dev.get(); + secondaries.emplace_back(std::move(sec_dev)); + co_await p_sec_dev->start(store_shard_nums); + magic_t magic = (magic_t)std::rand(); + device_id_t id = 0x1; + sds.emplace(id, device_spec_t{magic, dtype, btype, id}); + co_await p_sec_dev->mkfs( + device_config_t::create_secondary(new_osd_fsid, id, dtype, btype, magic) + ).handle_error(crimson::ct_error::assert_all("not possible")); + co_await set_secondaries(); + } device_id_t id = 0; device_type_t d_type = device->get_device_type(); backend_type_t b_type = device->get_backend_type(); diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index dedd6060b9e..831ef725eab 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -1331,6 +1331,59 @@ TransactionManager::promote_extent( t, *mapping.direct_cursor, std::move(promoted_extents)); } +TransactionManager::promote_extent_ret +TransactionManager::promote_extents_from_disk( + Transaction &t, + paddr_t paddr) +{ + using scan_device_func_t = BackrefManager::scan_device_func_t; + std::size_t size = 0; + scan_device_func_t func = [this, &t, &size]( + paddr_t paddr, extent_len_t length, extent_types_t type, laddr_t laddr) + -> base_iertr::future { + if (type != extent_types_t::OBJECT_DATA_BLOCK) { + co_return seastar::stop_iteration::no; + } + auto cursor = co_await lba_manager->get_cursor(t, laddr + ).handle_error_interruptible( + crimson::ct_error::enoent::handle([](auto e) { + // Another no_conflict transaction should have removed + // the mapping between the backref retrieval and the + // lba search, ignore it. + return seastar::make_ready_future(); + }), + crimson::ct_error::pass_further_all{} + ); + if (!cursor || cursor->is_end() || + !cursor->get_paddr().is_absolute() || + !cache->is_on_cold_tier(cursor->get_paddr())) { + // the mapping has been modified and the extent is + // either removed or already on the hot tier, skip it. + co_return seastar::stop_iteration::no; + } + assert(cursor->is_direct()); + assert(!cursor->has_shadow_paddr()); + auto extent = co_await read_cursor_by_type(t, std::move(cursor), type); + if (extent->is_stable_dirty()) { + // dirty extents shouldn't be promoted as is in + // the real world + co_return seastar::stop_iteration::no; + } + auto &pinboard = *cache->get_extent_pinboard(); + pinboard.remove(*extent); + extent->set_pin_state(extent_pin_state_t::Promoting); + co_await promote_extent(t, extent); + size += length; + if (size >= crimson::common::get_conf< + Option::size_t>("seastore_cache_promotion_size")) { + co_return seastar::stop_iteration::yes; + } else { + co_return seastar::stop_iteration::no; + } + }; + co_await backref_manager->scan_device(t, paddr, func); +} + TransactionManager::rewrite_extents_ret TransactionManager::rewrite_extents( Transaction &t, std::vector &extents, @@ -1655,7 +1708,10 @@ TransactionManagerRef make_transaction_manager( store_index, *backref_manager, trimmer_config, backend_type, roll_start, roll_size, - !pure_rbm_backend); + !pure_rbm_backend + || crimson::common::get_conf( + "seastore_logical_bucket_cache_test_stress") + ); AsyncCleanerRef cleaner; JournalRef journal; diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index 9ffb08719a6..f1e350afdde 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -964,7 +964,11 @@ public: using ExtentCallbackInterface::promote_extent_ret; promote_extent_ret promote_extent( Transaction &t, - CachedExtentRef extent); + CachedExtentRef extent) final; + + promote_extent_ret promote_extents_from_disk( + Transaction &t, + paddr_t paddr) final; using ExtentCallbackInterface::demote_region_res_t; using ExtentCallbackInterface::demote_region_ret; diff --git a/src/test/crimson/seastore/test_transaction_manager.cc b/src/test/crimson/seastore/test_transaction_manager.cc index f5b5c9f77a3..e34d003a3e0 100644 --- a/src/test/crimson/seastore/test_transaction_manager.cc +++ b/src/test/crimson/seastore/test_transaction_manager.cc @@ -922,7 +922,7 @@ struct transaction_manager_test_t : if (run_clean) { return epm->run_background_work_until_halt(); } else { - return epm->background_process.trimmer->trim(); + return epm->background_process.trimmer->trim(false); } }).handle_error( crimson::ct_error::assert_all( From c48848ba9fa42b73dc37fc452de7ef572bf8f179 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Sat, 9 May 2026 10:15:24 +0800 Subject: [PATCH 25/51] test/crimson/seastore/test_transaction_manager: the scatter_allocation case shouldn't submit the trans In scatter_allocation, an extent is split into several small ones, and the last allocation is supposed to fail, in which case, the small extents that belong to the same allocation wouldn't be attached to the parent lba leaf node. Sumitting the transaction would trigger assertions, and is not consistent with the behavior of nornal crimson OSDs. Signed-off-by: Xuehan Xu --- src/test/crimson/seastore/test_transaction_manager.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test/crimson/seastore/test_transaction_manager.cc b/src/test/crimson/seastore/test_transaction_manager.cc index e34d003a3e0..71edd935a28 100644 --- a/src/test/crimson/seastore/test_transaction_manager.cc +++ b/src/test/crimson/seastore/test_transaction_manager.cc @@ -1904,13 +1904,12 @@ TEST_P(tm_random_block_device_test_t, scatter_allocation) epm->prefill_fragmented_devices(); auto t = create_transaction(); for (int i = 0; i < 1958; i++) { + logger().info("scatter_allocation: {}", i); auto extents = alloc_extents(t, (ADDR + i * 16384).checked_to_laddr(), 16384, 'a'); } alloc_extents_deemed_fail(t, (ADDR + 1958 * 16384).checked_to_laddr(), 16384, 'a'); check_mappings(t); check(); - submit_transaction(std::move(t)); - check(); }); } From 660c7d86f48bb19ccc1828764a940b7dc6afddf3 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Mon, 11 May 2026 14:27:25 +0800 Subject: [PATCH 26/51] crimson/os/seastore/cache: count promote/demote related metrics Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/cache.cc | 54 +++++++++++++++++++++++++++++++- src/crimson/os/seastore/cache.h | 4 +++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc index 7c7d44d32ed..5e7e4586219 100644 --- a/src/crimson/os/seastore/cache.cc +++ b/src/crimson/os/seastore/cache.cc @@ -122,6 +122,8 @@ void Cache::register_metrics(store_index_t store_index) last_dirty_io_by_src_ext = {}; last_trim_rewrites = {}; last_reclaim_rewrites = {}; + last_promote_rewrites = {};; + last_demote_rewrites = {}; last_access = {}; last_cache_absent_by_src = {}; last_access_by_src_ext = {}; @@ -779,6 +781,34 @@ void Cache::register_metrics(store_index_t store_index) sm::description("sum of the version from rewrite-reclaim extents"), {sm::label_instance("shard_store_index", std::to_string(store_index))} ), + sm::make_counter( + "version_count_promote", + [this] { + return stats.promote_rewrites.get_num_rewrites(); + }, + sm::description("total number of rewrite-promote extents"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} + ), + sm::make_counter( + "version_sum_promote", + stats.promote_rewrites.dirty_version, + sm::description("sum of the version from rewrite-promote extents"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} + ), + sm::make_counter( + "version_count_demote", + [this] { + return stats.demote_rewrites.get_num_rewrites(); + }, + sm::description("total number of rewrite-demote extents"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} + ), + sm::make_counter( + "version_sum_demote", + stats.demote_rewrites.dirty_version, + sm::description("sum of the version from rewrite-demote extents"), + {sm::label_instance("shard_store_index", std::to_string(store_index))} + ), } ); } @@ -1979,6 +2009,10 @@ record_t Cache::prepare_record( } else if (trans_src == Transaction::src_t::CLEANER_MAIN || trans_src == Transaction::src_t::CLEANER_COLD) { stats.reclaim_rewrites.add(rewrite_stats); + } else if (trans_src == Transaction::src_t::PROMOTE) { + stats.promote_rewrites.add(rewrite_stats); + } else if (trans_src == Transaction::src_t::DEMOTE) { + stats.demote_rewrites.add(rewrite_stats); } else { assert(rewrite_stats.is_clear()); } @@ -2876,6 +2910,10 @@ cache_stats_t Cache::get_stats( _trim_rewrites.minus(last_trim_rewrites); rewrite_stats_t _reclaim_rewrites = stats.reclaim_rewrites; _reclaim_rewrites.minus(last_reclaim_rewrites); + rewrite_stats_t _promote_rewrites = stats.promote_rewrites; + _promote_rewrites.minus(last_promote_rewrites); + rewrite_stats_t _demote_rewrites = stats.demote_rewrites; + _demote_rewrites.minus(last_demote_rewrites); oss << "\nrewrite trim ndirty=" << fmt::format(dfmt, _trim_rewrites.num_n_dirty/seconds) << "ps, dirty=" @@ -2887,7 +2925,19 @@ cache_stats_t Cache::get_stats( << "ps, dirty=" << fmt::format(dfmt, _reclaim_rewrites.num_dirty/seconds) << "ps, dversion=" - << fmt::format(dfmt, _reclaim_rewrites.get_avg_version()); + << fmt::format(dfmt, _reclaim_rewrites.get_avg_version()) + << "; promote ndirty=" + << fmt::format(dfmt, _promote_rewrites.num_n_dirty/seconds) + << "ps, dirty=" + << fmt::format(dfmt, _promote_rewrites.num_dirty/seconds) + << "ps, dversion=" + << fmt::format(dfmt, _promote_rewrites.get_avg_version()) + << "; demote ndirty=" + << fmt::format(dfmt, _demote_rewrites.num_n_dirty/seconds) + << "ps, dirty=" + << fmt::format(dfmt, _demote_rewrites.num_dirty/seconds) + << "ps, dversion=" + << fmt::format(dfmt, _demote_rewrites.get_avg_version()); oss << "\ncache total" << cache_size_stats_t{extents_index.get_bytes(), extents_index.size()}; @@ -2946,6 +2996,8 @@ cache_stats_t Cache::get_stats( last_dirty_io_by_src_ext = stats.dirty_io_by_src_ext; last_trim_rewrites = stats.trim_rewrites; last_reclaim_rewrites = stats.reclaim_rewrites; + last_promote_rewrites = stats.promote_rewrites; + last_demote_rewrites = stats.demote_rewrites; last_cache_absent_by_src = stats.cache_absent_by_src; last_access_by_src_ext = stats.access_by_src_ext; } diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h index d92e42802b3..8a47ab7104b 100644 --- a/src/crimson/os/seastore/cache.h +++ b/src/crimson/os/seastore/cache.h @@ -1901,6 +1901,8 @@ private: rewrite_stats_t trim_rewrites; rewrite_stats_t reclaim_rewrites; + rewrite_stats_t promote_rewrites; + rewrite_stats_t demote_rewrites; } stats; mutable dirty_io_stats_t last_dirty_io; @@ -1908,6 +1910,8 @@ private: last_dirty_io_by_src_ext; mutable rewrite_stats_t last_trim_rewrites; mutable rewrite_stats_t last_reclaim_rewrites; + mutable rewrite_stats_t last_promote_rewrites; + mutable rewrite_stats_t last_demote_rewrites; mutable cache_access_stats_t last_access; mutable counter_by_src_t last_cache_absent_by_src; mutable counter_by_src_t > From c8769a0f3416ddc53001ce5463361988beea9d87 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Wed, 13 May 2026 15:13:28 +0800 Subject: [PATCH 27/51] qa/config/crimson_qa_overrides.yaml: add test workload for LBC Signed-off-by: Xuehan Xu --- qa/config/crimson_logical_bucket_cache.yaml | 5 +++++ qa/config/crimson_qa_overrides.yaml | 2 ++ .../seastore/rbm$/crimson_seastore_rbm.yaml | 5 ++++- .../seastore/rbm$/crimson_seastore_rbm_2q.yaml | 5 ++++- .../seastore/segmented$/crimson_seastore_segmented.yaml | 2 ++ .../seastore/segmented$/crimson_seastore_segmented_2q.yaml | 2 ++ qa/suites/crimson-rados/basic/deploy/ceph.yaml | 5 +++++ .../crimson-rados/cephfs/crimson_logical_bucket_cache.yaml | 1 + .../objectstore_tool/crimson_logical_bucket_cache.yaml | 1 + .../osd_shards/crimson_logical_bucket_cache.yaml | 1 + .../crimson-rados/rbd/crimson_logical_bucket_cache.yaml | 1 + .../crimson-rados/rgw/crimson_logical_bucket_cache.yaml | 1 + .../crimson-rados/thrash/crimson_logical_bucket_cache.yaml | 1 + qa/tasks/ceph_objectstore_tool.py | 4 ++-- 14 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 qa/config/crimson_logical_bucket_cache.yaml create mode 120000 qa/suites/crimson-rados/cephfs/crimson_logical_bucket_cache.yaml create mode 120000 qa/suites/crimson-rados/objectstore_tool/crimson_logical_bucket_cache.yaml create mode 120000 qa/suites/crimson-rados/osd_shards/crimson_logical_bucket_cache.yaml create mode 120000 qa/suites/crimson-rados/rbd/crimson_logical_bucket_cache.yaml create mode 120000 qa/suites/crimson-rados/rgw/crimson_logical_bucket_cache.yaml create mode 120000 qa/suites/crimson-rados/thrash/crimson_logical_bucket_cache.yaml diff --git a/qa/config/crimson_logical_bucket_cache.yaml b/qa/config/crimson_logical_bucket_cache.yaml new file mode 100644 index 00000000000..c0223f80507 --- /dev/null +++ b/qa/config/crimson_logical_bucket_cache.yaml @@ -0,0 +1,5 @@ +overrides: + ceph: + conf: + osd: + seastore logical bucket cache test stress: true diff --git a/qa/config/crimson_qa_overrides.yaml b/qa/config/crimson_qa_overrides.yaml index ce095c423af..3551d7e0870 100644 --- a/qa/config/crimson_qa_overrides.yaml +++ b/qa/config/crimson_qa_overrides.yaml @@ -14,6 +14,8 @@ overrides: debug ms: 20 seastore cachepin size pershard: 64M seastore max concurrent transactions: 8 + client: + debug_objecter: 20 crimson_compat: true workunit: env: diff --git a/qa/objectstore_crimson/seastore/rbm$/crimson_seastore_rbm.yaml b/qa/objectstore_crimson/seastore/rbm$/crimson_seastore_rbm.yaml index 1674a1d8a02..0e8111a15ef 100644 --- a/qa/objectstore_crimson/seastore/rbm$/crimson_seastore_rbm.yaml +++ b/qa/objectstore_crimson/seastore/rbm$/crimson_seastore_rbm.yaml @@ -4,7 +4,10 @@ overrides: osd: # crimson's osd objectstore option osd objectstore: seastore - seastore main device type: RANDOM_BLOCK_SSD + seastore hot device type: RANDOM_BLOCK_SSD + seastore hot backend type: RANDOM_BLOCK + seastore cold device type: RANDOM_BLOCK_HDD + seastore cold backend type: RANDOM_BLOCK debug seastore: 20 debug seastore onode: 20 debug seastore odata: 20 diff --git a/qa/objectstore_crimson/seastore/rbm$/crimson_seastore_rbm_2q.yaml b/qa/objectstore_crimson/seastore/rbm$/crimson_seastore_rbm_2q.yaml index 27467c10c6a..a74bc32a243 100644 --- a/qa/objectstore_crimson/seastore/rbm$/crimson_seastore_rbm_2q.yaml +++ b/qa/objectstore_crimson/seastore/rbm$/crimson_seastore_rbm_2q.yaml @@ -5,7 +5,10 @@ overrides: # crimson's osd objectstore option osd objectstore: seastore seastore cachepin type: 2Q - seastore main device type: RANDOM_BLOCK_SSD + seastore hot device type: RANDOM_BLOCK_SSD + seastore hot backend type: RANDOM_BLOCK + seastore cold device type: RANDOM_BLOCK_HDD + seastore cold backend type: RANDOM_BLOCK debug seastore: 20 debug seastore onode: 20 debug seastore odata: 20 diff --git a/qa/objectstore_crimson/seastore/segmented$/crimson_seastore_segmented.yaml b/qa/objectstore_crimson/seastore/segmented$/crimson_seastore_segmented.yaml index 148de206da3..8bd32f33085 100644 --- a/qa/objectstore_crimson/seastore/segmented$/crimson_seastore_segmented.yaml +++ b/qa/objectstore_crimson/seastore/segmented$/crimson_seastore_segmented.yaml @@ -4,6 +4,8 @@ overrides: osd: # crimson's osd objectstore option osd objectstore: seastore + seastore cold device type: RANDOM_BLOCK_HDD + seastore cold backend type: RANDOM_BLOCK debug seastore: 20 debug seastore onode: 20 debug seastore odata: 20 diff --git a/qa/objectstore_crimson/seastore/segmented$/crimson_seastore_segmented_2q.yaml b/qa/objectstore_crimson/seastore/segmented$/crimson_seastore_segmented_2q.yaml index 1ceadfb0c24..8069fce3d77 100644 --- a/qa/objectstore_crimson/seastore/segmented$/crimson_seastore_segmented_2q.yaml +++ b/qa/objectstore_crimson/seastore/segmented$/crimson_seastore_segmented_2q.yaml @@ -5,6 +5,8 @@ overrides: # crimson's osd objectstore option osd objectstore: seastore seastore cachepin type: 2Q + seastore cold device type: RANDOM_BLOCK_HDD + seastore cold backend type: RANDOM_BLOCK debug seastore: 20 debug seastore onode: 20 debug seastore odata: 20 diff --git a/qa/suites/crimson-rados/basic/deploy/ceph.yaml b/qa/suites/crimson-rados/basic/deploy/ceph.yaml index 1af2f4b1bd2..4d836918199 100644 --- a/qa/suites/crimson-rados/basic/deploy/ceph.yaml +++ b/qa/suites/crimson-rados/basic/deploy/ceph.yaml @@ -11,3 +11,8 @@ tasks: mon osdmap full prune min: 15 mon osdmap full prune interval: 2 mon osdmap full prune txsize: 2 +overrides: + ceph: + conf: + osd: + seastore logical bucket cache test stress: true diff --git a/qa/suites/crimson-rados/cephfs/crimson_logical_bucket_cache.yaml b/qa/suites/crimson-rados/cephfs/crimson_logical_bucket_cache.yaml new file mode 120000 index 00000000000..d963ced9047 --- /dev/null +++ b/qa/suites/crimson-rados/cephfs/crimson_logical_bucket_cache.yaml @@ -0,0 +1 @@ +.qa/config/crimson_logical_bucket_cache.yaml \ No newline at end of file diff --git a/qa/suites/crimson-rados/objectstore_tool/crimson_logical_bucket_cache.yaml b/qa/suites/crimson-rados/objectstore_tool/crimson_logical_bucket_cache.yaml new file mode 120000 index 00000000000..d963ced9047 --- /dev/null +++ b/qa/suites/crimson-rados/objectstore_tool/crimson_logical_bucket_cache.yaml @@ -0,0 +1 @@ +.qa/config/crimson_logical_bucket_cache.yaml \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/crimson_logical_bucket_cache.yaml b/qa/suites/crimson-rados/osd_shards/crimson_logical_bucket_cache.yaml new file mode 120000 index 00000000000..d963ced9047 --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/crimson_logical_bucket_cache.yaml @@ -0,0 +1 @@ +.qa/config/crimson_logical_bucket_cache.yaml \ No newline at end of file diff --git a/qa/suites/crimson-rados/rbd/crimson_logical_bucket_cache.yaml b/qa/suites/crimson-rados/rbd/crimson_logical_bucket_cache.yaml new file mode 120000 index 00000000000..d963ced9047 --- /dev/null +++ b/qa/suites/crimson-rados/rbd/crimson_logical_bucket_cache.yaml @@ -0,0 +1 @@ +.qa/config/crimson_logical_bucket_cache.yaml \ No newline at end of file diff --git a/qa/suites/crimson-rados/rgw/crimson_logical_bucket_cache.yaml b/qa/suites/crimson-rados/rgw/crimson_logical_bucket_cache.yaml new file mode 120000 index 00000000000..d963ced9047 --- /dev/null +++ b/qa/suites/crimson-rados/rgw/crimson_logical_bucket_cache.yaml @@ -0,0 +1 @@ +.qa/config/crimson_logical_bucket_cache.yaml \ No newline at end of file diff --git a/qa/suites/crimson-rados/thrash/crimson_logical_bucket_cache.yaml b/qa/suites/crimson-rados/thrash/crimson_logical_bucket_cache.yaml new file mode 120000 index 00000000000..d963ced9047 --- /dev/null +++ b/qa/suites/crimson-rados/thrash/crimson_logical_bucket_cache.yaml @@ -0,0 +1 @@ +.qa/config/crimson_logical_bucket_cache.yaml \ No newline at end of file diff --git a/qa/tasks/ceph_objectstore_tool.py b/qa/tasks/ceph_objectstore_tool.py index ba978dbb299..444694bd906 100644 --- a/qa/tasks/ceph_objectstore_tool.py +++ b/qa/tasks/ceph_objectstore_tool.py @@ -209,8 +209,8 @@ def task(ctx, config): manager.raw_cluster_cmd('osd', 'set', 'nodown') if CRIMSON: - CRIMSON_DEVICE_TYPE = manager.get_config('osd', 0, 'seastore_main_device_type') - log.info('seastore_main_device_type is {}...'.format(CRIMSON_DEVICE_TYPE)) + CRIMSON_DEVICE_TYPE = manager.get_config('osd', 0, 'seastore_hot_device_type') + log.info('seastore_hot_device_type is {}...'.format(CRIMSON_DEVICE_TYPE)) PGNUM = config.get('pgnum', 12) log.info("pgnum: {num}".format(num=PGNUM)) From e4979f98ed5be34f4fce9eb17cff27b14e9985bd Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Wed, 13 May 2026 19:51:58 +0800 Subject: [PATCH 28/51] crimson/os/seastore: drop backref when all backends are RBMs Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/async_cleaner.h | 2 +- src/crimson/os/seastore/extent_placement_manager.h | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/crimson/os/seastore/async_cleaner.h b/src/crimson/os/seastore/async_cleaner.h index 59f0311a72b..1efc06b3c57 100644 --- a/src/crimson/os/seastore/async_cleaner.h +++ b/src/crimson/os/seastore/async_cleaner.h @@ -702,7 +702,7 @@ private: } bool can_drop_backref() const { - return get_backend_type() == backend_type_t::RANDOM_BLOCK; + return !tail_include_alloc; } bool should_trim_alloc() const { diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index 11463df8fb3..8bd25d67219 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -689,8 +689,9 @@ public: bool is_pure_rbm() const { return get_main_backend_type() == backend_type_t::RANDOM_BLOCK && - // as of now, cold tier can only be segmented. - !background_process.has_cold_tier(); + (!background_process.has_cold_tier() || + (background_process.get_cold_tier_backend_type() == + backend_type_t::RANDOM_BLOCK)); } bool has_cold_tier() const { @@ -945,6 +946,11 @@ private: return cold_cleaner.get() != nullptr; } + backend_type_t get_cold_tier_backend_type() const { + assert(cold_cleaner); + return cold_cleaner->get_backend_type(); + } + bool is_cold_device(device_id_t id) const { if (!has_cold_tier()) { return false; From eeba1411a29d4f3540f2e038acac2fc865e3e564 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Sat, 16 May 2026 13:40:35 +0800 Subject: [PATCH 29/51] crimson/os/seastore/transaction_manager: wait for the demotion when the rbm main device is not enough Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/async_cleaner.h | 8 +- .../seastore/backref/btree_backref_manager.cc | 20 +- .../os/seastore/btree/fixed_kv_btree.h | 371 ++++++++++-------- src/crimson/os/seastore/cache.h | 22 +- .../os/seastore/extent_placement_manager.cc | 13 +- .../os/seastore/extent_placement_manager.h | 34 +- .../os/seastore/lba/btree_lba_manager.cc | 7 +- .../os/seastore/transaction_manager.cc | 120 ++++-- src/crimson/os/seastore/transaction_manager.h | 45 ++- .../seastore/test_btree_lba_manager.cc | 4 +- 10 files changed, 404 insertions(+), 240 deletions(-) diff --git a/src/crimson/os/seastore/async_cleaner.h b/src/crimson/os/seastore/async_cleaner.h index 1efc06b3c57..647e0b233b3 100644 --- a/src/crimson/os/seastore/async_cleaner.h +++ b/src/crimson/os/seastore/async_cleaner.h @@ -1285,6 +1285,8 @@ public: virtual bool should_clean_space() const = 0; + virtual double get_alive_ratio() const = 0; + using clean_space_ertr = base_ertr; using clean_space_ret = clean_space_ertr::future<>; virtual clean_space_ret clean_space() = 0; @@ -1663,7 +1665,7 @@ private: if (segments.get_unavailable_bytes() == 0) return 0; return (double)get_unavailable_unused_bytes() / (double)segments.get_unavailable_bytes(); } - double get_alive_ratio() const { + double get_alive_ratio() const final { return stats.used_bytes / (double)segments.get_total_bytes(); } @@ -1839,6 +1841,10 @@ public: return st; } + double get_alive_ratio() const final { + return stats.used_bytes / (double)get_total_bytes(); + } + void print(std::ostream &, bool is_detailed) const final; mount_ret mount() final; diff --git a/src/crimson/os/seastore/backref/btree_backref_manager.cc b/src/crimson/os/seastore/backref/btree_backref_manager.cc index 2190d0327b8..505ed21790d 100644 --- a/src/crimson/os/seastore/backref/btree_backref_manager.cc +++ b/src/crimson/os/seastore/backref/btree_backref_manager.cc @@ -72,16 +72,16 @@ BtreeBackrefManager::mkfs( { LOG_PREFIX(BtreeBackrefManager::mkfs); INFOT("start", t); - return cache.get_root(t).si_then([this, &t](auto croot) { - assert(croot->is_mutation_pending()); - croot->get_root().backref_root = BackrefBtree::mkfs(croot, get_context(t)); - return mkfs_iertr::now(); - }).handle_error_interruptible( - mkfs_iertr::pass_further{}, - crimson::ct_error::assert_all( - "Invalid error in BtreeBackrefManager::mkfs" - ) - ); + + auto croot = co_await cache.get_root(t); + croot->get_root().backref_root = + co_await BackrefBtree::mkfs(croot, get_context(t) + ).handle_error_interruptible( + mkfs_iertr::pass_further{}, + crimson::ct_error::assert_all( + "Invalid error in BtreeBackrefManager::mkfs" + ) + ); } BtreeBackrefManager::get_mapping_ret diff --git a/src/crimson/os/seastore/btree/fixed_kv_btree.h b/src/crimson/os/seastore/btree/fixed_kv_btree.h index d8256c7552a..59b7a150ee0 100644 --- a/src/crimson/os/seastore/btree/fixed_kv_btree.h +++ b/src/crimson/os/seastore/btree/fixed_kv_btree.h @@ -649,11 +649,25 @@ public: * leaf node as the root, links it to the RootBlock, and returns the * tree's root descriptor. */ - using mkfs_ret = phy_tree_root_t; + using mkfs_iertr = base_iertr; + using mkfs_ret = mkfs_iertr::future; static mkfs_ret mkfs(RootBlockRef &root_block, op_context_t c) { assert(root_block->is_mutation_pending()); - auto root_leaf = c.cache.template alloc_new_non_data_extent( - c.trans, node_size, {placement_hint_t::HOT, INIT_GENERATION}); + LeafNodeRef root_leaf; + while (!root_leaf) { + try { + root_leaf = c.cache.template alloc_new_non_data_extent( + c.trans, node_size, {placement_hint_t::HOT, INIT_GENERATION}); + } catch (crimson::ct_error::eagain&) { + } catch (crimson::ct_error::enospc&) { + ceph_abort("shouldn't have no space"); + } + if (!root_leaf) { + c.cache.get_epm().maybe_wake_background(); + co_await trans_intr::make_interruptible( + c.cache.get_epm().wait_background()); + } + } root_leaf->set_size(0); fixed_kv_node_meta_t meta{min_max_t::min, min_max_t::max, 1}; root_leaf->set_meta(meta); @@ -661,7 +675,7 @@ public: get_tree_stats(c.trans).depth = 1u; get_tree_stats(c.trans).extents_num_delta++; TreeRootLinker::link_root(root_block, root_leaf.get()); - return phy_tree_root_t{root_leaf->get_paddr(), 1u}; + co_return phy_tree_root_t{root_leaf->get_paddr(), 1u}; } /** @@ -1727,18 +1741,29 @@ public: LOG_PREFIX(FixedKVBtree::rewrite_extent); assert(is_lba_backref_node(e->get_type())); - auto do_rewrite = [&](auto &fixed_kv_extent) { + auto do_rewrite = [&](auto &fixed_kv_extent) -> rewrite_extent_ret { auto opt = Cache::alloc_option_t { fixed_kv_extent.get_user_hint(), // get target rewrite generation fixed_kv_extent.get_rewrite_generation() }; - auto n_fixed_kv_extent = c.cache.template alloc_new_non_data_extent< - std::remove_reference_t - >( - c.trans, - fixed_kv_extent.get_length(), - opt); + TCachedExtentRef> n_fixed_kv_extent; + while (!n_fixed_kv_extent) { + try { + n_fixed_kv_extent = c.cache.template alloc_new_non_data_extent< + std::remove_reference_t + >( + c.trans, + fixed_kv_extent.get_length(), + opt); + } catch (crimson::ct_error::eagain&) {} + if (!n_fixed_kv_extent) { + c.cache.get_epm().maybe_wake_background(); + co_await trans_intr::make_interruptible( + c.cache.get_epm().wait_background()); + } + } n_fixed_kv_extent->rewrite(c.trans, fixed_kv_extent, 0); SUBTRACET( @@ -1747,26 +1772,24 @@ public: c.trans, fixed_kv_extent, *n_fixed_kv_extent); - - return update_internal_mapping( + + co_await update_internal_mapping( c, n_fixed_kv_extent->get_node_meta().depth, n_fixed_kv_extent->get_node_meta().begin, e->get_paddr(), n_fixed_kv_extent->get_paddr(), - n_fixed_kv_extent - ).si_then([c, e] { - c.cache.retire_extent(c.trans, e); - }); + n_fixed_kv_extent); + c.cache.retire_extent(c.trans, e); }; if (e->get_type() == internal_node_t::TYPE) { auto lint = e->cast(); - return do_rewrite(*lint); + co_await do_rewrite(*lint); } else { assert(e->get_type() == leaf_node_t::TYPE); auto lleaf = e->cast(); - return do_rewrite(*lleaf); + co_await do_rewrite(*lleaf); } } @@ -2601,126 +2624,150 @@ private: { LOG_PREFIX(FixedKVBtree::handle_split); - return iter.check_split(c - ).si_then([FNAME, this, c, &iter](auto split_from) { - SUBTRACET(seastore_fixedkv_tree, - "split_from {}, depth {}", c.trans, split_from, iter.get_depth()); + auto split_from = co_await iter.check_split(c); + SUBTRACET(seastore_fixedkv_tree, + "split_from {}, depth {}", c.trans, split_from, iter.get_depth()); - if (split_from == iter.get_depth()) { - assert(iter.is_full()); - auto nroot = c.cache.template alloc_new_non_data_extent( - c.trans, node_size, {placement_hint_t::HOT, INIT_GENERATION}); - fixed_kv_node_meta_t meta{ - min_max_t::min, min_max_t::max, iter.get_depth() + 1}; - nroot->set_meta(meta); - nroot->range = meta; - nroot->journal_insert( - nroot->begin(), - min_max_t::min, - get_root().get_location(), - nullptr); - iter.internal.push_back({nroot, 0}); + if (split_from == iter.get_depth()) { + assert(iter.is_full()); + InternalNodeRef nroot; + while (!nroot) { + try { + nroot = c.cache.template alloc_new_non_data_extent( + c.trans, node_size, {placement_hint_t::HOT, INIT_GENERATION}); + } catch (crimson::ct_error::eagain&) {} + if (!nroot) { + c.cache.get_epm().maybe_wake_background(); + co_await trans_intr::make_interruptible( + c.cache.get_epm().wait_background()); + } + } + fixed_kv_node_meta_t meta{ + min_max_t::min, min_max_t::max, iter.get_depth() + 1}; + nroot->set_meta(meta); + nroot->range = meta; + nroot->journal_insert( + nroot->begin(), + min_max_t::min, + get_root().get_location(), + nullptr); + iter.internal.push_back({nroot, 0}); - get_tree_stats(c.trans).depth = iter.get_depth(); - get_tree_stats(c.trans).extents_num_delta++; + get_tree_stats(c.trans).depth = iter.get_depth(); + get_tree_stats(c.trans).extents_num_delta++; - root_block = c.cache.duplicate_for_write( - c.trans, root_block)->template cast(); - get_root().set_location(nroot->get_paddr()); - get_root().set_depth(iter.get_depth()); - ceph_assert(get_root().get_depth() <= MAX_DEPTH); - set_root_node(nroot); + root_block = c.cache.duplicate_for_write( + c.trans, root_block)->template cast(); + get_root().set_location(nroot->get_paddr()); + get_root().set_depth(iter.get_depth()); + ceph_assert(get_root().get_depth() <= MAX_DEPTH); + set_root_node(nroot); + } + + /* pos may be either node_position_t or + * node_position_t */ + auto split_level = [&, c, FNAME](auto &parent_pos, auto &pos) + -> handle_split_iertr::future> { + decltype(pos.node) left, right; + node_key_t pivot = min_max_t::null; + while (!left) { + assert(!right); + assert(pivot == min_max_t::null); + try { + auto t = pos.node->make_split_children(c); + left = std::get<0>(t); + right = std::get<1>(t); + pivot = std::get<2>(t); + } catch (crimson::ct_error::eagain&) {} + if (!left) { + c.cache.get_epm().maybe_wake_background(); + co_await trans_intr::make_interruptible( + c.cache.get_epm().wait_background()); + } } - /* pos may be either node_position_t or - * node_position_t */ - auto split_level = [&, c, FNAME](auto &parent_pos, auto &pos) { - auto [left, right, pivot] = pos.node->make_split_children(c); + auto parent_node = parent_pos.node; + auto parent_iter = parent_pos.get_iter(); - auto parent_node = parent_pos.node; - auto parent_iter = parent_pos.get_iter(); + parent_node->update( + parent_iter, + left->get_paddr(), + left.get()); + parent_node->insert( + parent_iter + 1, + pivot, + right->get_paddr(), + right.get()); - parent_node->update( - parent_iter, - left->get_paddr(), - left.get()); - parent_node->insert( - parent_iter + 1, - pivot, - right->get_paddr(), - right.get()); + SUBTRACET( + seastore_fixedkv_tree, + "splitted {} into left: {}, right: {}", + c.trans, + *pos.node, + *left, + *right); + c.cache.retire_extent(c.trans, pos.node); + get_tree_stats(c.trans).extents_num_delta++; + co_return std::make_pair(left, right); + }; + + for (; split_from > 0; --split_from) { + auto &parent_pos = iter.get_internal(split_from + 1); + if (!parent_pos.node->is_mutable()) { + parent_pos.node = c.cache.duplicate_for_write( + c.trans, parent_pos.node + )->template cast(); + } + + if (split_from > 1) { + auto &pos = iter.get_internal(split_from); SUBTRACET( seastore_fixedkv_tree, - "splitted {} into left: {}, right: {}", + "splitting internal {} at depth {}, parent: {} at pos: {}", c.trans, *pos.node, - *left, - *right); - c.cache.retire_extent(c.trans, pos.node); + split_from, + *parent_pos.node, + parent_pos.pos); + auto [left, right] = co_await split_level(parent_pos, pos); - get_tree_stats(c.trans).extents_num_delta++; - return std::make_pair(left, right); - }; - - for (; split_from > 0; --split_from) { - auto &parent_pos = iter.get_internal(split_from + 1); - if (!parent_pos.node->is_mutable()) { - parent_pos.node = c.cache.duplicate_for_write( - c.trans, parent_pos.node - )->template cast(); - } - - if (split_from > 1) { - auto &pos = iter.get_internal(split_from); - SUBTRACET( - seastore_fixedkv_tree, - "splitting internal {} at depth {}, parent: {} at pos: {}", - c.trans, - *pos.node, - split_from, - *parent_pos.node, - parent_pos.pos); - auto [left, right] = split_level(parent_pos, pos); - - if (pos.pos < left->get_size()) { - pos.node = left; - } else { - pos.node = right; - pos.pos -= left->get_size(); - - parent_pos.pos += 1; - } + if (pos.pos < left->get_size()) { + pos.node = left; } else { - auto &pos = iter.leaf; - SUBTRACET( - seastore_fixedkv_tree, - "splitting leaf {}, parent: {} at pos: {}", - c.trans, - *pos.node, - *parent_pos.node, - parent_pos.pos); - auto [left, right] = split_level(parent_pos, pos); + pos.node = right; + pos.pos -= left->get_size(); - /* right->get_node_meta().begin == pivot == right->begin()->get_key() - * Thus, if pos.pos == left->get_size(), we want iter to point to - * left with pos.pos at the end rather than right with pos.pos = 0 - * since the insertion would be to the left of the first element - * of right and thus necessarily less than right->get_node_meta().begin. - */ - if (pos.pos <= left->get_size()) { - pos.node = left; - } else { - pos.node = right; - pos.pos -= left->get_size(); + parent_pos.pos += 1; + } + } else { + auto &pos = iter.leaf; + SUBTRACET( + seastore_fixedkv_tree, + "splitting leaf {}, parent: {} at pos: {}", + c.trans, + *pos.node, + *parent_pos.node, + parent_pos.pos); + auto [left, right] = co_await split_level(parent_pos, pos); - parent_pos.pos += 1; - } + /* right->get_node_meta().begin == pivot == right->begin()->get_key() + * Thus, if pos.pos == left->get_size(), we want iter to point to + * left with pos.pos at the end rather than right with pos.pos = 0 + * since the insertion would be to the left of the first element + * of right and thus necessarily less than right->get_node_meta().begin. + */ + if (pos.pos <= left->get_size()) { + pos.node = left; + } else { + pos.node = right; + pos.pos -= left->get_size(); + + parent_pos.pos += 1; } } - - return seastar::now(); - }); + } } @@ -2896,7 +2943,7 @@ private: SUBTRACET(seastore_fixedkv_tree, "parent: {}, node: {}", c.trans, *parent_pos.node, *pos.node); auto do_merge = [c, iter, donor_iter, donor_is_left, &parent_pos, &pos]( - typename NodeType::Ref donor) { + typename NodeType::Ref donor) -> handle_merge_ret { LOG_PREFIX(FixedKVBtree::merge_level); auto [l, r] = donor_is_left ? std::make_pair(donor, pos.node) : std::make_pair(pos.node, donor); @@ -2905,7 +2952,17 @@ private: std::make_pair(donor_iter, iter) : std::make_pair(iter, donor_iter); if (donor->at_min_capacity()) { - auto replacement = l->make_full_merge(c, r); + typename NodeType::Ref replacement; + while (!replacement) { + try { + replacement = l->make_full_merge(c, r); + } catch (crimson::ct_error::eagain&) {} + if (!replacement) { + c.cache.get_epm().maybe_wake_background(); + co_await trans_intr::make_interruptible( + c.cache.get_epm().wait_background()); + } + } parent_pos.node->update( liter, @@ -2926,11 +2983,23 @@ private: } else { auto pivot_idx = l->get_balance_pivot_idx(*l, *r); LOG_PREFIX(FixedKVBtree::merge_level); - auto [replacement_l, replacement_r, pivot] = - l->make_balanced( - c, - r, - pivot_idx); + typename NodeType::Ref replacement_l, replacement_r; + node_key_t pivot = min_max_t::null; + while (!replacement_l) { + assert(!replacement_r); + assert(pivot == min_max_t::null); + try { + auto t = l->make_balanced(c, r, pivot_idx); + replacement_l = std::get<0>(t); + replacement_r = std::get<1>(t); + pivot = std::get<2>(t); + } catch (crimson::ct_error::eagain&) {} + if (!replacement_l) { + c.cache.get_epm().maybe_wake_background(); + co_await trans_intr::make_interruptible( + c.cache.get_epm().wait_background()); + } + } parent_pos.node->update( liter, @@ -2966,8 +3035,6 @@ private: c.cache.retire_extent(c.trans, l); c.cache.retire_extent(c.trans, r); } - - return seastar::now(); }; auto v = parent_pos.node->template get_child( @@ -2975,30 +3042,26 @@ private: // checking the lba child must be atomic with creating // and linking the absent child if (v.has_child()) { - return std::move(v.get_child_fut() - ).si_then([do_merge=std::move(do_merge), &pos, - donor_iter, donor_is_left, c, parent_pos](auto child) { - LOG_PREFIX(FixedKVBtree::merge_level); - SUBTRACET(seastore_fixedkv_tree, - "got child on {}, pos: {}, res: {}", - c.trans, - *parent_pos.node, - donor_iter.get_offset(), - *child); - std::ignore = donor_is_left; - std::ignore = pos; - [[maybe_unused]] auto &node = (typename internal_node_t::base_t&)*child; - assert(donor_is_left ? - node.get_node_meta().end == pos.node->get_node_meta().begin : - node.get_node_meta().begin == pos.node->get_node_meta().end); - assert(node.get_node_meta().begin == donor_iter.get_key()); - assert(node.get_node_meta().end > donor_iter.get_key()); - return do_merge(child->template cast()); - }); + auto child = co_await std::move(v.get_child_fut()); + SUBTRACET(seastore_fixedkv_tree, + "got child on {}, pos: {}, res: {}", + c.trans, + *parent_pos.node, + donor_iter.get_offset(), + *child); + std::ignore = donor_is_left; + std::ignore = pos; + [[maybe_unused]] auto &node = (typename internal_node_t::base_t&)*child; + assert(donor_is_left ? + node.get_node_meta().end == pos.node->get_node_meta().begin : + node.get_node_meta().begin == pos.node->get_node_meta().end); + assert(node.get_node_meta().begin == donor_iter.get_key()); + assert(node.get_node_meta().end > donor_iter.get_key()); + co_return co_await do_merge(child->template cast()); } auto child_pos = v.get_child_pos(); - return get_node( + auto donor = co_await get_node( c, depth, donor_iter.get_val().maybe_relative_to(parent_pos.node->get_paddr()), @@ -3006,10 +3069,8 @@ private: end, std::make_optional>( child_pos.get_parent(), - child_pos.get_pos()) - ).si_then([do_merge=std::move(do_merge)](typename NodeType::Ref donor) { - return do_merge(donor); - }); + child_pos.get_pos())); + co_await do_merge(donor); } }; diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h index 8a47ab7104b..71e4f0bb296 100644 --- a/src/crimson/os/seastore/cache.h +++ b/src/crimson/os/seastore/cache.h @@ -742,6 +742,10 @@ public: } } + ExtentPlacementManager& get_epm() { + return epm; + } + extent_len_t get_block_size() const { return epm.get_block_size(); } @@ -1189,8 +1193,13 @@ public: t, T::TYPE, length, opt.hint, rewrite_gen_printer_t{opt.gen}); auto result = epm.alloc_new_non_data_extent(t, T::TYPE, length, opt); if (!result) { - SUBERRORT(seastore_cache, "insufficient space", t); - std::rethrow_exception(crimson::ct_error::enospc::exception_ptr()); + if (epm.is_full()) { + SUBERRORT(seastore_cache, "insufficient space", t); + std::rethrow_exception(crimson::ct_error::enospc::exception_ptr()); + } else { + SUBERRORT(seastore_cache, "insufficient space, wait for demoting", t); + std::rethrow_exception(crimson::ct_error::eagain::exception_ptr()); + } } auto ret = CachedExtent::make_cached_extent_ref(std::move(result->bp)); assert(is_rewrite_generation( @@ -1226,8 +1235,13 @@ public: t, T::TYPE, length, opt.hint, rewrite_gen_printer_t{opt.gen}); auto results = epm.alloc_new_data_extents(t, T::TYPE, length, opt); if (results.empty()) { - SUBERRORT(seastore_cache, "insufficient space", t); - std::rethrow_exception(crimson::ct_error::enospc::exception_ptr()); + if (epm.is_full()) { + SUBERRORT(seastore_cache, "insufficient space", t); + std::rethrow_exception(crimson::ct_error::enospc::exception_ptr()); + } else { + SUBERRORT(seastore_cache, "insufficient space, wait for demoting", t); + std::rethrow_exception(crimson::ct_error::eagain::exception_ptr()); + } } std::vector> extents; for (auto &result : results) { diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc index 7ece18312ab..9ed94046036 100644 --- a/src/crimson/os/seastore/extent_placement_manager.cc +++ b/src/crimson/os/seastore/extent_placement_manager.cc @@ -794,7 +794,6 @@ ExtentPlacementManager::BackgroundProcess::reserve_projected_usage( if (!is_ready()) { co_return; } - ceph_assert(!blocking_io); // The pipeline configuration prevents another IO from entering // prepare until the prior one exits and clears this. ++stats.io_count; @@ -824,15 +823,16 @@ ExtentPlacementManager::BackgroundProcess::reserve_projected_usage( // IO blocked -> needs cleaner -> cleaner sleeping -> nothing runs -> deadlock. // Kick the background so it can free space and call maybe_wake_blocked_io(). auto arm_blocking_io_and_wake = [this] { - blocking_io = seastar::promise<>(); + if (!blocking_io) { + blocking_io = seastar::shared_promise<>(); + } do_wake_background(); }; arm_blocking_io_and_wake(); // we just blocked this IO, now wait until // maybe_wake_blocked_io will set value to blocking_io - do { - co_await blocking_io->get_future(); - ceph_assert(!blocking_io); + while (true) { + co_await blocking_io->get_shared_future(); auto res = try_reserve_io(usage); if (res.is_successful()) { DEBUG("unblocked"); @@ -842,6 +842,7 @@ ExtentPlacementManager::BackgroundProcess::reserve_projected_usage( auto duration = end_time - begin_time; stats.io_blocked_time += std::chrono::duration_cast< std::chrono::milliseconds>(duration).count(); + break; } else { DEBUG("blocked again: inline={}, main={}, cold={}, usage={}", res.reserve_inline_success, @@ -857,7 +858,7 @@ ExtentPlacementManager::BackgroundProcess::reserve_projected_usage( } arm_blocking_io_and_wake(); } - } while (blocking_io); + } } } diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index 8bd25d67219..d1eb7606971 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -396,6 +396,18 @@ public: void set_primary_device(Device *device); + bool is_full() const { + return background_process.is_full(); + } + + void maybe_wake_background() { + background_process.maybe_wake_background(); + } + + seastar::future<> wait_background() { + return background_process.wait_background(); + } + void set_extent_callback(ExtentCallbackInterface *cb) { background_process.set_extent_callback(cb); } @@ -1086,9 +1098,11 @@ private: return !trimmer || !main_cleaner; } - protected: - state_t get_state() const final { - return state; + bool is_full() const { + if (has_cold_tier()) { + return cold_cleaner->get_alive_ratio() >= 0.99; + } + return main_cleaner->get_alive_ratio() >= 0.99; } void maybe_wake_background() final { @@ -1100,6 +1114,18 @@ private: } } + seastar::future<> wait_background() { + if (!blocking_io) { + blocking_io = seastar::shared_promise<>(); + } + return blocking_io->get_shared_future(); + } + + protected: + state_t get_state() const final { + return state; + } + void maybe_wake_blocked_io() final; void maybe_wake_promote() final { @@ -1348,7 +1374,7 @@ private: std::optional> process_join; std::optional> blocking_background; - std::optional> blocking_io; + std::optional> blocking_io; // Set by maybe_wake_blocked_io() whenever it actually unblocks a // user IO; consumed by run() to yield exactly once on that edge, // giving the woken continuation a chance to retry the reservation diff --git a/src/crimson/os/seastore/lba/btree_lba_manager.cc b/src/crimson/os/seastore/lba/btree_lba_manager.cc index 6120eeb3749..62b2622f695 100644 --- a/src/crimson/os/seastore/lba/btree_lba_manager.cc +++ b/src/crimson/os/seastore/lba/btree_lba_manager.cc @@ -173,7 +173,12 @@ BtreeLBAManager::mkfs( auto croot = co_await cache.get_root(t); assert(croot); assert(croot->is_mutation_pending()); - croot->get_root().lba_root = LBABtree::mkfs(croot, get_context(t)); + croot->get_root().lba_root = + co_await LBABtree::mkfs(croot, get_context(t) + ).handle_error_interruptible( + mkfs_iertr::pass_further{}, + crimson::ct_error::assert_all("unexpected error") + ); } /** diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index 831ef725eab..aa769a1d0b6 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -885,15 +885,24 @@ TransactionManager::rewrite_logical_extent( if (get_extent_category(extent->get_type()) == data_category_t::METADATA) { assert(extent->is_fully_loaded()); cache->retire_extent(t, extent); - auto nextent = cache->alloc_new_non_data_extent_by_type( - t, - extent->get_type(), - extent->get_length(), - extent->get_user_hint(), - // get target rewrite generation - extent->get_rewrite_generation(), - paddr_hint, - is_tracked)->cast(); + LogicalChildNodeRef nextent; + while (!nextent) { + try { + nextent = cache->alloc_new_non_data_extent_by_type( + t, + extent->get_type(), + extent->get_length(), + extent->get_user_hint(), + // get target rewrite generation + extent->get_rewrite_generation(), + paddr_hint, + is_tracked)->cast(); + } catch (crimson::ct_error::eagain&) {} + if (!nextent) { + epm->maybe_wake_background(); + co_await trans_intr::make_interruptible(epm->wait_background()); + } + } assert(nextent->get_write_policy() != write_policy_t::WRITE_THROUGH); nextent->rewrite(t, *extent, 0); @@ -933,20 +942,29 @@ TransactionManager::rewrite_logical_extent( t, std::move(extent), 0, length); assert(extent->is_fully_loaded()); cache->retire_extent(t, extent); - auto extents = cache->alloc_new_data_extents_by_type( - t, - extent->get_type(), - extent->get_length(), - { - extent->get_user_hint(), - // get target rewrite generation - extent->get_rewrite_generation(), - is_tracked, - paddr_hint, - // WRITH_THROUGH is only effective for client io, so - // always set the write policy to WRITE_BACK here - write_policy_t::WRITE_BACK - }); + std::vector extents; + while (extents.empty()) { + try { + extents = cache->alloc_new_data_extents_by_type( + t, + extent->get_type(), + extent->get_length(), + { + extent->get_user_hint(), + // get target rewrite generation + extent->get_rewrite_generation(), + is_tracked, + paddr_hint, + // WRITH_THROUGH is only effective for client io, so + // always set the write policy to WRITE_BACK here + write_policy_t::WRITE_BACK + }); + } catch (crimson::ct_error::eagain&) {} + if (extents.empty()) { + epm->maybe_wake_background(); + co_await trans_intr::make_interruptible(epm->wait_background()); + } + } extent_len_t off = 0; auto left = extent->get_length(); extent_ref_count_t refcount = 0; @@ -1237,17 +1255,26 @@ TransactionManager::promote_extent( cache->retire_extent(t, extent); if (get_extent_category(extent->get_type()) == data_category_t::DATA) { - auto promoted_raw_extents = cache->alloc_new_data_extents_by_type( - t, - orig_ext->get_type(), - orig_ext->get_length(), - { - placement_hint_t::HOT, - INIT_GENERATION, - true, - P_ADDR_NULL, - write_policy_t::WRITE_BACK - }); + std::vector promoted_raw_extents; + while (promoted_raw_extents.empty()) { + try { + promoted_raw_extents = cache->alloc_new_data_extents_by_type( + t, + orig_ext->get_type(), + orig_ext->get_length(), + { + placement_hint_t::HOT, + INIT_GENERATION, + true, + P_ADDR_NULL, + write_policy_t::WRITE_BACK + }); + } catch (crimson::ct_error::eagain&) {} + if (promoted_raw_extents.empty()) { + epm->maybe_wake_background(); + co_await trans_intr::make_interruptible(epm->wait_background()); + } + } t.touch_laddr_prefix(orig_ext->get_laddr().get_object_prefix()); promoted_extents.reserve(promoted_raw_extents.size()); @@ -1287,14 +1314,23 @@ TransactionManager::promote_extent( } ceph_assert(offset == orig_length); } else { - auto promoted_extent = cache->alloc_new_non_data_extent_by_type( - t, - orig_ext->get_type(), - orig_ext->get_length(), - placement_hint_t::HOT, - INIT_GENERATION, - P_ADDR_NULL, - true); + CachedExtentRef promoted_extent; + while (!promoted_extent) { + try { + promoted_extent = cache->alloc_new_non_data_extent_by_type( + t, + orig_ext->get_type(), + orig_ext->get_length(), + placement_hint_t::HOT, + INIT_GENERATION, + P_ADDR_NULL, + true); + } catch (crimson::ct_error::eagain&) {} + if (!promoted_extent) { + epm->maybe_wake_background(); + co_await trans_intr::make_interruptible(epm->wait_background()); + } + } auto lext = promoted_extent->cast(); lext->set_laddr(orig_ext->get_laddr()); lext->rewrite(t, *orig_ext, 0); diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index f1e350afdde..2eeaf13736d 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -574,21 +574,27 @@ public: LOG_PREFIX(TransactionManager::alloc_non_data_extent); SUBDEBUGT(seastore_tm, "{} hint {}~0x{:x} phint={} ...", t, T::TYPE, laddr_hint, len, placement_hint); - auto ext = cache->alloc_new_non_data_extent( - t, len, {placement_hint, INIT_GENERATION}); + TCachedExtentRef ext; + while (!ext) { + try { + ext = cache->alloc_new_non_data_extent( + t, len, {placement_hint, INIT_GENERATION}); + } catch(crimson::ct_error::eagain&) {} + if (!ext) { + epm->maybe_wake_background(); + co_await trans_intr::make_interruptible(epm->wait_background()); + } + } // user must initialize the logical extent themselves. assert(is_user_transaction(t.get_src())); ext->set_seen_by_users(); - return lba_manager->alloc_extent( + co_await lba_manager->alloc_extent( t, laddr_hint, *ext, - EXTENT_DEFAULT_REF_COUNT - ).si_then([ext=std::move(ext), &t, FNAME](auto &&) mutable { - SUBDEBUGT(seastore_tm, "allocated {}", t, *ext); - return alloc_extent_iertr::make_ready_future>( - std::move(ext)); - }); + EXTENT_DEFAULT_REF_COUNT); + SUBDEBUGT(seastore_tm, "allocated {}", t, *ext); + co_return ext; } /** @@ -612,12 +618,21 @@ public: LOG_PREFIX(TransactionManager::alloc_data_extents); SUBDEBUGT(seastore_tm, "{} hint {}~0x{:x} phint={} ...", t, T::TYPE, laddr_hint, len, placement_hint); - auto exts = cache->alloc_new_data_extents( - t, len, - { - placement_hint, INIT_GENERATION, false, P_ADDR_NULL, - epm->get_write_policy(T::TYPE, len) - }); + std::vector> exts; + while (exts.empty()) { + try { + exts = cache->alloc_new_data_extents( + t, len, + { + placement_hint, INIT_GENERATION, false, P_ADDR_NULL, + epm->get_write_policy(T::TYPE, len) + }); + } catch (crimson::ct_error::eagain&) {} + if (exts.empty()) { + epm->maybe_wake_background(); + co_await trans_intr::make_interruptible(epm->wait_background()); + } + } // user must initialize the logical extent themselves assert(is_user_transaction(t.get_src())); for (auto& ext : exts) { diff --git a/src/test/crimson/seastore/test_btree_lba_manager.cc b/src/test/crimson/seastore/test_btree_lba_manager.cc index c37758fc5d6..79d54b9c4fe 100644 --- a/src/test/crimson/seastore/test_btree_lba_manager.cc +++ b/src/test/crimson/seastore/test_btree_lba_manager.cc @@ -231,12 +231,12 @@ struct lba_btree_test : btree_test_base { LBAManager::mkfs_ret test_structure_setup(Transaction &t) final { return cache->get_root( t - ).si_then([this, &t](RootBlockRef croot) { + ).si_then([this, &t](RootBlockRef croot) -> LBAManager::mkfs_ret { auto mut_croot = cache->duplicate_for_write( t, croot )->cast(); mut_croot->root.lba_root = - LBABtree::mkfs(mut_croot, get_op_context(t)); + co_await LBABtree::mkfs(mut_croot, get_op_context(t)); }); } From e3c1baf8c8c47d0186888c732eed399e7284756f Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Mon, 18 May 2026 17:55:35 +0800 Subject: [PATCH 30/51] crimson/os/seastore: demote/promote background processes are also rewrite transactions So we need to merge the extents promoted/demoted with their counterparts in client transactions, just like trim/cleaner transaction do. The difference between demote/promote transactions and trim/cleaner transactions is that the former also involves remapping extents while the latter only involves rewriting and mutating extents. With the logical bucket cache in place, all rewrite transactions need to handle shadow extents carafully, too. Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/cache.cc | 50 ++++- src/crimson/os/seastore/cache.h | 3 + src/crimson/os/seastore/cached_extent.cc | 44 +++++ src/crimson/os/seastore/cached_extent.h | 2 + .../os/seastore/lba/btree_lba_manager.cc | 15 +- src/crimson/os/seastore/lba/lba_btree_node.h | 19 +- src/crimson/os/seastore/logical_child_node.h | 16 ++ src/crimson/os/seastore/seastore_types.h | 4 +- src/crimson/os/seastore/transaction.h | 33 ++++ .../os/seastore/transaction_manager.cc | 171 ++++++++++++------ src/crimson/os/seastore/transaction_manager.h | 35 +++- 11 files changed, 323 insertions(+), 69 deletions(-) diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc index 5e7e4586219..7d75f8012d6 100644 --- a/src/crimson/os/seastore/cache.cc +++ b/src/crimson/os/seastore/cache.cc @@ -1817,7 +1817,7 @@ record_t Cache::prepare_record( if (i->is_exist_clean()) { assert(i->version == 0); - assert(!i->prior_instance); + assert(!i->prior_instance || t.get_src() == transaction_type_t::DEMOTE); // no set_io_wait(), skip complete_commit() assert(!i->is_pending_io()); i->pending_for_transaction = TRANS_ID_NULL; @@ -1828,13 +1828,22 @@ record_t Cache::prepare_record( should_use_no_conflict_publish(t, i->get_type())); } - // exist mutation pending extents must be in t.mutated_block_list - add_extent(i); - const auto t_src = t.get_src(); - if (i->is_stable_dirty()) { - add_to_dirty(i, &t_src); + assert(i->is_logical()); + if (t.get_src() == transaction_type_t::DEMOTE) { + assert(!i->committer); + assert(!i->get_prior_instance()->committer); + i->new_committer(t); + assert(i->committer); + i->get_prior_instance()->committer = i->committer; } else { - touch_extent_fully(*i, &t_src, t.get_cache_hint()); + // exist mutation pending extents must be in t.mutated_block_list + add_extent(i); + const auto t_src = t.get_src(); + if (i->is_stable_dirty()) { + add_to_dirty(i, &t_src); + } else { + touch_extent_fully(*i, &t_src, t.get_cache_hint()); + } } alloc_delta.alloc_blk_ranges.emplace_back( @@ -2110,6 +2119,9 @@ void Cache::complete_commit( } if (i->is_logical()) { committer.maybe_sync_copied_lba_key(); + if (t.get_src() == transaction_type_t::PROMOTE) { + committer.commit_shadow_promote(t); + } } touch_extent_fully(prior, &t_src, t.get_cache_hint()); committer.sync_version(); @@ -2218,6 +2230,30 @@ void Cache::complete_commit( continue; } epm.mark_space_used(i->get_paddr(), i->get_length()); + assert(i->is_logical()); + auto t_src = t.get_src(); + if (t.get_src() == transaction_type_t::DEMOTE) { + assert(i->committer); + auto &committer = *i->committer; + auto &prior = static_cast( + *i->get_prior_instance()); + ceph_assert(prior.is_valid()); + TRACET("committing rewritten extent into " + "existing -- {}, prior={}", + t, *i, prior); + prior.pending_for_transaction = TRANS_ID_NULL; + if (auto shadow = prior.get_shadow(); shadow) { + committer.commit_shadow_demote(t); + prior.reset_shadow(); + } + committer.commit_state(); + committer.sync_checksum(); + committer.commit_and_share_paddr(); + touch_extent_fully(prior, &t_src, t.get_cache_hint()); + committer.sync_version(); + i->committer.reset(); + prior.committer.reset(); + } } for (auto &i: t.pre_alloc_list) { if (!i->is_valid()) { diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h index 71e4f0bb296..a2cf356f66f 100644 --- a/src/crimson/os/seastore/cache.h +++ b/src/crimson/os/seastore/cache.h @@ -1729,6 +1729,9 @@ public: read_extent_futs, [](auto &fut) { return std::move(fut); }); } + bool is_on_cold_tier(paddr_t paddr) const { + return epm.is_cold_device(paddr.get_device_id()); + } private: void touch_extent_fully( CachedExtent &ext, diff --git a/src/crimson/os/seastore/cached_extent.cc b/src/crimson/os/seastore/cached_extent.cc index 70e9e101035..9babb8264ec 100644 --- a/src/crimson/os/seastore/cached_extent.cc +++ b/src/crimson/os/seastore/cached_extent.cc @@ -537,4 +537,48 @@ void CachedExtent::new_committer(Transaction &t) { prior_instance->committer = committer; } +void ExtentCommitter::commit_shadow_demote(Transaction &t) { + LOG_PREFIX(ExtentCommitter::commit_shadow_demote); + assert(t.get_src() == transaction_type_t::DEMOTE); + auto &prior = *extent.prior_instance->template cast(); + auto shadow = prior.get_shadow(); + assert(shadow); + for (auto &trans_view : prior.retired_transactions) { + assert(trans_view.t != nullptr); + auto view_tid = trans_view.t->get_trans_id(); + if (view_tid == t.get_trans_id()) { + continue; + } + TRACET("removing shadow {} from retired_set of t.{}", t, *shadow, view_tid); + [[maybe_unused]] bool removed = + trans_view.t->remove_from_retired_set(*shadow); + assert(removed); + trans_view.t->remove_shadow_from_write_set( + shadow->get_paddr(), shadow->get_length()); + } +} + +void ExtentCommitter::commit_shadow_promote(Transaction &t) { + LOG_PREFIX(ExtentCommitter::commit_shadow_promote); + assert(t.get_src() == transaction_type_t::PROMOTE); + assert(extent.is_logical()); + auto &lprior = static_cast(*extent.prior_instance); + ceph_assert(lprior.get_pin_state() == extent_pin_state_t::Promoting); + auto &lext = static_cast(extent); + auto shadow = lext.get_shadow(); + assert(shadow); + assert(shadow->is_shadow_extent()); + lprior.set_shadow(shadow); + for (auto &trans_view : lprior.retired_transactions) { + assert(trans_view.t != nullptr); + auto view_tid = trans_view.t->get_trans_id(); + if (view_tid == t.get_trans_id()) { + continue; + } + TRACET("adding shadow {} from t.{}", t, *shadow, view_tid); + trans_view.t->add_absent_to_retired_set(shadow); + } + lprior.set_pin_state(extent_pin_state_t::Fresh); +} + } diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h index d40e9e632bf..9b8900b9b99 100644 --- a/src/crimson/os/seastore/cached_extent.h +++ b/src/crimson/os/seastore/cached_extent.h @@ -299,6 +299,8 @@ public: void commit_and_share_paddr(); void maybe_sync_copied_lba_key(); + void commit_shadow_demote(Transaction&); + void commit_shadow_promote(Transaction&); private: // the rewritten extent CachedExtent &extent; diff --git a/src/crimson/os/seastore/lba/btree_lba_manager.cc b/src/crimson/os/seastore/lba/btree_lba_manager.cc index 62b2622f695..dbdcfb2109c 100644 --- a/src/crimson/os/seastore/lba/btree_lba_manager.cc +++ b/src/crimson/os/seastore/lba/btree_lba_manager.cc @@ -1132,13 +1132,22 @@ BtreeLBAManager::update_mappings( return this->_update_mapping( c.trans, *cursor, - [prev_addr, addr, len, checksum]( + [prev_addr, addr, len, checksum, extent, c]( const lba_map_val_t &in) { lba_map_val_t ret = in; ceph_assert(in.pladdr.is_paddr()); - ceph_assert(in.pladdr.get_paddr() == prev_addr); ceph_assert(in.len == len); - ret.pladdr = addr; + if (likely(in.pladdr.get_paddr() == prev_addr)) { + ret.pladdr = addr; + } else { + // this can only happen when the extent is EXIST_CLEAN + // and is demoted onto the cold tier by a DEMOTE trans. + assert(in.shadow_paddr == P_ADDR_NULL); + assert(extent->is_exist_clean()); + assert(extent->get_paddr() == in.pladdr.get_paddr()); + assert(c.cache.is_on_cold_tier(extent->get_paddr())); + assert(!c.cache.is_on_cold_tier(prev_addr)); + } ret.checksum = checksum; return ret; }, diff --git a/src/crimson/os/seastore/lba/lba_btree_node.h b/src/crimson/os/seastore/lba/lba_btree_node.h index 53eb51c9d05..62a728296cc 100644 --- a/src/crimson/os/seastore/lba/lba_btree_node.h +++ b/src/crimson/os/seastore/lba/lba_btree_node.h @@ -324,6 +324,7 @@ struct LBALeafNode iterator &iter) { LOG_PREFIX(LBALeafNode::merge_content_to); + SUBTRACET(seastore_lba, "merging with {}", t, pending_version); std::map modified; auto it = pending_version.begin(); while (it != pending_version.end() && iter != this->end()) { @@ -345,14 +346,20 @@ struct LBALeafNode ceph_abort(); } if (is_valid_child_ptr(child)) { - if ((child->_is_mutable() || child->_is_pending_io())) { - // skip the ones that the pending version is also modifying + if (// skip the ones that the pending version is also modifying + (child->_is_mutable() || child->_is_pending_io()) || + // EXIST_CLEAN extents created by DEMOTE transactions also + // updates their paddrs, so they should also be skpped. + (pending_version.t->get_src() == transaction_type_t::DEMOTE)) { + SUBTRACET(seastore_lba, "skipping {}~{}", t, it->get_key(), it->get_val()); it++; continue; } else { assert(child->_is_exist_clean() || child->_is_exist_mutation_pending()); } } + SUBTRACET(seastore_lba, "examing v2: {}~{}, v1: {}~{}", + t, it->get_key(), it->get_val(), iter->get_key(), iter->get_val()); auto pending_key = it->get_key(); auto stable_key = iter->get_key(); auto stable_end = stable_key + v1.len; @@ -362,13 +369,21 @@ struct LBALeafNode if (pending_key != stable_key) { assert(v2.pladdr != v1.pladdr); assert(is_valid_child_ptr(child)); + assert(child->_is_exist_clean()); } if (v2.pladdr != v1.pladdr) { auto m_v2 = v2; auto off = pending_key.get_byte_distance(stable_key); auto paddr = v1.pladdr.get_paddr(); paddr = paddr + off; + SUBTRACET(seastore_lba, "merging to {}, paddr: {} -> {}", + t, pending_version, m_v2.pladdr, paddr); m_v2.pladdr = paddr; + if (v1.shadow_paddr == P_ADDR_NULL) { + m_v2.shadow_paddr = P_ADDR_NULL; + } else { + m_v2.shadow_paddr = (v1.shadow_paddr + off); + } SUBTRACET(seastore_lba, "merging to {}, paddr: {} -> {}", t, pending_version, m_v2.pladdr, paddr); if (!is_valid_child_ptr(child)) { diff --git a/src/crimson/os/seastore/logical_child_node.h b/src/crimson/os/seastore/logical_child_node.h index ea4bb9c6a76..977b96aaf21 100644 --- a/src/crimson/os/seastore/logical_child_node.h +++ b/src/crimson/os/seastore/logical_child_node.h @@ -44,6 +44,20 @@ public: laddr_t get_end() const { return (get_laddr() + get_length()).checked_to_laddr(); } + + TCachedExtentRef get_shadow() const { + return shadow; + } + + void set_shadow(TCachedExtentRef &s) { + assert(!shadow); + shadow = s; + } + + void reset_shadow() { + shadow.reset(); + } + protected: void on_replace_prior(Transaction &t) final { assert(is_seen_by_users()); @@ -56,6 +70,8 @@ protected: void on_data_commit() final { ceph_abort("impossible"); } +private: + TCachedExtentRef shadow; }; using LogicalChildNodeRef = TCachedExtentRef; } // namespace crimson::os::seastore diff --git a/src/crimson/os/seastore/seastore_types.h b/src/crimson/os/seastore/seastore_types.h index 4f02157111d..31c68b8cd47 100644 --- a/src/crimson/os/seastore/seastore_types.h +++ b/src/crimson/os/seastore/seastore_types.h @@ -2757,7 +2757,9 @@ constexpr bool is_background_transaction(transaction_type_t type) { constexpr bool is_rewrite_transaction(transaction_type_t type) { return type == transaction_type_t::TRIM_DIRTY || type == transaction_type_t::CLEANER_MAIN || - type == transaction_type_t::CLEANER_COLD; + type == transaction_type_t::CLEANER_COLD || + type == transaction_type_t::DEMOTE || + type == transaction_type_t::PROMOTE; } constexpr bool is_trim_transaction(transaction_type_t type) { diff --git a/src/crimson/os/seastore/transaction.h b/src/crimson/os/seastore/transaction.h index aee35094449..1a4a039ddb4 100644 --- a/src/crimson/os/seastore/transaction.h +++ b/src/crimson/os/seastore/transaction.h @@ -364,6 +364,21 @@ public: } } + bool remove_from_retired_set(CachedExtent &ext) { + auto it = retired_set.find(ext.get_paddr()); + if (it == retired_set.end()) { + return false; + } + auto &extent = it->extent; + if (extent->get_paddr() != ext.get_paddr()) { + return false; + } else { + assert(ext.get_length() == extent->get_length()); + retired_set.erase(it); + return true; + } + } + std::pair pre_stable_extent_paddr_mod( read_set_item_t &item) { @@ -432,6 +447,24 @@ public: write_set.insert(mextent); } } + void remove_shadow_from_write_set( + const paddr_t &shadow_paddr, extent_len_t len) { + std::vector exts; + for (auto [bottom, top] = write_set.get_overlap(shadow_paddr, len); + bottom != top; + bottom++) { + auto &mextent = *bottom; + if (mextent.is_initial_pending()) { + assert(!mextent.is_shadow_extent()); + continue; + } + assert(mextent.is_shadow_extent() && mextent.is_exist_clean()); + exts.emplace_back(&mextent); + } + for (auto i :exts) { + write_set.erase(*i); + } + } template auto for_each_finalized_fresh_block(F &&f) const { diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index aa769a1d0b6..c454cd1d9cc 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -259,13 +259,14 @@ TransactionManager::ref_ret TransactionManager::remove( auto laddr = ref->get_laddr(); cache->retire_absent_extent_addr_by_type( t, laddr, shadow_addr, length, ref->get_type(), - [laddr](auto &extent) { + [ref, laddr](auto &extent) { auto lextent = extent.template cast(); assert(extent.is_logical()); assert(!lextent->has_laddr()); assert(!extent.has_been_invalidated()); lextent->set_laddr(laddr); extent.set_shadow_extent(true); + ref->set_shadow(lextent); }); } } @@ -332,11 +333,32 @@ TransactionManager::_remove( LogicalChildNode >(); ceph_assert(extent); - cache->retire_extent(t, std::move(extent)); + cache->retire_extent(t, extent); + if (mapping.has_shadow_val()) { + if (auto shadow = extent->get_shadow(); shadow) { + cache->retire_extent(t, shadow); + } else { + auto laddr = mapping.get_intermediate_base(); + std::ignore = cache->retire_absent_extent_addr_by_type( + t, laddr, + mapping.get_shadow_val(), + mapping.get_intermediate_length(), + mapping.get_extent_type(), + [extent, laddr](auto &ext) { + auto lextent = ext.template cast(); + assert(ext.is_logical()); + assert(!lextent->has_laddr()); + assert(!ext.has_been_invalidated()); + lextent->set_laddr(laddr); + ext.set_shadow_extent(true); + extent->set_shadow(lextent); + }); + } + } } else { auto &child_pos = maybe_mapped_extent.get_child_pos(); auto laddr = mapping.get_intermediate_base(); - std::ignore = cache->retire_absent_extent_addr_by_type( + auto ext = cache->retire_absent_extent_addr_by_type( t, laddr, mapping.get_val(), mapping.get_intermediate_length(), @@ -349,13 +371,23 @@ TransactionManager::_remove( child_pos.link_child(lextent.get()); lextent->set_laddr(laddr); } - ); - } - if (mapping.has_shadow_val()) { - cache->retire_absent_extent_addr( - t, mapping.get_intermediate_base(), - mapping.get_shadow_val(), - mapping.get_intermediate_length()); + )->template cast(); + if (mapping.has_shadow_val()) { + std::ignore = cache->retire_absent_extent_addr_by_type( + t, mapping.get_intermediate_base(), + mapping.get_shadow_val(), + mapping.get_intermediate_length(), + mapping.get_extent_type(), + [laddr, ext](auto &extent) { + auto lextent = extent.template cast(); + assert(extent.is_logical()); + assert(!lextent->has_laddr()); + assert(!extent.has_been_invalidated()); + lextent->set_laddr(laddr); + extent.set_shadow_extent(true); + ext->set_shadow(lextent); + }); + } } } @@ -465,7 +497,7 @@ TransactionManager::refs_ret TransactionManager::remove( base_iertr::future TransactionManager::relocate_logical_extent( - Transaction &t, LBAMapping mapping) + Transaction &t, LBAMapping mapping, laddr_t new_laddr) { LOG_PREFIX(TransactionManager::relocate_logical_extent); SUBDEBUGT(seastore_tm, "relocate {}", t, mapping); @@ -476,7 +508,7 @@ TransactionManager::relocate_logical_extent( if (!v.has_child()) { auto &child_pos = v.get_child_pos(); auto laddr = mapping.get_key(); - std::ignore = cache->retire_absent_extent_addr_by_type( + auto extent = cache->retire_absent_extent_addr_by_type( t, laddr, mapping.get_val(), @@ -490,29 +522,53 @@ TransactionManager::relocate_logical_extent( child_pos.link_child(lextent.get()); lextent->set_laddr(laddr); } - ); - co_return cache->alloc_remapped_extent_by_type( - t, mapping.get_extent_type(), mapping.get_key(), - mapping.get_val(), 0, mapping.get_length(), std::nullopt - )->cast(); - } - - auto extent = co_await v.get_child_fut().si_then([](auto ext) { - return ext; - }); - - if (extent->is_stable()) { - cache->retire_extent(t, extent); - co_return cache->alloc_remapped_extent_by_type( - t, mapping.get_extent_type(), mapping.get_key(), - mapping.get_val(), 0, mapping.get_length(), std::nullopt )->cast(); + if (mapping.has_shadow_val()) { + std::ignore = cache->retire_absent_extent_addr_by_type( + t, laddr, + mapping.get_shadow_val(), + mapping.get_intermediate_length(), + mapping.get_extent_type(), + [extent, laddr](auto &ext) { + auto lextent = ext.template cast(); + assert(ext.is_logical()); + assert(!lextent->has_laddr()); + assert(!ext.has_been_invalidated()); + lextent->set_laddr(laddr); + ext.set_shadow_extent(true); + extent->set_shadow(lextent); + }); + } } else { - //TODO: relocating logical extents doesn't support - // mutation pending extents yet. - assert(extent->is_initial_pending() || extent->is_exist_clean()); - co_return extent; + auto extent = co_await v.get_child_fut_as(); + + if (extent->is_stable()) { + cache->retire_extent(t, extent); + } else { + //TODO: relocating logical extents doesn't support + // mutation pending extents yet. + assert(extent->is_initial_pending() || extent->is_exist_clean()); + extent->set_laddr(new_laddr); + if (mapping.has_shadow_val()) { + assert(extent->get_shadow()); + extent->get_shadow()->set_laddr(new_laddr); + } + co_return extent; + } } + auto remapped_extent = cache->alloc_remapped_extent_by_type( + t, mapping.get_extent_type(), new_laddr, + mapping.get_val(), 0, mapping.get_length(), std::nullopt + )->cast(); + if (mapping.has_shadow_val()) { + auto remapped_shadow = cache->alloc_remapped_extent_by_type( + t, mapping.get_extent_type(), new_laddr, + mapping.get_shadow_val(), 0, mapping.get_length(), std::nullopt + )->cast(); + remapped_shadow->set_shadow_extent(true); + remapped_extent->set_shadow(remapped_shadow); + } + co_return remapped_extent; } base_iertr::future @@ -525,8 +581,9 @@ TransactionManager::relocate_shadow_extent( assert(!mapping.is_zero_reserved()); assert(mapping.is_viewable()); assert(!mapping.is_indirect()); + assert(t.get_src() == transaction_type_t::DEMOTE); auto v = get_extent_if_linked(t, *mapping.direct_cursor); - CachedExtentRef extent; + LogicalChildNodeRef extent; auto laddr = mapping.get_intermediate_base(); if (!v.has_child()) { auto &child_pos = v.get_child_pos(); @@ -544,26 +601,35 @@ TransactionManager::relocate_shadow_extent( child_pos.link_child(lextent.get()); lextent->set_laddr(laddr); } - ); + )->template cast(); } else { - auto extent = co_await std::move(v.get_child_fut()); + extent = co_await std::move(v.get_child_fut()); cache->retire_extent(t, extent); } - auto shadow_paddr = mapping.get_shadow_val(); - std::ignore = cache->retire_absent_extent_addr_by_type( - t, laddr, shadow_paddr, mapping.get_length(), mapping.get_extent_type(), - [laddr](auto &ext) { - auto lextent = ext.template cast(); - assert(ext.is_logical()); - assert(!lextent->has_laddr()); - assert(!ext.has_been_invalidated()); - lextent->set_laddr(laddr); - } - ); - co_return cache->alloc_remapped_extent_by_type( + if (auto shadow = extent->get_shadow(); shadow) { + cache->retire_extent(t, shadow); + } else { + auto shadow_paddr = mapping.get_shadow_val(); + std::ignore = cache->retire_absent_extent_addr_by_type( + t, laddr, shadow_paddr, mapping.get_length(), mapping.get_extent_type(), + [laddr, extent](auto &ext) { + auto lextent = ext.template cast(); + assert(ext.is_logical()); + assert(!lextent->has_laddr()); + assert(!ext.has_been_invalidated()); + lextent->set_laddr(laddr); + ext.set_shadow_extent(true); + extent->set_shadow(lextent); + } + ); + } + auto nextent = cache->alloc_remapped_extent_by_type( t, mapping.get_extent_type(), laddr, mapping.get_shadow_val(), 0, mapping.get_length(), std::nullopt )->cast(); + nextent->set_prior_instance(extent); + nextent->set_last_committed_crc(extent->get_last_committed_crc()); + co_return nextent; } TransactionManager::submit_transaction_iertr::future<> @@ -1046,7 +1112,6 @@ TransactionManager::rewrite_extent_ret TransactionManager::rewrite_extent( rewrite_gen_printer_t{target_generation}, sea_time_point_printer_t{modify_time}, *extent); - ceph_assert(!extent->is_pending_io()); } assert(extent->is_valid() && !extent->is_initial_pending()); @@ -1200,9 +1265,9 @@ TransactionManager::move_region( dst = co_await dst.refresh(); } } else if (!src.is_zero_reserved()) { - auto extent = co_await relocate_logical_extent(t, src); auto laddr = calc_dst_key(); - extent->set_laddr(laddr); + auto extent = co_await relocate_logical_extent(t, src, laddr); + assert(extent->get_laddr() == laddr); auto ret = co_await lba_manager->move_direct_mapping( t, src.get_effective_cursor_ref(), laddr, dst.get_effective_cursor_ref(), *extent); @@ -1309,6 +1374,8 @@ TransactionManager::promote_extent( slice_length, std::nullopt); remapped_cold_extent->set_shadow_extent(true); + auto lremapped = remapped_cold_extent->template cast(); + lext->set_shadow(lremapped); offset += slice_length; } @@ -1352,8 +1419,8 @@ TransactionManager::promote_extent( orig_ext->get_length(), std::nullopt); remapped_cold_extent->set_shadow_extent(true); - - remapped_cold_extent->set_shadow_extent(true); + auto lremapped = remapped_cold_extent->template cast(); + lext->set_shadow(lremapped); } auto cursor = co_await lba_manager->get_cursor( diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index 2eeaf13736d..87799086e92 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -113,7 +113,8 @@ public: */ base_iertr::future relocate_logical_extent( Transaction &t, - LBAMapping mapping); + LBAMapping mapping, + laddr_t new_laddr); base_iertr::future relocate_shadow_extent( Transaction &t, @@ -1592,7 +1593,7 @@ private: SUBTRACET(seastore_tm, "retire extent place holder...", t); auto &child_pos = ret.get_child_pos(); auto laddr = pin.get_key(); - std::ignore = cache->retire_absent_extent_addr_by_type( + auto ext = cache->retire_absent_extent_addr_by_type( t, laddr, pin.get_val(), original_len, pin.get_extent_type(), [&child_pos, laddr](auto &extent) mutable { auto lextent = extent.template cast(); @@ -1602,17 +1603,19 @@ private: child_pos.link_child(lextent.get()); lextent->set_laddr(laddr); } - ); + )->template cast(); if (pin.has_shadow_val()) { cache->retire_absent_extent_addr_by_type( t, pin.get_key(), pin.get_shadow_val(), original_len, pin.get_extent_type(), - [laddr](auto &extent) { + [laddr, ext](auto &extent) { auto lextent = extent.template cast(); assert(extent.is_logical()); assert(!lextent->has_laddr()); assert(!extent.has_been_invalidated()); lextent->set_laddr(laddr); + extent.set_shadow_extent(true); + ext->set_shadow(lextent); } ); } @@ -1640,6 +1643,26 @@ private: SUBTRACET(seastore_tm, "retire extent...", t); assert(extent->is_seen_by_users()); cache->retire_extent(t, extent); + if (pin.has_shadow_val()) { + if (auto shadow = extent->get_shadow()) { + cache->retire_extent(t, shadow); + } else { + auto laddr = pin.get_key(); + cache->retire_absent_extent_addr_by_type( + t, laddr, pin.get_shadow_val(), + original_len, pin.get_extent_type(), + [laddr, extent](auto &ext) { + auto lextent = ext.template cast(); + assert(ext.is_logical()); + assert(!lextent->has_laddr()); + assert(!ext.has_been_invalidated()); + lextent->set_laddr(laddr); + ext.set_shadow_extent(true); + extent->set_shadow(lextent); + } + ); + } + } } for (auto &remap : remaps) { auto remap_offset = remap.offset; @@ -1675,6 +1698,10 @@ private: remap_len, std::nullopt); cold_ext->set_shadow_extent(true); + auto &lremapped = static_cast< + LogicalChildNode&>(*remapped_extent); + auto lcold_ext = cold_ext->template cast(); + lremapped.set_shadow(lcold_ext); } // user must initialize the logical extent themselves. remapped_extent->set_seen_by_users(); From a9220d9470c2dc4b14f5af2d54ec0bc346b2f4bc Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Thu, 21 May 2026 21:52:46 +0800 Subject: [PATCH 31/51] crimson/os/seastore/cached_extent: let pending cached extents hold pointers to the transactions This is necessary for the promote transaction to find the transactions that are retiring the extents promoted Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/btree/fixed_kv_node.h | 14 ++-- src/crimson/os/seastore/cache.cc | 20 +++--- src/crimson/os/seastore/cache.h | 10 +-- src/crimson/os/seastore/cached_extent.cc | 68 ++++++++++++++++++- src/crimson/os/seastore/cached_extent.h | 67 ++++-------------- src/crimson/os/seastore/lba/lba_btree_node.cc | 6 +- src/crimson/os/seastore/lba/lba_btree_node.h | 6 +- src/crimson/os/seastore/linked_tree_node.h | 20 ++++-- src/crimson/os/seastore/transaction.h | 6 +- 9 files changed, 127 insertions(+), 90 deletions(-) diff --git a/src/crimson/os/seastore/btree/fixed_kv_node.h b/src/crimson/os/seastore/btree/fixed_kv_node.h index d2e4c272b59..faa3f7fc2f5 100644 --- a/src/crimson/os/seastore/btree/fixed_kv_node.h +++ b/src/crimson/os/seastore/btree/fixed_kv_node.h @@ -85,7 +85,7 @@ struct FixedKVNode : CachedExtent { void on_delta_write(paddr_t record_block_offset) final { // All in-memory relative addrs are necessarily record-relative assert(get_prior_instance()); - assert(pending_for_transaction); + assert(t != nullptr); resolve_relative_addrs(record_block_offset); } @@ -264,8 +264,9 @@ struct FixedKVInternalNode paddr_t addr, base_child_t* nextent) { LOG_PREFIX(FixedKVInternalNode::update); + assert(this->t != nullptr); SUBTRACE(seastore_fixedkv_tree, "trans.{}, pos {}, {}", - this->pending_for_transaction, + this->t->get_trans_id(), iter.get_offset(), (void*)nextent); this->update_child_ptr(iter.get_offset(), nextent); @@ -281,8 +282,9 @@ struct FixedKVInternalNode paddr_t addr, base_child_t* nextent) { LOG_PREFIX(FixedKVInternalNode::insert); + assert(this->t != nullptr); SUBTRACE(seastore_fixedkv_tree, "trans.{}, pos {}, key {}, {}", - this->pending_for_transaction, + this->t->get_trans_id(), iter.get_offset(), pivot, (void*)nextent); @@ -296,8 +298,9 @@ struct FixedKVInternalNode void remove(internal_const_iterator_t iter) { LOG_PREFIX(FixedKVInternalNode::remove); + assert(this->t != nullptr); SUBTRACE(seastore_fixedkv_tree, "trans.{}, pos {}, key {}", - this->pending_for_transaction, + this->t->get_trans_id(), iter.get_offset(), iter.get_key()); this->remove_child_ptr(iter.get_offset()); @@ -312,8 +315,9 @@ struct FixedKVInternalNode paddr_t addr, base_child_t* nextent) { LOG_PREFIX(FixedKVInternalNode::replace); + assert(this->t != nullptr); SUBTRACE(seastore_fixedkv_tree, "trans.{}, pos {}, old key {}, key {}, {}", - this->pending_for_transaction, + this->t->get_trans_id(), iter.get_offset(), iter.get_key(), pivot, diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc index 7d75f8012d6..8fa3f69518c 100644 --- a/src/crimson/os/seastore/cache.cc +++ b/src/crimson/os/seastore/cache.cc @@ -1386,7 +1386,7 @@ CachedExtentRef Cache::duplicate_for_write( } auto ret = i->duplicate_for_write(t); - ret->pending_for_transaction = t.get_trans_id(); + ret->t = &t; ret->set_prior_instance(i); if (!is_root_type(ret->get_type())) { assert(ret->get_paddr().is_absolute()); @@ -1787,7 +1787,7 @@ record_t Cache::prepare_record( } assert(i->state == CachedExtent::extent_state_t::DIRTY); assert(i->version > 0); - assert(i->pending_for_transaction == TRANS_ID_NULL); + assert(i->t == nullptr); assert(!i->prior_instance); remove_from_dirty(i, &trans_src); // set the version to zero because the extent state is now clean @@ -1820,7 +1820,7 @@ record_t Cache::prepare_record( assert(!i->prior_instance || t.get_src() == transaction_type_t::DEMOTE); // no set_io_wait(), skip complete_commit() assert(!i->is_pending_io()); - i->pending_for_transaction = TRANS_ID_NULL; + i->t = nullptr; i->state = CachedExtent::extent_state_t::CLEAN; } else { assert(i->is_exist_mutation_pending()); @@ -2099,7 +2099,7 @@ void Cache::complete_commit( assert(i->get_last_committed_crc() == CRC_NULL); } #endif - i->pending_for_transaction = TRANS_ID_NULL; + i->t = nullptr; i->on_initial_write(); const auto t_src = t.get_src(); if (should_use_no_conflict_publish(t, i->get_type())) { @@ -2110,7 +2110,7 @@ void Cache::complete_commit( TRACET("committing rewritten extent into " "existing, inline={} -- {}, prior={}", t, is_inline, *i, prior); - prior.pending_for_transaction = TRANS_ID_NULL; + prior.t = nullptr; committer.commit_state(); committer.sync_checksum(); committer.commit_and_share_paddr(); @@ -2206,7 +2206,7 @@ void Cache::complete_commit( committer.commit_state(); committer.sync_checksum(); auto &prior = *i->prior_instance; - prior.pending_for_transaction = TRANS_ID_NULL; + prior.t = nullptr; ceph_assert(prior.is_valid()); if (is_lba_backref_node(i->get_type())) { committer.commit_data(); @@ -2218,7 +2218,7 @@ void Cache::complete_commit( prior.committer.reset(); } - i->pending_for_transaction = TRANS_ID_NULL; + i->t = nullptr; i->reset_prior_instance(); assert(i->version > 0); i->complete_io(); @@ -2241,7 +2241,7 @@ void Cache::complete_commit( TRACET("committing rewritten extent into " "existing -- {}, prior={}", t, *i, prior); - prior.pending_for_transaction = TRANS_ID_NULL; + prior.t = nullptr; if (auto shadow = prior.get_shadow(); shadow) { committer.commit_shadow_demote(t); prior.reset_shadow(); @@ -2297,7 +2297,7 @@ void Cache::init() P_ADDR_ROOT, PLACEMENT_HINT_NULL, NULL_GENERATION, - TRANS_ID_NULL, + nullptr, write_policy_t::WRITE_BACK); root->set_modify_time(seastar::lowres_system_clock::now()); INFO("init root -- {}", *root); @@ -2722,7 +2722,7 @@ Cache::_get_absent_extent_by_type( offset, PLACEMENT_HINT_NULL, NULL_GENERATION, - TRANS_ID_NULL, + nullptr, write_policy_t::WRITE_BACK); DEBUGT("{} length=0x{:x} is absent, add extent ... -- {}", t, type, length, *ret); diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h index a2cf356f66f..66487069924 100644 --- a/src/crimson/os/seastore/cache.h +++ b/src/crimson/os/seastore/cache.h @@ -866,7 +866,7 @@ private: offset, PLACEMENT_HINT_NULL, NULL_GENERATION, - TRANS_ID_NULL, + nullptr, write_policy_t::WRITE_BACK); SUBDEBUGT(seastore_cache, "{} {}~0x{:x} is absent, add extent and reading range 0x{:x}~0x{:x} ... -- {}", @@ -906,7 +906,7 @@ private: offset, PLACEMENT_HINT_NULL, NULL_GENERATION, - TRANS_ID_NULL, + nullptr, write_policy_t::WRITE_BACK); SUBDEBUG(seastore_cache, "{} {}~0x{:x} is absent, add extent and reading range 0x{:x}~0x{:x} ... -- {}", @@ -1209,7 +1209,7 @@ public: result->paddr, opt.hint, result->gen, - t.get_trans_id(), + &t, write_policy_t::WRITE_BACK); t.add_fresh_extent(ret); SUBDEBUGT(seastore_cache, @@ -1253,7 +1253,7 @@ public: result.paddr, opt.hint, result.gen, - t.get_trans_id(), + &t, opt.write_policy); t.add_fresh_extent(ret); SUBDEBUGT(seastore_cache, @@ -1296,7 +1296,7 @@ public: remap_paddr, PLACEMENT_HINT_NULL, NULL_GENERATION, - t.get_trans_id(), + &t, write_policy_t::WRITE_BACK); auto extent = ext->template cast(); diff --git a/src/crimson/os/seastore/cached_extent.cc b/src/crimson/os/seastore/cached_extent.cc index 9babb8264ec..a863d0a0341 100644 --- a/src/crimson/os/seastore/cached_extent.cc +++ b/src/crimson/os/seastore/cached_extent.cc @@ -89,6 +89,34 @@ std::ostream &operator<<(std::ostream &out, const CachedExtent &ext) return ext.print(out); } +trans_spec_view_t::trans_spec_view_t( + Transaction &t) : t(&t) {} + +bool trans_spec_view_t::cmp_t::operator()( + const trans_spec_view_t &lhs, + const trans_spec_view_t &rhs) const +{ + transaction_id_t l = ((lhs.t == nullptr) ? 0 : lhs.t->get_trans_id()); + transaction_id_t r = ((rhs.t == nullptr) ? 0 : rhs.t->get_trans_id()); + return l < r; +} + +bool trans_spec_view_t::cmp_t::operator()( + const transaction_id_t &lhs, + const trans_spec_view_t &rhs) const +{ + transaction_id_t r = ((rhs.t == nullptr) ? 0 : rhs.t->get_trans_id()); + return lhs < r; +} + +bool trans_spec_view_t::cmp_t::operator()( + const trans_spec_view_t &lhs, + const transaction_id_t &rhs) const +{ + transaction_id_t l = ((lhs.t == nullptr) ? 0 : lhs.t->get_trans_id()); + return l < rhs; +} + CachedExtent::~CachedExtent() { if (parent_index) { @@ -124,6 +152,44 @@ CachedExtent* CachedExtent::maybe_get_transactional_view(Transaction &t) { return this; } +bool CachedExtent::is_pending_in_trans(transaction_id_t id) const { + auto trans_id = ((t == nullptr) ? 0 : t->get_trans_id()); + return is_pending() && trans_id == id; +} + +std::ostream &CachedExtent::print(std::ostream &out) const { + std::string prior_poffset_str = prior_poffset + ? fmt::format("{}", *prior_poffset) + : "nullopt"; + out << "CachedExtent(addr=" << this + << ", type=" << get_type() + << ", trans=" << ((t == nullptr) ? 0 : t->get_trans_id()) + << ", version=" << version + << ", dirty_from=" << dirty_from + << ", modify_time=" << sea_time_point_printer_t{modify_time} + << ", paddr=" << get_paddr() + << ", prior_paddr=" << prior_poffset_str + << std::hex << ", length=0x" << get_length() + << ", loaded=0x" << get_loaded_length() << std::dec + << ", state=" << state + << ", pin_state=" << pin_state + << ", last_committed_crc=" << last_committed_crc + << ", refcount=" << use_count() + << ", user_hint=" << user_hint + << ", write_policy=" << write_policy + << ", rewrite_gen=" << rewrite_gen_printer_t{rewrite_generation} + << ", pending_io="; + if (is_pending_io()) { + out << io_wait->from_state; + } else { + out << "N/A"; + } + if (is_valid() && is_fully_loaded() && !is_stable_clean_pending()) { + print_detail(out); + } + return out << ")"; +} + std::ostream &LogicalCachedExtent::print_detail(std::ostream &out) const { out << ", laddr=" << laddr @@ -410,7 +476,7 @@ void ExtentCommitter::commit_state() { SUBTRACET(seastore_cache, "{} prior={}", t, extent, *extent.prior_instance); auto &prior = *extent.prior_instance; - prior.pending_for_transaction = extent.pending_for_transaction; + prior.t = extent.t; prior.modify_time = extent.modify_time; prior.last_committed_crc = extent.last_committed_crc; prior.dirty_from = extent.dirty_from; diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h index 9b8900b9b99..bb66e20428a 100644 --- a/src/crimson/os/seastore/cached_extent.h +++ b/src/crimson/os/seastore/cached_extent.h @@ -141,32 +141,22 @@ template using read_trans_set_t = typename read_set_item_t::trans_set_t; struct trans_spec_view_t { - // if the extent is pending, contains the id of the owning transaction; - // TRANS_ID_NULL otherwise - transaction_id_t pending_for_transaction = TRANS_ID_NULL; + // if the extent is pending, this points to the transaction + Transaction *t = nullptr; trans_spec_view_t() = default; - trans_spec_view_t(transaction_id_t id) : pending_for_transaction(id) {} + trans_spec_view_t(Transaction &t); virtual ~trans_spec_view_t() = default; struct cmp_t { bool operator()( const trans_spec_view_t &lhs, - const trans_spec_view_t &rhs) const - { - return lhs.pending_for_transaction < rhs.pending_for_transaction; - } + const trans_spec_view_t &rhs) const; bool operator()( const transaction_id_t &lhs, - const trans_spec_view_t &rhs) const - { - return lhs < rhs.pending_for_transaction; - } + const trans_spec_view_t &rhs) const; bool operator()( const trans_spec_view_t &lhs, - const transaction_id_t &rhs) const - { - return lhs.pending_for_transaction < rhs; - } + const transaction_id_t &rhs) const; }; using trans_view_hook_t = @@ -362,13 +352,13 @@ public: paddr_t paddr, placement_hint_t hint, rewrite_gen_t gen, - transaction_id_t trans_id, + Transaction *trans, write_policy_t policy) { state = _state; set_paddr(paddr); user_hint = hint; rewrite_generation = gen; - pending_for_transaction = trans_id; + t = trans; write_policy = policy; } @@ -522,38 +512,7 @@ public: friend std::ostream &operator<<(std::ostream &, extent_state_t); virtual std::ostream &print_detail(std::ostream &out) const { return out; } - std::ostream &print(std::ostream &out) const { - std::string prior_poffset_str = prior_poffset - ? fmt::format("{}", *prior_poffset) - : "nullopt"; - out << "CachedExtent(addr=" << this - << ", type=" << get_type() - << ", trans=" << pending_for_transaction - << ", version=" << version - << ", dirty_from=" << dirty_from - << ", modify_time=" << sea_time_point_printer_t{modify_time} - << ", paddr=" << get_paddr() - << ", prior_paddr=" << prior_poffset_str - << std::hex << ", length=0x" << get_length() - << ", loaded=0x" << get_loaded_length() << std::dec - << ", state=" << state - << ", pin_state=" << pin_state - << ", last_committed_crc=" << last_committed_crc - << ", refcount=" << use_count() - << ", user_hint=" << user_hint - << ", write_policy=" << write_policy - << ", rewrite_gen=" << rewrite_gen_printer_t{rewrite_generation} - << ", pending_io="; - if (is_pending_io()) { - out << io_wait->from_state; - } else { - out << "N/A"; - } - if (is_valid() && is_fully_loaded() && !is_stable_clean_pending()) { - print_detail(out); - } - return out << ")"; - } + std::ostream &print(std::ostream &out) const; /** * get_delta @@ -852,9 +811,7 @@ public: } /// Returns true if the extent part of the open transaction - bool is_pending_in_trans(transaction_id_t id) const { - return is_pending() && pending_for_transaction == id; - } + bool is_pending_in_trans(transaction_id_t id) const; enum class viewable_state_t { stable, // viewable @@ -1331,8 +1288,8 @@ struct trans_retired_extent_link_t { // Otherwise, we have to search through each extent's "retired_transactions" // to remove the transaction trans_spec_view_t trans_view; - trans_retired_extent_link_t(CachedExtentRef extent, transaction_id_t id) - : extent(extent), trans_view{id} + trans_retired_extent_link_t(CachedExtentRef extent, Transaction &t) + : extent(extent), trans_view{t} { assert(extent->is_stable()); extent->retired_transactions.insert(trans_view); diff --git a/src/crimson/os/seastore/lba/lba_btree_node.cc b/src/crimson/os/seastore/lba/lba_btree_node.cc index 1c97d078a11..c94f262c364 100644 --- a/src/crimson/os/seastore/lba/lba_btree_node.cc +++ b/src/crimson/os/seastore/lba/lba_btree_node.cc @@ -50,8 +50,9 @@ void LBALeafNode::update( modification_t mod) { LOG_PREFIX(LBALeafNode::update); + assert(this->t != nullptr); SUBTRACE(seastore_fixedkv_tree, "trans.{}, pos {}", - this->pending_for_transaction, + this->t->get_trans_id(), iter.get_offset()); if (likely(mod == modification_t::USER_MODIFY)) { this->on_modify(); @@ -71,8 +72,9 @@ LBALeafNode::internal_const_iterator_t LBALeafNode::insert( lba_map_val_t val) { LOG_PREFIX(LBALeafNode::insert); + assert(this->t != nullptr); SUBTRACE(seastore_fixedkv_tree, "trans.{}, pos {}, key {}", - this->pending_for_transaction, + this->t->get_trans_id(), iter.get_offset(), addr); this->on_modify(); diff --git a/src/crimson/os/seastore/lba/lba_btree_node.h b/src/crimson/os/seastore/lba/lba_btree_node.h index 62a728296cc..00239082f19 100644 --- a/src/crimson/os/seastore/lba/lba_btree_node.h +++ b/src/crimson/os/seastore/lba/lba_btree_node.h @@ -165,8 +165,9 @@ struct LBALeafNode laddr_t pivot, lba_map_val_t val) { LOG_PREFIX(FixedKVInternalNode::replace); + assert(this->t != nullptr); SUBTRACE(seastore_fixedkv_tree, "trans.{}, pos {}, old key {}, key {}", - this->pending_for_transaction, + this->t->get_trans_id(), iter.get_offset(), iter.get_key(), pivot); @@ -180,8 +181,9 @@ struct LBALeafNode void remove(internal_const_iterator_t iter) final { LOG_PREFIX(LBALeafNode::remove); + assert(this->t != nullptr); SUBTRACE(seastore_fixedkv_tree, "trans.{}, pos {}, key {}", - this->pending_for_transaction, + this->t->get_trans_id(), iter.get_offset(), iter.get_key()); assert(iter != this->end()); diff --git a/src/crimson/os/seastore/linked_tree_node.h b/src/crimson/os/seastore/linked_tree_node.h index 1b136c8a6a2..f6dc6f828db 100644 --- a/src/crimson/os/seastore/linked_tree_node.h +++ b/src/crimson/os/seastore/linked_tree_node.h @@ -117,7 +117,7 @@ protected: auto pi = me.get_prior_instance(); auto &prior = *pi->template cast(); ceph_assert(prior.parent_of_root); - ceph_assert(me.pending_for_transaction); + ceph_assert(me.t != nullptr); parent_of_root = prior.parent_of_root; TreeRootLinker::link_root(parent_of_root, &me); return; @@ -441,7 +441,7 @@ public: // see Transaction::views struct copy_dests_t : trans_spec_view_t { std::set, Comparator> dests_by_key; - copy_dests_t(Transaction &t) : trans_spec_view_t{t.get_trans_id()} {} + copy_dests_t(Transaction &t) : trans_spec_view_t(t) {} ~copy_dests_t() { LOG_PREFIX(~copy_dests_t); SUBTRACE(seastore_fixedkv_tree, "copy_dests_t destroyed"); @@ -480,8 +480,9 @@ protected: auto mut_iter = me.mutation_pending_extents.find( t.get_trans_id(), trans_spec_view_t::cmp_t()); assert(mut_iter != me.mutation_pending_extents.end()); - assert(copy_dests_by_trans.find(t.get_trans_id()) == - copy_dests_by_trans.end()); + assert(copy_dests_by_trans.find( + t.get_trans_id(), trans_spec_view_t::cmp_t()) == + copy_dests_by_trans.end()); return static_cast(&(*mut_iter)); } ceph_assert(hint == CachedExtent::viewable_state_t::stable_become_retired); @@ -501,7 +502,8 @@ protected: template void for_each_copy_dest_set(Transaction &t, Func &&f) { for (auto &dests : copy_dests_by_trans) { - if (dests.pending_for_transaction == t.get_trans_id()) { + assert(dests.t != nullptr); + if (dests.t->get_trans_id() == t.get_trans_id()) { continue; } auto ©_dests = static_cast(dests); @@ -515,8 +517,11 @@ protected: auto tid = t.get_trans_id(); auto iter = copy_dests_by_trans.lower_bound( tid, trans_spec_view_t::cmp_t()); + if (iter != copy_dests_by_trans.end()) { + assert(iter->t != nullptr); + } if (iter == copy_dests_by_trans.end() || - iter->pending_for_transaction != tid) { + iter->t->get_trans_id() != tid) { iter = copy_dests_by_trans.insert_before( iter, t.add_transactional_view(t)); } @@ -548,8 +553,9 @@ protected: auto &me = down_cast(); LOG_PREFIX(ParentNode::remove_child_ptr); auto raw_children = children.data(); + assert(me.t != nullptr); SUBTRACE(seastore_fixedkv_tree, "trans.{}, pos {}, total size {}, extent {}", - me.pending_for_transaction, + me.t->get_trans_id(), offset, me.get_size(), (void*)raw_children[offset]); diff --git a/src/crimson/os/seastore/transaction.h b/src/crimson/os/seastore/transaction.h index 1a4a039ddb4..8e907a692e9 100644 --- a/src/crimson/os/seastore/transaction.h +++ b/src/crimson/os/seastore/transaction.h @@ -162,7 +162,7 @@ public: ref->set_invalid(*this); write_set.erase(*ref); assert(ref->prior_instance); - retired_set.emplace(ref->prior_instance, trans_id); + retired_set.emplace(ref->prior_instance, *this); assert(read_set.count(ref->prior_instance->get_paddr(), extent_cmp_t{})); ref->reset_prior_instance(); } else { @@ -171,7 +171,7 @@ public: // XXX: prevent double retire -- retired_set.count(ref->get_paddr()) == 0 // If it's already in the set, insert here will be a noop, // which is what we want. - retired_set.emplace(ref, trans_id); + retired_set.emplace(ref, *this); } } @@ -408,7 +408,7 @@ public: bool retired) { read_set.insert(item); if (retired) { - retired_set.emplace(item.ref, trans_id); + retired_set.emplace(item.ref, *this); } } void maybe_update_pending_paddr( From a5693801a6cc6451b3f1f7fde95854933109c1bc Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Wed, 3 Jun 2026 17:36:42 +0800 Subject: [PATCH 32/51] crimson/os/seastore/lba,TM: remove shadows when trimming dirty non-data extents Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/async_cleaner.cc | 5 ++- src/crimson/os/seastore/async_cleaner.h | 12 +++++++ .../os/seastore/lba/btree_lba_manager.cc | 5 +++ src/crimson/os/seastore/transaction_manager.h | 36 +++++++++++++++++++ 4 files changed, 57 insertions(+), 1 deletion(-) diff --git a/src/crimson/os/seastore/async_cleaner.cc b/src/crimson/os/seastore/async_cleaner.cc index 9d515e13cd9..7ccf710d9e2 100644 --- a/src/crimson/os/seastore/async_cleaner.cc +++ b/src/crimson/os/seastore/async_cleaner.cc @@ -717,8 +717,11 @@ JournalTrimmerImpl::trim_dirty() return trans_intr::do_for_each( dirty_list, [this, &t](auto &e) { - return extent_callback->rewrite_extent( + return extent_callback->maybe_remove_shadow(t, *e + ).si_then([this, &t, e] { + return extent_callback->rewrite_extent( t, e, INIT_GENERATION, NULL_TIME); + }); }); }); }).si_then([this, &t] { diff --git a/src/crimson/os/seastore/async_cleaner.h b/src/crimson/os/seastore/async_cleaner.h index 647e0b233b3..9751dc15eea 100644 --- a/src/crimson/os/seastore/async_cleaner.h +++ b/src/crimson/os/seastore/async_cleaner.h @@ -398,6 +398,18 @@ public: laddr_t prefix, std::size_t max_proceed_size) = 0; + /** + * maybe_remove_shadow + * + * Remove the shadow of the extent if it exists + */ + using maybe_remove_shadow_iertr = base_iertr; + using maybe_remove_shadow_ret = + maybe_remove_shadow_iertr::future<>; + virtual maybe_remove_shadow_ret maybe_remove_shadow( + Transaction &t, + CachedExtent &e) = 0; + /** * get_extents_if_live * diff --git a/src/crimson/os/seastore/lba/btree_lba_manager.cc b/src/crimson/os/seastore/lba/btree_lba_manager.cc index dbdcfb2109c..10a38fa9d32 100644 --- a/src/crimson/os/seastore/lba/btree_lba_manager.cc +++ b/src/crimson/os/seastore/lba/btree_lba_manager.cc @@ -1139,6 +1139,11 @@ BtreeLBAManager::update_mappings( ceph_assert(in.len == len); if (likely(in.pladdr.get_paddr() == prev_addr)) { ret.pladdr = addr; + if (c.trans.get_src() == transaction_type_t::TRIM_DIRTY) { + // This is a dirty onode/omap extent that are rewritten, + // the shadow extent should be removed. + ret.shadow_paddr = P_ADDR_NULL; + } } else { // this can only happen when the extent is EXIST_CLEAN // and is demoted onto the cold tier by a DEMOTE trans. diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index 87799086e92..2ca10ed8101 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -993,6 +993,42 @@ public: laddr_t start, std::size_t max_proceed_size) final; + maybe_remove_shadow_ret maybe_remove_shadow( + Transaction &t, + CachedExtent &e) { + if (!e.is_logical()) { + co_return; + } + auto &extent = static_cast(e); + auto cursor = co_await lba_manager->get_cursor(t, extent + ).handle_error_interruptible( + crimson::ct_error::input_output_error::pass_further{}, + crimson::ct_error::enoent::assert_failure{} + ); + if (cursor->has_shadow_paddr()) { + if (auto shadow = extent.get_shadow(); shadow) { + cache->retire_extent(t, shadow); + } else { + auto laddr = cursor->get_laddr(); + std::ignore = cache->retire_absent_extent_addr_by_type( + t, laddr, + cursor->get_shadow_paddr(), + cursor->get_length(), + cursor->get_extent_type(), + [&extent, laddr](auto &ext) { + auto lextent = ext.template cast(); + assert(ext.is_logical()); + assert(!lextent->has_laddr()); + assert(!ext.has_been_invalidated()); + lextent->set_laddr(laddr); + ext.set_shadow_extent(true); + extent.set_shadow(lextent); + } + ); + } + } + } + using ExtentCallbackInterface::get_extents_if_live_ret; get_extents_if_live_ret get_extents_if_live( Transaction &t, From e141ac3f380f319ba76497a4906785d4a6e8d360 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Thu, 4 Jun 2026 13:54:01 +0800 Subject: [PATCH 33/51] crimson/os/seastore/extent_pinboard: don't add extents under promotion to cache This is to avoid an under-promotion extent from being added back to and evicted from the cache again before the promotion ends. Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/cached_extent.cc | 2 ++ src/crimson/os/seastore/cached_extent.h | 5 +++-- src/crimson/os/seastore/extent_pinboard.cc | 20 +++++++++++++------ .../os/seastore/transaction_manager.cc | 10 ++++++---- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/crimson/os/seastore/cached_extent.cc b/src/crimson/os/seastore/cached_extent.cc index a863d0a0341..f2efd08cb9a 100644 --- a/src/crimson/os/seastore/cached_extent.cc +++ b/src/crimson/os/seastore/cached_extent.cc @@ -72,6 +72,8 @@ std::ostream &operator<<(std::ostream &out, const extent_pin_state_t &s) { return out << "Fresh"; case extent_pin_state_t::PendingPromote: return out << "PendingPromote"; + case extent_pin_state_t::Promoting: + return out << "Promoting"; case extent_pin_state_t::WarmIn: return out << "WarmIn"; case extent_pin_state_t::Hot: diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h index bb66e20428a..2a528a2a259 100644 --- a/src/crimson/os/seastore/cached_extent.h +++ b/src/crimson/os/seastore/cached_extent.h @@ -260,6 +260,7 @@ enum class extent_pin_state_t : uint8_t { // shared state between LRU and 2Q impl Fresh = 0, PendingPromote, + Promoting, // 2Q impl only WarmIn, Hot, @@ -836,7 +837,7 @@ public: using crimson::common::get_conf; auto type = get_conf("seastore_cachepin_type"); if (type == "LRU") { - assert(pin_state <= extent_pin_state_t::PendingPromote); + assert(pin_state <= extent_pin_state_t::Promoting); } else if (type == "2Q") { assert(pin_state < extent_pin_state_t::Max); } else { @@ -852,7 +853,7 @@ public: using crimson::common::get_conf; auto type = get_conf("seastore_cachepin_type"); if (type == "LRU") { - assert(pin_state <= extent_pin_state_t::PendingPromote); + assert(pin_state <= extent_pin_state_t::Promoting); } else if (type == "2Q") { assert(pin_state < extent_pin_state_t::Max); } else { diff --git a/src/crimson/os/seastore/extent_pinboard.cc b/src/crimson/os/seastore/extent_pinboard.cc index e149d3b0bad..f9f856d33ec 100644 --- a/src/crimson/os/seastore/extent_pinboard.cc +++ b/src/crimson/os/seastore/extent_pinboard.cc @@ -386,7 +386,7 @@ public: extents.emplace_back(&extent); } for (auto &extent : extents) { - remove_extent(*extent, extent_pin_state_t::Fresh); + remove_extent(*extent, extent_pin_state_t::Promoting); } promoting_extents.insert( promoting_extents.end(), @@ -541,7 +541,8 @@ public: promoter.remove_extent(extent, extent_pin_state_t::Fresh); } } else { - ceph_assert(s == extent_pin_state_t::Fresh); + ceph_assert(s == extent_pin_state_t::Fresh || + s == extent_pin_state_t::Promoting); } } @@ -563,8 +564,10 @@ public: const Transaction::src_t* p_src, extent_len_t /*load_start*/, extent_len_t /*load_length*/) final { - if (extent.is_linked_to_list()) { - auto s = extent.get_pin_state(); + if (auto s = extent.get_pin_state(); + s == extent_pin_state_t::Promoting) { + assert(!extent.is_linked_to_list()); + } else if (extent.is_linked_to_list()) { assert(s <= extent_pin_state_t::PendingPromote); if (s == extent_pin_state_t::Fresh) { lru.move_to_top(extent, p_src); @@ -588,7 +591,7 @@ public: lru.increase_cached_size(extent, increased_length, p_src); } else { // promoter take the complete extent size for content size calculation - assert(extent.get_pin_state() <= extent_pin_state_t::PendingPromote); + assert(extent.get_pin_state() <= extent_pin_state_t::Promoting); } } @@ -782,7 +785,8 @@ public: } extent.set_pin_state(extent_pin_state_t::Fresh); } else { - ceph_assert(s == extent_pin_state_t::Fresh); + ceph_assert(s == extent_pin_state_t::Fresh || + s == extent_pin_state_t::Promoting); } } @@ -793,6 +797,10 @@ public: extent_len_t load_length) final { auto state = extent.get_pin_state(); auto type = extent.get_type(); + if (state == extent_pin_state_t::Promoting) { + assert(!extent.is_linked_to_list()); + return; + } if (extent.is_linked_to_list()) { if (state == extent_pin_state_t::Hot) { hot.move_to_top(extent, p_src); diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index c454cd1d9cc..09655e51357 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -1353,8 +1353,9 @@ TransactionManager::promote_extent( auto slice_laddr = (orig_laddr + offset).checked_to_laddr(); auto slice_length = extent->get_length(); extent->rewrite(t, *orig_ext, offset); - assert(!extent->get_paddr().is_absolute() || - !cache->is_on_cold_tier(extent->get_paddr())); + if (extent->get_paddr().is_absolute()) { + assert(!cache->is_on_cold_tier(extent->get_paddr())); + } auto lext = extent->cast(); lext->set_laddr(slice_laddr); @@ -1401,8 +1402,9 @@ TransactionManager::promote_extent( auto lext = promoted_extent->cast(); lext->set_laddr(orig_ext->get_laddr()); lext->rewrite(t, *orig_ext, 0); - assert(!extent->get_paddr().is_absolute() || - !cache->is_on_cold_tier(lext->get_paddr())); + if (lext->get_paddr().is_absolute()) { + assert(!cache->is_on_cold_tier(lext->get_paddr())); + } t.touch_laddr_prefix(orig_ext->get_laddr().get_object_prefix()); //TODO: this memory copy should be saved orig_ext->get_bptr().copy_out( From 6db2d97c74fbeae2fa76ec0861ba913463dfdc57 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Thu, 4 Jun 2026 13:59:36 +0800 Subject: [PATCH 34/51] crimson/os/seastore/lba: merge the whole mapping into the delta Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/cached_extent.cc | 2 +- src/crimson/os/seastore/lba/lba_btree_node.h | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/crimson/os/seastore/cached_extent.cc b/src/crimson/os/seastore/cached_extent.cc index f2efd08cb9a..cdf5378c42b 100644 --- a/src/crimson/os/seastore/cached_extent.cc +++ b/src/crimson/os/seastore/cached_extent.cc @@ -554,7 +554,7 @@ void ExtentCommitter::_share_prior_data_to_mutations() { if (it != merged.end()) { TRACE("{} -> {}, {} -> {}", me, mextent, (pladdr_t)buf.val.pladdr, it->second); - buf.val.pladdr = pladdr_le_t(it->second); + buf.val = lba::lba_map_val_le_t(it->second); } } }); diff --git a/src/crimson/os/seastore/lba/lba_btree_node.h b/src/crimson/os/seastore/lba/lba_btree_node.h index 00239082f19..f6dd325a8ba 100644 --- a/src/crimson/os/seastore/lba/lba_btree_node.h +++ b/src/crimson/os/seastore/lba/lba_btree_node.h @@ -320,14 +320,14 @@ struct LBALeafNode std::ostream &print_detail(std::ostream &out) const final; - std::map merge_content_to( + std::map merge_content_to( Transaction &t, LBALeafNode &pending_version, iterator &iter) { LOG_PREFIX(LBALeafNode::merge_content_to); SUBTRACET(seastore_lba, "merging with {}", t, pending_version); - std::map modified; + std::map modified; auto it = pending_version.begin(); while (it != pending_version.end() && iter != this->end()) { const auto &v1 = iter->get_val(); @@ -395,7 +395,7 @@ struct LBALeafNode m_v2.checksum = v1.checksum; } it->set_val(m_v2); - auto [_it, inserted] = modified.emplace(it->get_key(), paddr); + auto [_it, inserted] = modified.emplace(it->get_key(), m_v2); ceph_assert(inserted); } it++; From a4f338e77fa7ee22a2e1090b41d74fdec031fa38 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Thu, 4 Jun 2026 17:41:48 +0800 Subject: [PATCH 35/51] crimson/os/seastore: also update lba mappings' shadow fields when updating paddr synchronously Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/cached_extent.cc | 25 +++++++++++++++++-- .../os/seastore/lba/btree_lba_manager.cc | 23 ++++++++++++++--- .../os/seastore/lba/btree_lba_manager.h | 3 ++- src/crimson/os/seastore/transaction.h | 7 +++--- 4 files changed, 49 insertions(+), 9 deletions(-) diff --git a/src/crimson/os/seastore/cached_extent.cc b/src/crimson/os/seastore/cached_extent.cc index cdf5378c42b..5d22fef0b7c 100644 --- a/src/crimson/os/seastore/cached_extent.cc +++ b/src/crimson/os/seastore/cached_extent.cc @@ -504,8 +504,29 @@ void ExtentCommitter::maybe_sync_copied_lba_key() { auto &lextent = static_cast(extent); auto &prior = *extent.prior_instance; for (auto &item : prior.read_transactions) { - item.t->maybe_sync_copied_lba_key( - lextent.get_laddr(), lextent.get_paddr()); + switch (t.get_src()) { + case transaction_type_t::PROMOTE: + { + auto &shadow = *lextent.get_shadow(); + item.t->maybe_sync_copied_lba_key( + lextent.get_laddr(), + lextent.get_paddr(), + shadow.get_paddr()); + break; + } + case transaction_type_t::DEMOTE: + item.t->maybe_sync_copied_lba_key( + lextent.get_laddr(), + lextent.get_paddr(), + P_ADDR_NULL); + break; + default: + item.t->maybe_sync_copied_lba_key( + lextent.get_laddr(), + lextent.get_paddr(), + std::nullopt); + break; + } } } diff --git a/src/crimson/os/seastore/lba/btree_lba_manager.cc b/src/crimson/os/seastore/lba/btree_lba_manager.cc index 10a38fa9d32..10cb09f5490 100644 --- a/src/crimson/os/seastore/lba/btree_lba_manager.cc +++ b/src/crimson/os/seastore/lba/btree_lba_manager.cc @@ -1495,7 +1495,8 @@ BtreeLBAManager::remap_mappings( void BtreeLBAManager::update_paddr_sync( Transaction &t, laddr_t laddr, - paddr_t paddr) + paddr_t paddr, + std::optional shadow) { LOG_PREFIX(BtreeLBAManager::update_paddr_sync); DEBUGT("laddr={}, paddr={}", t, laddr, paddr); @@ -1513,12 +1514,28 @@ void BtreeLBAManager::update_paddr_sync( ceph_assert(child->is_exist_clean()); auto cursor = iter.get_cursor(c); assert(cursor->get_laddr() == laddr); + paddr_t shadow_paddr; + if (shadow) { + // the committing txn changed the shadow + // to *shadow + shadow_paddr = *shadow; + } else if (cursor->has_shadow_paddr()) { + // shadow is preserved by the committer, + // and the copy inherited one, so the source + // was promoted when it's copied + shadow_paddr = cursor->get_shadow_paddr(); + } else { + // shadow is preserved, and nothing is + // inherited: not currently promoted + shadow_paddr = P_ADDR_NULL; + } btree.update( c, std::move(iter), lba_map_val_t{ cursor->get_length(), pladdr_t{std::move(paddr)}, + shadow_paddr, cursor->get_refcount(), cursor->get_checksum(), cursor->get_extent_type()}, @@ -1577,8 +1594,8 @@ BtreeLBAManager::_copy_mapping( c.trans.new_lba_key_copied( ret.src->get_key(), dest_laddr, - [this, c](laddr_t laddr, paddr_t paddr) { - update_paddr_sync(c.trans, laddr, paddr); + [this, c](laddr_t laddr, paddr_t paddr, std::optional shadow) { + update_paddr_sync(c.trans, laddr, paddr, shadow); }); auto [niter, inserted] = co_await btree.copy( c, diff --git a/src/crimson/os/seastore/lba/btree_lba_manager.h b/src/crimson/os/seastore/lba/btree_lba_manager.h index c78289bbf41..1bbd3a07ef2 100644 --- a/src/crimson/os/seastore/lba/btree_lba_manager.h +++ b/src/crimson/os/seastore/lba/btree_lba_manager.h @@ -617,7 +617,8 @@ private: void update_paddr_sync( Transaction &t, laddr_t laddr, - paddr_t paddr); + paddr_t paddr, + std::optional shadow); /** diff --git a/src/crimson/os/seastore/transaction.h b/src/crimson/os/seastore/transaction.h index 8e907a692e9..ceab4491e6d 100644 --- a/src/crimson/os/seastore/transaction.h +++ b/src/crimson/os/seastore/transaction.h @@ -748,7 +748,7 @@ public: bool force_rewrite_conflict = false; using update_copied_lba_key_func_t = - std::function; + std::function)>; void new_lba_key_copied( laddr_t src, laddr_t dest, @@ -758,7 +758,8 @@ public: update_copied_lba_key = std::move(func); } } - void maybe_sync_copied_lba_key(laddr_t laddr, paddr_t paddr) { + void maybe_sync_copied_lba_key( + laddr_t laddr, paddr_t paddr, std::optional shadow) { if (likely(copied_lba_keys.empty())) { return; } @@ -768,7 +769,7 @@ public: return; } laddr_t key = it->second; - update_copied_lba_key(key, paddr); + update_copied_lba_key(key, paddr, shadow); } RootBlockRef peek_root() { return root; From 0a0a7e3da0040bd7fe7f04f7552f5a22d99a527d Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Sun, 7 Jun 2026 16:31:03 +0800 Subject: [PATCH 36/51] crimson/os/seastore/cache: allow LogNode to be remapped Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/cache.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc index 8fa3f69518c..1620b077f4b 100644 --- a/src/crimson/os/seastore/cache.cc +++ b/src/crimson/os/seastore/cache.cc @@ -1256,6 +1256,9 @@ CachedExtentRef Cache::alloc_remapped_extent_by_type( case extent_types_t::TEST_BLOCK: return alloc_remapped_extent( t, remap_laddr, remap_paddr, remap_offset, remap_length, original_bptr); + case extent_types_t::LOG_NODE: + return alloc_remapped_extent( + t, remap_laddr, remap_paddr, remap_offset, remap_length, original_bptr); default: ceph_abort("invalid extent type"); return CachedExtentRef(); From 43f048c81e14f4bcac25170a27ef613351b22db9 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Sun, 7 Jun 2026 16:32:37 +0800 Subject: [PATCH 37/51] crimson/os/seastore/epm: avoid adjust generation for eviction if the main device is not SEGMENTED Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/extent_placement_manager.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index d1eb7606971..cbe53168e78 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -1053,7 +1053,8 @@ private: } rewrite_gen_t adjust_generation(rewrite_gen_t gen) { - if (has_cold_tier()) { + if (has_cold_tier() && + get_main_backend_type() == backend_type_t::SEGMENTED) { return eviction_state.adjust_generation_with_eviction(gen); } else { return gen; From f18fe72baaf4f550529af453f741c6e9636cfefc Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Tue, 9 Jun 2026 15:33:24 +0800 Subject: [PATCH 38/51] crimson/os/seastore/transaction_manager: indicate the extents as COLD when demoting them Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/async_cleaner.cc | 2 +- src/crimson/os/seastore/async_cleaner.h | 1 + src/crimson/os/seastore/cached_extent.h | 10 ++++++++-- src/crimson/os/seastore/extent_placement_manager.h | 12 ++++++++---- src/crimson/os/seastore/transaction_manager.cc | 8 +++++--- src/crimson/os/seastore/transaction_manager.h | 1 + 6 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/crimson/os/seastore/async_cleaner.cc b/src/crimson/os/seastore/async_cleaner.cc index 7ccf710d9e2..6ff73f7995e 100644 --- a/src/crimson/os/seastore/async_cleaner.cc +++ b/src/crimson/os/seastore/async_cleaner.cc @@ -1395,7 +1395,7 @@ do_reclaim_space_ret do_reclaim_space( reclaimed += ext->get_length(); } return extent_callback.rewrite_extents( - t, extents, target_generation, modify_time); + t, extents, target_generation, PLACEMENT_HINT_NULL, modify_time); }); }).si_then([&extent_callback, &t] { return extent_callback.submit_transaction_direct(t); diff --git a/src/crimson/os/seastore/async_cleaner.h b/src/crimson/os/seastore/async_cleaner.h index 9751dc15eea..6f38b9f33fc 100644 --- a/src/crimson/os/seastore/async_cleaner.h +++ b/src/crimson/os/seastore/async_cleaner.h @@ -357,6 +357,7 @@ public: Transaction &t, std::vector &extents, rewrite_gen_t target_generation, + placement_hint_t hint, sea_time_point modify_time) = 0; /** diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h index 2a528a2a259..7a3ccf979ac 100644 --- a/src/crimson/os/seastore/cached_extent.h +++ b/src/crimson/os/seastore/cached_extent.h @@ -776,8 +776,14 @@ public: } /// assign the target rewrite generation for the followup rewrite - void set_target_rewrite_generation(rewrite_gen_t gen) { - user_hint = placement_hint_t::REWRITE; + void set_target_rewrite_generation( + rewrite_gen_t gen, + placement_hint_t hint = PLACEMENT_HINT_NULL) { + if (hint != PLACEMENT_HINT_NULL) { + user_hint = hint; + } else { + user_hint = placement_hint_t::REWRITE; + } rewrite_generation = gen; } diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index cbe53168e78..7f88f426ed1 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -488,7 +488,9 @@ public: ) { assert(opt.hint < placement_hint_t::NUM_HINTS); assert(is_target_rewrite_generation(opt.gen, dynamic_max_rewrite_generation)); - assert(opt.gen == INIT_GENERATION || opt.hint == placement_hint_t::REWRITE); + assert(opt.gen == INIT_GENERATION || + opt.hint == placement_hint_t::REWRITE || + opt.hint == placement_hint_t::COLD); data_category_t category = get_extent_category(type); opt.gen = adjust_generation( @@ -529,7 +531,9 @@ public: LOG_PREFIX(ExtentPlacementManager::alloc_new_data_extents); assert(opt.hint < placement_hint_t::NUM_HINTS); assert(is_target_rewrite_generation(opt.gen, dynamic_max_rewrite_generation)); - assert(opt.gen == INIT_GENERATION || opt.hint == placement_hint_t::REWRITE); + assert(opt.gen == INIT_GENERATION || + opt.hint == placement_hint_t::REWRITE || + opt.hint == placement_hint_t::COLD); data_category_t category = get_extent_category(type); opt.gen = adjust_generation( @@ -768,7 +772,6 @@ private: is_lba_backref_node(type)) { gen = INLINE_GENERATION; } else if (hint == placement_hint_t::COLD) { - assert(gen == INIT_GENERATION); if (background_process.has_cold_tier()) { gen = hot_tier_generations; } else { @@ -806,7 +809,8 @@ private: } if (is_tracked && gen >= hot_tier_generations && - hint != placement_hint_t::REWRITE) { + hint != placement_hint_t::REWRITE && + hint != placement_hint_t::COLD) { gen = hot_tier_generations - 1; } diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index 09655e51357..3fe575e0517 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -1493,18 +1493,19 @@ TransactionManager::rewrite_extents_ret TransactionManager::rewrite_extents( Transaction &t, std::vector &extents, rewrite_gen_t target_generation, + placement_hint_t hint, sea_time_point modify_time) { LOG_PREFIX(TransactionManager::rewrite_extents); return seastar::do_with( P_ADDR_NULL, L_ADDR_NULL, - [this, &t, target_generation, modify_time, &extents, FNAME] + [this, &t, target_generation, modify_time, &extents, FNAME, hint] (auto &paddr_hint, auto &next_laddr) { return trans_intr::do_for_each( extents, [this, &t, target_generation, modify_time, FNAME, - &paddr_hint, &next_laddr](auto &extent) { + &paddr_hint, &next_laddr, hint](auto &extent) { { auto updated = cache->update_extent_from_transaction(t, extent); if (!updated) { @@ -1531,7 +1532,7 @@ TransactionManager::rewrite_extents_ret TransactionManager::rewrite_extents( } extent->set_target_rewrite_generation(INIT_GENERATION); } else { - extent->set_target_rewrite_generation(target_generation); + extent->set_target_rewrite_generation(target_generation, hint); ceph_assert(modify_time != NULL_TIME); extent->set_modify_time(modify_time); } @@ -1616,6 +1617,7 @@ TransactionManager::demote_region( co_await rewrite_extents( t, extents, epm->get_max_hot_gen() + 1, + placement_hint_t::COLD, seastar::lowres_system_clock::now()); co_return ret; diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index 2ca10ed8101..1f7c84f7a33 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -975,6 +975,7 @@ public: Transaction &t, std::vector &extents, rewrite_gen_t target_generation, + placement_hint_t hint, sea_time_point modify_time) final; using ExtentCallbackInterface::promote_extent_ret; From b899ade928621684bbd839ddc8bd3429053f4730 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Wed, 17 Jun 2026 19:21:06 +0800 Subject: [PATCH 39/51] crimson/os/seastore/lba: add more rigorous assertions Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/lba/lba_btree_node.h | 6 +++++- src/crimson/os/seastore/linked_tree_node.h | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/crimson/os/seastore/lba/lba_btree_node.h b/src/crimson/os/seastore/lba/lba_btree_node.h index f6dd325a8ba..9f6e1ed296a 100644 --- a/src/crimson/os/seastore/lba/lba_btree_node.h +++ b/src/crimson/os/seastore/lba/lba_btree_node.h @@ -357,7 +357,11 @@ struct LBALeafNode it++; continue; } else { - assert(child->_is_exist_clean() || child->_is_exist_mutation_pending()); + assert(child->_is_exist_clean() || + child->_is_exist_mutation_pending() || + // a pending child might have been invalidated and + // its corresponding mapping has not been removed + !child->_is_valid()); } } SUBTRACET(seastore_lba, "examing v2: {}~{}, v1: {}~{}", diff --git a/src/crimson/os/seastore/linked_tree_node.h b/src/crimson/os/seastore/linked_tree_node.h index f6dc6f828db..c9cf58902ba 100644 --- a/src/crimson/os/seastore/linked_tree_node.h +++ b/src/crimson/os/seastore/linked_tree_node.h @@ -215,9 +215,9 @@ public: virtual bool _is_mutable() const = 0; virtual bool _is_exist_clean() const = 0; virtual bool _is_exist_mutation_pending() const = 0; + virtual bool _is_valid() const = 0; protected: parent_tracker_ref parent_tracker; - virtual bool _is_valid() const = 0; virtual bool _is_stable() const = 0; template friend class ParentNode; From aba5a77872f30a86a673f889627bb37f3f080cdd Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Mon, 22 Jun 2026 17:48:15 +0800 Subject: [PATCH 40/51] crimson/os/seastore/lba: make sure all mappings in the copied range is examined when updating paddrs for the copied mappings Signed-off-by: Xuehan Xu --- .../os/seastore/btree/fixed_kv_btree.h | 61 ++++++++++ .../os/seastore/lba/btree_lba_manager.cc | 110 +++++++++++------- .../os/seastore/lba/btree_lba_manager.h | 1 + src/crimson/os/seastore/linked_tree_node.h | 1 - src/crimson/os/seastore/transaction.h | 17 ++- 5 files changed, 145 insertions(+), 45 deletions(-) diff --git a/src/crimson/os/seastore/btree/fixed_kv_btree.h b/src/crimson/os/seastore/btree/fixed_kv_btree.h index 59b7a150ee0..c9cc522cd54 100644 --- a/src/crimson/os/seastore/btree/fixed_kv_btree.h +++ b/src/crimson/os/seastore/btree/fixed_kv_btree.h @@ -216,6 +216,21 @@ public: } + // return true if reached the next mapping and false otherwise + bool next_sync(op_context_t c) { +#ifndef NDEBUG + assert_valid(); +#endif + assert(is_full()); + assert(!is_end()); + leaf.pos++; + if (at_boundary()) { + return handle_boundary_sync(c); + } else { + return true; + } + } + /** * Move to the previous entry. If already at position 0 in the current * leaf, walks up the internal stack (ensure_internal_bottom_up) to find @@ -520,6 +535,50 @@ public: }); } + // return true if reached the next mapping and false otherwise + bool handle_boundary_sync(op_context_t c) { + assert(at_boundary()); + assert(is_full()); + depth_t depth = 2; + // find the first level (from the leaf's parent up) with a right sibling + for (; depth <= get_depth(); depth++) { + auto &internal = get_internal(depth); + auto it = internal.node->iter_idx(internal.pos + 1); + if (it != internal.node->end()) { + break; + } + } + if (depth > get_depth()) { + return false; // no next leaf (end) + } + // step to the right sibling at that level, then descend leftmost to the leaf + get_internal(depth).pos++; + while (depth > 2) { + auto &internal = get_internal(depth); + auto it = get_internal(depth).node->iter_idx(internal.pos); + auto child = get_internal(depth).node->template get_child_sync< + internal_node_t>(c.trans, c.cache, internal.pos, it.get_key()); + if (!is_valid_child_ptr(child.get())) { + return false; + } + depth--; + get_internal(depth).node = child; + get_internal(depth).pos = 0; + } + assert(depth == 2); + // position `leaf` at the next leaf + auto &parent = get_internal(depth); + auto it = parent.node->begin(); + auto child = parent.node->template get_child_sync< + leaf_node_t>(c.trans, c.cache, parent.pos, it.get_key()); + if (!is_valid_child_ptr(child.get())) { + return false; + } + leaf.node = child; + leaf.pos = 0; + return true; + } + /** * Called when leaf.pos has advanced past the end of the current leaf * (at_boundary() == true). Walks up the internal stack to find the @@ -763,10 +822,12 @@ public: if (depth > 1) { auto child = entry.node->template get_child_sync( c.trans, c.cache, entry.pos, riter.get_key()); + ceph_assert(is_valid_child_ptr(child.get())); iter.get_internal(depth).node = child; } else { auto child = entry.node->template get_child_sync( c.trans, c.cache, entry.pos, riter.get_key()); + ceph_assert(is_valid_child_ptr(child.get())); iter.leaf.node = child; } } diff --git a/src/crimson/os/seastore/lba/btree_lba_manager.cc b/src/crimson/os/seastore/lba/btree_lba_manager.cc index 10cb09f5490..843861a299b 100644 --- a/src/crimson/os/seastore/lba/btree_lba_manager.cc +++ b/src/crimson/os/seastore/lba/btree_lba_manager.cc @@ -1496,6 +1496,7 @@ void BtreeLBAManager::update_paddr_sync( Transaction &t, laddr_t laddr, paddr_t paddr, + extent_len_t len, std::optional shadow) { LOG_PREFIX(BtreeLBAManager::update_paddr_sync); @@ -1503,44 +1504,73 @@ void BtreeLBAManager::update_paddr_sync( auto c = get_context(t); auto btree = get_btree_sync(c); auto iter = btree.lower_bound_sync(c, laddr); - assert(iter.get_leaf_node()->is_pending()); - auto child = iter.get_leaf_node()->get_child_sync( - c.trans, c.cache, iter.get_leaf_pos(), iter.get_key()); - ceph_assert(child); - if (child->is_initial_pending()) { - TRACET("{} is initial_pending, skipping", t, *child); - return; + while (iter.get_key() + iter.get_val().len <= laddr + len) { + assert(iter.get_leaf_node()->is_pending()); + if (iter.get_val().pladdr.is_laddr() || + iter.get_val().pladdr.get_paddr().is_zero()) { + TRACET("skipping mapping {}~{}", t, iter.get_key(), iter.get_val()); + if (!iter.next_sync(c)) { + // can't reach the next mapping, which means the next + // mapping mustn't have been touched by the current + // transactions, we don't need to continue, just leave + return; + } + continue; + } + auto child = iter.get_leaf_node()->get_child_sync( + c.trans, c.cache, iter.get_leaf_pos(), iter.get_key()); + ceph_assert(is_valid_child_ptr(child.get())); + if (child->is_initial_pending()) { + TRACET("{} is initial_pending, skipping", t, *child); + if (!iter.next_sync(c)) { + // can't reach the next mapping, which means the next + // mapping mustn't have been touched by the current + // transactions, we don't need to continue, just leave + return; + } + continue; + } + ceph_assert(child->is_exist_clean()); + auto cursor = iter.get_cursor(c); + extent_len_t off = cursor->get_laddr().get_byte_distance< + extent_len_t>(laddr); + paddr_t shadow_paddr; + if (shadow) { + // the committing txn changed the shadow + // to *shadow + shadow_paddr = *shadow; + if (shadow_paddr != P_ADDR_NULL) { + shadow_paddr = shadow_paddr + off; + } + } else if (cursor->has_shadow_paddr()) { + // shadow is preserved by the committer, + // and the copy inherited one, so the source + // was promoted when it's copied + shadow_paddr = cursor->get_shadow_paddr(); + } else { + // shadow is preserved, and nothing is + // inherited: not currently promoted + shadow_paddr = P_ADDR_NULL; + } + iter = btree.update( + c, + std::move(iter), + lba_map_val_t{ + cursor->get_length(), + pladdr_t{paddr + off}, + shadow_paddr, + cursor->get_refcount(), + cursor->get_checksum(), + cursor->get_extent_type()}, + nullptr, + modification_t::TRANS_SYNC); + if (!iter.next_sync(c)) { + // can't reach the next mapping, which means the next + // mapping mustn't have been touched by the current + // transactions, we don't need to continue, just leave + return; + } } - ceph_assert(child->is_exist_clean()); - auto cursor = iter.get_cursor(c); - assert(cursor->get_laddr() == laddr); - paddr_t shadow_paddr; - if (shadow) { - // the committing txn changed the shadow - // to *shadow - shadow_paddr = *shadow; - } else if (cursor->has_shadow_paddr()) { - // shadow is preserved by the committer, - // and the copy inherited one, so the source - // was promoted when it's copied - shadow_paddr = cursor->get_shadow_paddr(); - } else { - // shadow is preserved, and nothing is - // inherited: not currently promoted - shadow_paddr = P_ADDR_NULL; - } - btree.update( - c, - std::move(iter), - lba_map_val_t{ - cursor->get_length(), - pladdr_t{std::move(paddr)}, - shadow_paddr, - cursor->get_refcount(), - cursor->get_checksum(), - cursor->get_extent_type()}, - nullptr, - modification_t::TRANS_SYNC); } // --------------------------------------------------------------------------- @@ -1594,8 +1624,10 @@ BtreeLBAManager::_copy_mapping( c.trans.new_lba_key_copied( ret.src->get_key(), dest_laddr, - [this, c](laddr_t laddr, paddr_t paddr, std::optional shadow) { - update_paddr_sync(c.trans, laddr, paddr, shadow); + ret.src->get_length(), + [this, c](laddr_t laddr, paddr_t paddr, + extent_len_t len, std::optional shadow) { + update_paddr_sync(c.trans, laddr, paddr, len, shadow); }); auto [niter, inserted] = co_await btree.copy( c, diff --git a/src/crimson/os/seastore/lba/btree_lba_manager.h b/src/crimson/os/seastore/lba/btree_lba_manager.h index 1bbd3a07ef2..ae8e88c9904 100644 --- a/src/crimson/os/seastore/lba/btree_lba_manager.h +++ b/src/crimson/os/seastore/lba/btree_lba_manager.h @@ -618,6 +618,7 @@ private: Transaction &t, laddr_t laddr, paddr_t paddr, + extent_len_t len, std::optional shadow); diff --git a/src/crimson/os/seastore/linked_tree_node.h b/src/crimson/os/seastore/linked_tree_node.h index c9cf58902ba..0e500305357 100644 --- a/src/crimson/os/seastore/linked_tree_node.h +++ b/src/crimson/os/seastore/linked_tree_node.h @@ -350,7 +350,6 @@ public: auto &sparent = me.get_stable_for_key(key); auto spos = sparent.lower_bound(key).get_offset(); child = sparent.children[spos]; - assert(is_valid_child_ptr(child)); auto ret = etvr.get_extent_viewable_by_trans_sync( t, static_cast(child)); return ret->template cast(); diff --git a/src/crimson/os/seastore/transaction.h b/src/crimson/os/seastore/transaction.h index ceab4491e6d..c256e5b8f99 100644 --- a/src/crimson/os/seastore/transaction.h +++ b/src/crimson/os/seastore/transaction.h @@ -747,13 +747,19 @@ public: bool force_rewrite_conflict = false; + struct lmapping_t { + laddr_t dest = L_ADDR_NULL; + extent_len_t len = 0; + }; using update_copied_lba_key_func_t = - std::function)>; + std::function)>; void new_lba_key_copied( laddr_t src, laddr_t dest, + extent_len_t len, update_copied_lba_key_func_t &&func) { - copied_lba_keys.emplace(src, dest); + copied_lba_keys.emplace(src, lmapping_t{dest, len}); if (!update_copied_lba_key) { update_copied_lba_key = std::move(func); } @@ -768,8 +774,9 @@ public: if (it == copied_lba_keys.end()) { return; } - laddr_t key = it->second; - update_copied_lba_key(key, paddr, shadow); + laddr_t key = it->second.dest; + extent_len_t len = it->second.len; + update_copied_lba_key(key, paddr, len, shadow); } RootBlockRef peek_root() { return root; @@ -1002,7 +1009,7 @@ private: cache_hint_t cache_hint = CACHE_HINT_TOUCH; - std::map copied_lba_keys; + std::map copied_lba_keys; update_copied_lba_key_func_t update_copied_lba_key; }; using TransactionRef = Transaction::Ref; From 92a569aa4aa883a012aafd43f1f445e794c9399f Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Tue, 30 Jun 2026 16:57:17 +0800 Subject: [PATCH 41/51] crimson/os/seastore/btree: define fmt::formatter for lba_val_map_t in btree_types.h Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/btree/btree_types.h | 4 ++++ src/crimson/os/seastore/lba/lba_btree_node.h | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/crimson/os/seastore/btree/btree_types.h b/src/crimson/os/seastore/btree/btree_types.h index 0958e1933e4..076f58e1ef3 100644 --- a/src/crimson/os/seastore/btree/btree_types.h +++ b/src/crimson/os/seastore/btree/btree_types.h @@ -298,3 +298,7 @@ std::ostream &operator<<( } } // namespace crimson::os::seastore + +#if FMT_VERSION >= 90000 +template <> struct fmt::formatter : fmt::ostream_formatter {}; +#endif diff --git a/src/crimson/os/seastore/lba/lba_btree_node.h b/src/crimson/os/seastore/lba/lba_btree_node.h index 9f6e1ed296a..500ff523df0 100644 --- a/src/crimson/os/seastore/lba/lba_btree_node.h +++ b/src/crimson/os/seastore/lba/lba_btree_node.h @@ -531,7 +531,6 @@ using LBACursorRef = boost::intrusive_ptr; #if FMT_VERSION >= 90000 template <> struct fmt::formatter : fmt::ostream_formatter {}; -template <> struct fmt::formatter : fmt::ostream_formatter {}; template <> struct fmt::formatter : fmt::ostream_formatter {}; template <> struct fmt::formatter : fmt::ostream_formatter {}; #endif From aad3524f7a05cefe3292723946c099baf0dc82ab Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Fri, 3 Jul 2026 21:32:53 +0800 Subject: [PATCH 42/51] crimson/os/seastore/journal: lift the alloc_map scanning from CircularBoundedJournal to Journal Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/journal.cc | 34 +++++++++++++++++-- src/crimson/os/seastore/journal.h | 18 ++++++++++ .../journal/circular_bounded_journal.cc | 25 ++------------ .../journal/circular_bounded_journal.h | 10 ++---- .../os/seastore/journal/segmented_journal.h | 14 ++++++++ 5 files changed, 69 insertions(+), 32 deletions(-) diff --git a/src/crimson/os/seastore/journal.cc b/src/crimson/os/seastore/journal.cc index e55b62b045a..2f866de5db3 100644 --- a/src/crimson/os/seastore/journal.cc +++ b/src/crimson/os/seastore/journal.cc @@ -5,7 +5,35 @@ #include "journal/segmented_journal.h" #include "journal/circular_bounded_journal.h" -namespace crimson::os::seastore::journal { +namespace crimson::os::seastore { + +Journal::scan_alloc_map_ret +Journal::scan_alloc_map() { + alloc_map_t map; + journal_seq_t tail = get_dirty_tail() <= get_alloc_tail() ? + get_dirty_tail() : get_alloc_tail(); + auto build_paddr_seq_map = [&map]( + const auto &offsets, + const auto &e, + sea_time_point modify_time) + { + if (e.type == extent_types_t::ALLOC_INFO) { + alloc_delta_t alloc_delta; + decode(alloc_delta, e.bl); + if (alloc_delta.op == alloc_delta_t::op_types_t::CLEAR) { + for (auto &alloc_blk : alloc_delta.alloc_blk_ranges) { + map[alloc_blk.paddr] = offsets.write_result.start_seq; + } + } + } + return replay_ertr::make_ready_future(true); + }; + // build the paddr->journal_seq_t map from extent allocations + co_await scan_valid_record_delta(std::move(build_paddr_seq_map), tail); + co_return map; +} + +namespace journal { JournalRef make_segmented( store_index_t store_index, @@ -24,4 +52,6 @@ JournalRef make_circularbounded( return std::make_unique(store_index, trimmer, device, path); } -} +} // namespace journal + +} // namespace crimson::os::seastore diff --git a/src/crimson/os/seastore/journal.h b/src/crimson/os/seastore/journal.h index ea7debc0d4c..6b757d02fa9 100644 --- a/src/crimson/os/seastore/journal.h +++ b/src/crimson/os/seastore/journal.h @@ -106,6 +106,24 @@ public: virtual backend_type_t get_type() = 0; virtual bool is_checksum_needed() = 0; + +protected: + using alloc_map_t = std::map; + using scan_alloc_map_ertr = replay_ertr; + using scan_alloc_map_ret = scan_alloc_map_ertr::future; + scan_alloc_map_ret scan_alloc_map(); + + using scan_delta_handler_t = std::function< + replay_ertr::future( + const record_locator_t&, + const delta_info_t&, + sea_time_point modify_time)>; + virtual replay_ret scan_valid_record_delta( + scan_delta_handler_t &&delta_handler, + journal_seq_t tail) = 0; + + virtual journal_seq_t get_dirty_tail() const = 0; + virtual journal_seq_t get_alloc_tail() const = 0; }; using JournalRef = std::unique_ptr; diff --git a/src/crimson/os/seastore/journal/circular_bounded_journal.cc b/src/crimson/os/seastore/journal/circular_bounded_journal.cc index 4bb31172fd4..b7fe14c0512 100644 --- a/src/crimson/os/seastore/journal/circular_bounded_journal.cc +++ b/src/crimson/os/seastore/journal/circular_bounded_journal.cc @@ -141,7 +141,7 @@ CircularBoundedJournal::submit_record( } Journal::replay_ret CircularBoundedJournal::replay_segment( - cbj_delta_handler_t &handler, scan_valid_records_cursor& cursor) + scan_delta_handler_t &handler, scan_valid_records_cursor& cursor) { LOG_PREFIX(Journal::replay_segment); return seastar::do_with( @@ -232,7 +232,7 @@ Journal::replay_ret CircularBoundedJournal::replay_segment( Journal::replay_ret CircularBoundedJournal::scan_valid_record_delta( - cbj_delta_handler_t &&handler, journal_seq_t tail) + scan_delta_handler_t &&handler, journal_seq_t tail) { LOG_PREFIX(Journal::scan_valid_record_delta); INFO("starting at {} ", tail); @@ -341,7 +341,6 @@ Journal::replay_ret CircularBoundedJournal::replay( cjs.set_cbj_header(head); DEBUG("header : {}", cjs.get_cbj_header()); cjs.set_initialized(true); - std::map map; std::map> crc_info; auto tail = get_dirty_tail() <= get_alloc_tail() ? get_dirty_tail() : get_alloc_tail(); @@ -360,27 +359,9 @@ Journal::replay_ret CircularBoundedJournal::replay( return replay_ertr::make_ready_future(true); }; co_await scan_valid_record_delta(std::move(find_tail), tail); - tail = get_dirty_tail() <= get_alloc_tail() ? - get_dirty_tail() : get_alloc_tail(); - auto build_paddr_seq_map = [&map]( - const auto &offsets, - const auto &e, - sea_time_point modify_time) - { - if (e.type == extent_types_t::ALLOC_INFO) { - alloc_delta_t alloc_delta; - decode(alloc_delta, e.bl); - if (alloc_delta.op == alloc_delta_t::op_types_t::CLEAR) { - for (auto &alloc_blk : alloc_delta.alloc_blk_ranges) { - map[alloc_blk.paddr] = offsets.write_result.start_seq; - } - } - } - return replay_ertr::make_ready_future(true); - }; // The second pass to build the paddr->journal_seq_t map // from extent allocations - co_await scan_valid_record_delta(std::move(build_paddr_seq_map), tail); + alloc_map_t map = co_await scan_alloc_map(); auto call_d_handler_if_valid = [this, &map, &d_handler, &crc_info]( const auto &offsets, const auto &e, diff --git a/src/crimson/os/seastore/journal/circular_bounded_journal.h b/src/crimson/os/seastore/journal/circular_bounded_journal.h index 9facd14d654..e6af4c556ec 100644 --- a/src/crimson/os/seastore/journal/circular_bounded_journal.h +++ b/src/crimson/os/seastore/journal/circular_bounded_journal.h @@ -147,14 +147,8 @@ public: return cjs.get_records_start(); } - using cbj_delta_handler_t = std::function< - replay_ertr::future( - const record_locator_t&, - const delta_info_t&, - sea_time_point modify_time)>; - Journal::replay_ret scan_valid_record_delta( - cbj_delta_handler_t &&delta_handler, + scan_delta_handler_t &&delta_handler, journal_seq_t tail); void try_read_rolled_header(scan_valid_records_cursor &cursor) { @@ -170,7 +164,7 @@ public: }; Journal::replay_ret replay_segment( - cbj_delta_handler_t &handler, scan_valid_records_cursor& cursor); + scan_delta_handler_t &handler, scan_valid_records_cursor& cursor); read_ret read(paddr_t start, size_t len) final; diff --git a/src/crimson/os/seastore/journal/segmented_journal.h b/src/crimson/os/seastore/journal/segmented_journal.h index 03c1e8721c6..05c191b6c25 100644 --- a/src/crimson/os/seastore/journal/segmented_journal.h +++ b/src/crimson/os/seastore/journal/segmented_journal.h @@ -110,6 +110,20 @@ private: delta_handler_t &delta_handler, ///< [in] processes deltas in order replay_stats_t &stats ///< [out] replay stats ); + + journal_seq_t get_dirty_tail() const final { + return trimmer.get_dirty_tail(); + } + + journal_seq_t get_alloc_tail() const final { + return trimmer.get_alloc_tail(); + } + + replay_ret scan_valid_record_delta( + scan_delta_handler_t &&delta_handler, + journal_seq_t tail) final { + return replay_ertr::now(); + } }; } From f56b2a5f42ef000a85c9c3f231b6ed21f3c78192 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Fri, 3 Jul 2026 21:47:23 +0800 Subject: [PATCH 43/51] crimson/os/seastore/journal: coroutinize SegmentedJournal::replay() Signed-off-by: Xuehan Xu --- .../os/seastore/journal/segmented_journal.cc | 40 +++++++------------ 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/src/crimson/os/seastore/journal/segmented_journal.cc b/src/crimson/os/seastore/journal/segmented_journal.cc index 82336f42a16..664110333b5 100644 --- a/src/crimson/os/seastore/journal/segmented_journal.cc +++ b/src/crimson/os/seastore/journal/segmented_journal.cc @@ -329,32 +329,20 @@ SegmentedJournal::replay_ret SegmentedJournal::replay( delta_handler_t &&delta_handler) { LOG_PREFIX(Journal::replay); - return sm_group.find_journal_segment_headers( - ).safe_then([this, FNAME, delta_handler=std::move(delta_handler)] - (auto &&segment_headers) mutable -> replay_ret { - INFO("got {} segments", segment_headers.size()); - return seastar::do_with( - std::move(delta_handler), - replay_segments_t(), - replay_stats_t(), - [this, segment_headers=std::move(segment_headers), FNAME] - (auto &handler, auto &segments, auto &stats) mutable -> replay_ret { - return prep_replay_segments(std::move(segment_headers) - ).safe_then([this, &handler, &segments, &stats](auto replay_segs) mutable { - segments = std::move(replay_segs); - return crimson::do_for_each(segments,[this, &handler, &stats](auto i) mutable { - return replay_segment(i.first, i.second, handler, stats); - }); - }).safe_then([&stats, FNAME] { - INFO("replay done, record_groups={}, records={}, " - "alloc_deltas={}, dirty_deltas={}", - stats.num_record_groups, - stats.num_records, - stats.num_alloc_deltas, - stats.num_dirty_deltas); - }); - }); - }); + auto handler = std::move(delta_handler); + auto segment_headers = co_await sm_group.find_journal_segment_headers(); + INFO("got {} segments", segment_headers.size()); + replay_stats_t stats; + auto segments = co_await prep_replay_segments(std::move(segment_headers)); + for (auto &[seq, header] : segments) { + co_await replay_segment(seq, header, handler, stats); + } + INFO("replay done, record_groups={}, records={}, " + "alloc_deltas={}, dirty_deltas={}", + stats.num_record_groups, + stats.num_records, + stats.num_alloc_deltas, + stats.num_dirty_deltas); } seastar::future<> SegmentedJournal::flush(OrderingHandle &handle) From acdb2923619f6b9422434e9c0376d49da40eeb99 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Fri, 3 Jul 2026 22:05:54 +0800 Subject: [PATCH 44/51] crimson/os/seastore/journal: refactor SegmentedJournal::replay() to adapt to Journal::scan_valid_record_delta() Signed-off-by: Xuehan Xu --- .../os/seastore/journal/segmented_journal.cc | 90 ++++++++++--------- .../os/seastore/journal/segmented_journal.h | 18 ++-- 2 files changed, 58 insertions(+), 50 deletions(-) diff --git a/src/crimson/os/seastore/journal/segmented_journal.cc b/src/crimson/os/seastore/journal/segmented_journal.cc index 664110333b5..130054f0fd1 100644 --- a/src/crimson/os/seastore/journal/segmented_journal.cc +++ b/src/crimson/os/seastore/journal/segmented_journal.cc @@ -10,6 +10,7 @@ #include "segmented_journal.h" #include "crimson/common/config_proxy.h" +#include "crimson/common/coroutine.h" #include "crimson/os/seastore/logging.h" SET_SUBSYS(seastore_journal); @@ -108,8 +109,8 @@ SegmentedJournal::prep_replay_segments( return scan_last_segment(last_segment_id, last_header ).safe_then([this, FNAME, segments=std::move(segments)] { INFO("dirty_tail={}, alloc_tail={}", - trimmer.get_dirty_tail(), - trimmer.get_alloc_tail()); + get_dirty_tail(), + get_alloc_tail()); auto journal_tail = trimmer.get_journal_tail(); auto journal_tail_paddr = journal_tail.offset; ceph_assert(journal_tail != JOURNAL_SEQ_NULL); @@ -129,9 +130,9 @@ SegmentedJournal::prep_replay_segments( auto num_segments = segments.end() - from; INFO("{} segments to replay", num_segments); - auto ret = replay_segments_t(num_segments); + replay_segments.resize(num_segments); std::transform( - from, segments.end(), ret.begin(), + from, segments.end(), replay_segments.begin(), [this](const auto &p) { auto ret = journal_seq_t{ p.second.segment_seq, @@ -141,10 +142,9 @@ SegmentedJournal::prep_replay_segments( }; return std::make_pair(ret, p.second); }); - ret[0].first.offset = journal_tail_paddr; + replay_segments[0].first.offset = journal_tail_paddr; return prep_replay_segments_fut( - replay_ertr::ready_future_marker{}, - std::move(ret)); + replay_ertr::ready_future_marker{}); }); } @@ -229,15 +229,14 @@ SegmentedJournal::replay_ertr::future<> SegmentedJournal::replay_segment( journal_seq_t seq, segment_header_t header, - delta_handler_t &handler, - replay_stats_t &stats) + scan_delta_handler_t &handler) { LOG_PREFIX(Journal::replay_segment); INFO("starting at {} -- {}", seq, header); return seastar::do_with( scan_valid_records_cursor(seq), SegmentManagerGroup::found_record_handler_t( - [&handler, this, &stats]( + [&handler, this]( record_locator_t locator, const record_group_header_t& header, const bufferlist& mdbuf) @@ -259,16 +258,14 @@ SegmentedJournal::replay_segment( [write_result=locator.write_result, this, FNAME, - &handler, - &stats](auto& record_deltas_list) + &handler](auto& record_deltas_list) { return crimson::do_for_each( record_deltas_list, [write_result, this, FNAME, - &handler, - &stats](record_deltas_t& record_deltas) + &handler](record_deltas_t& record_deltas) { ++stats.num_records; auto locator = record_locator_t{ @@ -281,30 +278,11 @@ SegmentedJournal::replay_segment( return crimson::do_for_each( record_deltas.deltas, [locator, - this, - &handler, - &stats](auto &p) + &handler](auto &p) { auto& modify_time = p.first; auto& delta = p.second; - return handler( - locator, - delta, - trimmer.get_dirty_tail(), - trimmer.get_alloc_tail(), - modify_time - ).safe_then([&stats, delta_type=delta.type](auto ret) { - auto [is_applied, ext] = ret; - if (is_applied) { - // see Cache::replay_delta() - assert(delta_type != extent_types_t::JOURNAL_TAIL); - if (delta_type == extent_types_t::ALLOC_INFO) { - ++stats.num_alloc_deltas; - } else { - ++stats.num_dirty_deltas; - } - } - }); + return handler(locator, delta, modify_time).discard_result(); }); }); }); @@ -325,6 +303,17 @@ SegmentedJournal::replay_segment( ); } +SegmentedJournal::replay_ret +SegmentedJournal::scan_valid_record_delta( + scan_delta_handler_t &&delta_handler, + journal_seq_t tail) +{ + auto handler = std::move(delta_handler); + for (auto &[seq, header] : replay_segments) { + co_await replay_segment(seq, header, handler); + } +} + SegmentedJournal::replay_ret SegmentedJournal::replay( delta_handler_t &&delta_handler) { @@ -332,11 +321,32 @@ SegmentedJournal::replay_ret SegmentedJournal::replay( auto handler = std::move(delta_handler); auto segment_headers = co_await sm_group.find_journal_segment_headers(); INFO("got {} segments", segment_headers.size()); - replay_stats_t stats; - auto segments = co_await prep_replay_segments(std::move(segment_headers)); - for (auto &[seq, header] : segments) { - co_await replay_segment(seq, header, handler, stats); - } + co_await prep_replay_segments(std::move(segment_headers)); + auto d_handler = [&handler, this]( + const record_locator_t &locator, + const delta_info_t &delta, + sea_time_point modify_time) -> replay_ertr::future { + auto ret = co_await handler( + locator, + delta, + get_dirty_tail(), + get_alloc_tail(), + modify_time); + auto [is_applied, ext] = ret; + if (is_applied) { + // see Cache::replay_delta() + assert(delta.type != extent_types_t::JOURNAL_TAIL); + if (delta.type == extent_types_t::ALLOC_INFO) { + ++stats.num_alloc_deltas; + } else { + ++stats.num_dirty_deltas; + } + } + co_return true; + }; + journal_seq_t tail = get_dirty_tail() <= get_alloc_tail() ? + get_dirty_tail() : get_alloc_tail(); + co_return co_await scan_valid_record_delta(std::move(d_handler), tail); INFO("replay done, record_groups={}, records={}, " "alloc_deltas={}, dirty_deltas={}", stats.num_record_groups, diff --git a/src/crimson/os/seastore/journal/segmented_journal.h b/src/crimson/os/seastore/journal/segmented_journal.h index 05c191b6c25..2b7135b2711 100644 --- a/src/crimson/os/seastore/journal/segmented_journal.h +++ b/src/crimson/os/seastore/journal/segmented_journal.h @@ -83,10 +83,7 @@ private: WritePipeline* write_pipeline = nullptr; /// return ordered vector of segments to replay - using replay_segments_t = std::vector< - std::pair>; - using prep_replay_segments_fut = replay_ertr::future< - replay_segments_t>; + using prep_replay_segments_fut = replay_ertr::future<>; prep_replay_segments_fut prep_replay_segments( std::vector> segments); @@ -100,15 +97,18 @@ private: std::size_t num_records = 0; std::size_t num_alloc_deltas = 0; std::size_t num_dirty_deltas = 0; - }; + } stats; + + using replay_segments_t = std::vector< + std::pair>; + replay_segments_t replay_segments; /// replays records starting at start through end of segment replay_ertr::future<> replay_segment( journal_seq_t start, ///< [in] starting addr, seq segment_header_t header, ///< [in] segment header - delta_handler_t &delta_handler, ///< [in] processes deltas in order - replay_stats_t &stats ///< [out] replay stats + scan_delta_handler_t &delta_handler ///< [in] processes deltas in order ); journal_seq_t get_dirty_tail() const final { @@ -121,9 +121,7 @@ private: replay_ret scan_valid_record_delta( scan_delta_handler_t &&delta_handler, - journal_seq_t tail) final { - return replay_ertr::now(); - } + journal_seq_t tail) final; }; } From 64263d609d9c6979160ac98e8fd427515a76c7be Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Fri, 3 Jul 2026 18:10:54 +0800 Subject: [PATCH 45/51] crimson/os/seastore/journal/segmented_journal: scan the journal for the allocation map on boot if the cold tier is an RBM one Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/journal.cc | 6 ++++-- src/crimson/os/seastore/journal.h | 3 ++- .../os/seastore/journal/segmented_journal.cc | 17 ++++++++++++++--- .../os/seastore/journal/segmented_journal.h | 4 +++- src/crimson/os/seastore/transaction_manager.cc | 5 ++++- .../crimson/seastore/test_btree_lba_manager.cc | 2 +- .../crimson/seastore/test_seastore_journal.cc | 4 ++-- 7 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/crimson/os/seastore/journal.cc b/src/crimson/os/seastore/journal.cc index 2f866de5db3..250fb76b18c 100644 --- a/src/crimson/os/seastore/journal.cc +++ b/src/crimson/os/seastore/journal.cc @@ -38,9 +38,11 @@ namespace journal { JournalRef make_segmented( store_index_t store_index, SegmentProvider &provider, - JournalTrimmer &trimmer) + JournalTrimmer &trimmer, + bool scan_alloc_on_boot) { - return std::make_unique(store_index, provider, trimmer); + return std::make_unique( + store_index, provider, trimmer, scan_alloc_on_boot); } JournalRef make_circularbounded( diff --git a/src/crimson/os/seastore/journal.h b/src/crimson/os/seastore/journal.h index 6b757d02fa9..259ed714ab5 100644 --- a/src/crimson/os/seastore/journal.h +++ b/src/crimson/os/seastore/journal.h @@ -132,7 +132,8 @@ namespace journal { JournalRef make_segmented( store_index_t store_index, SegmentProvider &provider, - JournalTrimmer &trimmer); + JournalTrimmer &trimmer, + bool scan_alloc_on_boot); JournalRef make_circularbounded( store_index_t store_index, diff --git a/src/crimson/os/seastore/journal/segmented_journal.cc b/src/crimson/os/seastore/journal/segmented_journal.cc index 130054f0fd1..8557cb51e3d 100644 --- a/src/crimson/os/seastore/journal/segmented_journal.cc +++ b/src/crimson/os/seastore/journal/segmented_journal.cc @@ -30,7 +30,8 @@ namespace crimson::os::seastore::journal { SegmentedJournal::SegmentedJournal( store_index_t store_index, SegmentProvider &segment_provider, - JournalTrimmer &trimmer) + JournalTrimmer &trimmer, + bool scan_alloc_on_startup) : store_index(store_index), segment_seq_allocator( new SegmentSeqAllocator(segment_type_t::JOURNAL)), @@ -49,7 +50,8 @@ SegmentedJournal::SegmentedJournal( "seastore_journal_batch_preferred_fullness"), journal_segment_allocator), sm_group(*segment_provider.get_segment_manager_group()), - trimmer{trimmer} + trimmer{trimmer}, + scan_alloc_on_startup(scan_alloc_on_startup) { } @@ -322,10 +324,19 @@ SegmentedJournal::replay_ret SegmentedJournal::replay( auto segment_headers = co_await sm_group.find_journal_segment_headers(); INFO("got {} segments", segment_headers.size()); co_await prep_replay_segments(std::move(segment_headers)); - auto d_handler = [&handler, this]( + alloc_map_t alloc_map; + if (scan_alloc_on_startup) { + alloc_map = co_await scan_alloc_map(); + } + auto d_handler = [&handler, this, &alloc_map]( const record_locator_t &locator, const delta_info_t &delta, sea_time_point modify_time) -> replay_ertr::future { + if (auto it = alloc_map.find(delta.paddr); + it != alloc_map.end() && + it->second > locator.write_result.start_seq) { + co_return true; + } auto ret = co_await handler( locator, delta, diff --git a/src/crimson/os/seastore/journal/segmented_journal.h b/src/crimson/os/seastore/journal/segmented_journal.h index 2b7135b2711..312fa48fbfc 100644 --- a/src/crimson/os/seastore/journal/segmented_journal.h +++ b/src/crimson/os/seastore/journal/segmented_journal.h @@ -28,7 +28,8 @@ public: SegmentedJournal( store_index_t store_index, SegmentProvider &segment_provider, - JournalTrimmer &trimmer); + JournalTrimmer &trimmer, + bool scan_alloc_on_startup); ~SegmentedJournal() {} JournalTrimmer &get_trimmer() final { @@ -99,6 +100,7 @@ private: std::size_t num_dirty_deltas = 0; } stats; + const bool scan_alloc_on_startup = false; using replay_segments_t = std::vector< std::pair>; replay_segments_t replay_segments; diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index 3fe575e0517..f0fd2a5d222 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -1824,6 +1824,7 @@ TransactionManagerRef make_transaction_manager( JournalRef journal; AsyncCleanerRef cold_cleaner = nullptr; + bool scan_alloc_on_boot = false; if (cold_sms) { assert(!cold_rbs); @@ -1844,6 +1845,7 @@ TransactionManagerRef make_transaction_manager( } cold_cleaner = std::move(segment_cleaner); } else if (cold_rbs) { + scan_alloc_on_boot = true; cold_cleaner = RBMCleaner::create( store_index, std::move(cold_rbs), @@ -1871,7 +1873,8 @@ TransactionManagerRef make_transaction_manager( journal = journal::make_segmented( store_index, *segment_cleaner, - *journal_trimmer); + *journal_trimmer, + scan_alloc_on_boot); } else { cleaner = RBMCleaner::create( store_index, diff --git a/src/test/crimson/seastore/test_btree_lba_manager.cc b/src/test/crimson/seastore/test_btree_lba_manager.cc index 79d54b9c4fe..1973226de4c 100644 --- a/src/test/crimson/seastore/test_btree_lba_manager.cc +++ b/src/test/crimson/seastore/test_btree_lba_manager.cc @@ -138,7 +138,7 @@ struct btree_test_base : segment_manager::get_ephemeral_device_config(0, 1, 0)); }).safe_then([this] { sms.reset(new SegmentManagerGroup()); - journal = journal::make_segmented(0, *this, *this); + journal = journal::make_segmented(0, *this, *this, false); rewrite_gen_t hot_tier_generations = crimson::common::get_conf( "seastore_hot_tier_generations"); rewrite_gen_t cold_tier_generations = crimson::common::get_conf( diff --git a/src/test/crimson/seastore/test_seastore_journal.cc b/src/test/crimson/seastore/test_seastore_journal.cc index 77d57acf8b9..13e738269c4 100644 --- a/src/test/crimson/seastore/test_seastore_journal.cc +++ b/src/test/crimson/seastore/test_seastore_journal.cc @@ -157,7 +157,7 @@ struct journal_test_t : seastar_test_suite_t, SegmentProvider, JournalTrimmer { block_size = segment_manager->get_block_size(); sms.reset(new SegmentManagerGroup()); next = segment_id_t(segment_manager->get_device_id(), 0); - journal = journal::make_segmented(0, *this, *this); + journal = journal::make_segmented(0, *this, *this, false); journal->set_write_pipeline(&pipeline); sms->add_segment_manager(segment_manager.get()); return journal->open_for_mkfs(); @@ -182,7 +182,7 @@ struct journal_test_t : seastar_test_suite_t, SegmentProvider, JournalTrimmer { auto replay(T &&f) { return journal->close( ).safe_then([this, f=std::move(f)]() mutable { - journal = journal::make_segmented(0, *this, *this); + journal = journal::make_segmented(0, *this, *this, false); journal->set_write_pipeline(&pipeline); return journal->replay(std::forward(std::move(f))); }).safe_then([this] { From ba0655657c932dd5362b68cf1cb8bfae66696d12 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Sat, 4 Jul 2026 13:03:28 +0800 Subject: [PATCH 46/51] crimson/os/seastore/transaction: mutation_pending extents' paddrs should be on the same device as the stable prior's old paddr Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/transaction.h | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/crimson/os/seastore/transaction.h b/src/crimson/os/seastore/transaction.h index c256e5b8f99..3f698b41531 100644 --- a/src/crimson/os/seastore/transaction.h +++ b/src/crimson/os/seastore/transaction.h @@ -431,15 +431,18 @@ public: auto &mextent = *i; write_set.erase(mextent); extent_len_t off = 0; - if (new_paddr.is_absolute_segmented()) { - assert(mextent.get_paddr().as_seg_paddr().get_segment_id() - == old_paddr.as_seg_paddr().get_segment_id()); + if (old_paddr.is_absolute_segmented()) { + assert(mextent.get_paddr().as_seg_paddr().get_segment_id() == + old_paddr.as_seg_paddr().get_segment_id()); + assert(mextent.get_paddr().as_seg_paddr().get_segment_off() >= + old_paddr.as_seg_paddr().get_segment_off()); assert(mextent.get_paddr().as_seg_paddr().get_segment_off() - >= old_paddr.as_seg_paddr().get_segment_off()); - off = mextent.get_paddr().as_seg_paddr().get_segment_off() - - old_paddr.as_seg_paddr().get_segment_off(); + + mextent.get_length() <= + old_paddr.as_seg_paddr().get_segment_off() + len); + off = mextent.get_paddr().as_seg_paddr().get_segment_off() - + old_paddr.as_seg_paddr().get_segment_off(); } else { - assert(new_paddr.is_absolute_random_block()); + assert(mextent.get_paddr().is_absolute_random_block()); off = mextent.get_paddr().as_blk_paddr().get_device_off() - old_paddr.as_blk_paddr().get_device_off(); } From 953710c5e61f18a20bd030bdaad042489429a5aa Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Sat, 4 Jul 2026 23:21:23 +0800 Subject: [PATCH 47/51] crimson/os/seastore/transaction_manager: add the shadow to the trans' read_set when getting extents Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/transaction_manager.h | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index 1f7c84f7a33..50aa55d1588 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -1521,8 +1521,19 @@ private: assert(cursor.get_pos() != BTREENODE_POS_NULL); ceph_assert(t.get_trans_id() == cursor.ctx.trans.get_trans_id()); auto p = cursor.parent->cast(); - return p->template get_child( + auto v = p->template get_child( t, cursor.ctx.cache, cursor.get_pos(), cursor.key); + if (v.has_child()) { + return v.get_child_fut_as( + ).si_then([&t](auto lext) { + if (auto shadow = lext->get_shadow();\ + shadow && shadow->is_stable()) { + std::ignore = t.maybe_add_to_read_set(shadow); + } + return lext; + }); + } + return v; } base_iertr::future read_cursor_by_type( From 528bba37b9b383d765df6791506c421f7b544571 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Thu, 9 Jul 2026 13:36:51 +0800 Subject: [PATCH 48/51] crimson/os/seastore/epm: add try_reserve_main and abort_main_usage As the counterpart of try_reserve_cold and abort_cold_usage Signed-off-by: Xuehan Xu --- .../os/seastore/extent_placement_manager.cc | 32 +++++++++++++------ .../os/seastore/extent_placement_manager.h | 2 ++ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc index 9ed94046036..64e9ff1ae3e 100644 --- a/src/crimson/os/seastore/extent_placement_manager.cc +++ b/src/crimson/os/seastore/extent_placement_manager.cc @@ -981,23 +981,35 @@ void ExtentPlacementManager::BackgroundProcess::abort_cold_usage( } } +bool ExtentPlacementManager::BackgroundProcess::try_reserve_main( + std::size_t usage) +{ + return main_cleaner->try_reserve_projected_usage(usage); +} + reserve_cleaner_result_t ExtentPlacementManager::BackgroundProcess::try_reserve_cleaner( const cleaner_usage_t &usage) { return { - main_cleaner->try_reserve_projected_usage(usage.main_usage), + try_reserve_main(usage.main_usage), try_reserve_cold(usage.cold_ool_usage) }; } +void ExtentPlacementManager::BackgroundProcess::abort_main_usage( + std::size_t usage, bool success) +{ + if (success) { + main_cleaner->release_projected_usage(usage); + } +} + void ExtentPlacementManager::BackgroundProcess::abort_cleaner_usage( const cleaner_usage_t &usage, const reserve_cleaner_result_t &result) { - if (result.reserve_main_success) { - main_cleaner->release_projected_usage(usage.main_usage); - } + abort_main_usage(usage.main_usage, result.reserve_main_success); abort_cold_usage(usage.cold_ool_usage, result.reserve_cold_success); } @@ -1184,16 +1196,16 @@ seastar::future<> ExtentPlacementManager::BackgroundProcess::run_promote() return seastar::futurize_invoke([this] { if (pinboard->should_promote()) { - auto usage = cleaner_usage_t{pinboard->get_promotion_size(), 0}; - auto res = try_reserve_cleaner(usage); - if (res.is_successful()) { + auto usage = pinboard->get_promotion_size(); + auto reserved = try_reserve_main(usage); + if (reserved) { return pinboard->promote( - ).finally([this, usage, res] { - abort_cleaner_usage(usage, res); + ).finally([this, usage] { + abort_main_usage(usage, true); }); } else { // reserve usage failed, block - abort_cleaner_usage(usage, res); + abort_main_usage(usage, false); } } // shouldn't promote, block diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index 7f88f426ed1..a6e220a273b 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -1145,7 +1145,9 @@ private: private: // reserve helpers bool try_reserve_cold(std::size_t usage); + bool try_reserve_main(std::size_t usage); void abort_cold_usage(std::size_t usage, bool success); + void abort_main_usage(std::size_t usage, bool success); reserve_cleaner_result_t try_reserve_cleaner(const cleaner_usage_t &usage); void abort_cleaner_usage(const cleaner_usage_t &usage, From e89e2104a8b86ece07e44cff3546ba60d47260a8 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Thu, 9 Jul 2026 14:40:12 +0800 Subject: [PATCH 49/51] crimson/os/seastore/epm: reserve the space in the cold tier before demoting Signed-off-by: Xuehan Xu --- .../os/seastore/extent_placement_manager.cc | 16 ++++++++++++++-- .../os/seastore/extent_placement_manager.h | 5 ++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc index 64e9ff1ae3e..6fca0efb8fd 100644 --- a/src/crimson/os/seastore/extent_placement_manager.cc +++ b/src/crimson/os/seastore/extent_placement_manager.cc @@ -1129,8 +1129,17 @@ ExtentPlacementManager::BackgroundProcess::do_background_cycle() } } + if (proceed_demote && + !try_reserve_cold(logical_bucket_demote_size_per_cycle)) { + abort_cold_usage(logical_bucket_demote_size_per_cycle, false); + proceed_demote = false; + } + if (!proceed_clean_main && !proceed_clean_cold && !proceed_demote) { - ceph_abort_msg("no background process will start"); + // abort when the system is full, following the enospc handling + // in ObjectDataHandler + ceph_abort_msg("no background process will start, " + "this probably means the underlying disks are full"); } return seastar::when_all( [this, FNAME, proceed_clean_main, abort_cold_cleaner_usage, @@ -1178,7 +1187,10 @@ ExtentPlacementManager::BackgroundProcess::do_background_cycle() if (!proceed_demote) { return seastar::now(); } - return logical_bucket->demote(); + return logical_bucket->demote( + ).finally([this] { + abort_cold_usage(logical_bucket_demote_size_per_cycle, true); + }); } ).discard_result(); } diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index a6e220a273b..ee4a8a4a90c 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -926,9 +926,11 @@ private: ceph_assert(pinboard != nullptr); pinboard->set_background_callback(this); + logical_bucket_demote_size_per_cycle = + get_conf("seastore_logical_bucket_proceed_size_per_cycle"); logical_bucket = create_logical_bucket( get_conf("seastore_logical_bucket_capacity"), - get_conf("seastore_logical_bucket_proceed_size_per_cycle")); + logical_bucket_demote_size_per_cycle); logical_bucket->set_background_callback(this); } LOG_PREFIX(BackgroundProcess::init); @@ -1372,6 +1374,7 @@ private: AsyncCleanerRef main_cleaner; ExtentPinboard *pinboard = nullptr; LogicalBucketRef logical_bucket; + std::size_t logical_bucket_demote_size_per_cycle = 0; /* * cold tier (optional, see has_cold_tier()) From 6b6c8d4da3d42a2580aa2fab49a5be05c96e77d7 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Mon, 20 Jul 2026 10:18:12 +0800 Subject: [PATCH 50/51] crimson/os/seastore/transaction_manager: allow non-existing lba regions when demoting them It is possible that, when we do demote_region, the target region has already been removed from the lba tree, for example, a temp recovering object might have already been renamed to the real one, which means all lba mappings within its original region have been moved away. Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/transaction_manager.cc | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index f0fd2a5d222..025a23c7900 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -1579,10 +1579,21 @@ TransactionManager::demote_region( auto cursor = co_await lba_manager->upper_bound_right( t, start ).handle_error_interruptible( + crimson::ct_error::enoent::handle([](auto) { + // It's possible that there has been no lba mappings left + // when demoting a region, for example, a temp recovering + // object may have been renamed, which makes the lba mappings + // in its own region moved. + return seastar::make_ready_future(); + }), demote_region_iertr::pass_further{}, crimson::ct_error::assert_all("unexpected enoent")); - auto it = co_await resolve_cursor_to_mapping(t, std::move(cursor)); demote_region_res_t ret{0, 0, false}; + if (!cursor) { + ret.complete = true; + co_return ret; + } + auto it = co_await resolve_cursor_to_mapping(t, std::move(cursor)); std::vector extents; while ((ret.demoted_size + ret.evicted_size) < max_proceed_size) { if (it.is_end() || it.get_key().get_object_prefix() != prefix) { From b8f9beb9c19155cdffafc511b880308fa7015b18 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Tue, 28 Jul 2026 12:42:59 +0800 Subject: [PATCH 51/51] crimson/os/seastore/transaction_manager: don't demote dirty extents Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/transaction_manager.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index 025a23c7900..4d0803a5859 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -604,6 +604,11 @@ TransactionManager::relocate_shadow_extent( )->template cast(); } else { extent = co_await std::move(v.get_child_fut()); + if (extent->is_stable_dirty()) { + // the extent is dirty, skip it. + DEBUGT("skipping dirty extent: {}", t, *extent); + co_return LogicalChildNodeRef(); + } cache->retire_extent(t, extent); } if (auto shadow = extent->get_shadow(); shadow) { @@ -1607,6 +1612,10 @@ TransactionManager::demote_region( if (it.has_shadow_val()) { DEBUGT("demote shadow {}", t, it); auto extent = co_await relocate_shadow_extent(t, it); + if (!extent) { + DEBUGT("{} can't be demoted", t, it); + continue; + } ret.demoted_size += extent->get_length(); auto cursor = co_await lba_manager->demote_extent( t, *it.direct_cursor, *extent);