From 4b74d0af8fb968d3cd0a777da78e88288ea60b41 Mon Sep 17 00:00:00 2001 From: Igor Fedotov Date: Mon, 20 Apr 2026 23:06:51 +0300 Subject: [PATCH 1/4] os/bluestore: make 'bluestore_debug_enforce_settings' control rotational/non-rotational OSD behavior for DB/WAL devices. Prior to this commit this parameter determined what device class settings to apply for BlueStore only. OSD had partially applied actual journal/db/main device class for some operations, e.g. benchmark on init or allocmap persistence, irrespective to this setting value. This commit allows 'bluestore_debug_enforce_settings" to override the actual device class completely. Signed-off-by: Igor Fedotov --- src/common/options/global.yaml.in | 10 ++++--- src/os/bluestore/BlueFS.h | 1 + src/os/bluestore/BlueStore.cc | 48 ++++++++++++++++++++++-------- src/os/bluestore/BlueStore.h | 1 + src/test/objectstore/store_test.cc | 8 ++--- 5 files changed, 47 insertions(+), 21 deletions(-) diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in index 59317a8d6b2..35b434450a9 100644 --- a/src/common/options/global.yaml.in +++ b/src/common/options/global.yaml.in @@ -5874,13 +5874,15 @@ options: type: str level: dev desc: Enforces specific hardware profile settings - long_desc: '''hdd'' enforces settings intended for BlueStore on a rotational - drive. ''ssd'' enforces settings intended for BlueStore on an SSD. ''default'' - indicates that BlueStore is to use settings based on the detected hardware.' + long_desc: '''hybrid'' enforces osd settings applicable to hybrid + (AKA main=hdd, db/wal=ssd) disk setup. + ''ssd'' enforces osd settings applicable to all-flash disk setup. + ''default'' specifies automatic settings selection based on real + detected hardware.' default: default enum_values: - default - - hdd + - hybrid - ssd with_legacy: true - name: bluestore_allocator_lookup_policy diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h index 84b9bac6611..eebd70a0ba0 100644 --- a/src/os/bluestore/BlueFS.h +++ b/src/os/bluestore/BlueFS.h @@ -903,6 +903,7 @@ public: int unlink(std::string_view dirname, std::string_view filename); int mkdir(std::string_view dirname); int rmdir(std::string_view dirname); + bool wal_is_rotational(); bool db_is_rotational(); diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index d2afa093e52..a5aa845fdec 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -7681,12 +7681,13 @@ int BlueStore::_lock_fsid() return 0; } -bool BlueStore::is_rotational() +bool BlueStore::_is_main_rotational() { if (bdev) { return bdev->is_rotational(); } + // Do open main devicet if not yet opened. bool rotational = true; int r = _open_path(); if (r < 0) @@ -7713,37 +7714,58 @@ bool BlueStore::is_rotational() return rotational; } +bool BlueStore::is_rotational() +{ + return _use_rotational_settings(); +} + bool BlueStore::is_journal_rotational() { if (!bluefs) { - dout(5) << __func__ << " bluefs disabled, default to store media type" + dout(5) << __func__ << " bluefs disabled, default to main device type" << dendl; - return is_rotational(); + return _use_rotational_settings(); } - dout(10) << __func__ << " " << (int)bluefs->wal_is_rotational() << dendl; - return bluefs->wal_is_rotational(); + if (cct->_conf->bluestore_debug_enforce_settings == "ssd" || + cct->_conf->bluestore_debug_enforce_settings == "hybrid") { + dout(10) << __func__ << " overriden to ssd mode." << dendl; + return false; + } + bool r = bluefs->wal_is_rotational(); + dout(10) << __func__ << " " << (int)r << dendl; + return r; } bool BlueStore::is_db_rotational() { if (!bluefs) { - dout(5) << __func__ << " bluefs disabled, default to store media type" + dout(5) << __func__ << " bluefs disabled, default to main device type" << dendl; - return is_rotational(); + return _use_rotational_settings(); } - dout(10) << __func__ << " " << (int)bluefs->db_is_rotational() << dendl; - return bluefs->db_is_rotational(); + if (cct->_conf->bluestore_debug_enforce_settings == "ssd" || + cct->_conf->bluestore_debug_enforce_settings == "hybrid") { + dout(10) << __func__ << " overriden to ssd mode." << dendl; + return false; + } + bool r = bluefs->db_is_rotational(); + dout(10) << __func__ << " " << (int)r << dendl; + return r; } bool BlueStore::_use_rotational_settings() { - if (cct->_conf->bluestore_debug_enforce_settings == "hdd") { - return true; - } if (cct->_conf->bluestore_debug_enforce_settings == "ssd") { + dout(10) << __func__ << " overriden to ssd." << dendl; return false; } - return bdev->is_rotational(); + if (cct->_conf->bluestore_debug_enforce_settings == "hybrid") { + dout(10) << __func__ << " overriden to hdd." << dendl; + return true; + } + bool r = _is_main_rotational(); + dout(0) << __func__ << " returns " << r << dendl; + return r; } bool BlueStore::is_statfs_recoverable() const diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index c4ef3c0e9e6..7a10dcde001 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -4048,6 +4048,7 @@ private: std::array, 5> alloc_stats_history = { std::make_tuple(0ul, 0ul, 0ul) }; + bool _is_main_rotational(); inline bool _use_rotational_settings(); public: diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc index 655a9d25a0c..5ba86a13e05 100644 --- a/src/test/objectstore/store_test.cc +++ b/src/test/objectstore/store_test.cc @@ -1685,7 +1685,7 @@ TEST_P(StoreTestSpecificAUSize, ReproBug41901Test) { SetVal(g_conf(), "bluestore_write_v2", "false"); SetVal(g_conf(), "bluestore_max_blob_size", "524288"); - SetVal(g_conf(), "bluestore_debug_enforce_settings", "hdd"); + SetVal(g_conf(), "bluestore_debug_enforce_settings", "hybrid"); g_conf().apply_changes(nullptr); StartDeferred(65536); @@ -10036,7 +10036,7 @@ TEST_P(StoreTestSpecificAUSize, ReproBug56488Test) { size_t alloc_size = 65536; size_t write_size = 4096; SetVal(g_conf(), "bluestore_write_v2", "false"); - SetVal(g_conf(), "bluestore_debug_enforce_settings", "hdd"); + SetVal(g_conf(), "bluestore_debug_enforce_settings", "hybrid"); SetVal(g_conf(), "bluestore_block_db_create", "true"); SetVal(g_conf(), "bluestore_block_db_size", stringify(1 << 30).c_str()); @@ -11858,7 +11858,7 @@ TEST_P(StoreTestSpecificAUSize, BluestoreEnforceHWSettingsHdd) { return; SetVal(g_conf(), "bluestore_write_v2", "false"); - SetVal(g_conf(), "bluestore_debug_enforce_settings", "hdd"); + SetVal(g_conf(), "bluestore_debug_enforce_settings", "hybrid"); StartDeferred(0x1000); int r; @@ -12216,7 +12216,7 @@ TEST_P(StoreTestSpecificAUSize, Ticket45195Repro) { SetVal(g_conf(), "bluestore_default_buffered_write", "true"); SetVal(g_conf(), "bluestore_max_blob_size", "65536"); - SetVal(g_conf(), "bluestore_debug_enforce_settings", "hdd"); + SetVal(g_conf(), "bluestore_debug_enforce_settings", "hybrid"); SetVal(g_conf(), "bluestore_fsck_on_mount", "false"); g_conf().apply_changes(nullptr); From d6e021d76b895d8fd924ca3cd1b3db65548484f9 Mon Sep 17 00:00:00 2001 From: Igor Fedotov Date: Tue, 21 Apr 2026 15:21:07 +0300 Subject: [PATCH 2/4] os/bluestore: do not add expanded space to allocator after allocmap recovery. The recovery procedure performs that itself by design. Fixes: https://tracker.ceph.com/issues/75852 Signed-off-by: Igor Fedotov --- src/os/bluestore/BlueStore.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index a5aa845fdec..5081bbc11a3 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -7534,6 +7534,13 @@ int BlueStore::_init_alloc() } if (restore_allocator(alloc, &num, &bytes) == 0) { dout(5) << __func__ << "::NCB::restore_allocator() completed successfully alloc=" << alloc << dendl; + if (before_expansion_bdev_size > 0 && + before_expansion_bdev_size < bdev_label.size) { + // we grow the allocation range, must reflect it in the allocation file + alloc->init_add_free(before_expansion_bdev_size, + bdev_label.size - before_expansion_bdev_size); + need_to_destage_allocation_file = true; + } } else { // This must mean that we had an unplanned shutdown and didn't manage to destage the allocator dout(0) << __func__ << "::NCB::restore_allocator() failed! Run Full Recovery from ONodes (might take a while) ..." << dendl; @@ -7545,13 +7552,6 @@ int BlueStore::_init_alloc() return -ENOTRECOVERABLE; } } - if (before_expansion_bdev_size > 0 && - before_expansion_bdev_size < bdev_label.size) { - // we grow the allocation range, must reflect it in the allocation file - alloc->init_add_free(before_expansion_bdev_size, - bdev_label.size - before_expansion_bdev_size); - need_to_destage_allocation_file = true; - } } before_expansion_bdev_size = 0; From f88927d283b08df6f1a13c547ef693ab074a6e59 Mon Sep 17 00:00:00 2001 From: Igor Fedotov Date: Tue, 21 Apr 2026 15:22:43 +0300 Subject: [PATCH 3/4] qa/osd-bluefs-volume-ops: add more bluestore expansion test cases. Reproduces: https://tracker.ceph.com/issues/75852 Signed-off-by: Igor Fedotov --- qa/standalone/osd/osd-bluefs-volume-ops.sh | 94 ++++++++++++++++++++-- 1 file changed, 89 insertions(+), 5 deletions(-) diff --git a/qa/standalone/osd/osd-bluefs-volume-ops.sh b/qa/standalone/osd/osd-bluefs-volume-ops.sh index 8f4f14cf14a..05242c20b5f 100755 --- a/qa/standalone/osd/osd-bluefs-volume-ops.sh +++ b/qa/standalone/osd/osd-bluefs-volume-ops.sh @@ -338,6 +338,8 @@ function TEST_bluestore() { timeout 60 rados bench -p foo 15 write -b 4096 --no-cleanup #|| return 1 wait_for_clean || return 1 + echo PASSED + return 0 } function TEST_bluestore2() { @@ -401,6 +403,9 @@ function TEST_bluestore2() { osd_pid0=$(cat $dir/osd.0.pid) wait_for_clean || return 1 + + echo PASSED + return 0 } function TEST_bluestore_expand() { @@ -485,15 +490,14 @@ function TEST_bluestore_expand() { ceph osd down 0 ceph-bluestore-tool --log-file $dir/bluestore_tool.log --path $dir/0 qfsck || return 1 + + echo PASSED + return 0 } function TEST_bluestore_expand_online() { local dir=$1 - local flimit=$(ulimit -n) - if [ $flimit -lt 1536 ]; then - echo "Low open file limit ($flimit), test may fail. Increase to 1536 or higher and retry if that happens." - fi export CEPH_MON="127.0.0.1:7146" # git grep '\<7146\>' : there must be only one export CEPH_ARGS CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " @@ -571,9 +575,89 @@ function TEST_bluestore_expand_online() { ceph-bluestore-tool --path $dir/1 fsck || return 1 ceph-bluestore-tool --path $dir/2 fsck || return 1 ceph-bluestore-tool --path $dir/3 fsck || return 1 - } +function TEST_bluestore_expand_with_allocmap_recovery() { + local dir=$1 + + export CEPH_MON="127.0.0.1:7146" # git grep '\<7146\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + CEPH_ARGS+="--bluestore_block_size=2G " + CEPH_ARGS+="--bluestore_block_db_create=true " + CEPH_ARGS+="--bluestore_block_db_size=1G " + CEPH_ARGS+="--bluestore_block_wal_size=1G " + CEPH_ARGS+="--bluestore_block_wal_create=true " + CEPH_ARGS+="--bluestore_fsck_on_mount=true " + CEPH_ARGS+="--osd_pool_default_size=1 " + CEPH_ARGS+="--osd_pool_default_min_size=1 " + CEPH_ARGS+="--bluestore_debug_enforce_settings=ssd " + CEPH_ARGS+="--bluestore_allocation_from_file=1 " + + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + osd_pid0=$(cat $dir/osd.0.pid) + + run_osd $dir 1 || return 1 + osd_pid1=$(cat $dir/osd.1.pid) + + sleep 5 + + create_pool foo 16 + + # write some objects + timeout 60 rados bench -p foo 5 write -b 4096 --no-cleanup #|| return 1 + + echo "after bench" + + # kill OSD.0 non-gracefully + while kill -9 $osd_pid0; do sleep 1 ; done + ceph osd down 0 + + # kill OSD.1 [hopefully] gracefully + while kill $osd_pid1; do sleep 1 ; done + ceph osd down 1 + + available0=$( ceph-objectstore-tool --data-path $dir/0 --op statfs --no-mon-config --format json-pretty | jq ".available" ) + truncate $dir/0/block -s +2G # 4GB total + + ceph-bluestore-tool --path $dir/0 bluefs-bdev-expand || return 1 + + available=$( ceph-objectstore-tool --data-path $dir/0 --op statfs --no-mon-config --format json-pretty | jq ".available" ) + if [ $(($available-$available0)) -le $((1<<30)) ]; then + # Failed to get large enough available space increase + echo $available0 " -> " $available ", failed to expand?" + return 1 + fi + + available0=$( ceph-objectstore-tool --data-path $dir/1 --op statfs --no-mon-config --format json-pretty | jq ".available" ) + truncate $dir/1/block -s +2G # 4GB total + + ceph-bluestore-tool --path $dir/1 bluefs-bdev-expand || return 1 + + available=$( ceph-objectstore-tool --data-path $dir/1 --op statfs --no-mon-config --format json-pretty | jq ".available" ) + if [ $(($available-$available0)) -le $((1<<30)) ]; then + # Failed to get large enough available space increase + echo $available0 " -> " $available ", failed to expand?" + return 1 + fi + + ceph-bluestore-tool --path $dir/0 fsck || return 1 + ceph-bluestore-tool --path $dir/1 fsck || return 1 + + activate_osd $dir 0 || return 1 + osd_pid0=$(cat $dir/osd.0.pid) + + activate_osd $dir 1 || return 1 + osd_pid1=$(cat $dir/osd.1.pid) + + wait_for_clean || return 1 + + echo PASSED + return 0 +} main osd-bluefs-volume-ops "$@" # Local Variables: From 641038fb0e132997de0b7cd052435cdd3171f037 Mon Sep 17 00:00:00 2001 From: Igor Fedotov Date: Wed, 20 May 2026 16:27:30 +0300 Subject: [PATCH 4/4] test/osd-bluefs-volume-ops: minor cleanup Signed-off-by: Igor Fedotov --- qa/standalone/osd/osd-bluefs-volume-ops.sh | 26 ++++++---------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/qa/standalone/osd/osd-bluefs-volume-ops.sh b/qa/standalone/osd/osd-bluefs-volume-ops.sh index 05242c20b5f..5893457a5e3 100755 --- a/qa/standalone/osd/osd-bluefs-volume-ops.sh +++ b/qa/standalone/osd/osd-bluefs-volume-ops.sh @@ -19,18 +19,14 @@ function run() { function TEST_bluestore() { local dir=$1 - local flimit=$(ulimit -n) - if [ $flimit -lt 1536 ]; then - echo "Low open file limit ($flimit), test may fail. Increase to 1536 or higher and retry if that happens." - fi export CEPH_MON="127.0.0.1:7146" # git grep '\<7146\>' : there must be only one export CEPH_ARGS CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " CEPH_ARGS+="--mon-host=$CEPH_MON " - CEPH_ARGS+="--bluestore_block_size=2147483648 " + CEPH_ARGS+="--bluestore_block_size=2G " CEPH_ARGS+="--bluestore_block_db_create=true " - CEPH_ARGS+="--bluestore_block_db_size=1073741824 " - CEPH_ARGS+="--bluestore_block_wal_size=1073741824 " + CEPH_ARGS+="--bluestore_block_db_size=1G " + CEPH_ARGS+="--bluestore_block_wal_size=1G " CEPH_ARGS+="--bluestore_block_wal_create=true " CEPH_ARGS+="--bluestore_fsck_on_mount=true " #choosing randomly allocation from file @@ -345,17 +341,13 @@ function TEST_bluestore() { function TEST_bluestore2() { local dir=$1 - local flimit=$(ulimit -n) - if [ $flimit -lt 1536 ]; then - echo "Low open file limit ($flimit), test may fail. Increase to 1536 or higher and retry if that happens." - fi export CEPH_MON="127.0.0.1:7146" # git grep '\<7146\>' : there must be only one export CEPH_ARGS CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " CEPH_ARGS+="--mon-host=$CEPH_MON " - CEPH_ARGS+="--bluestore_block_size=4294967296 " + CEPH_ARGS+="--bluestore_block_size=4G " CEPH_ARGS+="--bluestore_block_db_create=true " - CEPH_ARGS+="--bluestore_block_db_size=1073741824 " + CEPH_ARGS+="--bluestore_block_db_size=1G " CEPH_ARGS+="--bluestore_block_wal_create=false " CEPH_ARGS+="--bluestore_fsck_on_mount=true " CEPH_ARGS+="--osd_pool_default_size=1 " @@ -411,17 +403,13 @@ function TEST_bluestore2() { function TEST_bluestore_expand() { local dir=$1 - local flimit=$(ulimit -n) - if [ $flimit -lt 1536 ]; then - echo "Low open file limit ($flimit), test may fail. Increase to 1536 or higher and retry if that happens." - fi export CEPH_MON="127.0.0.1:7146" # git grep '\<7146\>' : there must be only one export CEPH_ARGS CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " CEPH_ARGS+="--mon-host=$CEPH_MON " - CEPH_ARGS+="--bluestore_block_size=4294967296 " + CEPH_ARGS+="--bluestore_block_size=4G " CEPH_ARGS+="--bluestore_block_db_create=true " - CEPH_ARGS+="--bluestore_block_db_size=1073741824 " + CEPH_ARGS+="--bluestore_block_db_size=1G " CEPH_ARGS+="--bluestore_block_wal_create=false " CEPH_ARGS+="--bluestore_fsck_on_mount=true " CEPH_ARGS+="--osd_pool_default_size=1 "