mirror of
https://github.com/ceph/ceph
synced 2026-08-01 22:45:39 +00:00
Merge pull request #68503 from ifed01/wip-ifed-fix-ncb-expand
os/bluestore: do not add expanded space to allocator after allocmap recovery. Reviewed-by: Adam Kupczyk <akupczyk@ibm.com>
This commit is contained in:
Igor Fedotov
GitHub
commit
881d8c9144
@ -19,18 +19,14 @@ function run() {
|
||||
function TEST_bluestore() {
|
||||
local dir=$1
|
||||
|
||||
local flimit=$(ulimit -n)
|
||||
if [ $flimit -lt 1536 ]; then
|
||||
echo "Low open file limit ($flimit), test may fail. Increase to 1536 or higher and retry if that happens."
|
||||
fi
|
||||
export CEPH_MON="127.0.0.1:7146" # git grep '\<7146\>' : there must be only one
|
||||
export CEPH_ARGS
|
||||
CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
|
||||
CEPH_ARGS+="--mon-host=$CEPH_MON "
|
||||
CEPH_ARGS+="--bluestore_block_size=2147483648 "
|
||||
CEPH_ARGS+="--bluestore_block_size=2G "
|
||||
CEPH_ARGS+="--bluestore_block_db_create=true "
|
||||
CEPH_ARGS+="--bluestore_block_db_size=1073741824 "
|
||||
CEPH_ARGS+="--bluestore_block_wal_size=1073741824 "
|
||||
CEPH_ARGS+="--bluestore_block_db_size=1G "
|
||||
CEPH_ARGS+="--bluestore_block_wal_size=1G "
|
||||
CEPH_ARGS+="--bluestore_block_wal_create=true "
|
||||
CEPH_ARGS+="--bluestore_fsck_on_mount=true "
|
||||
#choosing randomly allocation from file
|
||||
@ -338,22 +334,20 @@ function TEST_bluestore() {
|
||||
timeout 60 rados bench -p foo 15 write -b 4096 --no-cleanup #|| return 1
|
||||
|
||||
wait_for_clean || return 1
|
||||
echo PASSED
|
||||
return 0
|
||||
}
|
||||
|
||||
function TEST_bluestore2() {
|
||||
local dir=$1
|
||||
|
||||
local flimit=$(ulimit -n)
|
||||
if [ $flimit -lt 1536 ]; then
|
||||
echo "Low open file limit ($flimit), test may fail. Increase to 1536 or higher and retry if that happens."
|
||||
fi
|
||||
export CEPH_MON="127.0.0.1:7146" # git grep '\<7146\>' : there must be only one
|
||||
export CEPH_ARGS
|
||||
CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
|
||||
CEPH_ARGS+="--mon-host=$CEPH_MON "
|
||||
CEPH_ARGS+="--bluestore_block_size=4294967296 "
|
||||
CEPH_ARGS+="--bluestore_block_size=4G "
|
||||
CEPH_ARGS+="--bluestore_block_db_create=true "
|
||||
CEPH_ARGS+="--bluestore_block_db_size=1073741824 "
|
||||
CEPH_ARGS+="--bluestore_block_db_size=1G "
|
||||
CEPH_ARGS+="--bluestore_block_wal_create=false "
|
||||
CEPH_ARGS+="--bluestore_fsck_on_mount=true "
|
||||
CEPH_ARGS+="--osd_pool_default_size=1 "
|
||||
@ -401,22 +395,21 @@ function TEST_bluestore2() {
|
||||
osd_pid0=$(cat $dir/osd.0.pid)
|
||||
|
||||
wait_for_clean || return 1
|
||||
|
||||
echo PASSED
|
||||
return 0
|
||||
}
|
||||
|
||||
function TEST_bluestore_expand() {
|
||||
local dir=$1
|
||||
|
||||
local flimit=$(ulimit -n)
|
||||
if [ $flimit -lt 1536 ]; then
|
||||
echo "Low open file limit ($flimit), test may fail. Increase to 1536 or higher and retry if that happens."
|
||||
fi
|
||||
export CEPH_MON="127.0.0.1:7146" # git grep '\<7146\>' : there must be only one
|
||||
export CEPH_ARGS
|
||||
CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
|
||||
CEPH_ARGS+="--mon-host=$CEPH_MON "
|
||||
CEPH_ARGS+="--bluestore_block_size=4294967296 "
|
||||
CEPH_ARGS+="--bluestore_block_size=4G "
|
||||
CEPH_ARGS+="--bluestore_block_db_create=true "
|
||||
CEPH_ARGS+="--bluestore_block_db_size=1073741824 "
|
||||
CEPH_ARGS+="--bluestore_block_db_size=1G "
|
||||
CEPH_ARGS+="--bluestore_block_wal_create=false "
|
||||
CEPH_ARGS+="--bluestore_fsck_on_mount=true "
|
||||
CEPH_ARGS+="--osd_pool_default_size=1 "
|
||||
@ -485,15 +478,14 @@ function TEST_bluestore_expand() {
|
||||
ceph osd down 0
|
||||
|
||||
ceph-bluestore-tool --log-file $dir/bluestore_tool.log --path $dir/0 qfsck || return 1
|
||||
|
||||
echo PASSED
|
||||
return 0
|
||||
}
|
||||
|
||||
function TEST_bluestore_expand_online() {
|
||||
local dir=$1
|
||||
|
||||
local flimit=$(ulimit -n)
|
||||
if [ $flimit -lt 1536 ]; then
|
||||
echo "Low open file limit ($flimit), test may fail. Increase to 1536 or higher and retry if that happens."
|
||||
fi
|
||||
export CEPH_MON="127.0.0.1:7146" # git grep '\<7146\>' : there must be only one
|
||||
export CEPH_ARGS
|
||||
CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
|
||||
@ -571,9 +563,89 @@ function TEST_bluestore_expand_online() {
|
||||
ceph-bluestore-tool --path $dir/1 fsck || return 1
|
||||
ceph-bluestore-tool --path $dir/2 fsck || return 1
|
||||
ceph-bluestore-tool --path $dir/3 fsck || return 1
|
||||
|
||||
}
|
||||
|
||||
function TEST_bluestore_expand_with_allocmap_recovery() {
|
||||
local dir=$1
|
||||
|
||||
export CEPH_MON="127.0.0.1:7146" # git grep '\<7146\>' : there must be only one
|
||||
export CEPH_ARGS
|
||||
CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
|
||||
CEPH_ARGS+="--mon-host=$CEPH_MON "
|
||||
CEPH_ARGS+="--bluestore_block_size=2G "
|
||||
CEPH_ARGS+="--bluestore_block_db_create=true "
|
||||
CEPH_ARGS+="--bluestore_block_db_size=1G "
|
||||
CEPH_ARGS+="--bluestore_block_wal_size=1G "
|
||||
CEPH_ARGS+="--bluestore_block_wal_create=true "
|
||||
CEPH_ARGS+="--bluestore_fsck_on_mount=true "
|
||||
CEPH_ARGS+="--osd_pool_default_size=1 "
|
||||
CEPH_ARGS+="--osd_pool_default_min_size=1 "
|
||||
CEPH_ARGS+="--bluestore_debug_enforce_settings=ssd "
|
||||
CEPH_ARGS+="--bluestore_allocation_from_file=1 "
|
||||
|
||||
run_mon $dir a || return 1
|
||||
run_mgr $dir x || return 1
|
||||
run_osd $dir 0 || return 1
|
||||
osd_pid0=$(cat $dir/osd.0.pid)
|
||||
|
||||
run_osd $dir 1 || return 1
|
||||
osd_pid1=$(cat $dir/osd.1.pid)
|
||||
|
||||
sleep 5
|
||||
|
||||
create_pool foo 16
|
||||
|
||||
# write some objects
|
||||
timeout 60 rados bench -p foo 5 write -b 4096 --no-cleanup #|| return 1
|
||||
|
||||
echo "after bench"
|
||||
|
||||
# kill OSD.0 non-gracefully
|
||||
while kill -9 $osd_pid0; do sleep 1 ; done
|
||||
ceph osd down 0
|
||||
|
||||
# kill OSD.1 [hopefully] gracefully
|
||||
while kill $osd_pid1; do sleep 1 ; done
|
||||
ceph osd down 1
|
||||
|
||||
available0=$( ceph-objectstore-tool --data-path $dir/0 --op statfs --no-mon-config --format json-pretty | jq ".available" )
|
||||
truncate $dir/0/block -s +2G # 4GB total
|
||||
|
||||
ceph-bluestore-tool --path $dir/0 bluefs-bdev-expand || return 1
|
||||
|
||||
available=$( ceph-objectstore-tool --data-path $dir/0 --op statfs --no-mon-config --format json-pretty | jq ".available" )
|
||||
if [ $(($available-$available0)) -le $((1<<30)) ]; then
|
||||
# Failed to get large enough available space increase
|
||||
echo $available0 " -> " $available ", failed to expand?"
|
||||
return 1
|
||||
fi
|
||||
|
||||
available0=$( ceph-objectstore-tool --data-path $dir/1 --op statfs --no-mon-config --format json-pretty | jq ".available" )
|
||||
truncate $dir/1/block -s +2G # 4GB total
|
||||
|
||||
ceph-bluestore-tool --path $dir/1 bluefs-bdev-expand || return 1
|
||||
|
||||
available=$( ceph-objectstore-tool --data-path $dir/1 --op statfs --no-mon-config --format json-pretty | jq ".available" )
|
||||
if [ $(($available-$available0)) -le $((1<<30)) ]; then
|
||||
# Failed to get large enough available space increase
|
||||
echo $available0 " -> " $available ", failed to expand?"
|
||||
return 1
|
||||
fi
|
||||
|
||||
ceph-bluestore-tool --path $dir/0 fsck || return 1
|
||||
ceph-bluestore-tool --path $dir/1 fsck || return 1
|
||||
|
||||
activate_osd $dir 0 || return 1
|
||||
osd_pid0=$(cat $dir/osd.0.pid)
|
||||
|
||||
activate_osd $dir 1 || return 1
|
||||
osd_pid1=$(cat $dir/osd.1.pid)
|
||||
|
||||
wait_for_clean || return 1
|
||||
|
||||
echo PASSED
|
||||
return 0
|
||||
}
|
||||
main osd-bluefs-volume-ops "$@"
|
||||
|
||||
# Local Variables:
|
||||
|
||||
@ -5874,13 +5874,15 @@ options:
|
||||
type: str
|
||||
level: dev
|
||||
desc: Enforces specific hardware profile settings
|
||||
long_desc: '''hdd'' enforces settings intended for BlueStore on a rotational
|
||||
drive. ''ssd'' enforces settings intended for BlueStore on an SSD. ''default''
|
||||
indicates that BlueStore is to use settings based on the detected hardware.'
|
||||
long_desc: '''hybrid'' enforces osd settings applicable to hybrid
|
||||
(AKA main=hdd, db/wal=ssd) disk setup.
|
||||
''ssd'' enforces osd settings applicable to all-flash disk setup.
|
||||
''default'' specifies automatic settings selection based on real
|
||||
detected hardware.'
|
||||
default: default
|
||||
enum_values:
|
||||
- default
|
||||
- hdd
|
||||
- hybrid
|
||||
- ssd
|
||||
with_legacy: true
|
||||
- name: bluestore_allocator_lookup_policy
|
||||
|
||||
@ -903,6 +903,7 @@ public:
|
||||
int unlink(std::string_view dirname, std::string_view filename);
|
||||
int mkdir(std::string_view dirname);
|
||||
int rmdir(std::string_view dirname);
|
||||
|
||||
bool wal_is_rotational();
|
||||
bool db_is_rotational();
|
||||
|
||||
|
||||
@ -7534,6 +7534,13 @@ int BlueStore::_init_alloc()
|
||||
}
|
||||
if (restore_allocator(alloc, &num, &bytes) == 0) {
|
||||
dout(5) << __func__ << "::NCB::restore_allocator() completed successfully alloc=" << alloc << dendl;
|
||||
if (before_expansion_bdev_size > 0 &&
|
||||
before_expansion_bdev_size < bdev_label.size) {
|
||||
// we grow the allocation range, must reflect it in the allocation file
|
||||
alloc->init_add_free(before_expansion_bdev_size,
|
||||
bdev_label.size - before_expansion_bdev_size);
|
||||
need_to_destage_allocation_file = true;
|
||||
}
|
||||
} else {
|
||||
// This must mean that we had an unplanned shutdown and didn't manage to destage the allocator
|
||||
dout(0) << __func__ << "::NCB::restore_allocator() failed! Run Full Recovery from ONodes (might take a while) ..." << dendl;
|
||||
@ -7545,13 +7552,6 @@ int BlueStore::_init_alloc()
|
||||
return -ENOTRECOVERABLE;
|
||||
}
|
||||
}
|
||||
if (before_expansion_bdev_size > 0 &&
|
||||
before_expansion_bdev_size < bdev_label.size) {
|
||||
// we grow the allocation range, must reflect it in the allocation file
|
||||
alloc->init_add_free(before_expansion_bdev_size,
|
||||
bdev_label.size - before_expansion_bdev_size);
|
||||
need_to_destage_allocation_file = true;
|
||||
}
|
||||
}
|
||||
before_expansion_bdev_size = 0;
|
||||
|
||||
@ -7681,12 +7681,13 @@ int BlueStore::_lock_fsid()
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool BlueStore::is_rotational()
|
||||
bool BlueStore::_is_main_rotational()
|
||||
{
|
||||
if (bdev) {
|
||||
return bdev->is_rotational();
|
||||
}
|
||||
|
||||
// Do open main devicet if not yet opened.
|
||||
bool rotational = true;
|
||||
int r = _open_path();
|
||||
if (r < 0)
|
||||
@ -7713,37 +7714,58 @@ bool BlueStore::is_rotational()
|
||||
return rotational;
|
||||
}
|
||||
|
||||
bool BlueStore::is_rotational()
|
||||
{
|
||||
return _use_rotational_settings();
|
||||
}
|
||||
|
||||
bool BlueStore::is_journal_rotational()
|
||||
{
|
||||
if (!bluefs) {
|
||||
dout(5) << __func__ << " bluefs disabled, default to store media type"
|
||||
dout(5) << __func__ << " bluefs disabled, default to main device type"
|
||||
<< dendl;
|
||||
return is_rotational();
|
||||
return _use_rotational_settings();
|
||||
}
|
||||
dout(10) << __func__ << " " << (int)bluefs->wal_is_rotational() << dendl;
|
||||
return bluefs->wal_is_rotational();
|
||||
if (cct->_conf->bluestore_debug_enforce_settings == "ssd" ||
|
||||
cct->_conf->bluestore_debug_enforce_settings == "hybrid") {
|
||||
dout(10) << __func__ << " overriden to ssd mode." << dendl;
|
||||
return false;
|
||||
}
|
||||
bool r = bluefs->wal_is_rotational();
|
||||
dout(10) << __func__ << " " << (int)r << dendl;
|
||||
return r;
|
||||
}
|
||||
|
||||
bool BlueStore::is_db_rotational()
|
||||
{
|
||||
if (!bluefs) {
|
||||
dout(5) << __func__ << " bluefs disabled, default to store media type"
|
||||
dout(5) << __func__ << " bluefs disabled, default to main device type"
|
||||
<< dendl;
|
||||
return is_rotational();
|
||||
return _use_rotational_settings();
|
||||
}
|
||||
dout(10) << __func__ << " " << (int)bluefs->db_is_rotational() << dendl;
|
||||
return bluefs->db_is_rotational();
|
||||
if (cct->_conf->bluestore_debug_enforce_settings == "ssd" ||
|
||||
cct->_conf->bluestore_debug_enforce_settings == "hybrid") {
|
||||
dout(10) << __func__ << " overriden to ssd mode." << dendl;
|
||||
return false;
|
||||
}
|
||||
bool r = bluefs->db_is_rotational();
|
||||
dout(10) << __func__ << " " << (int)r << dendl;
|
||||
return r;
|
||||
}
|
||||
|
||||
bool BlueStore::_use_rotational_settings()
|
||||
{
|
||||
if (cct->_conf->bluestore_debug_enforce_settings == "hdd") {
|
||||
return true;
|
||||
}
|
||||
if (cct->_conf->bluestore_debug_enforce_settings == "ssd") {
|
||||
dout(10) << __func__ << " overriden to ssd." << dendl;
|
||||
return false;
|
||||
}
|
||||
return bdev->is_rotational();
|
||||
if (cct->_conf->bluestore_debug_enforce_settings == "hybrid") {
|
||||
dout(10) << __func__ << " overriden to hdd." << dendl;
|
||||
return true;
|
||||
}
|
||||
bool r = _is_main_rotational();
|
||||
dout(0) << __func__ << " returns " << r << dendl;
|
||||
return r;
|
||||
}
|
||||
|
||||
bool BlueStore::is_statfs_recoverable() const
|
||||
|
||||
@ -4048,6 +4048,7 @@ private:
|
||||
std::array<std::tuple<uint64_t, uint64_t, uint64_t>, 5> alloc_stats_history =
|
||||
{ std::make_tuple(0ul, 0ul, 0ul) };
|
||||
|
||||
bool _is_main_rotational();
|
||||
inline bool _use_rotational_settings();
|
||||
|
||||
public:
|
||||
|
||||
@ -1685,7 +1685,7 @@ TEST_P(StoreTestSpecificAUSize, ReproBug41901Test) {
|
||||
|
||||
SetVal(g_conf(), "bluestore_write_v2", "false");
|
||||
SetVal(g_conf(), "bluestore_max_blob_size", "524288");
|
||||
SetVal(g_conf(), "bluestore_debug_enforce_settings", "hdd");
|
||||
SetVal(g_conf(), "bluestore_debug_enforce_settings", "hybrid");
|
||||
g_conf().apply_changes(nullptr);
|
||||
StartDeferred(65536);
|
||||
|
||||
@ -10036,7 +10036,7 @@ TEST_P(StoreTestSpecificAUSize, ReproBug56488Test) {
|
||||
size_t alloc_size = 65536;
|
||||
size_t write_size = 4096;
|
||||
SetVal(g_conf(), "bluestore_write_v2", "false");
|
||||
SetVal(g_conf(), "bluestore_debug_enforce_settings", "hdd");
|
||||
SetVal(g_conf(), "bluestore_debug_enforce_settings", "hybrid");
|
||||
SetVal(g_conf(), "bluestore_block_db_create", "true");
|
||||
SetVal(g_conf(), "bluestore_block_db_size", stringify(1 << 30).c_str());
|
||||
|
||||
@ -11858,7 +11858,7 @@ TEST_P(StoreTestSpecificAUSize, BluestoreEnforceHWSettingsHdd) {
|
||||
return;
|
||||
|
||||
SetVal(g_conf(), "bluestore_write_v2", "false");
|
||||
SetVal(g_conf(), "bluestore_debug_enforce_settings", "hdd");
|
||||
SetVal(g_conf(), "bluestore_debug_enforce_settings", "hybrid");
|
||||
StartDeferred(0x1000);
|
||||
|
||||
int r;
|
||||
@ -12216,7 +12216,7 @@ TEST_P(StoreTestSpecificAUSize, Ticket45195Repro) {
|
||||
|
||||
SetVal(g_conf(), "bluestore_default_buffered_write", "true");
|
||||
SetVal(g_conf(), "bluestore_max_blob_size", "65536");
|
||||
SetVal(g_conf(), "bluestore_debug_enforce_settings", "hdd");
|
||||
SetVal(g_conf(), "bluestore_debug_enforce_settings", "hybrid");
|
||||
SetVal(g_conf(), "bluestore_fsck_on_mount", "false");
|
||||
g_conf().apply_changes(nullptr);
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user
