gian/ceph-mirror
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2026-08-01 22:45:39 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge PR #68813 into main

Browse Source 
* refs/pull/68813/head:
	tools/cephfs: make journal pointer/header failure explicit
	doc/cephfs: add note explainin the new batching technique
	tools/cephfs: document --max-rss usage in first-damage.py
	qa: test cephfs-journal-recovery event recover_dentries summary
	tools/cephfs: flush every journal event as soon as it is scanned

Reviewed-by: Venky Shankar <vshankar@redhat.com>
Reviewed-by: Patrick Donnelly <pdonnell@ibm.com>
This commit is contained in:
Venky Shankar 2026-07-16 11:43:24 +05:30
commit 7d65abc9c1
8 changed files with 270 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
doc/cephfs/cephfs-journal-tool.rst
View File

@ -150,6 +150,8 @@ Filtering:
* ``--dname <string>`` only include events referring to this named dentry within a directory
fragment (may only be used in conjunction with ``--frag``)
* ``--client <int>`` only include events from this client session ID
* ``--max-rss <bytes>`` (works only with ``recover_dentries`` for now) limits the RSS of the cephfs-journal-tool
by scanning and flushing journal events in batches
Filters may be combined on an AND basis (i.e. only the intersection of events from each filter).

12
doc/cephfs/disaster-recovery-experts.rst
View File

@ -65,6 +65,18 @@ InoTables of each ``in`` MDS rank, to indicate that any written inodes' numbers
are now in use. In simple cases, this will result in an entirely valid backing
store state.
.. warning::
Large journal sizes can cause the tool's Resident Set Size (RSS) to spike
significantly. Use the ``--max-rss <bytes>`` flag to limit memory usage, keeping the following
in mind:
* **Performance Impact:** Do not set this value too low. Restricting memory excessively will
severely degrade runtime performance and prolong the recovery process.
* **30% Buffer Rule:** Maintain a safety buffer of at least 30% relative to the tool's total
memory budget. For example, if 100 GiB can be allotted to the tool, cap its RSS limit at
70 GiB: ``--max-rss 75161927680``.
.. warning::
The resulting state of the backing store is not guaranteed to be

12
doc/cephfs/disaster-recovery.rst
View File

@ -85,6 +85,18 @@ Using first-damage.py
cephfs-journal-tool --rank=<fs_name>:0 event recover_dentries summary
.. warning::
Large journal sizes can cause the tool's Resident Set Size (RSS) to
spike significantly. To prevent excessive memory consumption, use the
``--max-rss <bytes>`` flag to cap the RSS.
Avoid setting this value too low, as it will severely degrade runtime
performance and prolong recovery. It is best practice to maintain a memory
buffer of at least 30% of the tool's total budget. For example, if 100 GiB
is to be allotted to the tool, cap the RSS at 70 GiB
(``--max-rss 75161927680``).
#. Reset the journal:
.. prompt:: bash #

101
qa/tasks/cephfs/test_journal_repair.py
View File

@ -208,6 +208,107 @@ class TestJournalRepair(CephFSTestCase):
# Validate that the journal flush has trimmed the old journal objects
self.assertGreater(len(journal_objs_before_reset), len(journal_objs_after_reset))
def test_cephfs_journal_tool_recover_dentries_with_huge_journal(self):
"""
That after having a pile of unflushed dentries and mds crashed,
invocation of `cephfs-journal-tool recover_dentries summary` can
flush dentries into RADOS without getting consuming too much RSS
or getting OOM killed.
NOTE: this test runs through 10M iterations each with 6 ops, totalling
to 60K ops. Please run this with caution.
"""
# Register cleanup of the config_set so that a partial run can still
# reset the state
self.addCleanup(self.config_rm, 'mds', 'debug_mds')
self.addCleanup(self.config_rm, 'mds', 'debug_ms')
self.addCleanup(self.config_rm, 'mds', 'mds_cache_memory_limit')
self.addCleanup(self.config_rm, 'mds', 'mds_log_max_segments')
self.addCleanup(self.config_rm, 'mds', 'mds_log_warn_factor')
self.addCleanup(self.config_rm, 'mds', 'mds_log_trim_upkeep_interval')
self.addCleanup(self.config_rm, 'mds', 'mds_verify_scatter')
self.addCleanup(self.config_rm, 'mds', 'mds_debug_scatterstat')
# debugging can be too slow given how long the file names are being
# used in the test case therefore turn off the mds
self.config_set('mds', 'debug_mds', '0')
self.config_set('mds', 'debug_ms', '0')
# We do not want any unintended failover
self.fs.set_joinable(False)
# Set the MDS cache limit to 32GiB
self.config_set('mds', 'mds_cache_memory_limit', '34359738368')
# 100K segments limit is more than enough for 60M metadata ops
# with mds_log_events_per_segment being 1024 (default)
self.config_set('mds', 'mds_log_max_segments', '100000')
# eases generating heath warnings
self.config_set('mds', 'mds_log_warn_factor', '1')
# Sleep trim() for a day
self.config_set('mds', 'mds_log_trim_upkeep_interval', '86400')
# Clutter the journal
self.mount_a.run_shell_payload(
"""
set -u
rm -rf w*
A200=$(printf 'A%.0s' {1..200})
X400=$(printf 'X%.0s' {1..400})
export A200 X400
for w in $(seq 0 19); do
(
d="w${w}"
mkdir -p "$d/a" "$d/b" "$d/c"
for ((i = 0; i < 10000; i++)); do
touch "$d/a/${A200}_f${i}"
ln "$d/a/${A200}_f${i}" "$d/b/${A200}_l${i}"
mv "$d/b/${A200}_l${i}" "$d/c/${A200}_r${i}"
ln "$d/a/${A200}_f${i}" "$d/c/${A200}_h${i}"
mv "$d/a/${A200}_f${i}" "$d/b/${A200}_m${i}"
setfattr -n user.chaos -v "$X400" "$d/b/${A200}_m${i}"
done
) &
done
wait
""",timeout=43200)
# kill the mds
self.fs.rank_fail(rank=0)
# Try to access files from the client
blocked_ls = self.mount_a.run_shell(["ls", "w0/a"], wait=False)
log.info("Sleeping to check ls is blocked...")
time.sleep(60)
self.assertFalse(blocked_ls.finished)
self.mount_a.kill()
self.mount_a.kill_cleanup()
# flush dentries into RADOS
self.fs.fail()
result = self.fs.journal_tool(["event", "recover_dentries", "summary",
"--max-rss", "524288000"], 0,
quiet=True)
log.info(f"recover_dentries result:\n{result}")
# test dentries by truncating the journal
self.fs.journal_tool(['journal', 'reset', '--yes-i-really-really-mean-it'], 0)
# Dir stats maybe inconsistent post journal reset
self.config_set('mds', 'mds_verify_scatter', 'false')
self.config_set('mds', 'mds_debug_scatterstat', 'false')
self.fs.set_joinable(True)
status = self.fs.wait_for_daemons()
self.assertEqual(len(list(self.fs.get_ranks(status=status))), 1)
self.mount_a.mount_wait()
# checking one of the dirs should be enough
self.mount_a.run_shell(["ls", "w0"], wait=True)
@for_teuthology # 308s
def test_reset(self):
"""

49
src/tools/cephfs/JournalScanner.cc
View File

@ -31,7 +31,7 @@
* that we were able to apply our checks, it does *not* mean that the journal is
* healthy.
*/
int JournalScanner::scan(bool const full)
int JournalScanner::scan(bool const full, EventCallback cb)
{
int r = 0;
@ -40,17 +40,26 @@ int JournalScanner::scan(bool const full)
return r;
}
if (!is_mdlog || pointer_present) {
if (is_mdlog && !pointer_present) {
derr << "Aborting journal scan" << dendl;
return 0;
} else {
r = scan_header();
if (r < 0) {
return r;
}
}
if (full && header_present) {
r = scan_events();
if (r < 0) {
return r;
// NOTE:: ensure the caller checks for header validity before scanning events
if (full) {
if (!header_valid) {
derr << "Aborting journal scan" << dendl;
return 0;
} else {
r = scan_events(cb);
if (r < 0) {
return r;
}
}
}
@ -151,7 +160,7 @@ int JournalScanner::scan_header()
}
int JournalScanner::scan_events()
int JournalScanner::scan_events(EventCallback cb)
{
uint64_t object_size = g_conf()->mds_log_segment_size;
if (object_size == 0) {
@ -208,7 +217,7 @@ int JournalScanner::scan_events()
} else {
dout(4) << "Read 0x" << std::hex << this_object.length() << std::dec
<< " bytes from " << oid << " gap=" << gap << dendl;
objects_valid.push_back(oid);
++num_objects_valid;
this_object.begin().copy(this_object.length(), read_buf);
}
@ -302,7 +311,12 @@ int JournalScanner::scan_events()
}
if (filter.apply(read_offset, *le)) {
events.insert_or_assign(read_offset, EventRecord(std::move(le), consumed));
EventRecord er(std::move(le), consumed);
if (cb) {
cb(read_offset, er);
} else {
events.insert_or_assign(read_offset, std::move(er));
}
}
} else {
valid_entry = false;
@ -313,7 +327,12 @@ int JournalScanner::scan_events()
auto q = le_bl.cbegin();
pi->decode(q);
if (filter.apply(read_offset, *pi)) {
events.insert_or_assign(read_offset, EventRecord(std::move(pi), consumed));
EventRecord er(std::move(pi), consumed);
if (cb) {
cb(read_offset, er);
} else {
events.insert_or_assign(read_offset, std::move(er));
}
}
} catch (const buffer::error &err) {
valid_entry = false;
@ -328,7 +347,7 @@ int JournalScanner::scan_events()
read_offset += consumed;
break;
} else {
events_valid.push_back(read_offset);
++num_events_valid;
read_offset += consumed;
}
}
@ -340,10 +359,12 @@ int JournalScanner::scan_events()
ranges_invalid.push_back(Range(gap_start, -1));
}
dout(4) << "Scanned objects, " << objects_missing.size() << " missing, " << objects_valid.size() << " valid" << dendl;
dout(4) << "Scanned objects, " << objects_missing.size() << " missing, " << num_objects_valid << " valid" << dendl;
dout(4) << "Events scanned, " << ranges_invalid.size() << " gaps" << dendl;
dout(4) << "Found " << events_valid.size() << " valid events" << dendl;
dout(4) << "Selected " << events.size() << " events events for processing" << dendl;
dout(4) << "Found " << num_events_valid << " valid events" << dendl;
if (!cb) {
dout(4) << "Selected " << events.size() << " events for processing" << dendl;
}
return 0;
}

33
src/tools/cephfs/JournalScanner.h
View File

@ -15,6 +15,8 @@
#ifndef JOURNAL_SCANNER_H
#define JOURNAL_SCANNER_H
#include <functional>
#include "include/rados/librados_fwd.hpp"
// For Journaler::Header, can't forward-declare nested classes
@ -74,18 +76,6 @@ class JournalScanner
~JournalScanner();
int set_journal_ino();
int scan(bool const full=true);
int scan_pointer();
int scan_header();
int scan_events();
void report(std::ostream &out) const;
std::string obj_name(uint64_t offset) const;
std::string obj_name(inodeno_t ino, uint64_t offset) const;
// The results of the scan
inodeno_t ino; // Corresponds to journal ino according their type
struct EventRecord {
EventRecord(std::unique_ptr<LogEvent> le, uint32_t rs) : log_event(std::move(le)), raw_size(rs) {}
EventRecord(std::unique_ptr<PurgeItem> p, uint32_t rs) : pi(std::move(p)), raw_size(rs) {}
@ -94,6 +84,21 @@ class JournalScanner
uint32_t raw_size = 0; //< Size from start offset including all encoding overhead
};
using EventCallback = std::function<void(uint64_t, EventRecord&)>;
int set_journal_ino();
int scan(bool const full=true, EventCallback cb = nullptr);
int scan_pointer();
int scan_header();
int scan_events(EventCallback cb = nullptr);
void report(std::ostream &out) const;
std::string obj_name(uint64_t offset) const;
std::string obj_name(inodeno_t ino, uint64_t offset) const;
// The results of the scan
inodeno_t ino; // Corresponds to journal ino according their type
class EventError {
public:
int r;
@ -114,10 +119,10 @@ class JournalScanner
bool is_healthy() const;
bool is_readable() const;
std::vector<std::string> objects_valid;
std::vector<uint64_t> objects_missing;
std::vector<Range> ranges_invalid;
std::vector<uint64_t> events_valid;
uint64_t num_objects_valid{0};
uint64_t num_events_valid{0};
EventMap events;
// For events present in ::events (i.e. scanned successfully),

114
src/tools/cephfs/JournalTool.cc
View File

@ -72,7 +72,9 @@ void JournalTool::usage()
<< "\n"
<< "Special options\n"
<< " --alternate-pool <name> Alternative metadata pool to target\n"
<< " when using recover_dentries.\n";
<< " when using recover_dentries.\n"
<< " --max-rss <bytes> Maximum RSS allowed per batch during\n"
<< " event recover_dentries.\n";
generic_client_usage();
}
@ -454,6 +456,9 @@ int JournalTool::main_event(std::vector<const char*> &argv)
}
std::string output_path = "dump";
uint64_t batch_size = 0;
uint64_t batch_bytes = 0;
JournalScanner::EventCallback flush = nullptr;
while(arg != argv.end()) {
std::string arg_str;
if (ceph_argparse_witharg(argv, arg, &arg_str, "--path", (char*)NULL)) {
@ -464,6 +469,16 @@ int JournalTool::main_event(std::vector<const char*> &argv)
int r = rados.ioctx_create(arg_str.c_str(), output);
ceph_assert(r == 0);
other_pool = true;
} else if (ceph_argparse_witharg(argv, arg, &arg_str, "--max-rss",
nullptr)) {
std::string parse_err;
/* an estimate based on calculating journal event sizes, a decoded event
holds ~3x its encoded stream */
batch_size = strict_strtoll(arg_str.c_str(), 0, &parse_err) / 3;
if (!parse_err.empty()) {
derr << "Invalid --max-rss value '" << arg_str << "': " << parse_err << dendl;
return -EINVAL;
}
} else {
cerr << "Unknown argument: '" << *arg << "'" << std::endl;
return -EINVAL;
@ -482,45 +497,74 @@ int JournalTool::main_event(std::vector<const char*> &argv)
return r;
}
} else if (command == "recover_dentries") {
r = js.scan();
if (r) {
std::set<inodeno_t> consumed_inos;
progress_tracker->set_operation_name("Processing events");
// decode the journal first to check if the journal pointer and header are valid
r = js.scan(false);
if (r < 0) {
derr << "Failed to scan journal (" << cpp_strerror(r) << ")" << dendl;
return r;
}
/**
* Iterate over log entries, attempting to scavenge from each one
*/
std::set<inodeno_t> consumed_inos;
uint64_t event_count = js.events.size();
progress_tracker->set_operation_name("Processing events");
progress_tracker->start(event_count);
for (JournalScanner::EventMap::iterator i = js.events.begin();
i != js.events.end(); ++i) {
auto& le = i->second.log_event;
EMetaBlob const *mb = le->get_metablob();
if (mb) {
int scav_r = recover_dentries(*mb, dry_run, &consumed_inos);
if (scav_r) {
dout(1) << "Error processing event 0x" << std::hex << i->first << std::dec
<< ": " << cpp_strerror(scav_r) << ", continuing..." << dendl;
if (r == 0) {
r = scav_r;
}
// Our goal is to read all we can, so don't stop on errors, but
// do record them for possible later output
js.errors.insert(std::make_pair(i->first,
JournalScanner::EventError(scav_r, cpp_strerror(r))));
}
}
progress_tracker->increment();
progress_tracker->display_progress();
if (!js.pointer_present) {
derr << "Cannot recover dentries: journal pointer is missing" << dendl;
return 0;
}
// if header_valid is false, header_present is also false
if (!js.header_valid) {
derr << "Cannot recover dentries: journal header is missing or invalid" << dendl;
return 0;
}
progress_tracker->display_final_summary();
// start the progress tracker with the diff b/w write and expire position
progress_tracker->start(js.header->write_pos - js.header->expire_pos);
auto flush_events = [&]() {
for (auto it = js.events.begin(); it != js.events.end(); ) {
const auto& le = it->second.log_event;
EMetaBlob const *mb = le->get_metablob();
if (mb) {
int scav_r = recover_dentries(*mb, dry_run, &consumed_inos);
if (scav_r) {
dout(1) << "Error processing event 0x" << std::hex << it->first << std::dec
<< ": " << cpp_strerror(scav_r) << ", continuing..." << dendl;
js.errors.insert(std::make_pair(it->first,
JournalScanner::EventError(scav_r, cpp_strerror(scav_r))));
}
}
it = js.events.erase(it);
progress_tracker->increment();
progress_tracker->display_progress();
}
batch_bytes = 0;
};
if (batch_size) {
std::cout << "Batch size: " << batch_size << " bytes" << std::endl;
flush = [&](uint64_t offset, JournalScanner::EventRecord& er) {
batch_bytes += er.raw_size;
bool flushable = er.log_event
// factor in mds_debug_subtrees conf
&& (er.log_event->get_type() == EVENT_SUBTREEMAP || er.log_event->get_type() == EVENT_SUBTREEMAP_TEST)
&& batch_bytes >= batch_size;
js.events.insert_or_assign(offset, std::move(er));
if (flushable) {
flush_events();
}
};
}
r = js.scan_events(flush);
flush_events();
if (r) {
derr << "Failed to scan events (" << cpp_strerror(r) << ")" << dendl;
return r;
}
progress_tracker->display_final_summary();
/**

10
src/tools/cephfs/first-damage.py
View File

@ -23,6 +23,16 @@
#
# cephfs-journal-tool --rank=<fs_name>:0 event recover_dentries summary
#
# NOTE: Large journal sizes can cause the tool's Resident Set Size (RSS) to
# spike significantly. To prevent excessive memory consumption, use the
# `--max-rss <bytes>` flag to cap the RSS.
#
# Avoid setting this value too low, as it will severely degrade runtime
# performance and prolong recovery. It is best practice to maintain a memory
# buffer of at least 30% of the tool's total budget. For example, if 100 GiB
# is to be allotted to the tool, cap the RSS at 70 GiB
# (`--max-rss 75161927680`).
#
# 4b) If all good so far, reset the journal:
#
# cephfs-journal-tool --rank=<fs_name>:0 journal reset --yes-i-really-really-mean-it