gian/ceph-mirror
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2026-08-01 22:45:39 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #66757 from senolcolak/wip-v2-mgr-inventory-crush-host

Browse Source 
mgr: override OSD hostname with CRUSH physical host in all metadata paths
This commit is contained in:
SrinivasaBharathKanta 2026-07-29 08:58:21 +05:30 committed by GitHub
commit bfebe2ea25
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 67 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/mgr/DaemonServer.cc
View File

@ -541,7 +541,7 @@ void DaemonServer::fetch_missing_metadata(const DaemonKey& key,
if (!daemon_state.is_updating(key) &&
(key.type == "osd" || key.type == "mds" || key.type == "mon")) {
std::ostringstream oss;
auto c = new MetadataUpdate(daemon_state, key);
auto c = new MetadataUpdate(daemon_state, cluster_state, key);
if (key.type == "osd") {
oss << "{\"prefix\": \"osd metadata\", \"id\": "
<< key.name<< "}";
@ -3667,7 +3667,7 @@ void DaemonServer::got_mgr_map()
cluster_state.with_mgrmap([&](const MgrMap& mgrmap) {
auto md_update = [&] (DaemonKey key) {
std::ostringstream oss;
auto c = new MetadataUpdate(daemon_state, key);
auto c = new MetadataUpdate(daemon_state, cluster_state, key);
// FIXME remove post-nautilus: include 'id' for luminous mons
oss << "{\"prefix\": \"mgr metadata\", \"who\": \""
<< key.name << "\", \"id\": \"" << key.name << "\"}";

70
src/mgr/Mgr.cc
View File

@ -83,6 +83,21 @@ Mgr::~Mgr()
{
}
static std::string crush_hostname_for_osd(ClusterState& cluster_state, int osd_id)
{
std::string hostname;
cluster_state.with_osdmap([&](const OSDMap& osdmap) {
if (osdmap.crush) {
auto loc = osdmap.crush->get_full_location(osd_id);
auto it = loc.find("host");
if (it != loc.end()) {
hostname = it->second;
}
}
});
return hostname;
}
void MetadataUpdate::finish(int r)
{
daemon_state.clear_updating(key);
@ -120,11 +135,21 @@ void MetadataUpdate::finish(int r)
if (daemon_state.exists(key)) {
DaemonStatePtr state = daemon_state.get(key);
// Resolve CRUSH host before taking state->lock to preserve lock ordering
// (Objecter::rwlock must not be acquired under state->lock)
std::string crush_host;
if (key.type == "osd") {
try {
crush_host = crush_hostname_for_osd(cluster_state, std::stoi(key.name));
} catch (const std::exception& e) {
dout(5) << "cannot derive CRUSH hostname for " << key
<< ": " << e.what() << dendl;
}
}
std::map<string,string> m;
{
std::lock_guard l(state->lock);
state->hostname = daemon_meta.at("hostname").get_str();
std::string reported_hostname = daemon_meta.at("hostname").get_str();
if (key.type == "mds" || key.type == "mgr" || key.type == "mon") {
daemon_meta.erase("name");
} else if (key.type == "osd") {
@ -134,12 +159,19 @@ void MetadataUpdate::finish(int r)
for (const auto &[key, val] : daemon_meta) {
m.emplace(key, val.get_str());
}
// prefer CRUSH physical host over container/pod hostname (tracker.ceph.com/issues/73080)
// fall back to the reported hostname when CRUSH does not place the OSD
// under a host bucket, and for mds/mgr/mon which don't use CRUSH
m["hostname"] = !crush_host.empty() ? crush_host : reported_hostname;
}
// update_metadata calls _rm then _insert using state->hostname read from
// the map via set_metadata, so hostname must be set through m, not
// directly on state->hostname before the call (that would corrupt by_server)
daemon_state.update_metadata(state, m);
} else {
auto state = std::make_shared<DaemonState>(daemon_state.types);
state->key = key;
state->hostname = daemon_meta.at("hostname").get_str();
std::string reported_hostname = daemon_meta.at("hostname").get_str();
if (key.type == "mds" || key.type == "mgr" || key.type == "mon") {
daemon_meta.erase("name");
@ -152,6 +184,20 @@ void MetadataUpdate::finish(int r)
for (const auto &[key, val] : daemon_meta) {
m.emplace(key, val.get_str());
}
// prefer CRUSH physical host over container/pod hostname (tracker.ceph.com/issues/73080)
// fall back to the reported hostname when CRUSH does not place the OSD
// under a host bucket, and for mds/mgr/mon which don't use CRUSH
std::string crush_host;
if (key.type == "osd") {
try {
crush_host = crush_hostname_for_osd(cluster_state,
std::stoi(key.name));
} catch (const std::exception& e) {
dout(5) << "cannot derive CRUSH hostname for " << key
<< ": " << e.what() << dendl;
}
}
m["hostname"] = !crush_host.empty() ? crush_host : reported_hostname;
state->set_metadata(m);
daemon_state.insert(state);
@ -478,13 +524,19 @@ void Mgr::load_all_metadata()
dout(1) << "Skipping incomplete metadata entry" << dendl;
continue;
}
dout(4) << osd_metadata.at("hostname").get_str() << dendl;
DaemonStatePtr dm = std::make_shared<DaemonState>(daemon_state.types);
dm->key = DaemonKey{"osd",
stringify(osd_metadata.at("id").get_int())};
int osd_id = osd_metadata.at("id").get_int();
dm->key = DaemonKey{"osd", stringify(osd_id)};
dm->hostname = osd_metadata.at("hostname").get_str();
// prefer CRUSH physical host over container/pod hostname (tracker.ceph.com/issues/73080)
std::string crush_host = crush_hostname_for_osd(cluster_state, osd_id);
if (!crush_host.empty()) {
dm->hostname = crush_host;
}
dout(4) << dm->hostname << dendl;
osd_metadata.erase("id");
osd_metadata.erase("hostname");
@ -550,7 +602,7 @@ void Mgr::handle_osd_map()
update_meta = true;
}
if (update_meta) {
auto c = new MetadataUpdate(daemon_state, k);
auto c = new MetadataUpdate(daemon_state, cluster_state, k);
std::ostringstream cmd;
cmd << "{\"prefix\": \"osd metadata\", \"id\": "
<< osd_id << "}";
@ -597,7 +649,7 @@ void Mgr::handle_mon_map()
if (daemon_state.is_updating(k)) {
continue;
}
auto c = new MetadataUpdate(daemon_state, k);
auto c = new MetadataUpdate(daemon_state, cluster_state, k);
constexpr std::string_view cmd = R"({{"prefix": "mon metadata", "id": "{}"}})";
monc->start_mon_command({fmt::format(cmd, name)}, {},
&c->outbl, &c->outs, c);
@ -738,7 +790,7 @@ void Mgr::handle_fs_map(ref_t<MFSMap> m)
}
if (update) {
auto c = new MetadataUpdate(daemon_state, k);
auto c = new MetadataUpdate(daemon_state, cluster_state, k);
// Older MDS daemons don't have addr in the metadata, so
// fake it if the returned metadata doesn't have the field.

6
src/mgr/Mgr.h
View File

@ -115,6 +115,7 @@ class MetadataUpdate : public Context
private:
DaemonStateIndex &daemon_state;
ClusterState &cluster_state;
DaemonKey key;
std::map<std::string, std::string> defaults;
@ -123,8 +124,9 @@ public:
bufferlist outbl;
std::string outs;
MetadataUpdate(DaemonStateIndex &daemon_state_, const DaemonKey &key_)
: daemon_state(daemon_state_), key(key_)
MetadataUpdate(DaemonStateIndex &daemon_state_, ClusterState &cluster_state_,
const DaemonKey &key_)
: daemon_state(daemon_state_), cluster_state(cluster_state_), key(key_)
{
daemon_state.notify_updating(key);
}