gian/ceph-mirror
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2026-08-01 22:45:39 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

convert spirit_json to Glaze

Browse Source 
Consolidate codecs; views, buffer::list, etc..

Move JSON encoding from source file into inline header functions;
reorganize. Consolidate mostly-similar implementations with templates.

Assisted-by: Codex:GPT-5

Signed-off-by: Jesse F. Williamson <jfw@ibm.com>
This commit is contained in:
Jesse F. Williamson 2026-01-13 14:19:22 -08:00
parent 25187a4389
commit cb5a7d467b
44 changed files with 2730 additions and 1359 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
CMakeLists.txt
View File

@ -161,7 +161,6 @@ if(FREEBSD)
list(APPEND CMAKE_REQUIRED_INCLUDES /usr/local/include)
endif(FREEBSD)
#put all the libs and binaries in one place
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
@ -200,6 +199,35 @@ if(NOT HAVE_CXX11_ATOMIC)
" ${LIBATOMIC_LINK_FLAGS}")
endif()
include(CephAddCPM)
add_cpm("Glaze" FALSE "gh:stephenberry/glaze@7.6.0#v7.6.0")
# Some CPM/FetchContent paths populate Glaze without adding its alias target.
if(NOT TARGET glaze::glaze)
if(DEFINED glaze_SOURCE_DIR)
set(CEPH_GLAZE_SOURCE_DIR "${glaze_SOURCE_DIR}")
elseif(DEFINED CPM_PACKAGE_glaze_SOURCE_DIR)
set(CEPH_GLAZE_SOURCE_DIR "${CPM_PACKAGE_glaze_SOURCE_DIR}")
elseif(DEFINED Glaze_SOURCE_DIR)
set(CEPH_GLAZE_SOURCE_DIR "${Glaze_SOURCE_DIR}")
endif()
if(NOT CEPH_GLAZE_SOURCE_DIR OR NOT EXISTS "${CEPH_GLAZE_SOURCE_DIR}/include/glaze/glaze.hpp")
message(FATAL_ERROR
"Glaze was added by CPM, but no glaze::glaze target or source directory was found")
endif()
add_library(glaze_glaze INTERFACE)
add_library(glaze::glaze ALIAS glaze_glaze)
target_include_directories(glaze_glaze SYSTEM INTERFACE
"${CEPH_GLAZE_SOURCE_DIR}/include")
endif()
if(TARGET glaze_glaze)
target_compile_definitions(glaze_glaze INTERFACE GLZ_USE_STD_FORMAT_FLOAT=0)
endif()
get_target_property(GLAZE_INCLUDE_DIRS glaze::glaze INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM ${GLAZE_INCLUDE_DIRS})
option(WITH_RDMA "Enable RDMA in async messenger" ON)
if(WITH_RDMA)
find_package(verbs REQUIRED)
@ -538,27 +566,7 @@ option(WITH_CATCH2 "Enable Catch2 unit tests" ON)
option(WITH_SYSTEM_CATCH2 "Enable Catch2 unit tests (require system packages)" OFF)
if(WITH_CATCH2)
# Save pre-existing definitions (if any):
set(ORIG_CPM_LOCAL_PACKAGES_ONLY ${CPM_USE_LOCAL_PACKAGES_ONLY})
if(WITH_SYSTEM_CATCH2)
set(CPM_LOCAL_PACKAGES_ONLY ON)
endif()
include(${CMAKE_MODULE_PATH}/CPM.cmake)
CPMAddPackage("gh:catchorg/Catch2@3.8.1")
# Restore the original CPM settings in case someone else wants to use the module:
set(CPM_USE_LOCAL_PACKAGES_ONLY, ${ORIG_CPM_USE_LOCAL_PACKAGES_ONLY})
# CPM skips the fetch silently if downloads are disabled,
# e.g. with FETCHCONTENT_FULLY_DISCONNECTED=ON on deb builds
if(NOT TARGET Catch2::Catch2WithMain)
message(WARNING "Catch2 unavailable, disabling Catch2 tests")
set(WITH_CATCH2 OFF)
else()
message("-- Enabled Catch2 support")
endif()
add_cpm("Catch2" FALSE "gh:catchorg/Catch2@3.8.1")
endif()
# enable breakpad unless win32 or power

30
cmake/modules/CephAddCPM.cmake Normal file
View File

@ -0,0 +1,30 @@
# Add CPM packages in a consistent, Ceph-friendly way:
set(CEPH_ORIG_CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}")
include("${CMAKE_CURRENT_LIST_DIR}/CPM.cmake")
set(CMAKE_MODULE_PATH "${CEPH_ORIG_CMAKE_MODULE_PATH}")
# Two settings govern CPM's behavior with respect to local packages:
# CPM_USE_LOCAL_PACKAGES and CPM_LOCAL_PACKAGES_ONLY. The first prefers local
# packages; the second requires them, e.g. for an offline-only build.
#
# This macro saves and restores both settings. If CPM_LOCAL_PACKAGES_ONLY is
# already set, it is honored.
macro(add_cpm PACKAGE_NAME USE_LOCAL_FLAG)
set(_ORIG_CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}")
set(_ORIG_CPM_USE_LOCAL_PACKAGES "${CPM_USE_LOCAL_PACKAGES}")
set(_ORIG_CPM_LOCAL_PACKAGES_ONLY "${CPM_LOCAL_PACKAGES_ONLY}")
if(${USE_LOCAL_FLAG})
set(CPM_USE_LOCAL_PACKAGES ON)
endif()
CPMAddPackage(${ARGN})
set(CPM_USE_LOCAL_PACKAGES "${_ORIG_CPM_USE_LOCAL_PACKAGES}")
set(CPM_LOCAL_PACKAGES_ONLY "${_ORIG_CPM_LOCAL_PACKAGES_ONLY}")
set(CMAKE_MODULE_PATH "${_ORIG_CMAKE_MODULE_PATH}")
message(STATUS "-- Enabled ${PACKAGE_NAME} support.")
endmacro()

1
src/CMakeLists.txt
View File

@ -609,6 +609,7 @@ set(ceph_common_objs
$<TARGET_OBJECTS:erasure_code>)
set(ceph_common_deps
common_utf8 extblkdev arch crc32
ceph-json
${LIB_RESOLV}
Boost::thread
Boost::context

21
src/cls/CMakeLists.txt
View File

@ -100,9 +100,9 @@ endif (WITH_RADOSGW)
# cls_refcount
set(cls_refcount_srcs
refcount/cls_refcount.cc
refcount/cls_refcount_ops.cc
${CMAKE_SOURCE_DIR}/src/common/ceph_json.cc)
refcount/cls_refcount_ops.cc)
add_library(cls_refcount SHARED ${cls_refcount_srcs})
target_link_libraries(cls_refcount ceph-json)
set_target_properties(cls_refcount PROPERTIES
VERSION "1.0.0"
SOVERSION "1"
@ -202,10 +202,9 @@ if (WITH_RADOSGW)
set(cls_rgw_srcs
rgw/cls_rgw.cc
rgw/cls_rgw_ops.cc
rgw/cls_rgw_types.cc
${CMAKE_SOURCE_DIR}/src/common/ceph_json.cc)
rgw/cls_rgw_types.cc)
add_library(cls_rgw SHARED ${cls_rgw_srcs})
target_link_libraries(cls_rgw ${FMT_LIB})
target_link_libraries(cls_rgw ${FMT_LIB} ceph-json)
target_include_directories(cls_rgw
PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw/driver/rados"
PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw")
@ -300,9 +299,9 @@ install(TARGETS cls_cas DESTINATION ${cls_dir})
#cls_queue
set(cls_queue_srcs
queue/cls_queue.cc
queue/cls_queue_src.cc
${CMAKE_SOURCE_DIR}/src/common/ceph_json.cc)
queue/cls_queue_src.cc)
add_library(cls_queue SHARED ${cls_queue_srcs})
target_link_libraries(cls_queue ceph-json)
set_target_properties(cls_queue PROPERTIES
VERSION "1.0.0"
SOVERSION "1"
@ -318,9 +317,9 @@ add_library(cls_queue_client STATIC ${cls_queue_client_srcs})
if (WITH_RADOSGW)
set(cls_rgw_gc_srcs
rgw_gc/cls_rgw_gc.cc
queue/cls_queue_src.cc
${CMAKE_SOURCE_DIR}/src/common/ceph_json.cc)
queue/cls_queue_src.cc)
add_library(cls_rgw_gc SHARED ${cls_rgw_gc_srcs})
target_link_libraries(cls_rgw_gc ceph-json)
target_include_directories(cls_rgw_gc
PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw/driver/rados"
PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw")
@ -352,9 +351,9 @@ endif (WITH_RADOSGW)
#cls_2pc_queue
set(cls_2pc_queue_srcs
2pc_queue/cls_2pc_queue.cc
queue/cls_queue_src.cc
${CMAKE_SOURCE_DIR}/src/common/ceph_json.cc)
queue/cls_queue_src.cc)
add_library(cls_2pc_queue SHARED ${cls_2pc_queue_srcs})
target_link_libraries(cls_2pc_queue ceph-json)
set_target_properties(cls_2pc_queue PROPERTIES
VERSION "1.0.0"
SOVERSION "1"

6
src/common/CMakeLists.txt
View File

@ -8,6 +8,9 @@ add_library(common_prioritycache_obj OBJECT
PriorityCache.cc)
target_link_libraries(common_prioritycache_obj legacy-option-headers)
add_library(ceph-json OBJECT ceph_json.cc)
target_link_libraries(ceph-json PUBLIC glaze::glaze)
if(WIN32)
add_library(dlfcn_win32 STATIC win32/dlfcn.cc win32/errno.cc)
endif()
@ -59,7 +62,6 @@ set(common_srcs
ceph_frag.cc
ceph_fs.cc
ceph_hash.cc
ceph_json.cc
ceph_strings.cc
ceph_releases.cc
ceph_time.cc
@ -209,7 +211,7 @@ target_compile_definitions(common-common-objs PRIVATE
"CEPH_INSTALL_FULL_PKGLIBDIR=\"${CEPH_INSTALL_FULL_PKGLIBDIR}\""
"CEPH_INSTALL_DATADIR=\"${CEPH_INSTALL_DATADIR}\""
$<TARGET_PROPERTY:${FMT_LIB},INTERFACE_COMPILE_DEFINITIONS>)
target_link_libraries(common-common-objs PUBLIC legacy-option-headers)
target_link_libraries(common-common-objs PUBLIC legacy-option-headers ceph-json)
if(WITH_BREAKPAD)
target_link_libraries(common-common-objs PUBLIC
Breakpad::client)

911
src/common/ceph_json.cc
View File

File diff suppressed because it is too large Load Diff

1256
src/common/ceph_json.h
View File

File diff suppressed because it is too large Load Diff

6
src/common/cmdparse.cc
View File

@ -21,7 +21,9 @@
#include "common/strtol.h"
#include "include/ceph_assert.h" // boost clobbers this
#include "include/types.h" // for operator<<(std::vector)
#include "json_spirit/json_spirit.h"
#include "common/ceph_json.h"
#include <ostream>
#include <sstream>
@ -324,13 +326,15 @@ cmdmap_from_json(const vector<string>& cmd, cmdmap_t *mapp, std::ostream& ss)
json_spirit::mValue v;
string fullcmd;
// First, join all cmd strings
for (auto& c : cmd)
fullcmd += c;
try {
if (!json_spirit::read(fullcmd, v))
throw std::runtime_error("unparseable JSON " + fullcmd);
throw std::runtime_error("unparseable JSON: |" + fullcmd + '|');
if (v.type() != json_spirit::obj_type)
throw std::runtime_error("not JSON object " + fullcmd);

22
src/common/dout_fmt.h
View File

@ -54,3 +54,25 @@ inline void dout_fmt_use_prefix(std::ostream&) {}
fmt::print(*_dout, __VA_ARGS__); \
*_dout << dendl; \
}
// This is useful for debugging during development, please do NOT leave calls to it lying around in production:
inline void dout_trace(const std::string_view info, const std::string_view msg_tag = "dout-TRACE", const std::source_location sl = std::source_location::current())
{
// ...this is an attempt only, as the extension is non-standard-- do NOT count on getting the values back:
#pragma push_macro("dout_subsys")
#pragma push_macro("dout_context")
#define dout_subsys ceph_subsys_rgw
#define dout_context g_ceph_context
dout_fmt(0, "{} [{}]: {}", msg_tag, fmt::format("{}#{}", sl.file_name(), sl.function_name()), info);
#pragma pop_macro("dout_subsys")
#pragma pop_macro("dout_context")
}
inline void dout_trace(const std::source_location sl = std::source_location::current())
{
dout_trace("", "dout-TRACE", sl);
}

2
src/crimson/CMakeLists.txt
View File

@ -60,7 +60,6 @@ add_library(crimson-common STATIC
${PROJECT_SOURCE_DIR}/src/common/hex.cc
${PROJECT_SOURCE_DIR}/src/common/frag.cc
${PROJECT_SOURCE_DIR}/src/common/fs_types.cc
${PROJECT_SOURCE_DIR}/src/common/ceph_json.cc
${PROJECT_SOURCE_DIR}/src/common/histogram.cc
${PROJECT_SOURCE_DIR}/src/common/hobject.cc
${PROJECT_SOURCE_DIR}/src/common/hostname.cc
@ -153,6 +152,7 @@ target_link_libraries(crimson-common
crimson::cflags
PRIVATE
crc32 arch
ceph-json
Boost::iostreams
Boost::random
json_spirit

2
src/crimson/admin/pg_commands.cc
View File

@ -17,6 +17,8 @@
#include "crimson/osd/osd.h"
#include "crimson/osd/pg.h"
#include "json_spirit/json_spirit.h"
SET_SUBSYS(osd);
using crimson::osd::OSD;

2
src/include/encoding.h
View File

@ -246,7 +246,7 @@ inline void decode_nohead(unsigned len, std::string& s, bufferlist::const_iterat
// const char* (encode only, string compatible)
inline void encode(const char *s, bufferlist& bl)
{
encode(std::string_view(s, strlen(s)), bl);
encode(std::string_view(s), bl);
}
// opaque byte vectors

47
src/librados/RadosClient.cc
View File

@ -17,7 +17,11 @@
#include <sys/stat.h>
#include <fcntl.h>
#include <algorithm>
#include <iostream>
#include <iterator>
#include <ranges>
#include <span>
#include <string>
#include <sstream>
#include <pthread.h>
@ -28,7 +32,6 @@
#include "common/common_init.h"
#include "common/ceph_json.h"
#include "common/errno.h"
#include "common/ceph_json.h"
#include "common/async/blocked_completion.h"
#include "include/buffer.h"
#include "include/stringify.h"
@ -1140,33 +1143,31 @@ int librados::RadosClient::get_inconsistent_pgs(int64_t pool_id,
// no pg returned
return 0;
}
JSONParser parser;
if (!parser.parse(outbl.c_str(), outbl.length())) {
ceph_json::value response;
if (!ceph_json::parse(outbl, response)) {
return -EINVAL;
}
vector<string> v;
if (!parser.is_array()) {
JSONObj *pgstat_obj = parser.find_obj("pg_stats");
if (!pgstat_obj)
return 0;
auto s = pgstat_obj->get_data();
JSONParser pg_stats;
if (!pg_stats.parse(s.c_str(), s.length())) {
return -EINVAL;
const auto pg_entries = [&]() -> std::span<const ceph_json::value> {
if (response.is_array()) {
return ceph_json::array_view(response);
}
v = pg_stats.get_array_elements();
} else {
v = parser.get_array_elements();
}
for (auto i : v) {
JSONParser pg_json;
if (!pg_json.parse(i.c_str(), i.length())) {
return -EINVAL;
auto* pg_stats = ceph_json::find_value(response, "pg_stats");
if (!pg_stats) {
return {};
}
return ceph_json::array_view(*pg_stats);
}();
std::ranges::transform(pg_entries, std::back_inserter(*pgs), [] (const auto& pg) {
string pgid;
JSONDecoder::decode_json("pgid", pgid, &pg_json);
pgs->emplace_back(std::move(pgid));
}
ceph_json::decode_field("pgid", pgid, pg);
return pgid;
});
return 0;
}

2
src/mgr/ActivePyModules.cc
View File

@ -43,6 +43,8 @@
#include "PyModuleRegistry.h"
#include "PyUtil.h"
#include "json_spirit/json_spirit.h"
#define dout_context g_ceph_context
#define dout_subsys ceph_subsys_mgr
#undef dout_prefix

15
src/mgr/JSONCommand.h
View File

@ -4,6 +4,7 @@
* Ceph - scalable distributed file system
*
* Copyright (C) 2016 John Spray <john.spray@redhat.com>
* Copyright (C) 2026 IBM
*
* This is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@ -20,18 +21,18 @@
class JSONCommand : public Command
{
public:
json_spirit::mValue json_result;
glz::generic json_result;
void wait() override
{
Command::wait();
if (r == 0) {
bool read_ok = json_spirit::read(
outbl.to_str(), json_result);
if (!read_ok) {
r = -EINVAL;
}
if (0 != r) {
return;
}
if (auto ec = glz::read_json(json_result, outbl.to_str()); ec) {
r = -EINVAL;
}
}
};

57
src/mgr/Mgr.cc
View File

@ -14,6 +14,8 @@
#include <Python.h>
#include "common/ceph_json.h"
#include "osdc/Objecter.h"
#include "common/errno.h"
#include "mon/MonClient.h"
@ -104,21 +106,21 @@ void MetadataUpdate::finish(int r)
if (r == 0) {
if (key.type == "mds" || key.type == "osd" ||
key.type == "mgr" || key.type == "mon") {
json_spirit::mValue json_result;
bool read_ok = json_spirit::read(
outbl.to_str(), json_result);
if (!read_ok) {
glz::generic json_result;
if (auto ec = glz::read_json(json_result, outbl.to_str()); ec) {
dout(1) << "mon returned invalid JSON for " << key << dendl;
return;
}
if (json_result.type() != json_spirit::obj_type) {
if (!json_result.is_object()) {
dout(1) << "mon returned valid JSON " << key
<< " but not an object: '" << outbl.to_str() << "'" << dendl;
return;
}
dout(4) << "mon returned valid metadata JSON for " << key << dendl;
json_spirit::mObject daemon_meta = json_result.get_obj();
auto daemon_meta = json_result.get_object();
// Skip daemon who doesn't have hostname yet
if (daemon_meta.count("hostname") == 0) {
@ -149,7 +151,7 @@ void MetadataUpdate::finish(int r)
std::map<string,string> m;
{
std::lock_guard l(state->lock);
std::string reported_hostname = daemon_meta.at("hostname").get_str();
const std::string reported_hostname = daemon_meta.at("hostname").get_string();
if (key.type == "mds" || key.type == "mgr" || key.type == "mon") {
daemon_meta.erase("name");
} else if (key.type == "osd") {
@ -157,7 +159,7 @@ void MetadataUpdate::finish(int r)
}
daemon_meta.erase("hostname");
for (const auto &[key, val] : daemon_meta) {
m.emplace(key, val.get_str());
m.emplace(key, val.get_string());
}
// prefer CRUSH physical host over container/pod hostname (tracker.ceph.com/issues/73080)
// fall back to the reported hostname when CRUSH does not place the OSD
@ -171,7 +173,7 @@ void MetadataUpdate::finish(int r)
} else {
auto state = std::make_shared<DaemonState>(daemon_state.types);
state->key = key;
std::string reported_hostname = daemon_meta.at("hostname").get_str();
const std::string reported_hostname = daemon_meta.at("hostname").get_string();
if (key.type == "mds" || key.type == "mgr" || key.type == "mon") {
daemon_meta.erase("name");
@ -182,7 +184,7 @@ void MetadataUpdate::finish(int r)
std::map<string,string> m;
for (const auto &[key, val] : daemon_meta) {
m.emplace(key, val.get_str());
m.emplace(key, val.get_string());
}
// prefer CRUSH physical host over container/pod hostname (tracker.ceph.com/issues/73080)
// fall back to the reported hostname when CRUSH does not place the OSD
@ -250,7 +252,7 @@ std::map<std::string, std::string> Mgr::load_store()
std::map<std::string, std::string> loaded;
for (auto &key_str : cmd.json_result.get_array()) {
std::string const key = key_str.get_str();
auto key = key_str.get_string();
dout(20) << "saw key '" << key << "'" << dendl;
@ -473,45 +475,44 @@ void Mgr::load_all_metadata()
ceph_assert(osd_cmd.r == 0);
for (auto &metadata_val : mds_cmd.json_result.get_array()) {
json_spirit::mObject daemon_meta = metadata_val.get_obj();
auto daemon_meta = metadata_val.get_object();
if (daemon_meta.count("hostname") == 0) {
dout(1) << "Skipping incomplete metadata entry" << dendl;
continue;
}
DaemonStatePtr dm = std::make_shared<DaemonState>(daemon_state.types);
dm->key = DaemonKey{"mds",
daemon_meta.at("name").get_str()};
dm->hostname = daemon_meta.at("hostname").get_str();
dm->key = DaemonKey{"mds", daemon_meta.at("name").get_string()};
dm->hostname = daemon_meta.at("hostname").get_string();
daemon_meta.erase("name");
daemon_meta.erase("hostname");
for (const auto &[key, val] : daemon_meta) {
dm->metadata.emplace(key, val.get_str());
dm->metadata.emplace(key, val.get_string());
}
daemon_state.insert(dm);
}
for (auto &metadata_val : mon_cmd.json_result.get_array()) {
json_spirit::mObject daemon_meta = metadata_val.get_obj();
auto daemon_meta = metadata_val.get_object();
if (daemon_meta.count("hostname") == 0) {
dout(1) << "Skipping incomplete metadata entry" << dendl;
continue;
}
DaemonStatePtr dm = std::make_shared<DaemonState>(daemon_state.types);
dm->key = DaemonKey{"mon",
daemon_meta.at("name").get_str()};
dm->hostname = daemon_meta.at("hostname").get_str();
dm->key = DaemonKey{"mon", daemon_meta.at("name").get_string() };
dm->hostname = daemon_meta.at("hostname").get_string();
daemon_meta.erase("name");
daemon_meta.erase("hostname");
std::map<string,string> m;
for (const auto &[key, val] : daemon_meta) {
m.emplace(key, val.get_str());
m.emplace(key, val.get_string());
}
dm->set_metadata(m);
@ -519,19 +520,19 @@ void Mgr::load_all_metadata()
}
for (auto &osd_metadata_val : osd_cmd.json_result.get_array()) {
json_spirit::mObject osd_metadata = osd_metadata_val.get_obj();
auto osd_metadata = osd_metadata_val.get_object();
if (osd_metadata.count("hostname") == 0) {
dout(1) << "Skipping incomplete metadata entry" << dendl;
continue;
}
DaemonStatePtr dm = std::make_shared<DaemonState>(daemon_state.types);
int osd_id = osd_metadata.at("id").get_int();
const auto osd_id = static_cast<int>(osd_metadata.at("id").get_number());
dm->key = DaemonKey{"osd", stringify(osd_id)};
dm->hostname = osd_metadata.at("hostname").get_str();
dm->hostname = osd_metadata.at("hostname").get_string();
// prefer CRUSH physical host over container/pod hostname (tracker.ceph.com/issues/73080)
std::string crush_host = crush_hostname_for_osd(cluster_state, osd_id);
const std::string crush_host = crush_hostname_for_osd(cluster_state, osd_id);
if (!crush_host.empty()) {
dm->hostname = crush_host;
}
@ -540,9 +541,9 @@ void Mgr::load_all_metadata()
osd_metadata.erase("id");
osd_metadata.erase("hostname");
std::map<string,string> m;
for (const auto &i : osd_metadata) {
m[i.first] = i.second.get_str();
map<string,string> m;
for (const auto &[k, v] : osd_metadata) {
m[k] = v.get_string();
}
dm->set_metadata(m);

61
src/rgw/radosgw-admin/radosgw-admin.cc
View File

@ -2,8 +2,8 @@
// vim: ts=8 sw=2 sts=2 expandtab ft=cpp
/*
* Copyright (C) 2025 IBM
*/
* Copyright (C) 2025-2026 IBM
*/
#include <cerrno>
#include <string>
@ -22,6 +22,8 @@ extern "C" {
#include <fmt/format.h>
#include <liboath/oath.h>
#include "auth/Crypto.h"
#include "compressor/Compressor.h"
@ -1468,7 +1470,7 @@ static int read_decode_json(const string& infile, T& t)
return ret;
}
JSONParser p;
if (!p.parse(bl.c_str(), bl.length())) {
if (!p.parse(bl)) {
cout << "failed to parse JSON" << std::endl;
return -EINVAL;
}
@ -1492,7 +1494,7 @@ static int read_decode_json(const string& infile, T& t, K *k)
return ret;
}
JSONParser p;
if (!p.parse(bl.c_str(), bl.length())) {
if (!p.parse(bl)) {
cout << "failed to parse JSON" << std::endl;
return -EINVAL;
}
@ -2075,7 +2077,7 @@ static int send_to_remote_gateway(RGWRESTConn* conn, req_info& info,
return ret;
}
ret = parser.parse(response.c_str(), response.length());
ret = parser.parse(response);
if (ret < 0) {
cerr << "failed to parse response" << std::endl;
return ret;
@ -2105,14 +2107,15 @@ static int send_to_url(const string& url,
if (!result) {
return result.error();
}
int ret = rgw_http_error_to_errno(*result);
if (ret < 0) {
return ret;
}
ret = parser.parse(response.c_str(), response.length());
ret = parser.parse(response);
if (ret < 0) {
cout << "failed to parse response" << std::endl;
cerr << "failed to parse remote response" << std::endl;
return ret;
}
return 0;
@ -2157,7 +2160,7 @@ static int commit_period(rgw::sal::ConfigStore* cfgstore,
realm, realm_writer, current_period,
period, cerr, force, *site);
if (ret < 0) {
cerr << "failed to commit period: " << cpp_strerror(-ret) << std::endl;
cerr << "commit_period(): failed to commit period: " << cpp_strerror(-ret) << std::endl;
}
(void) cfgstore->realm_notify_new_period(dpp(), null_yield, period);
return ret;
@ -2166,7 +2169,6 @@ static int commit_period(rgw::sal::ConfigStore* cfgstore,
if (remote.empty() && url.empty()) {
// use the new master zone's connection
remote = master_zone;
cerr << "Sending period to new master zone " << remote << std::endl;
}
boost::optional<RGWRESTConn> conn;
RGWRESTConn *remote_conn = nullptr;
@ -2211,13 +2213,15 @@ static int commit_period(rgw::sal::ConfigStore* cfgstore,
try {
decode_json_obj(period, &p);
} catch (const JSONDecoder::err& e) {
cout << "failed to decode JSON input: " << e.what() << std::endl;
cerr << "failed to decode JSON input: " << e.what() << std::endl;
return -EINVAL;
}
if (period.get_id().empty()) {
cerr << "Period commit got back an empty period id" << std::endl;
cerr << "commit_period(): Period commit got back an empty period id" << std::endl;
return -EINVAL;
}
// the master zone gave us back the period that it committed, so it's
// safe to save it as our latest epoch
constexpr bool exclusive = false;
@ -2288,18 +2292,21 @@ static int update_period(rgw::sal::ConfigStore* cfgstore,
constexpr bool exclusive = false;
ret = cfgstore->create_period(dpp(), null_yield, exclusive, period);
if (ret < 0) {
cerr << "failed to driver period: " << cpp_strerror(-ret) << std::endl;
return ret;
}
if (commit) {
ret = commit_period(cfgstore, realm, *realm_writer, period, remote, url,
opt_region, access, secret, force, site);
if (ret < 0) {
cerr << "failed to commit period: " << cpp_strerror(-ret) << std::endl;
cerr << "update_period(): failed to commit period: " << cpp_strerror(-ret) << std::endl;
return ret;
}
}
encode_json("period", period, formatter);
formatter->flush(cout);
return 0;
@ -2351,6 +2358,7 @@ static int do_period_pull(rgw::sal::ConfigStore* cfgstore,
cerr << "request failed: " << cpp_strerror(-ret) << std::endl;
return ret;
}
try {
decode_json_obj(*period, &p);
} catch (const JSONDecoder::err& e) {
@ -3724,6 +3732,7 @@ void init_realm_param(CephContext *cct, string& var, std::optional<string>& opt_
// would need to be terminated, so the warning is simply suppressed.
// coverity[root_function:SUPPRESS]
int main(int argc, const char **argv)
try
{
auto args = argv_to_vec(argc, argv);
if (args.empty()) {
@ -5677,10 +5686,19 @@ int main(int argc, const char **argv)
info.request_uri = "/admin/realm";
map<string, string> &params = info.args.get_params();
if (!realm_id.empty())
params["id"] = realm_id;
if (!realm_name.empty())
params["name"] = realm_name;
if (realm_name.empty()) {
cerr << "missing realm name" << std::endl;
return EINVAL;
}
if (realm_id.empty()) {
cerr << "missing realm id" << std::endl;
return EINVAL;
}
params["id"] = realm_id;
params["name"] = realm_name;
bufferlist bl;
JSONParser p;
@ -5694,6 +5712,7 @@ int main(int argc, const char **argv)
}
return -ret;
}
RGWRealm realm;
try {
decode_json_obj(realm, &p);
@ -5701,6 +5720,7 @@ int main(int argc, const char **argv)
cerr << "failed to decode JSON response: " << e.what() << std::endl;
return EINVAL;
}
RGWPeriod period;
auto& current_period = realm.get_current_period();
if (!current_period.empty()) {
@ -7374,7 +7394,7 @@ int main(int argc, const char **argv)
remote, url, opt_region, access_key, secret_key,
yes_i_really_mean_it, site.get());
if (ret < 0) {
cerr << "failed to commit period: " << cpp_strerror(-ret) << std::endl;
cerr << "OPT::PERIOD_COMMIT: failed to commit period: " << cpp_strerror(-ret) << std::endl;
return -ret;
}
@ -12311,6 +12331,7 @@ next:
if (rgw::all_zonegroups_support(*site, rgw::zone_features::notification_v2) &&
driver->stat_topics_v1(tenant, null_yield, dpp()) == -ENOENT) {
show_topics_info_v2(topic, subscribed_buckets, formatter.get());
} else {
encode_json("topic", topic, formatter.get());
}
@ -12791,3 +12812,9 @@ next:
}
return 0;
}
catch(const std::exception& e)
{
std::cerr << "exception: " << e.what() << std::endl;
return 1;
}

39
src/rgw/rgw_acl_swift.cc
View File

@ -285,38 +285,27 @@ int create_account_policy(const DoutPrefixProvider* dpp,
policy.create_default(owner.id, owner.display_name);
auto& acl = policy.get_acl();
JSONParser parser;
if (!parser.parse(acl_str.c_str(), acl_str.length())) {
ceph_json::value acl_json;
if (!ceph_json::parse(acl_str, acl_json)) {
ldpp_dout(dpp, 0) << "ERROR: JSONParser::parse returned error=" << dendl;
return -EINVAL;
}
JSONObjIter iter = parser.find_first("admin");
if (!iter.end() && (*iter)->is_array()) {
std::vector<std::string> admin;
decode_json_obj(admin, *iter);
ldpp_dout(dpp, 0) << "admins: " << admin << dendl;
const auto add_acl_grants = [&] (std::string_view name, std::string_view label, int perm) {
const auto* values = ceph_json::find_value(acl_json, name);
if (!values || !values->is_array()) {
return;
}
add_grants(dpp, driver, admin, SWIFT_PERM_ADMIN, acl);
}
auto grants = ceph_json::decode_value<std::vector<std::string>>(*values);
ldpp_dout(dpp, 0) << label << ": " << grants << dendl;
iter = parser.find_first("read-write");
if (!iter.end() && (*iter)->is_array()) {
std::vector<std::string> readwrite;
decode_json_obj(readwrite, *iter);
ldpp_dout(dpp, 0) << "read-write: " << readwrite << dendl;
add_grants(dpp, driver, grants, perm, acl);
};
add_grants(dpp, driver, readwrite, SWIFT_PERM_RWRT, acl);
}
iter = parser.find_first("read-only");
if (!iter.end() && (*iter)->is_array()) {
std::vector<std::string> readonly;
decode_json_obj(readonly, *iter);
ldpp_dout(dpp, 0) << "read-only: " << readonly << dendl;
add_grants(dpp, driver, readonly, SWIFT_PERM_READ, acl);
}
add_acl_grants("admin", "admins", SWIFT_PERM_ADMIN);
add_acl_grants("read-write", "read-write", SWIFT_PERM_RWRT);
add_acl_grants("read-only", "read-only", SWIFT_PERM_READ);
return 0;
}

18
src/rgw/rgw_auth_keystone.cc
View File

@ -91,9 +91,6 @@ admin_token_retry:
ret = validate.process(dpp, y);
/* NULL terminate for debug output. */
token_body_bl.append(static_cast<char>(0));
/* Detect Keystone rejection earlier than during the token parsing.
* Although failure at the parsing phase doesn't impose a threat,
* this allows to return proper error code (EACCESS instead of EINVAL
@ -128,7 +125,7 @@ admin_token_retry:
}
ldpp_dout(dpp, 20) << "received response status=" << validate.get_http_status()
<< ", body=" << token_body_bl.c_str() << dendl;
<< ", body=" << std::string_view(token_body_bl.c_str()) << dendl;
TokenEngine::token_envelope_t token_body;
ret = token_body.parse(dpp, token, token_body_bl);
@ -559,22 +556,17 @@ auto EC2Engine::get_secret_from_keystone(const DoutPrefixProvider* dpp,
/* now parse response */
JSONParser parser;
if (! parser.parse(token_body_bl.c_str(), token_body_bl.length())) {
ceph_json::value response;
if (!ceph_json::parse(token_body_bl, response)) {
ldpp_dout(dpp, 0) << "Keystone credential parse error: malformed json" << dendl;
return make_pair(boost::none, -EINVAL);
}
JSONObjIter credential_iter = parser.find_first("credential");
std::string secret_string;
try {
if (!credential_iter.end()) {
JSONDecoder::decode_json("secret", secret_string, *credential_iter, true);
} else {
ldpp_dout(dpp, 0) << "Keystone credential not present in return from server" << dendl;
return make_pair(boost::none, -EINVAL);
}
const auto& credential = ceph_json::require_object(response, "credential");
secret_string = ceph_json::require<std::string>(credential, "secret");
} catch (const JSONDecoder::err& err) {
ldpp_dout(dpp, 0) << "Keystone credential parse error: " << err.what() << dendl;
return make_pair(boost::none, -EINVAL);

2
src/rgw/rgw_keystone.cc
View File

@ -263,7 +263,7 @@ int TokenEnvelope::parse(const DoutPrefixProvider *dpp,
ceph::bufferlist& bl)
{
JSONParser parser;
if (! parser.parse(bl.c_str(), bl.length())) {
if (! parser.parse(bl)) {
ldpp_dout(dpp, 0) << "Keystone token parse error: malformed json" << dendl;
return -EINVAL;
}

2
src/rgw/rgw_metadata.cc
View File

@ -311,7 +311,7 @@ int RGWMetadataManager::put(string& metadata_key, bufferlist& bl,
}
JSONParser parser;
if (!parser.parse(bl.c_str(), bl.length())) {
if (!parser.parse(bl)) {
return -EINVAL;
}

2
src/rgw/rgw_op.cc
View File

@ -192,7 +192,7 @@ int rgw_forward_request_to_master(const DoutPrefixProvider* dpp,
if (ret < 0) {
return ret;
}
if (jp && !jp->parse(outdata.c_str(), outdata.length())) {
if (jp && !jp->parse(outdata)) {
ldpp_dout(dpp, 0) << "failed parsing response from master zonegroup" << dendl;
return -EINVAL;
}

2
src/rgw/rgw_op.h
View File

@ -165,7 +165,7 @@ int rgw_rest_get_json_input(CephContext *cct, req_state *s, T& out,
JSONParser parser;
if (!parser.parse(data.c_str(), data.length())) {
if (!parser.parse(data)) {
return -EINVAL;
}

9
src/rgw/rgw_opa.cc
View File

@ -78,16 +78,15 @@ int rgw_opa_authorize(RGWOp *& op,
}
/* check OPA response */
JSONParser parser;
if (!parser.parse(bl.c_str(), bl.length())) {
ceph_json::value response;
if (!ceph_json::parse(bl, response)) {
ldpp_dout(op, 2) << "OPA parse error: malformed json" << dendl;
return -EINVAL;
}
bool opa_result;
JSONDecoder::decode_json("result", opa_result, &parser);
const auto opa_result = ceph_json::require<bool>(response, "result");
if (opa_result == false) {
if (false == opa_result) {
ldpp_dout(op, 2) << "OPA rejecting request" << dendl;
return -EPERM;
}

1
src/rgw/rgw_period.cc
View File

@ -154,6 +154,7 @@ void RGWPeriod::dump(Formatter *f) const
encode_json("realm_epoch", realm_epoch, f);
}
// Reads internal fields /from/ JSONObj:
void RGWPeriod::decode_json(JSONObj *obj)
{
JSONDecoder::decode_json("id", id, obj);

2
src/rgw/rgw_period_puller.cc
View File

@ -53,7 +53,7 @@ int pull_period(const DoutPrefixProvider *dpp, RGWRESTConn* conn, const std::str
}
JSONParser parser;
r = parser.parse(data.c_str(), data.length());
r = parser.parse(data);
if (r < 0) {
ldpp_dout(dpp, -1) << "request failed: " << cpp_strerror(-r) << dendl;
return r;

18
src/rgw/rgw_policy_s3.cc
View File

@ -243,14 +243,12 @@ int RGWPolicy::check(RGWPolicyEnv *env, string& err_msg)
return 0;
}
int RGWPolicy::from_json(bufferlist& bl, string& err_msg)
int RGWPolicy::from_json(std::string_view json_in, string& err_msg)
{
JSONParser parser;
if (!parser.parse(bl.c_str(), bl.length())) {
err_msg = "Malformed JSON";
dout(0) << "malformed json" << dendl;
if (!parser.parse(json_in)) {
err_msg = fmt::format("malformed JSON (RGWPolicy) length = {}:\n{}\n----\n", json_in.length(), json_in);
dout(0) << err_msg << dendl;
return -EINVAL;
}
@ -258,7 +256,7 @@ int RGWPolicy::from_json(bufferlist& bl, string& err_msg)
JSONObjIter iter = parser.find_first("expiration");
if (iter.end()) {
err_msg = "Policy missing expiration";
dout(0) << "expiration not found" << dendl;
dout(0) << err_msg << dendl;
return -EINVAL; // change to a "no expiration" error following S3
}
@ -312,3 +310,9 @@ int RGWPolicy::from_json(bufferlist& bl, string& err_msg)
}
return 0;
}
int RGWPolicy::from_json(bufferlist& bl, string& err_msg)
{
return from_json(std::string_view { bl.c_str(), bl.length() }, err_msg);
}

1
src/rgw/rgw_policy_s3.h
View File

@ -53,5 +53,6 @@ public:
}
int check(RGWPolicyEnv *env, std::string& err_msg);
int from_json(std::string_view bl, std::string& err_msg);
int from_json(bufferlist& bl, std::string& err_msg);
};

3
src/rgw/rgw_rest.h
View File

@ -61,8 +61,7 @@ std::tuple<int, bufferlist > rgw_rest_get_json_input_keep_data(CephContext *cct,
}
JSONParser parser;
if (!parser.parse(data.c_str(), data.length())) {
if (!parser.parse(data)) {
return std::make_tuple(-EINVAL, std::move(data));
}

837
src/rgw/rgw_rest.sts.cc Normal file
View File

@ -0,0 +1,837 @@
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab ft=cpp
#include <vector>
#include <string>
#include <array>
#include <string_view>
#include <sstream>
#include <memory>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/format.hpp>
#include <boost/optional.hpp>
#include <boost/utility/in_place_factory.hpp>
#include <boost/tokenizer.hpp>
#include "ceph_ver.h"
#include "common/Formatter.h"
#include "common/utf8.h"
#include "common/ceph_json.h"
#include "rgw_rest.h"
#include "rgw_account.h"
#include "rgw_auth.h"
#include "rgw_auth_registry.h"
#include "jwt-cpp/jwt.h"
#include "rgw_rest_sts.h"
#include "rgw_formats.h"
#include "rgw_client_io.h"
#include "rgw_request.h"
#include "rgw_process.h"
#include "rgw_iam_policy.h"
#include "rgw_iam_policy_keywords.h"
#include "rgw_sts.h"
#include "rgw_rest_oidc_provider.h"
#define dout_context g_ceph_context
#define dout_subsys ceph_subsys_rgw
using namespace std;
namespace rgw::auth::sts {
bool
WebTokenEngine::is_applicable(const std::string& token) const noexcept
{
return ! token.empty();
}
std::string
WebTokenEngine::get_role_tenant(const string& role_arn) const
{
string tenant;
auto r_arn = rgw::ARN::parse(role_arn);
if (r_arn) {
tenant = r_arn->account;
}
return tenant;
}
std::string
WebTokenEngine::get_role_name(const string& role_arn) const
{
string role_name;
auto r_arn = rgw::ARN::parse(role_arn);
if (r_arn) {
role_name = r_arn->resource;
}
if (!role_name.empty()) {
auto pos = role_name.find_last_of('/');
if(pos != string::npos) {
role_name = role_name.substr(pos + 1);
}
}
return role_name;
}
int WebTokenEngine::load_provider(const DoutPrefixProvider* dpp, optional_yield y,
const string& role_arn, const string& iss,
RGWOIDCProviderInfo& info) const
{
string tenant = get_role_tenant(role_arn);
string idp_url = iss;
auto pos = idp_url.find("http://");
if (pos == std::string::npos) {
pos = idp_url.find("https://");
if (pos != std::string::npos) {
idp_url.erase(pos, 8);
} else {
pos = idp_url.find("www.");
if (pos != std::string::npos) {
idp_url.erase(pos, 4);
}
}
} else {
idp_url.erase(pos, 7);
}
return driver->load_oidc_provider(dpp, y, tenant, idp_url, info);
}
bool
WebTokenEngine::is_client_id_valid(vector<string>& client_ids, const string& client_id) const
{
for (auto it : client_ids) {
if (it == client_id) {
return true;
}
}
return false;
}
bool
WebTokenEngine::is_cert_valid(const vector<string>& thumbprints, const string& cert) const
{
//calculate thumbprint of cert
std::unique_ptr<BIO, decltype(&BIO_free_all)> certbio(BIO_new_mem_buf(cert.data(), cert.size()), BIO_free_all);
std::unique_ptr<BIO, decltype(&BIO_free_all)> keybio(BIO_new(BIO_s_mem()), BIO_free_all);
string pw="";
std::unique_ptr<X509, decltype(&X509_free)> x_509cert(PEM_read_bio_X509(certbio.get(), nullptr, nullptr, const_cast<char*>(pw.c_str())), X509_free);
const EVP_MD* fprint_type = EVP_sha1();
unsigned int fprint_size;
unsigned char fprint[EVP_MAX_MD_SIZE];
if (!X509_digest(x_509cert.get(), fprint_type, fprint, &fprint_size)) {
return false;
}
stringstream ss;
for (unsigned int i = 0; i < fprint_size; i++) {
ss << std::setfill('0') << std::setw(2) << std::hex << (0xFF & (unsigned int)fprint[i]);
}
std::string digest = ss.str();
for (auto& it : thumbprints) {
if (boost::iequals(it,digest)) {
return true;
}
}
return false;
}
template <typename T>
void
WebTokenEngine::recurse_and_insert(const string& key, const jwt::claim& c, T& t) const
{
string s_val;
jwt::claim::type c_type = c.get_type();
switch(c_type) {
case jwt::claim::type::null:
break;
case jwt::claim::type::boolean:
case jwt::claim::type::number:
case jwt::claim::type::int64:
{
s_val = c.to_json().serialize();
t.emplace(std::make_pair(key, s_val));
break;
}
case jwt::claim::type::string:
{
s_val = c.to_json().to_str();
t.emplace(std::make_pair(key, s_val));
break;
}
case jwt::claim::type::array:
{
const picojson::array& arr = c.as_array();
for (auto& a : arr) {
recurse_and_insert(key, jwt::claim(a), t);
}
break;
}
case jwt::claim::type::object:
{
const picojson::object& obj = c.as_object();
for (auto& m : obj) {
recurse_and_insert(m.first, jwt::claim(m.second), t);
}
break;
}
}
return;
}
//Extract all token claims so that they can be later used in the Condition element of Role's trust policy
WebTokenEngine::token_t
WebTokenEngine::get_token_claims(const jwt::decoded_jwt& decoded) const
{
WebTokenEngine::token_t token;
const auto& claims = decoded.get_payload_claims();
for (auto& c : claims) {
if (c.first == string(princTagsNamespace)) {
continue;
}
recurse_and_insert(c.first, c.second, token);
}
return token;
}
//Offline validation of incoming Web Token which is a signed JWT (JSON Web Token)
std::tuple<boost::optional<WebTokenEngine::token_t>, boost::optional<WebTokenEngine::principal_tags_t>>
WebTokenEngine::get_from_jwt(const DoutPrefixProvider* dpp, const std::string& token, const req_state* const s,
optional_yield y) const
{
WebTokenEngine::token_t t;
WebTokenEngine::principal_tags_t principal_tags;
try {
const auto& decoded = jwt::decode(token);
auto& payload = decoded.get_payload();
ldpp_dout(dpp, 20) << " payload = " << payload << dendl;
t = get_token_claims(decoded);
string iss;
if (decoded.has_issuer()) {
iss = decoded.get_issuer();
}
set<string> aud;
if (decoded.has_audience()) {
aud = decoded.get_audience();
}
string client_id;
if (decoded.has_payload_claim("client_id")) {
client_id = decoded.get_payload_claim("client_id").as_string();
}
if (client_id.empty() && decoded.has_payload_claim("clientId")) {
client_id = decoded.get_payload_claim("clientId").as_string();
}
string azp;
if (decoded.has_payload_claim("azp")) {
azp = decoded.get_payload_claim("azp").as_string();
}
string role_arn = s->info.args.get("RoleArn");
RGWOIDCProviderInfo provider;
int r = load_provider(dpp, y, role_arn, iss, provider);
if (r < 0) {
ldpp_dout(dpp, 0) << "Couldn't get oidc provider info using input iss" << iss << dendl;
throw -EACCES;
}
if (decoded.has_payload_claim(string(princTagsNamespace))) {
auto& cl = decoded.get_payload_claim(string(princTagsNamespace));
if (cl.get_type() == jwt::claim::type::object || cl.get_type() == jwt::claim::type::array) {
recurse_and_insert("dummy", cl, principal_tags);
for (auto it : principal_tags) {
ldpp_dout(dpp, 5) << "Key: " << it.first << " Value: " << it.second << dendl;
}
} else {
ldpp_dout(dpp, 0) << "Malformed principal tags" << cl.as_string() << dendl;
throw -EINVAL;
}
}
if (! provider.client_ids.empty()) {
bool found = false;
for (auto& it : aud) {
if (is_client_id_valid(provider.client_ids, it)) {
found = true;
break;
}
}
if (! found && ! is_client_id_valid(provider.client_ids, client_id) && ! is_client_id_valid(provider.client_ids, azp)) {
ldpp_dout(dpp, 0) << "Client id in token doesn't match with that registered with oidc provider" << dendl;
throw -EACCES;
}
}
//Validate signature
if (decoded.has_algorithm()) {
auto& algorithm = decoded.get_algorithm();
try {
validate_signature(dpp, decoded, algorithm, iss, provider.thumbprints, y);
} catch (...) {
throw -EACCES;
}
} else {
return {boost::none, boost::none};
}
} catch (int error) {
if (error == -EACCES) {
throw -EACCES;
}
ldpp_dout(dpp, 5) << "Invalid JWT token" << dendl;
return {boost::none, boost::none};
}
catch (...) {
ldpp_dout(dpp, 5) << "Invalid JWT token" << dendl;
return {boost::none, boost::none};
}
return {t, principal_tags};
}
std::string
WebTokenEngine::get_cert_url(const string& iss, const DoutPrefixProvider *dpp, optional_yield y) const
{
string cert_url;
string openidc_wellknown_url = iss;
bufferlist openidc_resp;
if (openidc_wellknown_url.back() == '/') {
openidc_wellknown_url.pop_back();
}
openidc_wellknown_url.append("/.well-known/openid-configuration");
RGWHTTPTransceiver openidc_req(cct, "GET", openidc_wellknown_url, &openidc_resp);
//Headers
openidc_req.append_header("Content-Type", "application/x-www-form-urlencoded");
int res = openidc_req.process(dpp, y);
if (res < 0) {
ldpp_dout(dpp, 10) << "HTTP request res: " << res << dendl;
throw -EINVAL;
}
//Debug only
ldpp_dout(dpp, 20) << "HTTP status: " << openidc_req.get_http_status() << dendl;
ldpp_dout(dpp, 20) << "JSON Response is: " << openidc_resp.c_str() << dendl;
JSONParser parser;
if (parser.parse(openidc_resp)) {
JSONObj::data_val val;
if (parser.get_data("jwks_uri", &val)) {
cert_url = val.str.c_str();
ldpp_dout(dpp, 20) << "Cert URL is: " << cert_url.c_str() << dendl;
} else {
ldpp_dout(dpp, 0) << "Malformed json returned while fetching openidc url" << dendl;
}
}
return cert_url;
}
void
WebTokenEngine::validate_signature(const DoutPrefixProvider* dpp, const jwt::decoded_jwt& decoded, const string& algorithm, const string& iss, const vector<string>& thumbprints, optional_yield y) const
{
if (algorithm != "HS256" && algorithm != "HS384" && algorithm != "HS512") {
string cert_url = get_cert_url(iss, dpp, y);
if (cert_url.empty()) {
throw -EINVAL;
}
// Get certificate
bufferlist cert_resp;
RGWHTTPTransceiver cert_req(cct, "GET", cert_url, &cert_resp);
//Headers
cert_req.append_header("Content-Type", "application/x-www-form-urlencoded");
int res = cert_req.process(dpp, y);
if (res < 0) {
ldpp_dout(dpp, 10) << "HTTP request res: " << res << dendl;
throw -EINVAL;
}
//Debug only
ldpp_dout(dpp, 20) << "HTTP status: " << cert_req.get_http_status() << dendl;
ldpp_dout(dpp, 20) << "JSON Response is: " << cert_resp.c_str() << dendl;
JSONParser parser;
if (parser.parse(cert_resp)) {
JSONObj::data_val val;
if (parser.get_data("keys", &val)) {
if (val.str[0] == '[') {
val.str.erase(0, 1);
}
if (val.str[val.str.size() - 1] == ']') {
val.str = val.str.erase(val.str.size() - 1, 1);
}
if (parser.parse(val)) {
vector<string> x5c;
if (JSONDecoder::decode_json("x5c", x5c, &parser)) {
string cert;
bool found_valid_cert = false;
for (auto& it : x5c) {
cert = "-----BEGIN CERTIFICATE-----\n" + it + "\n-----END CERTIFICATE-----";
ldpp_dout(dpp, 20) << "Certificate is: " << cert.c_str() << dendl;
if (is_cert_valid(thumbprints, cert)) {
found_valid_cert = true;
break;
}
}
if (! found_valid_cert) {
ldpp_dout(dpp, 0) << "Cert doesn't match that with the thumbprints registered with oidc provider: " << cert.c_str() << dendl;
throw -EINVAL;
}
try {
//verify method takes care of expired tokens also
if (algorithm == "RS256") {
auto verifier = jwt::verify()
.allow_algorithm(jwt::algorithm::rs256{cert});
verifier.verify(decoded);
} else if (algorithm == "RS384") {
auto verifier = jwt::verify()
.allow_algorithm(jwt::algorithm::rs384{cert});
verifier.verify(decoded);
} else if (algorithm == "RS512") {
auto verifier = jwt::verify()
.allow_algorithm(jwt::algorithm::rs512{cert});
verifier.verify(decoded);
} else if (algorithm == "ES256") {
auto verifier = jwt::verify()
.allow_algorithm(jwt::algorithm::es256{cert});
verifier.verify(decoded);
} else if (algorithm == "ES384") {
auto verifier = jwt::verify()
.allow_algorithm(jwt::algorithm::es384{cert});
verifier.verify(decoded);
} else if (algorithm == "ES512") {
auto verifier = jwt::verify()
.allow_algorithm(jwt::algorithm::es512{cert});
verifier.verify(decoded);
} else if (algorithm == "PS256") {
auto verifier = jwt::verify()
.allow_algorithm(jwt::algorithm::ps256{cert});
verifier.verify(decoded);
} else if (algorithm == "PS384") {
auto verifier = jwt::verify()
.allow_algorithm(jwt::algorithm::ps384{cert});
verifier.verify(decoded);
} else if (algorithm == "PS512") {
auto verifier = jwt::verify()
.allow_algorithm(jwt::algorithm::ps512{cert});
verifier.verify(decoded);
} else {
ldpp_dout(dpp, 0) << "Unsupported algorithm: " << algorithm << dendl;
throw -EINVAL;
}
} catch (std::runtime_error& e) {
ldpp_dout(dpp, 0) << "Signature validation failed: " << e.what() << dendl;
throw;
}
catch (...) {
ldpp_dout(dpp, 0) << "Signature validation failed" << dendl;
throw;
}
} else {
ldpp_dout(dpp, 0) << "x5c not present" << dendl;
throw -EINVAL;
}
} else {
ldpp_dout(dpp, 0) << "Malformed JSON object for keys" << dendl;
throw -EINVAL;
}
} else {
ldpp_dout(dpp, 0) << "keys not present in JSON" << dendl;
throw -EINVAL;
} //if-else get-data
} else {
ldpp_dout(dpp, 0) << "Malformed json returned while fetching cert" << dendl;
throw -EINVAL;
} //if-else parser cert_resp
} else {
ldpp_dout(dpp, 0) << "JWT signed by HMAC algos are currently not supported" << dendl;
throw -EINVAL;
}
}
WebTokenEngine::result_t
WebTokenEngine::authenticate( const DoutPrefixProvider* dpp,
const std::string& token,
const req_state* const s,
optional_yield y) const
{
if (! is_applicable(token)) {
return result_t::deny();
}
try {
auto [t, princ_tags] = get_from_jwt(dpp, token, s, y);
if (t) {
string role_session = s->info.args.get("RoleSessionName");
if (role_session.empty()) {
ldout(s->cct, 0) << "Role Session Name is empty " << dendl;
return result_t::deny(-EACCES);
}
string role_arn = s->info.args.get("RoleArn");
string role_tenant = get_role_tenant(role_arn);
string role_name = get_role_name(role_arn);
rgw_account_id role_account;
if (rgw::account::validate_id(role_tenant)) {
role_account = std::move(role_tenant);
role_tenant.clear();
}
std::unique_ptr<rgw::sal::RGWRole> role = driver->get_role(role_name, role_tenant, role_account);
int ret = role->load_by_name(dpp, y);
if (ret < 0) {
ldpp_dout(dpp, 0) << "Role not found: name:" << role_name << " tenant: " << role_tenant << dendl;
return result_t::deny(-EACCES);
}
std::optional<RGWAccountInfo> account;
if (!role_account.empty()) {
account.emplace();
rgw::sal::Attrs attrs; // ignored
RGWObjVersionTracker objv; // ignored
ret = driver->load_account_by_id(dpp, y, role_account,
*account, attrs, objv);
if (ret < 0) {
ldpp_dout(dpp, 0) << "Role account " << role_account << " not found" << dendl;
return result_t::deny(-EACCES);
}
}
boost::optional<multimap<string,string>> role_tags = role->get_tags();
auto apl = apl_factory->create_apl_web_identity(
cct, s, role->get_id(), role_session, role_tenant,
*t, role_tags, princ_tags, std::move(account));
return result_t::grant(std::move(apl));
}
return result_t::deny(-EACCES);
}
catch (...) {
return result_t::deny(-EACCES);
}
}
} // namespace rgw::auth::sts
int RGWREST_STS::verify_permission(optional_yield y)
{
STS::STSService _sts(s->cct, driver, s->user->get_id(), s->auth.identity.get());
sts = std::move(_sts);
string rArn = s->info.args.get("RoleArn");
const auto& [ret, role] = sts.getRoleInfo(s, rArn, y);
if (ret < 0) {
ldpp_dout(this, 0) << "failed to get role info using role arn: " << rArn << dendl;
return ret;
}
string policy = role->get_assume_role_policy();
//Parse the policy
//TODO - This step should be part of Role Creation
try {
// resource policy is not restricted to the current tenant
const std::string* policy_tenant = nullptr;
const rgw::IAM::Policy p(s->cct, policy_tenant, policy, false);
if (!s->principal_tags.empty()) {
auto res = p.eval(s->env, *s->auth.identity, rgw::IAM::stsTagSession, boost::none);
if (res != rgw::IAM::Effect::Allow) {
ldout(s->cct, 0) << "evaluating policy for stsTagSession returned deny/pass" << dendl;
return -EPERM;
}
}
uint64_t op;
if (get_type() == RGW_STS_ASSUME_ROLE_WEB_IDENTITY) {
op = rgw::IAM::stsAssumeRoleWithWebIdentity;
} else {
op = rgw::IAM::stsAssumeRole;
}
auto res = p.eval(s->env, *s->auth.identity, op, boost::none);
if (res != rgw::IAM::Effect::Allow) {
ldout(s->cct, 0) << "evaluating policy for op: " << op << " returned deny/pass" << dendl;
return -EPERM;
}
} catch (rgw::IAM::PolicyParseException& e) {
ldpp_dout(this, 0) << "failed to parse policy: " << e.what() << dendl;
return -EPERM;
}
return 0;
}
void RGWREST_STS::send_response()
{
if (op_ret) {
set_req_state_err(s, op_ret);
}
dump_errno(s);
end_header(s);
}
int RGWSTSGetSessionToken::verify_permission(optional_yield y)
{
rgw::Partition partition = rgw::Partition::aws;
rgw::Service service = rgw::Service::s3;
if (!verify_user_permission(this,
s,
rgw::ARN(partition, service, "", s->user->get_tenant(), ""),
rgw::IAM::stsGetSessionToken)) {
ldpp_dout(this, 0) << "User does not have permission to perform GetSessionToken" << dendl;
return -EACCES;
}
return 0;
}
int RGWSTSGetSessionToken::get_params()
{
duration = s->info.args.get("DurationSeconds");
serialNumber = s->info.args.get("SerialNumber");
tokenCode = s->info.args.get("TokenCode");
if (! duration.empty()) {
string err;
uint64_t duration_in_secs = strict_strtoll(duration.c_str(), 10, &err);
if (!err.empty()) {
ldpp_dout(this, 0) << "Invalid value of input duration: " << duration << dendl;
return -EINVAL;
}
if (duration_in_secs < STS::GetSessionTokenRequest::getMinDuration() ||
duration_in_secs > s->cct->_conf->rgw_sts_max_session_duration) {
ldpp_dout(this, 0) << "Invalid duration in secs: " << duration_in_secs << dendl;
return -EINVAL;
}
}
return 0;
}
void RGWSTSGetSessionToken::execute(optional_yield y)
{
if (op_ret = get_params(); op_ret < 0) {
return;
}
STS::STSService sts(s->cct, driver, s->user->get_id(), s->auth.identity.get());
STS::GetSessionTokenRequest req(duration, serialNumber, tokenCode);
const auto& [ret, creds] = sts.getSessionToken(this, req);
op_ret = std::move(ret);
//Dump the output
if (op_ret == 0) {
s->formatter->open_object_section_in_ns("GetSessionTokenResponse", RGW_REST_STS_XMLNS);
s->formatter->open_object_section("GetSessionTokenResult");
s->formatter->open_object_section("Credentials");
creds.dump(s->formatter);
s->formatter->close_section();
s->formatter->close_section();
s->formatter->close_section();
}
}
int RGWSTSAssumeRoleWithWebIdentity::get_params()
{
duration = s->info.args.get("DurationSeconds");
providerId = s->info.args.get("ProviderId");
policy = s->info.args.get("Policy");
roleArn = s->info.args.get("RoleArn");
roleSessionName = s->info.args.get("RoleSessionName");
iss = s->info.args.get("provider_id");
sub = s->info.args.get("sub");
aud = s->info.args.get("aud");
if (roleArn.empty() || roleSessionName.empty() || sub.empty() || aud.empty()) {
ldpp_dout(this, 0) << "ERROR: one of role arn or role session name or token is empty" << dendl;
return -EINVAL;
}
if (! policy.empty()) {
try {
const rgw::IAM::Policy p(
s->cct, nullptr, policy,
s->cct->_conf.get_val<bool>("rgw_policy_reject_invalid_principals"));
}
catch (rgw::IAM::PolicyParseException& e) {
ldpp_dout(this, 5) << "failed to parse policy: " << e.what() << "policy" << policy << dendl;
s->err.message = e.what();
return -ERR_MALFORMED_DOC;
}
}
return 0;
}
void RGWSTSAssumeRoleWithWebIdentity::execute(optional_yield y)
{
if (op_ret = get_params(); op_ret < 0) {
return;
}
STS::AssumeRoleWithWebIdentityRequest req(s->cct, duration, providerId, policy, roleArn,
roleSessionName, iss, sub, aud, s->principal_tags);
STS::AssumeRoleWithWebIdentityResponse response = sts.assumeRoleWithWebIdentity(this, req);
op_ret = std::move(response.assumeRoleResp.retCode);
//Dump the output
if (op_ret == 0) {
s->formatter->open_object_section_in_ns("AssumeRoleWithWebIdentityResponse", RGW_REST_STS_XMLNS);
s->formatter->open_object_section("AssumeRoleWithWebIdentityResult");
encode_json("SubjectFromWebIdentityToken", response.sub , s->formatter);
encode_json("Audience", response.aud , s->formatter);
s->formatter->open_object_section("AssumedRoleUser");
response.assumeRoleResp.user.dump(s->formatter);
s->formatter->close_section();
s->formatter->open_object_section("Credentials");
response.assumeRoleResp.creds.dump(s->formatter);
s->formatter->close_section();
encode_json("Provider", response.providerId , s->formatter);
encode_json("PackedPolicySize", response.assumeRoleResp.packedPolicySize , s->formatter);
s->formatter->close_section();
s->formatter->close_section();
}
}
int RGWSTSAssumeRole::get_params()
{
duration = s->info.args.get("DurationSeconds");
externalId = s->info.args.get("ExternalId");
policy = s->info.args.get("Policy");
roleArn = s->info.args.get("RoleArn");
roleSessionName = s->info.args.get("RoleSessionName");
serialNumber = s->info.args.get("SerialNumber");
tokenCode = s->info.args.get("TokenCode");
if (roleArn.empty() || roleSessionName.empty()) {
ldpp_dout(this, 0) << "ERROR: one of role arn or role session name is empty" << dendl;
return -EINVAL;
}
if (! policy.empty()) {
try {
const rgw::IAM::Policy p(
s->cct, nullptr, policy,
s->cct->_conf.get_val<bool>("rgw_policy_reject_invalid_principals"));
}
catch (rgw::IAM::PolicyParseException& e) {
ldpp_dout(this, 0) << "failed to parse policy: " << e.what() << "policy" << policy << dendl;
s->err.message = e.what();
return -ERR_MALFORMED_DOC;
}
}
return 0;
}
void RGWSTSAssumeRole::execute(optional_yield y)
{
if (op_ret = get_params(); op_ret < 0) {
return;
}
STS::AssumeRoleRequest req(s->cct, duration, externalId, policy, roleArn,
roleSessionName, serialNumber, tokenCode);
STS::AssumeRoleResponse response = sts.assumeRole(s, req, y);
op_ret = std::move(response.retCode);
//Dump the output
if (op_ret == 0) {
s->formatter->open_object_section_in_ns("AssumeRoleResponse", RGW_REST_STS_XMLNS);
s->formatter->open_object_section("AssumeRoleResult");
s->formatter->open_object_section("Credentials");
response.creds.dump(s->formatter);
s->formatter->close_section();
s->formatter->open_object_section("AssumedRoleUser");
response.user.dump(s->formatter);
s->formatter->close_section();
encode_json("PackedPolicySize", response.packedPolicySize , s->formatter);
s->formatter->close_section();
s->formatter->close_section();
}
}
int RGW_Auth_STS::authorize(const DoutPrefixProvider *dpp,
rgw::sal::Driver* driver,
const rgw::auth::StrategyRegistry& auth_registry,
req_state *s, optional_yield y)
{
return rgw::auth::Strategy::apply(dpp, auth_registry.get_sts(), s, y);
}
using op_generator = RGWOp*(*)();
static const std::unordered_map<std::string_view, op_generator> op_generators = {
{"AssumeRole", []() -> RGWOp* {return new RGWSTSAssumeRole;}},
{"GetSessionToken", []() -> RGWOp* {return new RGWSTSGetSessionToken;}},
{"AssumeRoleWithWebIdentity", []() -> RGWOp* {return new RGWSTSAssumeRoleWithWebIdentity;}}
};
bool RGWHandler_REST_STS::action_exists(const req_state* s)
{
if (s->info.args.exists("Action")) {
const std::string action_name = s->info.args.get("Action");
return op_generators.contains(action_name);
}
return false;
}
RGWOp *RGWHandler_REST_STS::op_post()
{
if (s->info.args.exists("Action")) {
const std::string action_name = s->info.args.get("Action");
const auto action_it = op_generators.find(action_name);
if (action_it != op_generators.end()) {
return action_it->second();
}
ldpp_dout(s, 10) << "unknown action '" << action_name << "' for STS handler" << dendl;
} else {
ldpp_dout(s, 10) << "missing action argument in STS handler" << dendl;
}
return nullptr;
}
int RGWHandler_REST_STS::init(rgw::sal::Driver* driver,
req_state *s,
rgw::io::BasicClient *cio)
{
s->dialect = "sts";
s->prot_flags = RGW_REST_STS;
return RGWHandler_REST::init(driver, s, cio);
}
int RGWHandler_REST_STS::authorize(const DoutPrefixProvider* dpp, optional_yield y)
{
if (s->info.args.exists("Action") && s->info.args.get("Action") == "AssumeRoleWithWebIdentity") {
return RGW_Auth_STS::authorize(dpp, driver, auth_registry, s, y);
}
return RGW_Auth_S3::authorize(dpp, driver, auth_registry, s, y);
}
RGWHandler_REST*
RGWRESTMgr_STS::get_handler(rgw::sal::Driver* driver,
req_state* const s,
const rgw::auth::StrategyRegistry& auth_registry,
const std::string& frontend_prefix)
{
return new RGWHandler_REST_STS(auth_registry);
}

15
src/rgw/rgw_rest_account.cc
View File

@ -14,6 +14,7 @@
*/
#include "rgw_rest_account.h"
#include "common/ceph_json.h"
#include "rgw_account.h"
#include "rgw_process_env.h"
@ -73,9 +74,8 @@ void RGWOp_Account_Create::execute(optional_yield y)
if (!driver->is_meta_master()) {
bufferlist data;
JSONParser parser;
op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
&data, &parser, s->info, s->err, y);
&data, nullptr, s->info, s->err, y);
if (op_ret < 0) {
ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
return;
@ -83,7 +83,16 @@ void RGWOp_Account_Create::execute(optional_yield y)
// the master zone may have generated its own account id, use the same
std::string meta_master_id;
JSONDecoder::decode_json("id", meta_master_id, &parser);
try {
const auto response = ceph_json::parse(data);
meta_master_id = ceph_json::require<std::string>(response, "id");
} catch (const JSONDecoder::err& e) {
ldpp_dout(this, 4) << "forward_request_to_master returned invalid account id: "
<< e.what() << dendl;
op_ret = -EINVAL;
return;
}
if (meta_master_id.empty()) {
ldpp_dout(this, 4) << "forward_request_to_master returned empty account id" << dendl;
op_ret = -EINVAL;

2
src/rgw/rgw_rest_client.cc
View File

@ -446,8 +446,6 @@ auto RGWRESTSimpleRequest::forward_request(const DoutPrefixProvider *dpp, const
return tl::unexpected(-ERR_SERVICE_UNAVAILABLE);
}
response.append((char)0); /* NULL terminate response */
if (outbl) {
*outbl = std::move(response);
}

2
src/rgw/rgw_rest_conn.h
View File

@ -17,7 +17,7 @@ template<class T>
inline int parse_decode_json(T& t, bufferlist& bl)
{
JSONParser p;
if (!p.parse(bl.c_str(), bl.length())) {
if (!p.parse(bl)) {
return -EINVAL;
}

2
src/rgw/rgw_rest_role.cc
View File

@ -449,7 +449,7 @@ int RGWModifyRoleTrustPolicy::init_processing(optional_yield y)
}
JSONParser p;
if (!p.parse(trust_policy.c_str(), trust_policy.length())) {
if (!p.parse(trust_policy)) {
ldpp_dout(this, 20) << "ERROR: failed to parse assume role policy doc" << dendl;
return -ERR_MALFORMED_DOC;
}

9
src/rgw/rgw_rest_s3.cc
View File

@ -3488,16 +3488,17 @@ int RGWPostObj_ObjStore_S3::get_policy(optional_yield y)
return -EINVAL;
}
decoded_policy.append('\0'); // NULL terminate
if('\0' != decoded_policy.c_str()[decoded_policy.length() - 1])
decoded_policy.append('\0'); // NULL terminate
ldpp_dout(this, 20) << "POST policy: " << decoded_policy.c_str() << dendl;
int r = post_policy.from_json(decoded_policy, err_msg);
int r = post_policy.from_json(std::string_view { decoded_policy.c_str() }, err_msg);
if (r < 0) {
if (err_msg.empty()) {
err_msg = "Failed to parse policy";
}
ldpp_dout(this, 0) << "failed to parse policy" << dendl;
ldpp_dout(this, 0) << err_msg << dendl;
return -EINVAL;
}

51
src/rgw/rgw_rest_sts.cc
View File

@ -324,7 +324,7 @@ WebTokenEngine::get_cert_url(const string& iss, const DoutPrefixProvider *dpp, o
ldpp_dout(dpp, 20) << "JSON Response is: " << openidc_resp.c_str() << dendl;
JSONParser parser;
if (parser.parse(openidc_resp.c_str(), openidc_resp.length())) {
if (parser.parse(openidc_resp)) {
JSONObj::data_val val;
if (parser.get_data("jwks_uri", &val)) {
cert_url = val.str.c_str();
@ -633,24 +633,21 @@ WebTokenEngine::validate_signature(const DoutPrefixProvider* dpp, const jwt::dec
ldpp_dout(dpp, 20) << "HTTP status: " << cert_req.get_http_status() << dendl;
ldpp_dout(dpp, 20) << "JSON Response is: " << cert_resp.c_str() << dendl;
JSONParser parser;
if (parser.parse(cert_resp.c_str(), cert_resp.length())) {
JSONObj* val = parser.find_obj("keys");
if (val && val->is_array()) {
vector<string> keys = val->get_array_elements();
for (auto &key : keys) {
JSONParser k_parser;
vector<string> x5c;
std::string use, kid;
if (k_parser.parse(key.c_str(), key.size())) {
if (JSONDecoder::decode_json("kid", kid, &k_parser)) {
ldpp_dout(dpp, 20) << "Checking key id: " << kid << dendl;
}
if (JSONDecoder::decode_json("use", use, &k_parser) && use != "sig") {
continue;
}
ceph_json::value cert_json;
if (ceph_json::parse(cert_resp, cert_json)) {
try {
for (const auto& key : ceph_json::require_array(cert_json, "keys")) {
if (const auto kid = ceph_json::field<std::string>(key, "kid"); kid) {
ldpp_dout(dpp, 20) << "Checking key id: " << *kid << dendl;
}
if (JSONDecoder::decode_json("x5c", x5c, &k_parser)) {
if (const auto use = ceph_json::field<std::string>(key, "use");
use && "sig" != *use) {
continue;
}
if (const auto x5c = ceph_json::field<std::vector<std::string>>(key, "x5c");
x5c) {
string cert;
bool found_valid_cert = false;
bool skip_thumbprint_verification = cct->_conf.get_val<bool>(
@ -661,7 +658,7 @@ WebTokenEngine::validate_signature(const DoutPrefixProvider* dpp, const jwt::dec
<< dendl;
throw std::system_error(EINVAL, std::system_category());
}
for (auto& it : x5c) {
for (auto& it : *x5c) {
cert = "-----BEGIN CERTIFICATE-----\n" + it + "\n-----END CERTIFICATE-----";
ldpp_dout(dpp, 20) << "Certificate is: " << cert.c_str() << dendl;
if (skip_thumbprint_verification || is_cert_valid(thumbprints, cert)) {
@ -739,27 +736,27 @@ WebTokenEngine::validate_signature(const DoutPrefixProvider* dpp, const jwt::dec
} else {
// Try bare key validation
ldpp_dout(dpp, 20) << "Trying bare key validation" << dendl;
std::string kty;
if (JSONDecoder::decode_json("kty", kty, &k_parser) && kty != "RSA") {
if (const auto kty = ceph_json::field<std::string>(key, "kty");
kty && "RSA" != *kty) {
ldpp_dout(dpp, 10) << "Only RSA bare key validation is currently supported" << dendl;
continue;
}
if (algorithm == "RS256" || algorithm == "RS384" || algorithm == "RS512") {
std::string n, e; //modulus and exponent
if (JSONDecoder::decode_json("n", n, &k_parser) && JSONDecoder::decode_json("e", e, &k_parser)) {
if (validate_signature_using_n_e(dpp, decoded, algorithm, n, e)) {
const auto n = ceph_json::field<std::string>(key, "n"); // modulus
const auto e = ceph_json::field<std::string>(key, "e"); // exponent
if (n && e) {
if (validate_signature_using_n_e(dpp, decoded, algorithm, *n, *e)) {
return;
}
}
ldpp_dout(dpp, 10) << "Bare key parameters (n&e) are not present for key" << dendl;
}
}
} //end k_parser.parse
} //end for iterate through keys
}
ldpp_dout(dpp, 0) << "Signature can not be validated with the JWKS present." << dendl;
throw std::system_error(EINVAL, std::system_category());
} else { //end val->is_array
} catch (const JSONDecoder::err&) {
ldpp_dout(dpp, 0) << "keys not present in JSON" << dendl;
throw std::system_error(EINVAL, std::system_category());
}

4
src/rgw/rgw_token.h
View File

@ -82,13 +82,13 @@ namespace rgw {
explicit RGWToken(const string& json) {
JSONParser p;
p.parse(json.c_str(), json.length());
p.parse(json);
JSONDecoder::decode_json(RGWToken::type_name, *this, &p);
}
RGWToken& operator=(const std::string& json) {
JSONParser p;
p.parse(json.c_str(), json.length());
p.parse(json);
JSONDecoder::decode_json(RGWToken::type_name, *this, &p);
return *this;
}

34
src/test/common/test_json_formattable.cc
View File

@ -4,6 +4,7 @@
/*
* Ceph - scalable distributed file system
*
* Copyright (C) 2026 IBM
* Copyright (C) 2018 Red Hat Inc.
*
* This library is free software; you can redistribute it and/or
@ -38,7 +39,7 @@ static void get_jf(const string& s, JSONFormattable *f)
}
}
TEST(formatable, str) {
TEST(formattable, str) {
JSONFormattable f;
get_jf("{ \"foo\": \"bar\" }", &f);
ASSERT_EQ((string)f["foo"], "bar");
@ -46,7 +47,7 @@ TEST(formatable, str) {
ASSERT_EQ((string)f["fooz"]("lala"), "lala");
}
TEST(formatable, str2) {
TEST(formattable, str2) {
JSONFormattable f;
get_jf("{ \"foo\": \"bar\" }", &f);
ASSERT_EQ((string)f["foo"], "bar");
@ -62,13 +63,13 @@ TEST(formatable, str2) {
}
TEST(formatable, str3) {
TEST(formattable, str3) {
JSONFormattable f;
get_jf("{ \"foo\": \"1234bar56\" }", &f);
ASSERT_EQ((string)f["foo"], "1234bar56");
}
TEST(formatable, int) {
TEST(formattable, int) {
JSONFormattable f;
get_jf("{ \"foo\": 1 }", &f);
ASSERT_EQ((int)f["foo"], 1);
@ -83,7 +84,7 @@ TEST(formatable, int) {
ASSERT_EQ((int)f2["fooz"](111), 123);
}
TEST(formatable, bool) {
TEST(formattable, bool) {
JSONFormattable f;
get_jf("{ \"foo\": \"true\" }", &f);
ASSERT_EQ((bool)f["foo"], true);
@ -95,7 +96,7 @@ TEST(formatable, bool) {
ASSERT_EQ((bool)f["foo"], false);
}
TEST(formatable, nested) {
TEST(formattable, nested) {
JSONFormattable f;
get_jf("{ \"obj\": { \"foo\": 1, \"inobj\": { \"foo\": 2 } } }", &f);
ASSERT_EQ((int)f["foo"], 0);
@ -103,7 +104,7 @@ TEST(formatable, nested) {
ASSERT_EQ((int)f["obj"]["inobj"]["foo"], 2);
}
TEST(formatable, array) {
TEST(formattable, array) {
JSONFormattable f;
get_jf("{ \"arr\": [ { \"foo\": 1, \"inobj\": { \"foo\": 2 } },"
"{ \"foo\": 2 } ] }", &f);
@ -124,7 +125,7 @@ TEST(formatable, array) {
}
}
TEST(formatable, bin_encode) {
TEST(formattable, bin_encode) {
JSONFormattable f, f2;
get_jf("{ \"arr\": [ { \"foo\": 1, \"bar\": \"aaa\", \"inobj\": { \"foo\": 2 } },"
"{ \"foo\": 2, \"inobj\": { \"foo\": 3 } } ] }", &f);
@ -156,7 +157,7 @@ TEST(formatable, bin_encode) {
}
TEST(formatable, json_encode) {
TEST(formattable, json_encode) {
JSONFormattable f, f2;
get_jf("{ \"arr\": [ { \"foo\": 1, \"bar\": \"aaa\", \"inobj\": { \"foo\": 2 } },"
"{ \"foo\": 2, \"inobj\": { \"foo\": 3 } } ] }", &f);
@ -181,7 +182,7 @@ TEST(formatable, json_encode) {
}
TEST(formatable, set) {
TEST(formattable, set) {
JSONFormattable f, f2;
f.set("", "{ \"abc\": \"xyz\"}");
@ -200,13 +201,13 @@ TEST(formatable, set) {
ASSERT_EQ((int)f["obj"]["c"], 30);
}
TEST(formatable, set2) {
TEST(formattable, set2) {
JSONFormattable f;
f.set("foo", "1234bar56");
ASSERT_EQ((string)f["foo"], "1234bar56");
}
TEST(formatable, erase) {
TEST(formattable, erase) {
JSONFormattable f, f2;
f.set("", "{ \"abc\": \"xyz\"}");
@ -240,7 +241,7 @@ static void dumpf(const JSONFormattable& f) {
dumpt(f, "f");
}
TEST(formatable, set_array) {
TEST(formattable, set_array) {
JSONFormattable f, f2;
f.set("asd[0]", "\"xyz\"");
@ -272,7 +273,7 @@ TEST(formatable, set_array) {
ASSERT_EQ((string)f2[0]["field"], "xyz");
}
TEST(formatable, erase_array) {
TEST(formattable, erase_array) {
JSONFormattable f;
f.set("asd[0]", "\"xyz\"");
@ -315,14 +316,13 @@ void formatter_convert(JSONFormatter& formatter, JSONFormattable *dest)
get_jf(ss.str(), dest);
}
TEST(formatable, encode_simple) {
TEST(formattable, encode_simple) {
JSONFormattable f;
encode_json("foo", "bar", &f);
ASSERT_EQ((string)f["foo"], "bar");
JSONFormatter formatter;
{
Formatter::ObjectSection s(formatter, "os");
@ -422,7 +422,7 @@ struct struct2 {
};
TEST(formatable, encode_struct) {
TEST(formattable, encode_struct) {
JSONFormattable f;
struct2 s2;

523
src/test/common/test_json_formatter.cc
View File

@ -4,6 +4,7 @@
/*
* Ceph - scalable distributed file system
*
* Copyright (C) 2025-2026 IBM
* Copyright (C) 2018 Red Hat Inc.
*
* This library is free software; you can redistribute it and/or
@ -21,18 +22,18 @@
#include "common/JSONFormatter.h"
#include "common/StackStringStream.h"
#include <algorithm>
#include <ranges>
#include <sstream>
using namespace std;
const string outstring =
"{\"pg_ready\":true, \"pg_stats\":[ { \"pgid\":\"1.0\", \"version\":\"16'56\",\"reported_seq\":\"62\",\"reported_epoch\":\"20\",\"state\":\"active+clean+inconsistent\",\"last_fresh\":\"2018-12-18 15:21:22.173804\",\"last_change\":\"2018-12-18 15:21:22.173804\",\"last_active\":\"2018-12-18 15:21:22.173804\",\"last_peered\":\"2018-12-18 15:21:22.173804\",\"last_clean\":\"2018-12-18 15:21:22.173804\",\"last_became_active\":\"2018-12-18 15:21:21.347685\",\"last_became_peered\":\"2018-12-18 15:21:21.347685\",\"last_unstale\":\"2018-12-18 15:21:22.173804\",\"last_undegraded\":\"2018-12-18 15:21:22.173804\",\"last_fullsized\":\"2018-12-18 15:21:22.173804\",\"mapping_epoch\":19,\"log_start\":\"0'0\",\"ondisk_log_start\":\"0'0\",\"created\":7,\"last_epoch_clean\":20,\"parent\":\"0.0\",\"parent_split_bits\":0,\"last_scrub\":\"16'56\",\"last_scrub_stamp\":\"2018-12-18 15:21:22.173684\",\"last_deep_scrub\":\"0'0\",\"last_deep_scrub_stamp\":\"2018-12-18 15:21:06.514438\",\"last_clean_scrub_stamp\":\"2018-12-18 15:21:06.514438\",\"log_size\":56,\"ondisk_log_size\":56,\"stats_invalid\":false,\"dirty_stats_invalid\":false,\"omap_stats_invalid\":false,\"hitset_stats_invalid\":false,\"hitset_bytes_stats_invalid\":false,\"pin_stats_invalid\":false,\"manifest_stats_invalid\":false,\"snaptrimq_len\":0,\"stat_sum\":{\"num_bytes\":24448,\"num_objects\":36,\"num_object_clones\":20,\"num_object_copies\":36,\"num_objects_missing_on_primary\":0,\"num_objects_missing\":0,\"num_objects_degraded\":0,\"num_objects_misplaced\":0,\"num_objects_unfound\":0,\"num_objects_dirty\":36,\"num_whiteouts\":3,\"num_read\":0,\"num_read_kb\":0,\"num_write\":36,\"num_write_kb\":50,\"num_scrub_errors\":20,\"num_shallow_scrub_errors\":20,\"num_deep_scrub_errors\":0,\"num_objects_recovered\":0,\"num_bytes_recovered\":0,\"num_keys_recovered\":0,\"num_objects_omap\":0,\"num_objects_hit_set_archive\":0,\"num_bytes_hit_set_archive\":0,\"num_flush\":0,\"num_flush_kb\":0,\"num_evict\":0,\"num_evict_kb\":0,\"num_promote\":0,\"num_flush_mode_high\":0,\"num_flush_mode_low\":0,\"num_evict_mode_some\":0,\"num_evict_mode_full\":0,\"num_objects_pinned\":0,\"num_legacy_snapsets\":0,\"num_large_omap_objects\":0,\"num_objects_manifest\":0},\"up\":[0],\"acting\":[0],\"blocked_by\":[],\"up_primary\":0,\"acting_primary\":0,\"purged_snaps\":[] }]}";
TEST(formatter, bug_37706) {
vector<std::string> pgs;
string outstring =
"{\"pg_ready\":true, \"pg_stats\":[ { \"pgid\":\"1.0\", \"version\":\"16'56\",\"reported_seq\":\"62\",\"reported_epoch\":\"20\",\"state\":\"active+clean+inconsistent\",\"last_fresh\":\"2018-12-18 15:21:22.173804\",\"last_change\":\"2018-12-18 15:21:22.173804\",\"last_active\":\"2018-12-18 15:21:22.173804\",\"last_peered\":\"2018-12-18 15:21:22.173804\",\"last_clean\":\"2018-12-18 15:21:22.173804\",\"last_became_active\":\"2018-12-18 15:21:21.347685\",\"last_became_peered\":\"2018-12-18 15:21:21.347685\",\"last_unstale\":\"2018-12-18 15:21:22.173804\",\"last_undegraded\":\"2018-12-18 15:21:22.173804\",\"last_fullsized\":\"2018-12-18 15:21:22.173804\",\"mapping_epoch\":19,\"log_start\":\"0'0\",\"ondisk_log_start\":\"0'0\",\"created\":7,\"last_epoch_clean\":20,\"parent\":\"0.0\",\"parent_split_bits\":0,\"last_scrub\":\"16'56\",\"last_scrub_stamp\":\"2018-12-18 15:21:22.173684\",\"last_deep_scrub\":\"0'0\",\"last_deep_scrub_stamp\":\"2018-12-18 15:21:06.514438\",\"last_clean_scrub_stamp\":\"2018-12-18 15:21:06.514438\",\"log_size\":56,\"ondisk_log_size\":56,\"stats_invalid\":false,\"dirty_stats_invalid\":false,\"omap_stats_invalid\":false,\"hitset_stats_invalid\":false,\"hitset_bytes_stats_invalid\":false,\"pin_stats_invalid\":false,\"manifest_stats_invalid\":false,\"snaptrimq_len\":0,\"stat_sum\":{\"num_bytes\":24448,\"num_objects\":36,\"num_object_clones\":20,\"num_object_copies\":36,\"num_objects_missing_on_primary\":0,\"num_objects_missing\":0,\"num_objects_degraded\":0,\"num_objects_misplaced\":0,\"num_objects_unfound\":0,\"num_objects_dirty\":36,\"num_whiteouts\":3,\"num_read\":0,\"num_read_kb\":0,\"num_write\":36,\"num_write_kb\":50,\"num_scrub_errors\":20,\"num_shallow_scrub_errors\":20,\"num_deep_scrub_errors\":0,\"num_objects_recovered\":0,\"num_bytes_recovered\":0,\"num_keys_recovered\":0,\"num_objects_omap\":0,\"num_objects_hit_set_archive\":0,\"num_bytes_hit_set_archive\":0,\"num_flush\":0,\"num_flush_kb\":0,\"num_evict\":0,\"num_evict_kb\":0,\"num_promote\":0,\"num_flush_mode_high\":0,\"num_flush_mode_low\":0,\"num_evict_mode_some\":0,\"num_evict_mode_full\":0,\"num_objects_pinned\":0,\"num_legacy_snapsets\":0,\"num_large_omap_objects\":0,\"num_objects_manifest\":0},\"up\":[0],\"acting\":[0],\"blocked_by\":[],\"up_primary\":0,\"acting_primary\":0,\"purged_snaps\":[] }]}";
JSONParser parser;
ASSERT_TRUE(parser.parse(outstring.c_str(), outstring.size()));
@ -128,3 +129,517 @@ TEST(formatter, dump_large_item) {
EXPECT_TRUE(parser.parse(bl.c_str(), bl.length()));
EXPECT_EQ(parser.find_obj("Location")->get_data(), full_url);
}
TEST(formatter, parse_types) {
// Check that JSONParser::parse() works with expected data types
// (otherwise at least indirectly tested elsewhere):
const auto json_input = R"({"expiration": "2025-01-17T10:26:46Z", "conditions": [{"bucket": "user-hqfadib9zxyj2pygevewzf45t-1"}, ["starts-with", "$key", "foo"], {"acl": "private"}, ["starts-with", "$Content-Type", "text/plain"], ["content-length-range", 0, 1024]]})";
{
JSONParser parser;
ASSERT_TRUE(parser.parse(outstring));
JSONObj *pgstat_obj = parser.find_obj("pg_stats");
EXPECT_TRUE(pgstat_obj);
}
{
JSONParser parser;
ASSERT_TRUE(parser.parse(std::string_view { outstring }));
JSONObj *pgstat_obj = parser.find_obj("pg_stats");
EXPECT_TRUE(pgstat_obj);
}
{
JSONParser parser;
buffer::list bl;
bl.append(outstring);
EXPECT_TRUE(parser.parse(bl.c_str(), bl.length()));
JSONObj *pgstat_obj = parser.find_obj("pg_stats");
EXPECT_TRUE(pgstat_obj);
}
{
JSONParser parser;
buffer::list bl;
bl.append(json_input);
ASSERT_TRUE(parser.parse(bl.c_str(), bl.length()));
JSONObjIter oi = parser.find_first();
JSONObj *o0 = *oi;
ASSERT_TRUE(nullptr != o0);
++oi;
JSONObj *o1 = *oi;
ASSERT_TRUE(nullptr != o1);
JSONObj *conditions = parser.find_obj("conditions");
ASSERT_TRUE(nullptr != conditions);
JSONObj *expiration = parser.find_obj("expiration");
ASSERT_TRUE(nullptr != expiration);
}
}
/* JFW: both of these fail again, AS THEY SHOULD:
-- IMO we need to fix any b0rken feeds
-- but, let's re-run this in the test suite so that we can get a comparison
before we patch things...
TEST(formatter, malformed_input_as_string) {
// A couple of the common error cases I see now that we're using a more conformant JSON parser come
// from differences in what malformed input should be accepted as. The two most common cases are:
// 1) parsing fails at first character, a non-delimited field meant to be a string;
// 2) parsing fails just before the end, extra NULL byte
// ...ceph_json should honor these gnarly inputs, at least for now.
const string_view bare_url = "http://turkey.straw.com";
const char garbage_at_end[] = { 's', 't', 'r', '\x00', '\x00' };
JFW: tmp
{
JSONParser parser;
ASSERT_TRUE(parser.parse(bare_url));
}
{
JSONParser parser;
ASSERT_TRUE(parser.parse(garbage_at_end));
}
}*/
namespace {
void assert_parse(JSONParser& parser, std::string_view json)
{
ASSERT_TRUE(parser.parse(json.data(), json.size())) << json;
}
JSONObj* assert_find(JSONObj& obj, const char *name)
{
auto *child = obj.find_obj(name);
EXPECT_NE(nullptr, child) << name;
return child;
}
struct compat_record {
string name;
int count = 0;
bool enabled = false;
vector<int> values;
map<string, int> by_name;
void dump(Formatter *f) const {
encode_json("name", name, f);
encode_json("count", count, f);
encode_json("enabled", enabled, f);
encode_json("values", values, f);
encode_json("by_name", by_name, f);
}
void decode_json(JSONObj *obj) {
JSONDecoder::decode_json("name", name, obj, true);
JSONDecoder::decode_json("count", count, obj, true);
JSONDecoder::decode_json("enabled", enabled, obj, true);
JSONDecoder::decode_json("values", values, obj, true);
JSONDecoder::decode_json("by_name", by_name, obj, true);
}
};
} // anonymous namespace
TEST(formatter, ceph_json_scalar_root_compatibility)
{
struct scalar_case {
string_view input;
string_view data;
bool quoted;
};
const scalar_case cases[] = {
{R"("plain string")", "plain string", true},
{"-42", "-42", false},
{"42", "42", false},
{"true", "true", false},
{"false", "false", false},
{"null", "null", false},
};
for (const auto& c : cases) {
JSONParser parser;
assert_parse(parser, c.input);
EXPECT_FALSE(parser.is_array()) << c.input;
EXPECT_FALSE(parser.is_object()) << c.input;
EXPECT_EQ(c.data, parser.get_data()) << c.input;
EXPECT_EQ(c.quoted, parser.get_data_val().quoted) << c.input;
}
}
TEST(formatter, ceph_json_native_handle_access)
{
static_assert(std::same_as<JSONParser::native_handle_type, ceph_json::value>);
constexpr string_view json = R"({"name":"fast","values":[1,2,3]})";
JSONParser parser;
assert_parse(parser, json);
auto& native = parser.native_handle();
ASSERT_TRUE(native.is_object());
const auto& const_parser = parser;
static_assert(std::same_as<decltype(const_parser.native_handle()),
const JSONParser::native_handle_type&>);
EXPECT_TRUE(const_parser.native_handle().is_object());
auto *name = assert_find(parser, "name");
const auto& native_name = name->native_handle();
ASSERT_TRUE(native_name.is_string());
EXPECT_EQ("fast", native_name.as<std::string>());
auto *values_obj = assert_find(parser, "values");
const auto values = ceph_json::array_view(values_obj->native_handle());
EXPECT_EQ(3u, values.size());
}
TEST(formatter, ceph_json_native_field_helpers)
{
constexpr string_view json =
R"({"name":"fast","enabled":true,"nested":{"secret":"open"},"values":["a","b"]})";
ceph_json::value root;
ASSERT_TRUE(ceph_json::parse(json, root));
const auto parsed_root = ceph_json::parse(json);
string name;
ASSERT_TRUE(ceph_json::decode_field("name", name, root));
EXPECT_EQ("fast", name);
EXPECT_EQ("fast", ceph_json::require<string>(parsed_root, "name"));
bool enabled = false;
ceph_json::decode_required_field("enabled", enabled, root);
EXPECT_TRUE(enabled);
EXPECT_TRUE(ceph_json::require<bool>(parsed_root, "enabled"));
const auto& members = ceph_json::require_object(parsed_root);
EXPECT_EQ(4u, std::ranges::size(members));
auto* nested = ceph_json::find_value(root, "nested");
ASSERT_NE(nullptr, nested);
string secret;
ceph_json::decode_required_field("secret", secret, *nested);
EXPECT_EQ("open", secret);
EXPECT_EQ("open", ceph_json::require<string>(
ceph_json::require_object(parsed_root, "nested"), "secret"));
vector<string> values;
ceph_json::decode_required_field("values", values, root);
ASSERT_EQ(2u, values.size());
EXPECT_EQ("a", values[0]);
EXPECT_EQ("b", values[1]);
EXPECT_EQ(values, ceph_json::require<vector<string>>(parsed_root, "values"));
const auto value_nodes = ceph_json::require_array(parsed_root, "values");
ASSERT_EQ(2u, value_nodes.size());
EXPECT_EQ("a", ceph_json::decode_value<string>(value_nodes.front()));
EXPECT_EQ("b", ceph_json::decode_value<string>(value_nodes.back()));
vector<string> viewed_values;
std::ranges::transform(
value_nodes, back_inserter(viewed_values),
[] (const auto& item) {
return ceph_json::decode_value<string>(item);
});
EXPECT_EQ(values, viewed_values);
int missing = 7;
EXPECT_FALSE(ceph_json::decode_field("missing", missing, root));
EXPECT_EQ(0, missing);
EXPECT_EQ(9, ceph_json::field_or(parsed_root, "missing", 9));
EXPECT_EQ(nullopt, ceph_json::field<int>(parsed_root, "missing"));
EXPECT_THROW(ceph_json::decode_required_field("missing", missing, root),
JSONDecoder::err);
EXPECT_THROW(ceph_json::require<int>(parsed_root, "missing"),
JSONDecoder::err);
EXPECT_THROW(ceph_json::require_object(parsed_root, "name"),
JSONDecoder::err);
EXPECT_THROW(ceph_json::require_array(parsed_root, "name"),
JSONDecoder::err);
EXPECT_THROW(ceph_json::require_array(parsed_root, "missing"),
JSONDecoder::err);
EXPECT_EQ(0u, ceph_json::array_view(parsed_root).size());
bufferlist bl;
bl.append(json);
bl.append("\0", 1);
ceph_json::value from_bl;
ASSERT_TRUE(ceph_json::parse(bl, from_bl));
string bl_name;
ceph_json::decode_required_field("name", bl_name, from_bl);
EXPECT_EQ("fast", bl_name);
EXPECT_EQ("fast", ceph_json::require<string>(ceph_json::parse(bl), "name"));
}
TEST(formatter, ceph_json_object_tree_compatibility)
{
constexpr string_view json =
R"({"name":"main","nested":{"n":7,"flag":false},)"
R"("tricky":"{not: [json], comma, quote \" }","escaped\"key":"ok",)"
R"("array":[1,"two",{"three":3},null],"empty":[]})";
JSONParser parser;
assert_parse(parser, json);
ASSERT_TRUE(parser.is_object());
auto *name = assert_find(parser, "name");
ASSERT_NE(nullptr, name);
EXPECT_EQ("main", name->get_data());
EXPECT_TRUE(name->get_data_val().quoted);
auto *tricky = assert_find(parser, "tricky");
ASSERT_NE(nullptr, tricky);
EXPECT_EQ(R"({not: [json], comma, quote " })", tricky->get_data());
auto *escaped_key = assert_find(parser, R"(escaped"key)");
ASSERT_NE(nullptr, escaped_key);
EXPECT_EQ("ok", escaped_key->get_data());
JSONObj::data_val name_attr;
ASSERT_TRUE(name->get_attr("name", name_attr));
EXPECT_EQ("main", name_attr.str);
EXPECT_TRUE(name_attr.quoted);
auto *nested = assert_find(parser, "nested");
ASSERT_NE(nullptr, nested);
EXPECT_TRUE(nested->is_object());
int n = 0;
ASSERT_TRUE(JSONDecoder::decode_json("n", n, nested, true));
EXPECT_EQ(7, n);
bool flag = true;
ASSERT_TRUE(JSONDecoder::decode_json("flag", flag, nested, true));
EXPECT_FALSE(flag);
auto *missing = parser.find_obj("missing");
EXPECT_EQ(nullptr, missing);
}
TEST(formatter, ceph_json_named_find_first_stops_after_named_child)
{
constexpr string_view json = R"({"alpha":1,"needle":"found","z_tail":2})";
JSONParser parser;
assert_parse(parser, json);
auto iter = parser.find_first("needle");
ASSERT_FALSE(iter.end());
EXPECT_EQ("needle", (*iter)->get_name());
EXPECT_EQ("found", (*iter)->get_data());
++iter;
EXPECT_TRUE(iter.end());
}
TEST(formatter, ceph_json_duplicate_object_members_known_regression)
{
/*
* The legacy JSONObj tree could preserve repeated object members with the
* same name. JSON itself only says object names SHOULD be unique, and an
* audit of Ceph and src/tools users found no semantic dependency on this:
* ordinary decoders use the first named field, map-like decoders expect
* explicit {"key", "val"} entries, and object iteration is used for arrays
* or generic display. We intentionally follow Glaze's object model here.
*/
constexpr string_view json =
R"({"dupe":"first","dupe":"second",)"
R"("nested":{"again":"first","again":"second"},)"
R"("array":[{"three":3,"three":4}]})";
JSONParser parser;
assert_parse(parser, json);
auto dupes = parser.find("dupe");
vector<string> values;
for (; !dupes.end(); ++dupes) {
values.push_back((*dupes)->get_data());
}
ASSERT_EQ(1u, values.size());
EXPECT_TRUE(values[0] == "first" || values[0] == "second");
auto *nested = assert_find(parser, "nested");
auto again = nested->find("again");
vector<string> nested_values;
for (; !again.end(); ++again) {
nested_values.push_back((*again)->get_data());
}
ASSERT_EQ(1u, nested_values.size());
EXPECT_TRUE(nested_values[0] == "first" || nested_values[0] == "second");
auto *array = assert_find(parser, "array");
const auto elements = array->get_array_elements();
ASSERT_EQ(1u, elements.size());
JSONParser object_element;
assert_parse(object_element, elements[0]);
auto threes = object_element.find("three");
vector<string> object_values;
for (; !threes.end(); ++threes) {
object_values.push_back((*threes)->get_data());
}
ASSERT_EQ(1u, object_values.size());
EXPECT_TRUE(object_values[0] == "3" || object_values[0] == "4");
}
TEST(formatter, ceph_json_array_elements_compatibility)
{
constexpr string_view json =
R"({"array":[1,"two",{"three":3},[4,5],null],"empty":[]})";
JSONParser parser;
assert_parse(parser, json);
auto *array = assert_find(parser, "array");
ASSERT_NE(nullptr, array);
ASSERT_TRUE(array->is_array());
const auto elements = array->get_array_elements();
ASSERT_EQ(5u, elements.size());
EXPECT_EQ("1", elements[0]);
EXPECT_EQ(R"("two")", elements[1]);
EXPECT_EQ("null", elements[4]);
JSONParser object_element;
assert_parse(object_element, elements[2]);
int three = 0;
ASSERT_TRUE(JSONDecoder::decode_json("three", three, &object_element, true));
EXPECT_EQ(3, three);
JSONParser array_element;
assert_parse(array_element, elements[3]);
vector<int> nested;
decode_json_obj(nested, &array_element);
ASSERT_EQ(2u, nested.size());
EXPECT_EQ(4, nested[0]);
EXPECT_EQ(5, nested[1]);
auto *empty = assert_find(parser, "empty");
ASSERT_NE(nullptr, empty);
EXPECT_TRUE(empty->get_array_elements().empty());
}
TEST(formatter, ceph_json_decoder_compatibility)
{
constexpr string_view json =
R"({"s":"text","i":-42,"u":42,"b_true":true,"b_false":false,)"
R"("b_num":1,"values":[1,2,3],)"
R"("by_name":[{"key":"alpha","val":10},{"key":"beta","val":20}],)"
R"("maybe":99})";
JSONParser parser;
assert_parse(parser, json);
string s;
ASSERT_TRUE(JSONDecoder::decode_json("s", s, &parser, true));
EXPECT_EQ("text", s);
int i = 0;
ASSERT_TRUE(JSONDecoder::decode_json("i", i, &parser, true));
EXPECT_EQ(-42, i);
unsigned u = 0;
ASSERT_TRUE(JSONDecoder::decode_json("u", u, &parser, true));
EXPECT_EQ(42u, u);
bool b = false;
ASSERT_TRUE(JSONDecoder::decode_json("b_true", b, &parser, true));
EXPECT_TRUE(b);
b = true;
ASSERT_TRUE(JSONDecoder::decode_json("b_false", b, &parser, true));
EXPECT_FALSE(b);
b = false;
ASSERT_TRUE(JSONDecoder::decode_json("b_num", b, &parser, true));
EXPECT_TRUE(b);
vector<int> values;
ASSERT_TRUE(JSONDecoder::decode_json("values", values, &parser, true));
ASSERT_EQ(3u, values.size());
EXPECT_EQ(1, values[0]);
EXPECT_EQ(2, values[1]);
EXPECT_EQ(3, values[2]);
map<string, int> by_name;
ASSERT_TRUE(JSONDecoder::decode_json("by_name", by_name, &parser, true));
ASSERT_EQ(2u, by_name.size());
EXPECT_EQ(10, by_name["alpha"]);
EXPECT_EQ(20, by_name["beta"]);
boost::optional<int> maybe;
ASSERT_TRUE(JSONDecoder::decode_json("maybe", maybe, &parser, true));
ASSERT_TRUE(maybe);
EXPECT_EQ(99, *maybe);
int missing = 123;
EXPECT_FALSE(JSONDecoder::decode_json("missing", missing, &parser));
EXPECT_EQ(0, missing);
boost::optional<int> missing_optional = 7;
EXPECT_FALSE(JSONDecoder::decode_json("missing_optional", missing_optional, &parser));
EXPECT_FALSE(missing_optional);
EXPECT_THROW(JSONDecoder::decode_json("missing_required", missing, &parser, true),
JSONDecoder::err);
}
TEST(formatter, ceph_json_encode_decode_compatibility)
{
compat_record input;
input.name = "encoded";
input.count = 17;
input.enabled = true;
input.values = {3, 5, 8};
input.by_name = {{"first", 1}, {"second", 2}};
JSONFormatter formatter;
formatter.open_object_section("root");
encode_json("record", input, &formatter);
formatter.close_section();
stringstream ss;
formatter.flush(ss);
JSONParser parser;
assert_parse(parser, ss.str());
auto *record_obj = assert_find(parser, "record");
ASSERT_NE(nullptr, record_obj);
compat_record output;
decode_json_obj(output, record_obj);
EXPECT_EQ(input.name, output.name);
EXPECT_EQ(input.count, output.count);
EXPECT_EQ(input.enabled, output.enabled);
EXPECT_EQ(input.values, output.values);
EXPECT_EQ(input.by_name, output.by_name);
}

2
src/test/objectstore/allocator_replay_test.cc
View File

@ -274,7 +274,7 @@ int replay_free_dump_and_apply_raw(
JSONParser p;
std::cout << "parsing..." << std::endl;
bool b = p.parse(fname);
bool b = p.parse_file(fname);
if (!b) {
std::cerr << "Failed to parse json: " << fname << std::endl;
return -1;

2
src/test/rgw/rgw_multi/multisite.py
View File

@ -1,6 +1,7 @@
from abc import ABCMeta, abstractmethod
from io import StringIO
import sys
import json
from .conn import get_gateway_connection, get_gateway_s3_resource, get_gateway_iam_connection, get_gateway_secure_connection, get_gateway_sns_client, get_gateway_sts_connection, get_gateway_temp_s3_client
@ -82,6 +83,7 @@ class SystemObject:
s, r = self.command(cluster, cmd, args or [], **kwargs)
if r == 0:
data = json.loads(s)
# To show the entire command: json.dump(data, sys.stdout)
self.load_from_json(data)
self.data = data
return self.data, r

4
src/tools/cephfs/MetaTool.cc
View File

@ -354,7 +354,7 @@ void MetaTool::inode_meta_t::encode(::ceph::bufferlist& bl, uint64_t features)
int MetaTool::_amend_meta(string& k, inode_meta_t& inode_meta, const string& fn, meta_op& op)
{
JSONParser parser;
if (!parser.parse(fn.c_str())) {
if (!parser.parse(fn)) {
cout << "Error parsing create user response" << std::endl;
return -1;
}
@ -479,7 +479,7 @@ int MetaTool::amend_fn(meta_op &op)
int MetaTool::_amend_fn(const string& fn, bool confirm)
{
JSONParser parser;
if (!parser.parse(fn.c_str())) {
if (!parser.parse(fn)) {
cout << "Error parsing create user response : " << fn << std::endl;
return -1;
}

8
src/tools/cephfs_mirror/InstanceWatcher.cc
View File

@ -89,10 +89,10 @@ void InstanceWatcher::handle_notify(uint64_t notify_id, uint64_t handle,
std::string mode;
bool purging = false;
try {
JSONDecoder jd(bl);
JSONDecoder::decode_json("dir_path", dir_path, &jd.parser, true);
JSONDecoder::decode_json("mode", mode, &jd.parser, true);
JSONDecoder::decode_json("purging", purging, &jd.parser, false);
const auto notify = ceph_json::parse(bl);
dir_path = ceph_json::require<std::string>(notify, "dir_path");
mode = ceph_json::require<std::string>(notify, "mode");
purging = ceph_json::field_or(notify, "purging", false);
} catch (const JSONDecoder::err &e) {
derr << ": failed to decode notify json: " << e.what() << dendl;
}

2
src/tools/rbd/action/Journal.cc
View File

@ -840,7 +840,7 @@ public:
n++;
error_count++;
JSONParser p;
if (!p.parse(bl.c_str(), bl.length())) {
if (!p.parse(bl)) {
std::cerr << "rbd: error parsing input (entry " << n << ")"
<< std::endl;
r = -EINVAL;