diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ade2bf75d8..c652242dd3d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -161,7 +161,6 @@ if(FREEBSD) list(APPEND CMAKE_REQUIRED_INCLUDES /usr/local/include) endif(FREEBSD) - #put all the libs and binaries in one place set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) @@ -200,6 +199,35 @@ if(NOT HAVE_CXX11_ATOMIC) " ${LIBATOMIC_LINK_FLAGS}") endif() +include(CephAddCPM) + +add_cpm("Glaze" FALSE "gh:stephenberry/glaze@7.6.0#v7.6.0") +# Some CPM/FetchContent paths populate Glaze without adding its alias target. +if(NOT TARGET glaze::glaze) + if(DEFINED glaze_SOURCE_DIR) + set(CEPH_GLAZE_SOURCE_DIR "${glaze_SOURCE_DIR}") + elseif(DEFINED CPM_PACKAGE_glaze_SOURCE_DIR) + set(CEPH_GLAZE_SOURCE_DIR "${CPM_PACKAGE_glaze_SOURCE_DIR}") + elseif(DEFINED Glaze_SOURCE_DIR) + set(CEPH_GLAZE_SOURCE_DIR "${Glaze_SOURCE_DIR}") + endif() + + if(NOT CEPH_GLAZE_SOURCE_DIR OR NOT EXISTS "${CEPH_GLAZE_SOURCE_DIR}/include/glaze/glaze.hpp") + message(FATAL_ERROR + "Glaze was added by CPM, but no glaze::glaze target or source directory was found") + endif() + + add_library(glaze_glaze INTERFACE) + add_library(glaze::glaze ALIAS glaze_glaze) + target_include_directories(glaze_glaze SYSTEM INTERFACE + "${CEPH_GLAZE_SOURCE_DIR}/include") +endif() +if(TARGET glaze_glaze) + target_compile_definitions(glaze_glaze INTERFACE GLZ_USE_STD_FORMAT_FLOAT=0) +endif() +get_target_property(GLAZE_INCLUDE_DIRS glaze::glaze INTERFACE_INCLUDE_DIRECTORIES) +include_directories(SYSTEM ${GLAZE_INCLUDE_DIRS}) + option(WITH_RDMA "Enable RDMA in async messenger" ON) if(WITH_RDMA) find_package(verbs REQUIRED) @@ -538,27 +566,7 @@ option(WITH_CATCH2 "Enable Catch2 unit tests" ON) option(WITH_SYSTEM_CATCH2 "Enable Catch2 unit tests (require system packages)" OFF) if(WITH_CATCH2) - # Save pre-existing definitions (if any): - set(ORIG_CPM_LOCAL_PACKAGES_ONLY ${CPM_USE_LOCAL_PACKAGES_ONLY}) - - if(WITH_SYSTEM_CATCH2) - set(CPM_LOCAL_PACKAGES_ONLY ON) - endif() - - include(${CMAKE_MODULE_PATH}/CPM.cmake) - CPMAddPackage("gh:catchorg/Catch2@3.8.1") - - # Restore the original CPM settings in case someone else wants to use the module: - set(CPM_USE_LOCAL_PACKAGES_ONLY, ${ORIG_CPM_USE_LOCAL_PACKAGES_ONLY}) - - # CPM skips the fetch silently if downloads are disabled, - # e.g. with FETCHCONTENT_FULLY_DISCONNECTED=ON on deb builds - if(NOT TARGET Catch2::Catch2WithMain) - message(WARNING "Catch2 unavailable, disabling Catch2 tests") - set(WITH_CATCH2 OFF) - else() - message("-- Enabled Catch2 support") - endif() + add_cpm("Catch2" FALSE "gh:catchorg/Catch2@3.8.1") endif() # enable breakpad unless win32 or power diff --git a/cmake/modules/CephAddCPM.cmake b/cmake/modules/CephAddCPM.cmake new file mode 100644 index 00000000000..d73dc01f93d --- /dev/null +++ b/cmake/modules/CephAddCPM.cmake @@ -0,0 +1,30 @@ +# Add CPM packages in a consistent, Ceph-friendly way: + +set(CEPH_ORIG_CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}") +include("${CMAKE_CURRENT_LIST_DIR}/CPM.cmake") +set(CMAKE_MODULE_PATH "${CEPH_ORIG_CMAKE_MODULE_PATH}") + +# Two settings govern CPM's behavior with respect to local packages: +# CPM_USE_LOCAL_PACKAGES and CPM_LOCAL_PACKAGES_ONLY. The first prefers local +# packages; the second requires them, e.g. for an offline-only build. +# +# This macro saves and restores both settings. If CPM_LOCAL_PACKAGES_ONLY is +# already set, it is honored. + +macro(add_cpm PACKAGE_NAME USE_LOCAL_FLAG) + set(_ORIG_CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}") + set(_ORIG_CPM_USE_LOCAL_PACKAGES "${CPM_USE_LOCAL_PACKAGES}") + set(_ORIG_CPM_LOCAL_PACKAGES_ONLY "${CPM_LOCAL_PACKAGES_ONLY}") + + if(${USE_LOCAL_FLAG}) + set(CPM_USE_LOCAL_PACKAGES ON) + endif() + + CPMAddPackage(${ARGN}) + + set(CPM_USE_LOCAL_PACKAGES "${_ORIG_CPM_USE_LOCAL_PACKAGES}") + set(CPM_LOCAL_PACKAGES_ONLY "${_ORIG_CPM_LOCAL_PACKAGES_ONLY}") + set(CMAKE_MODULE_PATH "${_ORIG_CMAKE_MODULE_PATH}") + + message(STATUS "-- Enabled ${PACKAGE_NAME} support.") +endmacro() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5a836c88cba..bcf017ec06b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -609,6 +609,7 @@ set(ceph_common_objs $) set(ceph_common_deps common_utf8 extblkdev arch crc32 + ceph-json ${LIB_RESOLV} Boost::thread Boost::context diff --git a/src/cls/CMakeLists.txt b/src/cls/CMakeLists.txt index cfcc2f86790..56e24e1891f 100644 --- a/src/cls/CMakeLists.txt +++ b/src/cls/CMakeLists.txt @@ -100,9 +100,9 @@ endif (WITH_RADOSGW) # cls_refcount set(cls_refcount_srcs refcount/cls_refcount.cc - refcount/cls_refcount_ops.cc - ${CMAKE_SOURCE_DIR}/src/common/ceph_json.cc) + refcount/cls_refcount_ops.cc) add_library(cls_refcount SHARED ${cls_refcount_srcs}) +target_link_libraries(cls_refcount ceph-json) set_target_properties(cls_refcount PROPERTIES VERSION "1.0.0" SOVERSION "1" @@ -202,10 +202,9 @@ if (WITH_RADOSGW) set(cls_rgw_srcs rgw/cls_rgw.cc rgw/cls_rgw_ops.cc - rgw/cls_rgw_types.cc - ${CMAKE_SOURCE_DIR}/src/common/ceph_json.cc) + rgw/cls_rgw_types.cc) add_library(cls_rgw SHARED ${cls_rgw_srcs}) - target_link_libraries(cls_rgw ${FMT_LIB}) + target_link_libraries(cls_rgw ${FMT_LIB} ceph-json) target_include_directories(cls_rgw PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw/driver/rados" PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw") @@ -300,9 +299,9 @@ install(TARGETS cls_cas DESTINATION ${cls_dir}) #cls_queue set(cls_queue_srcs queue/cls_queue.cc - queue/cls_queue_src.cc - ${CMAKE_SOURCE_DIR}/src/common/ceph_json.cc) + queue/cls_queue_src.cc) add_library(cls_queue SHARED ${cls_queue_srcs}) +target_link_libraries(cls_queue ceph-json) set_target_properties(cls_queue PROPERTIES VERSION "1.0.0" SOVERSION "1" @@ -318,9 +317,9 @@ add_library(cls_queue_client STATIC ${cls_queue_client_srcs}) if (WITH_RADOSGW) set(cls_rgw_gc_srcs rgw_gc/cls_rgw_gc.cc - queue/cls_queue_src.cc - ${CMAKE_SOURCE_DIR}/src/common/ceph_json.cc) + queue/cls_queue_src.cc) add_library(cls_rgw_gc SHARED ${cls_rgw_gc_srcs}) + target_link_libraries(cls_rgw_gc ceph-json) target_include_directories(cls_rgw_gc PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw/driver/rados" PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw") @@ -352,9 +351,9 @@ endif (WITH_RADOSGW) #cls_2pc_queue set(cls_2pc_queue_srcs 2pc_queue/cls_2pc_queue.cc - queue/cls_queue_src.cc - ${CMAKE_SOURCE_DIR}/src/common/ceph_json.cc) + queue/cls_queue_src.cc) add_library(cls_2pc_queue SHARED ${cls_2pc_queue_srcs}) +target_link_libraries(cls_2pc_queue ceph-json) set_target_properties(cls_2pc_queue PROPERTIES VERSION "1.0.0" SOVERSION "1" diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 8f8f1dd0ba0..7094b0acbfc 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -8,6 +8,9 @@ add_library(common_prioritycache_obj OBJECT PriorityCache.cc) target_link_libraries(common_prioritycache_obj legacy-option-headers) +add_library(ceph-json OBJECT ceph_json.cc) +target_link_libraries(ceph-json PUBLIC glaze::glaze) + if(WIN32) add_library(dlfcn_win32 STATIC win32/dlfcn.cc win32/errno.cc) endif() @@ -59,7 +62,6 @@ set(common_srcs ceph_frag.cc ceph_fs.cc ceph_hash.cc - ceph_json.cc ceph_strings.cc ceph_releases.cc ceph_time.cc @@ -209,7 +211,7 @@ target_compile_definitions(common-common-objs PRIVATE "CEPH_INSTALL_FULL_PKGLIBDIR=\"${CEPH_INSTALL_FULL_PKGLIBDIR}\"" "CEPH_INSTALL_DATADIR=\"${CEPH_INSTALL_DATADIR}\"" $) -target_link_libraries(common-common-objs PUBLIC legacy-option-headers) +target_link_libraries(common-common-objs PUBLIC legacy-option-headers ceph-json) if(WITH_BREAKPAD) target_link_libraries(common-common-objs PUBLIC Breakpad::client) diff --git a/src/common/ceph_json.cc b/src/common/ceph_json.cc index ccb5f32ecfb..32a54cef169 100644 --- a/src/common/ceph_json.cc +++ b/src/common/ceph_json.cc @@ -1,720 +1,287 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2025-2026 International Business Machines Corp. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * +*/ + #include "common/ceph_json.h" #include "include/utime.h" -// for testing DELETE ME -#include #include -#include #include #include +#include -#include "json_spirit/json_spirit_writer_template.h" +#include "glaze/glaze.hpp" -using namespace json_spirit; +#include +#include +#include +#include using std::ifstream; -using std::pair; using std::ostream; -using std::string; + +using std::begin; +using std::end; + +using std::pair; using std::vector; +using std::string; +using std::string_view; + using ceph::bufferlist; using ceph::Formatter; -#define dout_subsys ceph_subsys_rgw - -static JSONFormattable default_formattable; - -void encode_json(const char *name, const JSONObj::data_val& v, Formatter *f) -{ - if (v.quoted) { - encode_json(name, v.str, f); - } else { - f->dump_format_unquoted(name, "%s", v.str.c_str()); - } -} - -JSONObjIter::JSONObjIter() -{ -} - -JSONObjIter::~JSONObjIter() -{ -} - -void JSONObjIter::set(const JSONObjIter::map_iter_t &_cur, const JSONObjIter::map_iter_t &_last) -{ - cur = _cur; - last = _last; -} - -void JSONObjIter::operator++() -{ - if (cur != last) - ++cur; -} - -JSONObj *JSONObjIter::operator*() -{ - return cur->second; -} - // does not work, FIXME ostream& operator<<(ostream &out, const JSONObj &obj) { out << obj.name << ": " << obj.val; return out; } -JSONObj::~JSONObj() +JSONObj::JSONObj(JSONObj *parent_node, std::string_view name_in, + const ceph_json::value& data_in) + : parent { parent_node }, + name { name_in }, + data(data_in) { - for (auto iter = children.begin(); iter != children.end(); ++iter) { - JSONObj *obj = iter->second; - delete obj; + handle_value(data); +} + +void JSONObj::cache_data_value(std::string_view s, bool quoted) const +{ + val.set(s, quoted); + data_value_ready = true; +} + +void JSONObj::ensure_data_value() const +{ + if (data_value_ready) + return; + + if (data.is_string()) { + cache_data_value(data.as(), true); + return; } -} - -void JSONObj::add_child(string el, JSONObj *obj) -{ - children.insert(pair(el, obj)); -} - -bool JSONObj::get_attr(string name, data_val& attr) -{ - auto iter = attr_map.find(name); - if (iter == attr_map.end()) - return false; - attr = iter->second; - return true; -} - -JSONObjIter JSONObj::find(const string& name) -{ - JSONObjIter iter; - auto first = children.find(name); - if (first != children.end()) { - auto last = children.upper_bound(name); - iter.set(first, last); + std::string json_data; + if (auto ec = glz::write_json(data, json_data); ec) { + throw JSONDecoderErr(fmt::format("failed to parse JSON input: {}", + glz::format_error(ec, name))); } - return iter; -} -JSONObjIter JSONObj::find_first() -{ - JSONObjIter iter; - iter.set(children.begin(), children.end()); - return iter; -} - -JSONObjIter JSONObj::find_first(const string& name) -{ - JSONObjIter iter; - auto first = children.find(name); - iter.set(first, children.end()); - return iter; -} - -JSONObj *JSONObj::find_obj(const string& name) -{ - JSONObjIter iter = find(name); - if (iter.end()) - return NULL; - - return *iter; -} - -bool JSONObj::get_data(const string& key, data_val *dest) -{ - JSONObj *obj = find_obj(key); - if (!obj) - return false; - - *dest = obj->get_data_val(); - - return true; + cache_data_value(json_data, false); } /* accepts a JSON Array or JSON Object contained in - * a JSON Spirit Value, v, and creates a JSONObj for each + * a Glaze value, v, and creates a Ceph JSONObj for each * child contained in v */ -void JSONObj::handle_value(Value v) +void JSONObj::handle_value(const ceph_json::value& v) { - if (v.type() == obj_type) { - Object temp_obj = v.get_obj(); - for (Object::size_type i = 0; i < temp_obj.size(); i++) { - Pair temp_pair = temp_obj[i]; - string temp_name = temp_pair.name_; - Value temp_value = temp_pair.value_; - JSONObj *child = new JSONObj; - child->init(this, temp_value, temp_name); - add_child(temp_name, child); + children.clear(); + + if (auto *o = ceph_json::object(v)) { + children.reserve(o->size()); + + for (const auto& [k, v] : *o) { + children.emplace_back(std::make_unique(this, k, v)); } - } else if (v.type() == array_type) { - Array temp_array = v.get_array(); - Value value; - for (unsigned j = 0; j < temp_array.size(); j++) { - Value cur = temp_array[j]; - string temp_name; + std::ranges::sort(children, std::less<>{}, child_name); - JSONObj *child = new JSONObj; - child->init(this, cur, temp_name); - add_child(child->get_name(), child); + return; + } + + auto elements = ceph_json::array_view(v); + if (!elements.empty()) { + children.reserve(elements.size()); + + for (const auto& kv : elements) { + children.emplace_back(std::make_unique(this, "", kv)); } } -} -void JSONObj::init(JSONObj *p, Value v, string n) -{ - name = n; - parent = p; - data = v; - - handle_value(v); - if (v.type() == str_type) { - val.set(v.get_str(), true); - } else { - val.set(json_spirit::write_string(v), false); - } - attr_map.insert(pair(name, val)); -} - -JSONObj *JSONObj::get_parent() -{ - return parent; -} - -bool JSONObj::is_object() -{ - return (data.type() == obj_type); -} - -bool JSONObj::is_array() -{ - return (data.type() == array_type); + // unknown type is not-an-error (as per the original implementation) } vector JSONObj::get_array_elements() { - vector elements; - Array temp_array; + if (!data.is_array()) + return {}; - if (data.type() == array_type) - temp_array = data.get_array(); + const auto elements = ceph_json::array_view(data); - int array_size = temp_array.size(); - if (array_size > 0) - for (int i = 0; i < array_size; i++) { - Value temp_value = temp_array[i]; - string temp_string; - temp_string = write(temp_value, raw_utf8); - elements.push_back(temp_string); + vector decoded_elements; + decoded_elements.reserve(elements.size()); + + std::ranges::transform(elements, std::back_inserter(decoded_elements), [](const auto& element) { + auto json_str = glz::write_json(element); + + if (!json_str) { + throw JSONDecoder::err(glz::format_error(json_str)); } - return elements; -} + return std::move(*json_str); + }); -JSONParser::JSONParser() : buf_len(0), success(true) -{ -} - -JSONParser::~JSONParser() -{ -} - - - -void JSONParser::handle_data(const char *s, int len) -{ - json_buffer.append(s, len); // check for problems with null termination FIXME - buf_len += len; -} - -// parse a supplied JSON fragment -bool JSONParser::parse(const char *buf_, int len) -{ - if (!buf_) { - set_failure(); - return false; - } - - string json_string(buf_, len); - success = read(json_string, data); - if (success) { - handle_value(data); - if (data.type() != obj_type && - data.type() != array_type) { - if (data.type() == str_type) { - val.set(data.get_str(), true); - } else { - const std::string& s = json_spirit::write_string(data); - if (s.size() == (uint64_t)len) { /* Check if entire string is read */ - val.set(s, false); - } else { - set_failure(); - } - } - } - } else { - set_failure(); - } - - return success; -} - -// parse the internal json_buffer up to len -bool JSONParser::parse(int len) -{ - string json_string = json_buffer.substr(0, len); - success = read(json_string, data); - if (success) - handle_value(data); - else - set_failure(); - - return success; + return decoded_elements; } // parse the complete internal json_buffer bool JSONParser::parse() { - success = read(json_buffer, data); - if (success) - handle_value(data); - else - set_failure(); + if (!parse_json(json_buffer, data)) + return false; - return success; -} + handle_value(data); + data_value_ready = false; -// parse a supplied ifstream, for testing. DELETE ME -bool JSONParser::parse(const char *file_name) -{ - ifstream is(file_name); - success = read(is, data); - if (success) - handle_value(data); - else - set_failure(); - - return success; -} - - -void decode_json_obj(long& val, JSONObj *obj) -{ - string s = obj->get_data(); - const char *start = s.c_str(); - char *p; - - errno = 0; - val = strtol(start, &p, 10); - - /* Check for various possible errors */ - - if ((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN)) || - (errno != 0 && val == 0)) { - throw JSONDecoder::err("failed to parse number"); - } - - if (p == start) { - throw JSONDecoder::err("failed to parse number"); - } - - while (*p != '\0') { - if (!isspace(*p)) { - throw JSONDecoder::err("failed to parse number"); - } - p++; - } -} - -void decode_json_obj(unsigned long& val, JSONObj *obj) -{ - string s = obj->get_data(); - const char *start = s.c_str(); - char *p; - - errno = 0; - val = strtoul(start, &p, 10); - - /* Check for various possible errors */ - - if ((errno == ERANGE && val == ULONG_MAX) || - (errno != 0 && val == 0)) { - throw JSONDecoder::err("failed to number"); - } - - if (p == start) { - throw JSONDecoder::err("failed to parse number"); - } - - while (*p != '\0') { - if (!isspace(*p)) { - throw JSONDecoder::err("failed to parse number"); - } - p++; - } -} - -void decode_json_obj(long long& val, JSONObj *obj) -{ - string s = obj->get_data(); - const char *start = s.c_str(); - char *p; - - errno = 0; - val = strtoll(start, &p, 10); - - /* Check for various possible errors */ - - if ((errno == ERANGE && (val == LLONG_MAX || val == LLONG_MIN)) || - (errno != 0 && val == 0)) { - throw JSONDecoder::err("failed to parse number"); - } - - if (p == start) { - throw JSONDecoder::err("failed to parse number"); - } - - while (*p != '\0') { - if (!isspace(*p)) { - throw JSONDecoder::err("failed to parse number"); - } - p++; - } -} - -void decode_json_obj(unsigned long long& val, JSONObj *obj) -{ - string s = obj->get_data(); - const char *start = s.c_str(); - char *p; - - errno = 0; - val = strtoull(start, &p, 10); - - /* Check for various possible errors */ - - if ((errno == ERANGE && val == ULLONG_MAX) || - (errno != 0 && val == 0)) { - throw JSONDecoder::err("failed to number"); - } - - if (p == start) { - throw JSONDecoder::err("failed to parse number"); - } - - while (*p != '\0') { - if (!isspace(*p)) { - throw JSONDecoder::err("failed to parse number"); - } - p++; - } -} - -void decode_json_obj(int& val, JSONObj *obj) -{ - long l; - decode_json_obj(l, obj); -#if LONG_MAX > INT_MAX - if (l > INT_MAX || l < INT_MIN) { - throw JSONDecoder::err("integer out of range"); - } -#endif - - val = (int)l; -} - -void decode_json_obj(unsigned& val, JSONObj *obj) -{ - unsigned long l; - decode_json_obj(l, obj); -#if ULONG_MAX > UINT_MAX - if (l > UINT_MAX) { - throw JSONDecoder::err("unsigned integer out of range"); - } -#endif - - val = (unsigned)l; -} - -void decode_json_obj(bool& val, JSONObj *obj) -{ - string s = obj->get_data(); - if (strcasecmp(s.c_str(), "true") == 0) { - val = true; - return; - } - if (strcasecmp(s.c_str(), "false") == 0) { - val = false; - return; - } - int i; - decode_json_obj(i, obj); - val = (bool)i; -} - -void decode_json_obj(bufferlist& val, JSONObj *obj) -{ - string s = obj->get_data(); - - bufferlist bl; - bl.append(s.c_str(), s.size()); - try { - val.decode_base64(bl); - } catch (ceph::buffer::error& err) { - throw JSONDecoder::err("failed to decode base64"); - } -} - -void decode_json_obj(utime_t& val, JSONObj *obj) -{ - string s = obj->get_data(); - uint64_t epoch; - uint64_t nsec; - int r = utime_t::parse_date(s, &epoch, &nsec); - if (r == 0) { - val = utime_t(epoch, nsec); - } else { - throw JSONDecoder::err("failed to decode utime_t"); - } -} - -void decode_json_obj(ceph::real_time& val, JSONObj *obj) -{ - const std::string& s = obj->get_data(); - uint64_t epoch; - uint64_t nsec; - int r = utime_t::parse_date(s, &epoch, &nsec); - if (r == 0) { - using namespace std::chrono; - val = real_time{seconds(epoch) + nanoseconds(nsec)}; - } else { - throw JSONDecoder::err("failed to decode real_time"); - } -} - -void decode_json_obj(ceph::coarse_real_time& val, JSONObj *obj) -{ - const std::string& s = obj->get_data(); - uint64_t epoch; - uint64_t nsec; - int r = utime_t::parse_date(s, &epoch, &nsec); - if (r == 0) { - using namespace std::chrono; - val = coarse_real_time{seconds(epoch) + nanoseconds(nsec)}; - } else { - throw JSONDecoder::err("failed to decode coarse_real_time"); - } -} - -void decode_json_obj(ceph_dir_layout& i, JSONObj *obj){ - - unsigned tmp; - JSONDecoder::decode_json("dir_hash", tmp, obj, true); - i.dl_dir_hash = tmp; - JSONDecoder::decode_json("unused1", tmp, obj, true); - i.dl_unused1 = tmp; - JSONDecoder::decode_json("unused2", tmp, obj, true); - i.dl_unused2 = tmp; - JSONDecoder::decode_json("unused3", tmp, obj, true); - i.dl_unused3 = tmp; -} - -void encode_json(const char *name, std::string_view val, Formatter *f) -{ - f->dump_string(name, val); -} - -void encode_json(const char *name, const string& val, Formatter *f) -{ - f->dump_string(name, val); -} - -void encode_json(const char *name, const char *val, Formatter *f) -{ - f->dump_string(name, val); -} - -void encode_json(const char *name, bool val, Formatter *f) -{ - f->dump_bool(name, val); -} - -void encode_json(const char *name, int val, Formatter *f) -{ - f->dump_int(name, val); -} - -void encode_json(const char *name, long val, Formatter *f) -{ - f->dump_int(name, val); -} - -void encode_json(const char *name, unsigned val, Formatter *f) -{ - f->dump_unsigned(name, val); -} - -void encode_json(const char *name, unsigned long val, Formatter *f) -{ - f->dump_unsigned(name, val); -} - -void encode_json(const char *name, unsigned long long val, Formatter *f) -{ - f->dump_unsigned(name, val); -} - -void encode_json(const char *name, long long val, Formatter *f) -{ - f->dump_int(name, val); -} - -void encode_json(const char *name, const utime_t& val, Formatter *f) -{ - val.gmtime(f->dump_stream(name)); -} - -void encode_json(const char *name, const ceph::real_time& val, Formatter *f) -{ - encode_json(name, utime_t{val}, f); -} - -void encode_json(const char *name, const ceph::coarse_real_time& val, Formatter *f) -{ - encode_json(name, utime_t{val}, f); -} - -void encode_json(const char *name, const bufferlist& bl, Formatter *f) -{ - /* need to copy data from bl, as it is const bufferlist */ - bufferlist src = bl; - - bufferlist b64; - src.encode_base64(b64); - - string s(b64.c_str(), b64.length()); - - encode_json(name, s, f); -} - - - -/* JSONFormattable */ - -const JSONFormattable& JSONFormattable::operator[](const string& name) const -{ - auto i = obj.find(name); - if (i == obj.end()) { - return default_formattable; - } - return i->second; -} - -const JSONFormattable& JSONFormattable::operator[](size_t index) const -{ - if (index >= arr.size()) { - return default_formattable; - } - return arr[index]; -} - -JSONFormattable& JSONFormattable::operator[](const string& name) -{ - auto i = obj.find(name); - if (i == obj.end()) { - return default_formattable; - } - return i->second; -} - -JSONFormattable& JSONFormattable::operator[](size_t index) -{ - if (index >= arr.size()) { - return default_formattable; - } - return arr[index]; -} - -bool JSONFormattable::exists(const string& name) const -{ - auto i = obj.find(name); - return (i != obj.end()); -} - -bool JSONFormattable::exists(size_t index) const -{ - return (index < arr.size()); -} - -bool JSONFormattable::find(const string& name, string *val) const -{ - auto i = obj.find(name); - if (i == obj.end()) { - return false; - } - *val = i->second.val(); return true; } -int JSONFormattable::val_int() const { - return atoi(value.str.c_str()); +// parse the internal json_buffer up to len +bool JSONParser::parse(int len) +{ + const auto json = std::string_view { json_buffer }.substr(0, len); + + if (!parse_json(json, data)) + return false; + + handle_value(data); + data_value_ready = false; + + return true; } -long JSONFormattable::val_long() const { - return atol(value.str.c_str()); +// parse a supplied JSON fragment: +bool JSONParser::parse(std::string_view json_string_view) +{ + // The original implementation checked this, I'm not sure we have to but we're doing it for now: + if (json_string_view.empty()) + return false; + + if (!parse_json(json_string_view, data)) + return false; + + // recursively evaluate the result: + handle_value(data); + data_value_ready = false; + + if (data.is_object() or data.is_array()) + return true; + + if (data.is_string()) { + cache_data_value(data.get(), true); + return true; + } + + // For any other kind of value: + std::string s; + if(auto ec = glz::write_json(data, s); ec) { + throw JSONDecoder::err(glz::format_error(ec)); + } + + // Was the entire string read? + if (s.size() == static_cast(json_string_view.length())) { + cache_data_value(s, false); + return true; + } + + // Could not parse and convert: + return false; } -long long JSONFormattable::val_long_long() const { - return atoll(value.str.c_str()); +bool JSONParser::parse_file(const std::filesystem::path file_name) +{ + if (auto ec = glz::read_file_json(data, file_name.string(), std::string()); ec) { + return false; + } + + handle_value(data); + data_value_ready = false; + + return true; } +/* JSONFormattable */ + bool JSONFormattable::val_bool() const { return (boost::iequals(value.str, "true") || + boost::iequals(value.str, "1") || boost::iequals(value.str, "on") || - boost::iequals(value.str, "yes") || - boost::iequals(value.str, "1")); + boost::iequals(value.str, "yes")); } -string JSONFormattable::def(const string& def_val) const { - if (type == FMT_NONE) { - return def_val; +std::list JSONFormattable::generate_test_instances() { + std::list o; + o.emplace_back(); + o.emplace_back(); + o.back().set_type(FMT_VALUE); + o.back().value.str = "foo"; + o.back().value.quoted = true; + o.emplace_back(); + o.back().set_type(FMT_VALUE); + o.back().value.str = "foo"; + o.back().value.quoted = false; + o.emplace_back(); + o.back().set_type(FMT_ARRAY); + o.back().arr.push_back(JSONFormattable()); + o.back().arr.back().set_type(FMT_VALUE); + o.back().arr.back().value.str = "foo"; + o.back().arr.back().value.quoted = true; + o.back().arr.push_back(JSONFormattable()); + o.back().arr.back().set_type(FMT_VALUE); + o.back().arr.back().value.str = "bar"; + o.back().arr.back().value.quoted = true; + o.emplace_back(); + o.back().set_type(FMT_OBJ); + o.back().obj["foo"] = JSONFormattable(); + o.back().obj["foo"].set_type(FMT_VALUE); + o.back().obj["foo"].value.str = "bar"; + o.back().obj["foo"].value.quoted = true; + return o; } - return val(); -} -int JSONFormattable::def(int def_val) const { - if (type == FMT_NONE) { - return def_val; +void JSONFormattable::generate_test_instances(std::list& o) { + o.push_back(new JSONFormattable); + o.push_back(new JSONFormattable); + o.back()->set_type(FMT_VALUE); + o.back()->value.str = "foo"; + o.back()->value.quoted = true; + o.push_back(new JSONFormattable); + o.back()->set_type(FMT_VALUE); + o.back()->value.str = "foo"; + o.back()->value.quoted = false; + o.push_back(new JSONFormattable); + o.back()->set_type(FMT_ARRAY); + o.back()->arr.push_back(JSONFormattable()); + o.back()->arr.back().set_type(FMT_VALUE); + o.back()->arr.back().value.str = "foo"; + o.back()->arr.back().value.quoted = true; + o.back()->arr.push_back(JSONFormattable()); + o.back()->arr.back().set_type(FMT_VALUE); + o.back()->arr.back().value.str = "bar"; + o.back()->arr.back().value.quoted = true; + o.push_back(new JSONFormattable); + o.back()->set_type(FMT_OBJ); + o.back()->obj["foo"] = JSONFormattable(); + o.back()->obj["foo"].set_type(FMT_VALUE); + o.back()->obj["foo"].value.str = "bar"; + o.back()->obj["foo"].value.quoted = true; } - return val_int(); -} -bool JSONFormattable::def(bool def_val) const { - if (type == FMT_NONE) { - return def_val; - } - return val_bool(); -} - -string JSONFormattable::get(const string& name, const string& def_val) const -{ - return (*this)[name].def(def_val); -} - -int JSONFormattable::get_int(const string& name, int def_val) const -{ - return (*this)[name].def(def_val); -} - -bool JSONFormattable::get_bool(const string& name, bool def_val) const -{ - return (*this)[name].def(def_val); -} struct field_entity { bool is_obj{false}; /* either obj field or array entity */ @@ -722,7 +289,8 @@ struct field_entity { int index{0}; /* if array */ bool append{false}; - field_entity() {} + field_entity() = default; + explicit field_entity(std::string_view sv) : is_obj(true), name(sv) {} explicit field_entity(const string& n) : is_obj(true), name(n) {} explicit field_entity(int i) : is_obj(false), index(i) {} }; @@ -737,12 +305,14 @@ static int parse_entity(const string& s, vector *result) if (ofs != 0) { return -EINVAL; } - result->push_back(field_entity(s)); + result->emplace_back(field_entity(s)); return 0; } if (next_arr > ofs) { - string field = s.substr(ofs, next_arr - ofs); - result->push_back(field_entity(field)); + + auto field = string_view(ofs + begin(s), next_arr - ofs + begin(s)); + + result->emplace_back(field_entity(field)); ofs = next_arr; } size_t end_arr = s.find(']', next_arr + 1); @@ -750,16 +320,22 @@ static int parse_entity(const string& s, vector *result) return -EINVAL; } - string index_str = s.substr(next_arr + 1, end_arr - next_arr - 1); + auto index_str = string_view(s).substr(1 + next_arr, end_arr - next_arr - 1); ofs = end_arr + 1; if (!index_str.empty()) { - result->push_back(field_entity(atoi(index_str.c_str()))); + + int x; + if (auto [_, ec] = std::from_chars(begin(index_str), end(index_str), x); std::errc() == ec) + result->emplace_back(field_entity(x)); + else + throw std::invalid_argument(fmt::format("{}", index_str)); + } else { field_entity f; f.append = true; - result->push_back(f); + result->emplace_back(f); } } return 0; @@ -784,7 +360,7 @@ int JSONFormattable::set(const string& name, const string& val) JSONParser jp; - bool is_valid_json = jp.parse(val.c_str(), val.size()); + bool is_valid_json = jp.parse(val); for (const auto& i : tok) { vector v; @@ -919,36 +495,10 @@ void JSONFormattable::derive_from(const JSONFormattable& parent) } } -void encode_json(const char *name, const JSONFormattable& v, Formatter *f) -{ - v.encode_json(name, f); -} - -void JSONFormattable::encode_json(const char *name, Formatter *f) const -{ - switch (type) { - case JSONFormattable::FMT_VALUE: - ::encode_json(name, value, f); - break; - case JSONFormattable::FMT_ARRAY: - ::encode_json(name, arr, f); - break; - case JSONFormattable::FMT_OBJ: - f->open_object_section(name); - for (auto iter : obj) { - ::encode_json(iter.first.c_str(), iter.second, f); - } - f->close_section(); - break; - case JSONFormattable::FMT_NONE: - break; - } -} - bool JSONFormattable::handle_value(std::string_view name, std::string_view s, bool quoted) { JSONFormattable *new_val; if (cur_enc->is_array()) { - cur_enc->arr.push_back(JSONFormattable()); + cur_enc->arr.emplace_back(JSONFormattable()); new_val = &cur_enc->arr.back(); } else { cur_enc->set_type(JSONFormattable::FMT_OBJ); @@ -964,7 +514,7 @@ bool JSONFormattable::handle_open_section(std::string_view name, const char *ns, bool section_is_array) { if (cur_enc->is_array()) { - cur_enc->arr.push_back(JSONFormattable()); + cur_enc->arr.emplace_back(JSONFormattable()); cur_enc = &cur_enc->arr.back(); } else if (enc_stack.size() > 1) { /* only open a new section if already nested, @@ -972,7 +522,7 @@ bool JSONFormattable::handle_open_section(std::string_view name, */ cur_enc = &cur_enc->obj[string{name}]; } - enc_stack.push_back(cur_enc); + enc_stack.emplace_back(cur_enc); if (section_is_array) { cur_enc->set_type(JSONFormattable::FMT_ARRAY); @@ -1018,4 +568,3 @@ void JSONFormattable::decode(ceph::buffer::list::const_iterator& bl) { } DECODE_FINISH(bl); } - diff --git a/src/common/ceph_json.h b/src/common/ceph_json.h index a2e4fba5d7c..6928ed40fdb 100644 --- a/src/common/ceph_json.h +++ b/src/common/ceph_json.h @@ -1,50 +1,427 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2025-2026 International Business Machines Corp. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * +*/ + #ifndef CEPH_JSON_H #define CEPH_JSON_H +#include + +#include +#include #include #include +#include #include -#include -#include +#include + +#include +#include + +#include +#include +#include +#include + #include -#include "include/encoding.h" +#include + #include +#include + +// Some supported platforms still do not provide ; Ceph uses libfmt. +#ifndef GLZ_USE_STD_FORMAT_FLOAT +#define GLZ_USE_STD_FORMAT_FLOAT 0 +#endif + +#include "glaze/glaze.hpp" + +#include + #include #include -#include + +#include + +#include #include + +#include "common/strtol.h" #include "common/ceph_time.h" -#include "json_spirit/json_spirit.h" +#include #include "JSONFormatter.h" +#include "include/encoding.h" +class utime_t; class JSONObj; +class JSONFormattable; -class JSONObjIter { - typedef std::map::iterator map_iter_t; - map_iter_t cur; - map_iter_t last; +class JSONParser; -public: - JSONObjIter(); - ~JSONObjIter(); - void set(const JSONObjIter::map_iter_t &_cur, const JSONObjIter::map_iter_t &_end); - - void operator++(); - JSONObj *operator*(); - - bool end() const { - return (cur == last); - } +struct JSONDecoderErr : std::runtime_error { + using runtime_error::runtime_error; }; -class JSONObj +namespace ceph_json { +// Avoid referring to the library type in our user-facing code: +using value = glz::generic; + +namespace detail { + +inline std::string_view as_string_view(ceph::buffer::list& bl) { - JSONObj *parent; + auto out = std::string_view { bl.c_str(), bl.length() }; + if (!out.empty() && '\0' == out.back()) { + out.remove_suffix(1); + } + + return out; +} + +inline void read_json_or_throw(std::string_view input, value& data_out) +{ + if (auto ec = glz::read_json(data_out, input); ec) { + throw JSONDecoderErr(glz::format_error(ec, input)); + } +} + +} // namespace detail + +/* + * public: + * Prefer this API for new code or conversions; the older + * JSONObj/Parser/Decoder remains below for existing callsites that use its + * object model. + */ + +inline bool parse(std::string_view input, value& data_out) +{ + return !glz::read_json(data_out, input); +} + +inline value parse(std::string_view input) +{ + value out; + detail::read_json_or_throw(input, out); + return out; +} + +inline bool parse(ceph::buffer::list& bl, value& data_out) +{ + return parse(detail::as_string_view(bl), data_out); +} + +inline value parse(ceph::buffer::list& bl) +{ + return parse(detail::as_string_view(bl)); +} + +inline const value::object_t* object(const value& json) noexcept +{ + return json.get_if(); +} + +inline const value::array_t* array(const value& json) noexcept +{ + return json.get_if(); +} + +inline std::span array_view(const value::array_t& values) noexcept +{ + return values; +} + +inline std::span array_view(const value& json) noexcept +{ + auto* values = array(json); + if (!values) { + return {}; + } + + return array_view(*values); +} + +inline const value* find_value(const value& json, std::string_view name) noexcept +{ + auto* members = object(json); + if (!members) { + return nullptr; + } + + auto i = members->find(name); + if (std::end(*members) == i) { + return nullptr; + } + + return &i->second; +} + +template +void decode_value(const value& json, ValueT& out) +{ + if (auto ec = glz::read_json(out, json); ec) { + throw JSONDecoderErr(glz::format_error(ec)); + } +} + +template +ValueT decode_value(const value& json) +{ + ValueT out; + decode_value(json, out); + return out; +} + +template +bool decode_field(std::string_view name, ValueT& out, const value& json) +{ + auto* field = find_value(json, name); + if (!field) { + if constexpr (std::is_default_constructible_v) { + out = ValueT(); + } + + return false; + } + + try { + decode_value(*field, out); + } catch (const JSONDecoderErr& e) { + throw JSONDecoderErr(fmt::format("{}: {}", name, e.what())); + } + + return true; +} + +template +std::optional field(const value& json, std::string_view name) +{ + auto* item = find_value(json, name); + if (!item) { + return std::nullopt; + } + + try { + return decode_value(*item); + } catch (const JSONDecoderErr& e) { + throw JSONDecoderErr(fmt::format("{}: {}", name, e.what())); + } +} + +template +ValueT field_or(const value& json, std::string_view name, ValueT fallback) +{ + auto out = field(json, name); + if (!out) { + return fallback; + } + + return *out; +} + +template +void decode_required_field(std::string_view name, ValueT& out, const value& json) +{ + if (decode_field(name, out, json)) { + return; + } + + throw JSONDecoderErr(fmt::format("missing mandatory field {}", name)); +} + +template +ValueT require(const value& json, std::string_view name) +{ + auto out = field(json, name); + if (out) { + return *out; + } + + throw JSONDecoderErr(fmt::format("missing mandatory field {}", name)); +} + +inline const value::object_t& require_object(const value& json) +{ + auto* out = object(json); + if (!out) { + throw JSONDecoderErr("expected object"); + } + + return *out; +} + +inline const value& require_object(const value& json, std::string_view name) +{ + auto* out = find_value(json, name); + if (!out) { + throw JSONDecoderErr(fmt::format("missing mandatory field {}", name)); + } + + if (!out->is_object()) { + throw JSONDecoderErr(fmt::format("{}: expected object", name)); + } + + return *out; +} + +inline std::span require_array(const value& json) +{ + auto* out = array(json); + if (!out) { + throw JSONDecoderErr("expected array"); + } + + return array_view(*out); +} + +inline std::span require_array(const value& json, std::string_view name) +{ + auto* out = find_value(json, name); + if (!out) { + throw JSONDecoderErr(fmt::format("missing mandatory field {}", name)); + } + + auto* values = array(*out); + if (!values) { + throw JSONDecoderErr(fmt::format("{}: expected array", name)); + } + + return array_view(*values); +} +} // namespace ceph_json + +namespace ceph_json::detail { + +/* + * detail: + * Shared implementation constraints for the legacy compatibility API and the + * formatter helpers below. + */ + +template +consteval bool is_any_of() +{ + return (std::is_same_v || ...); +} + +/* Note: std::is_integer<> will also pick up bool, which in our +case we need to count as a non-integer type as the JSON codecs treat +it differently ("https://en.cppreference.com/w/cpp/types/is_integral"). + +From the extant code, the same rules should be applied to char, etc., +so I've also done so here: +*/ +template +concept json_integer = requires +{ + requires std::is_integral_v; + requires !std::is_same_v; + requires !is_any_of(); +}; + +template +concept json_signed_integer = requires +{ + requires json_integer && std::signed_integral; +}; + +template +concept json_unsigned_integer = requires +{ + requires json_integer && std::unsigned_integral; +}; + +/* Distinguish between containers with a value that's an associative kv-pair (a mapped type) and +those which are a "single" value. Note that this is not the same as the AssociativeContainer +named concept, as the rule there is that the container is key-indexed, and it does not necessarily +have to be a pair (e.g. std::set<> is an AssociativeContainer). Similarly, for sequence types +we don't want to capture standard strings and the like, even if we otherwise could consider them +a value sequence: +*/ +template +concept json_mapped_kv_seq = requires +{ + typename ContainerT::key_type; + typename ContainerT::key_compare; + + typename ContainerT::value_type; + typename ContainerT::mapped_type; +}; + +template +concept json_val_seq = requires +{ + typename ContainerT::value_type; + + requires !json_mapped_kv_seq; + requires !std::convertible_to; +}; + +} // namespace ceph_json::detail + +/* + * legacy compatibility: + * These types preserve the older Ceph JSON object model. Prefer the + * ceph_json::parse()/require()/field() family above when a caller can work + * directly from the native value tree. + */ + +class JSONObjIter final { + + using child_iter_t = std::vector>::iterator; + + child_iter_t cur; + child_iter_t last; + +private: + JSONObjIter(child_iter_t cur_, child_iter_t end_) + : cur(cur_), + last(end_) + {} + public: + JSONObjIter() = default; + +public: + void set(child_iter_t cur_, child_iter_t end_) { + cur = cur_; + last = end_; + } + + void operator++() { if (cur != last) ++cur; } + + // IMPORTANT: The returned pointer is intended as NON-OWNING (i.e. JSONObjIter + // is responsible for it): + JSONObj *operator*() { return cur->get(); } + + bool end() const { return (cur == last); } + +private: + friend JSONObj; +}; + +class JSONObj +{ + JSONObj *parent = nullptr; + +public: + using native_handle_type = ceph_json::value; + struct data_val { std::string str; bool quoted{false}; @@ -54,40 +431,117 @@ public: quoted = q; } }; + protected: + using child_seq_t = std::vector>; + std::string name; // corresponds to obj_type in XMLObj - json_spirit::Value data; - struct data_val val; - bool data_quoted{false}; - std::multimap children; - std::map attr_map; - void handle_value(json_spirit::Value v); + + ceph_json::value data; + + mutable data_val val; + + mutable bool data_value_ready = false; + + child_seq_t children; + + void cache_data_value(std::string_view s, bool quoted) const; + void ensure_data_value() const; + void handle_value(const ceph_json::value& v); + + static std::string_view child_name(const std::unique_ptr& child) noexcept { + return child->get_name(); + } + +protected: + static bool parse_json(std::string_view input, ceph_json::value& data_out) { + try { + ceph_json::detail::read_json_or_throw(input, data_out); + } catch (const JSONDecoderErr& e) { + throw std::runtime_error(fmt::format("JSON error:\n{}\n", e.what())); + } + + return true; + } public: + JSONObj() = default; - JSONObj() : parent(NULL){} + JSONObj(JSONObj *parent_node, std::string_view name_in, const ceph_json::value& data_in); - virtual ~JSONObj(); + virtual ~JSONObj() = default; - void init(JSONObj *p, json_spirit::Value v, std::string n); +public: + std::string& get_name() noexcept { return name; } - std::string& get_name() { return name; } - data_val& get_data_val() { return val; } - const std::string& get_data() { return val.str; } - bool get_data(const std::string& key, data_val *dest); - JSONObj *get_parent(); - void add_child(std::string el, JSONObj *child); - bool get_attr(std::string name, data_val& attr); - JSONObjIter find(const std::string& name); - JSONObjIter find_first(); - JSONObjIter find_first(const std::string& name); - JSONObj *find_obj(const std::string& name); + data_val& get_data_val() { + ensure_data_value(); + return val; + } + + const std::string& get_data() const { + ensure_data_value(); + return val.str; + } + + // Borrow access to the underlying Glaze value for performance-sensitive code. + native_handle_type& native_handle() noexcept { + data_value_ready = false; + return data; + } + + const native_handle_type& native_handle() const noexcept { return data; } + + bool get_data(std::string_view key, data_val *dest) { + + JSONObj *obj = find_obj(key); + if (!obj) + return false; + + *dest = obj->get_data_val(); + + return true; + } + + JSONObj *get_parent() const noexcept { return parent; }; + + bool get_attr(std::string_view attr_name, data_val& attr) { + if (name != attr_name) + return false; + + attr = get_data_val(); + return true; + } + + JSONObjIter find(std::string_view name) { + auto fst = std::ranges::lower_bound(children, name, std::less<>{}, child_name); + + if (std::end(children) != fst && name == (*fst)->get_name()) { + return { fst, std::ranges::upper_bound(children, name, std::less<>{}, child_name) }; + } + + return { std::end(children), std::end(children) }; + } + + JSONObjIter find_first() { + return { std::begin(children), std::end(children) }; + } + + JSONObjIter find_first(std::string_view name) { + return find(name); + } + + JSONObj *find_obj(std::string_view name) { + JSONObjIter i = this->find(name); + return i.end() ? nullptr : *i; + } friend std::ostream& operator<<(std::ostream &out, const JSONObj &obj); // does not work, FIXME - bool is_array(); - bool is_object(); + bool is_array() const noexcept { return data.is_array(); } + bool is_object() const noexcept { return data.is_object(); } + std::vector get_array_elements(); }; @@ -97,59 +551,78 @@ inline std::ostream& operator<<(std::ostream &out, const JSONObj::data_val& dv) return out; } -class JSONParser : public JSONObj +class JSONParser final : public JSONObj { - int buf_len; + int buf_len = 0; std::string json_buffer; - bool success; + public: - JSONParser(); - ~JSONParser() override; - void handle_data(const char *s, int len); + ~JSONParser() override = default; - bool parse(const char *buf_, int len); - bool parse(int len); +public: + // operate on the internal buffer: bool parse(); - bool parse(const char *file_name); + bool parse(int len); - const char *get_json() { return json_buffer.c_str(); } - void set_failure() { success = false; } + // operate on a string/stringlike range or object: + bool parse(std::string_view sv); + + bool parse(const char *buf_, int len) { + return buf_ ? + parse(std::string_view { buf_, static_cast(len) }) + : false; + } + + bool parse(ceph::buffer::list& bl) { + return parse(bl.c_str(), bl.length() - 1); + } + + [[deprecated("this may not be reliable")]] bool parse_file(const std::filesystem::path file_name); + +public: + const char *get_json() const noexcept{ return json_buffer.c_str(); } }; -void encode_json(const char *name, const JSONObj::data_val& v, ceph::Formatter *f); - -class JSONDecoder { +class JSONDecoder final { public: - struct err : std::runtime_error { - using runtime_error::runtime_error; - }; + using err = JSONDecoderErr; JSONParser parser; JSONDecoder(ceph::buffer::list& bl) { - if (!parser.parse(bl.c_str(), bl.length())) { - std::cout << "JSONDecoder::err()" << std::endl; + if (!parser.parse(bl.c_str(), bl.length())) throw JSONDecoder::err("failed to parse JSON input"); - } } template - static bool decode_json(const char *name, T& val, JSONObj *obj, bool mandatory = false); + static bool decode_json(std::string_view name, T& val, JSONObj *obj, bool mandatory = false); template - static bool decode_json(const char *name, C& container, void (*cb)(C&, JSONObj *obj), JSONObj *obj, bool mandatory = false); + static bool decode_json(std::string_view name, C& container, void (*cb)(C&, JSONObj *obj), JSONObj *obj, bool mandatory = false); template - static void decode_json(const char *name, T& val, const T& default_val, JSONObj *obj); + static void decode_json(std::string_view name, T& val, const T& default_val, JSONObj *obj); template - static bool decode_json(const char *name, boost::optional& val, JSONObj *obj, bool mandatory = false); + static bool decode_json(std::string_view name, boost::optional& val, JSONObj *obj, bool mandatory = false); template - static bool decode_json(const char *name, std::optional& val, JSONObj *obj, bool mandatory = false); - + static bool decode_json(std::string_view name, std::optional& val, JSONObj *obj, bool mandatory = false); }; + +template +requires ceph_json::detail::json_integer +void decode_json_obj(IntegerT& val, JSONObj *obj) +{ + auto r = ceph::parse(obj->get_data()); + + if(!r) + throw JSONDecoder::err(fmt::format("failed to parse number from JSON")); + + val = *r; +} + template void decode_json_obj(T& val, JSONObj *obj) { @@ -161,183 +634,149 @@ inline void decode_json_obj(std::string& val, JSONObj *obj) val = obj->get_data(); } -static inline void decode_json_obj(JSONObj::data_val& val, JSONObj *obj) +inline void decode_json_obj(JSONObj::data_val& val, JSONObj *obj) { val = obj->get_data_val(); } -void decode_json_obj(unsigned long long& val, JSONObj *obj); -void decode_json_obj(long long& val, JSONObj *obj); -void decode_json_obj(unsigned long& val, JSONObj *obj); -void decode_json_obj(long& val, JSONObj *obj); -void decode_json_obj(unsigned& val, JSONObj *obj); -void decode_json_obj(int& val, JSONObj *obj); -void decode_json_obj(bool& val, JSONObj *obj); -void decode_json_obj(ceph::buffer::list& val, JSONObj *obj); -class utime_t; -void decode_json_obj(utime_t& val, JSONObj *obj); -void decode_json_obj(ceph_dir_layout& i, JSONObj *obj); - -void decode_json_obj(ceph::real_time& val, JSONObj *obj); -void decode_json_obj(ceph::coarse_real_time& val, JSONObj *obj); - -template -void decode_json_obj(std::list& l, JSONObj *obj) +inline void decode_json_obj(bool& val, JSONObj *obj) { - l.clear(); + std::string_view sv(obj->get_data()); - JSONObjIter iter = obj->find_first(); + if (boost::iequals(sv, "true")) + { + val = true; + return; + } - for (; !iter.end(); ++iter) { - T val; - JSONObj *o = *iter; - decode_json_obj(val, o); - l.push_back(val); + if (boost::iequals(sv, "false")) + { + val = false; + return; + } + + // For 1, 0, anything else: + int i; + decode_json_obj(i, obj); + val = static_cast(i); +} + +inline void decode_json_obj(bufferlist& val, JSONObj *obj) +{ + bufferlist bl; + + std::string_view sv = obj->get_data(); + + bl.append(sv); + + try { + val.decode_base64(bl); + + } catch (ceph::buffer::error& err) { + throw JSONDecoder::err("failed to decode base64"); } } -template -void decode_json_obj(std::deque& l, JSONObj *obj) +inline void decode_json_obj(utime_t& val, JSONObj *obj) { - l.clear(); - - JSONObjIter iter = obj->find_first(); - - for (; !iter.end(); ++iter) { - T val; - JSONObj *o = *iter; - decode_json_obj(val, o); - l.push_back(val); + uint64_t epoch; + uint64_t nsec; + int r = utime_t::parse_date(obj->get_data(), &epoch, &nsec); + if (r == 0) { + val = utime_t(epoch, nsec); + } else { + throw JSONDecoder::err("failed to decode utime_t"); } } -template -void decode_json_obj(std::set& l, JSONObj *obj) +inline void decode_json_obj(ceph::real_time& val, JSONObj *obj) { - l.clear(); - - JSONObjIter iter = obj->find_first(); - - for (; !iter.end(); ++iter) { - T val; - JSONObj *o = *iter; - decode_json_obj(val, o); - l.insert(val); + uint64_t epoch; + uint64_t nsec; + int r = utime_t::parse_date(obj->get_data(), &epoch, &nsec); + if (r == 0) { + using namespace std::chrono; + val = real_time{seconds(epoch) + nanoseconds(nsec)}; + } else { + throw JSONDecoder::err("failed to decode real_time"); } } -template -void decode_json_obj(boost::container::flat_set& l, JSONObj *obj) +inline void decode_json_obj(ceph::coarse_real_time& val, JSONObj *obj) { - l.clear(); - - JSONObjIter iter = obj->find_first(); - - for (; !iter.end(); ++iter) { - T val; - JSONObj *o = *iter; - decode_json_obj(val, o); - l.insert(val); + uint64_t epoch; + uint64_t nsec; + int r = utime_t::parse_date(obj->get_data(), &epoch, &nsec); + if (r == 0) { + using namespace std::chrono; + val = coarse_real_time{seconds(epoch) + nanoseconds(nsec)}; + } else { + throw JSONDecoder::err("failed to decode coarse_real_time"); } } -template -void decode_json_obj(std::vector& l, JSONObj *obj) +inline void decode_json_obj(ceph_dir_layout& i, JSONObj *obj) { - l.clear(); - - JSONObjIter iter = obj->find_first(); - - for (; !iter.end(); ++iter) { - T val; - JSONObj *o = *iter; - decode_json_obj(val, o); - l.push_back(val); - } + unsigned tmp; + JSONDecoder::decode_json("dir_hash", tmp, obj, true); + i.dl_dir_hash = tmp; + JSONDecoder::decode_json("unused1", tmp, obj, true); + i.dl_unused1 = tmp; + JSONDecoder::decode_json("unused2", tmp, obj, true); + i.dl_unused2 = tmp; + JSONDecoder::decode_json("unused3", tmp, obj, true); + i.dl_unused3 = tmp; } -template > -void decode_json_obj(std::map& m, JSONObj *obj) +template +void decode_json_obj(SeqT& seq, JSONObj *obj) { - m.clear(); + seq.clear(); - JSONObjIter iter = obj->find_first(); + for (auto iter = obj->find_first(); !iter.end(); ++iter) { + typename SeqT::value_type val; + JSONObj *o = *iter; + decode_json_obj(val, o); - for (; !iter.end(); ++iter) { - K key; - V val; + if constexpr (requires { seq.emplace_back(val); }) + seq.emplace_back(val); + else + seq.emplace(val); + } +} + +template +void decode_json_obj(KVSeqT& kvs, JSONObj *obj) +{ + kvs.clear(); + + for (auto iter = obj->find_first(); !iter.end(); ++iter) { + typename KVSeqT::key_type key; + typename KVSeqT::mapped_type val; JSONObj *o = *iter; JSONDecoder::decode_json("key", key, o); JSONDecoder::decode_json("val", val, o); - m[key] = val; + + if constexpr(requires { kvs[key] = val; }) + kvs[key] = val; // i.e. insert_or_assign() + else + kvs.insert({key, val}); } } -template > -void decode_json_obj(boost::container::flat_map& m, JSONObj *obj) -{ - m.clear(); - - JSONObjIter iter = obj->find_first(); - - for (; !iter.end(); ++iter) { - K key; - V val; - JSONObj *o = *iter; - JSONDecoder::decode_json("key", key, o); - JSONDecoder::decode_json("val", val, o); - m[key] = val; - } -} - -template -void decode_json_obj(std::multimap& m, JSONObj *obj) -{ - m.clear(); - - JSONObjIter iter = obj->find_first(); - - for (; !iter.end(); ++iter) { - K key; - V val; - JSONObj *o = *iter; - JSONDecoder::decode_json("key", key, o); - JSONDecoder::decode_json("val", val, o); - m.insert(make_pair(key, val)); - } -} - -template -void decode_json_obj(boost::container::flat_map& m, JSONObj *obj) -{ - m.clear(); - - JSONObjIter iter = obj->find_first(); - - for (; !iter.end(); ++iter) { - K key; - V val; - JSONObj *o = *iter; - JSONDecoder::decode_json("key", key, o); - JSONDecoder::decode_json("val", val, o); - m[key] = val; - } -} template void decode_json_obj(C& container, void (*cb)(C&, JSONObj *obj), JSONObj *obj) { container.clear(); - JSONObjIter iter = obj->find_first(); - - for (; !iter.end(); ++iter) { + for (auto iter = obj->find_first(); !iter.end(); ++iter) { JSONObj *o = *iter; cb(container, o); } } template -bool JSONDecoder::decode_json(const char *name, T& val, JSONObj *obj, bool mandatory) +bool JSONDecoder::decode_json(std::string_view name, T& val, JSONObj *obj, bool mandatory) { JSONObjIter iter = obj->find_first(name); if (iter.end()) { @@ -363,7 +802,7 @@ bool JSONDecoder::decode_json(const char *name, T& val, JSONObj *obj, bool manda } template -bool JSONDecoder::decode_json(const char *name, C& container, void (*cb)(C&, JSONObj *), JSONObj *obj, bool mandatory) +bool JSONDecoder::decode_json(std::string_view name, C& container, void (*cb)(C&, JSONObj *), JSONObj *obj, bool mandatory) { container.clear(); @@ -388,7 +827,7 @@ bool JSONDecoder::decode_json(const char *name, C& container, void (*cb)(C&, JSO } template -void JSONDecoder::decode_json(const char *name, T& val, const T& default_val, JSONObj *obj) +void JSONDecoder::decode_json(std::string_view name, T& val, const T& default_val, JSONObj *obj) { JSONObjIter iter = obj->find_first(name); if (iter.end()) { @@ -407,7 +846,7 @@ void JSONDecoder::decode_json(const char *name, T& val, const T& default_val, JS } template -bool JSONDecoder::decode_json(const char *name, boost::optional& val, JSONObj *obj, bool mandatory) +bool JSONDecoder::decode_json(std::string_view name, boost::optional& val, JSONObj *obj, bool mandatory) { JSONObjIter iter = obj->find_first(name); if (iter.end()) { @@ -433,7 +872,7 @@ bool JSONDecoder::decode_json(const char *name, boost::optional& val, JSONObj } template -bool JSONDecoder::decode_json(const char *name, std::optional& val, JSONObj *obj, bool mandatory) +bool JSONDecoder::decode_json(std::string_view name, std::optional& val, JSONObj *obj, bool mandatory) { JSONObjIter iter = obj->find_first(name); if (iter.end()) { @@ -463,7 +902,7 @@ class JSONEncodeFilter public: class HandlerBase { public: - virtual ~HandlerBase() {} + virtual ~HandlerBase() = default; virtual std::type_index get_type() = 0; virtual void encode_json(const char *name, const void *pval, ceph::Formatter *) const = 0; @@ -472,15 +911,13 @@ public: template class Handler : public HandlerBase { public: - virtual ~Handler() {} - std::type_index get_type() override { return std::type_index(typeid(const T&)); } }; private: - std::map handlers; + boost::container::flat_map handlers; public: void register_type(HandlerBase *h) { @@ -499,8 +936,19 @@ public: } }; +void encode_json(const char *name, ceph_json::detail::json_signed_integer auto val, Formatter *f) +{ + f->dump_int(name, val); +} + +void encode_json(const char *name, ceph_json::detail::json_unsigned_integer auto val, Formatter *f) +{ + f->dump_unsigned(name, val); +} + template -static void encode_json_impl(const char *name, const T& val, ceph::Formatter *f) +requires requires(const T& val, ceph::Formatter *f) { val.dump(f); } +void encode_json_impl(const char *name, const T& val, ceph::Formatter *f) { f->open_object_section(name); val.dump(f); @@ -508,7 +956,8 @@ static void encode_json_impl(const char *name, const T& val, ceph::Formatter *f) } template -static void encode_json(const char *name, const T& val, ceph::Formatter *f) +requires requires(const T& val, ceph::Formatter *f) { encode_json_impl("", val, f); } +void encode_json(const char *name, const T& val, ceph::Formatter *f) { JSONEncodeFilter *filter = static_cast(f->get_external_feature_handler("JSONEncodeFilter")); @@ -518,150 +967,82 @@ static void encode_json(const char *name, const T& val, ceph::Formatter *f) } } -class utime_t; - -void encode_json(const char *name, std::string_view val, ceph::Formatter *f); -void encode_json(const char *name, const std::string& val, ceph::Formatter *f); -void encode_json(const char *name, const char *val, ceph::Formatter *f); -void encode_json(const char *name, bool val, ceph::Formatter *f); -void encode_json(const char *name, int val, ceph::Formatter *f); -void encode_json(const char *name, unsigned val, ceph::Formatter *f); -void encode_json(const char *name, long val, ceph::Formatter *f); -void encode_json(const char *name, unsigned long val, ceph::Formatter *f); -void encode_json(const char *name, long long val, ceph::Formatter *f); -void encode_json(const char *name, const utime_t& val, ceph::Formatter *f); -void encode_json(const char *name, const ceph::buffer::list& bl, ceph::Formatter *f); -void encode_json(const char *name, long long unsigned val, ceph::Formatter *f); - -void encode_json(const char *name, const ceph::real_time& val, ceph::Formatter *f); -void encode_json(const char *name, const ceph::coarse_real_time& val, ceph::Formatter *f); - -template -static void encode_json(const char *name, const std::list& l, ceph::Formatter *f) +inline void encode_json(const char *name, std::string_view val, Formatter *f) { - f->open_array_section(name); - for (auto iter = l.cbegin(); iter != l.cend(); ++iter) { - encode_json("obj", *iter, f); - } - f->close_section(); + f->dump_string(name, val); } -template -static void encode_json(const char *name, const std::deque& l, ceph::Formatter *f) +inline void encode_json(const char *name, const std::string& val, Formatter *f) { - f->open_array_section(name); - for (auto iter = l.cbegin(); iter != l.cend(); ++iter) { - encode_json("obj", *iter, f); - } - f->close_section(); + f->dump_string(name, val); } -template > -static void encode_json(const char *name, const std::set& l, ceph::Formatter *f) +inline void encode_json(const char *name, const char *val, Formatter *f) { - f->open_array_section(name); - for (auto iter = l.cbegin(); iter != l.cend(); ++iter) { - encode_json("obj", *iter, f); - } - f->close_section(); + f->dump_string(name, val); } -template -static void encode_json(const char *name, - const boost::container::flat_set& l, - ceph::Formatter *f) +inline void encode_json(const char *name, bool val, Formatter *f) { - f->open_array_section(name); - for (auto iter = l.cbegin(); iter != l.cend(); ++iter) { - encode_json("obj", *iter, f); - } - f->close_section(); + f->dump_bool(name, val); } -template -static void encode_json(const char *name, const std::vector& l, ceph::Formatter *f) +inline void encode_json(const char *name, const utime_t& val, Formatter *f) { - f->open_array_section(name); - for (auto iter = l.cbegin(); iter != l.cend(); ++iter) { - encode_json("obj", *iter, f); - } - f->close_section(); + val.gmtime(f->dump_stream(name)); } -template -static void encode_json(const char *name, const std::array& l, - ceph::Formatter *f) +inline void encode_json(const char *name, const ceph::real_time& val, Formatter *f) { - f->open_array_section(name); - for (auto iter = l.cbegin(); iter != l.cend(); ++iter) { - encode_json("obj", *iter, f); - } - f->close_section(); + encode_json(name, utime_t{val}, f); } -template> -static void encode_json(const char *name, const std::map& m, ceph::Formatter *f) +inline void encode_json(const char *name, const ceph::coarse_real_time& val, Formatter *f) { - f->open_array_section(name); - for (auto i = m.cbegin(); i != m.cend(); ++i) { - f->open_object_section("entry"); - encode_json("key", i->first, f); - encode_json("val", i->second, f); - f->close_section(); - } - f->close_section(); + encode_json(name, utime_t{val}, f); } -template > -static void encode_json(const char *name, const boost::container::flat_map& m, ceph::Formatter *f) +inline void encode_json(const char *name, const bufferlist& bl, Formatter *f) { - f->open_array_section(name); - for (auto i = m.cbegin(); i != m.cend(); ++i) { - f->open_object_section("entry"); - encode_json("key", i->first, f); - encode_json("val", i->second, f); - f->close_section(); - } - f->close_section(); + /* need to copy data from bl, as it is const bufferlist */ + bufferlist src = bl; + + bufferlist b64; + src.encode_base64(b64); + + std::string_view sv(b64.c_str(), b64.length()); + + encode_json(name, sv, f); } -template -static void encode_json(const char *name, const std::multimap& m, ceph::Formatter *f) +template +void encode_json(const char *name, const std::optional& o, ceph::Formatter *f) { - f->open_array_section(name); - for (auto i = m.begin(); i != m.end(); ++i) { - f->open_object_section("entry"); - encode_json("key", i->first, f); - encode_json("val", i->second, f); - f->close_section(); + if (!o) { + return; } - f->close_section(); + encode_json(name, *o, f); } -template -static void encode_json(const char *name, const boost::container::flat_map& m, ceph::Formatter *f) +inline void encode_json(const char *name, const JSONObj::data_val& v, Formatter *f) { - f->open_array_section(name); - for (auto i = m.begin(); i != m.end(); ++i) { - f->open_object_section("entry"); - encode_json("key", i->first, f); - encode_json("val", i->second, f); - f->close_section(); + if (v.quoted) { + encode_json(name, v.str, f); + } else { + f->dump_format_unquoted(name, "%s", v.str.c_str()); } - f->close_section(); } +inline void encode_json(const char *name, const JSONFormattable& v, Formatter *f); + template void encode_json_map(const char *name, const std::map& m, ceph::Formatter *f) { f->open_array_section(name); - for (auto iter = m.cbegin(); iter != m.cend(); ++iter) { - encode_json("obj", iter->second, f); - } + std::ranges::for_each(m, [&f](const auto& kv) { encode_json("obj", kv.second, f); }); f->close_section(); } - template void encode_json_map(const char *name, const char *index_name, const char *object_name, const char *value_name, @@ -700,27 +1081,17 @@ void encode_json_map(const char *name, const char *index_name, const char *object_name, const char *value_name, const std::map& m, ceph::Formatter *f) { - encode_json_map(name, index_name, object_name, value_name, NULL, NULL, m, f); + encode_json_map(name, index_name, object_name, value_name, nullptr, nullptr, m, f); } template void encode_json_map(const char *name, const char *index_name, const char *value_name, const std::map& m, ceph::Formatter *f) { - encode_json_map(name, index_name, NULL, value_name, NULL, NULL, m, f); + encode_json_map(name, index_name, nullptr, value_name, nullptr, nullptr, m, f); } -template -static void encode_json(const char *name, const std::optional& o, ceph::Formatter *f) -{ - if (!o) { - return; - } - encode_json(name, *o, f); -} - - -template +template void encode_json_map(const char *name, const boost::container::flat_map& m, ceph::Formatter *f) { f->open_array_section(name); @@ -730,7 +1101,6 @@ void encode_json_map(const char *name, const boost::container::flat_map& m f->close_section(); } - template void encode_json_map(const char *name, const char *index_name, const char *object_name, const char *value_name, @@ -769,21 +1139,42 @@ void encode_json_map(const char *name, const char *index_name, const char *object_name, const char *value_name, const boost::container::flat_map& m, ceph::Formatter *f) { - encode_json_map(name, index_name, object_name, value_name, NULL, NULL, m, f); + encode_json_map(name, index_name, object_name, value_name, nullptr, nullptr, m, f); } template void encode_json_map(const char *name, const char *index_name, const char *value_name, const boost::container::flat_map& m, ceph::Formatter *f) { - encode_json_map(name, index_name, NULL, value_name, NULL, NULL, m, f); + encode_json_map(name, index_name, nullptr, value_name, nullptr, nullptr, m, f); } +void encode_json(const char *name, const ceph_json::detail::json_val_seq auto& val, Formatter *f) +{ + f->open_array_section(name); + std::ranges::for_each(val, [&f](const auto &obj) { + ::encode_json("obj", obj, f); + }); + f->close_section(); +} + +void encode_json(const char *name, const ceph_json::detail::json_mapped_kv_seq auto& val, Formatter *f) +{ + f->open_array_section(name); + std::ranges::for_each(val, [&f](const auto& kv) { + f->open_object_section("entry"); + ::encode_json("key", kv.first, f); + ::encode_json("val", kv.second, f); + f->close_section(); + }); + f->close_section(); +} class JSONFormattable : public ceph::JSONFormatter { + JSONObj::data_val value; std::vector arr; - std::map obj; + std::map> obj; std::vector enc_stack; JSONFormattable *cur_enc; @@ -794,7 +1185,8 @@ protected: bool handle_close_section() override; public: - JSONFormattable(bool p = false) : JSONFormatter(p) { + JSONFormattable(bool p = false) + : JSONFormatter(p) { cur_enc = this; enc_stack.push_back(cur_enc); } @@ -857,114 +1249,50 @@ public: break; } } - static std::list generate_test_instances() { - std::list o; - o.emplace_back(); - o.emplace_back(); - o.back().set_type(FMT_VALUE); - o.back().value.str = "foo"; - o.back().value.quoted = true; - o.emplace_back(); - o.back().set_type(FMT_VALUE); - o.back().value.str = "foo"; - o.back().value.quoted = false; - o.emplace_back(); - o.back().set_type(FMT_ARRAY); - o.back().arr.push_back(JSONFormattable()); - o.back().arr.back().set_type(FMT_VALUE); - o.back().arr.back().value.str = "foo"; - o.back().arr.back().value.quoted = true; - o.back().arr.push_back(JSONFormattable()); - o.back().arr.back().set_type(FMT_VALUE); - o.back().arr.back().value.str = "bar"; - o.back().arr.back().value.quoted = true; - o.emplace_back(); - o.back().set_type(FMT_OBJ); - o.back().obj["foo"] = JSONFormattable(); - o.back().obj["foo"].set_type(FMT_VALUE); - o.back().obj["foo"].value.str = "bar"; - o.back().obj["foo"].value.quoted = true; - return o; - } - const std::string& val() const { - return value.str; - } + const std::map> object() const noexcept { return obj; } - int val_int() const; - long val_long() const; - long long val_long_long() const; - bool val_bool() const; - - const std::map object() const { - return obj; - } - - const std::vector& array() const { - return arr; - } - - const JSONFormattable& operator[](const std::string& name) const; - const JSONFormattable& operator[](size_t index) const; + const std::vector& array() const noexcept { return arr; } JSONFormattable& operator[](const std::string& name); + const JSONFormattable& operator[](const std::string& name) const; + JSONFormattable& operator[](size_t index); + const JSONFormattable& operator[](size_t index) const; - operator std::string() const { - return value.str; + const std::string& val() const noexcept { return value.str; } + int val_int() const { return atoi(value.str.c_str()); } + long val_long() const { return atol(value.str.c_str()); } + long long val_long_long() const { return atoll(value.str.c_str()); } + bool val_bool() const; + + operator std::string() const noexcept { return value.str; } + explicit operator int() const { return val_int(); } + explicit operator long() const { return val_long(); } + explicit operator long long() const { return val_long_long(); } + explicit operator bool() const { return val_bool(); } + + std::string def(const std::string& def_val) const { return FMT_NONE == type ? def_val : val(); } + int def(int def_val) const { return FMT_NONE == type ? def_val : val_int(); } + bool def(bool def_val) const { return FMT_NONE == type ? def_val : val_bool(); } + + std::string operator ()(const char *def_val) const { return def(std::string(def_val)); } + int operator()(int def_val) const { return def(def_val); } + bool operator()(bool def_val) const { return def(def_val); } + + bool exists(const std::string& name) const noexcept { return obj.contains(name); } + bool exists(size_t index) const noexcept { return (index < arr.size()); } + + bool find(const std::string& name, std::string *val) const noexcept { + if (auto i = obj.find(name); end(obj) != i) + return (*val = i->second.val()), true; + + return false; } - explicit operator int() const { - return val_int(); - } - - explicit operator long() const { - return val_long(); - } - - explicit operator long long() const { - return val_long_long(); - } - - explicit operator bool() const { - return val_bool(); - } - - template - T operator[](const std::string& name) const { - return this->operator[](name)(T()); - } - - template - T operator[](const std::string& name) { - return this->operator[](name)(T()); - } - - std::string operator ()(const char *def_val) const { - return def(std::string(def_val)); - } - - int operator()(int def_val) const { - return def(def_val); - } - - bool operator()(bool def_val) const { - return def(def_val); - } - - bool exists(const std::string& name) const; - bool exists(size_t index) const; - - std::string def(const std::string& def_val) const; - int def(int def_val) const; - bool def(bool def_val) const; - - bool find(const std::string& name, std::string *val) const; - - std::string get(const std::string& name, const std::string& def_val) const; - - int get_int(const std::string& name, int def_val) const; - bool get_bool(const std::string& name, bool def_val) const; + std::string get(const std::string& name, const std::string& def_val) const { return (*this)[name].def(def_val); } + int get_int(const std::string& name, int def_val) const { return (*this)[name].def(def_val); } + bool get_bool(const std::string& name, bool def_val) const { return (*this)[name].def(def_val); } int set(const std::string& name, const std::string& val); int erase(const std::string& name); @@ -973,12 +1301,62 @@ public: void encode_json(const char *name, ceph::Formatter *f) const; - bool is_array() const { - return (type == FMT_ARRAY); - } + bool is_array() const { return type == FMT_ARRAY; } + + // Intrusive test support: + private: + template friend class DencoderBase; + static std::list generate_test_instances(); + static void generate_test_instances(std::list& o); }; WRITE_CLASS_ENCODER(JSONFormattable) -void encode_json(const char *name, const JSONFormattable& v, ceph::Formatter *f); +static inline JSONFormattable default_formattable; + +// Note that this does NOT have the same semantics as an STL container: +inline JSONFormattable& JSONFormattable::operator[](const std::string& name) { + if (const auto i = obj.find(name); end(obj) != i) + return i->second; + + return default_formattable; +} + +inline const JSONFormattable& JSONFormattable::operator[](const std::string& name) const { + return const_cast(this)->operator[](name); +} + +inline JSONFormattable& JSONFormattable::operator[](size_t index) { + return index >= arr.size() ? default_formattable : arr[index]; +} + +inline const JSONFormattable& JSONFormattable::operator[](size_t index) const { + return const_cast(this)->operator[](index); +} + +inline void encode_json(const char *name, const JSONFormattable& v, Formatter *f) +{ + v.encode_json(name, f); +} + +inline void JSONFormattable::encode_json(const char *name, Formatter *f) const +{ + switch (type) { + case JSONFormattable::FMT_VALUE: + ::encode_json(name, value, f); + break; + case JSONFormattable::FMT_ARRAY: + ::encode_json(name, arr, f); + break; + case JSONFormattable::FMT_OBJ: + f->open_object_section(name); + for (auto iter : obj) { + ::encode_json(iter.first.c_str(), iter.second, f); + } + f->close_section(); + break; + case JSONFormattable::FMT_NONE: + break; + } +} #endif diff --git a/src/common/cmdparse.cc b/src/common/cmdparse.cc index e07015e8dfc..e17a5093b13 100644 --- a/src/common/cmdparse.cc +++ b/src/common/cmdparse.cc @@ -21,7 +21,9 @@ #include "common/strtol.h" #include "include/ceph_assert.h" // boost clobbers this #include "include/types.h" // for operator<<(std::vector) + #include "json_spirit/json_spirit.h" +#include "common/ceph_json.h" #include #include @@ -324,13 +326,15 @@ cmdmap_from_json(const vector& cmd, cmdmap_t *mapp, std::ostream& ss) json_spirit::mValue v; string fullcmd; + // First, join all cmd strings for (auto& c : cmd) fullcmd += c; try { if (!json_spirit::read(fullcmd, v)) - throw std::runtime_error("unparseable JSON " + fullcmd); + throw std::runtime_error("unparseable JSON: |" + fullcmd + '|'); + if (v.type() != json_spirit::obj_type) throw std::runtime_error("not JSON object " + fullcmd); diff --git a/src/common/dout_fmt.h b/src/common/dout_fmt.h index 036319fc6e3..88d1db2f613 100644 --- a/src/common/dout_fmt.h +++ b/src/common/dout_fmt.h @@ -54,3 +54,25 @@ inline void dout_fmt_use_prefix(std::ostream&) {} fmt::print(*_dout, __VA_ARGS__); \ *_dout << dendl; \ } + +// This is useful for debugging during development, please do NOT leave calls to it lying around in production: +inline void dout_trace(const std::string_view info, const std::string_view msg_tag = "dout-TRACE", const std::source_location sl = std::source_location::current()) +{ + // ...this is an attempt only, as the extension is non-standard-- do NOT count on getting the values back: +#pragma push_macro("dout_subsys") +#pragma push_macro("dout_context") + + #define dout_subsys ceph_subsys_rgw + #define dout_context g_ceph_context + + dout_fmt(0, "{} [{}]: {}", msg_tag, fmt::format("{}#{}", sl.file_name(), sl.function_name()), info); + +#pragma pop_macro("dout_subsys") +#pragma pop_macro("dout_context") +} + +inline void dout_trace(const std::source_location sl = std::source_location::current()) +{ + dout_trace("", "dout-TRACE", sl); +} + diff --git a/src/crimson/CMakeLists.txt b/src/crimson/CMakeLists.txt index e0fe354fd0f..c3e2081505b 100644 --- a/src/crimson/CMakeLists.txt +++ b/src/crimson/CMakeLists.txt @@ -60,7 +60,6 @@ add_library(crimson-common STATIC ${PROJECT_SOURCE_DIR}/src/common/hex.cc ${PROJECT_SOURCE_DIR}/src/common/frag.cc ${PROJECT_SOURCE_DIR}/src/common/fs_types.cc - ${PROJECT_SOURCE_DIR}/src/common/ceph_json.cc ${PROJECT_SOURCE_DIR}/src/common/histogram.cc ${PROJECT_SOURCE_DIR}/src/common/hobject.cc ${PROJECT_SOURCE_DIR}/src/common/hostname.cc @@ -153,6 +152,7 @@ target_link_libraries(crimson-common crimson::cflags PRIVATE crc32 arch + ceph-json Boost::iostreams Boost::random json_spirit diff --git a/src/crimson/admin/pg_commands.cc b/src/crimson/admin/pg_commands.cc index f8cc4c6a155..d96e458754e 100644 --- a/src/crimson/admin/pg_commands.cc +++ b/src/crimson/admin/pg_commands.cc @@ -17,6 +17,8 @@ #include "crimson/osd/osd.h" #include "crimson/osd/pg.h" +#include "json_spirit/json_spirit.h" + SET_SUBSYS(osd); using crimson::osd::OSD; diff --git a/src/include/encoding.h b/src/include/encoding.h index d073c1e7c62..a23465b19a3 100644 --- a/src/include/encoding.h +++ b/src/include/encoding.h @@ -246,7 +246,7 @@ inline void decode_nohead(unsigned len, std::string& s, bufferlist::const_iterat // const char* (encode only, string compatible) inline void encode(const char *s, bufferlist& bl) { - encode(std::string_view(s, strlen(s)), bl); + encode(std::string_view(s), bl); } // opaque byte vectors diff --git a/src/librados/RadosClient.cc b/src/librados/RadosClient.cc index c23f8ff27a5..b95d250048e 100644 --- a/src/librados/RadosClient.cc +++ b/src/librados/RadosClient.cc @@ -17,7 +17,11 @@ #include #include +#include #include +#include +#include +#include #include #include #include @@ -28,7 +32,6 @@ #include "common/common_init.h" #include "common/ceph_json.h" #include "common/errno.h" -#include "common/ceph_json.h" #include "common/async/blocked_completion.h" #include "include/buffer.h" #include "include/stringify.h" @@ -1140,33 +1143,31 @@ int librados::RadosClient::get_inconsistent_pgs(int64_t pool_id, // no pg returned return 0; } - JSONParser parser; - if (!parser.parse(outbl.c_str(), outbl.length())) { + + ceph_json::value response; + if (!ceph_json::parse(outbl, response)) { return -EINVAL; } - vector v; - if (!parser.is_array()) { - JSONObj *pgstat_obj = parser.find_obj("pg_stats"); - if (!pgstat_obj) - return 0; - auto s = pgstat_obj->get_data(); - JSONParser pg_stats; - if (!pg_stats.parse(s.c_str(), s.length())) { - return -EINVAL; + + const auto pg_entries = [&]() -> std::span { + if (response.is_array()) { + return ceph_json::array_view(response); } - v = pg_stats.get_array_elements(); - } else { - v = parser.get_array_elements(); - } - for (auto i : v) { - JSONParser pg_json; - if (!pg_json.parse(i.c_str(), i.length())) { - return -EINVAL; + + auto* pg_stats = ceph_json::find_value(response, "pg_stats"); + if (!pg_stats) { + return {}; } + + return ceph_json::array_view(*pg_stats); + }(); + + std::ranges::transform(pg_entries, std::back_inserter(*pgs), [] (const auto& pg) { string pgid; - JSONDecoder::decode_json("pgid", pgid, &pg_json); - pgs->emplace_back(std::move(pgid)); - } + ceph_json::decode_field("pgid", pgid, pg); + return pgid; + }); + return 0; } diff --git a/src/mgr/ActivePyModules.cc b/src/mgr/ActivePyModules.cc index bf8114346c6..e6f7c715136 100644 --- a/src/mgr/ActivePyModules.cc +++ b/src/mgr/ActivePyModules.cc @@ -43,6 +43,8 @@ #include "PyModuleRegistry.h" #include "PyUtil.h" +#include "json_spirit/json_spirit.h" + #define dout_context g_ceph_context #define dout_subsys ceph_subsys_mgr #undef dout_prefix diff --git a/src/mgr/JSONCommand.h b/src/mgr/JSONCommand.h index 2efc8e126f8..106fdb235ff 100644 --- a/src/mgr/JSONCommand.h +++ b/src/mgr/JSONCommand.h @@ -4,6 +4,7 @@ * Ceph - scalable distributed file system * * Copyright (C) 2016 John Spray + * Copyright (C) 2026 IBM * * This is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -20,18 +21,18 @@ class JSONCommand : public Command { public: - json_spirit::mValue json_result; + glz::generic json_result; void wait() override { Command::wait(); - if (r == 0) { - bool read_ok = json_spirit::read( - outbl.to_str(), json_result); - if (!read_ok) { - r = -EINVAL; - } + if (0 != r) { + return; + } + + if (auto ec = glz::read_json(json_result, outbl.to_str()); ec) { + r = -EINVAL; } } }; diff --git a/src/mgr/Mgr.cc b/src/mgr/Mgr.cc index 1d015585754..3e569c83aaa 100644 --- a/src/mgr/Mgr.cc +++ b/src/mgr/Mgr.cc @@ -14,6 +14,8 @@ #include +#include "common/ceph_json.h" + #include "osdc/Objecter.h" #include "common/errno.h" #include "mon/MonClient.h" @@ -104,21 +106,21 @@ void MetadataUpdate::finish(int r) if (r == 0) { if (key.type == "mds" || key.type == "osd" || key.type == "mgr" || key.type == "mon") { - json_spirit::mValue json_result; - bool read_ok = json_spirit::read( - outbl.to_str(), json_result); - if (!read_ok) { + + glz::generic json_result; + + if (auto ec = glz::read_json(json_result, outbl.to_str()); ec) { dout(1) << "mon returned invalid JSON for " << key << dendl; return; } - if (json_result.type() != json_spirit::obj_type) { + if (!json_result.is_object()) { dout(1) << "mon returned valid JSON " << key << " but not an object: '" << outbl.to_str() << "'" << dendl; return; } dout(4) << "mon returned valid metadata JSON for " << key << dendl; - json_spirit::mObject daemon_meta = json_result.get_obj(); + auto daemon_meta = json_result.get_object(); // Skip daemon who doesn't have hostname yet if (daemon_meta.count("hostname") == 0) { @@ -149,7 +151,7 @@ void MetadataUpdate::finish(int r) std::map m; { std::lock_guard l(state->lock); - std::string reported_hostname = daemon_meta.at("hostname").get_str(); + const std::string reported_hostname = daemon_meta.at("hostname").get_string(); if (key.type == "mds" || key.type == "mgr" || key.type == "mon") { daemon_meta.erase("name"); } else if (key.type == "osd") { @@ -157,7 +159,7 @@ void MetadataUpdate::finish(int r) } daemon_meta.erase("hostname"); for (const auto &[key, val] : daemon_meta) { - m.emplace(key, val.get_str()); + m.emplace(key, val.get_string()); } // prefer CRUSH physical host over container/pod hostname (tracker.ceph.com/issues/73080) // fall back to the reported hostname when CRUSH does not place the OSD @@ -171,7 +173,7 @@ void MetadataUpdate::finish(int r) } else { auto state = std::make_shared(daemon_state.types); state->key = key; - std::string reported_hostname = daemon_meta.at("hostname").get_str(); + const std::string reported_hostname = daemon_meta.at("hostname").get_string(); if (key.type == "mds" || key.type == "mgr" || key.type == "mon") { daemon_meta.erase("name"); @@ -182,7 +184,7 @@ void MetadataUpdate::finish(int r) std::map m; for (const auto &[key, val] : daemon_meta) { - m.emplace(key, val.get_str()); + m.emplace(key, val.get_string()); } // prefer CRUSH physical host over container/pod hostname (tracker.ceph.com/issues/73080) // fall back to the reported hostname when CRUSH does not place the OSD @@ -250,7 +252,7 @@ std::map Mgr::load_store() std::map loaded; for (auto &key_str : cmd.json_result.get_array()) { - std::string const key = key_str.get_str(); + auto key = key_str.get_string(); dout(20) << "saw key '" << key << "'" << dendl; @@ -473,45 +475,44 @@ void Mgr::load_all_metadata() ceph_assert(osd_cmd.r == 0); for (auto &metadata_val : mds_cmd.json_result.get_array()) { - json_spirit::mObject daemon_meta = metadata_val.get_obj(); + auto daemon_meta = metadata_val.get_object(); if (daemon_meta.count("hostname") == 0) { dout(1) << "Skipping incomplete metadata entry" << dendl; continue; } DaemonStatePtr dm = std::make_shared(daemon_state.types); - dm->key = DaemonKey{"mds", - daemon_meta.at("name").get_str()}; - dm->hostname = daemon_meta.at("hostname").get_str(); + + dm->key = DaemonKey{"mds", daemon_meta.at("name").get_string()}; + dm->hostname = daemon_meta.at("hostname").get_string(); daemon_meta.erase("name"); daemon_meta.erase("hostname"); for (const auto &[key, val] : daemon_meta) { - dm->metadata.emplace(key, val.get_str()); + dm->metadata.emplace(key, val.get_string()); } daemon_state.insert(dm); } for (auto &metadata_val : mon_cmd.json_result.get_array()) { - json_spirit::mObject daemon_meta = metadata_val.get_obj(); + auto daemon_meta = metadata_val.get_object(); if (daemon_meta.count("hostname") == 0) { dout(1) << "Skipping incomplete metadata entry" << dendl; continue; } DaemonStatePtr dm = std::make_shared(daemon_state.types); - dm->key = DaemonKey{"mon", - daemon_meta.at("name").get_str()}; - dm->hostname = daemon_meta.at("hostname").get_str(); + dm->key = DaemonKey{"mon", daemon_meta.at("name").get_string() }; + dm->hostname = daemon_meta.at("hostname").get_string(); daemon_meta.erase("name"); daemon_meta.erase("hostname"); std::map m; for (const auto &[key, val] : daemon_meta) { - m.emplace(key, val.get_str()); + m.emplace(key, val.get_string()); } dm->set_metadata(m); @@ -519,19 +520,19 @@ void Mgr::load_all_metadata() } for (auto &osd_metadata_val : osd_cmd.json_result.get_array()) { - json_spirit::mObject osd_metadata = osd_metadata_val.get_obj(); + auto osd_metadata = osd_metadata_val.get_object(); if (osd_metadata.count("hostname") == 0) { dout(1) << "Skipping incomplete metadata entry" << dendl; continue; } DaemonStatePtr dm = std::make_shared(daemon_state.types); - int osd_id = osd_metadata.at("id").get_int(); + const auto osd_id = static_cast(osd_metadata.at("id").get_number()); dm->key = DaemonKey{"osd", stringify(osd_id)}; - dm->hostname = osd_metadata.at("hostname").get_str(); + dm->hostname = osd_metadata.at("hostname").get_string(); // prefer CRUSH physical host over container/pod hostname (tracker.ceph.com/issues/73080) - std::string crush_host = crush_hostname_for_osd(cluster_state, osd_id); + const std::string crush_host = crush_hostname_for_osd(cluster_state, osd_id); if (!crush_host.empty()) { dm->hostname = crush_host; } @@ -540,9 +541,9 @@ void Mgr::load_all_metadata() osd_metadata.erase("id"); osd_metadata.erase("hostname"); - std::map m; - for (const auto &i : osd_metadata) { - m[i.first] = i.second.get_str(); + map m; + for (const auto &[k, v] : osd_metadata) { + m[k] = v.get_string(); } dm->set_metadata(m); diff --git a/src/rgw/radosgw-admin/radosgw-admin.cc b/src/rgw/radosgw-admin/radosgw-admin.cc index 70476396233..6d7d92f5c65 100644 --- a/src/rgw/radosgw-admin/radosgw-admin.cc +++ b/src/rgw/radosgw-admin/radosgw-admin.cc @@ -2,8 +2,8 @@ // vim: ts=8 sw=2 sts=2 expandtab ft=cpp /* - * Copyright (C) 2025 IBM - */ + * Copyright (C) 2025-2026 IBM +*/ #include #include @@ -22,6 +22,8 @@ extern "C" { #include +#include + #include "auth/Crypto.h" #include "compressor/Compressor.h" @@ -1468,7 +1470,7 @@ static int read_decode_json(const string& infile, T& t) return ret; } JSONParser p; - if (!p.parse(bl.c_str(), bl.length())) { + if (!p.parse(bl)) { cout << "failed to parse JSON" << std::endl; return -EINVAL; } @@ -1492,7 +1494,7 @@ static int read_decode_json(const string& infile, T& t, K *k) return ret; } JSONParser p; - if (!p.parse(bl.c_str(), bl.length())) { + if (!p.parse(bl)) { cout << "failed to parse JSON" << std::endl; return -EINVAL; } @@ -2075,7 +2077,7 @@ static int send_to_remote_gateway(RGWRESTConn* conn, req_info& info, return ret; } - ret = parser.parse(response.c_str(), response.length()); + ret = parser.parse(response); if (ret < 0) { cerr << "failed to parse response" << std::endl; return ret; @@ -2105,14 +2107,15 @@ static int send_to_url(const string& url, if (!result) { return result.error(); } + int ret = rgw_http_error_to_errno(*result); if (ret < 0) { return ret; } - ret = parser.parse(response.c_str(), response.length()); + ret = parser.parse(response); if (ret < 0) { - cout << "failed to parse response" << std::endl; + cerr << "failed to parse remote response" << std::endl; return ret; } return 0; @@ -2157,7 +2160,7 @@ static int commit_period(rgw::sal::ConfigStore* cfgstore, realm, realm_writer, current_period, period, cerr, force, *site); if (ret < 0) { - cerr << "failed to commit period: " << cpp_strerror(-ret) << std::endl; + cerr << "commit_period(): failed to commit period: " << cpp_strerror(-ret) << std::endl; } (void) cfgstore->realm_notify_new_period(dpp(), null_yield, period); return ret; @@ -2166,7 +2169,6 @@ static int commit_period(rgw::sal::ConfigStore* cfgstore, if (remote.empty() && url.empty()) { // use the new master zone's connection remote = master_zone; - cerr << "Sending period to new master zone " << remote << std::endl; } boost::optional conn; RGWRESTConn *remote_conn = nullptr; @@ -2211,13 +2213,15 @@ static int commit_period(rgw::sal::ConfigStore* cfgstore, try { decode_json_obj(period, &p); } catch (const JSONDecoder::err& e) { - cout << "failed to decode JSON input: " << e.what() << std::endl; + cerr << "failed to decode JSON input: " << e.what() << std::endl; return -EINVAL; } + if (period.get_id().empty()) { - cerr << "Period commit got back an empty period id" << std::endl; + cerr << "commit_period(): Period commit got back an empty period id" << std::endl; return -EINVAL; } + // the master zone gave us back the period that it committed, so it's // safe to save it as our latest epoch constexpr bool exclusive = false; @@ -2288,18 +2292,21 @@ static int update_period(rgw::sal::ConfigStore* cfgstore, constexpr bool exclusive = false; ret = cfgstore->create_period(dpp(), null_yield, exclusive, period); + if (ret < 0) { cerr << "failed to driver period: " << cpp_strerror(-ret) << std::endl; return ret; } + if (commit) { ret = commit_period(cfgstore, realm, *realm_writer, period, remote, url, opt_region, access, secret, force, site); if (ret < 0) { - cerr << "failed to commit period: " << cpp_strerror(-ret) << std::endl; + cerr << "update_period(): failed to commit period: " << cpp_strerror(-ret) << std::endl; return ret; } } + encode_json("period", period, formatter); formatter->flush(cout); return 0; @@ -2351,6 +2358,7 @@ static int do_period_pull(rgw::sal::ConfigStore* cfgstore, cerr << "request failed: " << cpp_strerror(-ret) << std::endl; return ret; } + try { decode_json_obj(*period, &p); } catch (const JSONDecoder::err& e) { @@ -3724,6 +3732,7 @@ void init_realm_param(CephContext *cct, string& var, std::optional& opt_ // would need to be terminated, so the warning is simply suppressed. // coverity[root_function:SUPPRESS] int main(int argc, const char **argv) +try { auto args = argv_to_vec(argc, argv); if (args.empty()) { @@ -5677,10 +5686,19 @@ int main(int argc, const char **argv) info.request_uri = "/admin/realm"; map ¶ms = info.args.get_params(); - if (!realm_id.empty()) - params["id"] = realm_id; - if (!realm_name.empty()) - params["name"] = realm_name; + + if (realm_name.empty()) { + cerr << "missing realm name" << std::endl; + return EINVAL; + } + + if (realm_id.empty()) { + cerr << "missing realm id" << std::endl; + return EINVAL; + } + + params["id"] = realm_id; + params["name"] = realm_name; bufferlist bl; JSONParser p; @@ -5694,6 +5712,7 @@ int main(int argc, const char **argv) } return -ret; } + RGWRealm realm; try { decode_json_obj(realm, &p); @@ -5701,6 +5720,7 @@ int main(int argc, const char **argv) cerr << "failed to decode JSON response: " << e.what() << std::endl; return EINVAL; } + RGWPeriod period; auto& current_period = realm.get_current_period(); if (!current_period.empty()) { @@ -7374,7 +7394,7 @@ int main(int argc, const char **argv) remote, url, opt_region, access_key, secret_key, yes_i_really_mean_it, site.get()); if (ret < 0) { - cerr << "failed to commit period: " << cpp_strerror(-ret) << std::endl; + cerr << "OPT::PERIOD_COMMIT: failed to commit period: " << cpp_strerror(-ret) << std::endl; return -ret; } @@ -12311,6 +12331,7 @@ next: if (rgw::all_zonegroups_support(*site, rgw::zone_features::notification_v2) && driver->stat_topics_v1(tenant, null_yield, dpp()) == -ENOENT) { show_topics_info_v2(topic, subscribed_buckets, formatter.get()); + } else { encode_json("topic", topic, formatter.get()); } @@ -12791,3 +12812,9 @@ next: } return 0; } +catch(const std::exception& e) +{ + std::cerr << "exception: " << e.what() << std::endl; + + return 1; +} diff --git a/src/rgw/rgw_acl_swift.cc b/src/rgw/rgw_acl_swift.cc index 5e5c575e793..b4ba65da485 100644 --- a/src/rgw/rgw_acl_swift.cc +++ b/src/rgw/rgw_acl_swift.cc @@ -285,38 +285,27 @@ int create_account_policy(const DoutPrefixProvider* dpp, policy.create_default(owner.id, owner.display_name); auto& acl = policy.get_acl(); - JSONParser parser; - if (!parser.parse(acl_str.c_str(), acl_str.length())) { + ceph_json::value acl_json; + if (!ceph_json::parse(acl_str, acl_json)) { ldpp_dout(dpp, 0) << "ERROR: JSONParser::parse returned error=" << dendl; return -EINVAL; } - JSONObjIter iter = parser.find_first("admin"); - if (!iter.end() && (*iter)->is_array()) { - std::vector admin; - decode_json_obj(admin, *iter); - ldpp_dout(dpp, 0) << "admins: " << admin << dendl; + const auto add_acl_grants = [&] (std::string_view name, std::string_view label, int perm) { + const auto* values = ceph_json::find_value(acl_json, name); + if (!values || !values->is_array()) { + return; + } - add_grants(dpp, driver, admin, SWIFT_PERM_ADMIN, acl); - } + auto grants = ceph_json::decode_value>(*values); + ldpp_dout(dpp, 0) << label << ": " << grants << dendl; - iter = parser.find_first("read-write"); - if (!iter.end() && (*iter)->is_array()) { - std::vector readwrite; - decode_json_obj(readwrite, *iter); - ldpp_dout(dpp, 0) << "read-write: " << readwrite << dendl; + add_grants(dpp, driver, grants, perm, acl); + }; - add_grants(dpp, driver, readwrite, SWIFT_PERM_RWRT, acl); - } - - iter = parser.find_first("read-only"); - if (!iter.end() && (*iter)->is_array()) { - std::vector readonly; - decode_json_obj(readonly, *iter); - ldpp_dout(dpp, 0) << "read-only: " << readonly << dendl; - - add_grants(dpp, driver, readonly, SWIFT_PERM_READ, acl); - } + add_acl_grants("admin", "admins", SWIFT_PERM_ADMIN); + add_acl_grants("read-write", "read-write", SWIFT_PERM_RWRT); + add_acl_grants("read-only", "read-only", SWIFT_PERM_READ); return 0; } diff --git a/src/rgw/rgw_auth_keystone.cc b/src/rgw/rgw_auth_keystone.cc index d06d125b77f..4833d9f27ea 100644 --- a/src/rgw/rgw_auth_keystone.cc +++ b/src/rgw/rgw_auth_keystone.cc @@ -91,9 +91,6 @@ admin_token_retry: ret = validate.process(dpp, y); - /* NULL terminate for debug output. */ - token_body_bl.append(static_cast(0)); - /* Detect Keystone rejection earlier than during the token parsing. * Although failure at the parsing phase doesn't impose a threat, * this allows to return proper error code (EACCESS instead of EINVAL @@ -128,7 +125,7 @@ admin_token_retry: } ldpp_dout(dpp, 20) << "received response status=" << validate.get_http_status() - << ", body=" << token_body_bl.c_str() << dendl; + << ", body=" << std::string_view(token_body_bl.c_str()) << dendl; TokenEngine::token_envelope_t token_body; ret = token_body.parse(dpp, token, token_body_bl); @@ -559,22 +556,17 @@ auto EC2Engine::get_secret_from_keystone(const DoutPrefixProvider* dpp, /* now parse response */ - JSONParser parser; - if (! parser.parse(token_body_bl.c_str(), token_body_bl.length())) { + ceph_json::value response; + if (!ceph_json::parse(token_body_bl, response)) { ldpp_dout(dpp, 0) << "Keystone credential parse error: malformed json" << dendl; return make_pair(boost::none, -EINVAL); } - JSONObjIter credential_iter = parser.find_first("credential"); std::string secret_string; try { - if (!credential_iter.end()) { - JSONDecoder::decode_json("secret", secret_string, *credential_iter, true); - } else { - ldpp_dout(dpp, 0) << "Keystone credential not present in return from server" << dendl; - return make_pair(boost::none, -EINVAL); - } + const auto& credential = ceph_json::require_object(response, "credential"); + secret_string = ceph_json::require(credential, "secret"); } catch (const JSONDecoder::err& err) { ldpp_dout(dpp, 0) << "Keystone credential parse error: " << err.what() << dendl; return make_pair(boost::none, -EINVAL); diff --git a/src/rgw/rgw_keystone.cc b/src/rgw/rgw_keystone.cc index bdfa848b6c0..747ade6d1a3 100644 --- a/src/rgw/rgw_keystone.cc +++ b/src/rgw/rgw_keystone.cc @@ -263,7 +263,7 @@ int TokenEnvelope::parse(const DoutPrefixProvider *dpp, ceph::bufferlist& bl) { JSONParser parser; - if (! parser.parse(bl.c_str(), bl.length())) { + if (! parser.parse(bl)) { ldpp_dout(dpp, 0) << "Keystone token parse error: malformed json" << dendl; return -EINVAL; } diff --git a/src/rgw/rgw_metadata.cc b/src/rgw/rgw_metadata.cc index 5de3d248930..ef306af4633 100644 --- a/src/rgw/rgw_metadata.cc +++ b/src/rgw/rgw_metadata.cc @@ -311,7 +311,7 @@ int RGWMetadataManager::put(string& metadata_key, bufferlist& bl, } JSONParser parser; - if (!parser.parse(bl.c_str(), bl.length())) { + if (!parser.parse(bl)) { return -EINVAL; } diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index f4d4845cfe4..ce958d9c53d 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -192,7 +192,7 @@ int rgw_forward_request_to_master(const DoutPrefixProvider* dpp, if (ret < 0) { return ret; } - if (jp && !jp->parse(outdata.c_str(), outdata.length())) { + if (jp && !jp->parse(outdata)) { ldpp_dout(dpp, 0) << "failed parsing response from master zonegroup" << dendl; return -EINVAL; } diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 5172508fb8d..472f276c430 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -165,7 +165,7 @@ int rgw_rest_get_json_input(CephContext *cct, req_state *s, T& out, JSONParser parser; - if (!parser.parse(data.c_str(), data.length())) { + if (!parser.parse(data)) { return -EINVAL; } diff --git a/src/rgw/rgw_opa.cc b/src/rgw/rgw_opa.cc index 6840227cffe..fb0074c1c1c 100644 --- a/src/rgw/rgw_opa.cc +++ b/src/rgw/rgw_opa.cc @@ -78,16 +78,15 @@ int rgw_opa_authorize(RGWOp *& op, } /* check OPA response */ - JSONParser parser; - if (!parser.parse(bl.c_str(), bl.length())) { + ceph_json::value response; + if (!ceph_json::parse(bl, response)) { ldpp_dout(op, 2) << "OPA parse error: malformed json" << dendl; return -EINVAL; } - bool opa_result; - JSONDecoder::decode_json("result", opa_result, &parser); + const auto opa_result = ceph_json::require(response, "result"); - if (opa_result == false) { + if (false == opa_result) { ldpp_dout(op, 2) << "OPA rejecting request" << dendl; return -EPERM; } diff --git a/src/rgw/rgw_period.cc b/src/rgw/rgw_period.cc index ca533ce7c88..82a6a4fcbc6 100644 --- a/src/rgw/rgw_period.cc +++ b/src/rgw/rgw_period.cc @@ -154,6 +154,7 @@ void RGWPeriod::dump(Formatter *f) const encode_json("realm_epoch", realm_epoch, f); } +// Reads internal fields /from/ JSONObj: void RGWPeriod::decode_json(JSONObj *obj) { JSONDecoder::decode_json("id", id, obj); diff --git a/src/rgw/rgw_period_puller.cc b/src/rgw/rgw_period_puller.cc index fd48612f5b2..f18f19f11fe 100644 --- a/src/rgw/rgw_period_puller.cc +++ b/src/rgw/rgw_period_puller.cc @@ -53,7 +53,7 @@ int pull_period(const DoutPrefixProvider *dpp, RGWRESTConn* conn, const std::str } JSONParser parser; - r = parser.parse(data.c_str(), data.length()); + r = parser.parse(data); if (r < 0) { ldpp_dout(dpp, -1) << "request failed: " << cpp_strerror(-r) << dendl; return r; diff --git a/src/rgw/rgw_policy_s3.cc b/src/rgw/rgw_policy_s3.cc index d8fadfdc3c6..e06b54d5dba 100644 --- a/src/rgw/rgw_policy_s3.cc +++ b/src/rgw/rgw_policy_s3.cc @@ -243,14 +243,12 @@ int RGWPolicy::check(RGWPolicyEnv *env, string& err_msg) return 0; } - -int RGWPolicy::from_json(bufferlist& bl, string& err_msg) +int RGWPolicy::from_json(std::string_view json_in, string& err_msg) { JSONParser parser; - - if (!parser.parse(bl.c_str(), bl.length())) { - err_msg = "Malformed JSON"; - dout(0) << "malformed json" << dendl; + if (!parser.parse(json_in)) { + err_msg = fmt::format("malformed JSON (RGWPolicy) length = {}:\n{}\n----\n", json_in.length(), json_in); + dout(0) << err_msg << dendl; return -EINVAL; } @@ -258,7 +256,7 @@ int RGWPolicy::from_json(bufferlist& bl, string& err_msg) JSONObjIter iter = parser.find_first("expiration"); if (iter.end()) { err_msg = "Policy missing expiration"; - dout(0) << "expiration not found" << dendl; + dout(0) << err_msg << dendl; return -EINVAL; // change to a "no expiration" error following S3 } @@ -312,3 +310,9 @@ int RGWPolicy::from_json(bufferlist& bl, string& err_msg) } return 0; } + +int RGWPolicy::from_json(bufferlist& bl, string& err_msg) +{ + return from_json(std::string_view { bl.c_str(), bl.length() }, err_msg); +} + diff --git a/src/rgw/rgw_policy_s3.h b/src/rgw/rgw_policy_s3.h index 40ad3c86a41..cfe94b5ecc0 100644 --- a/src/rgw/rgw_policy_s3.h +++ b/src/rgw/rgw_policy_s3.h @@ -53,5 +53,6 @@ public: } int check(RGWPolicyEnv *env, std::string& err_msg); + int from_json(std::string_view bl, std::string& err_msg); int from_json(bufferlist& bl, std::string& err_msg); }; diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h index fac9afd124f..c20dfb95d67 100644 --- a/src/rgw/rgw_rest.h +++ b/src/rgw/rgw_rest.h @@ -61,8 +61,7 @@ std::tuple rgw_rest_get_json_input_keep_data(CephContext *cct, } JSONParser parser; - - if (!parser.parse(data.c_str(), data.length())) { + if (!parser.parse(data)) { return std::make_tuple(-EINVAL, std::move(data)); } diff --git a/src/rgw/rgw_rest.sts.cc b/src/rgw/rgw_rest.sts.cc new file mode 100644 index 00000000000..15b86c3fda7 --- /dev/null +++ b/src/rgw/rgw_rest.sts.cc @@ -0,0 +1,837 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + + + +#include "ceph_ver.h" +#include "common/Formatter.h" +#include "common/utf8.h" +#include "common/ceph_json.h" + +#include "rgw_rest.h" +#include "rgw_account.h" +#include "rgw_auth.h" +#include "rgw_auth_registry.h" +#include "jwt-cpp/jwt.h" +#include "rgw_rest_sts.h" + +#include "rgw_formats.h" +#include "rgw_client_io.h" + +#include "rgw_request.h" +#include "rgw_process.h" +#include "rgw_iam_policy.h" +#include "rgw_iam_policy_keywords.h" + +#include "rgw_sts.h" +#include "rgw_rest_oidc_provider.h" + + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +namespace rgw::auth::sts { + +bool +WebTokenEngine::is_applicable(const std::string& token) const noexcept +{ + return ! token.empty(); +} + +std::string +WebTokenEngine::get_role_tenant(const string& role_arn) const +{ + string tenant; + auto r_arn = rgw::ARN::parse(role_arn); + if (r_arn) { + tenant = r_arn->account; + } + return tenant; +} + +std::string +WebTokenEngine::get_role_name(const string& role_arn) const +{ + string role_name; + auto r_arn = rgw::ARN::parse(role_arn); + if (r_arn) { + role_name = r_arn->resource; + } + if (!role_name.empty()) { + auto pos = role_name.find_last_of('/'); + if(pos != string::npos) { + role_name = role_name.substr(pos + 1); + } + } + return role_name; +} + +int WebTokenEngine::load_provider(const DoutPrefixProvider* dpp, optional_yield y, + const string& role_arn, const string& iss, + RGWOIDCProviderInfo& info) const +{ + string tenant = get_role_tenant(role_arn); + + string idp_url = iss; + auto pos = idp_url.find("http://"); + if (pos == std::string::npos) { + pos = idp_url.find("https://"); + if (pos != std::string::npos) { + idp_url.erase(pos, 8); + } else { + pos = idp_url.find("www."); + if (pos != std::string::npos) { + idp_url.erase(pos, 4); + } + } + } else { + idp_url.erase(pos, 7); + } + + return driver->load_oidc_provider(dpp, y, tenant, idp_url, info); +} + +bool +WebTokenEngine::is_client_id_valid(vector& client_ids, const string& client_id) const +{ + for (auto it : client_ids) { + if (it == client_id) { + return true; + } + } + return false; +} + +bool +WebTokenEngine::is_cert_valid(const vector& thumbprints, const string& cert) const +{ + //calculate thumbprint of cert + std::unique_ptr certbio(BIO_new_mem_buf(cert.data(), cert.size()), BIO_free_all); + std::unique_ptr keybio(BIO_new(BIO_s_mem()), BIO_free_all); + string pw=""; + std::unique_ptr x_509cert(PEM_read_bio_X509(certbio.get(), nullptr, nullptr, const_cast(pw.c_str())), X509_free); + const EVP_MD* fprint_type = EVP_sha1(); + unsigned int fprint_size; + unsigned char fprint[EVP_MAX_MD_SIZE]; + + if (!X509_digest(x_509cert.get(), fprint_type, fprint, &fprint_size)) { + return false; + } + stringstream ss; + for (unsigned int i = 0; i < fprint_size; i++) { + ss << std::setfill('0') << std::setw(2) << std::hex << (0xFF & (unsigned int)fprint[i]); + } + std::string digest = ss.str(); + + for (auto& it : thumbprints) { + if (boost::iequals(it,digest)) { + return true; + } + } + return false; +} + +template +void +WebTokenEngine::recurse_and_insert(const string& key, const jwt::claim& c, T& t) const +{ + string s_val; + jwt::claim::type c_type = c.get_type(); + switch(c_type) { + case jwt::claim::type::null: + break; + case jwt::claim::type::boolean: + case jwt::claim::type::number: + case jwt::claim::type::int64: + { + s_val = c.to_json().serialize(); + t.emplace(std::make_pair(key, s_val)); + break; + } + case jwt::claim::type::string: + { + s_val = c.to_json().to_str(); + t.emplace(std::make_pair(key, s_val)); + break; + } + case jwt::claim::type::array: + { + const picojson::array& arr = c.as_array(); + for (auto& a : arr) { + recurse_and_insert(key, jwt::claim(a), t); + } + break; + } + case jwt::claim::type::object: + { + const picojson::object& obj = c.as_object(); + for (auto& m : obj) { + recurse_and_insert(m.first, jwt::claim(m.second), t); + } + break; + } + } + return; +} + +//Extract all token claims so that they can be later used in the Condition element of Role's trust policy +WebTokenEngine::token_t +WebTokenEngine::get_token_claims(const jwt::decoded_jwt& decoded) const +{ + WebTokenEngine::token_t token; + const auto& claims = decoded.get_payload_claims(); + + for (auto& c : claims) { + if (c.first == string(princTagsNamespace)) { + continue; + } + recurse_and_insert(c.first, c.second, token); + } + return token; +} + +//Offline validation of incoming Web Token which is a signed JWT (JSON Web Token) +std::tuple, boost::optional> +WebTokenEngine::get_from_jwt(const DoutPrefixProvider* dpp, const std::string& token, const req_state* const s, + optional_yield y) const +{ + WebTokenEngine::token_t t; + WebTokenEngine::principal_tags_t principal_tags; + try { + const auto& decoded = jwt::decode(token); + + auto& payload = decoded.get_payload(); + ldpp_dout(dpp, 20) << " payload = " << payload << dendl; + + t = get_token_claims(decoded); + + string iss; + if (decoded.has_issuer()) { + iss = decoded.get_issuer(); + } + + set aud; + if (decoded.has_audience()) { + aud = decoded.get_audience(); + } + + string client_id; + if (decoded.has_payload_claim("client_id")) { + client_id = decoded.get_payload_claim("client_id").as_string(); + } + if (client_id.empty() && decoded.has_payload_claim("clientId")) { + client_id = decoded.get_payload_claim("clientId").as_string(); + } + string azp; + if (decoded.has_payload_claim("azp")) { + azp = decoded.get_payload_claim("azp").as_string(); + } + + string role_arn = s->info.args.get("RoleArn"); + RGWOIDCProviderInfo provider; + int r = load_provider(dpp, y, role_arn, iss, provider); + if (r < 0) { + ldpp_dout(dpp, 0) << "Couldn't get oidc provider info using input iss" << iss << dendl; + throw -EACCES; + } + if (decoded.has_payload_claim(string(princTagsNamespace))) { + auto& cl = decoded.get_payload_claim(string(princTagsNamespace)); + if (cl.get_type() == jwt::claim::type::object || cl.get_type() == jwt::claim::type::array) { + recurse_and_insert("dummy", cl, principal_tags); + for (auto it : principal_tags) { + ldpp_dout(dpp, 5) << "Key: " << it.first << " Value: " << it.second << dendl; + } + } else { + ldpp_dout(dpp, 0) << "Malformed principal tags" << cl.as_string() << dendl; + throw -EINVAL; + } + } + if (! provider.client_ids.empty()) { + bool found = false; + for (auto& it : aud) { + if (is_client_id_valid(provider.client_ids, it)) { + found = true; + break; + } + } + if (! found && ! is_client_id_valid(provider.client_ids, client_id) && ! is_client_id_valid(provider.client_ids, azp)) { + ldpp_dout(dpp, 0) << "Client id in token doesn't match with that registered with oidc provider" << dendl; + throw -EACCES; + } + } + //Validate signature + if (decoded.has_algorithm()) { + auto& algorithm = decoded.get_algorithm(); + try { + validate_signature(dpp, decoded, algorithm, iss, provider.thumbprints, y); + } catch (...) { + throw -EACCES; + } + } else { + return {boost::none, boost::none}; + } + } catch (int error) { + if (error == -EACCES) { + throw -EACCES; + } + ldpp_dout(dpp, 5) << "Invalid JWT token" << dendl; + return {boost::none, boost::none}; + } + catch (...) { + ldpp_dout(dpp, 5) << "Invalid JWT token" << dendl; + return {boost::none, boost::none}; + } + return {t, principal_tags}; +} + +std::string +WebTokenEngine::get_cert_url(const string& iss, const DoutPrefixProvider *dpp, optional_yield y) const +{ + string cert_url; + string openidc_wellknown_url = iss; + bufferlist openidc_resp; + + if (openidc_wellknown_url.back() == '/') { + openidc_wellknown_url.pop_back(); + } + openidc_wellknown_url.append("/.well-known/openid-configuration"); + + RGWHTTPTransceiver openidc_req(cct, "GET", openidc_wellknown_url, &openidc_resp); + + //Headers + openidc_req.append_header("Content-Type", "application/x-www-form-urlencoded"); + + int res = openidc_req.process(dpp, y); + if (res < 0) { + ldpp_dout(dpp, 10) << "HTTP request res: " << res << dendl; + throw -EINVAL; + } + + //Debug only + ldpp_dout(dpp, 20) << "HTTP status: " << openidc_req.get_http_status() << dendl; + ldpp_dout(dpp, 20) << "JSON Response is: " << openidc_resp.c_str() << dendl; + + JSONParser parser; + if (parser.parse(openidc_resp)) { + JSONObj::data_val val; + if (parser.get_data("jwks_uri", &val)) { + cert_url = val.str.c_str(); + ldpp_dout(dpp, 20) << "Cert URL is: " << cert_url.c_str() << dendl; + } else { + ldpp_dout(dpp, 0) << "Malformed json returned while fetching openidc url" << dendl; + } + } + return cert_url; +} + +void +WebTokenEngine::validate_signature(const DoutPrefixProvider* dpp, const jwt::decoded_jwt& decoded, const string& algorithm, const string& iss, const vector& thumbprints, optional_yield y) const +{ + if (algorithm != "HS256" && algorithm != "HS384" && algorithm != "HS512") { + string cert_url = get_cert_url(iss, dpp, y); + if (cert_url.empty()) { + throw -EINVAL; + } + + // Get certificate + bufferlist cert_resp; + RGWHTTPTransceiver cert_req(cct, "GET", cert_url, &cert_resp); + //Headers + cert_req.append_header("Content-Type", "application/x-www-form-urlencoded"); + + int res = cert_req.process(dpp, y); + if (res < 0) { + ldpp_dout(dpp, 10) << "HTTP request res: " << res << dendl; + throw -EINVAL; + } + //Debug only + ldpp_dout(dpp, 20) << "HTTP status: " << cert_req.get_http_status() << dendl; + ldpp_dout(dpp, 20) << "JSON Response is: " << cert_resp.c_str() << dendl; + + JSONParser parser; + if (parser.parse(cert_resp)) { + JSONObj::data_val val; + if (parser.get_data("keys", &val)) { + if (val.str[0] == '[') { + val.str.erase(0, 1); + } + if (val.str[val.str.size() - 1] == ']') { + val.str = val.str.erase(val.str.size() - 1, 1); + } + if (parser.parse(val)) { + vector x5c; + if (JSONDecoder::decode_json("x5c", x5c, &parser)) { + string cert; + bool found_valid_cert = false; + for (auto& it : x5c) { + cert = "-----BEGIN CERTIFICATE-----\n" + it + "\n-----END CERTIFICATE-----"; + ldpp_dout(dpp, 20) << "Certificate is: " << cert.c_str() << dendl; + if (is_cert_valid(thumbprints, cert)) { + found_valid_cert = true; + break; + } + } + if (! found_valid_cert) { + ldpp_dout(dpp, 0) << "Cert doesn't match that with the thumbprints registered with oidc provider: " << cert.c_str() << dendl; + throw -EINVAL; + } + try { + //verify method takes care of expired tokens also + if (algorithm == "RS256") { + auto verifier = jwt::verify() + .allow_algorithm(jwt::algorithm::rs256{cert}); + + verifier.verify(decoded); + } else if (algorithm == "RS384") { + auto verifier = jwt::verify() + .allow_algorithm(jwt::algorithm::rs384{cert}); + + verifier.verify(decoded); + } else if (algorithm == "RS512") { + auto verifier = jwt::verify() + .allow_algorithm(jwt::algorithm::rs512{cert}); + + verifier.verify(decoded); + } else if (algorithm == "ES256") { + auto verifier = jwt::verify() + .allow_algorithm(jwt::algorithm::es256{cert}); + + verifier.verify(decoded); + } else if (algorithm == "ES384") { + auto verifier = jwt::verify() + .allow_algorithm(jwt::algorithm::es384{cert}); + + verifier.verify(decoded); + } else if (algorithm == "ES512") { + auto verifier = jwt::verify() + .allow_algorithm(jwt::algorithm::es512{cert}); + + verifier.verify(decoded); + } else if (algorithm == "PS256") { + auto verifier = jwt::verify() + .allow_algorithm(jwt::algorithm::ps256{cert}); + + verifier.verify(decoded); + } else if (algorithm == "PS384") { + auto verifier = jwt::verify() + .allow_algorithm(jwt::algorithm::ps384{cert}); + + verifier.verify(decoded); + } else if (algorithm == "PS512") { + auto verifier = jwt::verify() + .allow_algorithm(jwt::algorithm::ps512{cert}); + + verifier.verify(decoded); + } else { + ldpp_dout(dpp, 0) << "Unsupported algorithm: " << algorithm << dendl; + throw -EINVAL; + } + } catch (std::runtime_error& e) { + ldpp_dout(dpp, 0) << "Signature validation failed: " << e.what() << dendl; + throw; + } + catch (...) { + ldpp_dout(dpp, 0) << "Signature validation failed" << dendl; + throw; + } + } else { + ldpp_dout(dpp, 0) << "x5c not present" << dendl; + throw -EINVAL; + } + } else { + ldpp_dout(dpp, 0) << "Malformed JSON object for keys" << dendl; + throw -EINVAL; + } + } else { + ldpp_dout(dpp, 0) << "keys not present in JSON" << dendl; + throw -EINVAL; + } //if-else get-data + } else { + ldpp_dout(dpp, 0) << "Malformed json returned while fetching cert" << dendl; + throw -EINVAL; + } //if-else parser cert_resp + } else { + ldpp_dout(dpp, 0) << "JWT signed by HMAC algos are currently not supported" << dendl; + throw -EINVAL; + } +} + +WebTokenEngine::result_t +WebTokenEngine::authenticate( const DoutPrefixProvider* dpp, + const std::string& token, + const req_state* const s, + optional_yield y) const +{ + if (! is_applicable(token)) { + return result_t::deny(); + } + + try { + auto [t, princ_tags] = get_from_jwt(dpp, token, s, y); + if (t) { + string role_session = s->info.args.get("RoleSessionName"); + if (role_session.empty()) { + ldout(s->cct, 0) << "Role Session Name is empty " << dendl; + return result_t::deny(-EACCES); + } + string role_arn = s->info.args.get("RoleArn"); + string role_tenant = get_role_tenant(role_arn); + string role_name = get_role_name(role_arn); + + rgw_account_id role_account; + if (rgw::account::validate_id(role_tenant)) { + role_account = std::move(role_tenant); + role_tenant.clear(); + } + + std::unique_ptr role = driver->get_role(role_name, role_tenant, role_account); + int ret = role->load_by_name(dpp, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "Role not found: name:" << role_name << " tenant: " << role_tenant << dendl; + return result_t::deny(-EACCES); + } + + std::optional account; + if (!role_account.empty()) { + account.emplace(); + rgw::sal::Attrs attrs; // ignored + RGWObjVersionTracker objv; // ignored + ret = driver->load_account_by_id(dpp, y, role_account, + *account, attrs, objv); + if (ret < 0) { + ldpp_dout(dpp, 0) << "Role account " << role_account << " not found" << dendl; + return result_t::deny(-EACCES); + } + } + + boost::optional> role_tags = role->get_tags(); + auto apl = apl_factory->create_apl_web_identity( + cct, s, role->get_id(), role_session, role_tenant, + *t, role_tags, princ_tags, std::move(account)); + return result_t::grant(std::move(apl)); + } + return result_t::deny(-EACCES); + } + catch (...) { + return result_t::deny(-EACCES); + } +} + +} // namespace rgw::auth::sts + +int RGWREST_STS::verify_permission(optional_yield y) +{ + STS::STSService _sts(s->cct, driver, s->user->get_id(), s->auth.identity.get()); + sts = std::move(_sts); + + string rArn = s->info.args.get("RoleArn"); + const auto& [ret, role] = sts.getRoleInfo(s, rArn, y); + if (ret < 0) { + ldpp_dout(this, 0) << "failed to get role info using role arn: " << rArn << dendl; + return ret; + } + string policy = role->get_assume_role_policy(); + + //Parse the policy + //TODO - This step should be part of Role Creation + try { + // resource policy is not restricted to the current tenant + const std::string* policy_tenant = nullptr; + + const rgw::IAM::Policy p(s->cct, policy_tenant, policy, false); + if (!s->principal_tags.empty()) { + auto res = p.eval(s->env, *s->auth.identity, rgw::IAM::stsTagSession, boost::none); + if (res != rgw::IAM::Effect::Allow) { + ldout(s->cct, 0) << "evaluating policy for stsTagSession returned deny/pass" << dendl; + return -EPERM; + } + } + uint64_t op; + if (get_type() == RGW_STS_ASSUME_ROLE_WEB_IDENTITY) { + op = rgw::IAM::stsAssumeRoleWithWebIdentity; + } else { + op = rgw::IAM::stsAssumeRole; + } + + auto res = p.eval(s->env, *s->auth.identity, op, boost::none); + if (res != rgw::IAM::Effect::Allow) { + ldout(s->cct, 0) << "evaluating policy for op: " << op << " returned deny/pass" << dendl; + return -EPERM; + } + } catch (rgw::IAM::PolicyParseException& e) { + ldpp_dout(this, 0) << "failed to parse policy: " << e.what() << dendl; + return -EPERM; + } + + return 0; +} + +void RGWREST_STS::send_response() +{ + if (op_ret) { + set_req_state_err(s, op_ret); + } + dump_errno(s); + end_header(s); +} + +int RGWSTSGetSessionToken::verify_permission(optional_yield y) +{ + rgw::Partition partition = rgw::Partition::aws; + rgw::Service service = rgw::Service::s3; + if (!verify_user_permission(this, + s, + rgw::ARN(partition, service, "", s->user->get_tenant(), ""), + rgw::IAM::stsGetSessionToken)) { + ldpp_dout(this, 0) << "User does not have permission to perform GetSessionToken" << dendl; + return -EACCES; + } + + return 0; +} + +int RGWSTSGetSessionToken::get_params() +{ + duration = s->info.args.get("DurationSeconds"); + serialNumber = s->info.args.get("SerialNumber"); + tokenCode = s->info.args.get("TokenCode"); + + if (! duration.empty()) { + string err; + uint64_t duration_in_secs = strict_strtoll(duration.c_str(), 10, &err); + if (!err.empty()) { + ldpp_dout(this, 0) << "Invalid value of input duration: " << duration << dendl; + return -EINVAL; + } + + if (duration_in_secs < STS::GetSessionTokenRequest::getMinDuration() || + duration_in_secs > s->cct->_conf->rgw_sts_max_session_duration) { + ldpp_dout(this, 0) << "Invalid duration in secs: " << duration_in_secs << dendl; + return -EINVAL; + } + } + + return 0; +} + +void RGWSTSGetSessionToken::execute(optional_yield y) +{ + if (op_ret = get_params(); op_ret < 0) { + return; + } + + STS::STSService sts(s->cct, driver, s->user->get_id(), s->auth.identity.get()); + + STS::GetSessionTokenRequest req(duration, serialNumber, tokenCode); + const auto& [ret, creds] = sts.getSessionToken(this, req); + op_ret = std::move(ret); + //Dump the output + if (op_ret == 0) { + s->formatter->open_object_section_in_ns("GetSessionTokenResponse", RGW_REST_STS_XMLNS); + s->formatter->open_object_section("GetSessionTokenResult"); + s->formatter->open_object_section("Credentials"); + creds.dump(s->formatter); + s->formatter->close_section(); + s->formatter->close_section(); + s->formatter->close_section(); + } +} + +int RGWSTSAssumeRoleWithWebIdentity::get_params() +{ + duration = s->info.args.get("DurationSeconds"); + providerId = s->info.args.get("ProviderId"); + policy = s->info.args.get("Policy"); + roleArn = s->info.args.get("RoleArn"); + roleSessionName = s->info.args.get("RoleSessionName"); + iss = s->info.args.get("provider_id"); + sub = s->info.args.get("sub"); + aud = s->info.args.get("aud"); + + if (roleArn.empty() || roleSessionName.empty() || sub.empty() || aud.empty()) { + ldpp_dout(this, 0) << "ERROR: one of role arn or role session name or token is empty" << dendl; + return -EINVAL; + } + + if (! policy.empty()) { + try { + const rgw::IAM::Policy p( + s->cct, nullptr, policy, + s->cct->_conf.get_val("rgw_policy_reject_invalid_principals")); + } + catch (rgw::IAM::PolicyParseException& e) { + ldpp_dout(this, 5) << "failed to parse policy: " << e.what() << "policy" << policy << dendl; + s->err.message = e.what(); + return -ERR_MALFORMED_DOC; + } + } + + return 0; +} + +void RGWSTSAssumeRoleWithWebIdentity::execute(optional_yield y) +{ + if (op_ret = get_params(); op_ret < 0) { + return; + } + + STS::AssumeRoleWithWebIdentityRequest req(s->cct, duration, providerId, policy, roleArn, + roleSessionName, iss, sub, aud, s->principal_tags); + STS::AssumeRoleWithWebIdentityResponse response = sts.assumeRoleWithWebIdentity(this, req); + op_ret = std::move(response.assumeRoleResp.retCode); + + //Dump the output + if (op_ret == 0) { + s->formatter->open_object_section_in_ns("AssumeRoleWithWebIdentityResponse", RGW_REST_STS_XMLNS); + s->formatter->open_object_section("AssumeRoleWithWebIdentityResult"); + encode_json("SubjectFromWebIdentityToken", response.sub , s->formatter); + encode_json("Audience", response.aud , s->formatter); + s->formatter->open_object_section("AssumedRoleUser"); + response.assumeRoleResp.user.dump(s->formatter); + s->formatter->close_section(); + s->formatter->open_object_section("Credentials"); + response.assumeRoleResp.creds.dump(s->formatter); + s->formatter->close_section(); + encode_json("Provider", response.providerId , s->formatter); + encode_json("PackedPolicySize", response.assumeRoleResp.packedPolicySize , s->formatter); + s->formatter->close_section(); + s->formatter->close_section(); + } +} + +int RGWSTSAssumeRole::get_params() +{ + duration = s->info.args.get("DurationSeconds"); + externalId = s->info.args.get("ExternalId"); + policy = s->info.args.get("Policy"); + roleArn = s->info.args.get("RoleArn"); + roleSessionName = s->info.args.get("RoleSessionName"); + serialNumber = s->info.args.get("SerialNumber"); + tokenCode = s->info.args.get("TokenCode"); + + if (roleArn.empty() || roleSessionName.empty()) { + ldpp_dout(this, 0) << "ERROR: one of role arn or role session name is empty" << dendl; + return -EINVAL; + } + + if (! policy.empty()) { + try { + const rgw::IAM::Policy p( + s->cct, nullptr, policy, + s->cct->_conf.get_val("rgw_policy_reject_invalid_principals")); + } + catch (rgw::IAM::PolicyParseException& e) { + ldpp_dout(this, 0) << "failed to parse policy: " << e.what() << "policy" << policy << dendl; + s->err.message = e.what(); + return -ERR_MALFORMED_DOC; + } + } + + return 0; +} + +void RGWSTSAssumeRole::execute(optional_yield y) +{ + if (op_ret = get_params(); op_ret < 0) { + return; + } + + STS::AssumeRoleRequest req(s->cct, duration, externalId, policy, roleArn, + roleSessionName, serialNumber, tokenCode); + STS::AssumeRoleResponse response = sts.assumeRole(s, req, y); + op_ret = std::move(response.retCode); + //Dump the output + if (op_ret == 0) { + s->formatter->open_object_section_in_ns("AssumeRoleResponse", RGW_REST_STS_XMLNS); + s->formatter->open_object_section("AssumeRoleResult"); + s->formatter->open_object_section("Credentials"); + response.creds.dump(s->formatter); + s->formatter->close_section(); + s->formatter->open_object_section("AssumedRoleUser"); + response.user.dump(s->formatter); + s->formatter->close_section(); + encode_json("PackedPolicySize", response.packedPolicySize , s->formatter); + s->formatter->close_section(); + s->formatter->close_section(); + } +} + +int RGW_Auth_STS::authorize(const DoutPrefixProvider *dpp, + rgw::sal::Driver* driver, + const rgw::auth::StrategyRegistry& auth_registry, + req_state *s, optional_yield y) +{ + return rgw::auth::Strategy::apply(dpp, auth_registry.get_sts(), s, y); +} + +using op_generator = RGWOp*(*)(); +static const std::unordered_map op_generators = { + {"AssumeRole", []() -> RGWOp* {return new RGWSTSAssumeRole;}}, + {"GetSessionToken", []() -> RGWOp* {return new RGWSTSGetSessionToken;}}, + {"AssumeRoleWithWebIdentity", []() -> RGWOp* {return new RGWSTSAssumeRoleWithWebIdentity;}} +}; + +bool RGWHandler_REST_STS::action_exists(const req_state* s) +{ + if (s->info.args.exists("Action")) { + const std::string action_name = s->info.args.get("Action"); + return op_generators.contains(action_name); + } + return false; +} + +RGWOp *RGWHandler_REST_STS::op_post() +{ + if (s->info.args.exists("Action")) { + const std::string action_name = s->info.args.get("Action"); + const auto action_it = op_generators.find(action_name); + if (action_it != op_generators.end()) { + return action_it->second(); + } + ldpp_dout(s, 10) << "unknown action '" << action_name << "' for STS handler" << dendl; + } else { + ldpp_dout(s, 10) << "missing action argument in STS handler" << dendl; + } + return nullptr; +} + +int RGWHandler_REST_STS::init(rgw::sal::Driver* driver, + req_state *s, + rgw::io::BasicClient *cio) +{ + s->dialect = "sts"; + s->prot_flags = RGW_REST_STS; + + return RGWHandler_REST::init(driver, s, cio); +} + +int RGWHandler_REST_STS::authorize(const DoutPrefixProvider* dpp, optional_yield y) +{ + if (s->info.args.exists("Action") && s->info.args.get("Action") == "AssumeRoleWithWebIdentity") { + return RGW_Auth_STS::authorize(dpp, driver, auth_registry, s, y); + } + return RGW_Auth_S3::authorize(dpp, driver, auth_registry, s, y); +} + +RGWHandler_REST* +RGWRESTMgr_STS::get_handler(rgw::sal::Driver* driver, + req_state* const s, + const rgw::auth::StrategyRegistry& auth_registry, + const std::string& frontend_prefix) +{ + return new RGWHandler_REST_STS(auth_registry); +} diff --git a/src/rgw/rgw_rest_account.cc b/src/rgw/rgw_rest_account.cc index 7eca38c5297..e90c7b71ba4 100644 --- a/src/rgw/rgw_rest_account.cc +++ b/src/rgw/rgw_rest_account.cc @@ -14,6 +14,7 @@ */ #include "rgw_rest_account.h" +#include "common/ceph_json.h" #include "rgw_account.h" #include "rgw_process_env.h" @@ -73,9 +74,8 @@ void RGWOp_Account_Create::execute(optional_yield y) if (!driver->is_meta_master()) { bufferlist data; - JSONParser parser; op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(), - &data, &parser, s->info, s->err, y); + &data, nullptr, s->info, s->err, y); if (op_ret < 0) { ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl; return; @@ -83,7 +83,16 @@ void RGWOp_Account_Create::execute(optional_yield y) // the master zone may have generated its own account id, use the same std::string meta_master_id; - JSONDecoder::decode_json("id", meta_master_id, &parser); + try { + const auto response = ceph_json::parse(data); + meta_master_id = ceph_json::require(response, "id"); + } catch (const JSONDecoder::err& e) { + ldpp_dout(this, 4) << "forward_request_to_master returned invalid account id: " + << e.what() << dendl; + op_ret = -EINVAL; + return; + } + if (meta_master_id.empty()) { ldpp_dout(this, 4) << "forward_request_to_master returned empty account id" << dendl; op_ret = -EINVAL; diff --git a/src/rgw/rgw_rest_client.cc b/src/rgw/rgw_rest_client.cc index 02b80c2f1ed..3db281dfe67 100644 --- a/src/rgw/rgw_rest_client.cc +++ b/src/rgw/rgw_rest_client.cc @@ -446,8 +446,6 @@ auto RGWRESTSimpleRequest::forward_request(const DoutPrefixProvider *dpp, const return tl::unexpected(-ERR_SERVICE_UNAVAILABLE); } - response.append((char)0); /* NULL terminate response */ - if (outbl) { *outbl = std::move(response); } diff --git a/src/rgw/rgw_rest_conn.h b/src/rgw/rgw_rest_conn.h index 82794867e56..8a88acf82e2 100644 --- a/src/rgw/rgw_rest_conn.h +++ b/src/rgw/rgw_rest_conn.h @@ -17,7 +17,7 @@ template inline int parse_decode_json(T& t, bufferlist& bl) { JSONParser p; - if (!p.parse(bl.c_str(), bl.length())) { + if (!p.parse(bl)) { return -EINVAL; } diff --git a/src/rgw/rgw_rest_role.cc b/src/rgw/rgw_rest_role.cc index 490803f59f5..6680d5b97fe 100644 --- a/src/rgw/rgw_rest_role.cc +++ b/src/rgw/rgw_rest_role.cc @@ -449,7 +449,7 @@ int RGWModifyRoleTrustPolicy::init_processing(optional_yield y) } JSONParser p; - if (!p.parse(trust_policy.c_str(), trust_policy.length())) { + if (!p.parse(trust_policy)) { ldpp_dout(this, 20) << "ERROR: failed to parse assume role policy doc" << dendl; return -ERR_MALFORMED_DOC; } diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 5e627b90f39..42c863c2142 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -3488,16 +3488,17 @@ int RGWPostObj_ObjStore_S3::get_policy(optional_yield y) return -EINVAL; } - decoded_policy.append('\0'); // NULL terminate + if('\0' != decoded_policy.c_str()[decoded_policy.length() - 1]) + decoded_policy.append('\0'); // NULL terminate + ldpp_dout(this, 20) << "POST policy: " << decoded_policy.c_str() << dendl; - - int r = post_policy.from_json(decoded_policy, err_msg); + int r = post_policy.from_json(std::string_view { decoded_policy.c_str() }, err_msg); if (r < 0) { if (err_msg.empty()) { err_msg = "Failed to parse policy"; } - ldpp_dout(this, 0) << "failed to parse policy" << dendl; + ldpp_dout(this, 0) << err_msg << dendl; return -EINVAL; } diff --git a/src/rgw/rgw_rest_sts.cc b/src/rgw/rgw_rest_sts.cc index d9b3f7ff952..6e8269fedf9 100644 --- a/src/rgw/rgw_rest_sts.cc +++ b/src/rgw/rgw_rest_sts.cc @@ -324,7 +324,7 @@ WebTokenEngine::get_cert_url(const string& iss, const DoutPrefixProvider *dpp, o ldpp_dout(dpp, 20) << "JSON Response is: " << openidc_resp.c_str() << dendl; JSONParser parser; - if (parser.parse(openidc_resp.c_str(), openidc_resp.length())) { + if (parser.parse(openidc_resp)) { JSONObj::data_val val; if (parser.get_data("jwks_uri", &val)) { cert_url = val.str.c_str(); @@ -633,24 +633,21 @@ WebTokenEngine::validate_signature(const DoutPrefixProvider* dpp, const jwt::dec ldpp_dout(dpp, 20) << "HTTP status: " << cert_req.get_http_status() << dendl; ldpp_dout(dpp, 20) << "JSON Response is: " << cert_resp.c_str() << dendl; - JSONParser parser; - if (parser.parse(cert_resp.c_str(), cert_resp.length())) { - JSONObj* val = parser.find_obj("keys"); - if (val && val->is_array()) { - vector keys = val->get_array_elements(); - for (auto &key : keys) { - JSONParser k_parser; - vector x5c; - std::string use, kid; - if (k_parser.parse(key.c_str(), key.size())) { - if (JSONDecoder::decode_json("kid", kid, &k_parser)) { - ldpp_dout(dpp, 20) << "Checking key id: " << kid << dendl; - } - if (JSONDecoder::decode_json("use", use, &k_parser) && use != "sig") { - continue; - } + ceph_json::value cert_json; + if (ceph_json::parse(cert_resp, cert_json)) { + try { + for (const auto& key : ceph_json::require_array(cert_json, "keys")) { + if (const auto kid = ceph_json::field(key, "kid"); kid) { + ldpp_dout(dpp, 20) << "Checking key id: " << *kid << dendl; + } - if (JSONDecoder::decode_json("x5c", x5c, &k_parser)) { + if (const auto use = ceph_json::field(key, "use"); + use && "sig" != *use) { + continue; + } + + if (const auto x5c = ceph_json::field>(key, "x5c"); + x5c) { string cert; bool found_valid_cert = false; bool skip_thumbprint_verification = cct->_conf.get_val( @@ -661,7 +658,7 @@ WebTokenEngine::validate_signature(const DoutPrefixProvider* dpp, const jwt::dec << dendl; throw std::system_error(EINVAL, std::system_category()); } - for (auto& it : x5c) { + for (auto& it : *x5c) { cert = "-----BEGIN CERTIFICATE-----\n" + it + "\n-----END CERTIFICATE-----"; ldpp_dout(dpp, 20) << "Certificate is: " << cert.c_str() << dendl; if (skip_thumbprint_verification || is_cert_valid(thumbprints, cert)) { @@ -739,27 +736,27 @@ WebTokenEngine::validate_signature(const DoutPrefixProvider* dpp, const jwt::dec } else { // Try bare key validation ldpp_dout(dpp, 20) << "Trying bare key validation" << dendl; - std::string kty; - if (JSONDecoder::decode_json("kty", kty, &k_parser) && kty != "RSA") { + if (const auto kty = ceph_json::field(key, "kty"); + kty && "RSA" != *kty) { ldpp_dout(dpp, 10) << "Only RSA bare key validation is currently supported" << dendl; continue; } if (algorithm == "RS256" || algorithm == "RS384" || algorithm == "RS512") { - std::string n, e; //modulus and exponent - if (JSONDecoder::decode_json("n", n, &k_parser) && JSONDecoder::decode_json("e", e, &k_parser)) { - if (validate_signature_using_n_e(dpp, decoded, algorithm, n, e)) { + const auto n = ceph_json::field(key, "n"); // modulus + const auto e = ceph_json::field(key, "e"); // exponent + if (n && e) { + if (validate_signature_using_n_e(dpp, decoded, algorithm, *n, *e)) { return; } } ldpp_dout(dpp, 10) << "Bare key parameters (n&e) are not present for key" << dendl; } } - } //end k_parser.parse - } //end for iterate through keys + } ldpp_dout(dpp, 0) << "Signature can not be validated with the JWKS present." << dendl; throw std::system_error(EINVAL, std::system_category()); - } else { //end val->is_array + } catch (const JSONDecoder::err&) { ldpp_dout(dpp, 0) << "keys not present in JSON" << dendl; throw std::system_error(EINVAL, std::system_category()); } diff --git a/src/rgw/rgw_token.h b/src/rgw/rgw_token.h index e76221fcafd..c810089848c 100644 --- a/src/rgw/rgw_token.h +++ b/src/rgw/rgw_token.h @@ -82,13 +82,13 @@ namespace rgw { explicit RGWToken(const string& json) { JSONParser p; - p.parse(json.c_str(), json.length()); + p.parse(json); JSONDecoder::decode_json(RGWToken::type_name, *this, &p); } RGWToken& operator=(const std::string& json) { JSONParser p; - p.parse(json.c_str(), json.length()); + p.parse(json); JSONDecoder::decode_json(RGWToken::type_name, *this, &p); return *this; } diff --git a/src/test/common/test_json_formattable.cc b/src/test/common/test_json_formattable.cc index 960eef6a3c5..6471ca9ea74 100644 --- a/src/test/common/test_json_formattable.cc +++ b/src/test/common/test_json_formattable.cc @@ -4,6 +4,7 @@ /* * Ceph - scalable distributed file system * + * Copyright (C) 2026 IBM * Copyright (C) 2018 Red Hat Inc. * * This library is free software; you can redistribute it and/or @@ -38,7 +39,7 @@ static void get_jf(const string& s, JSONFormattable *f) } } -TEST(formatable, str) { +TEST(formattable, str) { JSONFormattable f; get_jf("{ \"foo\": \"bar\" }", &f); ASSERT_EQ((string)f["foo"], "bar"); @@ -46,7 +47,7 @@ TEST(formatable, str) { ASSERT_EQ((string)f["fooz"]("lala"), "lala"); } -TEST(formatable, str2) { +TEST(formattable, str2) { JSONFormattable f; get_jf("{ \"foo\": \"bar\" }", &f); ASSERT_EQ((string)f["foo"], "bar"); @@ -62,13 +63,13 @@ TEST(formatable, str2) { } -TEST(formatable, str3) { +TEST(formattable, str3) { JSONFormattable f; get_jf("{ \"foo\": \"1234bar56\" }", &f); ASSERT_EQ((string)f["foo"], "1234bar56"); } -TEST(formatable, int) { +TEST(formattable, int) { JSONFormattable f; get_jf("{ \"foo\": 1 }", &f); ASSERT_EQ((int)f["foo"], 1); @@ -83,7 +84,7 @@ TEST(formatable, int) { ASSERT_EQ((int)f2["fooz"](111), 123); } -TEST(formatable, bool) { +TEST(formattable, bool) { JSONFormattable f; get_jf("{ \"foo\": \"true\" }", &f); ASSERT_EQ((bool)f["foo"], true); @@ -95,7 +96,7 @@ TEST(formatable, bool) { ASSERT_EQ((bool)f["foo"], false); } -TEST(formatable, nested) { +TEST(formattable, nested) { JSONFormattable f; get_jf("{ \"obj\": { \"foo\": 1, \"inobj\": { \"foo\": 2 } } }", &f); ASSERT_EQ((int)f["foo"], 0); @@ -103,7 +104,7 @@ TEST(formatable, nested) { ASSERT_EQ((int)f["obj"]["inobj"]["foo"], 2); } -TEST(formatable, array) { +TEST(formattable, array) { JSONFormattable f; get_jf("{ \"arr\": [ { \"foo\": 1, \"inobj\": { \"foo\": 2 } }," "{ \"foo\": 2 } ] }", &f); @@ -124,7 +125,7 @@ TEST(formatable, array) { } } -TEST(formatable, bin_encode) { +TEST(formattable, bin_encode) { JSONFormattable f, f2; get_jf("{ \"arr\": [ { \"foo\": 1, \"bar\": \"aaa\", \"inobj\": { \"foo\": 2 } }," "{ \"foo\": 2, \"inobj\": { \"foo\": 3 } } ] }", &f); @@ -156,7 +157,7 @@ TEST(formatable, bin_encode) { } -TEST(formatable, json_encode) { +TEST(formattable, json_encode) { JSONFormattable f, f2; get_jf("{ \"arr\": [ { \"foo\": 1, \"bar\": \"aaa\", \"inobj\": { \"foo\": 2 } }," "{ \"foo\": 2, \"inobj\": { \"foo\": 3 } } ] }", &f); @@ -181,7 +182,7 @@ TEST(formatable, json_encode) { } -TEST(formatable, set) { +TEST(formattable, set) { JSONFormattable f, f2; f.set("", "{ \"abc\": \"xyz\"}"); @@ -200,13 +201,13 @@ TEST(formatable, set) { ASSERT_EQ((int)f["obj"]["c"], 30); } -TEST(formatable, set2) { +TEST(formattable, set2) { JSONFormattable f; f.set("foo", "1234bar56"); ASSERT_EQ((string)f["foo"], "1234bar56"); } -TEST(formatable, erase) { +TEST(formattable, erase) { JSONFormattable f, f2; f.set("", "{ \"abc\": \"xyz\"}"); @@ -240,7 +241,7 @@ static void dumpf(const JSONFormattable& f) { dumpt(f, "f"); } -TEST(formatable, set_array) { +TEST(formattable, set_array) { JSONFormattable f, f2; f.set("asd[0]", "\"xyz\""); @@ -272,7 +273,7 @@ TEST(formatable, set_array) { ASSERT_EQ((string)f2[0]["field"], "xyz"); } -TEST(formatable, erase_array) { +TEST(formattable, erase_array) { JSONFormattable f; f.set("asd[0]", "\"xyz\""); @@ -315,14 +316,13 @@ void formatter_convert(JSONFormatter& formatter, JSONFormattable *dest) get_jf(ss.str(), dest); } -TEST(formatable, encode_simple) { +TEST(formattable, encode_simple) { JSONFormattable f; encode_json("foo", "bar", &f); ASSERT_EQ((string)f["foo"], "bar"); - JSONFormatter formatter; { Formatter::ObjectSection s(formatter, "os"); @@ -422,7 +422,7 @@ struct struct2 { }; -TEST(formatable, encode_struct) { +TEST(formattable, encode_struct) { JSONFormattable f; struct2 s2; diff --git a/src/test/common/test_json_formatter.cc b/src/test/common/test_json_formatter.cc index 79a20df22da..b3f29e69a9b 100644 --- a/src/test/common/test_json_formatter.cc +++ b/src/test/common/test_json_formatter.cc @@ -4,6 +4,7 @@ /* * Ceph - scalable distributed file system * + * Copyright (C) 2025-2026 IBM * Copyright (C) 2018 Red Hat Inc. * * This library is free software; you can redistribute it and/or @@ -21,18 +22,18 @@ #include "common/JSONFormatter.h" #include "common/StackStringStream.h" +#include +#include #include using namespace std; +const string outstring = +"{\"pg_ready\":true, \"pg_stats\":[ { \"pgid\":\"1.0\", \"version\":\"16'56\",\"reported_seq\":\"62\",\"reported_epoch\":\"20\",\"state\":\"active+clean+inconsistent\",\"last_fresh\":\"2018-12-18 15:21:22.173804\",\"last_change\":\"2018-12-18 15:21:22.173804\",\"last_active\":\"2018-12-18 15:21:22.173804\",\"last_peered\":\"2018-12-18 15:21:22.173804\",\"last_clean\":\"2018-12-18 15:21:22.173804\",\"last_became_active\":\"2018-12-18 15:21:21.347685\",\"last_became_peered\":\"2018-12-18 15:21:21.347685\",\"last_unstale\":\"2018-12-18 15:21:22.173804\",\"last_undegraded\":\"2018-12-18 15:21:22.173804\",\"last_fullsized\":\"2018-12-18 15:21:22.173804\",\"mapping_epoch\":19,\"log_start\":\"0'0\",\"ondisk_log_start\":\"0'0\",\"created\":7,\"last_epoch_clean\":20,\"parent\":\"0.0\",\"parent_split_bits\":0,\"last_scrub\":\"16'56\",\"last_scrub_stamp\":\"2018-12-18 15:21:22.173684\",\"last_deep_scrub\":\"0'0\",\"last_deep_scrub_stamp\":\"2018-12-18 15:21:06.514438\",\"last_clean_scrub_stamp\":\"2018-12-18 15:21:06.514438\",\"log_size\":56,\"ondisk_log_size\":56,\"stats_invalid\":false,\"dirty_stats_invalid\":false,\"omap_stats_invalid\":false,\"hitset_stats_invalid\":false,\"hitset_bytes_stats_invalid\":false,\"pin_stats_invalid\":false,\"manifest_stats_invalid\":false,\"snaptrimq_len\":0,\"stat_sum\":{\"num_bytes\":24448,\"num_objects\":36,\"num_object_clones\":20,\"num_object_copies\":36,\"num_objects_missing_on_primary\":0,\"num_objects_missing\":0,\"num_objects_degraded\":0,\"num_objects_misplaced\":0,\"num_objects_unfound\":0,\"num_objects_dirty\":36,\"num_whiteouts\":3,\"num_read\":0,\"num_read_kb\":0,\"num_write\":36,\"num_write_kb\":50,\"num_scrub_errors\":20,\"num_shallow_scrub_errors\":20,\"num_deep_scrub_errors\":0,\"num_objects_recovered\":0,\"num_bytes_recovered\":0,\"num_keys_recovered\":0,\"num_objects_omap\":0,\"num_objects_hit_set_archive\":0,\"num_bytes_hit_set_archive\":0,\"num_flush\":0,\"num_flush_kb\":0,\"num_evict\":0,\"num_evict_kb\":0,\"num_promote\":0,\"num_flush_mode_high\":0,\"num_flush_mode_low\":0,\"num_evict_mode_some\":0,\"num_evict_mode_full\":0,\"num_objects_pinned\":0,\"num_legacy_snapsets\":0,\"num_large_omap_objects\":0,\"num_objects_manifest\":0},\"up\":[0],\"acting\":[0],\"blocked_by\":[],\"up_primary\":0,\"acting_primary\":0,\"purged_snaps\":[] }]}"; TEST(formatter, bug_37706) { vector pgs; - string outstring = -"{\"pg_ready\":true, \"pg_stats\":[ { \"pgid\":\"1.0\", \"version\":\"16'56\",\"reported_seq\":\"62\",\"reported_epoch\":\"20\",\"state\":\"active+clean+inconsistent\",\"last_fresh\":\"2018-12-18 15:21:22.173804\",\"last_change\":\"2018-12-18 15:21:22.173804\",\"last_active\":\"2018-12-18 15:21:22.173804\",\"last_peered\":\"2018-12-18 15:21:22.173804\",\"last_clean\":\"2018-12-18 15:21:22.173804\",\"last_became_active\":\"2018-12-18 15:21:21.347685\",\"last_became_peered\":\"2018-12-18 15:21:21.347685\",\"last_unstale\":\"2018-12-18 15:21:22.173804\",\"last_undegraded\":\"2018-12-18 15:21:22.173804\",\"last_fullsized\":\"2018-12-18 15:21:22.173804\",\"mapping_epoch\":19,\"log_start\":\"0'0\",\"ondisk_log_start\":\"0'0\",\"created\":7,\"last_epoch_clean\":20,\"parent\":\"0.0\",\"parent_split_bits\":0,\"last_scrub\":\"16'56\",\"last_scrub_stamp\":\"2018-12-18 15:21:22.173684\",\"last_deep_scrub\":\"0'0\",\"last_deep_scrub_stamp\":\"2018-12-18 15:21:06.514438\",\"last_clean_scrub_stamp\":\"2018-12-18 15:21:06.514438\",\"log_size\":56,\"ondisk_log_size\":56,\"stats_invalid\":false,\"dirty_stats_invalid\":false,\"omap_stats_invalid\":false,\"hitset_stats_invalid\":false,\"hitset_bytes_stats_invalid\":false,\"pin_stats_invalid\":false,\"manifest_stats_invalid\":false,\"snaptrimq_len\":0,\"stat_sum\":{\"num_bytes\":24448,\"num_objects\":36,\"num_object_clones\":20,\"num_object_copies\":36,\"num_objects_missing_on_primary\":0,\"num_objects_missing\":0,\"num_objects_degraded\":0,\"num_objects_misplaced\":0,\"num_objects_unfound\":0,\"num_objects_dirty\":36,\"num_whiteouts\":3,\"num_read\":0,\"num_read_kb\":0,\"num_write\":36,\"num_write_kb\":50,\"num_scrub_errors\":20,\"num_shallow_scrub_errors\":20,\"num_deep_scrub_errors\":0,\"num_objects_recovered\":0,\"num_bytes_recovered\":0,\"num_keys_recovered\":0,\"num_objects_omap\":0,\"num_objects_hit_set_archive\":0,\"num_bytes_hit_set_archive\":0,\"num_flush\":0,\"num_flush_kb\":0,\"num_evict\":0,\"num_evict_kb\":0,\"num_promote\":0,\"num_flush_mode_high\":0,\"num_flush_mode_low\":0,\"num_evict_mode_some\":0,\"num_evict_mode_full\":0,\"num_objects_pinned\":0,\"num_legacy_snapsets\":0,\"num_large_omap_objects\":0,\"num_objects_manifest\":0},\"up\":[0],\"acting\":[0],\"blocked_by\":[],\"up_primary\":0,\"acting_primary\":0,\"purged_snaps\":[] }]}"; - - JSONParser parser; ASSERT_TRUE(parser.parse(outstring.c_str(), outstring.size())); @@ -128,3 +129,517 @@ TEST(formatter, dump_large_item) { EXPECT_TRUE(parser.parse(bl.c_str(), bl.length())); EXPECT_EQ(parser.find_obj("Location")->get_data(), full_url); } + +TEST(formatter, parse_types) { + // Check that JSONParser::parse() works with expected data types + // (otherwise at least indirectly tested elsewhere): + + const auto json_input = R"({"expiration": "2025-01-17T10:26:46Z", "conditions": [{"bucket": "user-hqfadib9zxyj2pygevewzf45t-1"}, ["starts-with", "$key", "foo"], {"acl": "private"}, ["starts-with", "$Content-Type", "text/plain"], ["content-length-range", 0, 1024]]})"; + + { + JSONParser parser; + + ASSERT_TRUE(parser.parse(outstring)); + + JSONObj *pgstat_obj = parser.find_obj("pg_stats"); + EXPECT_TRUE(pgstat_obj); + } + + { + JSONParser parser; + + ASSERT_TRUE(parser.parse(std::string_view { outstring })); + + JSONObj *pgstat_obj = parser.find_obj("pg_stats"); + EXPECT_TRUE(pgstat_obj); + } + + { + JSONParser parser; + + buffer::list bl; + bl.append(outstring); + EXPECT_TRUE(parser.parse(bl.c_str(), bl.length())); + + JSONObj *pgstat_obj = parser.find_obj("pg_stats"); + EXPECT_TRUE(pgstat_obj); + } + + { + JSONParser parser; + + buffer::list bl; + bl.append(json_input); + + ASSERT_TRUE(parser.parse(bl.c_str(), bl.length())); + + JSONObjIter oi = parser.find_first(); + + JSONObj *o0 = *oi; + ASSERT_TRUE(nullptr != o0); + + ++oi; + JSONObj *o1 = *oi; + ASSERT_TRUE(nullptr != o1); + JSONObj *conditions = parser.find_obj("conditions"); + ASSERT_TRUE(nullptr != conditions); + + JSONObj *expiration = parser.find_obj("expiration"); + ASSERT_TRUE(nullptr != expiration); + } +} + +/* JFW: both of these fail again, AS THEY SHOULD: + -- IMO we need to fix any b0rken feeds + -- but, let's re-run this in the test suite so that we can get a comparison + before we patch things... +TEST(formatter, malformed_input_as_string) { + + // A couple of the common error cases I see now that we're using a more conformant JSON parser come + // from differences in what malformed input should be accepted as. The two most common cases are: + // 1) parsing fails at first character, a non-delimited field meant to be a string; + // 2) parsing fails just before the end, extra NULL byte + // ...ceph_json should honor these gnarly inputs, at least for now. + + const string_view bare_url = "http://turkey.straw.com"; + const char garbage_at_end[] = { 's', 't', 'r', '\x00', '\x00' }; + + JFW: tmp + { + JSONParser parser; + ASSERT_TRUE(parser.parse(bare_url)); + } + + { + JSONParser parser; + ASSERT_TRUE(parser.parse(garbage_at_end)); + } +}*/ + +namespace { + +void assert_parse(JSONParser& parser, std::string_view json) +{ + ASSERT_TRUE(parser.parse(json.data(), json.size())) << json; +} + +JSONObj* assert_find(JSONObj& obj, const char *name) +{ + auto *child = obj.find_obj(name); + EXPECT_NE(nullptr, child) << name; + return child; +} + +struct compat_record { + string name; + int count = 0; + bool enabled = false; + vector values; + map by_name; + + void dump(Formatter *f) const { + encode_json("name", name, f); + encode_json("count", count, f); + encode_json("enabled", enabled, f); + encode_json("values", values, f); + encode_json("by_name", by_name, f); + } + + void decode_json(JSONObj *obj) { + JSONDecoder::decode_json("name", name, obj, true); + JSONDecoder::decode_json("count", count, obj, true); + JSONDecoder::decode_json("enabled", enabled, obj, true); + JSONDecoder::decode_json("values", values, obj, true); + JSONDecoder::decode_json("by_name", by_name, obj, true); + } +}; + +} // anonymous namespace + +TEST(formatter, ceph_json_scalar_root_compatibility) +{ + struct scalar_case { + string_view input; + string_view data; + bool quoted; + }; + + const scalar_case cases[] = { + {R"("plain string")", "plain string", true}, + {"-42", "-42", false}, + {"42", "42", false}, + {"true", "true", false}, + {"false", "false", false}, + {"null", "null", false}, + }; + + for (const auto& c : cases) { + JSONParser parser; + assert_parse(parser, c.input); + + EXPECT_FALSE(parser.is_array()) << c.input; + EXPECT_FALSE(parser.is_object()) << c.input; + EXPECT_EQ(c.data, parser.get_data()) << c.input; + EXPECT_EQ(c.quoted, parser.get_data_val().quoted) << c.input; + } +} + +TEST(formatter, ceph_json_native_handle_access) +{ + static_assert(std::same_as); + + constexpr string_view json = R"({"name":"fast","values":[1,2,3]})"; + + JSONParser parser; + assert_parse(parser, json); + + auto& native = parser.native_handle(); + ASSERT_TRUE(native.is_object()); + + const auto& const_parser = parser; + static_assert(std::same_as); + EXPECT_TRUE(const_parser.native_handle().is_object()); + + auto *name = assert_find(parser, "name"); + const auto& native_name = name->native_handle(); + ASSERT_TRUE(native_name.is_string()); + EXPECT_EQ("fast", native_name.as()); + + auto *values_obj = assert_find(parser, "values"); + const auto values = ceph_json::array_view(values_obj->native_handle()); + EXPECT_EQ(3u, values.size()); +} + +TEST(formatter, ceph_json_native_field_helpers) +{ + constexpr string_view json = + R"({"name":"fast","enabled":true,"nested":{"secret":"open"},"values":["a","b"]})"; + + ceph_json::value root; + ASSERT_TRUE(ceph_json::parse(json, root)); + const auto parsed_root = ceph_json::parse(json); + + string name; + ASSERT_TRUE(ceph_json::decode_field("name", name, root)); + EXPECT_EQ("fast", name); + EXPECT_EQ("fast", ceph_json::require(parsed_root, "name")); + + bool enabled = false; + ceph_json::decode_required_field("enabled", enabled, root); + EXPECT_TRUE(enabled); + EXPECT_TRUE(ceph_json::require(parsed_root, "enabled")); + + const auto& members = ceph_json::require_object(parsed_root); + EXPECT_EQ(4u, std::ranges::size(members)); + + auto* nested = ceph_json::find_value(root, "nested"); + ASSERT_NE(nullptr, nested); + + string secret; + ceph_json::decode_required_field("secret", secret, *nested); + EXPECT_EQ("open", secret); + EXPECT_EQ("open", ceph_json::require( + ceph_json::require_object(parsed_root, "nested"), "secret")); + + vector values; + ceph_json::decode_required_field("values", values, root); + ASSERT_EQ(2u, values.size()); + EXPECT_EQ("a", values[0]); + EXPECT_EQ("b", values[1]); + EXPECT_EQ(values, ceph_json::require>(parsed_root, "values")); + + const auto value_nodes = ceph_json::require_array(parsed_root, "values"); + ASSERT_EQ(2u, value_nodes.size()); + EXPECT_EQ("a", ceph_json::decode_value(value_nodes.front())); + EXPECT_EQ("b", ceph_json::decode_value(value_nodes.back())); + + vector viewed_values; + std::ranges::transform( + value_nodes, back_inserter(viewed_values), + [] (const auto& item) { + return ceph_json::decode_value(item); + }); + EXPECT_EQ(values, viewed_values); + + int missing = 7; + EXPECT_FALSE(ceph_json::decode_field("missing", missing, root)); + EXPECT_EQ(0, missing); + EXPECT_EQ(9, ceph_json::field_or(parsed_root, "missing", 9)); + EXPECT_EQ(nullopt, ceph_json::field(parsed_root, "missing")); + + EXPECT_THROW(ceph_json::decode_required_field("missing", missing, root), + JSONDecoder::err); + EXPECT_THROW(ceph_json::require(parsed_root, "missing"), + JSONDecoder::err); + EXPECT_THROW(ceph_json::require_object(parsed_root, "name"), + JSONDecoder::err); + EXPECT_THROW(ceph_json::require_array(parsed_root, "name"), + JSONDecoder::err); + EXPECT_THROW(ceph_json::require_array(parsed_root, "missing"), + JSONDecoder::err); + EXPECT_EQ(0u, ceph_json::array_view(parsed_root).size()); + + bufferlist bl; + bl.append(json); + bl.append("\0", 1); + + ceph_json::value from_bl; + ASSERT_TRUE(ceph_json::parse(bl, from_bl)); + + string bl_name; + ceph_json::decode_required_field("name", bl_name, from_bl); + EXPECT_EQ("fast", bl_name); + EXPECT_EQ("fast", ceph_json::require(ceph_json::parse(bl), "name")); +} + +TEST(formatter, ceph_json_object_tree_compatibility) +{ + constexpr string_view json = + R"({"name":"main","nested":{"n":7,"flag":false},)" + R"("tricky":"{not: [json], comma, quote \" }","escaped\"key":"ok",)" + R"("array":[1,"two",{"three":3},null],"empty":[]})"; + + JSONParser parser; + assert_parse(parser, json); + + ASSERT_TRUE(parser.is_object()); + + auto *name = assert_find(parser, "name"); + ASSERT_NE(nullptr, name); + EXPECT_EQ("main", name->get_data()); + EXPECT_TRUE(name->get_data_val().quoted); + + auto *tricky = assert_find(parser, "tricky"); + ASSERT_NE(nullptr, tricky); + EXPECT_EQ(R"({not: [json], comma, quote " })", tricky->get_data()); + + auto *escaped_key = assert_find(parser, R"(escaped"key)"); + ASSERT_NE(nullptr, escaped_key); + EXPECT_EQ("ok", escaped_key->get_data()); + + JSONObj::data_val name_attr; + ASSERT_TRUE(name->get_attr("name", name_attr)); + EXPECT_EQ("main", name_attr.str); + EXPECT_TRUE(name_attr.quoted); + + auto *nested = assert_find(parser, "nested"); + ASSERT_NE(nullptr, nested); + EXPECT_TRUE(nested->is_object()); + + int n = 0; + ASSERT_TRUE(JSONDecoder::decode_json("n", n, nested, true)); + EXPECT_EQ(7, n); + + bool flag = true; + ASSERT_TRUE(JSONDecoder::decode_json("flag", flag, nested, true)); + EXPECT_FALSE(flag); + + auto *missing = parser.find_obj("missing"); + EXPECT_EQ(nullptr, missing); +} + +TEST(formatter, ceph_json_named_find_first_stops_after_named_child) +{ + constexpr string_view json = R"({"alpha":1,"needle":"found","z_tail":2})"; + + JSONParser parser; + assert_parse(parser, json); + + auto iter = parser.find_first("needle"); + ASSERT_FALSE(iter.end()); + EXPECT_EQ("needle", (*iter)->get_name()); + EXPECT_EQ("found", (*iter)->get_data()); + + ++iter; + EXPECT_TRUE(iter.end()); +} + +TEST(formatter, ceph_json_duplicate_object_members_known_regression) +{ + /* + * The legacy JSONObj tree could preserve repeated object members with the + * same name. JSON itself only says object names SHOULD be unique, and an + * audit of Ceph and src/tools users found no semantic dependency on this: + * ordinary decoders use the first named field, map-like decoders expect + * explicit {"key", "val"} entries, and object iteration is used for arrays + * or generic display. We intentionally follow Glaze's object model here. + */ + constexpr string_view json = + R"({"dupe":"first","dupe":"second",)" + R"("nested":{"again":"first","again":"second"},)" + R"("array":[{"three":3,"three":4}]})"; + + JSONParser parser; + assert_parse(parser, json); + + auto dupes = parser.find("dupe"); + vector values; + for (; !dupes.end(); ++dupes) { + values.push_back((*dupes)->get_data()); + } + + ASSERT_EQ(1u, values.size()); + EXPECT_TRUE(values[0] == "first" || values[0] == "second"); + + auto *nested = assert_find(parser, "nested"); + auto again = nested->find("again"); + vector nested_values; + for (; !again.end(); ++again) { + nested_values.push_back((*again)->get_data()); + } + + ASSERT_EQ(1u, nested_values.size()); + EXPECT_TRUE(nested_values[0] == "first" || nested_values[0] == "second"); + + auto *array = assert_find(parser, "array"); + const auto elements = array->get_array_elements(); + ASSERT_EQ(1u, elements.size()); + + JSONParser object_element; + assert_parse(object_element, elements[0]); + + auto threes = object_element.find("three"); + vector object_values; + for (; !threes.end(); ++threes) { + object_values.push_back((*threes)->get_data()); + } + + ASSERT_EQ(1u, object_values.size()); + EXPECT_TRUE(object_values[0] == "3" || object_values[0] == "4"); +} + +TEST(formatter, ceph_json_array_elements_compatibility) +{ + constexpr string_view json = + R"({"array":[1,"two",{"three":3},[4,5],null],"empty":[]})"; + + JSONParser parser; + assert_parse(parser, json); + + auto *array = assert_find(parser, "array"); + ASSERT_NE(nullptr, array); + ASSERT_TRUE(array->is_array()); + + const auto elements = array->get_array_elements(); + ASSERT_EQ(5u, elements.size()); + EXPECT_EQ("1", elements[0]); + EXPECT_EQ(R"("two")", elements[1]); + EXPECT_EQ("null", elements[4]); + + JSONParser object_element; + assert_parse(object_element, elements[2]); + int three = 0; + ASSERT_TRUE(JSONDecoder::decode_json("three", three, &object_element, true)); + EXPECT_EQ(3, three); + + JSONParser array_element; + assert_parse(array_element, elements[3]); + vector nested; + decode_json_obj(nested, &array_element); + ASSERT_EQ(2u, nested.size()); + EXPECT_EQ(4, nested[0]); + EXPECT_EQ(5, nested[1]); + + auto *empty = assert_find(parser, "empty"); + ASSERT_NE(nullptr, empty); + EXPECT_TRUE(empty->get_array_elements().empty()); +} + +TEST(formatter, ceph_json_decoder_compatibility) +{ + constexpr string_view json = + R"({"s":"text","i":-42,"u":42,"b_true":true,"b_false":false,)" + R"("b_num":1,"values":[1,2,3],)" + R"("by_name":[{"key":"alpha","val":10},{"key":"beta","val":20}],)" + R"("maybe":99})"; + + JSONParser parser; + assert_parse(parser, json); + + string s; + ASSERT_TRUE(JSONDecoder::decode_json("s", s, &parser, true)); + EXPECT_EQ("text", s); + + int i = 0; + ASSERT_TRUE(JSONDecoder::decode_json("i", i, &parser, true)); + EXPECT_EQ(-42, i); + + unsigned u = 0; + ASSERT_TRUE(JSONDecoder::decode_json("u", u, &parser, true)); + EXPECT_EQ(42u, u); + + bool b = false; + ASSERT_TRUE(JSONDecoder::decode_json("b_true", b, &parser, true)); + EXPECT_TRUE(b); + + b = true; + ASSERT_TRUE(JSONDecoder::decode_json("b_false", b, &parser, true)); + EXPECT_FALSE(b); + + b = false; + ASSERT_TRUE(JSONDecoder::decode_json("b_num", b, &parser, true)); + EXPECT_TRUE(b); + + vector values; + ASSERT_TRUE(JSONDecoder::decode_json("values", values, &parser, true)); + ASSERT_EQ(3u, values.size()); + EXPECT_EQ(1, values[0]); + EXPECT_EQ(2, values[1]); + EXPECT_EQ(3, values[2]); + + map by_name; + ASSERT_TRUE(JSONDecoder::decode_json("by_name", by_name, &parser, true)); + ASSERT_EQ(2u, by_name.size()); + EXPECT_EQ(10, by_name["alpha"]); + EXPECT_EQ(20, by_name["beta"]); + + boost::optional maybe; + ASSERT_TRUE(JSONDecoder::decode_json("maybe", maybe, &parser, true)); + ASSERT_TRUE(maybe); + EXPECT_EQ(99, *maybe); + + int missing = 123; + EXPECT_FALSE(JSONDecoder::decode_json("missing", missing, &parser)); + EXPECT_EQ(0, missing); + + boost::optional missing_optional = 7; + EXPECT_FALSE(JSONDecoder::decode_json("missing_optional", missing_optional, &parser)); + EXPECT_FALSE(missing_optional); + + EXPECT_THROW(JSONDecoder::decode_json("missing_required", missing, &parser, true), + JSONDecoder::err); +} + +TEST(formatter, ceph_json_encode_decode_compatibility) +{ + compat_record input; + input.name = "encoded"; + input.count = 17; + input.enabled = true; + input.values = {3, 5, 8}; + input.by_name = {{"first", 1}, {"second", 2}}; + + JSONFormatter formatter; + formatter.open_object_section("root"); + encode_json("record", input, &formatter); + formatter.close_section(); + + stringstream ss; + formatter.flush(ss); + + JSONParser parser; + assert_parse(parser, ss.str()); + + auto *record_obj = assert_find(parser, "record"); + ASSERT_NE(nullptr, record_obj); + + compat_record output; + decode_json_obj(output, record_obj); + + EXPECT_EQ(input.name, output.name); + EXPECT_EQ(input.count, output.count); + EXPECT_EQ(input.enabled, output.enabled); + EXPECT_EQ(input.values, output.values); + EXPECT_EQ(input.by_name, output.by_name); +} diff --git a/src/test/objectstore/allocator_replay_test.cc b/src/test/objectstore/allocator_replay_test.cc index ddab87709de..9c62d101eeb 100644 --- a/src/test/objectstore/allocator_replay_test.cc +++ b/src/test/objectstore/allocator_replay_test.cc @@ -274,7 +274,7 @@ int replay_free_dump_and_apply_raw( JSONParser p; std::cout << "parsing..." << std::endl; - bool b = p.parse(fname); + bool b = p.parse_file(fname); if (!b) { std::cerr << "Failed to parse json: " << fname << std::endl; return -1; diff --git a/src/test/rgw/rgw_multi/multisite.py b/src/test/rgw/rgw_multi/multisite.py index 565caa83058..9d7e7432066 100644 --- a/src/test/rgw/rgw_multi/multisite.py +++ b/src/test/rgw/rgw_multi/multisite.py @@ -1,6 +1,7 @@ from abc import ABCMeta, abstractmethod from io import StringIO +import sys import json from .conn import get_gateway_connection, get_gateway_s3_resource, get_gateway_iam_connection, get_gateway_secure_connection, get_gateway_sns_client, get_gateway_sts_connection, get_gateway_temp_s3_client @@ -82,6 +83,7 @@ class SystemObject: s, r = self.command(cluster, cmd, args or [], **kwargs) if r == 0: data = json.loads(s) + # To show the entire command: json.dump(data, sys.stdout) self.load_from_json(data) self.data = data return self.data, r diff --git a/src/tools/cephfs/MetaTool.cc b/src/tools/cephfs/MetaTool.cc index 2081bf62020..77245cb8f30 100644 --- a/src/tools/cephfs/MetaTool.cc +++ b/src/tools/cephfs/MetaTool.cc @@ -354,7 +354,7 @@ void MetaTool::inode_meta_t::encode(::ceph::bufferlist& bl, uint64_t features) int MetaTool::_amend_meta(string& k, inode_meta_t& inode_meta, const string& fn, meta_op& op) { JSONParser parser; - if (!parser.parse(fn.c_str())) { + if (!parser.parse(fn)) { cout << "Error parsing create user response" << std::endl; return -1; } @@ -479,7 +479,7 @@ int MetaTool::amend_fn(meta_op &op) int MetaTool::_amend_fn(const string& fn, bool confirm) { JSONParser parser; - if (!parser.parse(fn.c_str())) { + if (!parser.parse(fn)) { cout << "Error parsing create user response : " << fn << std::endl; return -1; } diff --git a/src/tools/cephfs_mirror/InstanceWatcher.cc b/src/tools/cephfs_mirror/InstanceWatcher.cc index b8be5762497..75f1bd933ac 100644 --- a/src/tools/cephfs_mirror/InstanceWatcher.cc +++ b/src/tools/cephfs_mirror/InstanceWatcher.cc @@ -89,10 +89,10 @@ void InstanceWatcher::handle_notify(uint64_t notify_id, uint64_t handle, std::string mode; bool purging = false; try { - JSONDecoder jd(bl); - JSONDecoder::decode_json("dir_path", dir_path, &jd.parser, true); - JSONDecoder::decode_json("mode", mode, &jd.parser, true); - JSONDecoder::decode_json("purging", purging, &jd.parser, false); + const auto notify = ceph_json::parse(bl); + dir_path = ceph_json::require(notify, "dir_path"); + mode = ceph_json::require(notify, "mode"); + purging = ceph_json::field_or(notify, "purging", false); } catch (const JSONDecoder::err &e) { derr << ": failed to decode notify json: " << e.what() << dendl; } diff --git a/src/tools/rbd/action/Journal.cc b/src/tools/rbd/action/Journal.cc index f1bdd6b5289..a915ef4e7d1 100644 --- a/src/tools/rbd/action/Journal.cc +++ b/src/tools/rbd/action/Journal.cc @@ -840,7 +840,7 @@ public: n++; error_count++; JSONParser p; - if (!p.parse(bl.c_str(), bl.length())) { + if (!p.parse(bl)) { std::cerr << "rbd: error parsing input (entry " << n << ")" << std::endl; r = -EINVAL;