From 3a439bdbe933a3e017655f3b3c41b8fa5cbb1aee Mon Sep 17 00:00:00 2001 From: Csaba Imre Zempleni Date: Tue, 19 Dec 2023 14:45:00 +0100 Subject: [PATCH] Adding verbose error messages for logical combinations --- src/json-validator.cpp | 54 ++- test/CMakeLists.txt | 4 + test/issue-105-verbose-combination-errors.cpp | 338 ++++++++++++++++++ 3 files changed, 384 insertions(+), 12 deletions(-) create mode 100644 test/issue-105-verbose-combination-errors.cpp diff --git a/src/json-validator.cpp b/src/json-validator.cpp index 8b637a8..2a353fa 100644 --- a/src/json-validator.cpp +++ b/src/json-validator.cpp @@ -426,6 +426,32 @@ enum logical_combination_types { oneOf }; +class logical_combination_error_handler : public error_handler +{ +public: + struct error_entry + { + json::json_pointer ptr_; + json instance_; + std::string message_; + }; + + std::vector error_entry_list_; + + void error(const json::json_pointer &ptr, const json &instance, const std::string &message) override + { + error_entry_list_.push_back(error_entry{ ptr, instance, message }); + } + + void propagate(error_handler& e, const std::string& prefix) const + { + for (const error_entry& entry : error_entry_list_) + e.error(entry.ptr_, entry.instance_, prefix + entry.message_); + } + + operator bool() const { return !error_entry_list_.empty(); } +}; + template class logical_combination : public schema { @@ -434,29 +460,33 @@ class logical_combination : public schema void validate(const json::json_pointer &ptr, const json &instance, json_patch &patch, error_handler &e) const final { size_t count = 0; + logical_combination_error_handler error_summary; - for (auto &s : subschemata_) { - first_error_handler esub; + for (std::size_t index = 0; index < subschemata_.size(); ++index) { + const std::shared_ptr& s = subschemata_[index]; + logical_combination_error_handler esub; auto oldPatchSize = patch.get_json().size(); s->validate(ptr, instance, patch, esub); if (!esub) count++; - else + else { patch.get_json().get_ref().resize(oldPatchSize); + esub.propagate(error_summary, "case#" + std::to_string(index) + "] "); + } if (is_validate_complete(instance, ptr, e, esub, count)) return; } - // could accumulate esub details for anyOf and oneOf, but not clear how to select which subschema failure to report - // or how to report multiple such failures - if (count == 0) - e.error(ptr, instance, "no subschema has succeeded, but one of them is required to validate"); + if (count == 0) { + e.error(ptr, instance, "no subschema has succeeded, but one of them is required to validate. Type: " + key + ", number of failed subschemas: " + std::to_string(subschemata_.size())); + error_summary.propagate(e, "[combination: " + key + " / "); + } } // specialized for each of the logical_combination_types static const std::string key; - static bool is_validate_complete(const json &, const json::json_pointer &, error_handler &, const first_error_handler &, size_t); + static bool is_validate_complete(const json &, const json::json_pointer &, error_handler &, const logical_combination_error_handler &, size_t); public: logical_combination(json &sch, @@ -481,21 +511,21 @@ template <> const std::string logical_combination::key = "oneOf"; template <> -bool logical_combination::is_validate_complete(const json &, const json::json_pointer &, error_handler &e, const first_error_handler &esub, size_t) +bool logical_combination::is_validate_complete(const json &, const json::json_pointer &, error_handler &e, const logical_combination_error_handler &esub, size_t) { if (esub) - e.error(esub.ptr_, esub.instance_, "at least one subschema has failed, but all of them are required to validate - " + esub.message_); + e.error(esub.error_entry_list_.front().ptr_, esub.error_entry_list_.front().instance_, "at least one subschema has failed, but all of them are required to validate - " + esub.error_entry_list_.front().message_); return esub; } template <> -bool logical_combination::is_validate_complete(const json &, const json::json_pointer &, error_handler &, const first_error_handler &, size_t count) +bool logical_combination::is_validate_complete(const json &, const json::json_pointer &, error_handler &, const logical_combination_error_handler &, size_t count) { return count == 1; } template <> -bool logical_combination::is_validate_complete(const json &instance, const json::json_pointer &ptr, error_handler &e, const first_error_handler &, size_t count) +bool logical_combination::is_validate_complete(const json &instance, const json::json_pointer &ptr, error_handler &e, const logical_combination_error_handler &, size_t count) { if (count > 1) e.error(ptr, instance, "more than one subschema has succeeded, but exactly one of them is required to validate"); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 50ba0e6..51eec32 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -89,3 +89,7 @@ add_test(NAME issue-243-root-default-values COMMAND issue-243-root-default-value add_executable(issue-255-error-message-limit-precision issue-255-error-message-limit-precision.cpp) target_link_libraries(issue-255-error-message-limit-precision nlohmann_json_schema_validator) add_test(NAME issue-255-error-message-limit-precision COMMAND issue-255-error-message-limit-precision) + +add_executable(issue-105-verbose-combination-errors issue-105-verbose-combination-errors.cpp) +target_link_libraries(issue-105-verbose-combination-errors nlohmann_json_schema_validator) +add_test(NAME issue-105-verbose-combination-errors COMMAND issue-105-verbose-combination-errors) diff --git a/test/issue-105-verbose-combination-errors.cpp b/test/issue-105-verbose-combination-errors.cpp new file mode 100644 index 0000000..59f8492 --- /dev/null +++ b/test/issue-105-verbose-combination-errors.cpp @@ -0,0 +1,338 @@ +#include "nlohmann/json-schema.hpp" +#include "nlohmann/json.hpp" + +#include +#include +#include +#include + +//============================================================================== +// Test macros +//============================================================================== +#define LOG_ERROR(LOG_ERROR__ARGS) \ + std::cerr << __FILE__ << ":" << __LINE__ << ": " << LOG_ERROR__ARGS << std::endl + +#define EXPECT_THROW_WITH_MESSAGE(EXPRESSION, MESSAGE) \ + do \ + { \ + try \ + { \ + EXPRESSION; \ + LOG_ERROR("Expected exception not thrown with matching regex: \"" << MESSAGE << "\""); \ + ++g_error_count; \ + } catch (const std::exception& error) \ + { \ + const std::regex error_re{ MESSAGE }; \ + if (!std::regex_search(error.what(), error_re)) \ + { \ + LOG_ERROR("Expected exception with matching regex: \"" << MESSAGE << "\", but got this instead: " << error.what()); \ + ++g_error_count; \ + } \ + } \ + } while (false) + +#define ASSERT_OR_EXPECT_EQ(FIRST_THING, SECOND_THING, RETURN_IN_CASE_OF_ERROR) \ + do \ + { \ + if ((FIRST_THING) != (SECOND_THING)) \ + { \ + LOG_ERROR("The two values of " << (FIRST_THING) << " (" #FIRST_THING << ") and " << (SECOND_THING) << " (" #SECOND_THING << ") should be equal"); \ + if (RETURN_IN_CASE_OF_ERROR) \ + { \ + return; \ + } \ + } \ + } \ + while(false) + +#define ASSERT_EQ(FIRST_THING, SECOND_THING) ASSERT_OR_EXPECT_EQ(FIRST_THING, SECOND_THING, true) +#define EXPECT_EQ(FIRST_THING, SECOND_THING) ASSERT_OR_EXPECT_EQ(FIRST_THING, SECOND_THING, true) + +#define EXPECT_MATCH(STRING, REGEX) \ + do \ + { \ + if (!std::regex_search((STRING), std::regex{ (REGEX) })) \ + { \ + LOG_ERROR("String \"" << (STRING) << "\" doesn't match with regex: \"" << (REGEX) << "\""); \ + ++g_error_count; \ + } \ + } \ + while(false) + +namespace +{ + +//============================================================================== +// Test environment +//============================================================================== +int g_error_count = 0; + +//============================================================================== +// The schema used for testing +//============================================================================== +const std::string g_schema_template = R"( +{ + "properties": { + "first": { + "%COMBINATION_FIRST_LEVEL%": [ + { + "properties": { + "second": { + "%COMBINATION_SECOND_LEVEL%": [ + { + "minimum": 5, + "type": "integer" + }, + { + "multipleOf": 2, + "type": "integer" + } + ] + } + }, + "type": "object" + }, + { + "minimum": 20, + "type": "integer" + }, + { + "minLength": 10, + "type": "string" + } + ] + } + }, + "type": "object" +} +)"; + +auto generateSchema(const std::string& first_combination, const std::string& second_combination) -> nlohmann::json +{ + static const std::regex first_replace_re{"%COMBINATION_FIRST_LEVEL%"}; + static const std::regex second_replace_re{"%COMBINATION_SECOND_LEVEL%"}; + + std::string intermediate = std::regex_replace(g_schema_template, first_replace_re, first_combination); + + return nlohmann::json::parse(std::regex_replace(intermediate, second_replace_re, second_combination)); +} + +//============================================================================== +// Error handler to catch all the errors generated by the validator - also inside the combinations +//============================================================================== +class MyErrorHandler : public nlohmann::json_schema::error_handler +{ +public: + struct ErrorEntry + { + nlohmann::json::json_pointer ptr; + nlohmann::json intance; + std::string message; + }; + + using ErrorEntryList = std::vector; + + auto getErrors() const -> const ErrorEntryList& + { + return m_error_list; + } + +private: + auto error(const nlohmann::json::json_pointer& ptr, const nlohmann::json& instance, const std::string& message) -> void override + { + m_error_list.push_back(ErrorEntry{ptr, instance, message}); + } + + ErrorEntryList m_error_list; +}; + +//============================================================================== +// Error string helpers +//============================================================================== +auto operator<<(std::string first, const std::string& second) -> std::string +{ + first += ".*"; + first += second; + return first; +} + +auto rootError(const std::string& combination_type, std::size_t number_of_subschemas) -> std::string +{ + return "no subschema has succeeded, but one of them is required to validate. Type: " + combination_type + ", number of failed subschemas: " + std::to_string(number_of_subschemas); +} + +auto combinationError(const std::string& combination_type, std::size_t test_case_number) -> std::string +{ + return "[combination: " + combination_type + " / case#" + std::to_string(test_case_number) + "]"; +} + +//============================================================================== +// Validator function - for simplicity +//============================================================================== +auto validate(const nlohmann::json& schema, const nlohmann::json& instance, nlohmann::json_schema::error_handler* error_handler = nullptr) -> void +{ + nlohmann::json_schema::json_validator validator; + validator.set_root_schema(schema); + + if (error_handler) + { + validator.validate(instance, *error_handler); + } + else + { + validator.validate(instance); + } +} + +//============================================================================== +// The test cases +//============================================================================== +auto simpleTest(const std::string& first_combination, const std::string& second_combination) -> void +{ + const nlohmann::json schema = generateSchema(first_combination, second_combination); + EXPECT_THROW_WITH_MESSAGE(validate(schema, nlohmann::json{ { "first", { { "second", 1 } } } }), rootError(first_combination, 3)); + if (second_combination == "oneOf") + { + EXPECT_THROW_WITH_MESSAGE(validate(schema, nlohmann::json{ { "first", { { "second", 8 } } } }), rootError(first_combination, 3)); + } + EXPECT_THROW_WITH_MESSAGE(validate(schema, nlohmann::json{ { "first", 10 } }), rootError(first_combination, 3)); + EXPECT_THROW_WITH_MESSAGE(validate(schema, nlohmann::json{ { "first", "short" } }), rootError(first_combination, 3)); +} + +auto verboseTest(const std::string& first_combination, const std::string& second_combination) -> void +{ + const nlohmann::json schema = generateSchema(first_combination, second_combination); + + { + MyErrorHandler error_handler; + validate(schema, nlohmann::json{ { "first", { { "second", 1 } } } }, &error_handler); + + const MyErrorHandler::ErrorEntryList& error_list = error_handler.getErrors(); + EXPECT_EQ(error_list.size(), 6); + + EXPECT_EQ(error_list[0].ptr, nlohmann::json::json_pointer{ "/first" }); + EXPECT_MATCH(error_list[0].message, rootError(first_combination, 3)); + + EXPECT_EQ(error_list[1].ptr, nlohmann::json::json_pointer{ "/first/second" }); + EXPECT_MATCH(error_list[1].message, combinationError(first_combination, 0) << rootError(second_combination, 2)); + + EXPECT_EQ(error_list[2].ptr, nlohmann::json::json_pointer{ "/first/second" }); + EXPECT_MATCH(error_list[2].message, combinationError(first_combination, 0) << combinationError(second_combination, 0) << "instance is below minimum of 5"); + + EXPECT_EQ(error_list[3].ptr, nlohmann::json::json_pointer{ "/first/second" }); + EXPECT_MATCH(error_list[3].message, combinationError(first_combination, 0) << combinationError(second_combination, 1) << "instance is not a multiple of 2.0"); + + EXPECT_EQ(error_list[4].ptr, nlohmann::json::json_pointer{ "/first" }); + EXPECT_MATCH(error_list[4].message, combinationError(first_combination, 1) << "unexpected instance type"); + + EXPECT_EQ(error_list[5].ptr, nlohmann::json::json_pointer{ "/first" }); + EXPECT_MATCH(error_list[5].message, combinationError(first_combination, 2) << "unexpected instance type"); + } + + { + MyErrorHandler error_handler; + validate(schema, nlohmann::json{ { "first", { { "second", "not-an-integer" } } } }, &error_handler); + + const MyErrorHandler::ErrorEntryList& error_list = error_handler.getErrors(); + EXPECT_EQ(error_list.size(), 6); + + EXPECT_EQ(error_list[0].ptr, nlohmann::json::json_pointer{ "/first" }); + EXPECT_MATCH(error_list[0].message, rootError(first_combination, 3)); + + EXPECT_EQ(error_list[1].ptr, nlohmann::json::json_pointer{ "/first/second" }); + EXPECT_MATCH(error_list[1].message, combinationError(first_combination, 0) << rootError(second_combination, 2)); + + EXPECT_EQ(error_list[2].ptr, nlohmann::json::json_pointer{ "/first/second" }); + EXPECT_MATCH(error_list[2].message, combinationError(first_combination, 0) << combinationError(second_combination, 0) << "unexpected instance type"); + + EXPECT_EQ(error_list[3].ptr, nlohmann::json::json_pointer{ "/first/second" }); + EXPECT_MATCH(error_list[3].message, combinationError(first_combination, 0) << combinationError(second_combination, 1) << "unexpected instance type"); + + EXPECT_EQ(error_list[4].ptr, nlohmann::json::json_pointer{ "/first" }); + EXPECT_MATCH(error_list[4].message, combinationError(first_combination, 1) << "unexpected instance type"); + + EXPECT_EQ(error_list[5].ptr, nlohmann::json::json_pointer{ "/first" }); + EXPECT_MATCH(error_list[5].message, combinationError(first_combination, 2) << "unexpected instance type"); + } + + if (second_combination == "oneOf") + { + MyErrorHandler error_handler; + validate(schema, nlohmann::json{ { "first", { { "second", 8 } } } }, &error_handler); + + const MyErrorHandler::ErrorEntryList& error_list = error_handler.getErrors(); + EXPECT_EQ(error_list.size(), 4); + + EXPECT_EQ(error_list[0].ptr, nlohmann::json::json_pointer{ "/first" }); + EXPECT_MATCH(error_list[0].message, rootError(first_combination, 3)); + + EXPECT_EQ(error_list[1].ptr, nlohmann::json::json_pointer{ "/first/second" }); + EXPECT_MATCH(error_list[1].message, combinationError(first_combination, 0) << "more than one subschema has succeeded, but exactly one of them is required to validate"); + + EXPECT_EQ(error_list[2].ptr, nlohmann::json::json_pointer{ "/first" }); + EXPECT_MATCH(error_list[2].message, combinationError(first_combination, 1) << "unexpected instance type"); + + EXPECT_EQ(error_list[3].ptr, nlohmann::json::json_pointer{ "/first" }); + EXPECT_MATCH(error_list[3].message, combinationError(first_combination, 2) << "unexpected instance type"); + } + + { + MyErrorHandler error_handler; + validate(schema, nlohmann::json{ { "first", 10 } }, &error_handler); + + const MyErrorHandler::ErrorEntryList& error_list = error_handler.getErrors(); + EXPECT_EQ(error_list.size(), 4); + + EXPECT_EQ(error_list[0].ptr, nlohmann::json::json_pointer{ "/first" }); + EXPECT_MATCH(error_list[0].message, rootError(first_combination, 3)); + + EXPECT_EQ(error_list[1].ptr, nlohmann::json::json_pointer{ "/first" }); + EXPECT_MATCH(error_list[1].message, combinationError(first_combination, 0) << "unexpected instance type"); + + EXPECT_EQ(error_list[2].ptr, nlohmann::json::json_pointer{ "/first" }); + EXPECT_MATCH(error_list[2].message, combinationError(first_combination, 1) << "instance is below minimum of 20"); + + EXPECT_EQ(error_list[3].ptr, nlohmann::json::json_pointer{ "/first" }); + EXPECT_MATCH(error_list[3].message, combinationError(first_combination, 2) << "unexpected instance type"); + } + + { + MyErrorHandler error_handler; + validate(schema, nlohmann::json{ { "first", "short" } }, &error_handler); + + const MyErrorHandler::ErrorEntryList& error_list = error_handler.getErrors(); + EXPECT_EQ(error_list.size(), 4); + + EXPECT_EQ(error_list[0].ptr, nlohmann::json::json_pointer{ "/first" }); + EXPECT_MATCH(error_list[0].message, rootError(first_combination, 3)); + + EXPECT_EQ(error_list[1].ptr, nlohmann::json::json_pointer{ "/first" }); + EXPECT_MATCH(error_list[1].message, combinationError(first_combination, 0) << "unexpected instance type"); + + EXPECT_EQ(error_list[2].ptr, nlohmann::json::json_pointer{ "/first" }); + EXPECT_MATCH(error_list[2].message, combinationError(first_combination, 1) << "unexpected instance type"); + + EXPECT_EQ(error_list[3].ptr, nlohmann::json::json_pointer{ "/first" }); + EXPECT_MATCH(error_list[3].message, combinationError(first_combination, 2) << "instance is too short as per minLength:10"); + } +} + +} // namespace + +//============================================================================== +// MAIN - calling the test cases +//============================================================================== +auto main() -> int +{ + simpleTest("anyOf", "anyOf"); + simpleTest("anyOf", "oneOf"); + simpleTest("oneOf", "anyOf"); + simpleTest("oneOf", "oneOf"); + + verboseTest("anyOf", "anyOf"); + verboseTest("anyOf", "oneOf"); + verboseTest("oneOf", "anyOf"); + verboseTest("oneOf", "oneOf"); + + return g_error_count; +}