diff --git a/cpp/src/arrow/json/from_string.cc b/cpp/src/arrow/json/from_string.cc index c9d910667133..f573ae710651 100644 --- a/cpp/src/arrow/json/from_string.cc +++ b/cpp/src/arrow/json/from_string.cc @@ -106,7 +106,7 @@ class ConcreteConverter : public JSONConverter { int32_t num_elements = 0; for (auto element : json_array) { ARROW_ASSIGN_OR_RAISE(auto value, - internal::GetSimdjsonResult( + internal::ResolveSimdjsonResult( element, "Could not iterate elements of JSON array: ")); RETURN_NOT_OK(self->AppendValue(value)); num_elements++; @@ -287,7 +287,7 @@ Status ProcessJsonArrayElements( } ARROW_ASSIGN_OR_RAISE(sj::value element, - internal::GetSimdjsonResult( + internal::ResolveSimdjsonResult( *it, "Could not iterate elements of JSON array: ")); RETURN_NOT_OK(handler(element)); ++it; @@ -652,7 +652,7 @@ class MapConverter final : public ConcreteConverter { for (auto json_pair_result : array) { ARROW_ASSIGN_OR_RAISE( auto json_pair, - internal::GetSimdjsonResult( + internal::ResolveSimdjsonResult( json_pair_result, "Could not iterate elements of JSON array: ")); ARROW_ASSIGN_OR_RAISE(auto json_pair_array, internal::GetJsonAs(json_pair)); @@ -763,7 +763,7 @@ class StructConverter final : public ConcreteConverter { size_t i = 0; for (auto child : array) { ARROW_ASSIGN_OR_RAISE(auto child_value, - internal::GetSimdjsonResult( + internal::ResolveSimdjsonResult( child, "Could not iterate elements of JSON array: ")); RETURN_NOT_OK(child_converters_[i]->AppendValue(child_value)); ++i; @@ -779,7 +779,7 @@ class StructConverter final : public ConcreteConverter { std::vector field_seen(num_fields, false); for (auto field_result : object) { ARROW_ASSIGN_OR_RAISE(auto field, - internal::GetSimdjsonResult( + internal::ResolveSimdjsonResult( field_result, "Error getting field of object: ")); std::string_view key; if (field.unescaped_key(/*allow_replacement=*/false).get(key) != diff --git a/cpp/src/arrow/json/json_writer_internal.cc b/cpp/src/arrow/json/json_writer_internal.cc index 567694902497..6d3e9ff9d85d 100644 --- a/cpp/src/arrow/json/json_writer_internal.cc +++ b/cpp/src/arrow/json/json_writer_internal.cc @@ -108,14 +108,14 @@ Status JsonWriter::WriteValue(sj::value value) { for (auto field : object) { ARROW_ASSIGN_OR_RAISE( - auto key, internal::GetSimdjsonResult(field.unescaped_key(), - "Failed to get object key: ")); + auto key, internal::ResolveSimdjsonResult(field.unescaped_key(), + "Failed to get object key")); Key(key); - ARROW_ASSIGN_OR_RAISE( - auto field_value, - internal::GetSimdjsonResult(field.value(), "Failed to get object value: ")); + ARROW_ASSIGN_OR_RAISE(auto field_value, + internal::ResolveSimdjsonResult( + field.value(), "Failed to get object value")); RETURN_NOT_OK(WriteValue(field_value)); } @@ -130,7 +130,7 @@ Status JsonWriter::WriteValue(sj::value value) { for (auto element : array) { ARROW_ASSIGN_OR_RAISE( auto element_value, - internal::GetSimdjsonResult(element, "Failed to iterate JSON array: ")); + internal::ResolveSimdjsonResult(element, "Failed to iterate JSON array")); RETURN_NOT_OK(WriteValue(element_value)); } @@ -170,9 +170,9 @@ Status JsonWriter::WriteValue(sj::value value) { }, [&](sj::value value) -> Status { - ARROW_ASSIGN_OR_RAISE(auto raw_json, - internal::GetSimdjsonResult(simdjson::to_json_string(value), - "Failed to get raw JSON: ")); + ARROW_ASSIGN_OR_RAISE(auto raw_json, internal::ResolveSimdjsonResult( + simdjson::to_json_string(value), + "Failed to get raw JSON")); RawValue(raw_json); return Status::OK(); }); diff --git a/cpp/src/arrow/json/meson.build b/cpp/src/arrow/json/meson.build index edf92a46fdd2..ee2a26b2ccde 100644 --- a/cpp/src/arrow/json/meson.build +++ b/cpp/src/arrow/json/meson.build @@ -26,7 +26,7 @@ exc = executable( 'parser_test.cc', 'reader_test.cc', ], - dependencies: [arrow_test_dep, rapidjson_dep], + dependencies: [arrow_test_dep, rapidjson_dep, simdjson_dep], ) test('arrow-json-test', exc) diff --git a/cpp/src/arrow/meson.build b/cpp/src/arrow/meson.build index 955ef78973ed..b5443c28cf84 100644 --- a/cpp/src/arrow/meson.build +++ b/cpp/src/arrow/meson.build @@ -318,6 +318,22 @@ else rapidjson_dep = disabler() endif +if needs_json or needs_integration + simdjson_dep = dependency('simdjson', allow_fallback: false, required: false) + + if not simdjson_dep.found() + cmake = import('cmake') + simdjson_opts = cmake.subproject_options() + + simdjson_opts.add_cmake_defines({'SIMDJSON_EXCEPTIONS': 'OFF'}) + + simdjson_proj = cmake.subproject('simdjson', options: simdjson_opts) + simdjson_dep = simdjson_proj.dependency('simdjson') + endif +else + simdjson_dep = disabler() +endif + azure_dep = disabler() gcs_dep = disabler() s3_dep = disabler() @@ -520,7 +536,7 @@ if needs_json 'json/parser.cc', 'json/reader.cc', ], - 'dependencies': [rapidjson_dep], + 'dependencies': [rapidjson_dep, simdjson_dep], }, } endif diff --git a/cpp/src/arrow/util/simdjson_internal.h b/cpp/src/arrow/util/simdjson_internal.h index 8ffb741da42e..1badd99938b1 100644 --- a/cpp/src/arrow/util/simdjson_internal.h +++ b/cpp/src/arrow/util/simdjson_internal.h @@ -82,10 +82,11 @@ constexpr const char* JsonTypeName() { } template -Result GetSimdjsonResult(simdjson::simdjson_result result, std::string_view error) { +Result ResolveSimdjsonResult(simdjson::simdjson_result result, + std::string_view error) { T value; if (auto error_code = std::move(result).get(value); error_code != simdjson::SUCCESS) { - return Status::Invalid(error, simdjson::error_message(error_code)); + return Status::Invalid(error, ": ", simdjson::error_message(error_code)); } return value; } @@ -98,33 +99,34 @@ Status VisitJsonValue(simdjson::ondemand::value value, ObjectFn&& object_fn, NullFn&& null_fn, Int64Fn&& int64_fn, Uint64Fn&& uint64_fn, DoubleFn&& double_fn, BigIntegerFn&& big_integer_fn) { ARROW_ASSIGN_OR_RAISE( - auto type, GetSimdjsonResult(value.type(), "Failed to determine JSON type: ")); + auto type, ResolveSimdjsonResult(value.type(), "Failed to determine JSON type")); switch (type) { case simdjson::ondemand::json_type::object: { ARROW_ASSIGN_OR_RAISE( auto object, - GetSimdjsonResult(value.get_object(), "Failed to get JSON object: ")); + ResolveSimdjsonResult(value.get_object(), "Failed to get JSON object")); return object_fn(object); } case simdjson::ondemand::json_type::array: { ARROW_ASSIGN_OR_RAISE( - auto array, GetSimdjsonResult(value.get_array(), "Failed to get JSON array: ")); + auto array, + ResolveSimdjsonResult(value.get_array(), "Failed to get JSON array")); return array_fn(array); } case simdjson::ondemand::json_type::string: { ARROW_ASSIGN_OR_RAISE( auto string, - GetSimdjsonResult(value.get_string(), "Failed to get JSON string: ")); + ResolveSimdjsonResult(value.get_string(), "Failed to get JSON string")); return string_fn(string); } case simdjson::ondemand::json_type::boolean: { ARROW_ASSIGN_OR_RAISE( auto boolean, - GetSimdjsonResult(value.get_bool(), "Failed to get JSON boolean: ")); + ResolveSimdjsonResult(value.get_bool(), "Failed to get JSON boolean")); return bool_fn(boolean); } @@ -132,29 +134,30 @@ Status VisitJsonValue(simdjson::ondemand::value value, ObjectFn&& object_fn, return null_fn(); case simdjson::ondemand::json_type::number: { - ARROW_ASSIGN_OR_RAISE(auto number_type, - GetSimdjsonResult(value.get_number_type(), - "Failed to determine JSON number type: ")); + ARROW_ASSIGN_OR_RAISE( + auto number_type, + ResolveSimdjsonResult(value.get_number_type(), + "Failed to determine JSON number type")); switch (number_type) { case simdjson::ondemand::number_type::signed_integer: { ARROW_ASSIGN_OR_RAISE( auto number, - GetSimdjsonResult(value.get_int64(), "Failed to get signed integer: ")); + ResolveSimdjsonResult(value.get_int64(), "Failed to get signed integer")); return int64_fn(number); } case simdjson::ondemand::number_type::unsigned_integer: { - ARROW_ASSIGN_OR_RAISE( - auto number, - GetSimdjsonResult(value.get_uint64(), "Failed to get unsigned integer: ")); + ARROW_ASSIGN_OR_RAISE(auto number, + ResolveSimdjsonResult(value.get_uint64(), + "Failed to get unsigned integer")); return uint64_fn(number); } case simdjson::ondemand::number_type::floating_point_number: { ARROW_ASSIGN_OR_RAISE( - auto number, GetSimdjsonResult(value.get_double(), - "Failed to get floating-point number: ")); + auto number, ResolveSimdjsonResult(value.get_double(), + "Failed to get floating-point number")); return double_fn(number); } @@ -240,5 +243,90 @@ Result GetJsonAs(simdjson::ondemand::value& value) { return typed_value; } +template +Result GetJsonField(simdjson::ondemand::object& object, std::string_view key) { + for (auto field_result : object) { + ARROW_ASSIGN_OR_RAISE( + auto field, ResolveSimdjsonResult(field_result, "Failed to iterate JSON object")); + + ARROW_ASSIGN_OR_RAISE( + auto field_key, + ResolveSimdjsonResult(field.unescaped_key(), "Failed to get JSON object key")); + + if (field_key == key) { + auto value = field.value(); + + if constexpr (std::is_same_v) { + return value; + } else { + return GetJsonAs(value); + } + } + } + + return Status::KeyError("Missing JSON field: ", key); +} + +inline Result MinifyJson(std::string_view json) { + std::string minified(json.size(), '\0'); + size_t minified_len = 0; + + if (auto error = + simdjson::minify(json.data(), json.size(), minified.data(), minified_len); + error != simdjson::SUCCESS) { + return Status::Invalid("Failed to minify JSON: ", simdjson::error_message(error)); + } + + minified.resize(minified_len); + return minified; +} + +inline Status ValidateJsonObject(simdjson::ondemand::object object); + +inline Status ValidateJsonArray(simdjson::ondemand::array array); + +inline Status ConsumeJsonValue(simdjson::ondemand::value value) { + return VisitJsonValue( + value, ValidateJsonObject, ValidateJsonArray, + [](std::string_view) { return Status::OK(); }, [](bool) { return Status::OK(); }, + []() { return Status::OK(); }, [](int64_t) { return Status::OK(); }, + [](uint64_t) { return Status::OK(); }, [](double) { return Status::OK(); }, + [](simdjson::ondemand::value) { return Status::OK(); }); +} + +inline Status ValidateJsonObject(simdjson::ondemand::object object) { + for (auto field_result : object) { + ARROW_ASSIGN_OR_RAISE( + auto field, ResolveSimdjsonResult(field_result, "Failed to iterate JSON object")); + + RETURN_NOT_OK(ConsumeJsonValue(field.value())); + } + + return Status::OK(); +} + +inline Status ValidateJsonArray(simdjson::ondemand::array array) { + for (auto element_result : array) { + ARROW_ASSIGN_OR_RAISE( + auto value, + ResolveSimdjsonResult(element_result, "Failed to iterate JSON array")); + + RETURN_NOT_OK(ConsumeJsonValue(value)); + } + + return Status::OK(); +} + +inline Status ValidateJsonDocument(simdjson::ondemand::parser& parser, + simdjson::padded_string& json) { + ARROW_ASSIGN_OR_RAISE( + auto document, ResolveSimdjsonResult(parser.iterate(json), "Failed to parse JSON")); + + ARROW_ASSIGN_OR_RAISE(auto value, ResolveSimdjsonResult(document.get_value(), + "Failed to get JSON value")); + + return ConsumeJsonValue(value); +} + } // namespace internal } // namespace arrow diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt index e5860d891964..606dcdc0a9e3 100644 --- a/cpp/src/parquet/CMakeLists.txt +++ b/cpp/src/parquet/CMakeLists.txt @@ -263,9 +263,9 @@ endif() list(APPEND PARQUET_SHARED_LINK_LIBS arrow_shared) -# Add RapidJSON & simdjson libraries -list(APPEND PARQUET_SHARED_PRIVATE_LINK_LIBS RapidJSON simdjson::simdjson) -list(APPEND PARQUET_STATIC_LINK_LIBS RapidJSON simdjson::simdjson) +# Add simdjson libraries +list(APPEND PARQUET_SHARED_PRIVATE_LINK_LIBS simdjson::simdjson) +list(APPEND PARQUET_STATIC_LINK_LIBS simdjson::simdjson) # These are libraries that we will link privately with parquet_shared (as they # do not need to be linked transitively by other linkers) @@ -323,7 +323,7 @@ if(ARROW_TESTING) # "link" our dependencies so that include paths are configured # correctly target_link_libraries(parquet_testing PUBLIC ${ARROW_GTEST_GMOCK}) - list(APPEND PARQUET_TEST_LINK_LIBS parquet_testing RapidJSON) + list(APPEND PARQUET_TEST_LINK_LIBS parquet_testing simdjson::simdjson) endif() if(NOT ARROW_BUILD_SHARED) @@ -379,6 +379,7 @@ add_parquet_test(internals-test bloom_filter_test.cc geospatial/statistics_test.cc geospatial/util_internal_test.cc + geospatial/util_json_internal_test.cc metadata_test.cc page_index_test.cc properties_test.cc diff --git a/cpp/src/parquet/geospatial/util_json_internal.cc b/cpp/src/parquet/geospatial/util_json_internal.cc index 6278ab8873c8..862d320dbf5a 100644 --- a/cpp/src/parquet/geospatial/util_json_internal.cc +++ b/cpp/src/parquet/geospatial/util_json_internal.cc @@ -17,16 +17,15 @@ #include "parquet/geospatial/util_json_internal.h" +#include #include #include "arrow/extension_type.h" -#include "arrow/json/rapidjson_defs.h" // IWYU pragma: keep +#include "arrow/json/json_writer_internal.h" #include "arrow/result.h" +#include "arrow/util/simdjson_internal.h" #include "arrow/util/string.h" -#include -#include - #include "parquet/exception.h" #include "parquet/types.h" @@ -34,35 +33,94 @@ namespace parquet { namespace { ::arrow::Result GeospatialGeoArrowCrsToParquetCrs( - const ::arrow::rapidjson::Document& document) { - namespace rj = ::arrow::rapidjson; + simdjson::ondemand::object object) { + auto json_crs_result = + ::arrow::internal::GetJsonField(object, "crs"); - if (!document.HasMember("crs") || document["crs"].IsNull()) { - // Parquet GEOMETRY/GEOGRAPHY do not have a concept of a null/missing - // CRS, but an omitted one is more likely to have meant "lon/lat" than - // a truly unspecified one (i.e., Engineering CRS with arbitrary XY units) - return ""; + if (!json_crs_result.ok()) { + if (json_crs_result.status().IsKeyError()) { + // Parquet GEOMETRY/GEOGRAPHY do not have a concept of a null/missing + // CRS, but an omitted one is more likely to have meant "lon/lat" than + // a truly unspecified one (i.e., Engineering CRS with arbitrary XY units) + return ""; + } + + return json_crs_result.status(); } - const auto& json_crs = document["crs"]; - if (json_crs.IsString() && (json_crs == "EPSG:4326" || json_crs == "OGC:CRS84")) { - // crs can be left empty because these cases both correspond to - // longitude/latitude in WGS84 according to the Parquet specification + auto json_crs = *std::move(json_crs_result); + + ARROW_ASSIGN_OR_RAISE(bool is_null, ::arrow::internal::IsJsonNull(json_crs)); + if (is_null) { return ""; - } else if (json_crs.IsObject()) { - // Attempt to detect common PROJJSON representations of longitude/latitude and return - // an empty crs to maximize compatibility with readers that do not implement CRS - // support. PROJJSON stores this in the "id" member like: - // {..., "id": {"authority": "...", "code": "..."}} - if (json_crs.HasMember("id")) { - const auto& identifier = json_crs["id"]; - if (identifier.HasMember("authority") && identifier.HasMember("code")) { - if (identifier["authority"] == "OGC" && identifier["code"] == "CRS84") { - return ""; - } else if (identifier["authority"] == "EPSG" && identifier["code"] == "4326") { + } + + auto crs_string_result = ::arrow::internal::GetJsonAs(json_crs); + + if (crs_string_result.ok()) { + auto crs_string = *crs_string_result; + + if (crs_string == "EPSG:4326" || crs_string == "OGC:CRS84") { + // crs can be left empty because these cases both correspond to + // longitude/latitude in WGS84 according to the Parquet specification + return ""; + } + + // If we could not detect a longitude/latitude CRS, just write the string to the + // LogicalType crs (being sure to unescape a JSON string into a regular string) + return std::string(crs_string); + } + + ARROW_ASSIGN_OR_RAISE( + auto crs_object, + ::arrow::internal::GetJsonAs(json_crs)); + + // Attempt to detect common PROJJSON representations of longitude/latitude and return + // an empty crs to maximize compatibility with readers that do not implement CRS + // support. PROJJSON stores this in the "id" member like: + // {..., "id": {"authority": "...", "code": "..."}} + auto identifier_result = + ::arrow::internal::GetJsonField(crs_object, "id"); + + if (identifier_result.ok()) { + auto identifier = *std::move(identifier_result); + + std::optional authority_string; + std::optional code; + + for (auto field_result : identifier) { + ARROW_ASSIGN_OR_RAISE(auto field, + ::arrow::internal::ResolveSimdjsonResult( + field_result, "Failed to iterate JSON object")); + + ARROW_ASSIGN_OR_RAISE(auto key, + ::arrow::internal::ResolveSimdjsonResult( + field.unescaped_key(), "Failed to get JSON object key")); + + if (key == "authority") { + ARROW_ASSIGN_OR_RAISE( + auto authority, + ::arrow::internal::GetJsonAs(field.value())); + authority_string = authority; + } else if (key == "code") { + code = field.value(); + } + } + + if (authority_string && code) { + auto code_string_result = ::arrow::internal::GetJsonAs(*code); + + if (code_string_result.ok()) { + auto code_string = *code_string_result; + + if ((*authority_string == "OGC" && code_string == "CRS84") || + (*authority_string == "EPSG" && code_string == "4326")) { return ""; - } else if (identifier["authority"] == "EPSG" && identifier["code"].IsInt() && - identifier["code"].GetInt() == 4326) { + } + } else if (*authority_string == "EPSG") { + auto code_int_result = ::arrow::internal::GetJsonAs(*code); + + if (code_int_result.ok() && *code_int_result == 4326) { return ""; } } @@ -71,20 +129,21 @@ ::arrow::Result GeospatialGeoArrowCrsToParquetCrs( // If we could not detect a longitude/latitude CRS, just write the string to the // LogicalType crs (being sure to unescape a JSON string into a regular string) - if (json_crs.IsString()) { - return json_crs.GetString(); - } else { - rj::StringBuffer buffer; - rj::Writer writer(buffer); - json_crs.Accept(writer); - return buffer.GetString(); - } + RETURN_NOT_OK(::arrow::internal::ResolveSimdjsonResult(crs_object.reset(), + "Failed to reset 'crs' object") + .status()); + + ARROW_ASSIGN_OR_RAISE(auto raw_crs, + ::arrow::internal::ResolveSimdjsonResult( + crs_object.raw_json(), "Failed to get raw 'crs' JSON")); + + return ::arrow::internal::MinifyJson(raw_crs); } // Utility for ensuring that a Parquet CRS is valid JSON when written to // GeoArrow metadata (without escaping it if it is already valid JSON such as // a PROJJSON string) -std::string EscapeCrsAsJsonIfRequired(std::string_view crs); +::arrow::Result EscapeCrsAsJsonIfRequired(std::string_view crs); ::arrow::Result MakeGeoArrowCrsMetadata( std::string_view crs, @@ -113,64 +172,89 @@ ::arrow::Result MakeGeoArrowCrsMetadata( ARROW_ASSIGN_OR_RAISE(std::string projjson_value, metadata->Get(metadata_field)); // This value should be valid JSON, but if it is not, we escape it as a string such // that it can be inspected by the consumer of GeoArrow. - return R"("crs": )" + EscapeCrsAsJsonIfRequired(projjson_value) + - R"(, "crs_type": "projjson")"; + ARROW_ASSIGN_OR_RAISE(auto escaped, EscapeCrsAsJsonIfRequired(projjson_value)); + return R"("crs": )" + escaped + R"(, "crs_type": "projjson")"; } } // Pass on the string directly to GeoArrow. If the string is already valid JSON, // insert it directly into GeoArrow's "crs" field. Otherwise, escape it and pass it as a // string value. - return R"("crs": )" + EscapeCrsAsJsonIfRequired(crs); + ARROW_ASSIGN_OR_RAISE(auto escaped, EscapeCrsAsJsonIfRequired(crs)); + + return R"("crs": )" + escaped; } -std::string EscapeCrsAsJsonIfRequired(std::string_view crs) { - namespace rj = ::arrow::rapidjson; - rj::Document document; - if (document.Parse(crs.data(), crs.length()).HasParseError()) { - rj::StringBuffer buffer; - rj::Writer writer(buffer); - rj::Value v; - v.SetString(crs.data(), static_cast(crs.size())); - v.Accept(writer); - return std::string(buffer.GetString()); - } else { - return std::string(crs); +::arrow::Result EscapeJsonString(std::string_view value) { + ::arrow::json::JsonWriter writer; + writer.String(value); + + ARROW_ASSIGN_OR_RAISE(auto escaped, writer.GetString()); + return ::arrow::internal::MinifyJson(escaped); +} + +::arrow::Result EscapeCrsAsJsonIfRequired(std::string_view crs) { + simdjson::ondemand::parser parser; + simdjson::padded_string json(crs); + + if (!::arrow::internal::ValidateJsonDocument(parser, json).ok()) { + return EscapeJsonString(crs); } + + return std::string(crs); } } // namespace -::arrow::Result> LogicalTypeFromGeoArrowMetadata( - std::string_view serialized_data) { +::arrow::Result> +LogicalTypeFromGeoArrowMetadata(std::string_view serialized_data) { // Parquet has no way to interpret a null or missing CRS, so we choose the most likely // intent here (that the user meant to use the default Parquet CRS) if (serialized_data.empty() || serialized_data == "{}") { return LogicalType::Geometry(); } - namespace rj = ::arrow::rapidjson; - rj::Document document; - if (document.Parse(serialized_data.data(), serialized_data.length()).HasParseError()) { - return ::arrow::Status::Invalid("Invalid serialized JSON data: ", serialized_data); + simdjson::ondemand::parser parser; + simdjson::padded_string json(serialized_data); + + RETURN_NOT_OK(::arrow::internal::ValidateJsonDocument(parser, json)); + + // Reparse because validation consumes the On-Demand document. + ARROW_ASSIGN_OR_RAISE(auto document, ::arrow::internal::ResolveSimdjsonResult( + parser.iterate(json), "Failed to parse JSON")); + + ARROW_ASSIGN_OR_RAISE( + auto object, ::arrow::internal::ResolveSimdjsonResult(document.get_object(), + "Failed to get JSON object")); + + ARROW_ASSIGN_OR_RAISE(std::string crs, GeospatialGeoArrowCrsToParquetCrs(object)); + + auto edges_field = object["edges"]; + + if (edges_field.error() == simdjson::NO_SUCH_FIELD) { + return LogicalType::Geometry(crs); } - ARROW_ASSIGN_OR_RAISE(std::string crs, GeospatialGeoArrowCrsToParquetCrs(document)); + ARROW_ASSIGN_OR_RAISE(auto edges, ::arrow::internal::ResolveSimdjsonResult( + edges_field, "Failed to get 'edges' field")); - if (document.HasMember("edges") && document["edges"] == "planar") { + ARROW_ASSIGN_OR_RAISE(auto edges_string, + ::arrow::internal::GetJsonAs(edges)); + + if (edges_string == "planar") { return LogicalType::Geometry(crs); - } else if (document.HasMember("edges") && document["edges"] == "spherical") { + } + + if (edges_string == "spherical") { return LogicalType::Geography(crs, LogicalType::EdgeInterpolationAlgorithm::SPHERICAL); - } else if (document.HasMember("edges")) { - return ::arrow::Status::Invalid("Unsupported GeoArrow edge type: ", serialized_data); } - return LogicalType::Geometry(crs); + return ::arrow::Status::Invalid("Unsupported GeoArrow edge type: ", serialized_data); } ::arrow::Result> GeoArrowTypeFromLogicalType( - const LogicalType& logical_type, + const parquet::LogicalType& logical_type, const std::shared_ptr& metadata, const std::shared_ptr<::arrow::DataType>& storage_type) { // Check if we have a registered GeoArrow type to read into diff --git a/cpp/src/parquet/geospatial/util_json_internal.h b/cpp/src/parquet/geospatial/util_json_internal.h index 1d43b320f80c..5c18c55366e8 100644 --- a/cpp/src/parquet/geospatial/util_json_internal.h +++ b/cpp/src/parquet/geospatial/util_json_internal.h @@ -21,6 +21,7 @@ #include "arrow/util/key_value_metadata.h" +#include "parquet/platform.h" #include "parquet/types.h" namespace parquet { @@ -29,8 +30,8 @@ namespace parquet { /// GeoArrow `ARROW:extension:metadata` (JSON-encoded extension type metadata) /// /// Returns the appropriate LogicalType or Invalid if the metadata was invalid. -::arrow::Result> LogicalTypeFromGeoArrowMetadata( - std::string_view serialized_data); +PARQUET_EXPORT ::arrow::Result> +LogicalTypeFromGeoArrowMetadata(std::string_view serialized_data); /// \brief Compute a suitable DataType into which a GEOMETRY or GEOGRAPHY type should be /// read @@ -38,7 +39,8 @@ ::arrow::Result> LogicalTypeFromGeoArrowMetad /// The result of this function depends on whether or not "geoarrow.wkb" has been /// registered: if it has, the result will be the registered ExtensionType; if it has not, /// the result will be the given storage_type. -::arrow::Result> GeoArrowTypeFromLogicalType( +PARQUET_EXPORT ::arrow::Result> +GeoArrowTypeFromLogicalType( const LogicalType& logical_type, const std::shared_ptr& metadata, const std::shared_ptr<::arrow::DataType>& storage_type); diff --git a/cpp/src/parquet/geospatial/util_json_internal_test.cc b/cpp/src/parquet/geospatial/util_json_internal_test.cc new file mode 100644 index 000000000000..ae86a0cdfb3a --- /dev/null +++ b/cpp/src/parquet/geospatial/util_json_internal_test.cc @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "parquet/geospatial/util_json_internal.h" + +#include + +#include + +#include "arrow/testing/extension_type.h" +#include "arrow/testing/gtest_util.h" +#include "arrow/type.h" +#include "arrow/util/simdjson_internal.h" + +#include "parquet/test_util.h" + +namespace parquet { + +TEST(UtilJsonInternal, InvalidProjJsonIsEscaped) { + ::arrow::ExtensionTypeGuard guard(test::geoarrow_wkb()); + + auto metadata = ::arrow::key_value_metadata( + {"proj"}, {R"({"a":[1,2,]})"}); // Invalid JSON (trailing comma) + + auto logical_type = LogicalType::Geometry("projjson:proj"); + + ASSERT_OK_AND_ASSIGN( + auto type, GeoArrowTypeFromLogicalType(*logical_type, metadata, ::arrow::binary())); + + auto extension = std::dynamic_pointer_cast<::arrow::ExtensionType>(type); + ASSERT_NE(extension, nullptr); + + ASSERT_OK_AND_ASSIGN(auto actual, + ::arrow::internal::MinifyJson(extension->Serialize())); + + EXPECT_EQ(actual, "{\"crs\":\"{\\\"a\\\":[1,2,]}\",\"crs_type\":\"projjson\"}"); +} + +TEST(UtilJsonInternal, EscapedCrsKeyIsRecognized) { + std::string metadata = R"({"cr\u0073":"EPSG:3857","crs_type":"authority_code"})"; + + ASSERT_OK_AND_ASSIGN(auto logical_type, LogicalTypeFromGeoArrowMetadata(metadata)); + + ASSERT_EQ(logical_type->ToString(), "Geometry(crs=EPSG:3857)"); +} + +TEST(UtilJsonInternal, InvalidTrailingMetadataIsRejected) { + auto result = LogicalTypeFromGeoArrowMetadata( + R"({"crs":"EPSG:3857","edges":"planar","unused":[1,2,]})"); + + ASSERT_RAISES(Invalid, result); +} + +} // namespace parquet diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index 9069ccb5fd1a..f2aff7dfa1de 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -88,7 +88,7 @@ if not thrift_dep.found() thrift_dep = thrift_proj.dependency('thrift') endif -parquet_deps = [arrow_dep, rapidjson_dep, thrift_dep] +parquet_deps = [arrow_dep, simdjson_dep, thrift_dep] if needs_parquet_encryption or get_option('parquet_require_encryption').auto() openssl_dep = dependency('openssl', required: needs_parquet_encryption) @@ -198,6 +198,7 @@ parquet_tests = { 'encoding_test.cc', 'geospatial/statistics_test.cc', 'geospatial/util_internal_test.cc', + 'geospatial/util_json_internal_test.cc', 'metadata_test.cc', 'page_index_test.cc', 'properties_test.cc', @@ -291,6 +292,7 @@ parquet_test_dep = [ parquet_dep, parquet_test_support_dep, arrow_test_dep, + simdjson_dep, thrift_dep, ] diff --git a/cpp/src/parquet/reader_test.cc b/cpp/src/parquet/reader_test.cc index 7ae9021e35e9..cdeee116fbc3 100644 --- a/cpp/src/parquet/reader_test.cc +++ b/cpp/src/parquet/reader_test.cc @@ -27,12 +27,6 @@ #include #include -#include "arrow/json/rapidjson_defs.h" // IWYU pragma: keep - -#include -#include -#include - #include "arrow/array.h" #include "arrow/array/array_binary.h" #include "arrow/array/builder_binary.h" @@ -44,6 +38,7 @@ #include "arrow/util/checked_cast.h" #include "arrow/util/config.h" #include "arrow/util/range.h" +#include "arrow/util/simdjson_internal.h" #include "parquet/column_reader.h" #include "parquet/column_scanner.h" @@ -59,8 +54,6 @@ #include "parquet/test_util.h" #include "parquet/types.h" -namespace rj = arrow::rapidjson; - using arrow::internal::checked_pointer_cast; using arrow::internal::Zip; @@ -1230,14 +1223,11 @@ TEST_F(TestJSONWithLocalFile, JSONOutputSortColumns) { namespace { ::arrow::Status CheckJsonValid(std::string_view json_string) { - rj::Document json_doc; - constexpr auto kParseFlags = rj::kParseFullPrecisionFlag | rj::kParseNanAndInfFlag; - json_doc.Parse(json_string.data(), json_string.length()); - if (json_doc.HasParseError()) { - return ::arrow::Status::Invalid("JSON parse error at offset ", - json_doc.GetErrorOffset(), ": ", - rj::GetParseError_En(json_doc.GetParseError())); - } + simdjson::ondemand::parser parser; + auto padded_json = simdjson::padded_string(json_string); + + RETURN_NOT_OK(::arrow::internal::ValidateJsonDocument(parser, padded_json)); + return ::arrow::Status::OK(); } diff --git a/cpp/src/parquet/schema_test.cc b/cpp/src/parquet/schema_test.cc index 859f14a34d91..704b2da79c1b 100644 --- a/cpp/src/parquet/schema_test.cc +++ b/cpp/src/parquet/schema_test.cc @@ -1581,9 +1581,9 @@ TEST(TestLogicalTypeOperation, LogicalTypeRepresentation) { {LogicalType::Geometry(R"(crs with "quotes" and \backslashes\)"), R"(Geometry(crs=crs with "quotes" and \backslashes\))", R"({"Type": "Geometry", "crs": "crs with \"quotes\" and \\backslashes\\"})"}, - {LogicalType::Geometry("crs with control characters \u0001 and \u001F"), - "Geometry(crs=crs with control characters \u0001 and \u001F)", - R"({"Type": "Geometry", "crs": "crs with control characters \u0001 and \u001F"})"}, + {LogicalType::Geometry("crs with control characters \u0001 and \u001f"), + "Geometry(crs=crs with control characters \u0001 and \u001f)", + R"({"Type": "Geometry", "crs": "crs with control characters \u0001 and \u001f"})"}, {LogicalType::Geography(), "Geography(crs=, algorithm=spherical)", R"({"Type": "Geography"})"}, {LogicalType::Geography("srid:1234", diff --git a/cpp/src/parquet/types.cc b/cpp/src/parquet/types.cc index 9d7604faec30..cc3199f367af 100644 --- a/cpp/src/parquet/types.cc +++ b/cpp/src/parquet/types.cc @@ -24,16 +24,13 @@ #include #include -#include "arrow/json/rapidjson_defs.h" // IWYU pragma: keep +#include "arrow/json/json_writer_internal.h" #include "arrow/util/checked_cast.h" #include "arrow/util/compression.h" #include "arrow/util/decimal.h" #include "arrow/util/float16.h" #include "arrow/util/logging_internal.h" -#include -#include - #include "parquet/exception.h" #include "parquet/thrift_internal.h" #include "parquet/types.h" @@ -1785,13 +1782,9 @@ namespace { void WriteCrsKeyAndValue(const std::string_view crs, std::ostream& json) { // There is no restriction on the crs value here, and it may contain quotes // or backslashes that would result in invalid JSON if unescaped. - namespace rj = ::arrow::rapidjson; - rj::StringBuffer buffer; - rj::Writer writer(buffer); - rj::Value v; - v.SetString(crs.data(), static_cast(crs.size())); - v.Accept(writer); - json << R"(, "crs": )" << buffer.GetString(); + ::arrow::json::JsonWriter writer; + writer.String(crs); + json << R"(, "crs": )" << writer.GetString().ValueUnsafe(); } } // namespace diff --git a/cpp/subprojects/simdjson.wrap b/cpp/subprojects/simdjson.wrap new file mode 100644 index 000000000000..31d9de3c2750 --- /dev/null +++ b/cpp/subprojects/simdjson.wrap @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[wrap-file] +source_url = https://github.com/simdjson/simdjson/archive/refs/tags/v4.6.4.tar.gz +source_filename = simdjson-v4.6.4.tar.gz +source_hash = b091107844fe928158c5c2265c20360fff312889ddf7ebc4528a0f0f8f2ff9cd +directory = simdjson-v4.6.4 +method = cmake